1 // Copyright 2014 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
10 "runtime/internal/math"
14 // Should be a built-in for unsafe.Pointer?
17 func add(p unsafe.Pointer, x uintptr) unsafe.Pointer {
18 return unsafe.Pointer(uintptr(p) + x)
21 // getg returns the pointer to the current g.
22 // The compiler rewrites calls to this function into instructions
23 // that fetch the g directly (from TLS or from the dedicated register).
26 // mcall switches from the g to the g0 stack and invokes fn(g),
27 // where g is the goroutine that made the call.
28 // mcall saves g's current PC/SP in g->sched so that it can be restored later.
29 // It is up to fn to arrange for that later execution, typically by recording
30 // g in a data structure, causing something to call ready(g) later.
31 // mcall returns to the original goroutine g later, when g has been rescheduled.
32 // fn must not return at all; typically it ends by calling schedule, to let the m
33 // run other goroutines.
35 // mcall can only be called from g stacks (not g0, not gsignal).
37 // This must NOT be go:noescape: if fn is a stack-allocated closure,
38 // fn puts g on a run queue, and g executes before fn returns, the
39 // closure will be invalidated while it is still executing.
40 func mcall(fn func(*g))
42 // systemstack runs fn on a system stack.
43 // If systemstack is called from the per-OS-thread (g0) stack, or
44 // if systemstack is called from the signal handling (gsignal) stack,
45 // systemstack calls fn directly and returns.
46 // Otherwise, systemstack is being called from the limited stack
47 // of an ordinary goroutine. In this case, systemstack switches
48 // to the per-OS-thread stack, calls fn, and switches back.
49 // It is common to use a func literal as the argument, in order
50 // to share inputs and outputs with the code around the call
54 // systemstack(func() {
60 func systemstack(fn func())
63 //go:nowritebarrierrec
64 func badsystemstack() {
65 writeErrStr("fatal: systemstack called from unexpected goroutine")
68 // memclrNoHeapPointers clears n bytes starting at ptr.
70 // Usually you should use typedmemclr. memclrNoHeapPointers should be
71 // used only when the caller knows that *ptr contains no heap pointers
74 // *ptr is initialized memory and its type is pointer-free, or
76 // *ptr is uninitialized memory (e.g., memory that's being reused
77 // for a new allocation) and hence contains only "junk".
79 // memclrNoHeapPointers ensures that if ptr is pointer-aligned, and n
80 // is a multiple of the pointer size, then any pointer-aligned,
81 // pointer-sized portion is cleared atomically. Despite the function
82 // name, this is necessary because this function is the underlying
83 // implementation of typedmemclr and memclrHasPointers. See the doc of
84 // memmove for more details.
86 // The (CPU-specific) implementations of this function are in memclr_*.s.
89 func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr)
91 //go:linkname reflect_memclrNoHeapPointers reflect.memclrNoHeapPointers
92 func reflect_memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) {
93 memclrNoHeapPointers(ptr, n)
96 // memmove copies n bytes from "from" to "to".
98 // memmove ensures that any pointer in "from" is written to "to" with
99 // an indivisible write, so that racy reads cannot observe a
100 // half-written pointer. This is necessary to prevent the garbage
101 // collector from observing invalid pointers, and differs from memmove
102 // in unmanaged languages. However, memmove is only required to do
103 // this if "from" and "to" may contain pointers, which can only be the
104 // case if "from", "to", and "n" are all be word-aligned.
106 // Implementations are in memmove_*.s.
109 func memmove(to, from unsafe.Pointer, n uintptr)
111 // Outside assembly calls memmove. Make sure it has ABI wrappers.
113 //go:linkname memmove
115 //go:linkname reflect_memmove reflect.memmove
116 func reflect_memmove(to, from unsafe.Pointer, n uintptr) {
120 // exported value for testing
121 const hashLoad = float32(loadFactorNum) / float32(loadFactorDen)
124 func fastrand() uint32 {
126 // Implement wyrand: https://github.com/wangyi-fudan/wyhash
127 // Only the platform that math.Mul64 can be lowered
128 // by the compiler should be in this list.
129 if goarch.IsAmd64|goarch.IsArm64|goarch.IsPpc64|
130 goarch.IsPpc64le|goarch.IsMips64|goarch.IsMips64le|
131 goarch.IsS390x|goarch.IsRiscv64|goarch.IsLoong64 == 1 {
132 mp.fastrand += 0xa0761d6478bd642f
133 hi, lo := math.Mul64(mp.fastrand, mp.fastrand^0xe7037ed1a0b428db)
134 return uint32(hi ^ lo)
137 // Implement xorshift64+: 2 32-bit xorshift sequences added together.
138 // Shift triplet [17,7,16] was calculated as indicated in Marsaglia's
139 // Xorshift paper: https://www.jstatsoft.org/article/view/v008i14/xorshift.pdf
140 // This generator passes the SmallCrush suite, part of TestU01 framework:
141 // http://simul.iro.umontreal.ca/testu01/tu01.html
142 t := (*[2]uint32)(unsafe.Pointer(&mp.fastrand))
145 s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16
151 func fastrandn(n uint32) uint32 {
152 // This is similar to fastrand() % n, but faster.
153 // See https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/
154 return uint32(uint64(fastrand()) * uint64(n) >> 32)
157 func fastrand64() uint64 {
159 // Implement wyrand: https://github.com/wangyi-fudan/wyhash
160 // Only the platform that math.Mul64 can be lowered
161 // by the compiler should be in this list.
162 if goarch.IsAmd64|goarch.IsArm64|goarch.IsPpc64|
163 goarch.IsPpc64le|goarch.IsMips64|goarch.IsMips64le|
164 goarch.IsS390x|goarch.IsRiscv64 == 1 {
165 mp.fastrand += 0xa0761d6478bd642f
166 hi, lo := math.Mul64(mp.fastrand, mp.fastrand^0xe7037ed1a0b428db)
170 // Implement xorshift64+: 2 32-bit xorshift sequences added together.
171 // Xorshift paper: https://www.jstatsoft.org/article/view/v008i14/xorshift.pdf
172 // This generator passes the SmallCrush suite, part of TestU01 framework:
173 // http://simul.iro.umontreal.ca/testu01/tu01.html
174 t := (*[2]uint32)(unsafe.Pointer(&mp.fastrand))
177 s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16
182 s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16
183 r += uint64(s0+s1) << 32
189 func fastrandu() uint {
190 if goarch.PtrSize == 4 {
191 return uint(fastrand())
193 return uint(fastrand64())
196 //go:linkname rand_fastrand64 math/rand.fastrand64
197 func rand_fastrand64() uint64 { return fastrand64() }
199 //go:linkname rand2_fastrand64 math/rand/v2.fastrand64
200 func rand2_fastrand64() uint64 { return fastrand64() }
202 //go:linkname sync_fastrandn sync.fastrandn
203 func sync_fastrandn(n uint32) uint32 { return fastrandn(n) }
205 //go:linkname net_fastrandu net.fastrandu
206 func net_fastrandu() uint { return fastrandu() }
208 //go:linkname os_fastrand os.fastrand
209 func os_fastrand() uint32 { return fastrand() }
211 // in internal/bytealg/equal_*.s
214 func memequal(a, b unsafe.Pointer, size uintptr) bool
216 // noescape hides a pointer from escape analysis. noescape is
217 // the identity function but escape analysis doesn't think the
218 // output depends on the input. noescape is inlined and currently
219 // compiles down to zero instructions.
223 func noescape(p unsafe.Pointer) unsafe.Pointer {
225 return unsafe.Pointer(x ^ 0)
228 // noEscapePtr hides a pointer from escape analysis. See noescape.
232 func noEscapePtr[T any](p *T) *T {
233 x := uintptr(unsafe.Pointer(p))
234 return (*T)(unsafe.Pointer(x ^ 0))
237 // Not all cgocallback frames are actually cgocallback,
238 // so not all have these arguments. Mark them uintptr so that the GC
239 // does not misinterpret memory when the arguments are not present.
240 // cgocallback is not called from Go, only from crosscall2.
241 // This in turn calls cgocallbackg, which is where we'll find
242 // pointer-declared arguments.
244 // When fn is nil (frame is saved g), call dropm instead,
245 // this is used when the C thread is exiting.
246 func cgocallback(fn, frame, ctxt uintptr)
248 func gogo(buf *gobuf)
254 // reflectcall calls fn with arguments described by stackArgs, stackArgsSize,
255 // frameSize, and regArgs.
257 // Arguments passed on the stack and space for return values passed on the stack
258 // must be laid out at the space pointed to by stackArgs (with total length
259 // stackArgsSize) according to the ABI.
261 // stackRetOffset must be some value <= stackArgsSize that indicates the
262 // offset within stackArgs where the return value space begins.
264 // frameSize is the total size of the argument frame at stackArgs and must
265 // therefore be >= stackArgsSize. It must include additional space for spilling
266 // register arguments for stack growth and preemption.
268 // TODO(mknyszek): Once we don't need the additional spill space, remove frameSize,
269 // since frameSize will be redundant with stackArgsSize.
271 // Arguments passed in registers must be laid out in regArgs according to the ABI.
272 // regArgs will hold any return values passed in registers after the call.
274 // reflectcall copies stack arguments from stackArgs to the goroutine stack, and
275 // then copies back stackArgsSize-stackRetOffset bytes back to the return space
276 // in stackArgs once fn has completed. It also "unspills" argument registers from
277 // regArgs before calling fn, and spills them back into regArgs immediately
278 // following the call to fn. If there are results being returned on the stack,
279 // the caller should pass the argument frame type as stackArgsType so that
280 // reflectcall can execute appropriate write barriers during the copy.
282 // reflectcall expects regArgs.ReturnIsPtr to be populated indicating which
283 // registers on the return path will contain Go pointers. It will then store
284 // these pointers in regArgs.Ptrs such that they are visible to the GC.
286 // Package reflect passes a frame type. In package runtime, there is only
287 // one call that copies results back, in callbackWrap in syscall_windows.go, and it
288 // does NOT pass a frame type, meaning there are no write barriers invoked. See that
289 // call site for justification.
291 // Package reflect accesses this symbol through a linkname.
293 // Arguments passed through to reflectcall do not escape. The type is used
294 // only in a very limited callee of reflectcall, the stackArgs are copied, and
295 // regArgs is only used in the reflectcall frame.
298 func reflectcall(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
300 func procyield(cycles uint32)
302 type neverCallThisFunction struct{}
304 // goexit is the return stub at the top of every goroutine call stack.
305 // Each goroutine stack is constructed as if goexit called the
306 // goroutine's entry point function, so that when the entry point
307 // function returns, it will return to goexit, which will call goexit1
308 // to perform the actual exit.
310 // This function must never be called directly. Call goexit1 instead.
311 // gentraceback assumes that goexit terminates the stack. A direct
312 // call on the stack will cause gentraceback to stop walking the stack
313 // prematurely and if there is leftover state it may panic.
314 func goexit(neverCallThisFunction)
316 // publicationBarrier performs a store/store barrier (a "publication"
317 // or "export" barrier). Some form of synchronization is required
318 // between initializing an object and making that object accessible to
319 // another processor. Without synchronization, the initialization
320 // writes and the "publication" write may be reordered, allowing the
321 // other processor to follow the pointer and observe an uninitialized
322 // object. In general, higher-level synchronization should be used,
323 // such as locking or an atomic pointer write. publicationBarrier is
324 // for when those aren't an option, such as in the implementation of
325 // the memory manager.
327 // There's no corresponding barrier for the read side because the read
328 // side naturally has a data dependency order. All architectures that
329 // Go supports or seems likely to ever support automatically enforce
330 // data dependency ordering.
331 func publicationBarrier()
333 // getcallerpc returns the program counter (PC) of its caller's caller.
334 // getcallersp returns the stack pointer (SP) of its caller's caller.
335 // The implementation may be a compiler intrinsic; there is not
336 // necessarily code implementing this on every platform.
340 // func f(arg1, arg2, arg3 int) {
341 // pc := getcallerpc()
342 // sp := getcallersp()
345 // These two lines find the PC and SP immediately following
346 // the call to f (where f will return).
348 // The call to getcallerpc and getcallersp must be done in the
349 // frame being asked about.
351 // The result of getcallersp is correct at the time of the return,
352 // but it may be invalidated by any subsequent call to a function
353 // that might relocate the stack in order to grow or shrink it.
354 // A general rule is that the result of getcallersp should be used
355 // immediately and can only be passed to nosplit functions.
358 func getcallerpc() uintptr
361 func getcallersp() uintptr // implemented as an intrinsic on all platforms
363 // getclosureptr returns the pointer to the current closure.
364 // getclosureptr can only be used in an assignment statement
365 // at the entry of a function. Moreover, go:nosplit directive
366 // must be specified at the declaration of caller function,
367 // so that the function prolog does not clobber the closure register.
371 // func f(arg1, arg2, arg3 int) {
372 // dx := getclosureptr()
375 // The compiler rewrites calls to this function into instructions that fetch the
376 // pointer from a well-known register (DX on x86 architecture, etc.) directly.
377 func getclosureptr() uintptr
380 func asmcgocall(fn, arg unsafe.Pointer) int32
383 func morestack_noctxt()
386 // return0 is a stub used to return 0 from deferproc.
387 // It is called at the very end of deferproc to signal
388 // the calling Go function that it should not jump
394 // not called directly; definitions here supply type information for traceback.
395 // These must have the same signature (arg pointer map) as reflectcall.
396 func call16(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
397 func call32(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
398 func call64(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
399 func call128(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
400 func call256(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
401 func call512(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
402 func call1024(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
403 func call2048(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
404 func call4096(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
405 func call8192(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
406 func call16384(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
407 func call32768(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
408 func call65536(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
409 func call131072(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
410 func call262144(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
411 func call524288(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
412 func call1048576(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
413 func call2097152(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
414 func call4194304(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
415 func call8388608(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
416 func call16777216(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
417 func call33554432(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
418 func call67108864(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
419 func call134217728(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
420 func call268435456(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
421 func call536870912(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
422 func call1073741824(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
424 func systemstack_switch()
426 // alignUp rounds n up to a multiple of a. a must be a power of 2.
429 func alignUp(n, a uintptr) uintptr {
430 return (n + a - 1) &^ (a - 1)
433 // alignDown rounds n down to a multiple of a. a must be a power of 2.
436 func alignDown(n, a uintptr) uintptr {
440 // divRoundUp returns ceil(n / a).
441 func divRoundUp(n, a uintptr) uintptr {
442 // a is generally a power of two. This will get inlined and
443 // the compiler will optimize the division.
444 return (n + a - 1) / a
447 // checkASM reports whether assembly runtime checks have passed.
450 func memequal_varlen(a, b unsafe.Pointer) bool
452 // bool2int returns 0 if x is false or 1 if x is true.
453 func bool2int(x bool) int {
454 // Avoid branches. In the SSA compiler, this compiles to
455 // exactly what you would want it to.
456 return int(*(*uint8)(unsafe.Pointer(&x)))
459 // abort crashes the runtime in situations where even throw might not
460 // work. In general it should do something a debugger will recognize
461 // (e.g., an INT3 on x86). A crash in abort is recognized by the
462 // signal handler, which will attempt to tear down the runtime
466 // Called from compiled code; declared for vet; do NOT call from Go.
467 func gcWriteBarrier1()
468 func gcWriteBarrier2()
469 func gcWriteBarrier3()
470 func gcWriteBarrier4()
471 func gcWriteBarrier5()
472 func gcWriteBarrier6()
473 func gcWriteBarrier7()
474 func gcWriteBarrier8()
478 // Called from linker-generated .initarray; declared for go vet; do NOT call from Go.
481 // Injected by the signal handler for panicking signals.
482 // Initializes any registers that have fixed meaning at calls but
483 // are scratch in bodies and calls sigpanic.
484 // On many platforms it just jumps to sigpanic.
487 // intArgRegs is used by the various register assignment
488 // algorithm implementations in the runtime. These include:.
489 // - Finalizers (mfinal.go)
490 // - Windows callbacks (syscall_windows.go)
492 // Both are stripped-down versions of the algorithm since they
493 // only have to deal with a subset of cases (finalizers only
494 // take a pointer or interface argument, Go Windows callbacks
495 // don't support floating point).
497 // It should be modified with care and are generally only
498 // modified when testing this package.
500 // It should never be set higher than its internal/abi
501 // constant counterparts, because the system relies on a
502 // structure that is at least large enough to hold the
503 // registers the system supports.
505 // Protected by finlock.
506 var intArgRegs = abi.IntArgRegs