1 // Copyright 2013 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
12 "runtime/internal/atomic"
13 "runtime/internal/sys"
18 Stack layout parameters.
19 Included both by runtime (compiled via 6c) and linkers (compiled via gcc).
21 The per-goroutine g->stackguard is set to point StackGuard bytes
22 above the bottom of the stack. Each function compares its stack
23 pointer against g->stackguard to check for overflow. To cut one
24 instruction from the check sequence for functions with tiny frames,
25 the stack is allowed to protrude StackSmall bytes below the stack
26 guard. Functions with large frames don't bother with the check and
27 always call morestack. The sequences are (for amd64, others are
31 frame = function's stack frame size
32 argsize = size of function arguments (call + return)
34 stack frame size <= StackSmall:
37 MOVQ m->morearg, $(argsize << 32)
40 stack frame size > StackSmall but < StackBig
41 LEAQ (frame-StackSmall)(SP), R0
44 MOVQ m->morearg, $(argsize << 32)
47 stack frame size >= StackBig:
48 MOVQ m->morearg, $((argsize << 32) | frame)
51 The bottom StackGuard - StackSmall bytes are important: there has
52 to be enough room to execute functions that refuse to check for
53 stack overflow, either because they need to be adjacent to the
54 actual caller's frame (deferproc) or because they handle the imminent
55 stack overflow (morestack).
57 For example, deferproc might call malloc, which does one of the
58 above checks (without allocating a full frame), which might trigger
59 a call to morestack. This sequence needs to fit in the bottom
60 section of the stack. On amd64, morestack's frame is 40 bytes, and
61 deferproc's frame is 56 bytes. That fits well within the
62 StackGuard - StackSmall bytes at the bottom.
63 The linkers explore all possible call traces involving non-splitting
64 functions to make sure that this limit cannot be violated.
68 // stackSystem is a number of additional bytes to add
69 // to each stack below the usual guard area for OS-specific
70 // purposes like signal handling. Used on Windows, Plan 9,
71 // and iOS because they do not use a separate stack.
72 stackSystem = goos.IsWindows*512*goarch.PtrSize + goos.IsPlan9*512 + goos.IsIos*goarch.IsArm64*1024
74 // The minimum size of stack used by Go code
77 // The minimum stack size to allocate.
78 // The hackery here rounds fixedStack0 up to a power of 2.
79 fixedStack0 = stackMin + stackSystem
80 fixedStack1 = fixedStack0 - 1
81 fixedStack2 = fixedStack1 | (fixedStack1 >> 1)
82 fixedStack3 = fixedStack2 | (fixedStack2 >> 2)
83 fixedStack4 = fixedStack3 | (fixedStack3 >> 4)
84 fixedStack5 = fixedStack4 | (fixedStack4 >> 8)
85 fixedStack6 = fixedStack5 | (fixedStack5 >> 16)
86 fixedStack = fixedStack6 + 1
88 // stackNosplit is the maximum number of bytes that a chain of NOSPLIT
90 // This arithmetic must match that in cmd/internal/objabi/stack.go:StackNosplit.
91 stackNosplit = abi.StackNosplitBase * sys.StackGuardMultiplier
93 // The stack guard is a pointer this many bytes above the
94 // bottom of the stack.
96 // The guard leaves enough room for a stackNosplit chain of NOSPLIT calls
97 // plus one stackSmall frame plus stackSystem bytes for the OS.
98 // This arithmetic must match that in cmd/internal/objabi/stack.go:StackLimit.
99 stackGuard = stackNosplit + stackSystem + abi.StackSmall
103 // stackDebug == 0: no logging
104 // == 1: logging of per-stack operations
105 // == 2: logging of per-frame operations
106 // == 3: logging of per-word updates
107 // == 4: logging of per-word reads
109 stackFromSystem = 0 // allocate stacks from system memory instead of the heap
110 stackFaultOnFree = 0 // old stacks are mapped noaccess to detect use after free
111 stackNoCache = 0 // disable per-P small stack caches
113 // check the BP links during traceback.
118 stackPoisonCopy = 0 // fill stack that should not be accessed with garbage, to detect bad dereferences during copy
122 uintptrMask = 1<<(8*goarch.PtrSize) - 1
124 // The values below can be stored to g.stackguard0 to force
125 // the next stack check to fail.
126 // These are all larger than any real SP.
128 // Goroutine preemption request.
129 // 0xfffffade in hex.
130 stackPreempt = uintptrMask & -1314
132 // Thread is forking. Causes a split stack check failure.
133 // 0xfffffb2e in hex.
134 stackFork = uintptrMask & -1234
136 // Force a stack movement. Used for debugging.
137 // 0xfffffeed in hex.
138 stackForceMove = uintptrMask & -275
140 // stackPoisonMin is the lowest allowed stack poison value.
141 stackPoisonMin = uintptrMask & -4096
144 // Global pool of spans that have free stacks.
145 // Stacks are assigned an order according to size.
147 // order = log_2(size/FixedStack)
149 // There is a free list for each order.
150 var stackpool [_NumStackOrders]struct {
152 _ [(cpu.CacheLinePadSize - unsafe.Sizeof(stackpoolItem{})%cpu.CacheLinePadSize) % cpu.CacheLinePadSize]byte
155 type stackpoolItem struct {
161 // Global pool of large stack spans.
162 var stackLarge struct {
164 free [heapAddrBits - pageShift]mSpanList // free lists by log_2(s.npages)
168 if _StackCacheSize&_PageMask != 0 {
169 throw("cache size must be a multiple of page size")
171 for i := range stackpool {
172 stackpool[i].item.span.init()
173 lockInit(&stackpool[i].item.mu, lockRankStackpool)
175 for i := range stackLarge.free {
176 stackLarge.free[i].init()
177 lockInit(&stackLarge.lock, lockRankStackLarge)
181 // stacklog2 returns ⌊log_2(n)⌋.
182 func stacklog2(n uintptr) int {
191 // Allocates a stack from the free pool. Must be called with
192 // stackpool[order].item.mu held.
193 func stackpoolalloc(order uint8) gclinkptr {
194 list := &stackpool[order].item.span
196 lockWithRankMayAcquire(&mheap_.lock, lockRankMheap)
198 // no free stacks. Allocate another span worth.
199 s = mheap_.allocManual(_StackCacheSize>>_PageShift, spanAllocStack)
201 throw("out of memory")
203 if s.allocCount != 0 {
204 throw("bad allocCount")
206 if s.manualFreeList.ptr() != nil {
207 throw("bad manualFreeList")
210 s.elemsize = fixedStack << order
211 for i := uintptr(0); i < _StackCacheSize; i += s.elemsize {
212 x := gclinkptr(s.base() + i)
213 x.ptr().next = s.manualFreeList
218 x := s.manualFreeList
220 throw("span has no free stacks")
222 s.manualFreeList = x.ptr().next
224 if s.manualFreeList.ptr() == nil {
225 // all stacks in s are allocated.
231 // Adds stack x to the free pool. Must be called with stackpool[order].item.mu held.
232 func stackpoolfree(x gclinkptr, order uint8) {
233 s := spanOfUnchecked(uintptr(x))
234 if s.state.get() != mSpanManual {
235 throw("freeing stack not in a stack span")
237 if s.manualFreeList.ptr() == nil {
238 // s will now have a free stack
239 stackpool[order].item.span.insert(s)
241 x.ptr().next = s.manualFreeList
244 if gcphase == _GCoff && s.allocCount == 0 {
245 // Span is completely free. Return it to the heap
246 // immediately if we're sweeping.
248 // If GC is active, we delay the free until the end of
249 // GC to avoid the following type of situation:
251 // 1) GC starts, scans a SudoG but does not yet mark the SudoG.elem pointer
252 // 2) The stack that pointer points to is copied
253 // 3) The old stack is freed
254 // 4) The containing span is marked free
255 // 5) GC attempts to mark the SudoG.elem pointer. The
256 // marking fails because the pointer looks like a
257 // pointer into a free span.
259 // By not freeing, we prevent step #4 until GC is done.
260 stackpool[order].item.span.remove(s)
263 mheap_.freeManual(s, spanAllocStack)
267 // stackcacherefill/stackcacherelease implement a global pool of stack segments.
268 // The pool is required to prevent unlimited growth of per-thread caches.
271 func stackcacherefill(c *mcache, order uint8) {
273 print("stackcacherefill order=", order, "\n")
276 // Grab some stacks from the global cache.
277 // Grab half of the allowed capacity (to prevent thrashing).
280 lock(&stackpool[order].item.mu)
281 for size < _StackCacheSize/2 {
282 x := stackpoolalloc(order)
285 size += fixedStack << order
287 unlock(&stackpool[order].item.mu)
288 c.stackcache[order].list = list
289 c.stackcache[order].size = size
293 func stackcacherelease(c *mcache, order uint8) {
295 print("stackcacherelease order=", order, "\n")
297 x := c.stackcache[order].list
298 size := c.stackcache[order].size
299 lock(&stackpool[order].item.mu)
300 for size > _StackCacheSize/2 {
302 stackpoolfree(x, order)
304 size -= fixedStack << order
306 unlock(&stackpool[order].item.mu)
307 c.stackcache[order].list = x
308 c.stackcache[order].size = size
312 func stackcache_clear(c *mcache) {
314 print("stackcache clear\n")
316 for order := uint8(0); order < _NumStackOrders; order++ {
317 lock(&stackpool[order].item.mu)
318 x := c.stackcache[order].list
321 stackpoolfree(x, order)
324 c.stackcache[order].list = 0
325 c.stackcache[order].size = 0
326 unlock(&stackpool[order].item.mu)
330 // stackalloc allocates an n byte stack.
332 // stackalloc must run on the system stack because it uses per-P
333 // resources and must not split the stack.
336 func stackalloc(n uint32) stack {
337 // Stackalloc must be called on scheduler stack, so that we
338 // never try to grow the stack during the code that stackalloc runs.
339 // Doing so would cause a deadlock (issue 1547).
341 if thisg != thisg.m.g0 {
342 throw("stackalloc not on scheduler stack")
345 throw("stack size not a power of 2")
348 print("stackalloc ", n, "\n")
351 if debug.efence != 0 || stackFromSystem != 0 {
352 n = uint32(alignUp(uintptr(n), physPageSize))
353 v := sysAlloc(uintptr(n), &memstats.stacks_sys)
355 throw("out of memory (stackalloc)")
357 return stack{uintptr(v), uintptr(v) + uintptr(n)}
360 // Small stacks are allocated with a fixed-size free-list allocator.
361 // If we need a stack of a bigger size, we fall back on allocating
364 if n < fixedStack<<_NumStackOrders && n < _StackCacheSize {
367 for n2 > fixedStack {
372 if stackNoCache != 0 || thisg.m.p == 0 || thisg.m.preemptoff != "" {
373 // thisg.m.p == 0 can happen in the guts of exitsyscall
374 // or procresize. Just get a stack from the global pool.
375 // Also don't touch stackcache during gc
376 // as it's flushed concurrently.
377 lock(&stackpool[order].item.mu)
378 x = stackpoolalloc(order)
379 unlock(&stackpool[order].item.mu)
381 c := thisg.m.p.ptr().mcache
382 x = c.stackcache[order].list
384 stackcacherefill(c, order)
385 x = c.stackcache[order].list
387 c.stackcache[order].list = x.ptr().next
388 c.stackcache[order].size -= uintptr(n)
390 v = unsafe.Pointer(x)
393 npage := uintptr(n) >> _PageShift
394 log2npage := stacklog2(npage)
396 // Try to get a stack from the large stack cache.
397 lock(&stackLarge.lock)
398 if !stackLarge.free[log2npage].isEmpty() {
399 s = stackLarge.free[log2npage].first
400 stackLarge.free[log2npage].remove(s)
402 unlock(&stackLarge.lock)
404 lockWithRankMayAcquire(&mheap_.lock, lockRankMheap)
407 // Allocate a new stack from the heap.
408 s = mheap_.allocManual(npage, spanAllocStack)
410 throw("out of memory")
413 s.elemsize = uintptr(n)
415 v = unsafe.Pointer(s.base())
419 racemalloc(v, uintptr(n))
422 msanmalloc(v, uintptr(n))
425 asanunpoison(v, uintptr(n))
428 print(" allocated ", v, "\n")
430 return stack{uintptr(v), uintptr(v) + uintptr(n)}
433 // stackfree frees an n byte stack allocation at stk.
435 // stackfree must run on the system stack because it uses per-P
436 // resources and must not split the stack.
439 func stackfree(stk stack) {
441 v := unsafe.Pointer(stk.lo)
444 throw("stack not a power of 2")
446 if stk.lo+n < stk.hi {
447 throw("bad stack size")
450 println("stackfree", v, n)
451 memclrNoHeapPointers(v, n) // for testing, clobber stack data
453 if debug.efence != 0 || stackFromSystem != 0 {
454 if debug.efence != 0 || stackFaultOnFree != 0 {
457 sysFree(v, n, &memstats.stacks_sys)
467 if n < fixedStack<<_NumStackOrders && n < _StackCacheSize {
470 for n2 > fixedStack {
475 if stackNoCache != 0 || gp.m.p == 0 || gp.m.preemptoff != "" {
476 lock(&stackpool[order].item.mu)
477 stackpoolfree(x, order)
478 unlock(&stackpool[order].item.mu)
480 c := gp.m.p.ptr().mcache
481 if c.stackcache[order].size >= _StackCacheSize {
482 stackcacherelease(c, order)
484 x.ptr().next = c.stackcache[order].list
485 c.stackcache[order].list = x
486 c.stackcache[order].size += n
489 s := spanOfUnchecked(uintptr(v))
490 if s.state.get() != mSpanManual {
491 println(hex(s.base()), v)
492 throw("bad span state")
494 if gcphase == _GCoff {
495 // Free the stack immediately if we're
498 mheap_.freeManual(s, spanAllocStack)
500 // If the GC is running, we can't return a
501 // stack span to the heap because it could be
502 // reused as a heap span, and this state
503 // change would race with GC. Add it to the
504 // large stack cache instead.
505 log2npage := stacklog2(s.npages)
506 lock(&stackLarge.lock)
507 stackLarge.free[log2npage].insert(s)
508 unlock(&stackLarge.lock)
513 var maxstacksize uintptr = 1 << 20 // enough until runtime.main sets it for real
515 var maxstackceiling = maxstacksize
517 var ptrnames = []string{
522 // Stack frame layout
525 // +------------------+
526 // | args from caller |
527 // +------------------+ <- frame->argp
528 // | return address |
529 // +------------------+
530 // | caller's BP (*) | (*) if framepointer_enabled && varp > sp
531 // +------------------+ <- frame->varp
533 // +------------------+
534 // | args to callee |
535 // +------------------+ <- frame->sp
538 // +------------------+
539 // | args from caller |
540 // +------------------+ <- frame->argp
541 // | caller's retaddr |
542 // +------------------+
543 // | caller's FP (*) | (*) on ARM64, if framepointer_enabled && varp > sp
544 // +------------------+ <- frame->varp
546 // +------------------+
547 // | args to callee |
548 // +------------------+
549 // | return address |
550 // +------------------+ <- frame->sp
552 // varp > sp means that the function has a frame;
553 // varp == sp means frameless function.
555 type adjustinfo struct {
557 delta uintptr // ptr distance from old to new stack (newbase - oldbase)
559 // sghi is the highest sudog.elem on the stack.
563 // adjustpointer checks whether *vpp is in the old stack described by adjinfo.
564 // If so, it rewrites *vpp to point into the new stack.
565 func adjustpointer(adjinfo *adjustinfo, vpp unsafe.Pointer) {
566 pp := (*uintptr)(vpp)
569 print(" ", pp, ":", hex(p), "\n")
571 if adjinfo.old.lo <= p && p < adjinfo.old.hi {
572 *pp = p + adjinfo.delta
574 print(" adjust ptr ", pp, ":", hex(p), " -> ", hex(*pp), "\n")
579 // Information from the compiler about the layout of stack frames.
580 // Note: this type must agree with reflect.bitVector.
581 type bitvector struct {
586 // ptrbit returns the i'th bit in bv.
587 // ptrbit is less efficient than iterating directly over bitvector bits,
588 // and should only be used in non-performance-critical code.
589 // See adjustpointers for an example of a high-efficiency walk of a bitvector.
590 func (bv *bitvector) ptrbit(i uintptr) uint8 {
591 b := *(addb(bv.bytedata, i/8))
592 return (b >> (i % 8)) & 1
595 // bv describes the memory starting at address scanp.
596 // Adjust any pointers contained therein.
597 func adjustpointers(scanp unsafe.Pointer, bv *bitvector, adjinfo *adjustinfo, f funcInfo) {
598 minp := adjinfo.old.lo
599 maxp := adjinfo.old.hi
600 delta := adjinfo.delta
602 // If this frame might contain channel receive slots, use CAS
603 // to adjust pointers. If the slot hasn't been received into
604 // yet, it may contain stack pointers and a concurrent send
605 // could race with adjusting those pointers. (The sent value
606 // itself can never contain stack pointers.)
607 useCAS := uintptr(scanp) < adjinfo.sghi
608 for i := uintptr(0); i < num; i += 8 {
610 for j := uintptr(0); j < 8; j++ {
611 print(" ", add(scanp, (i+j)*goarch.PtrSize), ":", ptrnames[bv.ptrbit(i+j)], ":", hex(*(*uintptr)(add(scanp, (i+j)*goarch.PtrSize))), " # ", i, " ", *addb(bv.bytedata, i/8), "\n")
614 b := *(addb(bv.bytedata, i/8))
616 j := uintptr(sys.TrailingZeros8(b))
618 pp := (*uintptr)(add(scanp, (i+j)*goarch.PtrSize))
621 if f.valid() && 0 < p && p < minLegalPointer && debug.invalidptr != 0 {
622 // Looks like a junk value in a pointer slot.
623 // Live analysis wrong?
624 getg().m.traceback = 2
625 print("runtime: bad pointer in frame ", funcname(f), " at ", pp, ": ", hex(p), "\n")
626 throw("invalid pointer found on stack")
628 if minp <= p && p < maxp {
630 print("adjust ptr ", hex(p), " ", funcname(f), "\n")
633 ppu := (*unsafe.Pointer)(unsafe.Pointer(pp))
634 if !atomic.Casp1(ppu, unsafe.Pointer(p), unsafe.Pointer(p+delta)) {
645 // Note: the argument/return area is adjusted by the callee.
646 func adjustframe(frame *stkframe, adjinfo *adjustinfo) {
647 if frame.continpc == 0 {
653 print(" adjusting ", funcname(f), " frame=[", hex(frame.sp), ",", hex(frame.fp), "] pc=", hex(frame.pc), " continpc=", hex(frame.continpc), "\n")
656 // Adjust saved frame pointer if there is one.
657 if (goarch.ArchFamily == goarch.AMD64 || goarch.ArchFamily == goarch.ARM64) && frame.argp-frame.varp == 2*goarch.PtrSize {
662 // Frame pointers should always point to the next higher frame on
663 // the Go stack (or be nil, for the top frame on the stack).
664 bp := *(*uintptr)(unsafe.Pointer(frame.varp))
665 if bp != 0 && (bp < adjinfo.old.lo || bp >= adjinfo.old.hi) {
666 println("runtime: found invalid frame pointer")
667 print("bp=", hex(bp), " min=", hex(adjinfo.old.lo), " max=", hex(adjinfo.old.hi), "\n")
668 throw("bad frame pointer")
671 // On AMD64, this is the caller's frame pointer saved in the current
673 // On ARM64, this is the frame pointer of the caller's caller saved
674 // by the caller in its frame (one word below its SP).
675 adjustpointer(adjinfo, unsafe.Pointer(frame.varp))
678 locals, args, objs := frame.getStackMap(true)
680 // Adjust local variables if stack frame has been allocated.
682 size := uintptr(locals.n) * goarch.PtrSize
683 adjustpointers(unsafe.Pointer(frame.varp-size), &locals, adjinfo, f)
691 adjustpointers(unsafe.Pointer(frame.argp), &args, adjinfo, funcInfo{})
694 // Adjust pointers in all stack objects (whether they are live or not).
695 // See comments in mgcmark.go:scanframeworker.
697 for i := range objs {
700 base := frame.varp // locals base pointer
702 base = frame.argp // arguments and return values base pointer
704 p := base + uintptr(off)
706 // Object hasn't been allocated in the frame yet.
707 // (Happens when the stack bounds check fails and
708 // we call into morestack.)
711 ptrdata := obj.ptrdata()
712 gcdata := obj.gcdata()
715 // See comments in mgcmark.go:scanstack
716 s = materializeGCProg(ptrdata, gcdata)
717 gcdata = (*byte)(unsafe.Pointer(s.startAddr))
719 for i := uintptr(0); i < ptrdata; i += goarch.PtrSize {
720 if *addb(gcdata, i/(8*goarch.PtrSize))>>(i/goarch.PtrSize&7)&1 != 0 {
721 adjustpointer(adjinfo, unsafe.Pointer(p+i))
725 dematerializeGCProg(s)
731 func adjustctxt(gp *g, adjinfo *adjustinfo) {
732 adjustpointer(adjinfo, unsafe.Pointer(&gp.sched.ctxt))
733 if !framepointer_enabled {
738 if bp != 0 && (bp < adjinfo.old.lo || bp >= adjinfo.old.hi) {
739 println("runtime: found invalid top frame pointer")
740 print("bp=", hex(bp), " min=", hex(adjinfo.old.lo), " max=", hex(adjinfo.old.hi), "\n")
741 throw("bad top frame pointer")
745 adjustpointer(adjinfo, unsafe.Pointer(&gp.sched.bp))
746 if GOARCH == "arm64" {
747 // On ARM64, the frame pointer is saved one word *below* the SP,
748 // which is not copied or adjusted in any frame. Do it explicitly
750 if oldfp == gp.sched.sp-goarch.PtrSize {
751 memmove(unsafe.Pointer(gp.sched.bp), unsafe.Pointer(oldfp), goarch.PtrSize)
752 adjustpointer(adjinfo, unsafe.Pointer(gp.sched.bp))
757 func adjustdefers(gp *g, adjinfo *adjustinfo) {
758 // Adjust pointers in the Defer structs.
759 // We need to do this first because we need to adjust the
760 // defer.link fields so we always work on the new stack.
761 adjustpointer(adjinfo, unsafe.Pointer(&gp._defer))
762 for d := gp._defer; d != nil; d = d.link {
763 adjustpointer(adjinfo, unsafe.Pointer(&d.fn))
764 adjustpointer(adjinfo, unsafe.Pointer(&d.sp))
765 adjustpointer(adjinfo, unsafe.Pointer(&d.link))
769 func adjustpanics(gp *g, adjinfo *adjustinfo) {
770 // Panics are on stack and already adjusted.
771 // Update pointer to head of list in G.
772 adjustpointer(adjinfo, unsafe.Pointer(&gp._panic))
775 func adjustsudogs(gp *g, adjinfo *adjustinfo) {
776 // the data elements pointed to by a SudoG structure
777 // might be in the stack.
778 for s := gp.waiting; s != nil; s = s.waitlink {
779 adjustpointer(adjinfo, unsafe.Pointer(&s.elem))
783 func fillstack(stk stack, b byte) {
784 for p := stk.lo; p < stk.hi; p++ {
785 *(*byte)(unsafe.Pointer(p)) = b
789 func findsghi(gp *g, stk stack) uintptr {
791 for sg := gp.waiting; sg != nil; sg = sg.waitlink {
792 p := uintptr(sg.elem) + uintptr(sg.c.elemsize)
793 if stk.lo <= p && p < stk.hi && p > sghi {
800 // syncadjustsudogs adjusts gp's sudogs and copies the part of gp's
801 // stack they refer to while synchronizing with concurrent channel
802 // operations. It returns the number of bytes of stack copied.
803 func syncadjustsudogs(gp *g, used uintptr, adjinfo *adjustinfo) uintptr {
804 if gp.waiting == nil {
808 // Lock channels to prevent concurrent send/receive.
810 for sg := gp.waiting; sg != nil; sg = sg.waitlink {
812 // There is a ranking cycle here between gscan bit and
813 // hchan locks. Normally, we only allow acquiring hchan
814 // locks and then getting a gscan bit. In this case, we
815 // already have the gscan bit. We allow acquiring hchan
816 // locks here as a special case, since a deadlock can't
817 // happen because the G involved must already be
818 // suspended. So, we get a special hchan lock rank here
819 // that is lower than gscan, but doesn't allow acquiring
820 // any other locks other than hchan.
821 lockWithRank(&sg.c.lock, lockRankHchanLeaf)
827 adjustsudogs(gp, adjinfo)
829 // Copy the part of the stack the sudogs point in to
830 // while holding the lock to prevent races on
831 // send/receive slots.
833 if adjinfo.sghi != 0 {
834 oldBot := adjinfo.old.hi - used
835 newBot := oldBot + adjinfo.delta
836 sgsize = adjinfo.sghi - oldBot
837 memmove(unsafe.Pointer(newBot), unsafe.Pointer(oldBot), sgsize)
842 for sg := gp.waiting; sg != nil; sg = sg.waitlink {
852 // Copies gp's stack to a new stack of a different size.
853 // Caller must have changed gp status to Gcopystack.
854 func copystack(gp *g, newsize uintptr) {
855 if gp.syscallsp != 0 {
856 throw("stack growth not allowed in system call")
860 throw("nil stackbase")
862 used := old.hi - gp.sched.sp
863 // Add just the difference to gcController.addScannableStack.
864 // g0 stacks never move, so this will never account for them.
865 // It's also fine if we have no P, addScannableStack can deal with
867 gcController.addScannableStack(getg().m.p.ptr(), int64(newsize)-int64(old.hi-old.lo))
869 // allocate new stack
870 new := stackalloc(uint32(newsize))
871 if stackPoisonCopy != 0 {
875 print("copystack gp=", gp, " [", hex(old.lo), " ", hex(old.hi-used), " ", hex(old.hi), "]", " -> [", hex(new.lo), " ", hex(new.hi-used), " ", hex(new.hi), "]/", newsize, "\n")
878 // Compute adjustment.
879 var adjinfo adjustinfo
881 adjinfo.delta = new.hi - old.hi
883 // Adjust sudogs, synchronizing with channel ops if necessary.
885 if !gp.activeStackChans {
886 if newsize < old.hi-old.lo && gp.parkingOnChan.Load() {
887 // It's not safe for someone to shrink this stack while we're actively
888 // parking on a channel, but it is safe to grow since we do that
889 // ourselves and explicitly don't want to synchronize with channels
890 // since we could self-deadlock.
891 throw("racy sudog adjustment due to parking on channel")
893 adjustsudogs(gp, &adjinfo)
895 // sudogs may be pointing in to the stack and gp has
896 // released channel locks, so other goroutines could
897 // be writing to gp's stack. Find the highest such
898 // pointer so we can handle everything there and below
899 // carefully. (This shouldn't be far from the bottom
900 // of the stack, so there's little cost in handling
901 // everything below it carefully.)
902 adjinfo.sghi = findsghi(gp, old)
904 // Synchronize with channel ops and copy the part of
905 // the stack they may interact with.
906 ncopy -= syncadjustsudogs(gp, used, &adjinfo)
909 // Copy the stack (or the rest of it) to the new location
910 memmove(unsafe.Pointer(new.hi-ncopy), unsafe.Pointer(old.hi-ncopy), ncopy)
912 // Adjust remaining structures that have pointers into stacks.
913 // We have to do most of these before we traceback the new
914 // stack because gentraceback uses them.
915 adjustctxt(gp, &adjinfo)
916 adjustdefers(gp, &adjinfo)
917 adjustpanics(gp, &adjinfo)
918 if adjinfo.sghi != 0 {
919 adjinfo.sghi += adjinfo.delta
922 // Swap out old stack for new one
924 gp.stackguard0 = new.lo + stackGuard // NOTE: might clobber a preempt request
925 gp.sched.sp = new.hi - used
926 gp.stktopsp += adjinfo.delta
928 // Adjust pointers in the new stack.
930 for u.init(gp, 0); u.valid(); u.next() {
931 adjustframe(&u.frame, &adjinfo)
935 if stackPoisonCopy != 0 {
941 // round x up to a power of 2.
942 func round2(x int32) int32 {
950 // Called from runtime·morestack when more stack is needed.
951 // Allocate larger stack and relocate to new stack.
952 // Stack growth is multiplicative, for constant amortized cost.
954 // g->atomicstatus will be Grunning or Gscanrunning upon entry.
955 // If the scheduler is trying to stop this g, then it will set preemptStop.
957 // This must be nowritebarrierrec because it can be called as part of
958 // stack growth from other nowritebarrierrec functions, but the
959 // compiler doesn't check this.
961 //go:nowritebarrierrec
964 // TODO: double check all gp. shouldn't be getg().
965 if thisg.m.morebuf.g.ptr().stackguard0 == stackFork {
966 throw("stack growth after fork")
968 if thisg.m.morebuf.g.ptr() != thisg.m.curg {
969 print("runtime: newstack called from g=", hex(thisg.m.morebuf.g), "\n"+"\tm=", thisg.m, " m->curg=", thisg.m.curg, " m->g0=", thisg.m.g0, " m->gsignal=", thisg.m.gsignal, "\n")
970 morebuf := thisg.m.morebuf
971 traceback(morebuf.pc, morebuf.sp, morebuf.lr, morebuf.g.ptr())
972 throw("runtime: wrong goroutine in newstack")
977 if thisg.m.curg.throwsplit {
978 // Update syscallsp, syscallpc in case traceback uses them.
979 morebuf := thisg.m.morebuf
980 gp.syscallsp = morebuf.sp
981 gp.syscallpc = morebuf.pc
982 pcname, pcoff := "(unknown)", uintptr(0)
983 f := findfunc(gp.sched.pc)
986 pcoff = gp.sched.pc - f.entry()
988 print("runtime: newstack at ", pcname, "+", hex(pcoff),
989 " sp=", hex(gp.sched.sp), " stack=[", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n",
990 "\tmorebuf={pc:", hex(morebuf.pc), " sp:", hex(morebuf.sp), " lr:", hex(morebuf.lr), "}\n",
991 "\tsched={pc:", hex(gp.sched.pc), " sp:", hex(gp.sched.sp), " lr:", hex(gp.sched.lr), " ctxt:", gp.sched.ctxt, "}\n")
993 thisg.m.traceback = 2 // Include runtime frames
994 traceback(morebuf.pc, morebuf.sp, morebuf.lr, gp)
995 throw("runtime: stack split at bad time")
998 morebuf := thisg.m.morebuf
999 thisg.m.morebuf.pc = 0
1000 thisg.m.morebuf.lr = 0
1001 thisg.m.morebuf.sp = 0
1002 thisg.m.morebuf.g = 0
1004 // NOTE: stackguard0 may change underfoot, if another thread
1005 // is about to try to preempt gp. Read it just once and use that same
1006 // value now and below.
1007 stackguard0 := atomic.Loaduintptr(&gp.stackguard0)
1009 // Be conservative about where we preempt.
1010 // We are interested in preempting user Go code, not runtime code.
1011 // If we're holding locks, mallocing, or preemption is disabled, don't
1013 // This check is very early in newstack so that even the status change
1014 // from Grunning to Gwaiting and back doesn't happen in this case.
1015 // That status change by itself can be viewed as a small preemption,
1016 // because the GC might change Gwaiting to Gscanwaiting, and then
1017 // this goroutine has to wait for the GC to finish before continuing.
1018 // If the GC is in some way dependent on this goroutine (for example,
1019 // it needs a lock held by the goroutine), that small preemption turns
1020 // into a real deadlock.
1021 preempt := stackguard0 == stackPreempt
1023 if !canPreemptM(thisg.m) {
1024 // Let the goroutine keep running for now.
1025 // gp->preempt is set, so it will be preempted next time.
1026 gp.stackguard0 = gp.stack.lo + stackGuard
1027 gogo(&gp.sched) // never return
1031 if gp.stack.lo == 0 {
1032 throw("missing stack in newstack")
1035 if goarch.ArchFamily == goarch.AMD64 || goarch.ArchFamily == goarch.I386 || goarch.ArchFamily == goarch.WASM {
1036 // The call to morestack cost a word.
1037 sp -= goarch.PtrSize
1039 if stackDebug >= 1 || sp < gp.stack.lo {
1040 print("runtime: newstack sp=", hex(sp), " stack=[", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n",
1041 "\tmorebuf={pc:", hex(morebuf.pc), " sp:", hex(morebuf.sp), " lr:", hex(morebuf.lr), "}\n",
1042 "\tsched={pc:", hex(gp.sched.pc), " sp:", hex(gp.sched.sp), " lr:", hex(gp.sched.lr), " ctxt:", gp.sched.ctxt, "}\n")
1044 if sp < gp.stack.lo {
1045 print("runtime: gp=", gp, ", goid=", gp.goid, ", gp->status=", hex(readgstatus(gp)), "\n ")
1046 print("runtime: split stack overflow: ", hex(sp), " < ", hex(gp.stack.lo), "\n")
1047 throw("runtime: split stack overflow")
1051 if gp == thisg.m.g0 {
1052 throw("runtime: preempt g0")
1054 if thisg.m.p == 0 && thisg.m.locks == 0 {
1055 throw("runtime: g is running but p is not")
1058 if gp.preemptShrink {
1059 // We're at a synchronous safe point now, so
1060 // do the pending stack shrink.
1061 gp.preemptShrink = false
1066 preemptPark(gp) // never returns
1069 // Act like goroutine called runtime.Gosched.
1070 gopreempt_m(gp) // never return
1073 // Allocate a bigger segment and move the stack.
1074 oldsize := gp.stack.hi - gp.stack.lo
1075 newsize := oldsize * 2
1077 // Make sure we grow at least as much as needed to fit the new frame.
1078 // (This is just an optimization - the caller of morestack will
1079 // recheck the bounds on return.)
1080 if f := findfunc(gp.sched.pc); f.valid() {
1081 max := uintptr(funcMaxSPDelta(f))
1082 needed := max + stackGuard
1083 used := gp.stack.hi - gp.sched.sp
1084 for newsize-used < needed {
1089 if stackguard0 == stackForceMove {
1090 // Forced stack movement used for debugging.
1091 // Don't double the stack (or we may quickly run out
1092 // if this is done repeatedly).
1096 if newsize > maxstacksize || newsize > maxstackceiling {
1097 if maxstacksize < maxstackceiling {
1098 print("runtime: goroutine stack exceeds ", maxstacksize, "-byte limit\n")
1100 print("runtime: goroutine stack exceeds ", maxstackceiling, "-byte limit\n")
1102 print("runtime: sp=", hex(sp), " stack=[", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n")
1103 throw("stack overflow")
1106 // The goroutine must be executing in order to call newstack,
1107 // so it must be Grunning (or Gscanrunning).
1108 casgstatus(gp, _Grunning, _Gcopystack)
1110 // The concurrent GC will not scan the stack while we are doing the copy since
1111 // the gp is in a Gcopystack status.
1112 copystack(gp, newsize)
1113 if stackDebug >= 1 {
1114 print("stack grow done\n")
1116 casgstatus(gp, _Gcopystack, _Grunning)
1125 // adjust Gobuf as if it executed a call to fn
1126 // and then stopped before the first instruction in fn.
1127 func gostartcallfn(gobuf *gobuf, fv *funcval) {
1128 var fn unsafe.Pointer
1130 fn = unsafe.Pointer(fv.fn)
1132 fn = unsafe.Pointer(abi.FuncPCABIInternal(nilfunc))
1134 gostartcall(gobuf, fn, unsafe.Pointer(fv))
1137 // isShrinkStackSafe returns whether it's safe to attempt to shrink
1138 // gp's stack. Shrinking the stack is only safe when we have precise
1139 // pointer maps for all frames on the stack.
1140 func isShrinkStackSafe(gp *g) bool {
1141 // We can't copy the stack if we're in a syscall.
1142 // The syscall might have pointers into the stack and
1143 // often we don't have precise pointer maps for the innermost
1146 // We also can't copy the stack if we're at an asynchronous
1147 // safe-point because we don't have precise pointer maps for
1150 // We also can't *shrink* the stack in the window between the
1151 // goroutine calling gopark to park on a channel and
1152 // gp.activeStackChans being set.
1153 return gp.syscallsp == 0 && !gp.asyncSafePoint && !gp.parkingOnChan.Load()
1156 // Maybe shrink the stack being used by gp.
1158 // gp must be stopped and we must own its stack. It may be in
1159 // _Grunning, but only if this is our own user G.
1160 func shrinkstack(gp *g) {
1161 if gp.stack.lo == 0 {
1162 throw("missing stack in shrinkstack")
1164 if s := readgstatus(gp); s&_Gscan == 0 {
1165 // We don't own the stack via _Gscan. We could still
1166 // own it if this is our own user G and we're on the
1168 if !(gp == getg().m.curg && getg() != getg().m.curg && s == _Grunning) {
1169 // We don't own the stack.
1170 throw("bad status in shrinkstack")
1173 if !isShrinkStackSafe(gp) {
1174 throw("shrinkstack at bad time")
1176 // Check for self-shrinks while in a libcall. These may have
1177 // pointers into the stack disguised as uintptrs, but these
1178 // code paths should all be nosplit.
1179 if gp == getg().m.curg && gp.m.libcallsp != 0 {
1180 throw("shrinking stack in libcall")
1183 if debug.gcshrinkstackoff > 0 {
1186 f := findfunc(gp.startpc)
1187 if f.valid() && f.funcID == abi.FuncID_gcBgMarkWorker {
1188 // We're not allowed to shrink the gcBgMarkWorker
1189 // stack (see gcBgMarkWorker for explanation).
1193 oldsize := gp.stack.hi - gp.stack.lo
1194 newsize := oldsize / 2
1195 // Don't shrink the allocation below the minimum-sized stack
1197 if newsize < fixedStack {
1200 // Compute how much of the stack is currently in use and only
1201 // shrink the stack if gp is using less than a quarter of its
1202 // current stack. The currently used stack includes everything
1203 // down to the SP plus the stack guard space that ensures
1204 // there's room for nosplit functions.
1205 avail := gp.stack.hi - gp.stack.lo
1206 if used := gp.stack.hi - gp.sched.sp + stackNosplit; used >= avail/4 {
1211 print("shrinking stack ", oldsize, "->", newsize, "\n")
1214 copystack(gp, newsize)
1217 // freeStackSpans frees unused stack spans at the end of GC.
1218 func freeStackSpans() {
1219 // Scan stack pools for empty stack spans.
1220 for order := range stackpool {
1221 lock(&stackpool[order].item.mu)
1222 list := &stackpool[order].item.span
1223 for s := list.first; s != nil; {
1225 if s.allocCount == 0 {
1227 s.manualFreeList = 0
1229 mheap_.freeManual(s, spanAllocStack)
1233 unlock(&stackpool[order].item.mu)
1236 // Free large stack spans.
1237 lock(&stackLarge.lock)
1238 for i := range stackLarge.free {
1239 for s := stackLarge.free[i].first; s != nil; {
1241 stackLarge.free[i].remove(s)
1243 mheap_.freeManual(s, spanAllocStack)
1247 unlock(&stackLarge.lock)
1250 // A stackObjectRecord is generated by the compiler for each stack object in a stack frame.
1251 // This record must match the generator code in cmd/compile/internal/liveness/plive.go:emitStackObjects.
1252 type stackObjectRecord struct {
1254 // if negative, offset from varp
1255 // if non-negative, offset from argp
1258 _ptrdata int32 // ptrdata, or -ptrdata is GC prog is used
1259 gcdataoff uint32 // offset to gcdata from moduledata.rodata
1262 func (r *stackObjectRecord) useGCProg() bool {
1263 return r._ptrdata < 0
1266 func (r *stackObjectRecord) ptrdata() uintptr {
1274 // gcdata returns pointer map or GC prog of the type.
1275 func (r *stackObjectRecord) gcdata() *byte {
1276 ptr := uintptr(unsafe.Pointer(r))
1278 for datap := &firstmoduledata; datap != nil; datap = datap.next {
1279 if datap.gofunc <= ptr && ptr < datap.end {
1284 // If you get a panic here due to a nil mod,
1285 // you may have made a copy of a stackObjectRecord.
1286 // You must use the original pointer.
1287 res := mod.rodata + uintptr(r.gcdataoff)
1288 return (*byte)(unsafe.Pointer(res))
1291 // This is exported as ABI0 via linkname so obj can call it.
1294 //go:linkname morestackc
1296 throw("attempt to execute system stack code on user stack")
1299 // startingStackSize is the amount of stack that new goroutines start with.
1300 // It is a power of 2, and between _FixedStack and maxstacksize, inclusive.
1301 // startingStackSize is updated every GC by tracking the average size of
1302 // stacks scanned during the GC.
1303 var startingStackSize uint32 = fixedStack
1305 func gcComputeStartingStackSize() {
1306 if debug.adaptivestackstart == 0 {
1309 // For details, see the design doc at
1310 // https://docs.google.com/document/d/1YDlGIdVTPnmUiTAavlZxBI1d9pwGQgZT7IKFKlIXohQ/edit?usp=sharing
1311 // The basic algorithm is to track the average size of stacks
1312 // and start goroutines with stack equal to that average size.
1313 // Starting at the average size uses at most 2x the space that
1314 // an ideal algorithm would have used.
1315 // This is just a heuristic to avoid excessive stack growth work
1316 // early in a goroutine's lifetime. See issue 18138. Stacks that
1317 // are allocated too small can still grow, and stacks allocated
1318 // too large can still shrink.
1319 var scannedStackSize uint64
1320 var scannedStacks uint64
1321 for _, p := range allp {
1322 scannedStackSize += p.scannedStackSize
1323 scannedStacks += p.scannedStacks
1324 // Reset for next time
1325 p.scannedStackSize = 0
1328 if scannedStacks == 0 {
1329 startingStackSize = fixedStack
1332 avg := scannedStackSize/scannedStacks + stackGuard
1333 // Note: we add stackGuard to ensure that a goroutine that
1334 // uses the average space will not trigger a growth.
1335 if avg > uint64(maxstacksize) {
1336 avg = uint64(maxstacksize)
1338 if avg < fixedStack {
1341 // Note: maxstacksize fits in 30 bits, so avg also does.
1342 startingStackSize = uint32(round2(int32(avg)))