1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
8 "runtime/internal/atomic"
19 // If you add to this list, add to the list
20 // of "okay during garbage collection" status
23 _Grunnable // 1 runnable and on a run queue
27 _Gmoribund_unused // 5 currently unused, but hardcoded in gdb scripts
29 _Genqueue // 7 Only the Gscanenqueue is used.
30 _Gcopystack // 8 in this state when newstack is moving the stack
31 // the following encode that the GC is scanning the stack and what to do when it is done
32 _Gscan = 0x1000 // atomicstatus&~Gscan = the non-scan state,
33 // _Gscanidle = _Gscan + _Gidle, // Not used. Gidle only used with newly malloced gs
34 _Gscanrunnable = _Gscan + _Grunnable // 0x1001 When scanning completes make Grunnable (it is already on run queue)
35 _Gscanrunning = _Gscan + _Grunning // 0x1002 Used to tell preemption newstack routine to scan preempted stack.
36 _Gscansyscall = _Gscan + _Gsyscall // 0x1003 When scanning completes make it Gsyscall
37 _Gscanwaiting = _Gscan + _Gwaiting // 0x1004 When scanning completes make it Gwaiting
38 // _Gscanmoribund_unused, // not possible
39 // _Gscandead, // not possible
40 _Gscanenqueue = _Gscan + _Genqueue // When scanning completes make it Grunnable and put on runqueue
46 _Prunning // Only this P is allowed to change from _Prunning.
53 // Futex-based impl treats it as uint32 key,
54 // while sema-based impl as M* waitm.
55 // Used to be a union, but unions break precise GC.
60 // Futex-based impl treats it as uint32 key,
61 // while sema-based impl as M* waitm.
62 // Used to be a union, but unions break precise GC.
68 // variable-size, fn-specific data here
81 func efaceOf(ep *interface{}) *eface {
82 return (*eface)(unsafe.Pointer(ep))
85 // The guintptr, muintptr, and puintptr are all used to bypass write barriers.
86 // It is particularly important to avoid write barriers when the current P has
87 // been released, because the GC thinks the world is stopped, and an
88 // unexpected write barrier would not be synchronized with the GC,
89 // which can lead to a half-executed write barrier that has marked the object
90 // but not queued it. If the GC skips the object and completes before the
91 // queuing can occur, it will incorrectly free the object.
93 // We tried using special assignment functions invoked only when not
94 // holding a running P, but then some updates to a particular memory
95 // word went through write barriers and some did not. This breaks the
96 // write barrier shadow checking mode, and it is also scary: better to have
97 // a word that is completely ignored by the GC than to have one for which
98 // only a few updates are ignored.
100 // Gs, Ms, and Ps are always reachable via true pointers in the
101 // allgs, allm, and allp lists or (during allocation before they reach those lists)
102 // from stack variables.
104 // A guintptr holds a goroutine pointer, but typed as a uintptr
105 // to bypass write barriers. It is used in the Gobuf goroutine state
106 // and in scheduling lists that are manipulated without a P.
108 // The Gobuf.g goroutine pointer is almost always updated by assembly code.
109 // In one of the few places it is updated by Go code - func save - it must be
110 // treated as a uintptr to avoid a write barrier being emitted at a bad time.
111 // Instead of figuring out how to emit the write barriers missing in the
112 // assembly manipulation, we change the type of the field to uintptr,
113 // so that it does not require write barriers at all.
115 // Goroutine structs are published in the allg list and never freed.
116 // That will keep the goroutine structs from being collected.
117 // There is never a time that Gobuf.g's contain the only references
118 // to a goroutine: the publishing of the goroutine in allg comes first.
119 // Goroutine pointers are also kept in non-GC-visible places like TLS,
120 // so I can't see them ever moving. If we did want to start moving data
121 // in the GC, we'd need to allocate the goroutine structs from an
122 // alternate arena. Using guintptr doesn't make that problem any worse.
123 type guintptr uintptr
125 func (gp guintptr) ptr() *g { return (*g)(unsafe.Pointer(gp)) }
126 func (gp *guintptr) set(g *g) { *gp = guintptr(unsafe.Pointer(g)) }
127 func (gp *guintptr) cas(old, new guintptr) bool {
128 return atomic.Casuintptr((*uintptr)(unsafe.Pointer(gp)), uintptr(old), uintptr(new))
131 type puintptr uintptr
133 func (pp puintptr) ptr() *p { return (*p)(unsafe.Pointer(pp)) }
134 func (pp *puintptr) set(p *p) { *pp = puintptr(unsafe.Pointer(p)) }
136 type muintptr uintptr
138 func (mp muintptr) ptr() *m { return (*m)(unsafe.Pointer(mp)) }
139 func (mp *muintptr) set(m *m) { *mp = muintptr(unsafe.Pointer(m)) }
142 // The offsets of sp, pc, and g are known to (hard-coded in) libmach.
146 ctxt unsafe.Pointer // this has to be a pointer so that gc scans it
149 bp uintptr // for GOEXPERIMENT=framepointer
152 // Known to compiler.
153 // Changes here must also be made in src/cmd/internal/gc/select.go's selecttype.
159 elem unsafe.Pointer // data element
161 nrelease int32 // -1 for acquire
162 waitlink *sudog // g.waiting list
165 type gcstats struct {
166 // the struct must consist of only uint64's,
167 // because it is casted to uint64[].
175 type libcall struct {
177 n uintptr // number of parameters
178 args uintptr // parameters
179 r1 uintptr // return values
181 err uintptr // error number
184 // describes how to handle callback
185 type wincallbackcontext struct {
186 gobody unsafe.Pointer // go function to call
187 argsize uintptr // callback arguments size (in bytes)
188 restorestack uintptr // adjust stack on return by (in bytes) (386 only)
192 // Stack describes a Go execution stack.
193 // The bounds of the stack are exactly [lo, hi),
194 // with no implicit data structures on either side.
200 // stkbar records the state of a G's stack barrier.
202 savedLRPtr uintptr // location overwritten by stack barrier PC
203 savedLRVal uintptr // value overwritten at savedLRPtr
208 // stack describes the actual stack memory: [stack.lo, stack.hi).
209 // stackguard0 is the stack pointer compared in the Go stack growth prologue.
210 // It is stack.lo+StackGuard normally, but can be StackPreempt to trigger a preemption.
211 // stackguard1 is the stack pointer compared in the C stack growth prologue.
212 // It is stack.lo+StackGuard on g0 and gsignal stacks.
213 // It is ~0 on other goroutine stacks, to trigger a call to morestackc (and crash).
214 stack stack // offset known to runtime/cgo
215 stackguard0 uintptr // offset known to liblink
216 stackguard1 uintptr // offset known to liblink
218 _panic *_panic // innermost panic - offset known to liblink
219 _defer *_defer // innermost defer
220 m *m // current m; offset known to arm liblink
221 stackAlloc uintptr // stack allocation is [stack.lo,stack.lo+stackAlloc)
223 syscallsp uintptr // if status==Gsyscall, syscallsp = sched.sp to use during gc
224 syscallpc uintptr // if status==Gsyscall, syscallpc = sched.pc to use during gc
225 stkbar []stkbar // stack barriers, from low to high
226 stkbarPos uintptr // index of lowest stack barrier not hit
227 stktopsp uintptr // expected sp at top of stack, to check in traceback
228 param unsafe.Pointer // passed parameter on wakeup
230 stackLock uint32 // sigprof/scang lock; TODO: fold in to atomicstatus
232 waitsince int64 // approx time when the g become blocked
233 waitreason string // if status==Gwaiting
235 preempt bool // preemption signal, duplicates stackguard0 = stackpreempt
236 paniconfault bool // panic (instead of crash) on unexpected fault address
237 preemptscan bool // preempted g does scan for gc
238 gcscandone bool // g has scanned stack; protected by _Gscan bit in status
239 gcscanvalid bool // false at start of gc cycle, true if G has not run since last scan
240 throwsplit bool // must not split stack
241 raceignore int8 // ignore race detection events
242 sysblocktraced bool // StartTrace has emitted EvGoInSyscall about this goroutine
243 sysexitticks int64 // cputicks when syscall has returned (for tracing)
244 sysexitseq uint64 // trace seq when syscall has returned (for tracing)
251 gopc uintptr // pc of go statement that created this goroutine
252 startpc uintptr // pc of goroutine function
254 waiting *sudog // sudog structures this g is waiting on (that have a valid elem ptr)
256 // Per-G gcController state
258 // gcAssistBytes is this G's GC assist credit in terms of
259 // bytes allocated. If this is positive, then the G has credit
260 // to allocate gcAssistBytes bytes without assisting. If this
261 // is negative, then the G must correct this by performing
262 // scan work. We track this in bytes to make it fast to update
263 // and check for debt in the malloc hot path. The assist ratio
264 // determines how this corresponds to scan work debt.
269 g0 *g // goroutine with scheduling stack
270 morebuf gobuf // gobuf arg to morestack
271 divmod uint32 // div/mod denominator for arm - known to liblink
273 // Fields not known to debuggers.
274 procid uint64 // for debuggers, but offset not hard-coded
275 gsignal *g // signal-handling g
276 sigmask [4]uintptr // storage for saved signal mask
277 tls [4]uintptr // thread-local storage (for x86 extern register)
279 curg *g // current running goroutine
280 caughtsig guintptr // goroutine running during fatal signal
281 p puintptr // attached p for executing go code (nil if not executing go code)
286 preemptoff string // if != "", keep curg running on this m
292 spinning bool // m is out of work and is actively looking for work
293 blocked bool // m is blocked on a note
294 inwb bool // m is executing a write barrier
297 ncgocall uint64 // number of cgo calls in total
298 ncgo int32 // number of cgo calls currently in progress
300 alllink *m // on allm
302 machport uint32 // return address for mach ipc (os x)
305 createstack [32]uintptr // stack that created this thread.
306 freglo [16]uint32 // d[i] lsb and f[i]
307 freghi [16]uint32 // d[i] msb and f[i+16]
308 fflag uint32 // floating point compare flags
309 locked uint32 // tracking for lockosthread
310 nextwaitm uintptr // next m waiting for lock
311 waitsema uintptr // semaphore for parking on locks
317 waitunlockf unsafe.Pointer // todo go func(*g, unsafe.pointer) bool
318 waitlock unsafe.Pointer
323 //#ifdef GOOS_windows
324 thread uintptr // thread handle
325 // these are here because they are too large to be on the stack
326 // of low-level NOSPLIT functions.
328 libcallpc uintptr // for cpu profiler
331 syscall libcall // stores syscall parameters on windows
340 status uint32 // one of pidle/prunning/...
342 schedtick uint32 // incremented on every scheduler call
343 syscalltick uint32 // incremented on every system call
344 m muintptr // back-link to associated m (nil if idle)
347 deferpool [5][]*_defer // pool of available defer structs of different sizes (see panic.go)
348 deferpoolbuf [5][32]*_defer
350 // Cache of goroutine ids, amortizes accesses to runtimeĀ·sched.goidgen.
354 // Queue of runnable goroutines. Accessed without lock.
358 // runnext, if non-nil, is a runnable G that was ready'd by
359 // the current G and should be run next instead of what's in
360 // runq if there's time remaining in the running G's time
361 // slice. It will inherit the time left in the current time
362 // slice. If a set of goroutines is locked in a
363 // communicate-and-wait pattern, this schedules that set as a
364 // unit and eliminates the (potentially large) scheduling
365 // latency that otherwise arises from adding the ready'd
366 // goroutines to the end of the run queue.
369 // Available G's (status == Gdead)
378 palloc persistentAlloc // per-P to avoid mutex
381 gcAssistTime int64 // Nanoseconds in assistAlloc
383 gcMarkWorkerMode gcMarkWorkerMode
385 // gcw is this P's GC work buffer cache. The work buffer is
386 // filled by write barriers, drained by mutator assists, and
387 // disposed on certain GC state transitions.
390 runSafePointFn uint32 // if 1, run sched.safePointFn at next safe point
396 // The max value of GOMAXPROCS.
397 // There are no fundamental restrictions on the value.
398 _MaxGomaxprocs = 1 << 8
406 midle muintptr // idle m's waiting for work
407 nmidle int32 // number of idle m's waiting for work
408 nmidlelocked int32 // number of locked m's waiting for work
409 mcount int32 // number of m's that have been created
410 maxmcount int32 // maximum number of m's allowed (or die)
412 pidle puintptr // idle p's
414 nmspinning uint32 // limited to [0, 2^31-1]
416 // Global runnable queue.
421 // Global cache of dead G's.
426 // Central cache of sudog structs.
430 // Central pool of available defer structs of different sizes.
434 gcwaiting uint32 // gc is waiting to run
441 // safepointFn should be called on each P at the next GC
442 // safepoint if p.runSafePointFn is set.
447 profilehz int32 // cpu profiling rate
449 procresizetime int64 // nanotime() of last change to gomaxprocs
450 totaltime int64 // ā«gomaxprocs dt up to procresizetime
453 // The m->locked word holds two pieces of state counting active calls to LockOSThread/lockOSThread.
454 // The low bit (LockExternal) is a boolean reporting whether any LockOSThread call is active.
455 // External locks are not recursive; a second lock is silently ignored.
456 // The upper bits of m->locked record the nesting depth of calls to lockOSThread
457 // (counting up by LockInternal), popped by unlockOSThread (counting down by LockInternal).
458 // Internal locks can be recursive. For instance, a lock for cgo can occur while the main
459 // goroutine is holding the lock during the initialization phase.
465 type sigtabtt struct {
471 _SigNotify = 1 << iota // let signal.Notify have signal, even if from kernel
472 _SigKill // if signal.Notify doesn't take it, exit quietly
473 _SigThrow // if signal.Notify doesn't take it, exit loudly
474 _SigPanic // if the signal is from the kernel, panic
475 _SigDefault // if the signal isn't explicitly requested, don't monitor it
476 _SigHandling // our signal handler is registered
477 _SigIgnored // the signal was ignored before we registered for it
478 _SigGoExit // cause all runtime procs to exit (only used on Plan 9).
479 _SigSetStack // add SA_ONSTACK to libc handler
480 _SigUnblock // unblocked in minit
483 // Layout of in-memory per-function information prepared by linker
484 // See https://golang.org/s/go12symtab.
485 // Keep in sync with linker
486 // and with package debug/gosym and with symtab.go in package runtime.
488 entry uintptr // start pc
489 nameoff int32 // function name
491 args int32 // in/out args size
492 _ int32 // Previously: legacy frame size. TODO: Remove this.
501 // layout of Itab known to compilers
502 // allocated in non-garbage-collected memory
509 fun [1]uintptr // variable sized
512 // Lock-free stack node.
513 // // Also known to export_test.go.
519 type forcegcstate struct {
529 _Structrnd = sys.RegSize
532 // startup_random_data holds random bytes initialized at startup. These come from
533 // the ELF AT_RANDOM auxiliary vector (vdso_linux_amd64.go or os_linux_386.go).
534 var startupRandomData []byte
536 // extendRandom extends the random numbers in r[:n] to the whole slice r.
537 // Treats n<0 as n==0.
538 func extendRandom(r []byte, n int) {
543 // Extend random bits using hash function & time seed
548 h := memhash(unsafe.Pointer(&r[n-w]), uintptr(nanotime()), uintptr(w))
549 for i := 0; i < sys.PtrSize && n < len(r); i++ {
558 * deferred subroutine calls
563 sp uintptr // sp at time of defer
566 _panic *_panic // panic that is running defer
574 argp unsafe.Pointer // pointer to arguments of deferred call run during panic; cannot move - known to liblink
575 arg interface{} // argument to panic
576 link *_panic // link to earlier panic
577 recovered bool // whether this panic is over
578 aborted bool // the panic was aborted
585 type stkframe struct {
586 fn *_func // function being run
587 pc uintptr // program counter within fn
588 continpc uintptr // program counter where execution can continue, or 0 if not
589 lr uintptr // program counter at caller aka link register
590 sp uintptr // stack pointer at pc
591 fp uintptr // stack pointer at caller aka frame pointer
592 varp uintptr // top of local variables
593 argp uintptr // pointer to function arguments
594 arglen uintptr // number of bytes at argp
595 argmap *bitvector // force use of this argmap
599 _TraceRuntimeFrames = 1 << iota // include frames for internal runtime functions.
600 _TraceTrap // the initial PC, SP are from a trap, not a return PC from a call
601 _TraceJumpStack // if traceback is on a systemstack, resume trace at g that called into it
605 // The maximum number of frames we print for a traceback
606 _TracebackMaxFrames = 100
613 allp [_MaxGomaxprocs + 1]*p
621 // Information about what cpu features are available.
622 // Set on startup in asm_{x86,amd64}.s.
625 lfenceBeforeRdtsc bool
629 goarm uint8 // set by cmd/link on arm systems
632 // Set by the linker so the runtime can determine the buildmode.
634 islibrary bool // -buildmode=c-shared
635 isarchive bool // -buildmode=c-archive
639 * mutual exclusion locks. in the uncontended case,
640 * as fast as spin locks (just a few user-level instructions),
641 * but on the contention path they sleep in the kernel.
642 * a zeroed Mutex is unlocked (no need to initialize each lock).
646 * sleep and wakeup on one-time events.
647 * before any calls to notesleep or notewakeup,
648 * must call noteclear to initialize the Note.
649 * then, exactly one thread can call notesleep
650 * and exactly one thread can call notewakeup (once).
651 * once notewakeup has been called, the notesleep
652 * will return. future notesleep will return immediately.
653 * subsequent noteclear must be called only after
654 * previous notesleep has returned, e.g. it's disallowed
655 * to call noteclear straight after notewakeup.
657 * notetsleep is like notesleep but wakes up after
658 * a given number of nanoseconds even if the event
659 * has not yet happened. if a goroutine uses notetsleep to
660 * wake up early, it must wait to call noteclear until it
661 * can be sure that no other goroutine is calling
664 * notesleep/notetsleep are generally called on g0,
665 * notetsleepg is similar to notetsleep but is called on user g.
667 // bool runtimeĀ·notetsleep(Note*, int64); // false - timeout
668 // bool runtimeĀ·notetsleepg(Note*, int64); // false - timeout
672 * Initialize uint64 head to 0, compare with 0 to test for emptiness.
673 * The stack does not keep pointers to nodes,
674 * so they can be garbage collected if there are no other pointers to nodes.
677 // for mmap, we only pass the lower 32 bits of file offset to the
678 // assembly routine; the higher bits (if required), should be provided
679 // by the assembly routine as 0.