1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
8 "runtime/internal/atomic"
18 // If you add to this list, add to the list
19 // of "okay during garbage collection" status
22 _Grunnable // 1 runnable and on a run queue
26 _Gmoribund_unused // 5 currently unused, but hardcoded in gdb scripts
28 _Genqueue // 7 Only the Gscanenqueue is used.
29 _Gcopystack // 8 in this state when newstack is moving the stack
30 // the following encode that the GC is scanning the stack and what to do when it is done
31 _Gscan = 0x1000 // atomicstatus&~Gscan = the non-scan state,
32 // _Gscanidle = _Gscan + _Gidle, // Not used. Gidle only used with newly malloced gs
33 _Gscanrunnable = _Gscan + _Grunnable // 0x1001 When scanning completes make Grunnable (it is already on run queue)
34 _Gscanrunning = _Gscan + _Grunning // 0x1002 Used to tell preemption newstack routine to scan preempted stack.
35 _Gscansyscall = _Gscan + _Gsyscall // 0x1003 When scanning completes make it Gsyscall
36 _Gscanwaiting = _Gscan + _Gwaiting // 0x1004 When scanning completes make it Gwaiting
37 // _Gscanmoribund_unused, // not possible
38 // _Gscandead, // not possible
39 _Gscanenqueue = _Gscan + _Genqueue // When scanning completes make it Grunnable and put on runqueue
45 _Prunning // Only this P is allowed to change from _Prunning.
51 // The next line makes 'go generate' write the zgen_*.go files with
52 // per-OS and per-arch information, including constants
53 // named goos_$GOOS and goarch_$GOARCH for every
54 // known GOOS and GOARCH. The constant is 1 on the
55 // current system, 0 otherwise; multiplying by them is
56 // useful for defining GOOS- or GOARCH-specific constants.
57 //go:generate go run gengoos.go
60 // Futex-based impl treats it as uint32 key,
61 // while sema-based impl as M* waitm.
62 // Used to be a union, but unions break precise GC.
67 // Futex-based impl treats it as uint32 key,
68 // while sema-based impl as M* waitm.
69 // Used to be a union, but unions break precise GC.
75 // variable-size, fn-specific data here
88 func efaceOf(ep *interface{}) *eface {
89 return (*eface)(unsafe.Pointer(ep))
92 // The guintptr, muintptr, and puintptr are all used to bypass write barriers.
93 // It is particularly important to avoid write barriers when the current P has
94 // been released, because the GC thinks the world is stopped, and an
95 // unexpected write barrier would not be synchronized with the GC,
96 // which can lead to a half-executed write barrier that has marked the object
97 // but not queued it. If the GC skips the object and completes before the
98 // queuing can occur, it will incorrectly free the object.
100 // We tried using special assignment functions invoked only when not
101 // holding a running P, but then some updates to a particular memory
102 // word went through write barriers and some did not. This breaks the
103 // write barrier shadow checking mode, and it is also scary: better to have
104 // a word that is completely ignored by the GC than to have one for which
105 // only a few updates are ignored.
107 // Gs, Ms, and Ps are always reachable via true pointers in the
108 // allgs, allm, and allp lists or (during allocation before they reach those lists)
109 // from stack variables.
111 // A guintptr holds a goroutine pointer, but typed as a uintptr
112 // to bypass write barriers. It is used in the Gobuf goroutine state
113 // and in scheduling lists that are manipulated without a P.
115 // The Gobuf.g goroutine pointer is almost always updated by assembly code.
116 // In one of the few places it is updated by Go code - func save - it must be
117 // treated as a uintptr to avoid a write barrier being emitted at a bad time.
118 // Instead of figuring out how to emit the write barriers missing in the
119 // assembly manipulation, we change the type of the field to uintptr,
120 // so that it does not require write barriers at all.
122 // Goroutine structs are published in the allg list and never freed.
123 // That will keep the goroutine structs from being collected.
124 // There is never a time that Gobuf.g's contain the only references
125 // to a goroutine: the publishing of the goroutine in allg comes first.
126 // Goroutine pointers are also kept in non-GC-visible places like TLS,
127 // so I can't see them ever moving. If we did want to start moving data
128 // in the GC, we'd need to allocate the goroutine structs from an
129 // alternate arena. Using guintptr doesn't make that problem any worse.
130 type guintptr uintptr
132 func (gp guintptr) ptr() *g { return (*g)(unsafe.Pointer(gp)) }
133 func (gp *guintptr) set(g *g) { *gp = guintptr(unsafe.Pointer(g)) }
134 func (gp *guintptr) cas(old, new guintptr) bool {
135 return atomic.Casuintptr((*uintptr)(unsafe.Pointer(gp)), uintptr(old), uintptr(new))
138 type puintptr uintptr
140 func (pp puintptr) ptr() *p { return (*p)(unsafe.Pointer(pp)) }
141 func (pp *puintptr) set(p *p) { *pp = puintptr(unsafe.Pointer(p)) }
143 type muintptr uintptr
145 func (mp muintptr) ptr() *m { return (*m)(unsafe.Pointer(mp)) }
146 func (mp *muintptr) set(m *m) { *mp = muintptr(unsafe.Pointer(m)) }
149 // The offsets of sp, pc, and g are known to (hard-coded in) libmach.
153 ctxt unsafe.Pointer // this has to be a pointer so that gc scans it
156 bp uintptr // for GOEXPERIMENT=framepointer
159 // Known to compiler.
160 // Changes here must also be made in src/cmd/internal/gc/select.go's selecttype.
166 elem unsafe.Pointer // data element
168 nrelease int32 // -1 for acquire
169 waitlink *sudog // g.waiting list
172 type gcstats struct {
173 // the struct must consist of only uint64's,
174 // because it is casted to uint64[].
182 type libcall struct {
184 n uintptr // number of parameters
185 args uintptr // parameters
186 r1 uintptr // return values
188 err uintptr // error number
191 // describes how to handle callback
192 type wincallbackcontext struct {
193 gobody unsafe.Pointer // go function to call
194 argsize uintptr // callback arguments size (in bytes)
195 restorestack uintptr // adjust stack on return by (in bytes) (386 only)
199 // Stack describes a Go execution stack.
200 // The bounds of the stack are exactly [lo, hi),
201 // with no implicit data structures on either side.
207 // stkbar records the state of a G's stack barrier.
209 savedLRPtr uintptr // location overwritten by stack barrier PC
210 savedLRVal uintptr // value overwritten at savedLRPtr
215 // stack describes the actual stack memory: [stack.lo, stack.hi).
216 // stackguard0 is the stack pointer compared in the Go stack growth prologue.
217 // It is stack.lo+StackGuard normally, but can be StackPreempt to trigger a preemption.
218 // stackguard1 is the stack pointer compared in the C stack growth prologue.
219 // It is stack.lo+StackGuard on g0 and gsignal stacks.
220 // It is ~0 on other goroutine stacks, to trigger a call to morestackc (and crash).
221 stack stack // offset known to runtime/cgo
222 stackguard0 uintptr // offset known to liblink
223 stackguard1 uintptr // offset known to liblink
225 _panic *_panic // innermost panic - offset known to liblink
226 _defer *_defer // innermost defer
227 m *m // current m; offset known to arm liblink
228 stackAlloc uintptr // stack allocation is [stack.lo,stack.lo+stackAlloc)
230 syscallsp uintptr // if status==Gsyscall, syscallsp = sched.sp to use during gc
231 syscallpc uintptr // if status==Gsyscall, syscallpc = sched.pc to use during gc
232 stkbar []stkbar // stack barriers, from low to high
233 stkbarPos uintptr // index of lowest stack barrier not hit
234 stktopsp uintptr // expected sp at top of stack, to check in traceback
235 param unsafe.Pointer // passed parameter on wakeup
237 stackLock uint32 // sigprof/scang lock; TODO: fold in to atomicstatus
239 waitsince int64 // approx time when the g become blocked
240 waitreason string // if status==Gwaiting
242 preempt bool // preemption signal, duplicates stackguard0 = stackpreempt
243 paniconfault bool // panic (instead of crash) on unexpected fault address
244 preemptscan bool // preempted g does scan for gc
245 gcscandone bool // g has scanned stack; protected by _Gscan bit in status
246 gcscanvalid bool // false at start of gc cycle, true if G has not run since last scan
247 throwsplit bool // must not split stack
248 raceignore int8 // ignore race detection events
249 sysblocktraced bool // StartTrace has emitted EvGoInSyscall about this goroutine
250 sysexitticks int64 // cputicks when syscall has returned (for tracing)
251 sysexitseq uint64 // trace seq when syscall has returned (for tracing)
258 gopc uintptr // pc of go statement that created this goroutine
259 startpc uintptr // pc of goroutine function
261 waiting *sudog // sudog structures this g is waiting on (that have a valid elem ptr)
263 // Per-G gcController state
265 // gcAssistBytes is this G's GC assist credit in terms of
266 // bytes allocated. If this is positive, then the G has credit
267 // to allocate gcAssistBytes bytes without assisting. If this
268 // is negative, then the G must correct this by performing
269 // scan work. We track this in bytes to make it fast to update
270 // and check for debt in the malloc hot path. The assist ratio
271 // determines how this corresponds to scan work debt.
276 g0 *g // goroutine with scheduling stack
277 morebuf gobuf // gobuf arg to morestack
278 divmod uint32 // div/mod denominator for arm - known to liblink
280 // Fields not known to debuggers.
281 procid uint64 // for debuggers, but offset not hard-coded
282 gsignal *g // signal-handling g
283 sigmask [4]uintptr // storage for saved signal mask
284 tls [4]uintptr // thread-local storage (for x86 extern register)
286 curg *g // current running goroutine
287 caughtsig guintptr // goroutine running during fatal signal
288 p puintptr // attached p for executing go code (nil if not executing go code)
293 preemptoff string // if != "", keep curg running on this m
299 spinning bool // m is out of work and is actively looking for work
300 blocked bool // m is blocked on a note
301 inwb bool // m is executing a write barrier
304 ncgocall uint64 // number of cgo calls in total
305 ncgo int32 // number of cgo calls currently in progress
307 alllink *m // on allm
309 machport uint32 // return address for mach ipc (os x)
312 createstack [32]uintptr // stack that created this thread.
313 freglo [16]uint32 // d[i] lsb and f[i]
314 freghi [16]uint32 // d[i] msb and f[i+16]
315 fflag uint32 // floating point compare flags
316 locked uint32 // tracking for lockosthread
317 nextwaitm uintptr // next m waiting for lock
318 waitsema uintptr // semaphore for parking on locks
324 waitunlockf unsafe.Pointer // todo go func(*g, unsafe.pointer) bool
325 waitlock unsafe.Pointer
330 //#ifdef GOOS_windows
331 thread uintptr // thread handle
332 // these are here because they are too large to be on the stack
333 // of low-level NOSPLIT functions.
335 libcallpc uintptr // for cpu profiler
338 syscall libcall // stores syscall parameters on windows
347 status uint32 // one of pidle/prunning/...
349 schedtick uint32 // incremented on every scheduler call
350 syscalltick uint32 // incremented on every system call
351 m muintptr // back-link to associated m (nil if idle)
354 deferpool [5][]*_defer // pool of available defer structs of different sizes (see panic.go)
355 deferpoolbuf [5][32]*_defer
357 // Cache of goroutine ids, amortizes accesses to runtime·sched.goidgen.
361 // Queue of runnable goroutines. Accessed without lock.
365 // runnext, if non-nil, is a runnable G that was ready'd by
366 // the current G and should be run next instead of what's in
367 // runq if there's time remaining in the running G's time
368 // slice. It will inherit the time left in the current time
369 // slice. If a set of goroutines is locked in a
370 // communicate-and-wait pattern, this schedules that set as a
371 // unit and eliminates the (potentially large) scheduling
372 // latency that otherwise arises from adding the ready'd
373 // goroutines to the end of the run queue.
376 // Available G's (status == Gdead)
385 palloc persistentAlloc // per-P to avoid mutex
388 gcAssistTime int64 // Nanoseconds in assistAlloc
390 gcMarkWorkerMode gcMarkWorkerMode
392 // gcw is this P's GC work buffer cache. The work buffer is
393 // filled by write barriers, drained by mutator assists, and
394 // disposed on certain GC state transitions.
397 runSafePointFn uint32 // if 1, run sched.safePointFn at next safe point
403 // The max value of GOMAXPROCS.
404 // There are no fundamental restrictions on the value.
405 _MaxGomaxprocs = 1 << 8
413 midle muintptr // idle m's waiting for work
414 nmidle int32 // number of idle m's waiting for work
415 nmidlelocked int32 // number of locked m's waiting for work
416 mcount int32 // number of m's that have been created
417 maxmcount int32 // maximum number of m's allowed (or die)
419 pidle puintptr // idle p's
421 nmspinning uint32 // limited to [0, 2^31-1]
423 // Global runnable queue.
428 // Global cache of dead G's.
433 // Central cache of sudog structs.
437 // Central pool of available defer structs of different sizes.
441 gcwaiting uint32 // gc is waiting to run
448 // safepointFn should be called on each P at the next GC
449 // safepoint if p.runSafePointFn is set.
454 profilehz int32 // cpu profiling rate
456 procresizetime int64 // nanotime() of last change to gomaxprocs
457 totaltime int64 // ∫gomaxprocs dt up to procresizetime
460 // The m->locked word holds two pieces of state counting active calls to LockOSThread/lockOSThread.
461 // The low bit (LockExternal) is a boolean reporting whether any LockOSThread call is active.
462 // External locks are not recursive; a second lock is silently ignored.
463 // The upper bits of m->locked record the nesting depth of calls to lockOSThread
464 // (counting up by LockInternal), popped by unlockOSThread (counting down by LockInternal).
465 // Internal locks can be recursive. For instance, a lock for cgo can occur while the main
466 // goroutine is holding the lock during the initialization phase.
472 type sigtabtt struct {
478 _SigNotify = 1 << iota // let signal.Notify have signal, even if from kernel
479 _SigKill // if signal.Notify doesn't take it, exit quietly
480 _SigThrow // if signal.Notify doesn't take it, exit loudly
481 _SigPanic // if the signal is from the kernel, panic
482 _SigDefault // if the signal isn't explicitly requested, don't monitor it
483 _SigHandling // our signal handler is registered
484 _SigIgnored // the signal was ignored before we registered for it
485 _SigGoExit // cause all runtime procs to exit (only used on Plan 9).
486 _SigSetStack // add SA_ONSTACK to libc handler
487 _SigUnblock // unblocked in minit
490 // Layout of in-memory per-function information prepared by linker
491 // See https://golang.org/s/go12symtab.
492 // Keep in sync with linker
493 // and with package debug/gosym and with symtab.go in package runtime.
495 entry uintptr // start pc
496 nameoff int32 // function name
498 args int32 // in/out args size
499 _ int32 // Previously: legacy frame size. TODO: Remove this.
508 // layout of Itab known to compilers
509 // allocated in non-garbage-collected memory
516 fun [1]uintptr // variable sized
519 // Lock-free stack node.
520 // // Also known to export_test.go.
526 type forcegcstate struct {
539 // startup_random_data holds random bytes initialized at startup. These come from
540 // the ELF AT_RANDOM auxiliary vector (vdso_linux_amd64.go or os_linux_386.go).
541 var startupRandomData []byte
543 // extendRandom extends the random numbers in r[:n] to the whole slice r.
544 // Treats n<0 as n==0.
545 func extendRandom(r []byte, n int) {
550 // Extend random bits using hash function & time seed
555 h := memhash(unsafe.Pointer(&r[n-w]), uintptr(nanotime()), uintptr(w))
556 for i := 0; i < ptrSize && n < len(r); i++ {
565 * deferred subroutine calls
570 sp uintptr // sp at time of defer
573 _panic *_panic // panic that is running defer
581 argp unsafe.Pointer // pointer to arguments of deferred call run during panic; cannot move - known to liblink
582 arg interface{} // argument to panic
583 link *_panic // link to earlier panic
584 recovered bool // whether this panic is over
585 aborted bool // the panic was aborted
592 type stkframe struct {
593 fn *_func // function being run
594 pc uintptr // program counter within fn
595 continpc uintptr // program counter where execution can continue, or 0 if not
596 lr uintptr // program counter at caller aka link register
597 sp uintptr // stack pointer at pc
598 fp uintptr // stack pointer at caller aka frame pointer
599 varp uintptr // top of local variables
600 argp uintptr // pointer to function arguments
601 arglen uintptr // number of bytes at argp
602 argmap *bitvector // force use of this argmap
606 _TraceRuntimeFrames = 1 << iota // include frames for internal runtime functions.
607 _TraceTrap // the initial PC, SP are from a trap, not a return PC from a call
608 _TraceJumpStack // if traceback is on a systemstack, resume trace at g that called into it
612 // The maximum number of frames we print for a traceback
613 _TracebackMaxFrames = 100
620 allp [_MaxGomaxprocs + 1]*p
628 // Information about what cpu features are available.
629 // Set on startup in asm_{x86,amd64}.s.
632 lfenceBeforeRdtsc bool
636 goarm uint8 // set by cmd/link on arm systems
639 // Set by the linker so the runtime can determine the buildmode.
641 islibrary bool // -buildmode=c-shared
642 isarchive bool // -buildmode=c-archive
646 * mutual exclusion locks. in the uncontended case,
647 * as fast as spin locks (just a few user-level instructions),
648 * but on the contention path they sleep in the kernel.
649 * a zeroed Mutex is unlocked (no need to initialize each lock).
653 * sleep and wakeup on one-time events.
654 * before any calls to notesleep or notewakeup,
655 * must call noteclear to initialize the Note.
656 * then, exactly one thread can call notesleep
657 * and exactly one thread can call notewakeup (once).
658 * once notewakeup has been called, the notesleep
659 * will return. future notesleep will return immediately.
660 * subsequent noteclear must be called only after
661 * previous notesleep has returned, e.g. it's disallowed
662 * to call noteclear straight after notewakeup.
664 * notetsleep is like notesleep but wakes up after
665 * a given number of nanoseconds even if the event
666 * has not yet happened. if a goroutine uses notetsleep to
667 * wake up early, it must wait to call noteclear until it
668 * can be sure that no other goroutine is calling
671 * notesleep/notetsleep are generally called on g0,
672 * notetsleepg is similar to notetsleep but is called on user g.
674 // bool runtime·notetsleep(Note*, int64); // false - timeout
675 // bool runtime·notetsleepg(Note*, int64); // false - timeout
679 * Initialize uint64 head to 0, compare with 0 to test for emptiness.
680 * The stack does not keep pointers to nodes,
681 * so they can be garbage collected if there are no other pointers to nodes.
684 // for mmap, we only pass the lower 32 bits of file offset to the
685 // assembly routine; the higher bits (if required), should be provided
686 // by the assembly routine as 0.