1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
14 // Goroutine scheduler
15 // The scheduler's job is to distribute ready-to-run goroutines over worker threads.
17 // The main concepts are:
19 // M - worker thread, or machine.
20 // P - processor, a resource that is required to execute Go code.
21 // M must have an associated P to execute Go code, however it can be
22 // blocked or in a syscall w/o an associated P.
24 // Design doc at http://golang.org/s/go11sched.
27 // Number of goroutine ids to grab from sched.goidgen to local per-P cache at once.
28 // 16 seems to provide enough amortization, but other than that it's mostly arbitrary number.
38 G g0; // idle goroutine for m0
42 P* allp[MaxGomaxprocs+1];
47 Mutex allglock; // the following vars are protected by this lock or by stoptheworld
54 static void runqput(P*, G*);
55 static G* runqget(P*);
56 static bool runqputslow(P*, G*, uint32, uint32);
57 static G* runqsteal(P*, P*);
60 static void mcommoninit(M*);
61 static void schedule(void);
62 static void procresize(int32);
63 static void acquirep(P*);
64 static P* releasep(void);
65 static void newm(void(*)(void), P*);
66 static void stopm(void);
67 static void startm(P*, bool);
68 static void handoffp(P*);
69 static void wakep(void);
70 static void stoplockedm(void);
71 static void startlockedm(G*);
72 static void sysmon(void);
73 static uint32 retake(int64);
74 static void incidlelocked(int32);
75 static void checkdead(void);
76 static void exitsyscall0(G*);
78 static void goexit0(G*);
79 static void gfput(P*, G*);
81 static void gfpurge(P*);
82 static void globrunqput(G*);
83 static void globrunqputbatch(G*, G*, int32);
84 static G* globrunqget(P*, int32);
85 static P* pidleget(void);
86 static void pidleput(P*);
87 static void injectglist(G*);
88 static bool preemptall(void);
89 static bool preemptone(P*);
90 static bool exitsyscallfast(void);
91 static bool haveexperiment(int8*);
93 static void dropg(void);
95 extern String buildVersion;
98 // The bootstrap sequence is:
102 // make & queue new G
103 // call runtimeĀ·mstart
105 // The new G calls runtimeĀ·main.
107 // raceinit must be the first call to race detector.
108 // In particular, it must be done before mallocinit below calls racemapshadow.
111 _g_.racectx = raceinit()
114 sched.maxmcount = 10000
127 sched.lastpoll = uint64(nanotime())
129 if n := goatoi(gogetenv("GOMAXPROCS")); n > 0 {
130 if n > _MaxGomaxprocs {
135 procresize(int32(procs))
137 if buildVersion == "" {
138 // Condition should never trigger. This code just serves
139 // to ensure runtimeĀ·buildVersion is kept in the resulting binary.
140 buildVersion = "unknown"
148 func dumpgstatus(gp *g) {
150 print("runtime: gp: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", readgstatus(gp), "\n")
151 print("runtime: g: g=", _g_, ", goid=", _g_.goid, ", g->atomicstatus=", readgstatus(_g_), "\n")
155 // sched lock is held
156 if sched.mcount > sched.maxmcount {
157 print("runtime: program exceeds ", sched.maxmcount, "-thread limit\n")
158 gothrow("thread exhaustion")
162 func mcommoninit(mp *m) {
165 // g0 stack won't make sense for user (and is not necessary unwindable).
167 callers(1, &mp.createstack[0], len(mp.createstack))
170 mp.fastrand = 0x49f6428a + uint32(mp.id) + uint32(cputicks())
171 if mp.fastrand == 0 {
172 mp.fastrand = 0x49f6428a
180 if mp.gsignal != nil {
181 mp.gsignal.stackguard1 = mp.gsignal.stack.lo + _StackGuard
184 // Add to allm so garbage collector doesn't free g->m
185 // when it is just in a register or thread-local storage.
188 // NumCgoCall() iterates over allm w/o schedlock,
189 // so we need to publish it safely.
190 atomicstorep(unsafe.Pointer(&allm), unsafe.Pointer(mp))
194 // Mark gp ready to run.
196 status := readgstatus(gp)
200 _g_.m.locks++ // disable preemption because it can be holding p in a local var
201 if status&^_Gscan != _Gwaiting {
203 gothrow("bad g->status in ready")
206 // status is Gwaiting or Gscanwaiting, make Grunnable and put on runq
207 casgstatus(gp, _Gwaiting, _Grunnable)
209 if atomicload(&sched.npidle) != 0 && atomicload(&sched.nmspinning) == 0 { // TODO: fast atomic
213 if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack
214 _g_.stackguard0 = stackPreempt
218 func gcprocs() int32 {
219 // Figure out how many CPUs to use during GC.
220 // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc.
229 if n > sched.nmidle+1 { // one M is currently running
236 func needaddgcproc() bool {
245 n -= sched.nmidle + 1 // one M is currently running
250 func helpgc(nproc int32) {
254 for n := int32(1); n < nproc; n++ { // one M is currently running
255 if allp[pos].mcache == _g_.m.mcache {
260 gothrow("gcprocs inconsistency")
263 mp.mcache = allp[pos].mcache
270 // Similar to stoptheworld but best-effort and can be called several times.
271 // There is no reverse operation, used during crashing.
272 // This function must not lock any mutexes.
273 func freezetheworld() {
277 // stopwait and preemption requests can be lost
278 // due to races with concurrently executing threads,
279 // so try several times
280 for i := 0; i < 5; i++ {
281 // this should tell the scheduler to not start any new goroutines
282 sched.stopwait = 0x7fffffff
283 atomicstore(&sched.gcwaiting, 1)
284 // this should stop running goroutines
286 break // no running goroutines
296 func isscanstatus(status uint32) bool {
297 if status == _Gscan {
298 gothrow("isscanstatus: Bad status Gscan")
300 return status&_Gscan == _Gscan
303 // All reads and writes of g's status go through readgstatus, casgstatus
304 // castogscanstatus, casfrom_Gscanstatus.
306 func readgstatus(gp *g) uint32 {
307 return atomicload(&gp.atomicstatus)
310 // The Gscanstatuses are acting like locks and this releases them.
311 // If it proves to be a performance hit we should be able to make these
312 // simple atomic stores but for now we are going to throw if
313 // we see an inconsistent state.
314 func casfrom_Gscanstatus(gp *g, oldval, newval uint32) {
317 // Check that transition is valid.
320 print("runtime: casfrom_Gscanstatus bad oldval gp=", gp, ", oldval=", hex(oldval), ", newval=", hex(newval), "\n")
322 gothrow("casfrom_Gscanstatus:top gp->status is not in scan state")
327 if newval == oldval&^_Gscan {
328 success = cas(&gp.atomicstatus, oldval, newval)
331 if newval == _Gwaiting {
332 success = cas(&gp.atomicstatus, oldval, newval)
336 print("runtime: casfrom_Gscanstatus failed gp=", gp, ", oldval=", hex(oldval), ", newval=", hex(newval), "\n")
338 gothrow("casfrom_Gscanstatus: gp->status is not in scan state")
342 // This will return false if the gp is not in the expected status and the cas fails.
343 // This acts like a lock acquire while the casfromgstatus acts like a lock release.
344 func castogscanstatus(gp *g, oldval, newval uint32) bool {
349 if newval == oldval|_Gscan {
350 return cas(&gp.atomicstatus, oldval, newval)
353 if newval == _Gscanrunning || newval == _Gscanenqueue {
354 return cas(&gp.atomicstatus, oldval, newval)
357 print("runtime: castogscanstatus oldval=", hex(oldval), " newval=", hex(newval), "\n")
358 gothrow("castogscanstatus")
362 // If asked to move to or from a Gscanstatus this will throw. Use the castogscanstatus
363 // and casfrom_Gscanstatus instead.
364 // casgstatus will loop if the g->atomicstatus is in a Gscan status until the routine that
365 // put it in the Gscan state is finished.
367 func casgstatus(gp *g, oldval, newval uint32) {
368 if (oldval&_Gscan != 0) || (newval&_Gscan != 0) || oldval == newval {
370 print("casgstatus: oldval=", hex(oldval), " newval=", hex(newval), "\n")
371 gothrow("casgstatus: bad incoming values")
375 // loop if gp->atomicstatus is in a scan state giving
376 // GC time to finish and change the state to oldval.
377 for !cas(&gp.atomicstatus, oldval, newval) {
378 if oldval == _Gwaiting && gp.atomicstatus == _Grunnable {
380 gothrow("casgstatus: waiting for Gwaiting but is Grunnable")
383 // Help GC if needed.
384 // if gp.preemptscan && !gp.gcworkdone && (oldval == _Grunning || oldval == _Gsyscall) {
385 // gp.preemptscan = false
386 // systemstack(func() {
393 // casgstatus(gp, oldstatus, Gcopystack), assuming oldstatus is Gwaiting or Grunnable.
394 // Returns old status. Cannot call casgstatus directly, because we are racing with an
395 // async wakeup that might come in from netpoll. If we see Gwaiting from the readgstatus,
396 // it might have become Grunnable by the time we get to the cas. If we called casgstatus,
397 // it would loop waiting for the status to go back to Gwaiting, which it never will.
399 func casgcopystack(gp *g) uint32 {
401 oldstatus := readgstatus(gp) &^ _Gscan
402 if oldstatus != _Gwaiting && oldstatus != _Grunnable {
403 gothrow("copystack: bad status, not Gwaiting or Grunnable")
405 if cas(&gp.atomicstatus, oldstatus, _Gcopystack) {
411 // stopg ensures that gp is stopped at a GC safe point where its stack can be scanned
412 // or in the context of a moving collector the pointers can be flipped from pointing
413 // to old object to pointing to new objects.
414 // If stopg returns true, the caller knows gp is at a GC safe point and will remain there until
415 // the caller calls restartg.
416 // If stopg returns false, the caller is not responsible for calling restartg. This can happen
417 // if another thread, either the gp itself or another GC thread is taking the responsibility
418 // to do the GC work related to this thread.
419 func stopg(gp *g) bool {
425 switch s := readgstatus(gp); s {
428 gothrow("stopg: gp->atomicstatus is not valid")
434 // Loop until a new stack is in place.
439 // Claim goroutine by setting scan bit.
440 if !castogscanstatus(gp, s, s|_Gscan) {
443 // In scan state, do work.
450 // Goroutine already claimed by another GC helper.
454 // Claim goroutine, so we aren't racing with a status
455 // transition away from Grunning.
456 if !castogscanstatus(gp, _Grunning, _Gscanrunning) {
460 // Mark gp for preemption.
462 gp.preemptscan = true
464 gp.stackguard0 = stackPreempt
468 casfrom_Gscanstatus(gp, _Gscanrunning, _Grunning)
474 // The GC requests that this routine be moved from a scanmumble state to a mumble state.
475 func restartg(gp *g) {
480 gothrow("restartg: unexpected status")
488 casfrom_Gscanstatus(gp, s, s&^_Gscan)
490 // Scan is now completed.
491 // Goroutine now needs to be made runnable.
492 // We put it on the global run queue; ready blocks on the global scheduler lock.
494 casfrom_Gscanstatus(gp, _Gscanenqueue, _Gwaiting)
495 if gp != getg().m.curg {
496 gothrow("processing Gscanenqueue on wrong m")
503 func stopscanstart(gp *g) {
506 gothrow("GC not moved to G0")
509 if !isscanstatus(readgstatus(gp)) {
511 gothrow("GC not in scan state")
517 // Runs on g0 and does the actual work after putting the g back on the run queue.
518 func mquiesce(gpmaster *g) {
519 // enqueue the calling goroutine.
522 activeglen := len(allgs)
523 for i := 0; i < activeglen; i++ {
525 if readgstatus(gp) == _Gdead {
526 gp.gcworkdone = true // noop scan.
528 gp.gcworkdone = false
533 // Check that the G's gcwork (such as scanning) has been done. If not do it now.
534 // You can end up doing work here if the page trap on a Grunning Goroutine has
535 // not been sprung or in some race situations. For example a runnable goes dead
536 // and is started up again with a gp->gcworkdone set to false.
537 for i := 0; i < activeglen; i++ {
540 status := readgstatus(gp)
541 if status == _Gdead {
542 //do nothing, scan not needed.
543 gp.gcworkdone = true // scan is a noop
546 if status == _Grunning && gp.stackguard0 == uintptr(stackPreempt) && notetsleep(&sched.stopnote, 100*1000) { // nanosecond arg
547 noteclear(&sched.stopnote)
554 for i := 0; i < activeglen; i++ {
556 status := readgstatus(gp)
557 if isscanstatus(status) {
558 print("mstopandscang:bottom: post scan bad status gp=", gp, " has status ", hex(status), "\n")
561 if !gp.gcworkdone && status != _Gdead {
562 print("mstopandscang:bottom: post scan gp=", gp, "->gcworkdone still false\n")
567 schedule() // Never returns.
570 // quiesce moves all the goroutines to a GC safepoint which for now is a at preemption point.
571 // If the global gcphase is GCmark quiesce will ensure that all of the goroutine's stacks
572 // have been scanned before it returns.
573 func quiesce(mastergp *g) {
574 castogscanstatus(mastergp, _Grunning, _Gscanenqueue)
575 // Now move this to the g0 (aka m) stack.
576 // g0 will potentially scan this thread and put mastergp on the runqueue
580 // This is used by the GC as well as the routines that do stack dumps. In the case
581 // of GC all the routines can be reliably stopped. This is not always the case
582 // when the system is in panic or being exited.
583 func stoptheworld() {
586 // If we hold a lock, then we won't be able to stop another M
587 // that is blocked trying to acquire the lock.
589 gothrow("stoptheworld: holding locks")
593 sched.stopwait = gomaxprocs
594 atomicstore(&sched.gcwaiting, 1)
597 _g_.m.p.status = _Pgcstop // Pgcstop is only diagnostic.
599 // try to retake all P's in Psyscall status
600 for i := 0; i < int(gomaxprocs); i++ {
603 if s == _Psyscall && cas(&p.status, s, _Pgcstop) {
616 wait := sched.stopwait > 0
619 // wait for remaining P's to stop voluntarily
622 // wait for 100us, then try to re-preempt in case of any races
623 if notetsleep(&sched.stopnote, 100*1000) {
624 noteclear(&sched.stopnote)
630 if sched.stopwait != 0 {
631 gothrow("stoptheworld: not stopped")
633 for i := 0; i < int(gomaxprocs); i++ {
635 if p.status != _Pgcstop {
636 gothrow("stoptheworld: not stopped")
646 func starttheworld() {
649 _g_.m.locks++ // disable preemption because it can be holding p in a local var
650 gp := netpoll(false) // non-blocking
652 add := needaddgcproc()
658 procresize(gomaxprocs)
668 // procresize() puts p's with work at the beginning of the list.
669 // Once we reach a p without a run queue, the rest don't have one either.
670 if p.runqhead == p.runqtail {
678 if sched.sysmonwait != 0 {
680 notewakeup(&sched.sysmonnote)
691 gothrow("starttheworld: inconsistent mp->nextp")
696 // Start M to run P. Do not start another M below.
703 // If GC could have used another helper proc, start one now,
704 // in the hope that it will be available next time.
705 // It would have been even better to start it before the collection,
706 // but doing so requires allocating memory, so it's tricky to
707 // coordinate. This lazy approach works out in practice:
708 // we don't mind if the first couple gc rounds don't have quite
709 // the maximum number of procs.
713 if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack
714 _g_.stackguard0 = stackPreempt
718 // Called to start an M.
723 if _g_.stack.lo == 0 {
724 // Initialize stack bounds from system stack.
725 // Cgo may have left stack size in stack.hi.
730 _g_.stack.hi = uintptr(noescape(unsafe.Pointer(&size)))
731 _g_.stack.lo = _g_.stack.hi - size + 1024
733 // Initialize stack guards so that we can start calling
734 // both Go and C functions with stack growth prologues.
735 _g_.stackguard0 = _g_.stack.lo + _StackGuard
736 _g_.stackguard1 = _g_.stackguard0
744 gothrow("bad runtimeĀ·mstart")
747 // Record top of stack for use by mcall.
748 // Once we call schedule we're never coming back,
749 // so other calls can reuse this stack space.
750 gosave(&_g_.m.g0.sched)
751 _g_.m.g0.sched.pc = ^uintptr(0) // make sure it is never used
755 // Install signal handlers; after minit so that minit can
756 // prepare the thread to be able to handle the signals.
761 if _g_.m.mstartfn != nil {
762 fn := *(*func())(unsafe.Pointer(&_g_.m.mstartfn))
766 if _g_.m.helpgc != 0 {
769 } else if _g_.m != &m0 {
770 acquirep(_g_.m.nextp)
775 // TODO(brainman): This point is never reached, because scheduler
776 // does not release os threads at the moment. But once this path
777 // is enabled, we must remove our seh here.
780 // When running with cgo, we call _cgo_thread_start
781 // to start threads for us so that we can play nicely with
783 var cgoThreadStart unsafe.Pointer
785 type cgothreadstart struct {
791 // Allocate a new m unassociated with any thread.
792 // Can use p for allocation context if needed.
793 func allocm(_p_ *p) *m {
795 _g_.m.locks++ // disable GC because it can be called from sysmon
797 acquirep(_p_) // temporarily borrow p for mallocs in this function
802 // In case of cgo or Solaris, pthread_create will make us a stack.
803 // Windows and Plan 9 will layout sched stack on OS stack.
804 if iscgo || GOOS == "solaris" || GOOS == "windows" || GOOS == "plan9" {
815 if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack
816 _g_.stackguard0 = stackPreempt
826 // needm is called when a cgo callback happens on a
827 // thread without an m (a thread not created by Go).
828 // In this case, needm is expected to find an m to use
829 // and return with m, g initialized correctly.
830 // Since m and g are not set now (likely nil, but see below)
831 // needm is limited in what routines it can call. In particular
832 // it can only call nosplit functions (textflag 7) and cannot
833 // do any scheduling that requires an m.
835 // In order to avoid needing heavy lifting here, we adopt
836 // the following strategy: there is a stack of available m's
837 // that can be stolen. Using compare-and-swap
838 // to pop from the stack has ABA races, so we simulate
839 // a lock by doing an exchange (via casp) to steal the stack
840 // head and replace the top pointer with MLOCKED (1).
841 // This serves as a simple spin lock that we can use even
842 // without an m. The thread that locks the stack in this way
843 // unlocks the stack by storing a valid stack head pointer.
845 // In order to make sure that there is always an m structure
846 // available to be stolen, we maintain the invariant that there
847 // is always one more than needed. At the beginning of the
848 // program (if cgo is in use) the list is seeded with a single m.
849 // If needm finds that it has taken the last m off the list, its job
850 // is - once it has installed its own m so that it can do things like
851 // allocate memory - to create a spare m and put it on the list.
853 // Each of these extra m's also has a g0 and a curg that are
854 // pressed into service as the scheduling stack and current
855 // goroutine for the duration of the cgo callback.
857 // When the callback is done with the m, it calls dropm to
858 // put the m back on the list.
862 // Can happen if C/C++ code calls Go from a global ctor.
863 // Can not throw, because scheduler is not initialized yet.
865 // write(2, unsafe.Pointer("fatal error: cgo callback before cgo call\n"), sizeof("fatal error: cgo callback before cgo call\n") - 1)
869 // Lock extra list, take head, unlock popped list.
870 // nilokay=false is safe here because of the invariant above,
871 // that the extra list always contains or will soon contain
873 mp := lockextra(false)
875 // Set needextram when we've just emptied the list,
876 // so that the eventual call into cgocallbackg will
877 // allocate a new m for the extra list. We delay the
878 // allocation until then so that it can be done
879 // after exitsyscall makes sure it is okay to be
880 // running at all (that is, there's no garbage collection
881 // running right now).
882 mp.needextram = mp.schedlink == nil
883 unlockextra(mp.schedlink)
885 // Install g (= m->g0) and set the stack bounds
886 // to match the current stack. We don't actually know
887 // how big the stack is, like we don't know how big any
888 // scheduling stack is, but we assume there's at least 32 kB,
889 // which is more than enough for us.
892 _g_.stack.hi = uintptr(noescape(unsafe.Pointer(&x))) + 1024
893 _g_.stack.lo = uintptr(noescape(unsafe.Pointer(&x))) - 32*1024
894 _g_.stackguard0 = _g_.stack.lo + _StackGuard
896 // Initialize this thread to use the m.
901 // newextram allocates an m and puts it on the extra list.
902 // It is called with a working local m, so that it can do things
903 // like call schedlock and allocate.
905 // Create extra goroutine locked to extra m.
906 // The goroutine is the context in which the cgo callback will run.
907 // The sched.pc will never be returned to, but setting it to
908 // goexit makes clear to the traceback routines where
909 // the goroutine stack ends.
912 gp.sched.pc = funcPC(goexit) + _PCQuantum
913 gp.sched.sp = gp.stack.hi
914 gp.sched.sp -= 4 * regSize // extra space in case of reads slightly beyond frame
917 gp.syscallpc = gp.sched.pc
918 gp.syscallsp = gp.sched.sp
919 // malg returns status as Gidle, change to Gsyscall before adding to allg
920 // where GC will see it.
921 casgstatus(gp, _Gidle, _Gsyscall)
924 mp.locked = _LockInternal
927 gp.goid = int64(xadd64(&sched.goidgen, 1))
929 gp.racectx = racegostart(funcPC(newextram))
931 // put on allg for garbage collector
934 // Add m to the extra list.
935 mnext := lockextra(true)
940 // dropm is called when a cgo callback has called needm but is now
941 // done with the callback and returning back into the non-Go thread.
942 // It puts the current m back onto the extra list.
944 // The main expense here is the call to signalstack to release the
945 // m's signal stack, and then the call to needm on the next callback
946 // from this thread. It is tempting to try to save the m for next time,
947 // which would eliminate both these costs, but there might not be
948 // a next time: the current thread (which Go does not control) might exit.
949 // If we saved the m for that thread, there would be an m leak each time
950 // such a thread exited. Instead, we acquire and release an m on each
951 // call. These should typically not be scheduling operations, just a few
952 // atomics, so the cost should be small.
954 // TODO(rsc): An alternative would be to allocate a dummy pthread per-thread
955 // variable using pthread_key_create. Unlike the pthread keys we already use
956 // on OS X, this dummy key would never be read by Go code. It would exist
957 // only so that we could register at thread-exit-time destructor.
958 // That destructor would put the m back onto the extra list.
959 // This is purely a performance optimization. The current version,
960 // in which dropm happens on each cgo call, is still correct too.
961 // We may have to keep the current version on systems with cgo
962 // but without pthreads, like Windows.
964 // Undo whatever initialization minit did during needm.
967 // Clear m and g, and return m to the extra list.
968 // After the call to setmg we can only call nosplit functions.
972 mnext := lockextra(true)
979 // lockextra locks the extra list and returns the list head.
980 // The caller must unlock the list by storing a new list head
981 // to extram. If nilokay is true, then lockextra will
982 // return a nil list head if that's what it finds. If nilokay is false,
983 // lockextra will keep waiting until the list head is no longer nil.
985 func lockextra(nilokay bool) *m {
989 old := atomicloaduintptr(&extram)
995 if old == 0 && !nilokay {
999 if casuintptr(&extram, old, locked) {
1000 return (*m)(unsafe.Pointer(old))
1009 func unlockextra(mp *m) {
1010 atomicstoreuintptr(&extram, uintptr(unsafe.Pointer(mp)))
1013 // Create a new m. It will start off with a call to fn, or else the scheduler.
1014 func _newm(fn func(), _p_ *p) {
1017 mp.mstartfn = *(*unsafe.Pointer)(unsafe.Pointer(&fn))
1020 var ts cgothreadstart
1021 if _cgo_thread_start == nil {
1022 gothrow("_cgo_thread_start missing")
1025 ts.tls = (*uint64)(unsafe.Pointer(&mp.tls[0]))
1026 ts.fn = unsafe.Pointer(funcPC(mstart))
1027 asmcgocall(_cgo_thread_start, unsafe.Pointer(&ts))
1030 newosproc(mp, unsafe.Pointer(mp.g0.stack.hi))
1033 // Stops execution of the current m until new work is available.
1034 // Returns with acquired P.
1038 if _g_.m.locks != 0 {
1039 gothrow("stopm holding locks")
1042 gothrow("stopm holding p")
1045 _g_.m.spinning = false
1046 xadd(&sched.nmspinning, -1)
1053 notesleep(&_g_.m.park)
1054 noteclear(&_g_.m.park)
1055 if _g_.m.helpgc != 0 {
1061 acquirep(_g_.m.nextp)
1066 getg().m.spinning = true
1069 // Schedules some M to run the p (creates an M if necessary).
1070 // If p==nil, tries to get an idle P, if no idle P's does nothing.
1071 func startm(_p_ *p, spinning bool) {
1078 xadd(&sched.nmspinning, -1)
1094 gothrow("startm: m is spinning")
1096 if mp.nextp != nil {
1097 gothrow("startm: m has p")
1099 mp.spinning = spinning
1101 notewakeup(&mp.park)
1104 // Hands off P from syscall or locked M.
1105 func handoffp(_p_ *p) {
1106 // if it has local work, start it straight away
1107 if _p_.runqhead != _p_.runqtail || sched.runqsize != 0 {
1111 // no local work, check that there are no spinning/idle M's,
1112 // otherwise our help is not required
1113 if atomicload(&sched.nmspinning)+atomicload(&sched.npidle) == 0 && cas(&sched.nmspinning, 0, 1) { // TODO: fast atomic
1118 if sched.gcwaiting != 0 {
1119 _p_.status = _Pgcstop
1121 if sched.stopwait == 0 {
1122 notewakeup(&sched.stopnote)
1127 if sched.runqsize != 0 {
1132 // If this is the last running P and nobody is polling network,
1133 // need to wakeup another M to poll network.
1134 if sched.npidle == uint32(gomaxprocs-1) && atomicload64(&sched.lastpoll) != 0 {
1143 // Tries to add one more P to execute G's.
1144 // Called when a G is made runnable (newproc, ready).
1146 // be conservative about spinning threads
1147 if !cas(&sched.nmspinning, 0, 1) {
1153 // Stops execution of the current m that is locked to a g until the g is runnable again.
1154 // Returns with acquired P.
1155 func stoplockedm() {
1158 if _g_.m.lockedg == nil || _g_.m.lockedg.lockedm != _g_.m {
1159 gothrow("stoplockedm: inconsistent locking")
1162 // Schedule another M to run this p.
1167 // Wait until another thread schedules lockedg again.
1168 notesleep(&_g_.m.park)
1169 noteclear(&_g_.m.park)
1170 status := readgstatus(_g_.m.lockedg)
1171 if status&^_Gscan != _Grunnable {
1172 print("runtime:stoplockedm: g is not Grunnable or Gscanrunnable\n")
1174 gothrow("stoplockedm: not runnable")
1176 acquirep(_g_.m.nextp)
1180 // Schedules the locked m to run the locked gp.
1181 func startlockedm(gp *g) {
1186 gothrow("startlockedm: locked to me")
1188 if mp.nextp != nil {
1189 gothrow("startlockedm: m has p")
1191 // directly handoff current P to the locked m
1195 notewakeup(&mp.park)
1199 // Stops the current m for stoptheworld.
1200 // Returns when the world is restarted.
1204 if sched.gcwaiting == 0 {
1205 gothrow("gcstopm: not waiting for gc")
1208 _g_.m.spinning = false
1209 xadd(&sched.nmspinning, -1)
1213 _p_.status = _Pgcstop
1215 if sched.stopwait == 0 {
1216 notewakeup(&sched.stopnote)
1222 // Schedules gp to run on the current M.
1224 func execute(gp *g) {
1227 casgstatus(gp, _Grunnable, _Grunning)
1230 gp.stackguard0 = gp.stack.lo + _StackGuard
1235 // Check whether the profiler needs to be turned on or off.
1236 hz := sched.profilehz
1237 if _g_.m.profilehz != hz {
1238 resetcpuprofiler(hz)
1244 // Finds a runnable goroutine to execute.
1245 // Tries to steal from other P's, get g from global queue, poll network.
1246 func findrunnable() *g {
1250 if sched.gcwaiting != 0 {
1254 if fingwait && fingwake {
1255 if gp := wakefing(); gp != nil {
1261 if gp := runqget(_g_.m.p); gp != nil {
1266 if sched.runqsize != 0 {
1268 gp := globrunqget(_g_.m.p, 0)
1275 // poll network - returns list of goroutines
1276 if gp := netpoll(false); gp != nil { // non-blocking
1277 injectglist(gp.schedlink)
1278 casgstatus(gp, _Gwaiting, _Grunnable)
1282 // If number of spinning M's >= number of busy P's, block.
1283 // This is necessary to prevent excessive CPU consumption
1284 // when GOMAXPROCS>>1 but the program parallelism is low.
1285 if !_g_.m.spinning && 2*atomicload(&sched.nmspinning) >= uint32(gomaxprocs)-atomicload(&sched.npidle) { // TODO: fast atomic
1288 if !_g_.m.spinning {
1289 _g_.m.spinning = true
1290 xadd(&sched.nmspinning, 1)
1292 // random steal from other P's
1293 for i := 0; i < int(2*gomaxprocs); i++ {
1294 if sched.gcwaiting != 0 {
1297 _p_ := allp[fastrand1()%uint32(gomaxprocs)]
1302 gp = runqsteal(_g_.m.p, _p_)
1310 // return P and block
1312 if sched.gcwaiting != 0 {
1316 if sched.runqsize != 0 {
1317 gp := globrunqget(_g_.m.p, 0)
1325 _g_.m.spinning = false
1326 xadd(&sched.nmspinning, -1)
1329 // check all runqueues once again
1330 for i := 0; i < int(gomaxprocs); i++ {
1332 if _p_ != nil && _p_.runqhead != _p_.runqtail {
1345 if xchg64(&sched.lastpoll, 0) != 0 {
1347 gothrow("findrunnable: netpoll with p")
1350 gothrow("findrunnable: netpoll with spinning")
1352 gp := netpoll(true) // block until new work is available
1353 atomicstore64(&sched.lastpoll, uint64(nanotime()))
1360 injectglist(gp.schedlink)
1361 casgstatus(gp, _Gwaiting, _Grunnable)
1371 func resetspinning() {
1374 var nmspinning uint32
1376 _g_.m.spinning = false
1377 nmspinning = xadd(&sched.nmspinning, -1)
1379 gothrow("findrunnable: negative nmspinning")
1382 nmspinning = atomicload(&sched.nmspinning)
1385 // M wakeup policy is deliberately somewhat conservative (see nmspinning handling),
1386 // so see if we need to wakeup another P here.
1387 if nmspinning == 0 && atomicload(&sched.npidle) > 0 {
1392 // Injects the list of runnable G's into the scheduler.
1393 // Can run concurrently with GC.
1394 func injectglist(glist *g) {
1400 for n = 0; glist != nil; n++ {
1402 glist = gp.schedlink
1403 casgstatus(gp, _Gwaiting, _Grunnable)
1407 for ; n != 0 && sched.npidle != 0; n-- {
1412 // One round of scheduler: find a runnable goroutine and execute it.
1417 if _g_.m.locks != 0 {
1418 gothrow("schedule: holding locks")
1421 if _g_.m.lockedg != nil {
1423 execute(_g_.m.lockedg) // Never returns.
1427 if sched.gcwaiting != 0 {
1433 // Check the global runnable queue once in a while to ensure fairness.
1434 // Otherwise two goroutines can completely occupy the local runqueue
1435 // by constantly respawning each other.
1436 tick := _g_.m.p.schedtick
1437 // This is a fancy way to say tick%61==0,
1438 // it uses 2 MUL instructions instead of a single DIV and so is faster on modern processors.
1439 if uint64(tick)-((uint64(tick)*0x4325c53f)>>36)*61 == 0 && sched.runqsize > 0 {
1441 gp = globrunqget(_g_.m.p, 1)
1448 gp = runqget(_g_.m.p)
1449 if gp != nil && _g_.m.spinning {
1450 gothrow("schedule: spinning with local work")
1454 gp = findrunnable() // blocks until work is available
1458 if gp.lockedm != nil {
1459 // Hands off own p to the locked m,
1460 // then blocks waiting for a new p.
1468 // dropg removes the association between m and the current goroutine m->curg (gp for short).
1469 // Typically a caller sets gp's status away from Grunning and then
1470 // immediately calls dropg to finish the job. The caller is also responsible
1471 // for arranging that gp will be restarted using ready at an
1472 // appropriate time. After calling dropg and arranging for gp to be
1473 // readied later, the caller can do other work but eventually should
1474 // call schedule to restart the scheduling of goroutines on this m.
1478 if _g_.m.lockedg == nil {
1484 // Puts the current goroutine into a waiting state and calls unlockf.
1485 // If unlockf returns false, the goroutine is resumed.
1486 func park(unlockf func(*g, unsafe.Pointer) bool, lock unsafe.Pointer, reason string) {
1489 _g_.m.waitlock = lock
1490 _g_.m.waitunlockf = *(*unsafe.Pointer)(unsafe.Pointer(&unlockf))
1491 _g_.waitreason = reason
1495 func parkunlock_c(gp *g, lock unsafe.Pointer) bool {
1496 unlock((*mutex)(lock))
1500 // Puts the current goroutine into a waiting state and unlocks the lock.
1501 // The goroutine can be made runnable again by calling ready(gp).
1502 func parkunlock(lock *mutex, reason string) {
1503 park(parkunlock_c, unsafe.Pointer(lock), reason)
1506 // park continuation on g0.
1507 func park_m(gp *g) {
1510 casgstatus(gp, _Grunning, _Gwaiting)
1513 if _g_.m.waitunlockf != nil {
1514 fn := *(*func(*g, unsafe.Pointer) bool)(unsafe.Pointer(&_g_.m.waitunlockf))
1515 ok := fn(gp, _g_.m.waitlock)
1516 _g_.m.waitunlockf = nil
1517 _g_.m.waitlock = nil
1519 casgstatus(gp, _Gwaiting, _Grunnable)
1520 execute(gp) // Schedule it back, never returns.
1526 // Gosched continuation on g0.
1527 func gosched_m(gp *g) {
1528 status := readgstatus(gp)
1529 if status&^_Gscan != _Grunning {
1531 gothrow("bad g status")
1533 casgstatus(gp, _Grunning, _Grunnable)
1542 // Finishes execution of the current goroutine.
1543 // Must be NOSPLIT because it is called from Go. (TODO - probably not anymore)
1552 // goexit continuation on g0.
1553 func goexit0(gp *g) {
1556 casgstatus(gp, _Grunning, _Gdead)
1560 gp.paniconfault = false
1561 gp._defer = nil // should be true already but just in case.
1562 gp._panic = nil // non-nil for Goexit during panic. points at stack-allocated data.
1569 if _g_.m.locked&^_LockExternal != 0 {
1570 print("invalid m->locked = ", _g_.m.locked, "\n")
1571 gothrow("internal lockOSThread error")
1579 func save(pc, sp uintptr) {
1586 _g_.sched.ctxt = nil
1587 // write as uintptr to avoid write barrier, which will smash _g_.sched.
1588 *(*uintptr)(unsafe.Pointer(&_g_.sched.g)) = uintptr(unsafe.Pointer(_g_))
1591 // The goroutine g is about to enter a system call.
1592 // Record that it's not using the cpu anymore.
1593 // This is called only from the go syscall library and cgocall,
1594 // not from the low-level system calls used by the
1596 // Entersyscall cannot split the stack: the gosave must
1597 // make g->sched refer to the caller's stack segment, because
1598 // entersyscall is going to return immediately after.
1600 // Nothing entersyscall calls can split the stack either.
1601 // We cannot safely move the stack during an active call to syscall,
1602 // because we do not know which of the uintptr arguments are
1603 // really pointers (back into the stack).
1604 // In practice, this means that we make the fast path run through
1605 // entersyscall doing no-split things, and the slow path has to use systemstack
1606 // to run bigger things on the system stack.
1608 // reentersyscall is the entry point used by cgo callbacks, where explicitly
1609 // saved SP and PC are restored. This is needed when exitsyscall will be called
1610 // from a function further up in the call stack than the parent, as g->syscallsp
1611 // must always point to a valid stack frame. entersyscall below is the normal
1612 // entry point for syscalls, which obtains the SP and PC from the caller.
1614 func reentersyscall(pc, sp uintptr) {
1617 // Disable preemption because during this function g is in Gsyscall status,
1618 // but can have inconsistent g->sched, do not let GC observe it.
1621 // Entersyscall must not call any function that might split/grow the stack.
1622 // (See details in comment above.)
1623 // Catch calls that might, by replacing the stack guard with something that
1624 // will trip any stack check and leaving a flag to tell newstack to die.
1625 _g_.stackguard0 = stackPreempt
1626 _g_.throwsplit = true
1628 // Leave SP around for GC and traceback.
1632 casgstatus(_g_, _Grunning, _Gsyscall)
1633 if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp {
1634 systemstack(func() {
1635 print("entersyscall inconsistent ", hex(_g_.syscallsp), " [", hex(_g_.stack.lo), ",", hex(_g_.stack.hi), "]\n")
1636 gothrow("entersyscall")
1640 if atomicload(&sched.sysmonwait) != 0 { // TODO: fast atomic
1641 systemstack(entersyscall_sysmon)
1647 atomicstore(&_g_.m.p.status, _Psyscall)
1648 if sched.gcwaiting != 0 {
1649 systemstack(entersyscall_gcwait)
1653 // Goroutines must not split stacks in Gsyscall status (it would corrupt g->sched).
1654 // We set _StackGuard to StackPreempt so that first split stack check calls morestack.
1655 // Morestack detects this case and throws.
1656 _g_.stackguard0 = stackPreempt
1660 // Standard syscall entry used by the go syscall library and normal cgo calls.
1662 func entersyscall(dummy int32) {
1663 reentersyscall(getcallerpc(unsafe.Pointer(&dummy)), getcallersp(unsafe.Pointer(&dummy)))
1666 func entersyscall_sysmon() {
1668 if atomicload(&sched.sysmonwait) != 0 {
1669 atomicstore(&sched.sysmonwait, 0)
1670 notewakeup(&sched.sysmonnote)
1675 func entersyscall_gcwait() {
1679 if sched.stopwait > 0 && cas(&_g_.m.p.status, _Psyscall, _Pgcstop) {
1680 if sched.stopwait--; sched.stopwait == 0 {
1681 notewakeup(&sched.stopnote)
1687 // The same as entersyscall(), but with a hint that the syscall is blocking.
1689 func entersyscallblock(dummy int32) {
1692 _g_.m.locks++ // see comment in entersyscall
1693 _g_.throwsplit = true
1694 _g_.stackguard0 = stackPreempt // see comment in entersyscall
1696 // Leave SP around for GC and traceback.
1697 pc := getcallerpc(unsafe.Pointer(&dummy))
1698 sp := getcallersp(unsafe.Pointer(&dummy))
1700 _g_.syscallsp = _g_.sched.sp
1701 _g_.syscallpc = _g_.sched.pc
1702 if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp {
1705 sp3 := _g_.syscallsp
1706 systemstack(func() {
1707 print("entersyscallblock inconsistent ", hex(sp1), " ", hex(sp2), " ", hex(sp3), " [", hex(_g_.stack.lo), ",", hex(_g_.stack.hi), "]\n")
1708 gothrow("entersyscallblock")
1711 casgstatus(_g_, _Grunning, _Gsyscall)
1712 if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp {
1713 systemstack(func() {
1714 print("entersyscallblock inconsistent ", hex(sp), " ", hex(_g_.sched.sp), " ", hex(_g_.syscallsp), " [", hex(_g_.stack.lo), ",", hex(_g_.stack.hi), "]\n")
1715 gothrow("entersyscallblock")
1719 systemstack(entersyscallblock_handoff)
1721 // Resave for traceback during blocked call.
1722 save(getcallerpc(unsafe.Pointer(&dummy)), getcallersp(unsafe.Pointer(&dummy)))
1727 func entersyscallblock_handoff() {
1728 handoffp(releasep())
1731 // The goroutine g exited its system call.
1732 // Arrange for it to run on a cpu again.
1733 // This is called only from the go syscall library, not
1734 // from the low-level system calls used by the
1736 func exitsyscall(dummy int32) {
1739 _g_.m.locks++ // see comment in entersyscall
1740 if getcallersp(unsafe.Pointer(&dummy)) > _g_.syscallsp {
1741 gothrow("exitsyscall: syscall frame is no longer valid")
1745 if exitsyscallfast() {
1746 if _g_.m.mcache == nil {
1747 gothrow("lost mcache")
1749 // There's a cpu for us, so we can run.
1750 _g_.m.p.syscalltick++
1751 // We need to cas the status and scan before resuming...
1752 casgstatus(_g_, _Gsyscall, _Grunning)
1754 // Garbage collector isn't running (since we are),
1755 // so okay to clear syscallsp.
1759 // restore the preemption request in case we've cleared it in newstack
1760 _g_.stackguard0 = stackPreempt
1762 // otherwise restore the real _StackGuard, we've spoiled it in entersyscall/entersyscallblock
1763 _g_.stackguard0 = _g_.stack.lo + _StackGuard
1765 _g_.throwsplit = false
1771 // Call the scheduler.
1774 if _g_.m.mcache == nil {
1775 gothrow("lost mcache")
1778 // Scheduler returned, so we're allowed to run now.
1779 // Delete the syscallsp information that we left for
1780 // the garbage collector during the system call.
1781 // Must wait until now because until gosched returns
1782 // we don't know for sure that the garbage collector
1785 _g_.m.p.syscalltick++
1786 _g_.throwsplit = false
1790 func exitsyscallfast() bool {
1793 // Freezetheworld sets stopwait but does not retake P's.
1794 if sched.stopwait != 0 {
1800 // Try to re-acquire the last P.
1801 if _g_.m.p != nil && _g_.m.p.status == _Psyscall && cas(&_g_.m.p.status, _Psyscall, _Prunning) {
1802 // There's a cpu for us, so we can run.
1803 _g_.m.mcache = _g_.m.p.mcache
1808 // Try to get any other idle P.
1811 if sched.pidle != nil {
1813 systemstack(func() {
1814 ok = exitsyscallfast_pidle()
1823 func exitsyscallfast_pidle() bool {
1826 if _p_ != nil && atomicload(&sched.sysmonwait) != 0 {
1827 atomicstore(&sched.sysmonwait, 0)
1828 notewakeup(&sched.sysmonnote)
1838 // exitsyscall slow path on g0.
1839 // Failed to acquire P, enqueue gp as runnable.
1840 func exitsyscall0(gp *g) {
1843 casgstatus(gp, _Gsyscall, _Grunnable)
1849 } else if atomicload(&sched.sysmonwait) != 0 {
1850 atomicstore(&sched.sysmonwait, 0)
1851 notewakeup(&sched.sysmonnote)
1856 execute(gp) // Never returns.
1858 if _g_.m.lockedg != nil {
1859 // Wait until another thread schedules gp and so m again.
1861 execute(gp) // Never returns.
1864 schedule() // Never returns.
1870 // Fork can hang if preempted with signals frequently enough (see issue 5517).
1871 // Ensure that we stay on the same M where we disable profiling.
1873 if gp.m.profilehz != 0 {
1877 // This function is called before fork in syscall package.
1878 // Code between fork and exec must not allocate memory nor even try to grow stack.
1879 // Here we spoil g->_StackGuard to reliably detect any attempts to grow stack.
1880 // runtime_AfterFork will undo this in parent process, but not in child.
1881 gp.stackguard0 = stackFork
1884 // Called from syscall package before fork.
1886 func syscall_BeforeFork() {
1887 systemstack(beforefork)
1893 // See the comment in beforefork.
1894 gp.stackguard0 = gp.stack.lo + _StackGuard
1896 hz := sched.profilehz
1898 resetcpuprofiler(hz)
1903 // Called from syscall package after fork in parent.
1905 func syscall_AfterFork() {
1906 systemstack(afterfork)
1909 // Allocate a new g, with a stack big enough for stacksize bytes.
1910 func malg(stacksize int32) *g {
1913 stacksize = round2(_StackSystem + stacksize)
1914 systemstack(func() {
1915 newg.stack = stackalloc(uint32(stacksize))
1917 newg.stackguard0 = newg.stack.lo + _StackGuard
1918 newg.stackguard1 = ^uintptr(0)
1923 // Create a new g running fn with siz bytes of arguments.
1924 // Put it on the queue of g's waiting to run.
1925 // The compiler turns a go statement into a call to this.
1926 // Cannot split the stack because it assumes that the arguments
1927 // are available sequentially after &fn; they would not be
1928 // copied if a stack split occurred.
1930 func newproc(siz int32, fn *funcval) {
1931 argp := add(unsafe.Pointer(&fn), ptrSize)
1932 if hasLinkRegister {
1933 argp = add(argp, ptrSize) // skip caller's saved LR
1936 pc := getcallerpc(unsafe.Pointer(&siz))
1937 systemstack(func() {
1938 newproc1(fn, (*uint8)(argp), siz, 0, pc)
1942 // Create a new g running fn with narg bytes of arguments starting
1943 // at argp and returning nret bytes of results. callerpc is the
1944 // address of the go statement that created this. The new g is put
1945 // on the queue of g's waiting to run.
1946 func newproc1(fn *funcval, argp *uint8, narg int32, nret int32, callerpc uintptr) *g {
1950 _g_.m.throwing = -1 // do not dump full stacks
1951 gothrow("go of nil func value")
1953 _g_.m.locks++ // disable preemption because it can be holding p in a local var
1955 siz = (siz + 7) &^ 7
1957 // We could allocate a larger initial stack if necessary.
1958 // Not worth it: this is almost always an error.
1959 // 4*sizeof(uintreg): extra space added below
1960 // sizeof(uintreg): caller's LR (arm) or return address (x86, in gostartcall).
1961 if siz >= _StackMin-4*regSize-regSize {
1962 gothrow("newproc: function arguments too large for new goroutine")
1968 newg = malg(_StackMin)
1969 casgstatus(newg, _Gidle, _Gdead)
1970 allgadd(newg) // publishes with a g->status of Gdead so GC scanner doesn't look at uninitialized stack.
1972 if newg.stack.hi == 0 {
1973 gothrow("newproc1: newg missing stack")
1976 if readgstatus(newg) != _Gdead {
1977 gothrow("newproc1: new g is not Gdead")
1981 sp -= 4 * regSize // extra space in case of reads slightly beyond frame
1983 memmove(unsafe.Pointer(sp), unsafe.Pointer(argp), uintptr(narg))
1984 if hasLinkRegister {
1987 *(*unsafe.Pointer)(unsafe.Pointer(sp)) = nil
1990 memclr(unsafe.Pointer(&newg.sched), unsafe.Sizeof(newg.sched))
1992 newg.sched.pc = funcPC(goexit) + _PCQuantum // +PCQuantum so that previous instruction is in same function
1994 gostartcallfn(&newg.sched, fn)
1995 newg.gopc = callerpc
1996 casgstatus(newg, _Gdead, _Grunnable)
1998 if _p_.goidcache == _p_.goidcacheend {
1999 // Sched.goidgen is the last allocated id,
2000 // this batch must be [sched.goidgen+1, sched.goidgen+GoidCacheBatch].
2001 // At startup sched.goidgen=0, so main goroutine receives goid=1.
2002 _p_.goidcache = xadd64(&sched.goidgen, _GoidCacheBatch)
2003 _p_.goidcache -= _GoidCacheBatch - 1
2004 _p_.goidcacheend = _p_.goidcache + _GoidCacheBatch
2006 newg.goid = int64(_p_.goidcache)
2009 newg.racectx = racegostart(callerpc)
2013 if atomicload(&sched.npidle) != 0 && atomicload(&sched.nmspinning) == 0 && unsafe.Pointer(fn.fn) != unsafe.Pointer(funcPC(main)) { // TODO: fast atomic
2017 if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack
2018 _g_.stackguard0 = stackPreempt
2023 // Put on gfree list.
2024 // If local list is too long, transfer a batch to the global list.
2025 func gfput(_p_ *p, gp *g) {
2026 if readgstatus(gp) != _Gdead {
2027 gothrow("gfput: bad status (not Gdead)")
2030 stksize := gp.stack.hi - gp.stack.lo
2032 if stksize != _FixedStack {
2033 // non-standard stack size - free it.
2040 gp.schedlink = _p_.gfree
2043 if _p_.gfreecnt >= 64 {
2045 for _p_.gfreecnt >= 32 {
2048 _p_.gfree = gp.schedlink
2049 gp.schedlink = sched.gfree
2053 unlock(&sched.gflock)
2057 // Get from gfree list.
2058 // If local list is empty, grab a batch from global list.
2059 func gfget(_p_ *p) *g {
2062 if gp == nil && sched.gfree != nil {
2064 for _p_.gfreecnt < 32 && sched.gfree != nil {
2067 sched.gfree = gp.schedlink
2069 gp.schedlink = _p_.gfree
2072 unlock(&sched.gflock)
2076 _p_.gfree = gp.schedlink
2078 if gp.stack.lo == 0 {
2079 // Stack was deallocated in gfput. Allocate a new one.
2080 systemstack(func() {
2081 gp.stack = stackalloc(_FixedStack)
2083 gp.stackguard0 = gp.stack.lo + _StackGuard
2086 racemalloc(unsafe.Pointer(gp.stack.lo), gp.stack.hi-gp.stack.lo)
2093 // Purge all cached G's from gfree list to the global list.
2094 func gfpurge(_p_ *p) {
2096 for _p_.gfreecnt != 0 {
2099 _p_.gfree = gp.schedlink
2100 gp.schedlink = sched.gfree
2104 unlock(&sched.gflock)
2107 // Breakpoint executes a breakpoint trap.
2112 // dolockOSThread is called by LockOSThread and lockOSThread below
2113 // after they modify m.locked. Do not allow preemption during this call,
2114 // or else the m might be different in this function than in the caller.
2116 func dolockOSThread() {
2124 // LockOSThread wires the calling goroutine to its current operating system thread.
2125 // Until the calling goroutine exits or calls UnlockOSThread, it will always
2126 // execute in that thread, and no other goroutine can.
2127 func LockOSThread() {
2128 getg().m.locked |= _LockExternal
2133 func lockOSThread() {
2134 getg().m.locked += _LockInternal
2138 // dounlockOSThread is called by UnlockOSThread and unlockOSThread below
2139 // after they update m->locked. Do not allow preemption during this call,
2140 // or else the m might be in different in this function than in the caller.
2142 func dounlockOSThread() {
2144 if _g_.m.locked != 0 {
2153 // UnlockOSThread unwires the calling goroutine from its fixed operating system thread.
2154 // If the calling goroutine has not called LockOSThread, UnlockOSThread is a no-op.
2155 func UnlockOSThread() {
2156 getg().m.locked &^= _LockExternal
2161 func unlockOSThread() {
2163 if _g_.m.locked < _LockInternal {
2164 systemstack(badunlockosthread)
2166 _g_.m.locked -= _LockInternal
2170 func badunlockosthread() {
2171 gothrow("runtime: internal error: misuse of lockOSThread/unlockOSThread")
2174 func gcount() int32 {
2175 n := int32(allglen) - sched.ngfree
2184 // All these variables can be changed concurrently, so the result can be inconsistent.
2185 // But at least the current goroutine is running.
2192 func mcount() int32 {
2201 func _System() { _System() }
2202 func _ExternalCode() { _ExternalCode() }
2203 func _GC() { _GC() }
2207 // Called if we receive a SIGPROF signal.
2208 func sigprof(pc *uint8, sp *uint8, lr *uint8, gp *g, mp *m) {
2211 var stk [100]uintptr
2217 // Profiling runs concurrently with GC, so it must not allocate.
2220 // Define that a "user g" is a user-created goroutine, and a "system g"
2221 // is one that is m->g0 or m->gsignal. We've only made sure that we
2222 // can unwind user g's, so exclude the system g's.
2224 // It is not quite as easy as testing gp == m->curg (the current user g)
2225 // because we might be interrupted for profiling halfway through a
2226 // goroutine switch. The switch involves updating three (or four) values:
2227 // g, PC, SP, and (on arm) LR. The PC must be the last to be updated,
2228 // because once it gets updated the new g is running.
2230 // When switching from a user g to a system g, LR is not considered live,
2231 // so the update only affects g, SP, and PC. Since PC must be last, there
2232 // the possible partial transitions in ordinary execution are (1) g alone is updated,
2233 // (2) both g and SP are updated, and (3) SP alone is updated.
2234 // If g is updated, we'll see a system g and not look closer.
2235 // If SP alone is updated, we can detect the partial transition by checking
2236 // whether the SP is within g's stack bounds. (We could also require that SP
2237 // be changed only after g, but the stack bounds check is needed by other
2238 // cases, so there is no need to impose an additional requirement.)
2240 // There is one exceptional transition to a system g, not in ordinary execution.
2241 // When a signal arrives, the operating system starts the signal handler running
2242 // with an updated PC and SP. The g is updated last, at the beginning of the
2243 // handler. There are two reasons this is okay. First, until g is updated the
2244 // g and SP do not match, so the stack bounds check detects the partial transition.
2245 // Second, signal handlers currently run with signals disabled, so a profiling
2246 // signal cannot arrive during the handler.
2248 // When switching from a system g to a user g, there are three possibilities.
2250 // First, it may be that the g switch has no PC update, because the SP
2251 // either corresponds to a user g throughout (as in asmcgocall)
2252 // or because it has been arranged to look like a user g frame
2253 // (as in cgocallback_gofunc). In this case, since the entire
2254 // transition is a g+SP update, a partial transition updating just one of
2255 // those will be detected by the stack bounds check.
2257 // Second, when returning from a signal handler, the PC and SP updates
2258 // are performed by the operating system in an atomic update, so the g
2259 // update must be done before them. The stack bounds check detects
2260 // the partial transition here, and (again) signal handlers run with signals
2261 // disabled, so a profiling signal cannot arrive then anyway.
2263 // Third, the common case: it may be that the switch updates g, SP, and PC
2264 // separately, as in gogo.
2266 // Because gogo is the only instance, we check whether the PC lies
2267 // within that function, and if so, not ask for a traceback. This approach
2268 // requires knowing the size of the gogo function, which we
2269 // record in arch_*.h and check in runtime_test.go.
2271 // There is another apparently viable approach, recorded here in case
2272 // the "PC within gogo" check turns out not to be usable.
2273 // It would be possible to delay the update of either g or SP until immediately
2274 // before the PC update instruction. Then, because of the stack bounds check,
2275 // the only problematic interrupt point is just before that PC update instruction,
2276 // and the sigprof handler can detect that instruction and simulate stepping past
2277 // it in order to reach a consistent state. On ARM, the update of g must be made
2278 // in two places (in R10 and also in a TLS slot), so the delayed update would
2279 // need to be the SP update. The sigprof handler must read the instruction at
2280 // the current PC and if it was the known instruction (for example, JMP BX or
2281 // MOV R2, PC), use that other register in place of the PC value.
2282 // The biggest drawback to this solution is that it requires that we can tell
2283 // whether it's safe to read from the memory pointed at by PC.
2284 // In a correct program, we can test PC == nil and otherwise read,
2285 // but if a profiling signal happens at the instant that a program executes
2286 // a bad jump (before the program manages to handle the resulting fault)
2287 // the profiling handler could fault trying to read nonexistent memory.
2289 // To recap, there are no constraints on the assembly being used for the
2290 // transition. We simply require that g and SP match and that the PC is not
2293 usp := uintptr(unsafe.Pointer(sp))
2294 gogo := funcPC(gogo)
2295 if gp == nil || gp != mp.curg ||
2296 usp < gp.stack.lo || gp.stack.hi < usp ||
2297 (gogo <= uintptr(unsafe.Pointer(pc)) && uintptr(unsafe.Pointer(pc)) < gogo+_RuntimeGogoBytes) {
2303 n = int32(gentraceback(uintptr(unsafe.Pointer(pc)), uintptr(unsafe.Pointer(sp)), uintptr(unsafe.Pointer(lr)), gp, 0, &stk[0], len(stk), nil, nil, _TraceTrap))
2305 if !traceback || n <= 0 {
2306 // Normal traceback is impossible or has failed.
2307 // See if it falls into several common cases.
2309 if mp.ncgo > 0 && mp.curg != nil && mp.curg.syscallpc != 0 && mp.curg.syscallsp != 0 {
2310 // Cgo, we can't unwind and symbolize arbitrary C code,
2311 // so instead collect Go stack that leads to the cgo call.
2312 // This is especially important on windows, since all syscalls are cgo calls.
2313 n = int32(gentraceback(mp.curg.syscallpc, mp.curg.syscallsp, 0, mp.curg, 0, &stk[0], len(stk), nil, nil, 0))
2315 if GOOS == "windows" && n == 0 && mp.libcallg != nil && mp.libcallpc != 0 && mp.libcallsp != 0 {
2316 // Libcall, i.e. runtime syscall on windows.
2317 // Collect Go stack that leads to the call.
2318 n = int32(gentraceback(mp.libcallpc, mp.libcallsp, 0, mp.libcallg, 0, &stk[0], len(stk), nil, nil, 0))
2321 // If all of the above has failed, account it against abstract "System" or "GC".
2323 // "ExternalCode" is better than "etext".
2324 if uintptr(unsafe.Pointer(pc)) > uintptr(unsafe.Pointer(&etext)) {
2325 pc = (*uint8)(unsafe.Pointer(uintptr(funcPC(_ExternalCode) + _PCQuantum)))
2327 stk[0] = uintptr(unsafe.Pointer(pc))
2328 if mp.gcing != 0 || mp.helpgc != 0 {
2329 stk[1] = funcPC(_GC) + _PCQuantum
2331 stk[1] = funcPC(_System) + _PCQuantum
2337 // Simple cas-lock to coordinate with setcpuprofilerate.
2338 for !cas(&prof.lock, 0, 1) {
2342 cpuproftick(&stk[0], n)
2344 atomicstore(&prof.lock, 0)
2349 // Arrange to call fn with a traceback hz times a second.
2350 func setcpuprofilerate_m(hz int32) {
2351 // Force sane arguments.
2356 // Disable preemption, otherwise we can be rescheduled to another thread
2357 // that has profiling enabled.
2361 // Stop profiler on this thread so that it is safe to lock prof.
2362 // if a profiling signal came in while we had prof locked,
2363 // it would deadlock.
2366 for !cas(&prof.lock, 0, 1) {
2370 atomicstore(&prof.lock, 0)
2373 sched.profilehz = hz
2377 resetcpuprofiler(hz)
2383 // Change number of processors. The world is stopped, sched is locked.
2384 // gcworkbufs are not being modified by either the GC or
2385 // the write barrier code.
2386 func procresize(new int32) {
2388 if old < 0 || old > _MaxGomaxprocs || new <= 0 || new > _MaxGomaxprocs {
2389 gothrow("procresize: invalid arg")
2392 // initialize new P's
2393 for i := int32(0); i < new; i++ {
2399 atomicstorep(unsafe.Pointer(&allp[i]), unsafe.Pointer(p))
2401 if p.mcache == nil {
2402 if old == 0 && i == 0 {
2403 if getg().m.mcache == nil {
2404 gothrow("missing mcache?")
2406 p.mcache = getg().m.mcache // bootstrap
2408 p.mcache = allocmcache()
2413 // redistribute runnable G's evenly
2414 // collect all runnable goroutines in global queue preserving FIFO order
2415 // FIFO order is required to ensure fairness even during frequent GCs
2416 // see http://golang.org/issue/7126
2420 for i := int32(0); i < old; i++ {
2422 if p.runqhead == p.runqtail {
2426 // pop from tail of local queue
2428 gp := p.runq[p.runqtail%uint32(len(p.runq))]
2429 // push onto head of global queue
2430 gp.schedlink = sched.runqhead
2432 if sched.runqtail == nil {
2439 // fill local queues with at most len(p.runq)/2 goroutines
2440 // start at 1 because current M already executes some G and will acquire allp[0] below,
2441 // so if we have a spare G we want to put it into allp[1].
2443 for i := int32(1); i < new*int32(len(_p_.runq))/2 && sched.runqsize > 0; i++ {
2444 gp := sched.runqhead
2445 sched.runqhead = gp.schedlink
2446 if sched.runqhead == nil {
2447 sched.runqtail = nil
2450 runqput(allp[i%new], gp)
2454 for i := new; i < old; i++ {
2456 freemcache(p.mcache)
2460 // can't free P itself because it can be referenced by an M in syscall
2473 for i := new - 1; i > 0; i-- {
2478 var int32p *int32 = &gomaxprocs // make compiler check that gomaxprocs is an int32
2479 atomicstore((*uint32)(unsafe.Pointer(int32p)), uint32(new))
2482 // Associate p and the current m.
2483 func acquirep(_p_ *p) {
2486 if _g_.m.p != nil || _g_.m.mcache != nil {
2487 gothrow("acquirep: already in go")
2489 if _p_.m != nil || _p_.status != _Pidle {
2494 print("acquirep: p->m=", _p_.m, "(", id, ") p->status=", _p_.status, "\n")
2495 gothrow("acquirep: invalid p state")
2497 _g_.m.mcache = _p_.mcache
2500 _p_.status = _Prunning
2503 // Disassociate p and the current m.
2504 func releasep() *p {
2507 if _g_.m.p == nil || _g_.m.mcache == nil {
2508 gothrow("releasep: invalid arg")
2511 if _p_.m != _g_.m || _p_.mcache != _g_.m.mcache || _p_.status != _Prunning {
2512 print("releasep: m=", _g_.m, " m->p=", _g_.m.p, " p->m=", _p_.m, " m->mcache=", _g_.m.mcache, " p->mcache=", _p_.mcache, " p->status=", _p_.status, "\n")
2513 gothrow("releasep: invalid p state")
2522 func incidlelocked(v int32) {
2524 sched.nmidlelocked += v
2531 // Check for deadlock situation.
2532 // The check is based on number of running M's, if 0 -> deadlock.
2534 // If we are dying because of a signal caught on an already idle thread,
2535 // freezetheworld will cause all running threads to block.
2536 // And runtime will essentially enter into deadlock state,
2537 // except that there is a thread that will call exit soon.
2543 run := sched.mcount - sched.nmidle - sched.nmidlelocked - 1
2548 print("runtime: checkdead: nmidle=", sched.nmidle, " nmidlelocked=", sched.nmidlelocked, " mcount=", sched.mcount, "\n")
2549 gothrow("checkdead: inconsistent counts")
2554 for i := 0; i < len(allgs); i++ {
2559 s := readgstatus(gp)
2560 switch s &^ _Gscan {
2567 print("runtime: checkdead: find g ", gp.goid, " in status ", s, "\n")
2568 gothrow("checkdead: runnable g")
2572 if grunning == 0 { // possible if main goroutine calls runtimeĀ·Goexit()
2573 gothrow("no goroutines (main called runtime.Goexit) - deadlock!")
2576 // Maybe jump time forward for playground.
2579 casgstatus(gp, _Gwaiting, _Grunnable)
2583 gothrow("checkdead: no p for timer")
2590 notewakeup(&mp.park)
2595 getg().m.throwing = -1 // do not dump full stacks
2596 gothrow("all goroutines are asleep - deadlock!")
2600 // If we go two minutes without a garbage collection, force one to run.
2601 forcegcperiod := int64(2 * 60 * 1e9)
2603 // If a heap span goes unused for 5 minutes after a garbage collection,
2604 // we hand it back to the operating system.
2605 scavengelimit := int64(5 * 60 * 1e9)
2607 if debug.scavenge > 0 {
2608 // Scavenge-a-lot for testing.
2609 forcegcperiod = 10 * 1e6
2610 scavengelimit = 20 * 1e6
2613 lastscavenge := nanotime()
2616 // Make wake-up period small enough for the sampling to be correct.
2617 maxsleep := forcegcperiod / 2
2618 if scavengelimit < forcegcperiod {
2619 maxsleep = scavengelimit / 2
2622 lasttrace := int64(0)
2623 idle := 0 // how many cycles in succession we had not wokeup somebody
2626 if idle == 0 { // start with 20us sleep...
2628 } else if idle > 50 { // start doubling the sleep after 1ms...
2631 if delay > 10*1000 { // up to 10ms
2635 if debug.schedtrace <= 0 && (sched.gcwaiting != 0 || atomicload(&sched.npidle) == uint32(gomaxprocs)) { // TODO: fast atomic
2637 if atomicload(&sched.gcwaiting) != 0 || atomicload(&sched.npidle) == uint32(gomaxprocs) {
2638 atomicstore(&sched.sysmonwait, 1)
2640 notetsleep(&sched.sysmonnote, maxsleep)
2642 atomicstore(&sched.sysmonwait, 0)
2643 noteclear(&sched.sysmonnote)
2649 // poll network if not polled for more than 10ms
2650 lastpoll := int64(atomicload64(&sched.lastpoll))
2652 unixnow := unixnanotime()
2653 if lastpoll != 0 && lastpoll+10*1000*1000 < now {
2654 cas64(&sched.lastpoll, uint64(lastpoll), uint64(now))
2655 gp := netpoll(false) // non-blocking - returns list of goroutines
2657 // Need to decrement number of idle locked M's
2658 // (pretending that one more is running) before injectglist.
2659 // Otherwise it can lead to the following situation:
2660 // injectglist grabs all P's but before it starts M's to run the P's,
2661 // another M returns from syscall, finishes running its G,
2662 // observes that there is no work to do and no other running M's
2663 // and reports deadlock.
2669 // retake P's blocked in syscalls
2670 // and preempt long running G's
2671 if retake(now) != 0 {
2676 // check if we need to force a GC
2677 lastgc := int64(atomicload64(&memstats.last_gc))
2678 if lastgc != 0 && unixnow-lastgc > forcegcperiod && atomicload(&forcegc.idle) != 0 {
2681 forcegc.g.schedlink = nil
2682 injectglist(forcegc.g)
2683 unlock(&forcegc.lock)
2685 // scavenge heap once in a while
2686 if lastscavenge+scavengelimit/2 < now {
2687 mHeap_Scavenge(int32(nscavenge), uint64(now), uint64(scavengelimit))
2691 if debug.schedtrace > 0 && lasttrace+int64(debug.schedtrace*1000000) <= now {
2693 schedtrace(debug.scheddetail > 0)
2698 var pdesc [_MaxGomaxprocs]struct {
2705 func retake(now int64) uint32 {
2707 for i := int32(0); i < gomaxprocs; i++ {
2715 // Retake P from syscall if it's there for more than 1 sysmon tick (at least 20us).
2716 t := int64(_p_.syscalltick)
2717 if int64(pd.syscalltick) != t {
2718 pd.syscalltick = uint32(t)
2719 pd.syscallwhen = now
2722 // On the one hand we don't want to retake Ps if there is no other work to do,
2723 // but on the other hand we want to retake them eventually
2724 // because they can prevent the sysmon thread from deep sleep.
2725 if _p_.runqhead == _p_.runqtail && atomicload(&sched.nmspinning)+atomicload(&sched.npidle) > 0 && pd.syscallwhen+10*1000*1000 > now {
2728 // Need to decrement number of idle locked M's
2729 // (pretending that one more is running) before the CAS.
2730 // Otherwise the M from which we retake can exit the syscall,
2731 // increment nmidle and report deadlock.
2733 if cas(&_p_.status, s, _Pidle) {
2738 } else if s == _Prunning {
2739 // Preempt G if it's running for more than 10ms.
2740 t := int64(_p_.schedtick)
2741 if int64(pd.schedtick) != t {
2742 pd.schedtick = uint32(t)
2746 if pd.schedwhen+10*1000*1000 > now {
2755 // Tell all goroutines that they have been preempted and they should stop.
2756 // This function is purely best-effort. It can fail to inform a goroutine if a
2757 // processor just started running it.
2758 // No locks need to be held.
2759 // Returns true if preemption request was issued to at least one goroutine.
2760 func preemptall() bool {
2762 for i := int32(0); i < gomaxprocs; i++ {
2764 if _p_ == nil || _p_.status != _Prunning {
2767 if preemptone(_p_) {
2774 // Tell the goroutine running on processor P to stop.
2775 // This function is purely best-effort. It can incorrectly fail to inform the
2776 // goroutine. It can send inform the wrong goroutine. Even if it informs the
2777 // correct goroutine, that goroutine might ignore the request if it is
2778 // simultaneously executing newstack.
2779 // No lock needs to be held.
2780 // Returns true if preemption request was issued.
2781 // The actual preemption will happen at some point in the future
2782 // and will be indicated by the gp->status no longer being
2784 func preemptone(_p_ *p) bool {
2786 if mp == nil || mp == getg().m {
2790 if gp == nil || gp == mp.g0 {
2796 // Every call in a go routine checks for stack overflow by
2797 // comparing the current stack pointer to gp->stackguard0.
2798 // Setting gp->stackguard0 to StackPreempt folds
2799 // preemption into the normal stack overflow check.
2800 gp.stackguard0 = stackPreempt
2806 func schedtrace(detailed bool) {
2813 print("SCHED ", (now-starttime)/1e6, "ms: gomaxprocs=", gomaxprocs, " idleprocs=", sched.npidle, " threads=", sched.mcount, " spinningthreads=", sched.nmspinning, " idlethreads=", sched.nmidle, " runqueue=", sched.runqsize)
2815 print(" gcwaiting=", sched.gcwaiting, " nmidlelocked=", sched.nmidlelocked, " stopwait=", sched.stopwait, " sysmonwait=", sched.sysmonwait, "\n")
2817 // We must be careful while reading data from P's, M's and G's.
2818 // Even if we hold schedlock, most data can be changed concurrently.
2819 // E.g. (p->m ? p->m->id : -1) can crash if p->m changes from non-nil to nil.
2820 for i := int32(0); i < gomaxprocs; i++ {
2826 h := atomicload(&_p_.runqhead)
2827 t := atomicload(&_p_.runqtail)
2833 print(" P", i, ": status=", _p_.status, " schedtick=", _p_.schedtick, " syscalltick=", _p_.syscalltick, " m=", id, " runqsize=", t-h, " gfreecnt=", _p_.gfreecnt, "\n")
2835 // In non-detailed mode format lengths of per-P run queues as:
2836 // [len1 len2 len3 len4]
2842 if i == gomaxprocs-1 {
2853 for mp := allm; mp != nil; mp = mp.alllink {
2856 lockedg := mp.lockedg
2869 print(" M", mp.id, ": p=", id1, " curg=", id2, " mallocing=", mp.mallocing, " throwing=", mp.throwing, " gcing=", mp.gcing, ""+" locks=", mp.locks, " dying=", mp.dying, " helpgc=", mp.helpgc, " spinning=", mp.spinning, " blocked=", getg().m.blocked, " lockedg=", id3, "\n")
2873 for gi := 0; gi < len(allgs); gi++ {
2876 lockedm := gp.lockedm
2885 print(" G", gp.goid, ": status=", readgstatus(gp), "(", gp.waitreason, ") m=", id1, " lockedm=", id2, "\n")
2891 // Put mp on midle list.
2892 // Sched must be locked.
2894 mp.schedlink = sched.midle
2900 // Try to get an m from midle list.
2901 // Sched must be locked.
2905 sched.midle = mp.schedlink
2911 // Put gp on the global runnable queue.
2912 // Sched must be locked.
2913 func globrunqput(gp *g) {
2915 if sched.runqtail != nil {
2916 sched.runqtail.schedlink = gp
2924 // Put a batch of runnable goroutines on the global runnable queue.
2925 // Sched must be locked.
2926 func globrunqputbatch(ghead *g, gtail *g, n int32) {
2927 gtail.schedlink = nil
2928 if sched.runqtail != nil {
2929 sched.runqtail.schedlink = ghead
2931 sched.runqhead = ghead
2933 sched.runqtail = gtail
2937 // Try get a batch of G's from the global runnable queue.
2938 // Sched must be locked.
2939 func globrunqget(_p_ *p, max int32) *g {
2940 if sched.runqsize == 0 {
2944 n := sched.runqsize/gomaxprocs + 1
2945 if n > sched.runqsize {
2948 if max > 0 && n > max {
2951 if n > int32(len(_p_.runq))/2 {
2952 n = int32(len(_p_.runq)) / 2
2956 if sched.runqsize == 0 {
2957 sched.runqtail = nil
2960 gp := sched.runqhead
2961 sched.runqhead = gp.schedlink
2964 gp1 := sched.runqhead
2965 sched.runqhead = gp1.schedlink
2971 // Put p to on _Pidle list.
2972 // Sched must be locked.
2973 func pidleput(_p_ *p) {
2974 _p_.link = sched.pidle
2976 xadd(&sched.npidle, 1) // TODO: fast atomic
2979 // Try get a p from _Pidle list.
2980 // Sched must be locked.
2981 func pidleget() *p {
2984 sched.pidle = _p_.link
2985 xadd(&sched.npidle, -1) // TODO: fast atomic
2990 // Try to put g on local runnable queue.
2991 // If it's full, put onto global queue.
2992 // Executed only by the owner P.
2993 func runqput(_p_ *p, gp *g) {
2995 h := atomicload(&_p_.runqhead) // load-acquire, synchronize with consumers
2997 if t-h < uint32(len(_p_.runq)) {
2998 _p_.runq[t%uint32(len(_p_.runq))] = gp
2999 atomicstore(&_p_.runqtail, t+1) // store-release, makes the item available for consumption
3002 if runqputslow(_p_, gp, h, t) {
3005 // the queue is not full, now the put above must suceed
3009 // Put g and a batch of work from local runnable queue on global queue.
3010 // Executed only by the owner P.
3011 func runqputslow(_p_ *p, gp *g, h, t uint32) bool {
3012 var batch [len(_p_.runq)/2 + 1]*g
3014 // First, grab a batch from local queue.
3017 if n != uint32(len(_p_.runq)/2) {
3018 gothrow("runqputslow: queue is not full")
3020 for i := uint32(0); i < n; i++ {
3021 batch[i] = _p_.runq[(h+i)%uint32(len(_p_.runq))]
3023 if !cas(&_p_.runqhead, h, h+n) { // cas-release, commits consume
3028 // Link the goroutines.
3029 for i := uint32(0); i < n; i++ {
3030 batch[i].schedlink = batch[i+1]
3033 // Now put the batch on global queue.
3035 globrunqputbatch(batch[0], batch[n], int32(n+1))
3040 // Get g from local runnable queue.
3041 // Executed only by the owner P.
3042 func runqget(_p_ *p) *g {
3044 h := atomicload(&_p_.runqhead) // load-acquire, synchronize with other consumers
3049 gp := _p_.runq[h%uint32(len(_p_.runq))]
3050 if cas(&_p_.runqhead, h, h+1) { // cas-release, commits consume
3056 // Grabs a batch of goroutines from local runnable queue.
3057 // batch array must be of size len(p->runq)/2. Returns number of grabbed goroutines.
3058 // Can be executed by any P.
3059 func runqgrab(_p_ *p, batch []*g) uint32 {
3061 h := atomicload(&_p_.runqhead) // load-acquire, synchronize with other consumers
3062 t := atomicload(&_p_.runqtail) // load-acquire, synchronize with the producer
3068 if n > uint32(len(_p_.runq)/2) { // read inconsistent h and t
3071 for i := uint32(0); i < n; i++ {
3072 batch[i] = _p_.runq[(h+i)%uint32(len(_p_.runq))]
3074 if cas(&_p_.runqhead, h, h+n) { // cas-release, commits consume
3080 // Steal half of elements from local runnable queue of p2
3081 // and put onto local runnable queue of p.
3082 // Returns one of the stolen elements (or nil if failed).
3083 func runqsteal(_p_, p2 *p) *g {
3084 var batch [len(_p_.runq) / 2]*g
3086 n := runqgrab(p2, batch[:])
3095 h := atomicload(&_p_.runqhead) // load-acquire, synchronize with consumers
3097 if t-h+n >= uint32(len(_p_.runq)) {
3098 gothrow("runqsteal: runq overflow")
3100 for i := uint32(0); i < n; i++ {
3101 _p_.runq[(t+i)%uint32(len(_p_.runq))] = batch[i]
3103 atomicstore(&_p_.runqtail, t+n) // store-release, makes the item available for consumption
3107 func testSchedLocalQueue() {
3109 gs := make([]g, len(_p_.runq))
3110 for i := 0; i < len(_p_.runq); i++ {
3111 if runqget(_p_) != nil {
3112 gothrow("runq is not empty initially")
3114 for j := 0; j < i; j++ {
3115 runqput(_p_, &gs[i])
3117 for j := 0; j < i; j++ {
3118 if runqget(_p_) != &gs[i] {
3119 print("bad element at iter ", i, "/", j, "\n")
3120 gothrow("bad element")
3123 if runqget(_p_) != nil {
3124 gothrow("runq is not empty afterwards")
3129 func testSchedLocalQueueSteal() {
3132 gs := make([]g, len(p1.runq))
3133 for i := 0; i < len(p1.runq); i++ {
3134 for j := 0; j < i; j++ {
3138 gp := runqsteal(p2, p1)
3159 for j := 0; j < i; j++ {
3161 print("bad element ", j, "(", gs[j].sig, ") at iter ", i, "\n")
3162 gothrow("bad element")
3165 if s != i/2 && s != i/2+1 {
3166 print("bad steal ", s, ", want ", i/2, " or ", i/2+1, ", iter ", i, "\n")
3167 gothrow("bad steal")
3172 func setMaxThreads(in int) (out int) {
3174 out = int(sched.maxmcount)
3175 sched.maxmcount = int32(in)
3181 var goexperiment string = "GOEXPERIMENT" // TODO: defined in zaexperiment.h
3183 func haveexperiment(name string) bool {
3191 xname, x = x[:i], x[i+1:]
3201 func sync_procPin() int {
3210 func sync_procUnpin() {