1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
14 // Goroutine scheduler
15 // The scheduler's job is to distribute ready-to-run goroutines over worker threads.
17 // The main concepts are:
19 // M - worker thread, or machine.
20 // P - processor, a resource that is required to execute Go code.
21 // M must have an associated P to execute Go code, however it can be
22 // blocked or in a syscall w/o an associated P.
24 // Design doc at http://golang.org/s/go11sched.
27 // Number of goroutine ids to grab from sched.goidgen to local per-P cache at once.
28 // 16 seems to provide enough amortization, but other than that it's mostly arbitrary number.
38 G g0; // idle goroutine for m0
42 P* allp[MaxGomaxprocs+1];
47 Mutex allglock; // the following vars are protected by this lock or by stoptheworld
54 static void runqput(P*, G*);
55 static G* runqget(P*);
56 static bool runqputslow(P*, G*, uint32, uint32);
57 static G* runqsteal(P*, P*);
60 static void mcommoninit(M*);
61 static void schedule(void);
62 static void procresize(int32);
63 static void acquirep(P*);
64 static P* releasep(void);
65 static void newm(void(*)(void), P*);
66 static void stopm(void);
67 static void startm(P*, bool);
68 static void handoffp(P*);
69 static void wakep(void);
70 static void stoplockedm(void);
71 static void startlockedm(G*);
72 static void sysmon(void);
73 static uint32 retake(int64);
74 static void incidlelocked(int32);
75 static void checkdead(void);
76 static void exitsyscall0(G*);
78 static void goexit0(G*);
79 static void gfput(P*, G*);
81 static void gfpurge(P*);
82 static void globrunqput(G*);
83 static void globrunqputbatch(G*, G*, int32);
84 static G* globrunqget(P*, int32);
85 static P* pidleget(void);
86 static void pidleput(P*);
87 static void injectglist(G*);
88 static bool preemptall(void);
89 static bool preemptone(P*);
90 static bool exitsyscallfast(void);
91 static bool haveexperiment(int8*);
93 static void dropg(void);
95 extern String buildVersion;
98 // The bootstrap sequence is:
102 // make & queue new G
103 // call runtimeĀ·mstart
105 // The new G calls runtimeĀ·main.
107 // raceinit must be the first call to race detector.
108 // In particular, it must be done before mallocinit below calls racemapshadow.
111 _g_.racectx = raceinit()
114 sched.maxmcount = 10000
127 sched.lastpoll = uint64(nanotime())
129 if n := goatoi(gogetenv("GOMAXPROCS")); n > 0 {
130 if n > _MaxGomaxprocs {
135 procresize(int32(procs))
137 if buildVersion == "" {
138 // Condition should never trigger. This code just serves
139 // to ensure runtimeĀ·buildVersion is kept in the resulting binary.
140 buildVersion = "unknown"
148 func dumpgstatus(gp *g) {
150 print("runtime: gp: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", readgstatus(gp), "\n")
151 print("runtime: g: g=", _g_, ", goid=", _g_.goid, ", g->atomicstatus=", readgstatus(_g_), "\n")
155 // sched lock is held
156 if sched.mcount > sched.maxmcount {
157 print("runtime: program exceeds ", sched.maxmcount, "-thread limit\n")
158 gothrow("thread exhaustion")
162 func mcommoninit(mp *m) {
165 // g0 stack won't make sense for user (and is not necessary unwindable).
167 callers(1, &mp.createstack[0], len(mp.createstack))
170 mp.fastrand = 0x49f6428a + uint32(mp.id) + uint32(cputicks())
171 if mp.fastrand == 0 {
172 mp.fastrand = 0x49f6428a
180 if mp.gsignal != nil {
181 mp.gsignal.stackguard1 = mp.gsignal.stack.lo + _StackGuard
184 // Add to allm so garbage collector doesn't free g->m
185 // when it is just in a register or thread-local storage.
188 // NumCgoCall() iterates over allm w/o schedlock,
189 // so we need to publish it safely.
190 atomicstorep(unsafe.Pointer(&allm), unsafe.Pointer(mp))
194 // Mark gp ready to run.
196 status := readgstatus(gp)
200 _g_.m.locks++ // disable preemption because it can be holding p in a local var
201 if status&^_Gscan != _Gwaiting {
203 gothrow("bad g->status in ready")
206 // status is Gwaiting or Gscanwaiting, make Grunnable and put on runq
207 casgstatus(gp, _Gwaiting, _Grunnable)
209 if atomicload(&sched.npidle) != 0 && atomicload(&sched.nmspinning) == 0 { // TODO: fast atomic
213 if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack
214 _g_.stackguard0 = stackPreempt
218 func gcprocs() int32 {
219 // Figure out how many CPUs to use during GC.
220 // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc.
229 if n > sched.nmidle+1 { // one M is currently running
236 func needaddgcproc() bool {
245 n -= sched.nmidle + 1 // one M is currently running
250 func helpgc(nproc int32) {
254 for n := int32(1); n < nproc; n++ { // one M is currently running
255 if allp[pos].mcache == _g_.m.mcache {
260 gothrow("gcprocs inconsistency")
263 mp.mcache = allp[pos].mcache
270 // Similar to stoptheworld but best-effort and can be called several times.
271 // There is no reverse operation, used during crashing.
272 // This function must not lock any mutexes.
273 func freezetheworld() {
277 // stopwait and preemption requests can be lost
278 // due to races with concurrently executing threads,
279 // so try several times
280 for i := 0; i < 5; i++ {
281 // this should tell the scheduler to not start any new goroutines
282 sched.stopwait = 0x7fffffff
283 atomicstore(&sched.gcwaiting, 1)
284 // this should stop running goroutines
286 break // no running goroutines
296 func isscanstatus(status uint32) bool {
297 if status == _Gscan {
298 gothrow("isscanstatus: Bad status Gscan")
300 return status&_Gscan == _Gscan
303 // All reads and writes of g's status go through readgstatus, casgstatus
304 // castogscanstatus, casfrom_Gscanstatus.
306 func readgstatus(gp *g) uint32 {
307 return atomicload(&gp.atomicstatus)
310 // The Gscanstatuses are acting like locks and this releases them.
311 // If it proves to be a performance hit we should be able to make these
312 // simple atomic stores but for now we are going to throw if
313 // we see an inconsistent state.
314 func casfrom_Gscanstatus(gp *g, oldval, newval uint32) {
317 // Check that transition is valid.
323 if newval == oldval&^_Gscan {
324 success = cas(&gp.atomicstatus, oldval, newval)
327 if newval == _Gwaiting {
328 success = cas(&gp.atomicstatus, oldval, newval)
332 print("runtime: casfrom_Gscanstatus failed gp=", gp, ", oldval=", hex(oldval), ", newval=", hex(newval), "\n")
334 gothrow("casfrom_Gscanstatus: gp->status is not in scan state")
338 // This will return false if the gp is not in the expected status and the cas fails.
339 // This acts like a lock acquire while the casfromgstatus acts like a lock release.
340 func castogscanstatus(gp *g, oldval, newval uint32) bool {
345 if newval == oldval|_Gscan {
346 return cas(&gp.atomicstatus, oldval, newval)
349 if newval == _Gscanrunning || newval == _Gscanenqueue {
350 return cas(&gp.atomicstatus, oldval, newval)
353 print("runtime: castogscanstatus oldval=", hex(oldval), " newval=", hex(newval), "\n")
354 gothrow("castogscanstatus")
358 // If asked to move to or from a Gscanstatus this will throw. Use the castogscanstatus
359 // and casfrom_Gscanstatus instead.
360 // casgstatus will loop if the g->atomicstatus is in a Gscan status until the routine that
361 // put it in the Gscan state is finished.
363 func casgstatus(gp *g, oldval, newval uint32) {
364 if (oldval&_Gscan != 0) || (newval&_Gscan != 0) || oldval == newval {
366 print("casgstatus: oldval=", hex(oldval), " newval=", hex(newval), "\n")
367 gothrow("casgstatus: bad incoming values")
371 // loop if gp->atomicstatus is in a scan state giving
372 // GC time to finish and change the state to oldval.
373 for !cas(&gp.atomicstatus, oldval, newval) {
377 // stopg ensures that gp is stopped at a GC safe point where its stack can be scanned
378 // or in the context of a moving collector the pointers can be flipped from pointing
379 // to old object to pointing to new objects.
380 // If stopg returns true, the caller knows gp is at a GC safe point and will remain there until
381 // the caller calls restartg.
382 // If stopg returns false, the caller is not responsible for calling restartg. This can happen
383 // if another thread, either the gp itself or another GC thread is taking the responsibility
384 // to do the GC work related to this thread.
385 func stopg(gp *g) bool {
391 switch s := readgstatus(gp); s {
394 gothrow("stopg: gp->atomicstatus is not valid")
400 // Loop until a new stack is in place.
405 // Claim goroutine by setting scan bit.
406 if !castogscanstatus(gp, s, s|_Gscan) {
409 // In scan state, do work.
416 // Goroutine already claimed by another GC helper.
420 if gcphase == _GCscan {
421 // Running routines not scanned during
422 // GCscan phase, we only scan non-running routines.
427 // Claim goroutine, so we aren't racing with a status
428 // transition away from Grunning.
429 if !castogscanstatus(gp, _Grunning, _Gscanrunning) {
433 // Mark gp for preemption.
435 gp.preemptscan = true
437 gp.stackguard0 = stackPreempt
441 casfrom_Gscanstatus(gp, _Gscanrunning, _Grunning)
447 // The GC requests that this routine be moved from a scanmumble state to a mumble state.
448 func restartg(gp *g) {
453 gothrow("restartg: unexpected status")
461 casfrom_Gscanstatus(gp, s, s&^_Gscan)
463 // Scan is now completed.
464 // Goroutine now needs to be made runnable.
465 // We put it on the global run queue; ready blocks on the global scheduler lock.
467 casfrom_Gscanstatus(gp, _Gscanenqueue, _Gwaiting)
468 if gp != getg().m.curg {
469 gothrow("processing Gscanenqueue on wrong m")
476 func stopscanstart(gp *g) {
479 gothrow("GC not moved to G0")
482 if !isscanstatus(readgstatus(gp)) {
484 gothrow("GC not in scan state")
490 // Runs on g0 and does the actual work after putting the g back on the run queue.
491 func mquiesce(gpmaster *g) {
492 // enqueue the calling goroutine.
495 activeglen := len(allgs)
496 for i := 0; i < activeglen; i++ {
498 if readgstatus(gp) == _Gdead {
499 gp.gcworkdone = true // noop scan.
501 gp.gcworkdone = false
506 // Check that the G's gcwork (such as scanning) has been done. If not do it now.
507 // You can end up doing work here if the page trap on a Grunning Goroutine has
508 // not been sprung or in some race situations. For example a runnable goes dead
509 // and is started up again with a gp->gcworkdone set to false.
510 for i := 0; i < activeglen; i++ {
513 status := readgstatus(gp)
514 if status == _Gdead {
515 //do nothing, scan not needed.
516 gp.gcworkdone = true // scan is a noop
519 if status == _Grunning && gp.stackguard0 == uintptr(stackPreempt) && notetsleep(&sched.stopnote, 100*1000) { // nanosecond arg
520 noteclear(&sched.stopnote)
527 for i := 0; i < activeglen; i++ {
529 status := readgstatus(gp)
530 if isscanstatus(status) {
531 print("mstopandscang:bottom: post scan bad status gp=", gp, " has status ", hex(status), "\n")
534 if !gp.gcworkdone && status != _Gdead {
535 print("mstopandscang:bottom: post scan gp=", gp, "->gcworkdone still false\n")
540 schedule() // Never returns.
543 // quiesce moves all the goroutines to a GC safepoint which for now is a at preemption point.
544 // If the global gcphase is GCmark quiesce will ensure that all of the goroutine's stacks
545 // have been scanned before it returns.
546 func quiesce(mastergp *g) {
547 castogscanstatus(mastergp, _Grunning, _Gscanenqueue)
548 // Now move this to the g0 (aka m) stack.
549 // g0 will potentially scan this thread and put mastergp on the runqueue
553 // This is used by the GC as well as the routines that do stack dumps. In the case
554 // of GC all the routines can be reliably stopped. This is not always the case
555 // when the system is in panic or being exited.
556 func stoptheworld() {
559 // If we hold a lock, then we won't be able to stop another M
560 // that is blocked trying to acquire the lock.
562 gothrow("stoptheworld: holding locks")
566 sched.stopwait = gomaxprocs
567 atomicstore(&sched.gcwaiting, 1)
570 _g_.m.p.status = _Pgcstop // Pgcstop is only diagnostic.
572 // try to retake all P's in Psyscall status
573 for i := 0; i < int(gomaxprocs); i++ {
576 if s == _Psyscall && cas(&p.status, s, _Pgcstop) {
589 wait := sched.stopwait > 0
592 // wait for remaining P's to stop voluntarily
595 // wait for 100us, then try to re-preempt in case of any races
596 if notetsleep(&sched.stopnote, 100*1000) {
597 noteclear(&sched.stopnote)
603 if sched.stopwait != 0 {
604 gothrow("stoptheworld: not stopped")
606 for i := 0; i < int(gomaxprocs); i++ {
608 if p.status != _Pgcstop {
609 gothrow("stoptheworld: not stopped")
619 func starttheworld() {
622 _g_.m.locks++ // disable preemption because it can be holding p in a local var
623 gp := netpoll(false) // non-blocking
625 add := needaddgcproc()
631 procresize(gomaxprocs)
641 // procresize() puts p's with work at the beginning of the list.
642 // Once we reach a p without a run queue, the rest don't have one either.
643 if p.runqhead == p.runqtail {
651 if sched.sysmonwait != 0 {
653 notewakeup(&sched.sysmonnote)
664 gothrow("starttheworld: inconsistent mp->nextp")
669 // Start M to run P. Do not start another M below.
676 // If GC could have used another helper proc, start one now,
677 // in the hope that it will be available next time.
678 // It would have been even better to start it before the collection,
679 // but doing so requires allocating memory, so it's tricky to
680 // coordinate. This lazy approach works out in practice:
681 // we don't mind if the first couple gc rounds don't have quite
682 // the maximum number of procs.
686 if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack
687 _g_.stackguard0 = stackPreempt
691 // Called to start an M.
696 if _g_.stack.lo == 0 {
697 // Initialize stack bounds from system stack.
698 // Cgo may have left stack size in stack.hi.
703 _g_.stack.hi = uintptr(noescape(unsafe.Pointer(&size)))
704 _g_.stack.lo = _g_.stack.hi - size + 1024
706 // Initialize stack guards so that we can start calling
707 // both Go and C functions with stack growth prologues.
708 _g_.stackguard0 = _g_.stack.lo + _StackGuard
709 _g_.stackguard1 = _g_.stackguard0
717 gothrow("bad runtimeĀ·mstart")
720 // Record top of stack for use by mcall.
721 // Once we call schedule we're never coming back,
722 // so other calls can reuse this stack space.
723 gosave(&_g_.m.g0.sched)
724 _g_.m.g0.sched.pc = ^uintptr(0) // make sure it is never used
728 // Install signal handlers; after minit so that minit can
729 // prepare the thread to be able to handle the signals.
734 if _g_.m.mstartfn != nil {
735 fn := *(*func())(unsafe.Pointer(&_g_.m.mstartfn))
739 if _g_.m.helpgc != 0 {
742 } else if _g_.m != &m0 {
743 acquirep(_g_.m.nextp)
748 // TODO(brainman): This point is never reached, because scheduler
749 // does not release os threads at the moment. But once this path
750 // is enabled, we must remove our seh here.
753 // When running with cgo, we call _cgo_thread_start
754 // to start threads for us so that we can play nicely with
756 var cgoThreadStart unsafe.Pointer
758 type cgothreadstart struct {
764 // Allocate a new m unassociated with any thread.
765 // Can use p for allocation context if needed.
766 func allocm(_p_ *p) *m {
768 _g_.m.locks++ // disable GC because it can be called from sysmon
770 acquirep(_p_) // temporarily borrow p for mallocs in this function
775 // In case of cgo or Solaris, pthread_create will make us a stack.
776 // Windows and Plan 9 will layout sched stack on OS stack.
777 if iscgo || GOOS == "solaris" || GOOS == "windows" || GOOS == "plan9" {
788 if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack
789 _g_.stackguard0 = stackPreempt
799 // needm is called when a cgo callback happens on a
800 // thread without an m (a thread not created by Go).
801 // In this case, needm is expected to find an m to use
802 // and return with m, g initialized correctly.
803 // Since m and g are not set now (likely nil, but see below)
804 // needm is limited in what routines it can call. In particular
805 // it can only call nosplit functions (textflag 7) and cannot
806 // do any scheduling that requires an m.
808 // In order to avoid needing heavy lifting here, we adopt
809 // the following strategy: there is a stack of available m's
810 // that can be stolen. Using compare-and-swap
811 // to pop from the stack has ABA races, so we simulate
812 // a lock by doing an exchange (via casp) to steal the stack
813 // head and replace the top pointer with MLOCKED (1).
814 // This serves as a simple spin lock that we can use even
815 // without an m. The thread that locks the stack in this way
816 // unlocks the stack by storing a valid stack head pointer.
818 // In order to make sure that there is always an m structure
819 // available to be stolen, we maintain the invariant that there
820 // is always one more than needed. At the beginning of the
821 // program (if cgo is in use) the list is seeded with a single m.
822 // If needm finds that it has taken the last m off the list, its job
823 // is - once it has installed its own m so that it can do things like
824 // allocate memory - to create a spare m and put it on the list.
826 // Each of these extra m's also has a g0 and a curg that are
827 // pressed into service as the scheduling stack and current
828 // goroutine for the duration of the cgo callback.
830 // When the callback is done with the m, it calls dropm to
831 // put the m back on the list.
835 // Can happen if C/C++ code calls Go from a global ctor.
836 // Can not throw, because scheduler is not initialized yet.
838 // write(2, unsafe.Pointer("fatal error: cgo callback before cgo call\n"), sizeof("fatal error: cgo callback before cgo call\n") - 1)
842 // Lock extra list, take head, unlock popped list.
843 // nilokay=false is safe here because of the invariant above,
844 // that the extra list always contains or will soon contain
846 mp := lockextra(false)
848 // Set needextram when we've just emptied the list,
849 // so that the eventual call into cgocallbackg will
850 // allocate a new m for the extra list. We delay the
851 // allocation until then so that it can be done
852 // after exitsyscall makes sure it is okay to be
853 // running at all (that is, there's no garbage collection
854 // running right now).
855 mp.needextram = mp.schedlink == nil
856 unlockextra(mp.schedlink)
858 // Install g (= m->g0) and set the stack bounds
859 // to match the current stack. We don't actually know
860 // how big the stack is, like we don't know how big any
861 // scheduling stack is, but we assume there's at least 32 kB,
862 // which is more than enough for us.
865 _g_.stack.hi = uintptr(noescape(unsafe.Pointer(&x))) + 1024
866 _g_.stack.lo = uintptr(noescape(unsafe.Pointer(&x))) - 32*1024
867 _g_.stackguard0 = _g_.stack.lo + _StackGuard
869 // Initialize this thread to use the m.
874 // newextram allocates an m and puts it on the extra list.
875 // It is called with a working local m, so that it can do things
876 // like call schedlock and allocate.
878 // Create extra goroutine locked to extra m.
879 // The goroutine is the context in which the cgo callback will run.
880 // The sched.pc will never be returned to, but setting it to
881 // goexit makes clear to the traceback routines where
882 // the goroutine stack ends.
885 gp.sched.pc = funcPC(goexit) + _PCQuantum
886 gp.sched.sp = gp.stack.hi
887 gp.sched.sp -= 4 * regSize // extra space in case of reads slightly beyond frame
890 gp.syscallpc = gp.sched.pc
891 gp.syscallsp = gp.sched.sp
892 // malg returns status as Gidle, change to Gsyscall before adding to allg
893 // where GC will see it.
894 casgstatus(gp, _Gidle, _Gsyscall)
897 mp.locked = _LockInternal
900 gp.goid = int64(xadd64(&sched.goidgen, 1))
902 gp.racectx = racegostart(funcPC(newextram))
904 // put on allg for garbage collector
907 // Add m to the extra list.
908 mnext := lockextra(true)
913 // dropm is called when a cgo callback has called needm but is now
914 // done with the callback and returning back into the non-Go thread.
915 // It puts the current m back onto the extra list.
917 // The main expense here is the call to signalstack to release the
918 // m's signal stack, and then the call to needm on the next callback
919 // from this thread. It is tempting to try to save the m for next time,
920 // which would eliminate both these costs, but there might not be
921 // a next time: the current thread (which Go does not control) might exit.
922 // If we saved the m for that thread, there would be an m leak each time
923 // such a thread exited. Instead, we acquire and release an m on each
924 // call. These should typically not be scheduling operations, just a few
925 // atomics, so the cost should be small.
927 // TODO(rsc): An alternative would be to allocate a dummy pthread per-thread
928 // variable using pthread_key_create. Unlike the pthread keys we already use
929 // on OS X, this dummy key would never be read by Go code. It would exist
930 // only so that we could register at thread-exit-time destructor.
931 // That destructor would put the m back onto the extra list.
932 // This is purely a performance optimization. The current version,
933 // in which dropm happens on each cgo call, is still correct too.
934 // We may have to keep the current version on systems with cgo
935 // but without pthreads, like Windows.
937 // Undo whatever initialization minit did during needm.
940 // Clear m and g, and return m to the extra list.
941 // After the call to setmg we can only call nosplit functions.
945 mnext := lockextra(true)
952 // lockextra locks the extra list and returns the list head.
953 // The caller must unlock the list by storing a new list head
954 // to extram. If nilokay is true, then lockextra will
955 // return a nil list head if that's what it finds. If nilokay is false,
956 // lockextra will keep waiting until the list head is no longer nil.
958 func lockextra(nilokay bool) *m {
962 old := atomicloaduintptr(&extram)
968 if old == 0 && !nilokay {
972 if casuintptr(&extram, old, locked) {
973 return (*m)(unsafe.Pointer(old))
982 func unlockextra(mp *m) {
983 atomicstoreuintptr(&extram, uintptr(unsafe.Pointer(mp)))
986 // Create a new m. It will start off with a call to fn, or else the scheduler.
987 func _newm(fn func(), _p_ *p) {
990 mp.mstartfn = *(*unsafe.Pointer)(unsafe.Pointer(&fn))
993 var ts cgothreadstart
994 if _cgo_thread_start == nil {
995 gothrow("_cgo_thread_start missing")
998 ts.tls = (*uint64)(unsafe.Pointer(&mp.tls[0]))
999 ts.fn = unsafe.Pointer(funcPC(mstart))
1000 asmcgocall(_cgo_thread_start, unsafe.Pointer(&ts))
1003 newosproc(mp, unsafe.Pointer(mp.g0.stack.hi))
1006 // Stops execution of the current m until new work is available.
1007 // Returns with acquired P.
1011 if _g_.m.locks != 0 {
1012 gothrow("stopm holding locks")
1015 gothrow("stopm holding p")
1018 _g_.m.spinning = false
1019 xadd(&sched.nmspinning, -1)
1026 notesleep(&_g_.m.park)
1027 noteclear(&_g_.m.park)
1028 if _g_.m.helpgc != 0 {
1034 acquirep(_g_.m.nextp)
1039 getg().m.spinning = true
1042 // Schedules some M to run the p (creates an M if necessary).
1043 // If p==nil, tries to get an idle P, if no idle P's does nothing.
1044 func startm(_p_ *p, spinning bool) {
1051 xadd(&sched.nmspinning, -1)
1067 gothrow("startm: m is spinning")
1069 if mp.nextp != nil {
1070 gothrow("startm: m has p")
1072 mp.spinning = spinning
1074 notewakeup(&mp.park)
1077 // Hands off P from syscall or locked M.
1078 func handoffp(_p_ *p) {
1079 // if it has local work, start it straight away
1080 if _p_.runqhead != _p_.runqtail || sched.runqsize != 0 {
1084 // no local work, check that there are no spinning/idle M's,
1085 // otherwise our help is not required
1086 if atomicload(&sched.nmspinning)+atomicload(&sched.npidle) == 0 && cas(&sched.nmspinning, 0, 1) { // TODO: fast atomic
1091 if sched.gcwaiting != 0 {
1092 _p_.status = _Pgcstop
1094 if sched.stopwait == 0 {
1095 notewakeup(&sched.stopnote)
1100 if sched.runqsize != 0 {
1105 // If this is the last running P and nobody is polling network,
1106 // need to wakeup another M to poll network.
1107 if sched.npidle == uint32(gomaxprocs-1) && atomicload64(&sched.lastpoll) != 0 {
1116 // Tries to add one more P to execute G's.
1117 // Called when a G is made runnable (newproc, ready).
1119 // be conservative about spinning threads
1120 if !cas(&sched.nmspinning, 0, 1) {
1126 // Stops execution of the current m that is locked to a g until the g is runnable again.
1127 // Returns with acquired P.
1128 func stoplockedm() {
1131 if _g_.m.lockedg == nil || _g_.m.lockedg.lockedm != _g_.m {
1132 gothrow("stoplockedm: inconsistent locking")
1135 // Schedule another M to run this p.
1140 // Wait until another thread schedules lockedg again.
1141 notesleep(&_g_.m.park)
1142 noteclear(&_g_.m.park)
1143 status := readgstatus(_g_.m.lockedg)
1144 if status&^_Gscan != _Grunnable {
1145 print("runtime:stoplockedm: g is not Grunnable or Gscanrunnable\n")
1147 gothrow("stoplockedm: not runnable")
1149 acquirep(_g_.m.nextp)
1153 // Schedules the locked m to run the locked gp.
1154 func startlockedm(gp *g) {
1159 gothrow("startlockedm: locked to me")
1161 if mp.nextp != nil {
1162 gothrow("startlockedm: m has p")
1164 // directly handoff current P to the locked m
1168 notewakeup(&mp.park)
1172 // Stops the current m for stoptheworld.
1173 // Returns when the world is restarted.
1177 if sched.gcwaiting == 0 {
1178 gothrow("gcstopm: not waiting for gc")
1181 _g_.m.spinning = false
1182 xadd(&sched.nmspinning, -1)
1186 _p_.status = _Pgcstop
1188 if sched.stopwait == 0 {
1189 notewakeup(&sched.stopnote)
1195 // Schedules gp to run on the current M.
1197 func execute(gp *g) {
1200 casgstatus(gp, _Grunnable, _Grunning)
1203 gp.stackguard0 = gp.stack.lo + _StackGuard
1208 // Check whether the profiler needs to be turned on or off.
1209 hz := sched.profilehz
1210 if _g_.m.profilehz != hz {
1211 resetcpuprofiler(hz)
1217 // Finds a runnable goroutine to execute.
1218 // Tries to steal from other P's, get g from global queue, poll network.
1219 func findrunnable() *g {
1223 if sched.gcwaiting != 0 {
1227 if fingwait && fingwake {
1228 if gp := wakefing(); gp != nil {
1234 if gp := runqget(_g_.m.p); gp != nil {
1239 if sched.runqsize != 0 {
1241 gp := globrunqget(_g_.m.p, 0)
1248 // poll network - returns list of goroutines
1249 if gp := netpoll(false); gp != nil { // non-blocking
1250 injectglist(gp.schedlink)
1251 casgstatus(gp, _Gwaiting, _Grunnable)
1255 // If number of spinning M's >= number of busy P's, block.
1256 // This is necessary to prevent excessive CPU consumption
1257 // when GOMAXPROCS>>1 but the program parallelism is low.
1258 if !_g_.m.spinning && 2*atomicload(&sched.nmspinning) >= uint32(gomaxprocs)-atomicload(&sched.npidle) { // TODO: fast atomic
1261 if !_g_.m.spinning {
1262 _g_.m.spinning = true
1263 xadd(&sched.nmspinning, 1)
1265 // random steal from other P's
1266 for i := 0; i < int(2*gomaxprocs); i++ {
1267 if sched.gcwaiting != 0 {
1270 _p_ := allp[fastrand1()%uint32(gomaxprocs)]
1275 gp = runqsteal(_g_.m.p, _p_)
1283 // return P and block
1285 if sched.gcwaiting != 0 {
1289 if sched.runqsize != 0 {
1290 gp := globrunqget(_g_.m.p, 0)
1298 _g_.m.spinning = false
1299 xadd(&sched.nmspinning, -1)
1302 // check all runqueues once again
1303 for i := 0; i < int(gomaxprocs); i++ {
1305 if _p_ != nil && _p_.runqhead != _p_.runqtail {
1318 if xchg64(&sched.lastpoll, 0) != 0 {
1320 gothrow("findrunnable: netpoll with p")
1323 gothrow("findrunnable: netpoll with spinning")
1325 gp := netpoll(true) // block until new work is available
1326 atomicstore64(&sched.lastpoll, uint64(nanotime()))
1333 injectglist(gp.schedlink)
1334 casgstatus(gp, _Gwaiting, _Grunnable)
1344 func resetspinning() {
1347 var nmspinning uint32
1349 _g_.m.spinning = false
1350 nmspinning = xadd(&sched.nmspinning, -1)
1352 gothrow("findrunnable: negative nmspinning")
1355 nmspinning = atomicload(&sched.nmspinning)
1358 // M wakeup policy is deliberately somewhat conservative (see nmspinning handling),
1359 // so see if we need to wakeup another P here.
1360 if nmspinning == 0 && atomicload(&sched.npidle) > 0 {
1365 // Injects the list of runnable G's into the scheduler.
1366 // Can run concurrently with GC.
1367 func injectglist(glist *g) {
1373 for n = 0; glist != nil; n++ {
1375 glist = gp.schedlink
1376 casgstatus(gp, _Gwaiting, _Grunnable)
1380 for ; n != 0 && sched.npidle != 0; n-- {
1385 // One round of scheduler: find a runnable goroutine and execute it.
1390 if _g_.m.locks != 0 {
1391 gothrow("schedule: holding locks")
1394 if _g_.m.lockedg != nil {
1396 execute(_g_.m.lockedg) // Never returns.
1400 if sched.gcwaiting != 0 {
1406 // Check the global runnable queue once in a while to ensure fairness.
1407 // Otherwise two goroutines can completely occupy the local runqueue
1408 // by constantly respawning each other.
1409 tick := _g_.m.p.schedtick
1410 // This is a fancy way to say tick%61==0,
1411 // it uses 2 MUL instructions instead of a single DIV and so is faster on modern processors.
1412 if uint64(tick)-((uint64(tick)*0x4325c53f)>>36)*61 == 0 && sched.runqsize > 0 {
1414 gp = globrunqget(_g_.m.p, 1)
1421 gp = runqget(_g_.m.p)
1422 if gp != nil && _g_.m.spinning {
1423 gothrow("schedule: spinning with local work")
1427 gp = findrunnable() // blocks until work is available
1431 if gp.lockedm != nil {
1432 // Hands off own p to the locked m,
1433 // then blocks waiting for a new p.
1441 // dropg removes the association between m and the current goroutine m->curg (gp for short).
1442 // Typically a caller sets gp's status away from Grunning and then
1443 // immediately calls dropg to finish the job. The caller is also responsible
1444 // for arranging that gp will be restarted using ready at an
1445 // appropriate time. After calling dropg and arranging for gp to be
1446 // readied later, the caller can do other work but eventually should
1447 // call schedule to restart the scheduling of goroutines on this m.
1451 if _g_.m.lockedg == nil {
1457 // Puts the current goroutine into a waiting state and calls unlockf.
1458 // If unlockf returns false, the goroutine is resumed.
1459 func park(unlockf func(*g, unsafe.Pointer) bool, lock unsafe.Pointer, reason string) {
1462 _g_.m.waitlock = lock
1463 _g_.m.waitunlockf = *(*unsafe.Pointer)(unsafe.Pointer(&unlockf))
1464 _g_.waitreason = reason
1468 func parkunlock_c(gp *g, lock unsafe.Pointer) bool {
1469 unlock((*mutex)(lock))
1473 // Puts the current goroutine into a waiting state and unlocks the lock.
1474 // The goroutine can be made runnable again by calling ready(gp).
1475 func parkunlock(lock *mutex, reason string) {
1476 park(parkunlock_c, unsafe.Pointer(lock), reason)
1479 // park continuation on g0.
1480 func park_m(gp *g) {
1483 casgstatus(gp, _Grunning, _Gwaiting)
1486 if _g_.m.waitunlockf != nil {
1487 fn := *(*func(*g, unsafe.Pointer) bool)(unsafe.Pointer(&_g_.m.waitunlockf))
1488 ok := fn(gp, _g_.m.waitlock)
1489 _g_.m.waitunlockf = nil
1490 _g_.m.waitlock = nil
1492 casgstatus(gp, _Gwaiting, _Grunnable)
1493 execute(gp) // Schedule it back, never returns.
1499 // Gosched continuation on g0.
1500 func gosched_m(gp *g) {
1501 status := readgstatus(gp)
1502 if status&^_Gscan != _Grunning {
1504 gothrow("bad g status")
1506 casgstatus(gp, _Grunning, _Grunnable)
1515 // Finishes execution of the current goroutine.
1516 // Must be NOSPLIT because it is called from Go. (TODO - probably not anymore)
1525 // goexit continuation on g0.
1526 func goexit0(gp *g) {
1529 casgstatus(gp, _Grunning, _Gdead)
1533 gp.paniconfault = false
1534 gp._defer = nil // should be true already but just in case.
1535 gp._panic = nil // non-nil for Goexit during panic. points at stack-allocated data.
1542 if _g_.m.locked&^_LockExternal != 0 {
1543 print("invalid m->locked = ", _g_.m.locked, "\n")
1544 gothrow("internal lockOSThread error")
1552 func save(pc, sp uintptr) {
1559 _g_.sched.ctxt = nil
1560 // write as uintptr to avoid write barrier, which will smash _g_.sched.
1561 *(*uintptr)(unsafe.Pointer(&_g_.sched.g)) = uintptr(unsafe.Pointer(_g_))
1564 // The goroutine g is about to enter a system call.
1565 // Record that it's not using the cpu anymore.
1566 // This is called only from the go syscall library and cgocall,
1567 // not from the low-level system calls used by the
1569 // Entersyscall cannot split the stack: the gosave must
1570 // make g->sched refer to the caller's stack segment, because
1571 // entersyscall is going to return immediately after.
1573 // Nothing entersyscall calls can split the stack either.
1574 // We cannot safely move the stack during an active call to syscall,
1575 // because we do not know which of the uintptr arguments are
1576 // really pointers (back into the stack).
1577 // In practice, this means that we make the fast path run through
1578 // entersyscall doing no-split things, and the slow path has to use systemstack
1579 // to run bigger things on the system stack.
1581 // reentersyscall is the entry point used by cgo callbacks, where explicitly
1582 // saved SP and PC are restored. This is needed when exitsyscall will be called
1583 // from a function further up in the call stack than the parent, as g->syscallsp
1584 // must always point to a valid stack frame. entersyscall below is the normal
1585 // entry point for syscalls, which obtains the SP and PC from the caller.
1587 func reentersyscall(pc, sp uintptr) {
1590 // Disable preemption because during this function g is in Gsyscall status,
1591 // but can have inconsistent g->sched, do not let GC observe it.
1594 // Entersyscall must not call any function that might split/grow the stack.
1595 // (See details in comment above.)
1596 // Catch calls that might, by replacing the stack guard with something that
1597 // will trip any stack check and leaving a flag to tell newstack to die.
1598 _g_.stackguard0 = stackPreempt
1599 _g_.throwsplit = true
1601 // Leave SP around for GC and traceback.
1605 casgstatus(_g_, _Grunning, _Gsyscall)
1606 if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp {
1607 systemstack(func() {
1608 print("entersyscall inconsistent ", hex(_g_.syscallsp), " [", hex(_g_.stack.lo), ",", hex(_g_.stack.hi), "]\n")
1609 gothrow("entersyscall")
1613 if atomicload(&sched.sysmonwait) != 0 { // TODO: fast atomic
1614 systemstack(entersyscall_sysmon)
1620 atomicstore(&_g_.m.p.status, _Psyscall)
1621 if sched.gcwaiting != 0 {
1622 systemstack(entersyscall_gcwait)
1626 // Goroutines must not split stacks in Gsyscall status (it would corrupt g->sched).
1627 // We set _StackGuard to StackPreempt so that first split stack check calls morestack.
1628 // Morestack detects this case and throws.
1629 _g_.stackguard0 = stackPreempt
1633 // Standard syscall entry used by the go syscall library and normal cgo calls.
1635 func entersyscall(dummy int32) {
1636 reentersyscall(getcallerpc(unsafe.Pointer(&dummy)), getcallersp(unsafe.Pointer(&dummy)))
1639 func entersyscall_sysmon() {
1641 if atomicload(&sched.sysmonwait) != 0 {
1642 atomicstore(&sched.sysmonwait, 0)
1643 notewakeup(&sched.sysmonnote)
1648 func entersyscall_gcwait() {
1652 if sched.stopwait > 0 && cas(&_g_.m.p.status, _Psyscall, _Pgcstop) {
1653 if sched.stopwait--; sched.stopwait == 0 {
1654 notewakeup(&sched.stopnote)
1660 // The same as entersyscall(), but with a hint that the syscall is blocking.
1662 func entersyscallblock(dummy int32) {
1665 _g_.m.locks++ // see comment in entersyscall
1666 _g_.throwsplit = true
1667 _g_.stackguard0 = stackPreempt // see comment in entersyscall
1669 // Leave SP around for GC and traceback.
1670 pc := getcallerpc(unsafe.Pointer(&dummy))
1671 sp := getcallersp(unsafe.Pointer(&dummy))
1673 _g_.syscallsp = _g_.sched.sp
1674 _g_.syscallpc = _g_.sched.pc
1675 if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp {
1678 sp3 := _g_.syscallsp
1679 systemstack(func() {
1680 print("entersyscallblock inconsistent ", hex(sp1), " ", hex(sp2), " ", hex(sp3), " [", hex(_g_.stack.lo), ",", hex(_g_.stack.hi), "]\n")
1681 gothrow("entersyscallblock")
1684 casgstatus(_g_, _Grunning, _Gsyscall)
1685 if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp {
1686 systemstack(func() {
1687 print("entersyscallblock inconsistent ", hex(sp), " ", hex(_g_.sched.sp), " ", hex(_g_.syscallsp), " [", hex(_g_.stack.lo), ",", hex(_g_.stack.hi), "]\n")
1688 gothrow("entersyscallblock")
1692 systemstack(entersyscallblock_handoff)
1694 // Resave for traceback during blocked call.
1695 save(getcallerpc(unsafe.Pointer(&dummy)), getcallersp(unsafe.Pointer(&dummy)))
1700 func entersyscallblock_handoff() {
1701 handoffp(releasep())
1704 // The goroutine g exited its system call.
1705 // Arrange for it to run on a cpu again.
1706 // This is called only from the go syscall library, not
1707 // from the low-level system calls used by the
1709 func exitsyscall(dummy int32) {
1712 _g_.m.locks++ // see comment in entersyscall
1713 if getcallersp(unsafe.Pointer(&dummy)) > _g_.syscallsp {
1714 gothrow("exitsyscall: syscall frame is no longer valid")
1718 if exitsyscallfast() {
1719 if _g_.m.mcache == nil {
1720 gothrow("lost mcache")
1722 // There's a cpu for us, so we can run.
1723 _g_.m.p.syscalltick++
1724 // We need to cas the status and scan before resuming...
1725 casgstatus(_g_, _Gsyscall, _Grunning)
1727 // Garbage collector isn't running (since we are),
1728 // so okay to clear syscallsp.
1732 // restore the preemption request in case we've cleared it in newstack
1733 _g_.stackguard0 = stackPreempt
1735 // otherwise restore the real _StackGuard, we've spoiled it in entersyscall/entersyscallblock
1736 _g_.stackguard0 = _g_.stack.lo + _StackGuard
1738 _g_.throwsplit = false
1744 // Call the scheduler.
1747 if _g_.m.mcache == nil {
1748 gothrow("lost mcache")
1751 // Scheduler returned, so we're allowed to run now.
1752 // Delete the syscallsp information that we left for
1753 // the garbage collector during the system call.
1754 // Must wait until now because until gosched returns
1755 // we don't know for sure that the garbage collector
1758 _g_.m.p.syscalltick++
1759 _g_.throwsplit = false
1763 func exitsyscallfast() bool {
1766 // Freezetheworld sets stopwait but does not retake P's.
1767 if sched.stopwait != 0 {
1773 // Try to re-acquire the last P.
1774 if _g_.m.p != nil && _g_.m.p.status == _Psyscall && cas(&_g_.m.p.status, _Psyscall, _Prunning) {
1775 // There's a cpu for us, so we can run.
1776 _g_.m.mcache = _g_.m.p.mcache
1781 // Try to get any other idle P.
1784 if sched.pidle != nil {
1786 systemstack(func() {
1787 ok = exitsyscallfast_pidle()
1796 func exitsyscallfast_pidle() bool {
1799 if _p_ != nil && atomicload(&sched.sysmonwait) != 0 {
1800 atomicstore(&sched.sysmonwait, 0)
1801 notewakeup(&sched.sysmonnote)
1811 // exitsyscall slow path on g0.
1812 // Failed to acquire P, enqueue gp as runnable.
1813 func exitsyscall0(gp *g) {
1816 casgstatus(gp, _Gsyscall, _Grunnable)
1822 } else if atomicload(&sched.sysmonwait) != 0 {
1823 atomicstore(&sched.sysmonwait, 0)
1824 notewakeup(&sched.sysmonnote)
1829 execute(gp) // Never returns.
1831 if _g_.m.lockedg != nil {
1832 // Wait until another thread schedules gp and so m again.
1834 execute(gp) // Never returns.
1837 schedule() // Never returns.
1843 // Fork can hang if preempted with signals frequently enough (see issue 5517).
1844 // Ensure that we stay on the same M where we disable profiling.
1846 if gp.m.profilehz != 0 {
1850 // This function is called before fork in syscall package.
1851 // Code between fork and exec must not allocate memory nor even try to grow stack.
1852 // Here we spoil g->_StackGuard to reliably detect any attempts to grow stack.
1853 // runtime_AfterFork will undo this in parent process, but not in child.
1854 gp.stackguard0 = stackFork
1857 // Called from syscall package before fork.
1859 func syscall_BeforeFork() {
1860 systemstack(beforefork)
1866 // See the comment in beforefork.
1867 gp.stackguard0 = gp.stack.lo + _StackGuard
1869 hz := sched.profilehz
1871 resetcpuprofiler(hz)
1876 // Called from syscall package after fork in parent.
1878 func syscall_AfterFork() {
1879 systemstack(afterfork)
1882 // Allocate a new g, with a stack big enough for stacksize bytes.
1883 func malg(stacksize int32) *g {
1886 stacksize = round2(_StackSystem + stacksize)
1887 systemstack(func() {
1888 newg.stack = stackalloc(uint32(stacksize))
1890 newg.stackguard0 = newg.stack.lo + _StackGuard
1891 newg.stackguard1 = ^uintptr(0)
1896 // Create a new g running fn with siz bytes of arguments.
1897 // Put it on the queue of g's waiting to run.
1898 // The compiler turns a go statement into a call to this.
1899 // Cannot split the stack because it assumes that the arguments
1900 // are available sequentially after &fn; they would not be
1901 // copied if a stack split occurred.
1903 func newproc(siz int32, fn *funcval) {
1904 argp := add(unsafe.Pointer(&fn), ptrSize)
1905 if hasLinkRegister {
1906 argp = add(argp, ptrSize) // skip caller's saved LR
1909 pc := getcallerpc(unsafe.Pointer(&siz))
1910 systemstack(func() {
1911 newproc1(fn, (*uint8)(argp), siz, 0, pc)
1915 // Create a new g running fn with narg bytes of arguments starting
1916 // at argp and returning nret bytes of results. callerpc is the
1917 // address of the go statement that created this. The new g is put
1918 // on the queue of g's waiting to run.
1919 func newproc1(fn *funcval, argp *uint8, narg int32, nret int32, callerpc uintptr) *g {
1923 _g_.m.throwing = -1 // do not dump full stacks
1924 gothrow("go of nil func value")
1926 _g_.m.locks++ // disable preemption because it can be holding p in a local var
1928 siz = (siz + 7) &^ 7
1930 // We could allocate a larger initial stack if necessary.
1931 // Not worth it: this is almost always an error.
1932 // 4*sizeof(uintreg): extra space added below
1933 // sizeof(uintreg): caller's LR (arm) or return address (x86, in gostartcall).
1934 if siz >= _StackMin-4*regSize-regSize {
1935 gothrow("newproc: function arguments too large for new goroutine")
1941 newg = malg(_StackMin)
1942 casgstatus(newg, _Gidle, _Gdead)
1943 allgadd(newg) // publishes with a g->status of Gdead so GC scanner doesn't look at uninitialized stack.
1945 if newg.stack.hi == 0 {
1946 gothrow("newproc1: newg missing stack")
1949 if readgstatus(newg) != _Gdead {
1950 gothrow("newproc1: new g is not Gdead")
1954 sp -= 4 * regSize // extra space in case of reads slightly beyond frame
1956 memmove(unsafe.Pointer(sp), unsafe.Pointer(argp), uintptr(narg))
1957 if hasLinkRegister {
1960 *(*unsafe.Pointer)(unsafe.Pointer(sp)) = nil
1963 memclr(unsafe.Pointer(&newg.sched), unsafe.Sizeof(newg.sched))
1965 newg.sched.pc = funcPC(goexit) + _PCQuantum // +PCQuantum so that previous instruction is in same function
1967 gostartcallfn(&newg.sched, fn)
1968 newg.gopc = callerpc
1969 casgstatus(newg, _Gdead, _Grunnable)
1971 if _p_.goidcache == _p_.goidcacheend {
1972 // Sched.goidgen is the last allocated id,
1973 // this batch must be [sched.goidgen+1, sched.goidgen+GoidCacheBatch].
1974 // At startup sched.goidgen=0, so main goroutine receives goid=1.
1975 _p_.goidcache = xadd64(&sched.goidgen, _GoidCacheBatch)
1976 _p_.goidcache -= _GoidCacheBatch - 1
1977 _p_.goidcacheend = _p_.goidcache + _GoidCacheBatch
1979 newg.goid = int64(_p_.goidcache)
1982 newg.racectx = racegostart(callerpc)
1986 if atomicload(&sched.npidle) != 0 && atomicload(&sched.nmspinning) == 0 && unsafe.Pointer(fn.fn) != unsafe.Pointer(funcPC(main)) { // TODO: fast atomic
1990 if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack
1991 _g_.stackguard0 = stackPreempt
1996 // Put on gfree list.
1997 // If local list is too long, transfer a batch to the global list.
1998 func gfput(_p_ *p, gp *g) {
1999 if readgstatus(gp) != _Gdead {
2000 gothrow("gfput: bad status (not Gdead)")
2003 stksize := gp.stack.hi - gp.stack.lo
2005 if stksize != _FixedStack {
2006 // non-standard stack size - free it.
2013 gp.schedlink = _p_.gfree
2016 if _p_.gfreecnt >= 64 {
2018 for _p_.gfreecnt >= 32 {
2021 _p_.gfree = gp.schedlink
2022 gp.schedlink = sched.gfree
2026 unlock(&sched.gflock)
2030 // Get from gfree list.
2031 // If local list is empty, grab a batch from global list.
2032 func gfget(_p_ *p) *g {
2035 if gp == nil && sched.gfree != nil {
2037 for _p_.gfreecnt < 32 && sched.gfree != nil {
2040 sched.gfree = gp.schedlink
2042 gp.schedlink = _p_.gfree
2045 unlock(&sched.gflock)
2049 _p_.gfree = gp.schedlink
2051 if gp.stack.lo == 0 {
2052 // Stack was deallocated in gfput. Allocate a new one.
2053 systemstack(func() {
2054 gp.stack = stackalloc(_FixedStack)
2056 gp.stackguard0 = gp.stack.lo + _StackGuard
2059 racemalloc(unsafe.Pointer(gp.stack.lo), gp.stack.hi-gp.stack.lo)
2066 // Purge all cached G's from gfree list to the global list.
2067 func gfpurge(_p_ *p) {
2069 for _p_.gfreecnt != 0 {
2072 _p_.gfree = gp.schedlink
2073 gp.schedlink = sched.gfree
2077 unlock(&sched.gflock)
2080 // Breakpoint executes a breakpoint trap.
2085 // dolockOSThread is called by LockOSThread and lockOSThread below
2086 // after they modify m.locked. Do not allow preemption during this call,
2087 // or else the m might be different in this function than in the caller.
2089 func dolockOSThread() {
2097 // LockOSThread wires the calling goroutine to its current operating system thread.
2098 // Until the calling goroutine exits or calls UnlockOSThread, it will always
2099 // execute in that thread, and no other goroutine can.
2100 func LockOSThread() {
2101 getg().m.locked |= _LockExternal
2106 func lockOSThread() {
2107 getg().m.locked += _LockInternal
2111 // dounlockOSThread is called by UnlockOSThread and unlockOSThread below
2112 // after they update m->locked. Do not allow preemption during this call,
2113 // or else the m might be in different in this function than in the caller.
2115 func dounlockOSThread() {
2117 if _g_.m.locked != 0 {
2126 // UnlockOSThread unwires the calling goroutine from its fixed operating system thread.
2127 // If the calling goroutine has not called LockOSThread, UnlockOSThread is a no-op.
2128 func UnlockOSThread() {
2129 getg().m.locked &^= _LockExternal
2134 func unlockOSThread() {
2136 if _g_.m.locked < _LockInternal {
2137 systemstack(badunlockosthread)
2139 _g_.m.locked -= _LockInternal
2143 func badunlockosthread() {
2144 gothrow("runtime: internal error: misuse of lockOSThread/unlockOSThread")
2147 func gcount() int32 {
2148 n := int32(allglen) - sched.ngfree
2157 // All these variables can be changed concurrently, so the result can be inconsistent.
2158 // But at least the current goroutine is running.
2165 func mcount() int32 {
2174 func _System() { _System() }
2175 func _ExternalCode() { _ExternalCode() }
2176 func _GC() { _GC() }
2180 // Called if we receive a SIGPROF signal.
2181 func sigprof(pc *uint8, sp *uint8, lr *uint8, gp *g, mp *m) {
2184 var stk [100]uintptr
2190 // Profiling runs concurrently with GC, so it must not allocate.
2193 // Define that a "user g" is a user-created goroutine, and a "system g"
2194 // is one that is m->g0 or m->gsignal. We've only made sure that we
2195 // can unwind user g's, so exclude the system g's.
2197 // It is not quite as easy as testing gp == m->curg (the current user g)
2198 // because we might be interrupted for profiling halfway through a
2199 // goroutine switch. The switch involves updating three (or four) values:
2200 // g, PC, SP, and (on arm) LR. The PC must be the last to be updated,
2201 // because once it gets updated the new g is running.
2203 // When switching from a user g to a system g, LR is not considered live,
2204 // so the update only affects g, SP, and PC. Since PC must be last, there
2205 // the possible partial transitions in ordinary execution are (1) g alone is updated,
2206 // (2) both g and SP are updated, and (3) SP alone is updated.
2207 // If g is updated, we'll see a system g and not look closer.
2208 // If SP alone is updated, we can detect the partial transition by checking
2209 // whether the SP is within g's stack bounds. (We could also require that SP
2210 // be changed only after g, but the stack bounds check is needed by other
2211 // cases, so there is no need to impose an additional requirement.)
2213 // There is one exceptional transition to a system g, not in ordinary execution.
2214 // When a signal arrives, the operating system starts the signal handler running
2215 // with an updated PC and SP. The g is updated last, at the beginning of the
2216 // handler. There are two reasons this is okay. First, until g is updated the
2217 // g and SP do not match, so the stack bounds check detects the partial transition.
2218 // Second, signal handlers currently run with signals disabled, so a profiling
2219 // signal cannot arrive during the handler.
2221 // When switching from a system g to a user g, there are three possibilities.
2223 // First, it may be that the g switch has no PC update, because the SP
2224 // either corresponds to a user g throughout (as in asmcgocall)
2225 // or because it has been arranged to look like a user g frame
2226 // (as in cgocallback_gofunc). In this case, since the entire
2227 // transition is a g+SP update, a partial transition updating just one of
2228 // those will be detected by the stack bounds check.
2230 // Second, when returning from a signal handler, the PC and SP updates
2231 // are performed by the operating system in an atomic update, so the g
2232 // update must be done before them. The stack bounds check detects
2233 // the partial transition here, and (again) signal handlers run with signals
2234 // disabled, so a profiling signal cannot arrive then anyway.
2236 // Third, the common case: it may be that the switch updates g, SP, and PC
2237 // separately, as in gogo.
2239 // Because gogo is the only instance, we check whether the PC lies
2240 // within that function, and if so, not ask for a traceback. This approach
2241 // requires knowing the size of the gogo function, which we
2242 // record in arch_*.h and check in runtime_test.go.
2244 // There is another apparently viable approach, recorded here in case
2245 // the "PC within gogo" check turns out not to be usable.
2246 // It would be possible to delay the update of either g or SP until immediately
2247 // before the PC update instruction. Then, because of the stack bounds check,
2248 // the only problematic interrupt point is just before that PC update instruction,
2249 // and the sigprof handler can detect that instruction and simulate stepping past
2250 // it in order to reach a consistent state. On ARM, the update of g must be made
2251 // in two places (in R10 and also in a TLS slot), so the delayed update would
2252 // need to be the SP update. The sigprof handler must read the instruction at
2253 // the current PC and if it was the known instruction (for example, JMP BX or
2254 // MOV R2, PC), use that other register in place of the PC value.
2255 // The biggest drawback to this solution is that it requires that we can tell
2256 // whether it's safe to read from the memory pointed at by PC.
2257 // In a correct program, we can test PC == nil and otherwise read,
2258 // but if a profiling signal happens at the instant that a program executes
2259 // a bad jump (before the program manages to handle the resulting fault)
2260 // the profiling handler could fault trying to read nonexistent memory.
2262 // To recap, there are no constraints on the assembly being used for the
2263 // transition. We simply require that g and SP match and that the PC is not
2266 usp := uintptr(unsafe.Pointer(sp))
2267 gogo := funcPC(gogo)
2268 if gp == nil || gp != mp.curg ||
2269 usp < gp.stack.lo || gp.stack.hi < usp ||
2270 (gogo <= uintptr(unsafe.Pointer(pc)) && uintptr(unsafe.Pointer(pc)) < gogo+_RuntimeGogoBytes) {
2276 n = int32(gentraceback(uintptr(unsafe.Pointer(pc)), uintptr(unsafe.Pointer(sp)), uintptr(unsafe.Pointer(lr)), gp, 0, &stk[0], len(stk), nil, nil, _TraceTrap))
2278 if !traceback || n <= 0 {
2279 // Normal traceback is impossible or has failed.
2280 // See if it falls into several common cases.
2282 if mp.ncgo > 0 && mp.curg != nil && mp.curg.syscallpc != 0 && mp.curg.syscallsp != 0 {
2283 // Cgo, we can't unwind and symbolize arbitrary C code,
2284 // so instead collect Go stack that leads to the cgo call.
2285 // This is especially important on windows, since all syscalls are cgo calls.
2286 n = int32(gentraceback(mp.curg.syscallpc, mp.curg.syscallsp, 0, mp.curg, 0, &stk[0], len(stk), nil, nil, 0))
2288 if GOOS == "windows" && n == 0 && mp.libcallg != nil && mp.libcallpc != 0 && mp.libcallsp != 0 {
2289 // Libcall, i.e. runtime syscall on windows.
2290 // Collect Go stack that leads to the call.
2291 n = int32(gentraceback(mp.libcallpc, mp.libcallsp, 0, mp.libcallg, 0, &stk[0], len(stk), nil, nil, 0))
2294 // If all of the above has failed, account it against abstract "System" or "GC".
2296 // "ExternalCode" is better than "etext".
2297 if uintptr(unsafe.Pointer(pc)) > uintptr(unsafe.Pointer(&etext)) {
2298 pc = (*uint8)(unsafe.Pointer(uintptr(funcPC(_ExternalCode) + _PCQuantum)))
2300 stk[0] = uintptr(unsafe.Pointer(pc))
2301 if mp.gcing != 0 || mp.helpgc != 0 {
2302 stk[1] = funcPC(_GC) + _PCQuantum
2304 stk[1] = funcPC(_System) + _PCQuantum
2310 // Simple cas-lock to coordinate with setcpuprofilerate.
2311 for !cas(&prof.lock, 0, 1) {
2315 cpuproftick(&stk[0], n)
2317 atomicstore(&prof.lock, 0)
2322 // Arrange to call fn with a traceback hz times a second.
2323 func setcpuprofilerate_m(hz int32) {
2324 // Force sane arguments.
2329 // Disable preemption, otherwise we can be rescheduled to another thread
2330 // that has profiling enabled.
2334 // Stop profiler on this thread so that it is safe to lock prof.
2335 // if a profiling signal came in while we had prof locked,
2336 // it would deadlock.
2339 for !cas(&prof.lock, 0, 1) {
2343 atomicstore(&prof.lock, 0)
2346 sched.profilehz = hz
2350 resetcpuprofiler(hz)
2356 // Change number of processors. The world is stopped, sched is locked.
2357 // gcworkbufs are not being modified by either the GC or
2358 // the write barrier code.
2359 func procresize(new int32) {
2361 if old < 0 || old > _MaxGomaxprocs || new <= 0 || new > _MaxGomaxprocs {
2362 gothrow("procresize: invalid arg")
2365 // initialize new P's
2366 for i := int32(0); i < new; i++ {
2372 atomicstorep(unsafe.Pointer(&allp[i]), unsafe.Pointer(p))
2374 if p.mcache == nil {
2375 if old == 0 && i == 0 {
2376 if getg().m.mcache == nil {
2377 gothrow("missing mcache?")
2379 p.mcache = getg().m.mcache // bootstrap
2381 p.mcache = allocmcache()
2386 // redistribute runnable G's evenly
2387 // collect all runnable goroutines in global queue preserving FIFO order
2388 // FIFO order is required to ensure fairness even during frequent GCs
2389 // see http://golang.org/issue/7126
2393 for i := int32(0); i < old; i++ {
2395 if p.runqhead == p.runqtail {
2399 // pop from tail of local queue
2401 gp := p.runq[p.runqtail%uint32(len(p.runq))]
2402 // push onto head of global queue
2403 gp.schedlink = sched.runqhead
2405 if sched.runqtail == nil {
2412 // fill local queues with at most len(p.runq)/2 goroutines
2413 // start at 1 because current M already executes some G and will acquire allp[0] below,
2414 // so if we have a spare G we want to put it into allp[1].
2416 for i := int32(1); i < new*int32(len(_p_.runq))/2 && sched.runqsize > 0; i++ {
2417 gp := sched.runqhead
2418 sched.runqhead = gp.schedlink
2419 if sched.runqhead == nil {
2420 sched.runqtail = nil
2423 runqput(allp[i%new], gp)
2427 for i := new; i < old; i++ {
2429 freemcache(p.mcache)
2433 // can't free P itself because it can be referenced by an M in syscall
2446 for i := new - 1; i > 0; i-- {
2451 var int32p *int32 = &gomaxprocs // make compiler check that gomaxprocs is an int32
2452 atomicstore((*uint32)(unsafe.Pointer(int32p)), uint32(new))
2455 // Associate p and the current m.
2456 func acquirep(_p_ *p) {
2459 if _g_.m.p != nil || _g_.m.mcache != nil {
2460 gothrow("acquirep: already in go")
2462 if _p_.m != nil || _p_.status != _Pidle {
2467 print("acquirep: p->m=", _p_.m, "(", id, ") p->status=", _p_.status, "\n")
2468 gothrow("acquirep: invalid p state")
2470 _g_.m.mcache = _p_.mcache
2473 _p_.status = _Prunning
2476 // Disassociate p and the current m.
2477 func releasep() *p {
2480 if _g_.m.p == nil || _g_.m.mcache == nil {
2481 gothrow("releasep: invalid arg")
2484 if _p_.m != _g_.m || _p_.mcache != _g_.m.mcache || _p_.status != _Prunning {
2485 print("releasep: m=", _g_.m, " m->p=", _g_.m.p, " p->m=", _p_.m, " m->mcache=", _g_.m.mcache, " p->mcache=", _p_.mcache, " p->status=", _p_.status, "\n")
2486 gothrow("releasep: invalid p state")
2495 func incidlelocked(v int32) {
2497 sched.nmidlelocked += v
2504 // Check for deadlock situation.
2505 // The check is based on number of running M's, if 0 -> deadlock.
2507 // If we are dying because of a signal caught on an already idle thread,
2508 // freezetheworld will cause all running threads to block.
2509 // And runtime will essentially enter into deadlock state,
2510 // except that there is a thread that will call exit soon.
2516 run := sched.mcount - sched.nmidle - sched.nmidlelocked - 1
2521 print("runtime: checkdead: nmidle=", sched.nmidle, " nmidlelocked=", sched.nmidlelocked, " mcount=", sched.mcount, "\n")
2522 gothrow("checkdead: inconsistent counts")
2527 for i := 0; i < len(allgs); i++ {
2532 s := readgstatus(gp)
2533 switch s &^ _Gscan {
2540 print("runtime: checkdead: find g ", gp.goid, " in status ", s, "\n")
2541 gothrow("checkdead: runnable g")
2545 if grunning == 0 { // possible if main goroutine calls runtimeĀ·Goexit()
2546 gothrow("no goroutines (main called runtime.Goexit) - deadlock!")
2549 // Maybe jump time forward for playground.
2552 casgstatus(gp, _Gwaiting, _Grunnable)
2556 gothrow("checkdead: no p for timer")
2563 notewakeup(&mp.park)
2568 getg().m.throwing = -1 // do not dump full stacks
2569 gothrow("all goroutines are asleep - deadlock!")
2573 // If we go two minutes without a garbage collection, force one to run.
2574 forcegcperiod := int64(2 * 60 * 1e9)
2576 // If a heap span goes unused for 5 minutes after a garbage collection,
2577 // we hand it back to the operating system.
2578 scavengelimit := int64(5 * 60 * 1e9)
2580 if debug.scavenge > 0 {
2581 // Scavenge-a-lot for testing.
2582 forcegcperiod = 10 * 1e6
2583 scavengelimit = 20 * 1e6
2586 lastscavenge := nanotime()
2589 // Make wake-up period small enough for the sampling to be correct.
2590 maxsleep := forcegcperiod / 2
2591 if scavengelimit < forcegcperiod {
2592 maxsleep = scavengelimit / 2
2595 lasttrace := int64(0)
2596 idle := 0 // how many cycles in succession we had not wokeup somebody
2599 if idle == 0 { // start with 20us sleep...
2601 } else if idle > 50 { // start doubling the sleep after 1ms...
2604 if delay > 10*1000 { // up to 10ms
2608 if debug.schedtrace <= 0 && (sched.gcwaiting != 0 || atomicload(&sched.npidle) == uint32(gomaxprocs)) { // TODO: fast atomic
2610 if atomicload(&sched.gcwaiting) != 0 || atomicload(&sched.npidle) == uint32(gomaxprocs) {
2611 atomicstore(&sched.sysmonwait, 1)
2613 notetsleep(&sched.sysmonnote, maxsleep)
2615 atomicstore(&sched.sysmonwait, 0)
2616 noteclear(&sched.sysmonnote)
2622 // poll network if not polled for more than 10ms
2623 lastpoll := int64(atomicload64(&sched.lastpoll))
2625 unixnow := unixnanotime()
2626 if lastpoll != 0 && lastpoll+10*1000*1000 < now {
2627 cas64(&sched.lastpoll, uint64(lastpoll), uint64(now))
2628 gp := netpoll(false) // non-blocking - returns list of goroutines
2630 // Need to decrement number of idle locked M's
2631 // (pretending that one more is running) before injectglist.
2632 // Otherwise it can lead to the following situation:
2633 // injectglist grabs all P's but before it starts M's to run the P's,
2634 // another M returns from syscall, finishes running its G,
2635 // observes that there is no work to do and no other running M's
2636 // and reports deadlock.
2642 // retake P's blocked in syscalls
2643 // and preempt long running G's
2644 if retake(now) != 0 {
2649 // check if we need to force a GC
2650 lastgc := int64(atomicload64(&memstats.last_gc))
2651 if lastgc != 0 && unixnow-lastgc > forcegcperiod && atomicload(&forcegc.idle) != 0 {
2654 forcegc.g.schedlink = nil
2655 injectglist(forcegc.g)
2656 unlock(&forcegc.lock)
2658 // scavenge heap once in a while
2659 if lastscavenge+scavengelimit/2 < now {
2660 mHeap_Scavenge(int32(nscavenge), uint64(now), uint64(scavengelimit))
2664 if debug.schedtrace > 0 && lasttrace+int64(debug.schedtrace*1000000) <= now {
2666 schedtrace(debug.scheddetail > 0)
2671 var pdesc [_MaxGomaxprocs]struct {
2678 func retake(now int64) uint32 {
2680 for i := int32(0); i < gomaxprocs; i++ {
2688 // Retake P from syscall if it's there for more than 1 sysmon tick (at least 20us).
2689 t := int64(_p_.syscalltick)
2690 if int64(pd.syscalltick) != t {
2691 pd.syscalltick = uint32(t)
2692 pd.syscallwhen = now
2695 // On the one hand we don't want to retake Ps if there is no other work to do,
2696 // but on the other hand we want to retake them eventually
2697 // because they can prevent the sysmon thread from deep sleep.
2698 if _p_.runqhead == _p_.runqtail && atomicload(&sched.nmspinning)+atomicload(&sched.npidle) > 0 && pd.syscallwhen+10*1000*1000 > now {
2701 // Need to decrement number of idle locked M's
2702 // (pretending that one more is running) before the CAS.
2703 // Otherwise the M from which we retake can exit the syscall,
2704 // increment nmidle and report deadlock.
2706 if cas(&_p_.status, s, _Pidle) {
2711 } else if s == _Prunning {
2712 // Preempt G if it's running for more than 10ms.
2713 t := int64(_p_.schedtick)
2714 if int64(pd.schedtick) != t {
2715 pd.schedtick = uint32(t)
2719 if pd.schedwhen+10*1000*1000 > now {
2728 // Tell all goroutines that they have been preempted and they should stop.
2729 // This function is purely best-effort. It can fail to inform a goroutine if a
2730 // processor just started running it.
2731 // No locks need to be held.
2732 // Returns true if preemption request was issued to at least one goroutine.
2733 func preemptall() bool {
2735 for i := int32(0); i < gomaxprocs; i++ {
2737 if _p_ == nil || _p_.status != _Prunning {
2740 if preemptone(_p_) {
2747 // Tell the goroutine running on processor P to stop.
2748 // This function is purely best-effort. It can incorrectly fail to inform the
2749 // goroutine. It can send inform the wrong goroutine. Even if it informs the
2750 // correct goroutine, that goroutine might ignore the request if it is
2751 // simultaneously executing newstack.
2752 // No lock needs to be held.
2753 // Returns true if preemption request was issued.
2754 // The actual preemption will happen at some point in the future
2755 // and will be indicated by the gp->status no longer being
2757 func preemptone(_p_ *p) bool {
2759 if mp == nil || mp == getg().m {
2763 if gp == nil || gp == mp.g0 {
2769 // Every call in a go routine checks for stack overflow by
2770 // comparing the current stack pointer to gp->stackguard0.
2771 // Setting gp->stackguard0 to StackPreempt folds
2772 // preemption into the normal stack overflow check.
2773 gp.stackguard0 = stackPreempt
2779 func schedtrace(detailed bool) {
2786 print("SCHED ", (now-starttime)/1e6, "ms: gomaxprocs=", gomaxprocs, " idleprocs=", sched.npidle, " threads=", sched.mcount, " spinningthreads=", sched.nmspinning, " idlethreads=", sched.nmidle, " runqueue=", sched.runqsize)
2788 print(" gcwaiting=", sched.gcwaiting, " nmidlelocked=", sched.nmidlelocked, " stopwait=", sched.stopwait, " sysmonwait=", sched.sysmonwait, "\n")
2790 // We must be careful while reading data from P's, M's and G's.
2791 // Even if we hold schedlock, most data can be changed concurrently.
2792 // E.g. (p->m ? p->m->id : -1) can crash if p->m changes from non-nil to nil.
2793 for i := int32(0); i < gomaxprocs; i++ {
2799 h := atomicload(&_p_.runqhead)
2800 t := atomicload(&_p_.runqtail)
2806 print(" P", i, ": status=", _p_.status, " schedtick=", _p_.schedtick, " syscalltick=", _p_.syscalltick, " m=", id, " runqsize=", t-h, " gfreecnt=", _p_.gfreecnt, "\n")
2808 // In non-detailed mode format lengths of per-P run queues as:
2809 // [len1 len2 len3 len4]
2815 if i == gomaxprocs-1 {
2826 for mp := allm; mp != nil; mp = mp.alllink {
2829 lockedg := mp.lockedg
2842 print(" M", mp.id, ": p=", id1, " curg=", id2, " mallocing=", mp.mallocing, " throwing=", mp.throwing, " gcing=", mp.gcing, ""+" locks=", mp.locks, " dying=", mp.dying, " helpgc=", mp.helpgc, " spinning=", mp.spinning, " blocked=", getg().m.blocked, " lockedg=", id3, "\n")
2846 for gi := 0; gi < len(allgs); gi++ {
2849 lockedm := gp.lockedm
2858 print(" G", gp.goid, ": status=", readgstatus(gp), "(", gp.waitreason, ") m=", id1, " lockedm=", id2, "\n")
2864 // Put mp on midle list.
2865 // Sched must be locked.
2867 mp.schedlink = sched.midle
2873 // Try to get an m from midle list.
2874 // Sched must be locked.
2878 sched.midle = mp.schedlink
2884 // Put gp on the global runnable queue.
2885 // Sched must be locked.
2886 func globrunqput(gp *g) {
2888 if sched.runqtail != nil {
2889 sched.runqtail.schedlink = gp
2897 // Put a batch of runnable goroutines on the global runnable queue.
2898 // Sched must be locked.
2899 func globrunqputbatch(ghead *g, gtail *g, n int32) {
2900 gtail.schedlink = nil
2901 if sched.runqtail != nil {
2902 sched.runqtail.schedlink = ghead
2904 sched.runqhead = ghead
2906 sched.runqtail = gtail
2910 // Try get a batch of G's from the global runnable queue.
2911 // Sched must be locked.
2912 func globrunqget(_p_ *p, max int32) *g {
2913 if sched.runqsize == 0 {
2917 n := sched.runqsize/gomaxprocs + 1
2918 if n > sched.runqsize {
2921 if max > 0 && n > max {
2924 if n > int32(len(_p_.runq))/2 {
2925 n = int32(len(_p_.runq)) / 2
2929 if sched.runqsize == 0 {
2930 sched.runqtail = nil
2933 gp := sched.runqhead
2934 sched.runqhead = gp.schedlink
2937 gp1 := sched.runqhead
2938 sched.runqhead = gp1.schedlink
2944 // Put p to on _Pidle list.
2945 // Sched must be locked.
2946 func pidleput(_p_ *p) {
2947 _p_.link = sched.pidle
2949 xadd(&sched.npidle, 1) // TODO: fast atomic
2952 // Try get a p from _Pidle list.
2953 // Sched must be locked.
2954 func pidleget() *p {
2957 sched.pidle = _p_.link
2958 xadd(&sched.npidle, -1) // TODO: fast atomic
2963 // Try to put g on local runnable queue.
2964 // If it's full, put onto global queue.
2965 // Executed only by the owner P.
2966 func runqput(_p_ *p, gp *g) {
2968 h := atomicload(&_p_.runqhead) // load-acquire, synchronize with consumers
2970 if t-h < uint32(len(_p_.runq)) {
2971 _p_.runq[t%uint32(len(_p_.runq))] = gp
2972 atomicstore(&_p_.runqtail, t+1) // store-release, makes the item available for consumption
2975 if runqputslow(_p_, gp, h, t) {
2978 // the queue is not full, now the put above must suceed
2982 // Put g and a batch of work from local runnable queue on global queue.
2983 // Executed only by the owner P.
2984 func runqputslow(_p_ *p, gp *g, h, t uint32) bool {
2985 var batch [len(_p_.runq)/2 + 1]*g
2987 // First, grab a batch from local queue.
2990 if n != uint32(len(_p_.runq)/2) {
2991 gothrow("runqputslow: queue is not full")
2993 for i := uint32(0); i < n; i++ {
2994 batch[i] = _p_.runq[(h+i)%uint32(len(_p_.runq))]
2996 if !cas(&_p_.runqhead, h, h+n) { // cas-release, commits consume
3001 // Link the goroutines.
3002 for i := uint32(0); i < n; i++ {
3003 batch[i].schedlink = batch[i+1]
3006 // Now put the batch on global queue.
3008 globrunqputbatch(batch[0], batch[n], int32(n+1))
3013 // Get g from local runnable queue.
3014 // Executed only by the owner P.
3015 func runqget(_p_ *p) *g {
3017 h := atomicload(&_p_.runqhead) // load-acquire, synchronize with other consumers
3022 gp := _p_.runq[h%uint32(len(_p_.runq))]
3023 if cas(&_p_.runqhead, h, h+1) { // cas-release, commits consume
3029 // Grabs a batch of goroutines from local runnable queue.
3030 // batch array must be of size len(p->runq)/2. Returns number of grabbed goroutines.
3031 // Can be executed by any P.
3032 func runqgrab(_p_ *p, batch []*g) uint32 {
3034 h := atomicload(&_p_.runqhead) // load-acquire, synchronize with other consumers
3035 t := atomicload(&_p_.runqtail) // load-acquire, synchronize with the producer
3041 if n > uint32(len(_p_.runq)/2) { // read inconsistent h and t
3044 for i := uint32(0); i < n; i++ {
3045 batch[i] = _p_.runq[(h+i)%uint32(len(_p_.runq))]
3047 if cas(&_p_.runqhead, h, h+n) { // cas-release, commits consume
3053 // Steal half of elements from local runnable queue of p2
3054 // and put onto local runnable queue of p.
3055 // Returns one of the stolen elements (or nil if failed).
3056 func runqsteal(_p_, p2 *p) *g {
3057 var batch [len(_p_.runq) / 2]*g
3059 n := runqgrab(p2, batch[:])
3068 h := atomicload(&_p_.runqhead) // load-acquire, synchronize with consumers
3070 if t-h+n >= uint32(len(_p_.runq)) {
3071 gothrow("runqsteal: runq overflow")
3073 for i := uint32(0); i < n; i++ {
3074 _p_.runq[(t+i)%uint32(len(_p_.runq))] = batch[i]
3076 atomicstore(&_p_.runqtail, t+n) // store-release, makes the item available for consumption
3080 func testSchedLocalQueue() {
3082 gs := make([]g, len(_p_.runq))
3083 for i := 0; i < len(_p_.runq); i++ {
3084 if runqget(_p_) != nil {
3085 gothrow("runq is not empty initially")
3087 for j := 0; j < i; j++ {
3088 runqput(_p_, &gs[i])
3090 for j := 0; j < i; j++ {
3091 if runqget(_p_) != &gs[i] {
3092 print("bad element at iter ", i, "/", j, "\n")
3093 gothrow("bad element")
3096 if runqget(_p_) != nil {
3097 gothrow("runq is not empty afterwards")
3102 func testSchedLocalQueueSteal() {
3105 gs := make([]g, len(p1.runq))
3106 for i := 0; i < len(p1.runq); i++ {
3107 for j := 0; j < i; j++ {
3111 gp := runqsteal(p2, p1)
3132 for j := 0; j < i; j++ {
3134 print("bad element ", j, "(", gs[j].sig, ") at iter ", i, "\n")
3135 gothrow("bad element")
3138 if s != i/2 && s != i/2+1 {
3139 print("bad steal ", s, ", want ", i/2, " or ", i/2+1, ", iter ", i, "\n")
3140 gothrow("bad steal")
3145 func setMaxThreads(in int) (out int) {
3147 out = int(sched.maxmcount)
3148 sched.maxmcount = int32(in)
3154 var goexperiment string = "GOEXPERIMENT" // TODO: defined in zaexperiment.h
3156 func haveexperiment(name string) bool {
3164 xname, x = x[:i], x[i+1:]
3174 func sync_procPin() int {
3183 func sync_procUnpin() {