1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
14 // Goroutine scheduler
15 // The scheduler's job is to distribute ready-to-run goroutines over worker threads.
17 // The main concepts are:
19 // M - worker thread, or machine.
20 // P - processor, a resource that is required to execute Go code.
21 // M must have an associated P to execute Go code, however it can be
22 // blocked or in a syscall w/o an associated P.
24 // Design doc at http://golang.org/s/go11sched.
27 // Number of goroutine ids to grab from sched.goidgen to local per-P cache at once.
28 // 16 seems to provide enough amortization, but other than that it's mostly arbitrary number.
38 G g0; // idle goroutine for m0
42 P* allp[MaxGomaxprocs+1];
47 Mutex allglock; // the following vars are protected by this lock or by stoptheworld
54 static void runqput(P*, G*);
55 static G* runqget(P*);
56 static bool runqputslow(P*, G*, uint32, uint32);
57 static G* runqsteal(P*, P*);
60 static void mcommoninit(M*);
61 static void schedule(void);
62 static void procresize(int32);
63 static void acquirep(P*);
64 static P* releasep(void);
65 static void newm(void(*)(void), P*);
66 static void stopm(void);
67 static void startm(P*, bool);
68 static void handoffp(P*);
69 static void wakep(void);
70 static void stoplockedm(void);
71 static void startlockedm(G*);
72 static void sysmon(void);
73 static uint32 retake(int64);
74 static void incidlelocked(int32);
75 static void checkdead(void);
76 static void exitsyscall0(G*);
78 static void goexit0(G*);
79 static void gfput(P*, G*);
81 static void gfpurge(P*);
82 static void globrunqput(G*);
83 static void globrunqputbatch(G*, G*, int32);
84 static G* globrunqget(P*, int32);
85 static P* pidleget(void);
86 static void pidleput(P*);
87 static void injectglist(G*);
88 static bool preemptall(void);
89 static bool preemptone(P*);
90 static bool exitsyscallfast(void);
91 static bool haveexperiment(int8*);
93 static void dropg(void);
95 extern String buildVersion;
98 // The bootstrap sequence is:
102 // make & queue new G
103 // call runtimeĀ·mstart
105 // The new G calls runtimeĀ·main.
107 // raceinit must be the first call to race detector.
108 // In particular, it must be done before mallocinit below calls racemapshadow.
111 _g_.racectx = raceinit()
114 sched.maxmcount = 10000
127 sched.lastpoll = uint64(nanotime())
129 if n := goatoi(gogetenv("GOMAXPROCS")); n > 0 {
130 if n > _MaxGomaxprocs {
135 procresize(int32(procs))
137 if buildVersion == "" {
138 // Condition should never trigger. This code just serves
139 // to ensure runtimeĀ·buildVersion is kept in the resulting binary.
140 buildVersion = "unknown"
148 func dumpgstatus(gp *g) {
150 print("runtime: gp: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", readgstatus(gp), "\n")
151 print("runtime: g: g=", _g_, ", goid=", _g_.goid, ", g->atomicstatus=", readgstatus(_g_), "\n")
155 // sched lock is held
156 if sched.mcount > sched.maxmcount {
157 print("runtime: program exceeds ", sched.maxmcount, "-thread limit\n")
158 gothrow("thread exhaustion")
162 func mcommoninit(mp *m) {
165 // g0 stack won't make sense for user (and is not necessary unwindable).
167 callers(1, &mp.createstack[0], len(mp.createstack))
170 mp.fastrand = 0x49f6428a + uint32(mp.id) + uint32(cputicks())
171 if mp.fastrand == 0 {
172 mp.fastrand = 0x49f6428a
180 if mp.gsignal != nil {
181 mp.gsignal.stackguard1 = mp.gsignal.stack.lo + _StackGuard
184 // Add to allm so garbage collector doesn't free g->m
185 // when it is just in a register or thread-local storage.
188 // NumCgoCall() iterates over allm w/o schedlock,
189 // so we need to publish it safely.
190 atomicstorep(unsafe.Pointer(&allm), unsafe.Pointer(mp))
194 // Mark gp ready to run.
196 status := readgstatus(gp)
200 _g_.m.locks++ // disable preemption because it can be holding p in a local var
201 if status&^_Gscan != _Gwaiting {
203 gothrow("bad g->status in ready")
206 // status is Gwaiting or Gscanwaiting, make Grunnable and put on runq
207 casgstatus(gp, _Gwaiting, _Grunnable)
209 if atomicload(&sched.npidle) != 0 && atomicload(&sched.nmspinning) == 0 { // TODO: fast atomic
213 if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack
214 _g_.stackguard0 = stackPreempt
218 func gcprocs() int32 {
219 // Figure out how many CPUs to use during GC.
220 // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc.
229 if n > sched.nmidle+1 { // one M is currently running
236 func needaddgcproc() bool {
245 n -= sched.nmidle + 1 // one M is currently running
250 func helpgc(nproc int32) {
254 for n := int32(1); n < nproc; n++ { // one M is currently running
255 if allp[pos].mcache == _g_.m.mcache {
260 gothrow("gcprocs inconsistency")
263 mp.mcache = allp[pos].mcache
270 // Similar to stoptheworld but best-effort and can be called several times.
271 // There is no reverse operation, used during crashing.
272 // This function must not lock any mutexes.
273 func freezetheworld() {
277 // stopwait and preemption requests can be lost
278 // due to races with concurrently executing threads,
279 // so try several times
280 for i := 0; i < 5; i++ {
281 // this should tell the scheduler to not start any new goroutines
282 sched.stopwait = 0x7fffffff
283 atomicstore(&sched.gcwaiting, 1)
284 // this should stop running goroutines
286 break // no running goroutines
296 func isscanstatus(status uint32) bool {
297 if status == _Gscan {
298 gothrow("isscanstatus: Bad status Gscan")
300 return status&_Gscan == _Gscan
303 // All reads and writes of g's status go through readgstatus, casgstatus
304 // castogscanstatus, casfrom_Gscanstatus.
306 func readgstatus(gp *g) uint32 {
307 return atomicload(&gp.atomicstatus)
310 // The Gscanstatuses are acting like locks and this releases them.
311 // If it proves to be a performance hit we should be able to make these
312 // simple atomic stores but for now we are going to throw if
313 // we see an inconsistent state.
314 func casfrom_Gscanstatus(gp *g, oldval, newval uint32) {
317 // Check that transition is valid.
323 if newval == oldval&^_Gscan {
324 success = cas(&gp.atomicstatus, oldval, newval)
327 if newval == _Gwaiting {
328 success = cas(&gp.atomicstatus, oldval, newval)
332 print("runtime: casfrom_Gscanstatus failed gp=", gp, ", oldval=", hex(oldval), ", newval=", hex(newval), "\n")
334 gothrow("casfrom_Gscanstatus: gp->status is not in scan state")
338 // This will return false if the gp is not in the expected status and the cas fails.
339 // This acts like a lock acquire while the casfromgstatus acts like a lock release.
340 func castogscanstatus(gp *g, oldval, newval uint32) bool {
345 if newval == oldval|_Gscan {
346 return cas(&gp.atomicstatus, oldval, newval)
349 if newval == _Gscanrunning || newval == _Gscanenqueue {
350 return cas(&gp.atomicstatus, oldval, newval)
353 print("runtime: castogscanstatus oldval=", hex(oldval), " newval=", hex(newval), "\n")
354 gothrow("castogscanstatus")
358 // If asked to move to or from a Gscanstatus this will throw. Use the castogscanstatus
359 // and casfrom_Gscanstatus instead.
360 // casgstatus will loop if the g->atomicstatus is in a Gscan status until the routine that
361 // put it in the Gscan state is finished.
363 func casgstatus(gp *g, oldval, newval uint32) {
364 if (oldval&_Gscan != 0) || (newval&_Gscan != 0) || oldval == newval {
366 print("casgstatus: oldval=", hex(oldval), " newval=", hex(newval), "\n")
367 gothrow("casgstatus: bad incoming values")
371 // loop if gp->atomicstatus is in a scan state giving
372 // GC time to finish and change the state to oldval.
373 for !cas(&gp.atomicstatus, oldval, newval) {
374 // Help GC if needed.
375 if gp.preemptscan && !gp.gcworkdone && (oldval == _Grunning || oldval == _Gsyscall) {
376 gp.preemptscan = false
384 // stopg ensures that gp is stopped at a GC safe point where its stack can be scanned
385 // or in the context of a moving collector the pointers can be flipped from pointing
386 // to old object to pointing to new objects.
387 // If stopg returns true, the caller knows gp is at a GC safe point and will remain there until
388 // the caller calls restartg.
389 // If stopg returns false, the caller is not responsible for calling restartg. This can happen
390 // if another thread, either the gp itself or another GC thread is taking the responsibility
391 // to do the GC work related to this thread.
392 func stopg(gp *g) bool {
398 switch s := readgstatus(gp); s {
401 gothrow("stopg: gp->atomicstatus is not valid")
407 // Loop until a new stack is in place.
412 // Claim goroutine by setting scan bit.
413 if !castogscanstatus(gp, s, s|_Gscan) {
416 // In scan state, do work.
423 // Goroutine already claimed by another GC helper.
427 // Claim goroutine, so we aren't racing with a status
428 // transition away from Grunning.
429 if !castogscanstatus(gp, _Grunning, _Gscanrunning) {
433 // Mark gp for preemption.
435 gp.preemptscan = true
437 gp.stackguard0 = stackPreempt
441 casfrom_Gscanstatus(gp, _Gscanrunning, _Grunning)
447 // The GC requests that this routine be moved from a scanmumble state to a mumble state.
448 func restartg(gp *g) {
453 gothrow("restartg: unexpected status")
461 casfrom_Gscanstatus(gp, s, s&^_Gscan)
463 // Scan is now completed.
464 // Goroutine now needs to be made runnable.
465 // We put it on the global run queue; ready blocks on the global scheduler lock.
467 casfrom_Gscanstatus(gp, _Gscanenqueue, _Gwaiting)
468 if gp != getg().m.curg {
469 gothrow("processing Gscanenqueue on wrong m")
476 func stopscanstart(gp *g) {
479 gothrow("GC not moved to G0")
482 if !isscanstatus(readgstatus(gp)) {
484 gothrow("GC not in scan state")
490 // Runs on g0 and does the actual work after putting the g back on the run queue.
491 func mquiesce(gpmaster *g) {
492 activeglen := len(allgs)
493 // enqueue the calling goroutine.
495 for i := 0; i < activeglen; i++ {
497 if readgstatus(gp) == _Gdead {
498 gp.gcworkdone = true // noop scan.
500 gp.gcworkdone = false
505 // Check that the G's gcwork (such as scanning) has been done. If not do it now.
506 // You can end up doing work here if the page trap on a Grunning Goroutine has
507 // not been sprung or in some race situations. For example a runnable goes dead
508 // and is started up again with a gp->gcworkdone set to false.
509 for i := 0; i < activeglen; i++ {
512 status := readgstatus(gp)
513 if status == _Gdead {
514 //do nothing, scan not needed.
515 gp.gcworkdone = true // scan is a noop
518 if status == _Grunning && gp.stackguard0 == uintptr(stackPreempt) && notetsleep(&sched.stopnote, 100*1000) { // nanosecond arg
519 noteclear(&sched.stopnote)
526 for i := 0; i < activeglen; i++ {
528 status := readgstatus(gp)
529 if isscanstatus(status) {
530 print("mstopandscang:bottom: post scan bad status gp=", gp, " has status ", hex(status), "\n")
533 if !gp.gcworkdone && status != _Gdead {
534 print("mstopandscang:bottom: post scan gp=", gp, "->gcworkdone still false\n")
539 schedule() // Never returns.
542 // quiesce moves all the goroutines to a GC safepoint which for now is a at preemption point.
543 // If the global gcphase is GCmark quiesce will ensure that all of the goroutine's stacks
544 // have been scanned before it returns.
545 func quiesce(mastergp *g) {
546 castogscanstatus(mastergp, _Grunning, _Gscanenqueue)
547 // Now move this to the g0 (aka m) stack.
548 // g0 will potentially scan this thread and put mastergp on the runqueue
552 // This is used by the GC as well as the routines that do stack dumps. In the case
553 // of GC all the routines can be reliably stopped. This is not always the case
554 // when the system is in panic or being exited.
555 func stoptheworld() {
558 // If we hold a lock, then we won't be able to stop another M
559 // that is blocked trying to acquire the lock.
561 gothrow("stoptheworld: holding locks")
565 sched.stopwait = gomaxprocs
566 atomicstore(&sched.gcwaiting, 1)
569 _g_.m.p.status = _Pgcstop // Pgcstop is only diagnostic.
571 // try to retake all P's in Psyscall status
572 for i := 0; i < int(gomaxprocs); i++ {
575 if s == _Psyscall && cas(&p.status, s, _Pgcstop) {
588 wait := sched.stopwait > 0
591 // wait for remaining P's to stop voluntarily
594 // wait for 100us, then try to re-preempt in case of any races
595 if notetsleep(&sched.stopnote, 100*1000) {
596 noteclear(&sched.stopnote)
602 if sched.stopwait != 0 {
603 gothrow("stoptheworld: not stopped")
605 for i := 0; i < int(gomaxprocs); i++ {
607 if p.status != _Pgcstop {
608 gothrow("stoptheworld: not stopped")
618 func starttheworld() {
621 _g_.m.locks++ // disable preemption because it can be holding p in a local var
622 gp := netpoll(false) // non-blocking
624 add := needaddgcproc()
630 procresize(gomaxprocs)
640 // procresize() puts p's with work at the beginning of the list.
641 // Once we reach a p without a run queue, the rest don't have one either.
642 if p.runqhead == p.runqtail {
650 if sched.sysmonwait != 0 {
652 notewakeup(&sched.sysmonnote)
663 gothrow("starttheworld: inconsistent mp->nextp")
668 // Start M to run P. Do not start another M below.
675 // If GC could have used another helper proc, start one now,
676 // in the hope that it will be available next time.
677 // It would have been even better to start it before the collection,
678 // but doing so requires allocating memory, so it's tricky to
679 // coordinate. This lazy approach works out in practice:
680 // we don't mind if the first couple gc rounds don't have quite
681 // the maximum number of procs.
685 if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack
686 _g_.stackguard0 = stackPreempt
690 // Called to start an M.
695 if _g_.stack.lo == 0 {
696 // Initialize stack bounds from system stack.
697 // Cgo may have left stack size in stack.hi.
702 _g_.stack.hi = uintptr(noescape(unsafe.Pointer(&size)))
703 _g_.stack.lo = _g_.stack.hi - size + 1024
705 // Initialize stack guards so that we can start calling
706 // both Go and C functions with stack growth prologues.
707 _g_.stackguard0 = _g_.stack.lo + _StackGuard
708 _g_.stackguard1 = _g_.stackguard0
716 gothrow("bad runtimeĀ·mstart")
719 // Record top of stack for use by mcall.
720 // Once we call schedule we're never coming back,
721 // so other calls can reuse this stack space.
722 gosave(&_g_.m.g0.sched)
723 _g_.m.g0.sched.pc = ^uintptr(0) // make sure it is never used
727 // Install signal handlers; after minit so that minit can
728 // prepare the thread to be able to handle the signals.
733 if _g_.m.mstartfn != nil {
734 fn := *(*func())(unsafe.Pointer(&_g_.m.mstartfn))
738 if _g_.m.helpgc != 0 {
741 } else if _g_.m != &m0 {
742 acquirep(_g_.m.nextp)
747 // TODO(brainman): This point is never reached, because scheduler
748 // does not release os threads at the moment. But once this path
749 // is enabled, we must remove our seh here.
752 // When running with cgo, we call _cgo_thread_start
753 // to start threads for us so that we can play nicely with
755 var cgoThreadStart unsafe.Pointer
757 type cgothreadstart struct {
763 // Allocate a new m unassociated with any thread.
764 // Can use p for allocation context if needed.
765 func allocm(_p_ *p) *m {
767 _g_.m.locks++ // disable GC because it can be called from sysmon
769 acquirep(_p_) // temporarily borrow p for mallocs in this function
774 // In case of cgo or Solaris, pthread_create will make us a stack.
775 // Windows and Plan 9 will layout sched stack on OS stack.
776 if iscgo || GOOS == "solaris" || GOOS == "windows" || GOOS == "plan9" {
787 if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack
788 _g_.stackguard0 = stackPreempt
798 // needm is called when a cgo callback happens on a
799 // thread without an m (a thread not created by Go).
800 // In this case, needm is expected to find an m to use
801 // and return with m, g initialized correctly.
802 // Since m and g are not set now (likely nil, but see below)
803 // needm is limited in what routines it can call. In particular
804 // it can only call nosplit functions (textflag 7) and cannot
805 // do any scheduling that requires an m.
807 // In order to avoid needing heavy lifting here, we adopt
808 // the following strategy: there is a stack of available m's
809 // that can be stolen. Using compare-and-swap
810 // to pop from the stack has ABA races, so we simulate
811 // a lock by doing an exchange (via casp) to steal the stack
812 // head and replace the top pointer with MLOCKED (1).
813 // This serves as a simple spin lock that we can use even
814 // without an m. The thread that locks the stack in this way
815 // unlocks the stack by storing a valid stack head pointer.
817 // In order to make sure that there is always an m structure
818 // available to be stolen, we maintain the invariant that there
819 // is always one more than needed. At the beginning of the
820 // program (if cgo is in use) the list is seeded with a single m.
821 // If needm finds that it has taken the last m off the list, its job
822 // is - once it has installed its own m so that it can do things like
823 // allocate memory - to create a spare m and put it on the list.
825 // Each of these extra m's also has a g0 and a curg that are
826 // pressed into service as the scheduling stack and current
827 // goroutine for the duration of the cgo callback.
829 // When the callback is done with the m, it calls dropm to
830 // put the m back on the list.
834 // Can happen if C/C++ code calls Go from a global ctor.
835 // Can not throw, because scheduler is not initialized yet.
837 // write(2, unsafe.Pointer("fatal error: cgo callback before cgo call\n"), sizeof("fatal error: cgo callback before cgo call\n") - 1)
841 // Lock extra list, take head, unlock popped list.
842 // nilokay=false is safe here because of the invariant above,
843 // that the extra list always contains or will soon contain
845 mp := lockextra(false)
847 // Set needextram when we've just emptied the list,
848 // so that the eventual call into cgocallbackg will
849 // allocate a new m for the extra list. We delay the
850 // allocation until then so that it can be done
851 // after exitsyscall makes sure it is okay to be
852 // running at all (that is, there's no garbage collection
853 // running right now).
854 mp.needextram = mp.schedlink == nil
855 unlockextra(mp.schedlink)
857 // Install g (= m->g0) and set the stack bounds
858 // to match the current stack. We don't actually know
859 // how big the stack is, like we don't know how big any
860 // scheduling stack is, but we assume there's at least 32 kB,
861 // which is more than enough for us.
864 _g_.stack.hi = uintptr(noescape(unsafe.Pointer(&x))) + 1024
865 _g_.stack.lo = uintptr(noescape(unsafe.Pointer(&x))) - 32*1024
866 _g_.stackguard0 = _g_.stack.lo + _StackGuard
868 // Initialize this thread to use the m.
873 // newextram allocates an m and puts it on the extra list.
874 // It is called with a working local m, so that it can do things
875 // like call schedlock and allocate.
877 // Create extra goroutine locked to extra m.
878 // The goroutine is the context in which the cgo callback will run.
879 // The sched.pc will never be returned to, but setting it to
880 // goexit makes clear to the traceback routines where
881 // the goroutine stack ends.
884 gp.sched.pc = funcPC(goexit) + _PCQuantum
885 gp.sched.sp = gp.stack.hi
886 gp.sched.sp -= 4 * regSize // extra space in case of reads slightly beyond frame
889 gp.syscallpc = gp.sched.pc
890 gp.syscallsp = gp.sched.sp
891 // malg returns status as Gidle, change to Gsyscall before adding to allg
892 // where GC will see it.
893 casgstatus(gp, _Gidle, _Gsyscall)
896 mp.locked = _LockInternal
899 gp.goid = int64(xadd64(&sched.goidgen, 1))
901 gp.racectx = racegostart(funcPC(newextram))
903 // put on allg for garbage collector
906 // Add m to the extra list.
907 mnext := lockextra(true)
912 // dropm is called when a cgo callback has called needm but is now
913 // done with the callback and returning back into the non-Go thread.
914 // It puts the current m back onto the extra list.
916 // The main expense here is the call to signalstack to release the
917 // m's signal stack, and then the call to needm on the next callback
918 // from this thread. It is tempting to try to save the m for next time,
919 // which would eliminate both these costs, but there might not be
920 // a next time: the current thread (which Go does not control) might exit.
921 // If we saved the m for that thread, there would be an m leak each time
922 // such a thread exited. Instead, we acquire and release an m on each
923 // call. These should typically not be scheduling operations, just a few
924 // atomics, so the cost should be small.
926 // TODO(rsc): An alternative would be to allocate a dummy pthread per-thread
927 // variable using pthread_key_create. Unlike the pthread keys we already use
928 // on OS X, this dummy key would never be read by Go code. It would exist
929 // only so that we could register at thread-exit-time destructor.
930 // That destructor would put the m back onto the extra list.
931 // This is purely a performance optimization. The current version,
932 // in which dropm happens on each cgo call, is still correct too.
933 // We may have to keep the current version on systems with cgo
934 // but without pthreads, like Windows.
936 // Undo whatever initialization minit did during needm.
939 // Clear m and g, and return m to the extra list.
940 // After the call to setmg we can only call nosplit functions.
944 mnext := lockextra(true)
951 // lockextra locks the extra list and returns the list head.
952 // The caller must unlock the list by storing a new list head
953 // to extram. If nilokay is true, then lockextra will
954 // return a nil list head if that's what it finds. If nilokay is false,
955 // lockextra will keep waiting until the list head is no longer nil.
957 func lockextra(nilokay bool) *m {
961 old := atomicloaduintptr(&extram)
967 if old == 0 && !nilokay {
971 if casuintptr(&extram, old, locked) {
972 return (*m)(unsafe.Pointer(old))
981 func unlockextra(mp *m) {
982 atomicstoreuintptr(&extram, uintptr(unsafe.Pointer(mp)))
985 // Create a new m. It will start off with a call to fn, or else the scheduler.
986 func _newm(fn func(), _p_ *p) {
989 mp.mstartfn = *(*unsafe.Pointer)(unsafe.Pointer(&fn))
992 var ts cgothreadstart
993 if _cgo_thread_start == nil {
994 gothrow("_cgo_thread_start missing")
997 ts.tls = (*uint64)(unsafe.Pointer(&mp.tls[0]))
998 ts.fn = unsafe.Pointer(funcPC(mstart))
999 asmcgocall(_cgo_thread_start, unsafe.Pointer(&ts))
1002 newosproc(mp, unsafe.Pointer(mp.g0.stack.hi))
1005 // Stops execution of the current m until new work is available.
1006 // Returns with acquired P.
1010 if _g_.m.locks != 0 {
1011 gothrow("stopm holding locks")
1014 gothrow("stopm holding p")
1017 _g_.m.spinning = false
1018 xadd(&sched.nmspinning, -1)
1025 notesleep(&_g_.m.park)
1026 noteclear(&_g_.m.park)
1027 if _g_.m.helpgc != 0 {
1033 acquirep(_g_.m.nextp)
1038 getg().m.spinning = true
1041 // Schedules some M to run the p (creates an M if necessary).
1042 // If p==nil, tries to get an idle P, if no idle P's does nothing.
1043 func startm(_p_ *p, spinning bool) {
1050 xadd(&sched.nmspinning, -1)
1066 gothrow("startm: m is spinning")
1068 if mp.nextp != nil {
1069 gothrow("startm: m has p")
1071 mp.spinning = spinning
1073 notewakeup(&mp.park)
1076 // Hands off P from syscall or locked M.
1077 func handoffp(_p_ *p) {
1078 // if it has local work, start it straight away
1079 if _p_.runqhead != _p_.runqtail || sched.runqsize != 0 {
1083 // no local work, check that there are no spinning/idle M's,
1084 // otherwise our help is not required
1085 if atomicload(&sched.nmspinning)+atomicload(&sched.npidle) == 0 && cas(&sched.nmspinning, 0, 1) { // TODO: fast atomic
1090 if sched.gcwaiting != 0 {
1091 _p_.status = _Pgcstop
1093 if sched.stopwait == 0 {
1094 notewakeup(&sched.stopnote)
1099 if sched.runqsize != 0 {
1104 // If this is the last running P and nobody is polling network,
1105 // need to wakeup another M to poll network.
1106 if sched.npidle == uint32(gomaxprocs-1) && atomicload64(&sched.lastpoll) != 0 {
1115 // Tries to add one more P to execute G's.
1116 // Called when a G is made runnable (newproc, ready).
1118 // be conservative about spinning threads
1119 if !cas(&sched.nmspinning, 0, 1) {
1125 // Stops execution of the current m that is locked to a g until the g is runnable again.
1126 // Returns with acquired P.
1127 func stoplockedm() {
1130 if _g_.m.lockedg == nil || _g_.m.lockedg.lockedm != _g_.m {
1131 gothrow("stoplockedm: inconsistent locking")
1134 // Schedule another M to run this p.
1139 // Wait until another thread schedules lockedg again.
1140 notesleep(&_g_.m.park)
1141 noteclear(&_g_.m.park)
1142 status := readgstatus(_g_.m.lockedg)
1143 if status&^_Gscan != _Grunnable {
1144 print("runtime:stoplockedm: g is not Grunnable or Gscanrunnable\n")
1146 gothrow("stoplockedm: not runnable")
1148 acquirep(_g_.m.nextp)
1152 // Schedules the locked m to run the locked gp.
1153 func startlockedm(gp *g) {
1158 gothrow("startlockedm: locked to me")
1160 if mp.nextp != nil {
1161 gothrow("startlockedm: m has p")
1163 // directly handoff current P to the locked m
1167 notewakeup(&mp.park)
1171 // Stops the current m for stoptheworld.
1172 // Returns when the world is restarted.
1176 if sched.gcwaiting == 0 {
1177 gothrow("gcstopm: not waiting for gc")
1180 _g_.m.spinning = false
1181 xadd(&sched.nmspinning, -1)
1185 _p_.status = _Pgcstop
1187 if sched.stopwait == 0 {
1188 notewakeup(&sched.stopnote)
1194 // Schedules gp to run on the current M.
1196 func execute(gp *g) {
1199 casgstatus(gp, _Grunnable, _Grunning)
1202 gp.stackguard0 = gp.stack.lo + _StackGuard
1207 // Check whether the profiler needs to be turned on or off.
1208 hz := sched.profilehz
1209 if _g_.m.profilehz != hz {
1210 resetcpuprofiler(hz)
1216 // Finds a runnable goroutine to execute.
1217 // Tries to steal from other P's, get g from global queue, poll network.
1218 func findrunnable() *g {
1222 if sched.gcwaiting != 0 {
1226 if fingwait && fingwake {
1227 if gp := wakefing(); gp != nil {
1233 if gp := runqget(_g_.m.p); gp != nil {
1238 if sched.runqsize != 0 {
1240 gp := globrunqget(_g_.m.p, 0)
1247 // poll network - returns list of goroutines
1248 if gp := netpoll(false); gp != nil { // non-blocking
1249 injectglist(gp.schedlink)
1250 casgstatus(gp, _Gwaiting, _Grunnable)
1254 // If number of spinning M's >= number of busy P's, block.
1255 // This is necessary to prevent excessive CPU consumption
1256 // when GOMAXPROCS>>1 but the program parallelism is low.
1257 if !_g_.m.spinning && 2*atomicload(&sched.nmspinning) >= uint32(gomaxprocs)-atomicload(&sched.npidle) { // TODO: fast atomic
1260 if !_g_.m.spinning {
1261 _g_.m.spinning = true
1262 xadd(&sched.nmspinning, 1)
1264 // random steal from other P's
1265 for i := 0; i < int(2*gomaxprocs); i++ {
1266 if sched.gcwaiting != 0 {
1269 _p_ := allp[fastrand1()%uint32(gomaxprocs)]
1274 gp = runqsteal(_g_.m.p, _p_)
1282 // return P and block
1284 if sched.gcwaiting != 0 {
1288 if sched.runqsize != 0 {
1289 gp := globrunqget(_g_.m.p, 0)
1297 _g_.m.spinning = false
1298 xadd(&sched.nmspinning, -1)
1301 // check all runqueues once again
1302 for i := 0; i < int(gomaxprocs); i++ {
1304 if _p_ != nil && _p_.runqhead != _p_.runqtail {
1317 if xchg64(&sched.lastpoll, 0) != 0 {
1319 gothrow("findrunnable: netpoll with p")
1322 gothrow("findrunnable: netpoll with spinning")
1324 gp := netpoll(true) // block until new work is available
1325 atomicstore64(&sched.lastpoll, uint64(nanotime()))
1332 injectglist(gp.schedlink)
1333 casgstatus(gp, _Gwaiting, _Grunnable)
1343 func resetspinning() {
1346 var nmspinning uint32
1348 _g_.m.spinning = false
1349 nmspinning = xadd(&sched.nmspinning, -1)
1351 gothrow("findrunnable: negative nmspinning")
1354 nmspinning = atomicload(&sched.nmspinning)
1357 // M wakeup policy is deliberately somewhat conservative (see nmspinning handling),
1358 // so see if we need to wakeup another P here.
1359 if nmspinning == 0 && atomicload(&sched.npidle) > 0 {
1364 // Injects the list of runnable G's into the scheduler.
1365 // Can run concurrently with GC.
1366 func injectglist(glist *g) {
1372 for n = 0; glist != nil; n++ {
1374 glist = gp.schedlink
1375 casgstatus(gp, _Gwaiting, _Grunnable)
1379 for ; n != 0 && sched.npidle != 0; n-- {
1384 // One round of scheduler: find a runnable goroutine and execute it.
1389 if _g_.m.locks != 0 {
1390 gothrow("schedule: holding locks")
1393 if _g_.m.lockedg != nil {
1395 execute(_g_.m.lockedg) // Never returns.
1399 if sched.gcwaiting != 0 {
1405 // Check the global runnable queue once in a while to ensure fairness.
1406 // Otherwise two goroutines can completely occupy the local runqueue
1407 // by constantly respawning each other.
1408 tick := _g_.m.p.schedtick
1409 // This is a fancy way to say tick%61==0,
1410 // it uses 2 MUL instructions instead of a single DIV and so is faster on modern processors.
1411 if uint64(tick)-((uint64(tick)*0x4325c53f)>>36)*61 == 0 && sched.runqsize > 0 {
1413 gp = globrunqget(_g_.m.p, 1)
1420 gp = runqget(_g_.m.p)
1421 if gp != nil && _g_.m.spinning {
1422 gothrow("schedule: spinning with local work")
1426 gp = findrunnable() // blocks until work is available
1430 if gp.lockedm != nil {
1431 // Hands off own p to the locked m,
1432 // then blocks waiting for a new p.
1440 // dropg removes the association between m and the current goroutine m->curg (gp for short).
1441 // Typically a caller sets gp's status away from Grunning and then
1442 // immediately calls dropg to finish the job. The caller is also responsible
1443 // for arranging that gp will be restarted using ready at an
1444 // appropriate time. After calling dropg and arranging for gp to be
1445 // readied later, the caller can do other work but eventually should
1446 // call schedule to restart the scheduling of goroutines on this m.
1450 if _g_.m.lockedg == nil {
1456 // Puts the current goroutine into a waiting state and calls unlockf.
1457 // If unlockf returns false, the goroutine is resumed.
1458 func park(unlockf func(*g, unsafe.Pointer) bool, lock unsafe.Pointer, reason string) {
1461 _g_.m.waitlock = lock
1462 _g_.m.waitunlockf = *(*unsafe.Pointer)(unsafe.Pointer(&unlockf))
1463 _g_.waitreason = reason
1467 func parkunlock_c(gp *g, lock unsafe.Pointer) bool {
1468 unlock((*mutex)(lock))
1472 // Puts the current goroutine into a waiting state and unlocks the lock.
1473 // The goroutine can be made runnable again by calling ready(gp).
1474 func parkunlock(lock *mutex, reason string) {
1475 park(parkunlock_c, unsafe.Pointer(lock), reason)
1478 // park continuation on g0.
1479 func park_m(gp *g) {
1482 casgstatus(gp, _Grunning, _Gwaiting)
1485 if _g_.m.waitunlockf != nil {
1486 fn := *(*func(*g, unsafe.Pointer) bool)(unsafe.Pointer(&_g_.m.waitunlockf))
1487 ok := fn(gp, _g_.m.waitlock)
1488 _g_.m.waitunlockf = nil
1489 _g_.m.waitlock = nil
1491 casgstatus(gp, _Gwaiting, _Grunnable)
1492 execute(gp) // Schedule it back, never returns.
1498 // Gosched continuation on g0.
1499 func gosched_m(gp *g) {
1500 status := readgstatus(gp)
1501 if status&^_Gscan != _Grunning {
1503 gothrow("bad g status")
1505 casgstatus(gp, _Grunning, _Grunnable)
1514 // Finishes execution of the current goroutine.
1515 // Must be NOSPLIT because it is called from Go. (TODO - probably not anymore)
1524 // goexit continuation on g0.
1525 func goexit0(gp *g) {
1528 casgstatus(gp, _Grunning, _Gdead)
1532 gp.paniconfault = false
1533 gp._defer = nil // should be true already but just in case.
1534 gp._panic = nil // non-nil for Goexit during panic. points at stack-allocated data.
1541 if _g_.m.locked&^_LockExternal != 0 {
1542 print("invalid m->locked = ", _g_.m.locked, "\n")
1543 gothrow("internal lockOSThread error")
1551 func save(pc, sp uintptr) {
1558 _g_.sched.ctxt = nil
1562 // The goroutine g is about to enter a system call.
1563 // Record that it's not using the cpu anymore.
1564 // This is called only from the go syscall library and cgocall,
1565 // not from the low-level system calls used by the
1567 // Entersyscall cannot split the stack: the gosave must
1568 // make g->sched refer to the caller's stack segment, because
1569 // entersyscall is going to return immediately after.
1571 // Nothing entersyscall calls can split the stack either.
1572 // We cannot safely move the stack during an active call to syscall,
1573 // because we do not know which of the uintptr arguments are
1574 // really pointers (back into the stack).
1575 // In practice, this means that we make the fast path run through
1576 // entersyscall doing no-split things, and the slow path has to use systemstack
1577 // to run bigger things on the system stack.
1579 // reentersyscall is the entry point used by cgo callbacks, where explicitly
1580 // saved SP and PC are restored. This is needed when exitsyscall will be called
1581 // from a function further up in the call stack than the parent, as g->syscallsp
1582 // must always point to a valid stack frame. entersyscall below is the normal
1583 // entry point for syscalls, which obtains the SP and PC from the caller.
1585 func reentersyscall(pc, sp uintptr) {
1588 // Disable preemption because during this function g is in Gsyscall status,
1589 // but can have inconsistent g->sched, do not let GC observe it.
1592 // Entersyscall must not call any function that might split/grow the stack.
1593 // (See details in comment above.)
1594 // Catch calls that might, by replacing the stack guard with something that
1595 // will trip any stack check and leaving a flag to tell newstack to die.
1596 _g_.stackguard0 = stackPreempt
1597 _g_.throwsplit = true
1599 // Leave SP around for GC and traceback.
1603 casgstatus(_g_, _Grunning, _Gsyscall)
1604 if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp {
1605 systemstack(entersyscall_bad)
1608 if atomicload(&sched.sysmonwait) != 0 { // TODO: fast atomic
1609 systemstack(entersyscall_sysmon)
1615 atomicstore(&_g_.m.p.status, _Psyscall)
1616 if sched.gcwaiting != 0 {
1617 systemstack(entersyscall_gcwait)
1621 // Goroutines must not split stacks in Gsyscall status (it would corrupt g->sched).
1622 // We set _StackGuard to StackPreempt so that first split stack check calls morestack.
1623 // Morestack detects this case and throws.
1624 _g_.stackguard0 = stackPreempt
1628 // Standard syscall entry used by the go syscall library and normal cgo calls.
1630 func entersyscall(dummy int32) {
1631 reentersyscall(getcallerpc(unsafe.Pointer(&dummy)), getcallersp(unsafe.Pointer(&dummy)))
1634 func entersyscall_bad() {
1637 print("entersyscall inconsistent ", hex(gp.syscallsp), " [", hex(gp.stack.lo), ",", hex(gp.stack.hi), "]\n")
1638 gothrow("entersyscall")
1641 func entersyscall_sysmon() {
1643 if atomicload(&sched.sysmonwait) != 0 {
1644 atomicstore(&sched.sysmonwait, 0)
1645 notewakeup(&sched.sysmonnote)
1650 func entersyscall_gcwait() {
1654 if sched.stopwait > 0 && cas(&_g_.m.p.status, _Psyscall, _Pgcstop) {
1655 if sched.stopwait--; sched.stopwait == 0 {
1656 notewakeup(&sched.stopnote)
1662 // The same as entersyscall(), but with a hint that the syscall is blocking.
1664 func entersyscallblock(dummy int32) {
1667 _g_.m.locks++ // see comment in entersyscall
1668 _g_.throwsplit = true
1669 _g_.stackguard0 = stackPreempt // see comment in entersyscall
1671 // Leave SP around for GC and traceback.
1672 save(getcallerpc(unsafe.Pointer(&dummy)), getcallersp(unsafe.Pointer(&dummy)))
1673 _g_.syscallsp = _g_.sched.sp
1674 _g_.syscallpc = _g_.sched.pc
1675 casgstatus(_g_, _Grunning, _Gsyscall)
1676 if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp {
1677 systemstack(entersyscall_bad)
1680 systemstack(entersyscallblock_handoff)
1682 // Resave for traceback during blocked call.
1683 save(getcallerpc(unsafe.Pointer(&dummy)), getcallersp(unsafe.Pointer(&dummy)))
1688 func entersyscallblock_handoff() {
1689 handoffp(releasep())
1692 // The goroutine g exited its system call.
1693 // Arrange for it to run on a cpu again.
1694 // This is called only from the go syscall library, not
1695 // from the low-level system calls used by the
1697 func exitsyscall(dummy int32) {
1700 _g_.m.locks++ // see comment in entersyscall
1701 if getcallersp(unsafe.Pointer(&dummy)) > _g_.syscallsp {
1702 gothrow("exitsyscall: syscall frame is no longer valid")
1706 if exitsyscallfast() {
1707 if _g_.m.mcache == nil {
1708 gothrow("lost mcache")
1710 // There's a cpu for us, so we can run.
1711 _g_.m.p.syscalltick++
1712 // We need to cas the status and scan before resuming...
1713 casgstatus(_g_, _Gsyscall, _Grunning)
1715 // Garbage collector isn't running (since we are),
1716 // so okay to clear syscallsp.
1720 // restore the preemption request in case we've cleared it in newstack
1721 _g_.stackguard0 = stackPreempt
1723 // otherwise restore the real _StackGuard, we've spoiled it in entersyscall/entersyscallblock
1724 _g_.stackguard0 = _g_.stack.lo + _StackGuard
1726 _g_.throwsplit = false
1732 // Call the scheduler.
1735 if _g_.m.mcache == nil {
1736 gothrow("lost mcache")
1739 // Scheduler returned, so we're allowed to run now.
1740 // Delete the syscallsp information that we left for
1741 // the garbage collector during the system call.
1742 // Must wait until now because until gosched returns
1743 // we don't know for sure that the garbage collector
1746 _g_.m.p.syscalltick++
1747 _g_.throwsplit = false
1751 func exitsyscallfast() bool {
1754 // Freezetheworld sets stopwait but does not retake P's.
1755 if sched.stopwait != 0 {
1760 // Try to re-acquire the last P.
1761 if _g_.m.p != nil && _g_.m.p.status == _Psyscall && cas(&_g_.m.p.status, _Psyscall, _Prunning) {
1762 // There's a cpu for us, so we can run.
1763 _g_.m.mcache = _g_.m.p.mcache
1768 // Try to get any other idle P.
1770 if sched.pidle != nil {
1772 systemstack(func() {
1773 ok = exitsyscallfast_pidle()
1782 func exitsyscallfast_pidle() bool {
1785 if _p_ != nil && atomicload(&sched.sysmonwait) != 0 {
1786 atomicstore(&sched.sysmonwait, 0)
1787 notewakeup(&sched.sysmonnote)
1797 // exitsyscall slow path on g0.
1798 // Failed to acquire P, enqueue gp as runnable.
1799 func exitsyscall0(gp *g) {
1802 casgstatus(gp, _Gsyscall, _Grunnable)
1808 } else if atomicload(&sched.sysmonwait) != 0 {
1809 atomicstore(&sched.sysmonwait, 0)
1810 notewakeup(&sched.sysmonnote)
1815 execute(gp) // Never returns.
1817 if _g_.m.lockedg != nil {
1818 // Wait until another thread schedules gp and so m again.
1820 execute(gp) // Never returns.
1823 schedule() // Never returns.
1829 // Fork can hang if preempted with signals frequently enough (see issue 5517).
1830 // Ensure that we stay on the same M where we disable profiling.
1832 if gp.m.profilehz != 0 {
1836 // This function is called before fork in syscall package.
1837 // Code between fork and exec must not allocate memory nor even try to grow stack.
1838 // Here we spoil g->_StackGuard to reliably detect any attempts to grow stack.
1839 // runtime_AfterFork will undo this in parent process, but not in child.
1840 gp.stackguard0 = stackFork
1843 // Called from syscall package before fork.
1845 func syscall_BeforeFork() {
1846 systemstack(beforefork)
1852 // See the comment in beforefork.
1853 gp.stackguard0 = gp.stack.lo + _StackGuard
1855 hz := sched.profilehz
1857 resetcpuprofiler(hz)
1862 // Called from syscall package after fork in parent.
1864 func syscall_AfterFork() {
1865 systemstack(afterfork)
1868 // Allocate a new g, with a stack big enough for stacksize bytes.
1869 func malg(stacksize int32) *g {
1872 stacksize = round2(_StackSystem + stacksize)
1873 systemstack(func() {
1874 newg.stack = stackalloc(uint32(stacksize))
1876 newg.stackguard0 = newg.stack.lo + _StackGuard
1877 newg.stackguard1 = ^uintptr(0)
1882 // Create a new g running fn with siz bytes of arguments.
1883 // Put it on the queue of g's waiting to run.
1884 // The compiler turns a go statement into a call to this.
1885 // Cannot split the stack because it assumes that the arguments
1886 // are available sequentially after &fn; they would not be
1887 // copied if a stack split occurred.
1889 func newproc(siz int32, fn *funcval) {
1890 argp := add(unsafe.Pointer(&fn), ptrSize)
1891 if hasLinkRegister {
1892 argp = add(argp, ptrSize) // skip caller's saved LR
1895 pc := getcallerpc(unsafe.Pointer(&siz))
1896 systemstack(func() {
1897 newproc1(fn, (*uint8)(argp), siz, 0, pc)
1901 // Create a new g running fn with narg bytes of arguments starting
1902 // at argp and returning nret bytes of results. callerpc is the
1903 // address of the go statement that created this. The new g is put
1904 // on the queue of g's waiting to run.
1905 func newproc1(fn *funcval, argp *uint8, narg int32, nret int32, callerpc uintptr) *g {
1909 _g_.m.throwing = -1 // do not dump full stacks
1910 gothrow("go of nil func value")
1912 _g_.m.locks++ // disable preemption because it can be holding p in a local var
1914 siz = (siz + 7) &^ 7
1916 // We could allocate a larger initial stack if necessary.
1917 // Not worth it: this is almost always an error.
1918 // 4*sizeof(uintreg): extra space added below
1919 // sizeof(uintreg): caller's LR (arm) or return address (x86, in gostartcall).
1920 if siz >= _StackMin-4*regSize-regSize {
1921 gothrow("newproc: function arguments too large for new goroutine")
1927 newg = malg(_StackMin)
1928 casgstatus(newg, _Gidle, _Gdead)
1929 allgadd(newg) // publishes with a g->status of Gdead so GC scanner doesn't look at uninitialized stack.
1931 if newg.stack.hi == 0 {
1932 gothrow("newproc1: newg missing stack")
1935 if readgstatus(newg) != _Gdead {
1936 gothrow("newproc1: new g is not Gdead")
1940 sp -= 4 * regSize // extra space in case of reads slightly beyond frame
1942 memmove(unsafe.Pointer(sp), unsafe.Pointer(argp), uintptr(narg))
1943 if hasLinkRegister {
1946 *(*unsafe.Pointer)(unsafe.Pointer(sp)) = nil
1949 memclr(unsafe.Pointer(&newg.sched), unsafe.Sizeof(newg.sched))
1951 newg.sched.pc = funcPC(goexit) + _PCQuantum // +PCQuantum so that previous instruction is in same function
1953 gostartcallfn(&newg.sched, fn)
1954 newg.gopc = callerpc
1955 casgstatus(newg, _Gdead, _Grunnable)
1957 if _p_.goidcache == _p_.goidcacheend {
1958 // Sched.goidgen is the last allocated id,
1959 // this batch must be [sched.goidgen+1, sched.goidgen+GoidCacheBatch].
1960 // At startup sched.goidgen=0, so main goroutine receives goid=1.
1961 _p_.goidcache = xadd64(&sched.goidgen, _GoidCacheBatch)
1962 _p_.goidcache -= _GoidCacheBatch - 1
1963 _p_.goidcacheend = _p_.goidcache + _GoidCacheBatch
1965 newg.goid = int64(_p_.goidcache)
1968 newg.racectx = racegostart(callerpc)
1972 if atomicload(&sched.npidle) != 0 && atomicload(&sched.nmspinning) == 0 && unsafe.Pointer(fn.fn) != unsafe.Pointer(funcPC(main)) { // TODO: fast atomic
1976 if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack
1977 _g_.stackguard0 = stackPreempt
1982 // Put on gfree list.
1983 // If local list is too long, transfer a batch to the global list.
1984 func gfput(_p_ *p, gp *g) {
1985 if readgstatus(gp) != _Gdead {
1986 gothrow("gfput: bad status (not Gdead)")
1989 stksize := gp.stack.hi - gp.stack.lo
1991 if stksize != _FixedStack {
1992 // non-standard stack size - free it.
1999 gp.schedlink = _p_.gfree
2002 if _p_.gfreecnt >= 64 {
2004 for _p_.gfreecnt >= 32 {
2007 _p_.gfree = gp.schedlink
2008 gp.schedlink = sched.gfree
2012 unlock(&sched.gflock)
2016 // Get from gfree list.
2017 // If local list is empty, grab a batch from global list.
2018 func gfget(_p_ *p) *g {
2021 if gp == nil && sched.gfree != nil {
2023 for _p_.gfreecnt < 32 && sched.gfree != nil {
2026 sched.gfree = gp.schedlink
2028 gp.schedlink = _p_.gfree
2031 unlock(&sched.gflock)
2035 _p_.gfree = gp.schedlink
2037 if gp.stack.lo == 0 {
2038 // Stack was deallocated in gfput. Allocate a new one.
2039 systemstack(func() {
2040 gp.stack = stackalloc(_FixedStack)
2042 gp.stackguard0 = gp.stack.lo + _StackGuard
2045 racemalloc(unsafe.Pointer(gp.stack.lo), gp.stack.hi-gp.stack.lo)
2052 // Purge all cached G's from gfree list to the global list.
2053 func gfpurge(_p_ *p) {
2055 for _p_.gfreecnt != 0 {
2058 _p_.gfree = gp.schedlink
2059 gp.schedlink = sched.gfree
2063 unlock(&sched.gflock)
2066 // Breakpoint executes a breakpoint trap.
2071 // dolockOSThread is called by LockOSThread and lockOSThread below
2072 // after they modify m.locked. Do not allow preemption during this call,
2073 // or else the m might be different in this function than in the caller.
2075 func dolockOSThread() {
2083 // LockOSThread wires the calling goroutine to its current operating system thread.
2084 // Until the calling goroutine exits or calls UnlockOSThread, it will always
2085 // execute in that thread, and no other goroutine can.
2086 func LockOSThread() {
2087 getg().m.locked |= _LockExternal
2092 func lockOSThread() {
2093 getg().m.locked += _LockInternal
2097 // dounlockOSThread is called by UnlockOSThread and unlockOSThread below
2098 // after they update m->locked. Do not allow preemption during this call,
2099 // or else the m might be in different in this function than in the caller.
2101 func dounlockOSThread() {
2103 if _g_.m.locked != 0 {
2112 // UnlockOSThread unwires the calling goroutine from its fixed operating system thread.
2113 // If the calling goroutine has not called LockOSThread, UnlockOSThread is a no-op.
2114 func UnlockOSThread() {
2115 getg().m.locked &^= _LockExternal
2120 func unlockOSThread() {
2122 if _g_.m.locked < _LockInternal {
2123 systemstack(badunlockosthread)
2125 _g_.m.locked -= _LockInternal
2129 func badunlockosthread() {
2130 gothrow("runtime: internal error: misuse of lockOSThread/unlockOSThread")
2133 func gcount() int32 {
2134 n := int32(allglen) - sched.ngfree
2143 // All these variables can be changed concurrently, so the result can be inconsistent.
2144 // But at least the current goroutine is running.
2151 func mcount() int32 {
2160 func _System() { _System() }
2161 func _ExternalCode() { _ExternalCode() }
2162 func _GC() { _GC() }
2166 // Called if we receive a SIGPROF signal.
2167 func sigprof(pc *uint8, sp *uint8, lr *uint8, gp *g, mp *m) {
2170 var stk [100]uintptr
2176 // Profiling runs concurrently with GC, so it must not allocate.
2179 // Define that a "user g" is a user-created goroutine, and a "system g"
2180 // is one that is m->g0 or m->gsignal. We've only made sure that we
2181 // can unwind user g's, so exclude the system g's.
2183 // It is not quite as easy as testing gp == m->curg (the current user g)
2184 // because we might be interrupted for profiling halfway through a
2185 // goroutine switch. The switch involves updating three (or four) values:
2186 // g, PC, SP, and (on arm) LR. The PC must be the last to be updated,
2187 // because once it gets updated the new g is running.
2189 // When switching from a user g to a system g, LR is not considered live,
2190 // so the update only affects g, SP, and PC. Since PC must be last, there
2191 // the possible partial transitions in ordinary execution are (1) g alone is updated,
2192 // (2) both g and SP are updated, and (3) SP alone is updated.
2193 // If g is updated, we'll see a system g and not look closer.
2194 // If SP alone is updated, we can detect the partial transition by checking
2195 // whether the SP is within g's stack bounds. (We could also require that SP
2196 // be changed only after g, but the stack bounds check is needed by other
2197 // cases, so there is no need to impose an additional requirement.)
2199 // There is one exceptional transition to a system g, not in ordinary execution.
2200 // When a signal arrives, the operating system starts the signal handler running
2201 // with an updated PC and SP. The g is updated last, at the beginning of the
2202 // handler. There are two reasons this is okay. First, until g is updated the
2203 // g and SP do not match, so the stack bounds check detects the partial transition.
2204 // Second, signal handlers currently run with signals disabled, so a profiling
2205 // signal cannot arrive during the handler.
2207 // When switching from a system g to a user g, there are three possibilities.
2209 // First, it may be that the g switch has no PC update, because the SP
2210 // either corresponds to a user g throughout (as in asmcgocall)
2211 // or because it has been arranged to look like a user g frame
2212 // (as in cgocallback_gofunc). In this case, since the entire
2213 // transition is a g+SP update, a partial transition updating just one of
2214 // those will be detected by the stack bounds check.
2216 // Second, when returning from a signal handler, the PC and SP updates
2217 // are performed by the operating system in an atomic update, so the g
2218 // update must be done before them. The stack bounds check detects
2219 // the partial transition here, and (again) signal handlers run with signals
2220 // disabled, so a profiling signal cannot arrive then anyway.
2222 // Third, the common case: it may be that the switch updates g, SP, and PC
2223 // separately, as in gogo.
2225 // Because gogo is the only instance, we check whether the PC lies
2226 // within that function, and if so, not ask for a traceback. This approach
2227 // requires knowing the size of the gogo function, which we
2228 // record in arch_*.h and check in runtime_test.go.
2230 // There is another apparently viable approach, recorded here in case
2231 // the "PC within gogo" check turns out not to be usable.
2232 // It would be possible to delay the update of either g or SP until immediately
2233 // before the PC update instruction. Then, because of the stack bounds check,
2234 // the only problematic interrupt point is just before that PC update instruction,
2235 // and the sigprof handler can detect that instruction and simulate stepping past
2236 // it in order to reach a consistent state. On ARM, the update of g must be made
2237 // in two places (in R10 and also in a TLS slot), so the delayed update would
2238 // need to be the SP update. The sigprof handler must read the instruction at
2239 // the current PC and if it was the known instruction (for example, JMP BX or
2240 // MOV R2, PC), use that other register in place of the PC value.
2241 // The biggest drawback to this solution is that it requires that we can tell
2242 // whether it's safe to read from the memory pointed at by PC.
2243 // In a correct program, we can test PC == nil and otherwise read,
2244 // but if a profiling signal happens at the instant that a program executes
2245 // a bad jump (before the program manages to handle the resulting fault)
2246 // the profiling handler could fault trying to read nonexistent memory.
2248 // To recap, there are no constraints on the assembly being used for the
2249 // transition. We simply require that g and SP match and that the PC is not
2252 usp := uintptr(unsafe.Pointer(sp))
2253 gogo := funcPC(gogo)
2254 if gp == nil || gp != mp.curg ||
2255 usp < gp.stack.lo || gp.stack.hi < usp ||
2256 (gogo <= uintptr(unsafe.Pointer(pc)) && uintptr(unsafe.Pointer(pc)) < gogo+_RuntimeGogoBytes) {
2262 n = int32(gentraceback(uintptr(unsafe.Pointer(pc)), uintptr(unsafe.Pointer(sp)), uintptr(unsafe.Pointer(lr)), gp, 0, &stk[0], len(stk), nil, nil, _TraceTrap))
2264 if !traceback || n <= 0 {
2265 // Normal traceback is impossible or has failed.
2266 // See if it falls into several common cases.
2268 if mp.ncgo > 0 && mp.curg != nil && mp.curg.syscallpc != 0 && mp.curg.syscallsp != 0 {
2269 // Cgo, we can't unwind and symbolize arbitrary C code,
2270 // so instead collect Go stack that leads to the cgo call.
2271 // This is especially important on windows, since all syscalls are cgo calls.
2272 n = int32(gentraceback(mp.curg.syscallpc, mp.curg.syscallsp, 0, mp.curg, 0, &stk[0], len(stk), nil, nil, 0))
2274 if GOOS == "windows" && n == 0 && mp.libcallg != nil && mp.libcallpc != 0 && mp.libcallsp != 0 {
2275 // Libcall, i.e. runtime syscall on windows.
2276 // Collect Go stack that leads to the call.
2277 n = int32(gentraceback(mp.libcallpc, mp.libcallsp, 0, mp.libcallg, 0, &stk[0], len(stk), nil, nil, 0))
2280 // If all of the above has failed, account it against abstract "System" or "GC".
2282 // "ExternalCode" is better than "etext".
2283 if uintptr(unsafe.Pointer(pc)) > uintptr(unsafe.Pointer(&etext)) {
2284 pc = (*uint8)(unsafe.Pointer(uintptr(funcPC(_ExternalCode) + _PCQuantum)))
2286 stk[0] = uintptr(unsafe.Pointer(pc))
2287 if mp.gcing != 0 || mp.helpgc != 0 {
2288 stk[1] = funcPC(_GC) + _PCQuantum
2290 stk[1] = funcPC(_System) + _PCQuantum
2296 // Simple cas-lock to coordinate with setcpuprofilerate.
2297 for !cas(&prof.lock, 0, 1) {
2301 cpuproftick(&stk[0], n)
2303 atomicstore(&prof.lock, 0)
2308 // Arrange to call fn with a traceback hz times a second.
2309 func setcpuprofilerate_m(hz int32) {
2310 // Force sane arguments.
2315 // Disable preemption, otherwise we can be rescheduled to another thread
2316 // that has profiling enabled.
2320 // Stop profiler on this thread so that it is safe to lock prof.
2321 // if a profiling signal came in while we had prof locked,
2322 // it would deadlock.
2325 for !cas(&prof.lock, 0, 1) {
2329 atomicstore(&prof.lock, 0)
2332 sched.profilehz = hz
2336 resetcpuprofiler(hz)
2342 // Change number of processors. The world is stopped, sched is locked.
2343 func procresize(new int32) {
2345 if old < 0 || old > _MaxGomaxprocs || new <= 0 || new > _MaxGomaxprocs {
2346 gothrow("procresize: invalid arg")
2349 // initialize new P's
2350 for i := int32(0); i < new; i++ {
2356 atomicstorep(unsafe.Pointer(&allp[i]), unsafe.Pointer(p))
2358 if p.mcache == nil {
2359 if old == 0 && i == 0 {
2360 if getg().m.mcache == nil {
2361 gothrow("missing mcache?")
2363 p.mcache = getg().m.mcache // bootstrap
2365 p.mcache = allocmcache()
2370 // redistribute runnable G's evenly
2371 // collect all runnable goroutines in global queue preserving FIFO order
2372 // FIFO order is required to ensure fairness even during frequent GCs
2373 // see http://golang.org/issue/7126
2377 for i := int32(0); i < old; i++ {
2379 if p.runqhead == p.runqtail {
2383 // pop from tail of local queue
2385 gp := p.runq[p.runqtail%uint32(len(p.runq))]
2386 // push onto head of global queue
2387 gp.schedlink = sched.runqhead
2389 if sched.runqtail == nil {
2396 // fill local queues with at most len(p.runq)/2 goroutines
2397 // start at 1 because current M already executes some G and will acquire allp[0] below,
2398 // so if we have a spare G we want to put it into allp[1].
2400 for i := int32(1); i < new*int32(len(_p_.runq))/2 && sched.runqsize > 0; i++ {
2401 gp := sched.runqhead
2402 sched.runqhead = gp.schedlink
2403 if sched.runqhead == nil {
2404 sched.runqtail = nil
2407 runqput(allp[i%new], gp)
2411 for i := new; i < old; i++ {
2413 freemcache(p.mcache)
2417 // can't free P itself because it can be referenced by an M in syscall
2430 for i := new - 1; i > 0; i-- {
2435 var int32p *int32 = &gomaxprocs // make compiler check that gomaxprocs is an int32
2436 atomicstore((*uint32)(unsafe.Pointer(int32p)), uint32(new))
2439 // Associate p and the current m.
2440 func acquirep(_p_ *p) {
2443 if _g_.m.p != nil || _g_.m.mcache != nil {
2444 gothrow("acquirep: already in go")
2446 if _p_.m != nil || _p_.status != _Pidle {
2451 print("acquirep: p->m=", _p_.m, "(", id, ") p->status=", _p_.status, "\n")
2452 gothrow("acquirep: invalid p state")
2454 _g_.m.mcache = _p_.mcache
2457 _p_.status = _Prunning
2460 // Disassociate p and the current m.
2461 func releasep() *p {
2464 if _g_.m.p == nil || _g_.m.mcache == nil {
2465 gothrow("releasep: invalid arg")
2468 if _p_.m != _g_.m || _p_.mcache != _g_.m.mcache || _p_.status != _Prunning {
2469 print("releasep: m=", _g_.m, " m->p=", _g_.m.p, " p->m=", _p_.m, " m->mcache=", _g_.m.mcache, " p->mcache=", _p_.mcache, " p->status=", _p_.status, "\n")
2470 gothrow("releasep: invalid p state")
2479 func incidlelocked(v int32) {
2481 sched.nmidlelocked += v
2488 // Check for deadlock situation.
2489 // The check is based on number of running M's, if 0 -> deadlock.
2491 // If we are dying because of a signal caught on an already idle thread,
2492 // freezetheworld will cause all running threads to block.
2493 // And runtime will essentially enter into deadlock state,
2494 // except that there is a thread that will call exit soon.
2500 run := sched.mcount - sched.nmidle - sched.nmidlelocked - 1
2505 print("runtime: checkdead: nmidle=", sched.nmidle, " nmidlelocked=", sched.nmidlelocked, " mcount=", sched.mcount, "\n")
2506 gothrow("checkdead: inconsistent counts")
2511 for i := 0; i < len(allgs); i++ {
2516 s := readgstatus(gp)
2517 switch s &^ _Gscan {
2524 print("runtime: checkdead: find g ", gp.goid, " in status ", s, "\n")
2525 gothrow("checkdead: runnable g")
2529 if grunning == 0 { // possible if main goroutine calls runtimeĀ·Goexit()
2530 gothrow("no goroutines (main called runtime.Goexit) - deadlock!")
2533 // Maybe jump time forward for playground.
2536 casgstatus(gp, _Gwaiting, _Grunnable)
2540 gothrow("checkdead: no p for timer")
2547 notewakeup(&mp.park)
2552 getg().m.throwing = -1 // do not dump full stacks
2553 gothrow("all goroutines are asleep - deadlock!")
2557 // If we go two minutes without a garbage collection, force one to run.
2558 forcegcperiod := int64(2 * 60 * 1e9)
2560 // If a heap span goes unused for 5 minutes after a garbage collection,
2561 // we hand it back to the operating system.
2562 scavengelimit := int64(5 * 60 * 1e9)
2564 if debug.scavenge > 0 {
2565 // Scavenge-a-lot for testing.
2566 forcegcperiod = 10 * 1e6
2567 scavengelimit = 20 * 1e6
2570 lastscavenge := nanotime()
2573 // Make wake-up period small enough for the sampling to be correct.
2574 maxsleep := forcegcperiod / 2
2575 if scavengelimit < forcegcperiod {
2576 maxsleep = scavengelimit / 2
2579 lasttrace := int64(0)
2580 idle := 0 // how many cycles in succession we had not wokeup somebody
2583 if idle == 0 { // start with 20us sleep...
2585 } else if idle > 50 { // start doubling the sleep after 1ms...
2588 if delay > 10*1000 { // up to 10ms
2592 if debug.schedtrace <= 0 && (sched.gcwaiting != 0 || atomicload(&sched.npidle) == uint32(gomaxprocs)) { // TODO: fast atomic
2594 if atomicload(&sched.gcwaiting) != 0 || atomicload(&sched.npidle) == uint32(gomaxprocs) {
2595 atomicstore(&sched.sysmonwait, 1)
2597 notetsleep(&sched.sysmonnote, maxsleep)
2599 atomicstore(&sched.sysmonwait, 0)
2600 noteclear(&sched.sysmonnote)
2606 // poll network if not polled for more than 10ms
2607 lastpoll := int64(atomicload64(&sched.lastpoll))
2609 unixnow := unixnanotime()
2610 if lastpoll != 0 && lastpoll+10*1000*1000 < now {
2611 cas64(&sched.lastpoll, uint64(lastpoll), uint64(now))
2612 gp := netpoll(false) // non-blocking - returns list of goroutines
2614 // Need to decrement number of idle locked M's
2615 // (pretending that one more is running) before injectglist.
2616 // Otherwise it can lead to the following situation:
2617 // injectglist grabs all P's but before it starts M's to run the P's,
2618 // another M returns from syscall, finishes running its G,
2619 // observes that there is no work to do and no other running M's
2620 // and reports deadlock.
2626 // retake P's blocked in syscalls
2627 // and preempt long running G's
2628 if retake(now) != 0 {
2633 // check if we need to force a GC
2634 lastgc := int64(atomicload64(&memstats.last_gc))
2635 if lastgc != 0 && unixnow-lastgc > forcegcperiod && atomicload(&forcegc.idle) != 0 {
2638 forcegc.g.schedlink = nil
2639 injectglist(forcegc.g)
2640 unlock(&forcegc.lock)
2642 // scavenge heap once in a while
2643 if lastscavenge+scavengelimit/2 < now {
2644 mHeap_Scavenge(int32(nscavenge), uint64(now), uint64(scavengelimit))
2648 if debug.schedtrace > 0 && lasttrace+int64(debug.schedtrace*1000000) <= now {
2650 schedtrace(debug.scheddetail > 0)
2655 var pdesc [_MaxGomaxprocs]struct {
2662 func retake(now int64) uint32 {
2664 for i := int32(0); i < gomaxprocs; i++ {
2672 // Retake P from syscall if it's there for more than 1 sysmon tick (at least 20us).
2673 t := int64(_p_.syscalltick)
2674 if int64(pd.syscalltick) != t {
2675 pd.syscalltick = uint32(t)
2676 pd.syscallwhen = now
2679 // On the one hand we don't want to retake Ps if there is no other work to do,
2680 // but on the other hand we want to retake them eventually
2681 // because they can prevent the sysmon thread from deep sleep.
2682 if _p_.runqhead == _p_.runqtail && atomicload(&sched.nmspinning)+atomicload(&sched.npidle) > 0 && pd.syscallwhen+10*1000*1000 > now {
2685 // Need to decrement number of idle locked M's
2686 // (pretending that one more is running) before the CAS.
2687 // Otherwise the M from which we retake can exit the syscall,
2688 // increment nmidle and report deadlock.
2690 if cas(&_p_.status, s, _Pidle) {
2695 } else if s == _Prunning {
2696 // Preempt G if it's running for more than 10ms.
2697 t := int64(_p_.schedtick)
2698 if int64(pd.schedtick) != t {
2699 pd.schedtick = uint32(t)
2703 if pd.schedwhen+10*1000*1000 > now {
2712 // Tell all goroutines that they have been preempted and they should stop.
2713 // This function is purely best-effort. It can fail to inform a goroutine if a
2714 // processor just started running it.
2715 // No locks need to be held.
2716 // Returns true if preemption request was issued to at least one goroutine.
2717 func preemptall() bool {
2719 for i := int32(0); i < gomaxprocs; i++ {
2721 if _p_ == nil || _p_.status != _Prunning {
2724 if preemptone(_p_) {
2731 // Tell the goroutine running on processor P to stop.
2732 // This function is purely best-effort. It can incorrectly fail to inform the
2733 // goroutine. It can send inform the wrong goroutine. Even if it informs the
2734 // correct goroutine, that goroutine might ignore the request if it is
2735 // simultaneously executing newstack.
2736 // No lock needs to be held.
2737 // Returns true if preemption request was issued.
2738 // The actual preemption will happen at some point in the future
2739 // and will be indicated by the gp->status no longer being
2741 func preemptone(_p_ *p) bool {
2743 if mp == nil || mp == getg().m {
2747 if gp == nil || gp == mp.g0 {
2753 // Every call in a go routine checks for stack overflow by
2754 // comparing the current stack pointer to gp->stackguard0.
2755 // Setting gp->stackguard0 to StackPreempt folds
2756 // preemption into the normal stack overflow check.
2757 gp.stackguard0 = stackPreempt
2763 func schedtrace(detailed bool) {
2770 print("SCHED ", (now-starttime)/1e6, "ms: gomaxprocs=", gomaxprocs, " idleprocs=", sched.npidle, " threads=", sched.mcount, " spinningthreads=", sched.nmspinning, " idlethreads=", sched.nmidle, " runqueue=", sched.runqsize)
2772 print(" gcwaiting=", sched.gcwaiting, " nmidlelocked=", sched.nmidlelocked, " stopwait=", sched.stopwait, " sysmonwait=", sched.sysmonwait, "\n")
2774 // We must be careful while reading data from P's, M's and G's.
2775 // Even if we hold schedlock, most data can be changed concurrently.
2776 // E.g. (p->m ? p->m->id : -1) can crash if p->m changes from non-nil to nil.
2777 for i := int32(0); i < gomaxprocs; i++ {
2783 h := atomicload(&_p_.runqhead)
2784 t := atomicload(&_p_.runqtail)
2790 print(" P", i, ": status=", _p_.status, " schedtick=", _p_.schedtick, " syscalltick=", _p_.syscalltick, " m=", id, " runqsize=", t-h, " gfreecnt=", _p_.gfreecnt, "\n")
2792 // In non-detailed mode format lengths of per-P run queues as:
2793 // [len1 len2 len3 len4]
2799 if i == gomaxprocs-1 {
2810 for mp := allm; mp != nil; mp = mp.alllink {
2813 lockedg := mp.lockedg
2826 print(" M", mp.id, ": p=", id1, " curg=", id2, " mallocing=", mp.mallocing, " throwing=", mp.throwing, " gcing=", mp.gcing, ""+" locks=", mp.locks, " dying=", mp.dying, " helpgc=", mp.helpgc, " spinning=", mp.spinning, " blocked=", getg().m.blocked, " lockedg=", id3, "\n")
2830 for gi := 0; gi < len(allgs); gi++ {
2833 lockedm := gp.lockedm
2842 print(" G", gp.goid, ": status=", readgstatus(gp), "(", gp.waitreason, ") m=", id1, " lockedm=", id2, "\n")
2848 // Put mp on midle list.
2849 // Sched must be locked.
2851 mp.schedlink = sched.midle
2857 // Try to get an m from midle list.
2858 // Sched must be locked.
2862 sched.midle = mp.schedlink
2868 // Put gp on the global runnable queue.
2869 // Sched must be locked.
2870 func globrunqput(gp *g) {
2872 if sched.runqtail != nil {
2873 sched.runqtail.schedlink = gp
2881 // Put a batch of runnable goroutines on the global runnable queue.
2882 // Sched must be locked.
2883 func globrunqputbatch(ghead *g, gtail *g, n int32) {
2884 gtail.schedlink = nil
2885 if sched.runqtail != nil {
2886 sched.runqtail.schedlink = ghead
2888 sched.runqhead = ghead
2890 sched.runqtail = gtail
2894 // Try get a batch of G's from the global runnable queue.
2895 // Sched must be locked.
2896 func globrunqget(_p_ *p, max int32) *g {
2897 if sched.runqsize == 0 {
2901 n := sched.runqsize/gomaxprocs + 1
2902 if n > sched.runqsize {
2905 if max > 0 && n > max {
2908 if n > int32(len(_p_.runq))/2 {
2909 n = int32(len(_p_.runq)) / 2
2913 if sched.runqsize == 0 {
2914 sched.runqtail = nil
2917 gp := sched.runqhead
2918 sched.runqhead = gp.schedlink
2921 gp1 := sched.runqhead
2922 sched.runqhead = gp1.schedlink
2928 // Put p to on _Pidle list.
2929 // Sched must be locked.
2930 func pidleput(_p_ *p) {
2931 _p_.link = sched.pidle
2933 xadd(&sched.npidle, 1) // TODO: fast atomic
2936 // Try get a p from _Pidle list.
2937 // Sched must be locked.
2938 func pidleget() *p {
2941 sched.pidle = _p_.link
2942 xadd(&sched.npidle, -1) // TODO: fast atomic
2947 // Try to put g on local runnable queue.
2948 // If it's full, put onto global queue.
2949 // Executed only by the owner P.
2950 func runqput(_p_ *p, gp *g) {
2952 h := atomicload(&_p_.runqhead) // load-acquire, synchronize with consumers
2954 if t-h < uint32(len(_p_.runq)) {
2955 _p_.runq[t%uint32(len(_p_.runq))] = gp
2956 atomicstore(&_p_.runqtail, t+1) // store-release, makes the item available for consumption
2959 if runqputslow(_p_, gp, h, t) {
2962 // the queue is not full, now the put above must suceed
2966 // Put g and a batch of work from local runnable queue on global queue.
2967 // Executed only by the owner P.
2968 func runqputslow(_p_ *p, gp *g, h, t uint32) bool {
2969 var batch [len(_p_.runq)/2 + 1]*g
2971 // First, grab a batch from local queue.
2974 if n != uint32(len(_p_.runq)/2) {
2975 gothrow("runqputslow: queue is not full")
2977 for i := uint32(0); i < n; i++ {
2978 batch[i] = _p_.runq[(h+i)%uint32(len(_p_.runq))]
2980 if !cas(&_p_.runqhead, h, h+n) { // cas-release, commits consume
2985 // Link the goroutines.
2986 for i := uint32(0); i < n; i++ {
2987 batch[i].schedlink = batch[i+1]
2990 // Now put the batch on global queue.
2992 globrunqputbatch(batch[0], batch[n], int32(n+1))
2997 // Get g from local runnable queue.
2998 // Executed only by the owner P.
2999 func runqget(_p_ *p) *g {
3001 h := atomicload(&_p_.runqhead) // load-acquire, synchronize with other consumers
3006 gp := _p_.runq[h%uint32(len(_p_.runq))]
3007 if cas(&_p_.runqhead, h, h+1) { // cas-release, commits consume
3013 // Grabs a batch of goroutines from local runnable queue.
3014 // batch array must be of size len(p->runq)/2. Returns number of grabbed goroutines.
3015 // Can be executed by any P.
3016 func runqgrab(_p_ *p, batch []*g) uint32 {
3018 h := atomicload(&_p_.runqhead) // load-acquire, synchronize with other consumers
3019 t := atomicload(&_p_.runqtail) // load-acquire, synchronize with the producer
3025 if n > uint32(len(_p_.runq)/2) { // read inconsistent h and t
3028 for i := uint32(0); i < n; i++ {
3029 batch[i] = _p_.runq[(h+i)%uint32(len(_p_.runq))]
3031 if cas(&_p_.runqhead, h, h+n) { // cas-release, commits consume
3037 // Steal half of elements from local runnable queue of p2
3038 // and put onto local runnable queue of p.
3039 // Returns one of the stolen elements (or nil if failed).
3040 func runqsteal(_p_, p2 *p) *g {
3041 var batch [len(_p_.runq) / 2]*g
3043 n := runqgrab(p2, batch[:])
3052 h := atomicload(&_p_.runqhead) // load-acquire, synchronize with consumers
3054 if t-h+n >= uint32(len(_p_.runq)) {
3055 gothrow("runqsteal: runq overflow")
3057 for i := uint32(0); i < n; i++ {
3058 _p_.runq[(t+i)%uint32(len(_p_.runq))] = batch[i]
3060 atomicstore(&_p_.runqtail, t+n) // store-release, makes the item available for consumption
3064 func testSchedLocalQueue() {
3066 gs := make([]g, len(_p_.runq))
3067 for i := 0; i < len(_p_.runq); i++ {
3068 if runqget(_p_) != nil {
3069 gothrow("runq is not empty initially")
3071 for j := 0; j < i; j++ {
3072 runqput(_p_, &gs[i])
3074 for j := 0; j < i; j++ {
3075 if runqget(_p_) != &gs[i] {
3076 print("bad element at iter ", i, "/", j, "\n")
3077 gothrow("bad element")
3080 if runqget(_p_) != nil {
3081 gothrow("runq is not empty afterwards")
3086 func testSchedLocalQueueSteal() {
3089 gs := make([]g, len(p1.runq))
3090 for i := 0; i < len(p1.runq); i++ {
3091 for j := 0; j < i; j++ {
3095 gp := runqsteal(p2, p1)
3116 for j := 0; j < i; j++ {
3118 print("bad element ", j, "(", gs[j].sig, ") at iter ", i, "\n")
3119 gothrow("bad element")
3122 if s != i/2 && s != i/2+1 {
3123 print("bad steal ", s, ", want ", i/2, " or ", i/2+1, ", iter ", i, "\n")
3124 gothrow("bad steal")
3129 func setMaxThreads(in int) (out int) {
3131 out = int(sched.maxmcount)
3132 sched.maxmcount = int32(in)
3138 var goexperiment string = "GOEXPERIMENT" // TODO: defined in zaexperiment.h
3140 func haveexperiment(name string) bool {
3148 xname, x = x[:i], x[i+1:]
3158 func sync_procPin() int {
3167 func sync_procUnpin() {