1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
17 #ifdef USING_SPLIT_STACK
19 /* FIXME: These are not declared anywhere. */
21 extern void __splitstack_getcontext(void *context[10]);
23 extern void __splitstack_setcontext(void *context[10]);
25 extern void *__splitstack_makecontext(size_t, void *context[10], size_t *);
27 extern void * __splitstack_resetcontext(void *context[10], size_t *);
29 extern void *__splitstack_find(void *, void *, size_t *, void **, void **,
32 extern void __splitstack_block_signals (int *, int *);
34 extern void __splitstack_block_signals_context (void *context[10], int *,
39 #if defined(USING_SPLIT_STACK) && defined(LINKER_SUPPORTS_SPLIT_STACK)
40 # ifdef PTHREAD_STACK_MIN
41 # define StackMin PTHREAD_STACK_MIN
43 # define StackMin 8192
46 # define StackMin 2 * 1024 * 1024
49 static void schedule(G*);
51 typedef struct Sched Sched;
54 G runtime_g0; // idle goroutine for m0
63 #ifndef SETCONTEXT_CLOBBERS_TLS
71 fixcontext(ucontext_t *c __attribute__ ((unused)))
77 # if defined(__x86_64__) && defined(__sun__)
79 // x86_64 Solaris 10 and 11 have a bug: setcontext switches the %fs
80 // register to that of the thread which called getcontext. The effect
81 // is that the address of all __thread variables changes. This bug
82 // also affects pthread_self() and pthread_getspecific. We work
83 // around it by clobbering the context field directly to keep %fs the
86 static __thread greg_t fs;
94 fs = c.uc_mcontext.gregs[REG_FSBASE];
98 fixcontext(ucontext_t* c)
100 c->uc_mcontext.gregs[REG_FSBASE] = fs;
105 # error unknown case for SETCONTEXT_CLOBBERS_TLS
111 // We can not always refer to the TLS variables directly. The
112 // compiler will call tls_get_addr to get the address of the variable,
113 // and it may hold it in a register across a call to schedule. When
114 // we get back from the call we may be running in a different thread,
115 // in which case the register now points to the TLS variable for a
116 // different thread. We use non-inlinable functions to avoid this
119 G* runtime_g(void) __attribute__ ((noinline, no_split_stack));
127 M* runtime_m(void) __attribute__ ((noinline, no_split_stack));
135 int32 runtime_gcwaiting;
139 // The go scheduler's job is to match ready-to-run goroutines (`g's)
140 // with waiting-for-work schedulers (`m's). If there are ready g's
141 // and no waiting m's, ready() will start a new m running in a new
142 // OS thread, so that all ready g's can run simultaneously, up to a limit.
143 // For now, m's never go away.
145 // By default, Go keeps only one kernel thread (m) running user code
146 // at a single time; other threads may be blocked in the operating system.
147 // Setting the environment variable $GOMAXPROCS or calling
148 // runtime.GOMAXPROCS() will change the number of user threads
149 // allowed to execute simultaneously. $GOMAXPROCS is thus an
150 // approximation of the maximum number of cores to use.
152 // Even a program that can run without deadlock in a single process
153 // might use more m's if given the chance. For example, the prime
154 // sieve will use as many m's as there are primes (up to runtime_sched.mmax),
155 // allowing different stages of the pipeline to execute in parallel.
156 // We could revisit this choice, only kicking off new m's for blocking
157 // system calls, but that would limit the amount of parallel computation
158 // that go would try to do.
160 // In general, one could imagine all sorts of refinements to the
161 // scheduler, but the goal now is just to get something working on
167 G *gfree; // available g's (status == Gdead)
170 G *ghead; // g's waiting to run
172 int32 gwait; // number of g's waiting to run
173 int32 gcount; // number of g's that are alive
174 int32 grunning; // number of g's running on cpu or in syscall
176 M *mhead; // m's waiting for work
177 int32 mwait; // number of m's waiting for work
178 int32 mcount; // number of m's that have been created
180 volatile uint32 atomic; // atomic scheduling word (see below)
182 int32 profilehz; // cpu profiling rate
184 bool init; // running initialization
185 bool lockmain; // init called runtime.LockOSThread
187 Note stopped; // one g can set waitstop and wait here for m's to stop
190 // The atomic word in sched is an atomic uint32 that
191 // holds these fields.
193 // [15 bits] mcpu number of m's executing on cpu
194 // [15 bits] mcpumax max number of m's allowed on cpu
195 // [1 bit] waitstop some g is waiting on stopped
196 // [1 bit] gwaiting gwait != 0
198 // These fields are the information needed by entersyscall
199 // and exitsyscall to decide whether to coordinate with the
200 // scheduler. Packing them into a single machine word lets
201 // them use a fast path with a single atomic read/write and
202 // no lock/unlock. This greatly reduces contention in
203 // syscall- or cgo-heavy multithreaded programs.
205 // Except for entersyscall and exitsyscall, the manipulations
206 // to these fields only happen while holding the schedlock,
207 // so the routines holding schedlock only need to worry about
208 // what entersyscall and exitsyscall do, not the other routines
209 // (which also use the schedlock).
211 // In particular, entersyscall and exitsyscall only read mcpumax,
212 // waitstop, and gwaiting. They never write them. Thus, writes to those
213 // fields can be done (holding schedlock) without fear of write conflicts.
214 // There may still be logic conflicts: for example, the set of waitstop must
215 // be conditioned on mcpu >= mcpumax or else the wait may be a
216 // spurious sleep. The Promela model in proc.p verifies these accesses.
219 mcpuMask = (1<<mcpuWidth) - 1,
221 mcpumaxShift = mcpuShift + mcpuWidth,
222 waitstopShift = mcpumaxShift + mcpuWidth,
223 gwaitingShift = waitstopShift+1,
225 // The max value of GOMAXPROCS is constrained
226 // by the max value we can store in the bit fields
227 // of the atomic word. Reserve a few high values
228 // so that we can detect accidental decrement
230 maxgomaxprocs = mcpuMask - 10,
233 #define atomic_mcpu(v) (((v)>>mcpuShift)&mcpuMask)
234 #define atomic_mcpumax(v) (((v)>>mcpumaxShift)&mcpuMask)
235 #define atomic_waitstop(v) (((v)>>waitstopShift)&1)
236 #define atomic_gwaiting(v) (((v)>>gwaitingShift)&1)
239 int32 runtime_gomaxprocs;
240 bool runtime_singleproc;
242 static bool canaddmcpu(void);
244 // An m that is waiting for notewakeup(&m->havenextg). This may
245 // only be accessed while the scheduler lock is held. This is used to
246 // minimize the number of times we call notewakeup while the scheduler
247 // lock is held, since the m will normally move quickly to lock the
248 // scheduler itself, producing lock contention.
251 // Scheduling helpers. Sched must be locked.
252 static void gput(G*); // put/get on ghead/gtail
253 static G* gget(void);
254 static void mput(M*); // put/get on mhead
256 static void gfput(G*); // put/get on gfree
257 static G* gfget(void);
258 static void matchmg(void); // match m's to g's
259 static void readylocked(G*); // ready, but sched is locked
260 static void mnextg(M*, G*);
261 static void mcommoninit(M*);
269 v = runtime_sched.atomic;
271 w &= ~(mcpuMask<<mcpumaxShift);
272 w |= n<<mcpumaxShift;
273 if(runtime_cas(&runtime_sched.atomic, v, w))
278 // First function run by a new goroutine. This replaces gogocall.
284 fn = (void (*)(void*))(g->entry);
289 // Switch context to a different goroutine. This is like longjmp.
290 static void runtime_gogo(G*) __attribute__ ((noinline));
292 runtime_gogo(G* newg)
294 #ifdef USING_SPLIT_STACK
295 __splitstack_setcontext(&newg->stack_context[0]);
298 newg->fromgogo = true;
299 fixcontext(&newg->context);
300 setcontext(&newg->context);
301 runtime_throw("gogo setcontext returned");
304 // Save context and call fn passing g as a parameter. This is like
305 // setjmp. Because getcontext always returns 0, unlike setjmp, we use
306 // g->fromgogo as a code. It will be true if we got here via
307 // setcontext. g == nil the first time this is called in a new m.
308 static void runtime_mcall(void (*)(G*)) __attribute__ ((noinline));
310 runtime_mcall(void (*pfn)(G*))
314 #ifndef USING_SPLIT_STACK
318 // Ensure that all registers are on the stack for the garbage
320 __builtin_unwind_init();
325 runtime_throw("runtime: mcall called on m->g0 stack");
329 #ifdef USING_SPLIT_STACK
330 __splitstack_getcontext(&g->stack_context[0]);
334 gp->fromgogo = false;
335 getcontext(&gp->context);
337 // When we return from getcontext, we may be running
338 // in a new thread. That means that m and g may have
339 // changed. They are global variables so we will
340 // reload them, but the addresses of m and g may be
341 // cached in our local stack frame, and those
342 // addresses may be wrong. Call functions to reload
343 // the values for this thread.
347 if (gp == nil || !gp->fromgogo) {
348 #ifdef USING_SPLIT_STACK
349 __splitstack_setcontext(&mp->g0->stack_context[0]);
351 mp->g0->entry = (byte*)pfn;
354 // It's OK to set g directly here because this case
355 // can not occur if we got here via a setcontext to
356 // the getcontext call just above.
359 fixcontext(&mp->g0->context);
360 setcontext(&mp->g0->context);
361 runtime_throw("runtime: mcall function returned");
365 // Keep trace of scavenger's goroutine for deadlock detection.
368 // The bootstrap sequence is:
372 // make & queue new G
373 // call runtime_mstart
375 // The new G calls runtime_main.
377 runtime_schedinit(void)
391 runtime_mallocinit();
398 // Allocate internal symbol table representation now,
399 // so that we don't need to call malloc when we crash.
400 // runtime_findfunc(0);
402 runtime_gomaxprocs = 1;
403 p = runtime_getenv("GOMAXPROCS");
404 if(p != nil && (n = runtime_atoi(p)) != 0) {
405 if(n > maxgomaxprocs)
407 runtime_gomaxprocs = n;
409 // wait for the main goroutine to start before taking
410 // GOMAXPROCS into account.
412 runtime_singleproc = runtime_gomaxprocs == 1;
414 canaddmcpu(); // mcpu++ to account for bootstrap m
415 m->helpgc = 1; // flag to tell schedule() to mcpu--
416 runtime_sched.grunning++;
418 // Can not enable GC until all roots are registered.
419 // mstats.enablegc = 1;
423 extern void main_init(void) __asm__ ("__go_init_main");
424 extern void main_main(void) __asm__ ("main.main");
426 // The main goroutine.
430 // Lock the main goroutine onto this, the main OS thread,
431 // during initialization. Most programs won't care, but a few
432 // do require certain calls to be made by the main thread.
433 // Those can arrange for main.main to run in the main thread
434 // by calling runtime.LockOSThread during initialization
435 // to preserve the lock.
436 runtime_LockOSThread();
437 // From now on, newgoroutines may use non-main threads.
438 setmcpumax(runtime_gomaxprocs);
439 runtime_sched.init = true;
440 scvg = __go_go(runtime_MHeap_Scavenger, nil);
442 runtime_sched.init = false;
443 if(!runtime_sched.lockmain)
444 runtime_UnlockOSThread();
446 // For gccgo we have to wait until after main is initialized
447 // to enable GC, because initializing main registers the GC
457 // Lock the scheduler.
461 runtime_lock(&runtime_sched);
464 // Unlock the scheduler.
472 runtime_unlock(&runtime_sched);
474 runtime_notewakeup(&m->havenextg);
480 g->status = Gmoribund;
485 runtime_goroutineheader(G *g)
504 status = g->waitreason;
515 runtime_printf("goroutine %d [%s]:\n", g->goid, status);
519 runtime_tracebackothers(G *me)
523 for(g = runtime_allg; g != nil; g = g->alllink) {
524 if(g == me || g->status == Gdead)
526 runtime_printf("\n");
527 runtime_goroutineheader(g);
528 // runtime_traceback(g->sched.pc, g->sched.sp, 0, g);
532 // Mark this g as m's idle goroutine.
533 // This functionality might be used in environments where programs
534 // are limited to a single thread, to simulate a select-driven
535 // network server. It is not exposed via the standard runtime API.
537 runtime_idlegoroutine(void)
540 runtime_throw("g is already an idle goroutine");
547 m->id = runtime_sched.mcount++;
548 m->fastrand = 0x49f6428aUL + m->id + runtime_cputicks();
551 m->mcache = runtime_allocmcache();
553 runtime_callers(1, m->createstack, nelem(m->createstack));
555 // Add to runtime_allm so garbage collector doesn't free m
556 // when it is just in a register or thread-local storage.
557 m->alllink = runtime_allm;
558 // runtime_NumCgoCall() iterates over allm w/o schedlock,
559 // so we need to publish it safely.
560 runtime_atomicstorep(&runtime_allm, m);
563 // Try to increment mcpu. Report whether succeeded.
570 v = runtime_sched.atomic;
571 if(atomic_mcpu(v) >= atomic_mcpumax(v))
573 if(runtime_cas(&runtime_sched.atomic, v, v+(1<<mcpuShift)))
578 // Put on `g' queue. Sched must be locked.
584 // If g is wired, hand it off directly.
585 if((m = g->lockedm) != nil && canaddmcpu()) {
590 // If g is the idle goroutine for an m, hand it off.
591 if(g->idlem != nil) {
592 if(g->idlem->idleg != nil) {
593 runtime_printf("m%d idle out of sync: g%d g%d\n",
595 g->idlem->idleg->goid, g->goid);
596 runtime_throw("runtime: double idle");
603 if(runtime_sched.ghead == nil)
604 runtime_sched.ghead = g;
606 runtime_sched.gtail->schedlink = g;
607 runtime_sched.gtail = g;
610 // if it transitions to nonzero, set atomic gwaiting bit.
611 if(runtime_sched.gwait++ == 0)
612 runtime_xadd(&runtime_sched.atomic, 1<<gwaitingShift);
615 // Report whether gget would return something.
619 return runtime_sched.ghead != nil || m->idleg != nil;
622 // Get from `g' queue. Sched must be locked.
628 g = runtime_sched.ghead;
630 runtime_sched.ghead = g->schedlink;
631 if(runtime_sched.ghead == nil)
632 runtime_sched.gtail = nil;
634 // if it transitions to zero, clear atomic gwaiting bit.
635 if(--runtime_sched.gwait == 0)
636 runtime_xadd(&runtime_sched.atomic, -1<<gwaitingShift);
637 } else if(m->idleg != nil) {
644 // Put on `m' list. Sched must be locked.
648 m->schedlink = runtime_sched.mhead;
649 runtime_sched.mhead = m;
650 runtime_sched.mwait++;
653 // Get an `m' to run `g'. Sched must be locked.
659 // if g has its own m, use it.
660 if(g && (m = g->lockedm) != nil)
663 // otherwise use general m pool.
664 if((m = runtime_sched.mhead) != nil){
665 runtime_sched.mhead = m->schedlink;
666 runtime_sched.mwait--;
671 // Mark g ready to run.
680 // Mark g ready to run. Sched is already locked.
681 // G might be running already and about to stop.
682 // The sched lock protects g->status from changing underfoot.
687 // Running on another machine.
688 // Ready it when it stops.
694 if(g->status == Grunnable || g->status == Grunning) {
695 runtime_printf("goroutine %d has status %d\n", g->goid, g->status);
696 runtime_throw("bad g->status in ready");
698 g->status = Grunnable;
704 // Same as readylocked but a different symbol so that
705 // debuggers can set a breakpoint here and catch all
708 newprocreadylocked(G *g)
713 // Pass g to m for running.
714 // Caller has already incremented mcpu.
718 runtime_sched.grunning++;
723 runtime_notewakeup(&mwakeup->havenextg);
728 // Get the next goroutine that m should run.
729 // Sched must be locked on entry, is unlocked on exit.
730 // Makes sure that at most $GOMAXPROCS g's are
731 // running on cpus (not in system calls) at any given time.
739 if(atomic_mcpu(runtime_sched.atomic) >= maxgomaxprocs)
740 runtime_throw("negative mcpu");
742 // If there is a g waiting as m->nextg, the mcpu++
743 // happened before it was passed to mnextg.
744 if(m->nextg != nil) {
751 if(m->lockedg != nil) {
752 // We can only run one g, and it's not available.
753 // Make sure some other cpu is running to handle
754 // the ordinary run queue.
755 if(runtime_sched.gwait != 0) {
757 // m->lockedg might have been on the queue.
758 if(m->nextg != nil) {
766 // Look for work on global queue.
767 while(haveg() && canaddmcpu()) {
770 runtime_throw("gget inconsistency");
773 mnextg(gp->lockedm, gp);
776 runtime_sched.grunning++;
781 // The while loop ended either because the g queue is empty
782 // or because we have maxed out our m procs running go
783 // code (mcpu >= mcpumax). We need to check that
784 // concurrent actions by entersyscall/exitsyscall cannot
785 // invalidate the decision to end the loop.
787 // We hold the sched lock, so no one else is manipulating the
788 // g queue or changing mcpumax. Entersyscall can decrement
789 // mcpu, but if does so when there is something on the g queue,
790 // the gwait bit will be set, so entersyscall will take the slow path
791 // and use the sched lock. So it cannot invalidate our decision.
793 // Wait on global m queue.
797 // Look for deadlock situation.
798 if((scvg == nil && runtime_sched.grunning == 0) ||
799 (scvg != nil && runtime_sched.grunning == 1 && runtime_sched.gwait == 0 &&
800 (scvg->status == Grunning || scvg->status == Gsyscall))) {
801 runtime_throw("all goroutines are asleep - deadlock!");
806 runtime_noteclear(&m->havenextg);
808 // Stoptheworld is waiting for all but its cpu to go to stop.
809 // Entersyscall might have decremented mcpu too, but if so
810 // it will see the waitstop and take the slow path.
811 // Exitsyscall never increments mcpu beyond mcpumax.
812 v = runtime_atomicload(&runtime_sched.atomic);
813 if(atomic_waitstop(v) && atomic_mcpu(v) <= atomic_mcpumax(v)) {
814 // set waitstop = 0 (known to be 1)
815 runtime_xadd(&runtime_sched.atomic, -1<<waitstopShift);
816 runtime_notewakeup(&runtime_sched.stopped);
820 runtime_notesleep(&m->havenextg);
824 runtime_lock(&runtime_sched);
827 if((gp = m->nextg) == nil)
828 runtime_throw("bad m->nextg in nextgoroutine");
834 runtime_helpgc(bool *extra)
839 // Figure out how many CPUs to use.
840 // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc.
841 max = runtime_gomaxprocs;
842 if(max > runtime_ncpu)
843 max = runtime_ncpu > 0 ? runtime_ncpu : 1;
847 // We're going to use one CPU no matter what.
848 // Figure out the max number of additional CPUs.
851 runtime_lock(&runtime_sched);
853 while(n < max && (mp = mget(nil)) != nil) {
857 runtime_notewakeup(&mp->havenextg);
859 runtime_unlock(&runtime_sched);
866 runtime_stoptheworld(void)
871 runtime_gcwaiting = 1;
877 v = runtime_sched.atomic;
878 if(atomic_mcpu(v) <= 1)
881 // It would be unsafe for multiple threads to be using
882 // the stopped note at once, but there is only
883 // ever one thread doing garbage collection.
884 runtime_noteclear(&runtime_sched.stopped);
885 if(atomic_waitstop(v))
886 runtime_throw("invalid waitstop");
888 // atomic { waitstop = 1 }, predicated on mcpu <= 1 check above
890 if(!runtime_cas(&runtime_sched.atomic, v, v+(1<<waitstopShift)))
894 runtime_notesleep(&runtime_sched.stopped);
897 runtime_singleproc = runtime_gomaxprocs == 1;
902 runtime_starttheworld(bool extra)
907 runtime_gcwaiting = 0;
908 setmcpumax(runtime_gomaxprocs);
910 if(extra && canaddmcpu()) {
911 // Start a new m that will (we hope) be idle
912 // and so available to help when the next
913 // garbage collection happens.
914 // canaddmcpu above did mcpu++
915 // (necessary, because m will be doing various
916 // initialization work so is definitely running),
917 // but m is not running a specific goroutine,
918 // so set the helpgc flag as a signal to m's
919 // first schedule(nil) to mcpu-- and grunning--.
922 runtime_sched.grunning++;
927 // Called to start an M.
929 runtime_mstart(void* mp)
939 // Record top of stack for use by mcall.
940 // Once we call schedule we're never coming back,
941 // so other calls can reuse this stack space.
942 #ifdef USING_SPLIT_STACK
943 __splitstack_getcontext(&g->stack_context[0]);
945 g->gcinitial_sp = ∓
946 // Setting gcstack_size to 0 is a marker meaning that gcinitial_sp
947 // is the top of the stack, not the bottom.
951 getcontext(&g->context);
953 if(g->entry != nil) {
954 // Got here from mcall.
955 void (*pfn)(G*) = (void (*)(G*))g->entry;
956 G* gp = (G*)g->param;
962 #ifdef USING_SPLIT_STACK
964 int dont_block_signals = 0;
965 __splitstack_block_signals(&dont_block_signals, nil);
973 typedef struct CgoThreadStart CgoThreadStart;
974 struct CgoThreadStart
981 // Kick off new m's as needed (up to mcpumax).
989 if(m->mallocing || m->gcing)
992 while(haveg() && canaddmcpu()) {
995 runtime_throw("gget inconsistency");
997 // Find the m that will run gp.
998 if((mp = mget(gp)) == nil)
1004 // Create a new m. It will start off with a call to runtime_mstart.
1009 pthread_attr_t attr;
1012 m = runtime_malloc(sizeof(M));
1014 m->g0 = runtime_malg(-1, nil, nil);
1016 if(pthread_attr_init(&attr) != 0)
1017 runtime_throw("pthread_attr_init");
1018 if(pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0)
1019 runtime_throw("pthread_attr_setdetachstate");
1021 #ifndef PTHREAD_STACK_MIN
1022 #define PTHREAD_STACK_MIN 8192
1024 if(pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN) != 0)
1025 runtime_throw("pthread_attr_setstacksize");
1027 if(pthread_create(&tid, &attr, runtime_mstart, m) != 0)
1028 runtime_throw("pthread_create");
1033 // One round of scheduler: find a goroutine and run it.
1034 // The argument is the goroutine that was running before
1035 // schedule was called, or nil if this is the first call.
1045 // Just finished running gp.
1047 runtime_sched.grunning--;
1049 // atomic { mcpu-- }
1050 v = runtime_xadd(&runtime_sched.atomic, -1<<mcpuShift);
1051 if(atomic_mcpu(v) > maxgomaxprocs)
1052 runtime_throw("negative mcpu in scheduler");
1057 // Shouldn't have been running!
1058 runtime_throw("bad gp->status in sched");
1060 gp->status = Grunnable;
1071 if(--runtime_sched.gcount == 0)
1075 if(gp->readyonstop){
1076 gp->readyonstop = 0;
1079 } else if(m->helpgc) {
1080 // Bootstrap m or new m started by starttheworld.
1081 // atomic { mcpu-- }
1082 v = runtime_xadd(&runtime_sched.atomic, -1<<mcpuShift);
1083 if(atomic_mcpu(v) > maxgomaxprocs)
1084 runtime_throw("negative mcpu in scheduler");
1085 // Compensate for increment in starttheworld().
1086 runtime_sched.grunning--;
1088 } else if(m->nextg != nil) {
1089 // New m started by matchmg.
1091 runtime_throw("invalid m state in scheduler");
1094 // Find (or wait for) g to run. Unlocks runtime_sched.
1095 gp = nextgandunlock();
1096 gp->readyonstop = 0;
1097 gp->status = Grunning;
1101 // Check whether the profiler needs to be turned on or off.
1102 hz = runtime_sched.profilehz;
1103 if(m->profilehz != hz)
1104 runtime_resetcpuprofiler(hz);
1109 // Enter scheduler. If g->status is Grunning,
1110 // re-queues g and runs everyone else who is waiting
1111 // before running g again. If g->status is Gmoribund,
1114 runtime_gosched(void)
1117 runtime_throw("gosched holding locks");
1119 runtime_throw("gosched of g0");
1120 runtime_mcall(schedule);
1123 // The goroutine g is about to enter a system call.
1124 // Record that it's not using the cpu anymore.
1125 // This is called only from the go syscall library and cgocall,
1126 // not from the low-level system calls used by the runtime.
1128 // Entersyscall cannot split the stack: the runtime_gosave must
1129 // make g->sched refer to the caller's stack segment, because
1130 // entersyscall is going to return immediately after.
1131 // It's okay to call matchmg and notewakeup even after
1132 // decrementing mcpu, because we haven't released the
1133 // sched lock yet, so the garbage collector cannot be running.
1135 void runtime_entersyscall(void) __attribute__ ((no_split_stack));
1138 runtime_entersyscall(void)
1142 if(m->profilehz > 0)
1143 runtime_setprof(false);
1145 // Leave SP around for gc and traceback.
1146 #ifdef USING_SPLIT_STACK
1147 g->gcstack = __splitstack_find(NULL, NULL, &g->gcstack_size,
1148 &g->gcnext_segment, &g->gcnext_sp,
1151 g->gcnext_sp = (byte *) &v;
1154 // Save the registers in the g structure so that any pointers
1155 // held in registers will be seen by the garbage collector.
1156 // We could use getcontext here, but setjmp is more efficient
1157 // because it doesn't need to save the signal mask.
1160 g->status = Gsyscall;
1163 // The slow path inside the schedlock/schedunlock will get
1164 // through without stopping if it does:
1167 // waitstop && mcpu <= mcpumax not true
1168 // If we can do the same with a single atomic add,
1169 // then we can skip the locks.
1170 v = runtime_xadd(&runtime_sched.atomic, -1<<mcpuShift);
1171 if(!atomic_gwaiting(v) && (!atomic_waitstop(v) || atomic_mcpu(v) > atomic_mcpumax(v)))
1175 v = runtime_atomicload(&runtime_sched.atomic);
1176 if(atomic_gwaiting(v)) {
1178 v = runtime_atomicload(&runtime_sched.atomic);
1180 if(atomic_waitstop(v) && atomic_mcpu(v) <= atomic_mcpumax(v)) {
1181 runtime_xadd(&runtime_sched.atomic, -1<<waitstopShift);
1182 runtime_notewakeup(&runtime_sched.stopped);
1188 // The goroutine g exited its system call.
1189 // Arrange for it to run on a cpu again.
1190 // This is called only from the go syscall library, not
1191 // from the low-level system calls used by the runtime.
1193 runtime_exitsyscall(void)
1199 // If we can do the mcpu++ bookkeeping and
1200 // find that we still have mcpu <= mcpumax, then we can
1201 // start executing Go code immediately, without having to
1202 // schedlock/schedunlock.
1204 v = runtime_xadd(&runtime_sched.atomic, (1<<mcpuShift));
1205 if(m->profilehz == runtime_sched.profilehz && atomic_mcpu(v) <= atomic_mcpumax(v)) {
1206 // There's a cpu for us, so we can run.
1207 gp->status = Grunning;
1208 // Garbage collector isn't running (since we are),
1209 // so okay to clear gcstack.
1210 #ifdef USING_SPLIT_STACK
1213 gp->gcnext_sp = nil;
1214 runtime_memclr(gp->gcregs, sizeof gp->gcregs);
1216 if(m->profilehz > 0)
1217 runtime_setprof(true);
1221 // Tell scheduler to put g back on the run queue:
1222 // mostly equivalent to g->status = Grunning,
1223 // but keeps the garbage collector from thinking
1224 // that g is running right now, which it's not.
1225 gp->readyonstop = 1;
1227 // All the cpus are taken.
1228 // The scheduler will ready g and put this m to sleep.
1229 // When the scheduler takes g away from m,
1230 // it will undo the runtime_sched.mcpu++ above.
1233 // Gosched returned, so we're allowed to run now.
1234 // Delete the gcstack information that we left for
1235 // the garbage collector during the system call.
1236 // Must wait until now because until gosched returns
1237 // we don't know for sure that the garbage collector
1239 #ifdef USING_SPLIT_STACK
1242 gp->gcnext_sp = nil;
1243 runtime_memclr(gp->gcregs, sizeof gp->gcregs);
1246 // Allocate a new g, with a stack big enough for stacksize bytes.
1248 runtime_malg(int32 stacksize, byte** ret_stack, size_t* ret_stacksize)
1252 newg = runtime_malloc(sizeof(G));
1253 if(stacksize >= 0) {
1254 #if USING_SPLIT_STACK
1255 int dont_block_signals = 0;
1257 *ret_stack = __splitstack_makecontext(stacksize,
1258 &newg->stack_context[0],
1260 __splitstack_block_signals_context(&newg->stack_context[0],
1261 &dont_block_signals, nil);
1263 *ret_stack = runtime_mallocgc(stacksize, FlagNoProfiling|FlagNoGC, 0, 0);
1264 *ret_stacksize = stacksize;
1265 newg->gcinitial_sp = *ret_stack;
1266 newg->gcstack_size = stacksize;
1272 /* For runtime package testing. */
1274 void runtime_testing_entersyscall(void)
1275 __asm__("libgo_runtime.runtime.entersyscall");
1278 runtime_testing_entersyscall()
1280 runtime_entersyscall();
1283 void runtime_testing_exitsyscall(void)
1284 __asm__("libgo_runtime.runtime.exitsyscall");
1287 runtime_testing_exitsyscall()
1289 runtime_exitsyscall();
1293 __go_go(void (*fn)(void*), void* arg)
1297 G * volatile newg; // volatile to avoid longjmp warning
1301 if((newg = gfget()) != nil){
1302 #ifdef USING_SPLIT_STACK
1303 int dont_block_signals = 0;
1305 sp = __splitstack_resetcontext(&newg->stack_context[0],
1307 __splitstack_block_signals_context(&newg->stack_context[0],
1308 &dont_block_signals, nil);
1310 sp = newg->gcinitial_sp;
1311 spsize = newg->gcstack_size;
1313 runtime_throw("bad spsize in __go_go");
1314 newg->gcnext_sp = sp;
1317 newg = runtime_malg(StackMin, &sp, &spsize);
1318 if(runtime_lastg == nil)
1319 runtime_allg = newg;
1321 runtime_lastg->alllink = newg;
1322 runtime_lastg = newg;
1324 newg->status = Gwaiting;
1325 newg->waitreason = "new goroutine";
1327 newg->entry = (byte*)fn;
1329 newg->gopc = (uintptr)__builtin_return_address(0);
1331 runtime_sched.gcount++;
1332 runtime_sched.goidgen++;
1333 newg->goid = runtime_sched.goidgen;
1336 runtime_throw("nil g->stack0");
1338 getcontext(&newg->context);
1339 newg->context.uc_stack.ss_sp = sp;
1340 #ifdef MAKECONTEXT_STACK_TOP
1341 newg->context.uc_stack.ss_sp += spsize;
1343 newg->context.uc_stack.ss_size = spsize;
1344 makecontext(&newg->context, kickoff, 0);
1346 newprocreadylocked(newg);
1350 //printf(" goid=%d\n", newg->goid);
1353 // Put on gfree list. Sched must be locked.
1357 g->schedlink = runtime_sched.gfree;
1358 runtime_sched.gfree = g;
1361 // Get from gfree list. Sched must be locked.
1367 g = runtime_sched.gfree;
1369 runtime_sched.gfree = g->schedlink;
1373 // Run all deferred functions for the current goroutine.
1379 while((d = g->defer) != nil) {
1386 g->defer = d->__next;
1391 void runtime_Goexit (void) asm ("libgo_runtime.runtime.Goexit");
1394 runtime_Goexit(void)
1400 void runtime_Gosched (void) asm ("libgo_runtime.runtime.Gosched");
1403 runtime_Gosched(void)
1408 // Implementation of runtime.GOMAXPROCS.
1409 // delete when scheduler is stronger
1411 runtime_gomaxprocsfunc(int32 n)
1417 ret = runtime_gomaxprocs;
1420 if(n > maxgomaxprocs)
1422 runtime_gomaxprocs = n;
1423 if(runtime_gomaxprocs > 1)
1424 runtime_singleproc = false;
1425 if(runtime_gcwaiting != 0) {
1426 if(atomic_mcpumax(runtime_sched.atomic) != 1)
1427 runtime_throw("invalid mcpumax during gc");
1434 // If there are now fewer allowed procs
1435 // than procs running, stop.
1436 v = runtime_atomicload(&runtime_sched.atomic);
1437 if((int32)atomic_mcpu(v) > n) {
1442 // handle more procs
1449 runtime_LockOSThread(void)
1451 if(m == &runtime_m0 && runtime_sched.init) {
1452 runtime_sched.lockmain = true;
1460 runtime_UnlockOSThread(void)
1462 if(m == &runtime_m0 && runtime_sched.init) {
1463 runtime_sched.lockmain = false;
1471 runtime_lockedOSThread(void)
1473 return g->lockedm != nil && m->lockedg != nil;
1476 // for testing of callbacks
1478 _Bool runtime_golockedOSThread(void)
1479 asm("libgo_runtime.runtime.golockedOSThread");
1482 runtime_golockedOSThread(void)
1484 return runtime_lockedOSThread();
1487 // for testing of wire, unwire
1494 int32 runtime_NumGoroutine (void)
1495 __asm__ ("libgo_runtime.runtime.NumGoroutine");
1498 runtime_NumGoroutine()
1500 return runtime_sched.gcount;
1504 runtime_gcount(void)
1506 return runtime_sched.gcount;
1510 runtime_mcount(void)
1512 return runtime_sched.mcount;
1517 void (*fn)(uintptr*, int32);
1522 // Called if we receive a SIGPROF signal.
1524 runtime_sigprof(uint8 *pc __attribute__ ((unused)),
1525 uint8 *sp __attribute__ ((unused)),
1526 uint8 *lr __attribute__ ((unused)),
1527 G *gp __attribute__ ((unused)))
1531 if(prof.fn == nil || prof.hz == 0)
1534 runtime_lock(&prof);
1535 if(prof.fn == nil) {
1536 runtime_unlock(&prof);
1539 // n = runtime_gentraceback(pc, sp, lr, gp, 0, prof.pcbuf, nelem(prof.pcbuf));
1541 // prof.fn(prof.pcbuf, n);
1542 runtime_unlock(&prof);
1545 // Arrange to call fn with a traceback hz times a second.
1547 runtime_setcpuprofilerate(void (*fn)(uintptr*, int32), int32 hz)
1549 // Force sane arguments.
1557 // Stop profiler on this cpu so that it is safe to lock prof.
1558 // if a profiling signal came in while we had prof locked,
1559 // it would deadlock.
1560 runtime_resetcpuprofiler(0);
1562 runtime_lock(&prof);
1565 runtime_unlock(&prof);
1566 runtime_lock(&runtime_sched);
1567 runtime_sched.profilehz = hz;
1568 runtime_unlock(&runtime_sched);
1571 runtime_resetcpuprofiler(hz);