OSDN Git Service

1a605a1921c1dbf4bdf7b07f90f0bf025dcc796d
[pf3gnuchains/gcc-fork.git] / libgo / runtime / proc.c
1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include <limits.h>
6 #include <stdlib.h>
7 #include <pthread.h>
8 #include <unistd.h>
9
10 #include "config.h"
11 #include "runtime.h"
12 #include "arch.h"
13 #include "defs.h"
14 #include "malloc.h"
15 #include "go-defer.h"
16
17 #ifdef USING_SPLIT_STACK
18
19 /* FIXME: These are not declared anywhere.  */
20
21 extern void __splitstack_getcontext(void *context[10]);
22
23 extern void __splitstack_setcontext(void *context[10]);
24
25 extern void *__splitstack_makecontext(size_t, void *context[10], size_t *);
26
27 extern void * __splitstack_resetcontext(void *context[10], size_t *);
28
29 extern void *__splitstack_find(void *, void *, size_t *, void **, void **,
30                                void **);
31
32 extern void __splitstack_block_signals (int *, int *);
33
34 extern void __splitstack_block_signals_context (void *context[10], int *,
35                                                 int *);
36
37 #endif
38
39 #if defined(USING_SPLIT_STACK) && defined(LINKER_SUPPORTS_SPLIT_STACK)
40 # ifdef PTHREAD_STACK_MIN
41 #  define StackMin PTHREAD_STACK_MIN
42 # else
43 #  define StackMin 8192
44 # endif
45 #else
46 # define StackMin 2 * 1024 * 1024
47 #endif
48
49 uintptr runtime_stacks_sys;
50
51 static void schedule(G*);
52
53 static void gtraceback(G*);
54
55 typedef struct Sched Sched;
56
57 M       runtime_m0;
58 G       runtime_g0;     // idle goroutine for m0
59
60 #ifdef __rtems__
61 #define __thread
62 #endif
63
64 static __thread G *g;
65 static __thread M *m;
66
67 #ifndef SETCONTEXT_CLOBBERS_TLS
68
69 static inline void
70 initcontext(void)
71 {
72 }
73
74 static inline void
75 fixcontext(ucontext_t *c __attribute__ ((unused)))
76 {
77 }
78
79 # else
80
81 # if defined(__x86_64__) && defined(__sun__)
82
83 // x86_64 Solaris 10 and 11 have a bug: setcontext switches the %fs
84 // register to that of the thread which called getcontext.  The effect
85 // is that the address of all __thread variables changes.  This bug
86 // also affects pthread_self() and pthread_getspecific.  We work
87 // around it by clobbering the context field directly to keep %fs the
88 // same.
89
90 static __thread greg_t fs;
91
92 static inline void
93 initcontext(void)
94 {
95         ucontext_t c;
96
97         getcontext(&c);
98         fs = c.uc_mcontext.gregs[REG_FSBASE];
99 }
100
101 static inline void
102 fixcontext(ucontext_t* c)
103 {
104         c->uc_mcontext.gregs[REG_FSBASE] = fs;
105 }
106
107 # else
108
109 #  error unknown case for SETCONTEXT_CLOBBERS_TLS
110
111 # endif
112
113 #endif
114
115 // We can not always refer to the TLS variables directly.  The
116 // compiler will call tls_get_addr to get the address of the variable,
117 // and it may hold it in a register across a call to schedule.  When
118 // we get back from the call we may be running in a different thread,
119 // in which case the register now points to the TLS variable for a
120 // different thread.  We use non-inlinable functions to avoid this
121 // when necessary.
122
123 G* runtime_g(void) __attribute__ ((noinline, no_split_stack));
124
125 G*
126 runtime_g(void)
127 {
128         return g;
129 }
130
131 M* runtime_m(void) __attribute__ ((noinline, no_split_stack));
132
133 M*
134 runtime_m(void)
135 {
136         return m;
137 }
138
139 int32   runtime_gcwaiting;
140
141 // Go scheduler
142 //
143 // The go scheduler's job is to match ready-to-run goroutines (`g's)
144 // with waiting-for-work schedulers (`m's).  If there are ready g's
145 // and no waiting m's, ready() will start a new m running in a new
146 // OS thread, so that all ready g's can run simultaneously, up to a limit.
147 // For now, m's never go away.
148 //
149 // By default, Go keeps only one kernel thread (m) running user code
150 // at a single time; other threads may be blocked in the operating system.
151 // Setting the environment variable $GOMAXPROCS or calling
152 // runtime.GOMAXPROCS() will change the number of user threads
153 // allowed to execute simultaneously.  $GOMAXPROCS is thus an
154 // approximation of the maximum number of cores to use.
155 //
156 // Even a program that can run without deadlock in a single process
157 // might use more m's if given the chance.  For example, the prime
158 // sieve will use as many m's as there are primes (up to runtime_sched.mmax),
159 // allowing different stages of the pipeline to execute in parallel.
160 // We could revisit this choice, only kicking off new m's for blocking
161 // system calls, but that would limit the amount of parallel computation
162 // that go would try to do.
163 //
164 // In general, one could imagine all sorts of refinements to the
165 // scheduler, but the goal now is just to get something working on
166 // Linux and OS X.
167
168 struct Sched {
169         Lock;
170
171         G *gfree;       // available g's (status == Gdead)
172         int32 goidgen;
173
174         G *ghead;       // g's waiting to run
175         G *gtail;
176         int32 gwait;    // number of g's waiting to run
177         int32 gcount;   // number of g's that are alive
178         int32 grunning; // number of g's running on cpu or in syscall
179
180         M *mhead;       // m's waiting for work
181         int32 mwait;    // number of m's waiting for work
182         int32 mcount;   // number of m's that have been created
183
184         volatile uint32 atomic; // atomic scheduling word (see below)
185
186         int32 profilehz;        // cpu profiling rate
187
188         bool init;  // running initialization
189         bool lockmain;  // init called runtime.LockOSThread
190
191         Note    stopped;        // one g can set waitstop and wait here for m's to stop
192 };
193
194 // The atomic word in sched is an atomic uint32 that
195 // holds these fields.
196 //
197 //      [15 bits] mcpu          number of m's executing on cpu
198 //      [15 bits] mcpumax       max number of m's allowed on cpu
199 //      [1 bit] waitstop        some g is waiting on stopped
200 //      [1 bit] gwaiting        gwait != 0
201 //
202 // These fields are the information needed by entersyscall
203 // and exitsyscall to decide whether to coordinate with the
204 // scheduler.  Packing them into a single machine word lets
205 // them use a fast path with a single atomic read/write and
206 // no lock/unlock.  This greatly reduces contention in
207 // syscall- or cgo-heavy multithreaded programs.
208 //
209 // Except for entersyscall and exitsyscall, the manipulations
210 // to these fields only happen while holding the schedlock,
211 // so the routines holding schedlock only need to worry about
212 // what entersyscall and exitsyscall do, not the other routines
213 // (which also use the schedlock).
214 //
215 // In particular, entersyscall and exitsyscall only read mcpumax,
216 // waitstop, and gwaiting.  They never write them.  Thus, writes to those
217 // fields can be done (holding schedlock) without fear of write conflicts.
218 // There may still be logic conflicts: for example, the set of waitstop must
219 // be conditioned on mcpu >= mcpumax or else the wait may be a
220 // spurious sleep.  The Promela model in proc.p verifies these accesses.
221 enum {
222         mcpuWidth = 15,
223         mcpuMask = (1<<mcpuWidth) - 1,
224         mcpuShift = 0,
225         mcpumaxShift = mcpuShift + mcpuWidth,
226         waitstopShift = mcpumaxShift + mcpuWidth,
227         gwaitingShift = waitstopShift+1,
228
229         // The max value of GOMAXPROCS is constrained
230         // by the max value we can store in the bit fields
231         // of the atomic word.  Reserve a few high values
232         // so that we can detect accidental decrement
233         // beyond zero.
234         maxgomaxprocs = mcpuMask - 10,
235 };
236
237 #define atomic_mcpu(v)          (((v)>>mcpuShift)&mcpuMask)
238 #define atomic_mcpumax(v)       (((v)>>mcpumaxShift)&mcpuMask)
239 #define atomic_waitstop(v)      (((v)>>waitstopShift)&1)
240 #define atomic_gwaiting(v)      (((v)>>gwaitingShift)&1)
241
242 Sched runtime_sched;
243 int32 runtime_gomaxprocs;
244 bool runtime_singleproc;
245
246 static bool canaddmcpu(void);
247
248 // An m that is waiting for notewakeup(&m->havenextg).  This may
249 // only be accessed while the scheduler lock is held.  This is used to
250 // minimize the number of times we call notewakeup while the scheduler
251 // lock is held, since the m will normally move quickly to lock the
252 // scheduler itself, producing lock contention.
253 static M* mwakeup;
254
255 // Scheduling helpers.  Sched must be locked.
256 static void gput(G*);   // put/get on ghead/gtail
257 static G* gget(void);
258 static void mput(M*);   // put/get on mhead
259 static M* mget(G*);
260 static void gfput(G*);  // put/get on gfree
261 static G* gfget(void);
262 static void matchmg(void);      // match m's to g's
263 static void readylocked(G*);    // ready, but sched is locked
264 static void mnextg(M*, G*);
265 static void mcommoninit(M*);
266
267 void
268 setmcpumax(uint32 n)
269 {
270         uint32 v, w;
271
272         for(;;) {
273                 v = runtime_sched.atomic;
274                 w = v;
275                 w &= ~(mcpuMask<<mcpumaxShift);
276                 w |= n<<mcpumaxShift;
277                 if(runtime_cas(&runtime_sched.atomic, v, w))
278                         break;
279         }
280 }
281
282 // First function run by a new goroutine.  This replaces gogocall.
283 static void
284 kickoff(void)
285 {
286         void (*fn)(void*);
287
288         fn = (void (*)(void*))(g->entry);
289         fn(g->param);
290         runtime_goexit();
291 }
292
293 // Switch context to a different goroutine.  This is like longjmp.
294 static void runtime_gogo(G*) __attribute__ ((noinline));
295 static void
296 runtime_gogo(G* newg)
297 {
298 #ifdef USING_SPLIT_STACK
299         __splitstack_setcontext(&newg->stack_context[0]);
300 #endif
301         g = newg;
302         newg->fromgogo = true;
303         fixcontext(&newg->context);
304         setcontext(&newg->context);
305         runtime_throw("gogo setcontext returned");
306 }
307
308 // Save context and call fn passing g as a parameter.  This is like
309 // setjmp.  Because getcontext always returns 0, unlike setjmp, we use
310 // g->fromgogo as a code.  It will be true if we got here via
311 // setcontext.  g == nil the first time this is called in a new m.
312 static void runtime_mcall(void (*)(G*)) __attribute__ ((noinline));
313 static void
314 runtime_mcall(void (*pfn)(G*))
315 {
316         M *mp;
317         G *gp;
318 #ifndef USING_SPLIT_STACK
319         int i;
320 #endif
321
322         // Ensure that all registers are on the stack for the garbage
323         // collector.
324         __builtin_unwind_init();
325
326         mp = m;
327         gp = g;
328         if(gp == mp->g0)
329                 runtime_throw("runtime: mcall called on m->g0 stack");
330
331         if(gp != nil) {
332
333 #ifdef USING_SPLIT_STACK
334                 __splitstack_getcontext(&g->stack_context[0]);
335 #else
336                 gp->gcnext_sp = &i;
337 #endif
338                 gp->fromgogo = false;
339                 getcontext(&gp->context);
340
341                 // When we return from getcontext, we may be running
342                 // in a new thread.  That means that m and g may have
343                 // changed.  They are global variables so we will
344                 // reload them, but the addresses of m and g may be
345                 // cached in our local stack frame, and those
346                 // addresses may be wrong.  Call functions to reload
347                 // the values for this thread.
348                 mp = runtime_m();
349                 gp = runtime_g();
350
351                 if(gp->traceback != nil)
352                         gtraceback(gp);
353         }
354         if (gp == nil || !gp->fromgogo) {
355 #ifdef USING_SPLIT_STACK
356                 __splitstack_setcontext(&mp->g0->stack_context[0]);
357 #endif
358                 mp->g0->entry = (byte*)pfn;
359                 mp->g0->param = gp;
360
361                 // It's OK to set g directly here because this case
362                 // can not occur if we got here via a setcontext to
363                 // the getcontext call just above.
364                 g = mp->g0;
365
366                 fixcontext(&mp->g0->context);
367                 setcontext(&mp->g0->context);
368                 runtime_throw("runtime: mcall function returned");
369         }
370 }
371
372 // Keep trace of scavenger's goroutine for deadlock detection.
373 static G *scvg;
374
375 // The bootstrap sequence is:
376 //
377 //      call osinit
378 //      call schedinit
379 //      make & queue new G
380 //      call runtime_mstart
381 //
382 // The new G calls runtime_main.
383 void
384 runtime_schedinit(void)
385 {
386         int32 n;
387         const byte *p;
388
389         m = &runtime_m0;
390         g = &runtime_g0;
391         m->g0 = g;
392         m->curg = g;
393         g->m = m;
394
395         initcontext();
396
397         m->nomemprof++;
398         runtime_mallocinit();
399         mcommoninit(m);
400
401         runtime_goargs();
402         runtime_goenvs();
403
404         // For debugging:
405         // Allocate internal symbol table representation now,
406         // so that we don't need to call malloc when we crash.
407         // runtime_findfunc(0);
408
409         runtime_gomaxprocs = 1;
410         p = runtime_getenv("GOMAXPROCS");
411         if(p != nil && (n = runtime_atoi(p)) != 0) {
412                 if(n > maxgomaxprocs)
413                         n = maxgomaxprocs;
414                 runtime_gomaxprocs = n;
415         }
416         // wait for the main goroutine to start before taking
417         // GOMAXPROCS into account.
418         setmcpumax(1);
419         runtime_singleproc = runtime_gomaxprocs == 1;
420
421         canaddmcpu();   // mcpu++ to account for bootstrap m
422         m->helpgc = 1;  // flag to tell schedule() to mcpu--
423         runtime_sched.grunning++;
424
425         // Can not enable GC until all roots are registered.
426         // mstats.enablegc = 1;
427         m->nomemprof--;
428 }
429
430 extern void main_init(void) __asm__ ("__go_init_main");
431 extern void main_main(void) __asm__ ("main.main");
432
433 // The main goroutine.
434 void
435 runtime_main(void)
436 {
437         // Lock the main goroutine onto this, the main OS thread,
438         // during initialization.  Most programs won't care, but a few
439         // do require certain calls to be made by the main thread.
440         // Those can arrange for main.main to run in the main thread
441         // by calling runtime.LockOSThread during initialization
442         // to preserve the lock.
443         runtime_LockOSThread();
444         // From now on, newgoroutines may use non-main threads.
445         setmcpumax(runtime_gomaxprocs);
446         runtime_sched.init = true;
447         scvg = __go_go(runtime_MHeap_Scavenger, nil);
448         main_init();
449         runtime_sched.init = false;
450         if(!runtime_sched.lockmain)
451                 runtime_UnlockOSThread();
452
453         // For gccgo we have to wait until after main is initialized
454         // to enable GC, because initializing main registers the GC
455         // roots.
456         mstats.enablegc = 1;
457
458         // The deadlock detection has false negatives.
459         // Let scvg start up, to eliminate the false negative
460         // for the trivial program func main() { select{} }.
461         runtime_gosched();
462
463         main_main();
464         runtime_exit(0);
465         for(;;)
466                 *(int32*)0 = 0;
467 }
468
469 // Lock the scheduler.
470 static void
471 schedlock(void)
472 {
473         runtime_lock(&runtime_sched);
474 }
475
476 // Unlock the scheduler.
477 static void
478 schedunlock(void)
479 {
480         M *m;
481
482         m = mwakeup;
483         mwakeup = nil;
484         runtime_unlock(&runtime_sched);
485         if(m != nil)
486                 runtime_notewakeup(&m->havenextg);
487 }
488
489 void
490 runtime_goexit(void)
491 {
492         g->status = Gmoribund;
493         runtime_gosched();
494 }
495
496 void
497 runtime_goroutineheader(G *g)
498 {
499         const char *status;
500
501         switch(g->status) {
502         case Gidle:
503                 status = "idle";
504                 break;
505         case Grunnable:
506                 status = "runnable";
507                 break;
508         case Grunning:
509                 status = "running";
510                 break;
511         case Gsyscall:
512                 status = "syscall";
513                 break;
514         case Gwaiting:
515                 if(g->waitreason)
516                         status = g->waitreason;
517                 else
518                         status = "waiting";
519                 break;
520         case Gmoribund:
521                 status = "moribund";
522                 break;
523         default:
524                 status = "???";
525                 break;
526         }
527         runtime_printf("goroutine %d [%s]:\n", g->goid, status);
528 }
529
530 void
531 runtime_goroutinetrailer(G *g)
532 {
533         if(g != nil && g->gopc != 0 && g->goid != 1) {
534                 struct __go_string fn;
535                 struct __go_string file;
536                 int line;
537
538                 if(__go_file_line(g->gopc - 1, &fn, &file, &line)) {
539                         runtime_printf("created by %s\n", fn.__data);
540                         runtime_printf("\t%s:%d\n", file.__data, line);
541                 }
542         }
543 }
544
545 struct Traceback
546 {
547         G* gp;
548         uintptr pcbuf[100];
549         int32 c;
550 };
551
552 void
553 runtime_tracebackothers(G * volatile me)
554 {
555         G * volatile g;
556         Traceback traceback;
557
558         traceback.gp = me;
559         for(g = runtime_allg; g != nil; g = g->alllink) {
560                 if(g == me || g->status == Gdead)
561                         continue;
562                 runtime_printf("\n");
563                 runtime_goroutineheader(g);
564
565                 // Our only mechanism for doing a stack trace is
566                 // _Unwind_Backtrace.  And that only works for the
567                 // current thread, not for other random goroutines.
568                 // So we need to switch context to the goroutine, get
569                 // the backtrace, and then switch back.
570
571                 // This means that if g is running or in a syscall, we
572                 // can't reliably print a stack trace.  FIXME.
573                 if(g->status == Gsyscall || g->status == Grunning) {
574                         runtime_printf("no stack trace available\n");
575                         runtime_goroutinetrailer(g);
576                         continue;
577                 }
578
579                 g->traceback = &traceback;
580
581 #ifdef USING_SPLIT_STACK
582                 __splitstack_getcontext(&me->stack_context[0]);
583 #endif
584                 getcontext(&me->context);
585
586                 if(g->traceback != nil) {
587                         runtime_gogo(g);
588                 }
589
590                 runtime_printtrace(traceback.pcbuf, traceback.c);
591                 runtime_goroutinetrailer(g);
592         }
593 }
594
595 // Do a stack trace of gp, and then restore the context to
596 // gp->dotraceback.
597
598 static void
599 gtraceback(G* gp)
600 {
601         Traceback* traceback;
602
603         traceback = gp->traceback;
604         gp->traceback = nil;
605         traceback->c = runtime_callers(1, traceback->pcbuf,
606                 sizeof traceback->pcbuf / sizeof traceback->pcbuf[0]);
607         runtime_gogo(traceback->gp);
608 }
609
610 // Mark this g as m's idle goroutine.
611 // This functionality might be used in environments where programs
612 // are limited to a single thread, to simulate a select-driven
613 // network server.  It is not exposed via the standard runtime API.
614 void
615 runtime_idlegoroutine(void)
616 {
617         if(g->idlem != nil)
618                 runtime_throw("g is already an idle goroutine");
619         g->idlem = m;
620 }
621
622 static void
623 mcommoninit(M *m)
624 {
625         m->id = runtime_sched.mcount++;
626         m->fastrand = 0x49f6428aUL + m->id + runtime_cputicks();
627
628         if(m->mcache == nil)
629                 m->mcache = runtime_allocmcache();
630
631         runtime_callers(1, m->createstack, nelem(m->createstack));
632
633         // Add to runtime_allm so garbage collector doesn't free m
634         // when it is just in a register or thread-local storage.
635         m->alllink = runtime_allm;
636         // runtime_NumCgoCall() iterates over allm w/o schedlock,
637         // so we need to publish it safely.
638         runtime_atomicstorep(&runtime_allm, m);
639 }
640
641 // Try to increment mcpu.  Report whether succeeded.
642 static bool
643 canaddmcpu(void)
644 {
645         uint32 v;
646
647         for(;;) {
648                 v = runtime_sched.atomic;
649                 if(atomic_mcpu(v) >= atomic_mcpumax(v))
650                         return 0;
651                 if(runtime_cas(&runtime_sched.atomic, v, v+(1<<mcpuShift)))
652                         return 1;
653         }
654 }
655
656 // Put on `g' queue.  Sched must be locked.
657 static void
658 gput(G *g)
659 {
660         M *m;
661
662         // If g is wired, hand it off directly.
663         if((m = g->lockedm) != nil && canaddmcpu()) {
664                 mnextg(m, g);
665                 return;
666         }
667
668         // If g is the idle goroutine for an m, hand it off.
669         if(g->idlem != nil) {
670                 if(g->idlem->idleg != nil) {
671                         runtime_printf("m%d idle out of sync: g%d g%d\n",
672                                 g->idlem->id,
673                                 g->idlem->idleg->goid, g->goid);
674                         runtime_throw("runtime: double idle");
675                 }
676                 g->idlem->idleg = g;
677                 return;
678         }
679
680         g->schedlink = nil;
681         if(runtime_sched.ghead == nil)
682                 runtime_sched.ghead = g;
683         else
684                 runtime_sched.gtail->schedlink = g;
685         runtime_sched.gtail = g;
686
687         // increment gwait.
688         // if it transitions to nonzero, set atomic gwaiting bit.
689         if(runtime_sched.gwait++ == 0)
690                 runtime_xadd(&runtime_sched.atomic, 1<<gwaitingShift);
691 }
692
693 // Report whether gget would return something.
694 static bool
695 haveg(void)
696 {
697         return runtime_sched.ghead != nil || m->idleg != nil;
698 }
699
700 // Get from `g' queue.  Sched must be locked.
701 static G*
702 gget(void)
703 {
704         G *g;
705
706         g = runtime_sched.ghead;
707         if(g){
708                 runtime_sched.ghead = g->schedlink;
709                 if(runtime_sched.ghead == nil)
710                         runtime_sched.gtail = nil;
711                 // decrement gwait.
712                 // if it transitions to zero, clear atomic gwaiting bit.
713                 if(--runtime_sched.gwait == 0)
714                         runtime_xadd(&runtime_sched.atomic, -1<<gwaitingShift);
715         } else if(m->idleg != nil) {
716                 g = m->idleg;
717                 m->idleg = nil;
718         }
719         return g;
720 }
721
722 // Put on `m' list.  Sched must be locked.
723 static void
724 mput(M *m)
725 {
726         m->schedlink = runtime_sched.mhead;
727         runtime_sched.mhead = m;
728         runtime_sched.mwait++;
729 }
730
731 // Get an `m' to run `g'.  Sched must be locked.
732 static M*
733 mget(G *g)
734 {
735         M *m;
736
737         // if g has its own m, use it.
738         if(g && (m = g->lockedm) != nil)
739                 return m;
740
741         // otherwise use general m pool.
742         if((m = runtime_sched.mhead) != nil){
743                 runtime_sched.mhead = m->schedlink;
744                 runtime_sched.mwait--;
745         }
746         return m;
747 }
748
749 // Mark g ready to run.
750 void
751 runtime_ready(G *g)
752 {
753         schedlock();
754         readylocked(g);
755         schedunlock();
756 }
757
758 // Mark g ready to run.  Sched is already locked.
759 // G might be running already and about to stop.
760 // The sched lock protects g->status from changing underfoot.
761 static void
762 readylocked(G *g)
763 {
764         if(g->m){
765                 // Running on another machine.
766                 // Ready it when it stops.
767                 g->readyonstop = 1;
768                 return;
769         }
770
771         // Mark runnable.
772         if(g->status == Grunnable || g->status == Grunning) {
773                 runtime_printf("goroutine %d has status %d\n", g->goid, g->status);
774                 runtime_throw("bad g->status in ready");
775         }
776         g->status = Grunnable;
777
778         gput(g);
779         matchmg();
780 }
781
782 // Same as readylocked but a different symbol so that
783 // debuggers can set a breakpoint here and catch all
784 // new goroutines.
785 static void
786 newprocreadylocked(G *g)
787 {
788         readylocked(g);
789 }
790
791 // Pass g to m for running.
792 // Caller has already incremented mcpu.
793 static void
794 mnextg(M *m, G *g)
795 {
796         runtime_sched.grunning++;
797         m->nextg = g;
798         if(m->waitnextg) {
799                 m->waitnextg = 0;
800                 if(mwakeup != nil)
801                         runtime_notewakeup(&mwakeup->havenextg);
802                 mwakeup = m;
803         }
804 }
805
806 // Get the next goroutine that m should run.
807 // Sched must be locked on entry, is unlocked on exit.
808 // Makes sure that at most $GOMAXPROCS g's are
809 // running on cpus (not in system calls) at any given time.
810 static G*
811 nextgandunlock(void)
812 {
813         G *gp;
814         uint32 v;
815
816 top:
817         if(atomic_mcpu(runtime_sched.atomic) >= maxgomaxprocs)
818                 runtime_throw("negative mcpu");
819
820         // If there is a g waiting as m->nextg, the mcpu++
821         // happened before it was passed to mnextg.
822         if(m->nextg != nil) {
823                 gp = m->nextg;
824                 m->nextg = nil;
825                 schedunlock();
826                 return gp;
827         }
828
829         if(m->lockedg != nil) {
830                 // We can only run one g, and it's not available.
831                 // Make sure some other cpu is running to handle
832                 // the ordinary run queue.
833                 if(runtime_sched.gwait != 0) {
834                         matchmg();
835                         // m->lockedg might have been on the queue.
836                         if(m->nextg != nil) {
837                                 gp = m->nextg;
838                                 m->nextg = nil;
839                                 schedunlock();
840                                 return gp;
841                         }
842                 }
843         } else {
844                 // Look for work on global queue.
845                 while(haveg() && canaddmcpu()) {
846                         gp = gget();
847                         if(gp == nil)
848                                 runtime_throw("gget inconsistency");
849
850                         if(gp->lockedm) {
851                                 mnextg(gp->lockedm, gp);
852                                 continue;
853                         }
854                         runtime_sched.grunning++;
855                         schedunlock();
856                         return gp;
857                 }
858
859                 // The while loop ended either because the g queue is empty
860                 // or because we have maxed out our m procs running go
861                 // code (mcpu >= mcpumax).  We need to check that
862                 // concurrent actions by entersyscall/exitsyscall cannot
863                 // invalidate the decision to end the loop.
864                 //
865                 // We hold the sched lock, so no one else is manipulating the
866                 // g queue or changing mcpumax.  Entersyscall can decrement
867                 // mcpu, but if does so when there is something on the g queue,
868                 // the gwait bit will be set, so entersyscall will take the slow path
869                 // and use the sched lock.  So it cannot invalidate our decision.
870                 //
871                 // Wait on global m queue.
872                 mput(m);
873         }
874
875         // Look for deadlock situation.
876         // There is a race with the scavenger that causes false negatives:
877         // if the scavenger is just starting, then we have
878         //      scvg != nil && grunning == 0 && gwait == 0
879         // and we do not detect a deadlock.  It is possible that we should
880         // add that case to the if statement here, but it is too close to Go 1
881         // to make such a subtle change.  Instead, we work around the
882         // false negative in trivial programs by calling runtime.gosched
883         // from the main goroutine just before main.main.
884         // See runtime_main above.
885         //
886         // On a related note, it is also possible that the scvg == nil case is
887         // wrong and should include gwait, but that does not happen in
888         // standard Go programs, which all start the scavenger.
889         //
890         if((scvg == nil && runtime_sched.grunning == 0) ||
891            (scvg != nil && runtime_sched.grunning == 1 && runtime_sched.gwait == 0 &&
892             (scvg->status == Grunning || scvg->status == Gsyscall))) {
893                 runtime_throw("all goroutines are asleep - deadlock!");
894         }
895
896         m->nextg = nil;
897         m->waitnextg = 1;
898         runtime_noteclear(&m->havenextg);
899
900         // Stoptheworld is waiting for all but its cpu to go to stop.
901         // Entersyscall might have decremented mcpu too, but if so
902         // it will see the waitstop and take the slow path.
903         // Exitsyscall never increments mcpu beyond mcpumax.
904         v = runtime_atomicload(&runtime_sched.atomic);
905         if(atomic_waitstop(v) && atomic_mcpu(v) <= atomic_mcpumax(v)) {
906                 // set waitstop = 0 (known to be 1)
907                 runtime_xadd(&runtime_sched.atomic, -1<<waitstopShift);
908                 runtime_notewakeup(&runtime_sched.stopped);
909         }
910         schedunlock();
911
912         runtime_notesleep(&m->havenextg);
913         if(m->helpgc) {
914                 runtime_gchelper();
915                 m->helpgc = 0;
916                 runtime_lock(&runtime_sched);
917                 goto top;
918         }
919         if((gp = m->nextg) == nil)
920                 runtime_throw("bad m->nextg in nextgoroutine");
921         m->nextg = nil;
922         return gp;
923 }
924
925 int32
926 runtime_helpgc(bool *extra)
927 {
928         M *mp;
929         int32 n, max;
930
931         // Figure out how many CPUs to use.
932         // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc.
933         max = runtime_gomaxprocs;
934         if(max > runtime_ncpu)
935                 max = runtime_ncpu > 0 ? runtime_ncpu : 1;
936         if(max > MaxGcproc)
937                 max = MaxGcproc;
938
939         // We're going to use one CPU no matter what.
940         // Figure out the max number of additional CPUs.
941         max--;
942
943         runtime_lock(&runtime_sched);
944         n = 0;
945         while(n < max && (mp = mget(nil)) != nil) {
946                 n++;
947                 mp->helpgc = 1;
948                 mp->waitnextg = 0;
949                 runtime_notewakeup(&mp->havenextg);
950         }
951         runtime_unlock(&runtime_sched);
952         if(extra)
953                 *extra = n != max;
954         return n;
955 }
956
957 void
958 runtime_stoptheworld(void)
959 {
960         uint32 v;
961
962         schedlock();
963         runtime_gcwaiting = 1;
964
965         setmcpumax(1);
966
967         // while mcpu > 1
968         for(;;) {
969                 v = runtime_sched.atomic;
970                 if(atomic_mcpu(v) <= 1)
971                         break;
972
973                 // It would be unsafe for multiple threads to be using
974                 // the stopped note at once, but there is only
975                 // ever one thread doing garbage collection.
976                 runtime_noteclear(&runtime_sched.stopped);
977                 if(atomic_waitstop(v))
978                         runtime_throw("invalid waitstop");
979
980                 // atomic { waitstop = 1 }, predicated on mcpu <= 1 check above
981                 // still being true.
982                 if(!runtime_cas(&runtime_sched.atomic, v, v+(1<<waitstopShift)))
983                         continue;
984
985                 schedunlock();
986                 runtime_notesleep(&runtime_sched.stopped);
987                 schedlock();
988         }
989         runtime_singleproc = runtime_gomaxprocs == 1;
990         schedunlock();
991 }
992
993 void
994 runtime_starttheworld(bool extra)
995 {
996         M *m;
997
998         schedlock();
999         runtime_gcwaiting = 0;
1000         setmcpumax(runtime_gomaxprocs);
1001         matchmg();
1002         if(extra && canaddmcpu()) {
1003                 // Start a new m that will (we hope) be idle
1004                 // and so available to help when the next
1005                 // garbage collection happens.
1006                 // canaddmcpu above did mcpu++
1007                 // (necessary, because m will be doing various
1008                 // initialization work so is definitely running),
1009                 // but m is not running a specific goroutine,
1010                 // so set the helpgc flag as a signal to m's
1011                 // first schedule(nil) to mcpu-- and grunning--.
1012                 m = runtime_newm();
1013                 m->helpgc = 1;
1014                 runtime_sched.grunning++;
1015         }
1016         schedunlock();
1017 }
1018
1019 // Called to start an M.
1020 void*
1021 runtime_mstart(void* mp)
1022 {
1023         m = (M*)mp;
1024         g = m->g0;
1025
1026         initcontext();
1027
1028         g->entry = nil;
1029         g->param = nil;
1030
1031         // Record top of stack for use by mcall.
1032         // Once we call schedule we're never coming back,
1033         // so other calls can reuse this stack space.
1034 #ifdef USING_SPLIT_STACK
1035         __splitstack_getcontext(&g->stack_context[0]);
1036 #else
1037         g->gcinitial_sp = &mp;
1038         // Setting gcstack_size to 0 is a marker meaning that gcinitial_sp
1039         // is the top of the stack, not the bottom.
1040         g->gcstack_size = 0;
1041         g->gcnext_sp = &mp;
1042 #endif
1043         getcontext(&g->context);
1044
1045         if(g->entry != nil) {
1046                 // Got here from mcall.
1047                 void (*pfn)(G*) = (void (*)(G*))g->entry;
1048                 G* gp = (G*)g->param;
1049                 pfn(gp);
1050                 *(int*)0x21 = 0x21;
1051         }
1052         runtime_minit();
1053
1054 #ifdef USING_SPLIT_STACK
1055         {
1056           int dont_block_signals = 0;
1057           __splitstack_block_signals(&dont_block_signals, nil);
1058         }
1059 #endif
1060
1061         // Install signal handlers; after minit so that minit can
1062         // prepare the thread to be able to handle the signals.
1063         if(m == &runtime_m0)
1064                 runtime_initsig();
1065
1066         schedule(nil);
1067         return nil;
1068 }
1069
1070 typedef struct CgoThreadStart CgoThreadStart;
1071 struct CgoThreadStart
1072 {
1073         M *m;
1074         G *g;
1075         void (*fn)(void);
1076 };
1077
1078 // Kick off new m's as needed (up to mcpumax).
1079 // Sched is locked.
1080 static void
1081 matchmg(void)
1082 {
1083         G *gp;
1084         M *mp;
1085
1086         if(m->mallocing || m->gcing)
1087                 return;
1088
1089         while(haveg() && canaddmcpu()) {
1090                 gp = gget();
1091                 if(gp == nil)
1092                         runtime_throw("gget inconsistency");
1093
1094                 // Find the m that will run gp.
1095                 if((mp = mget(gp)) == nil)
1096                         mp = runtime_newm();
1097                 mnextg(mp, gp);
1098         }
1099 }
1100
1101 // Create a new m.  It will start off with a call to runtime_mstart.
1102 M*
1103 runtime_newm(void)
1104 {
1105         M *m;
1106         pthread_attr_t attr;
1107         pthread_t tid;
1108         size_t stacksize;
1109
1110         m = runtime_malloc(sizeof(M));
1111         mcommoninit(m);
1112         m->g0 = runtime_malg(-1, nil, nil);
1113
1114         if(pthread_attr_init(&attr) != 0)
1115                 runtime_throw("pthread_attr_init");
1116         if(pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0)
1117                 runtime_throw("pthread_attr_setdetachstate");
1118
1119 #ifndef PTHREAD_STACK_MIN
1120 #define PTHREAD_STACK_MIN 8192
1121 #endif
1122
1123         stacksize = PTHREAD_STACK_MIN;
1124
1125 #if 0
1126 #ifdef HAVE__DL_GET_TLS_STATIC_INFO
1127         {
1128                 /* On GNU/Linux the static TLS size is taken out of
1129                    the stack size, and we get an error or a crash if
1130                    there is not enough stack space left.  Add it back
1131                    in if we can, in case the program uses a lot of TLS
1132                    space.  */
1133 #ifndef internal_function
1134 #ifdef __i386__
1135 #define internal_function __attribute__ ((regparm (3), stdcall))
1136 #else
1137 #define internal_function
1138 #endif
1139 #endif
1140                 extern void _dl_get_tls_static_info(size_t*, size_t*) internal_function;
1141                 size_t tlssize, tlsalign;
1142                 _dl_get_tls_static_info(&tlssize, &tlsalign);
1143                 stacksize += tlssize;
1144         }
1145 #endif
1146 #endif
1147
1148         if(pthread_attr_setstacksize(&attr, stacksize) != 0)
1149                 runtime_throw("pthread_attr_setstacksize");
1150
1151         if(pthread_create(&tid, &attr, runtime_mstart, m) != 0)
1152                 runtime_throw("pthread_create");
1153
1154         return m;
1155 }
1156
1157 // One round of scheduler: find a goroutine and run it.
1158 // The argument is the goroutine that was running before
1159 // schedule was called, or nil if this is the first call.
1160 // Never returns.
1161 static void
1162 schedule(G *gp)
1163 {
1164         int32 hz;
1165         uint32 v;
1166
1167         schedlock();
1168         if(gp != nil) {
1169                 // Just finished running gp.
1170                 gp->m = nil;
1171                 runtime_sched.grunning--;
1172
1173                 // atomic { mcpu-- }
1174                 v = runtime_xadd(&runtime_sched.atomic, -1<<mcpuShift);
1175                 if(atomic_mcpu(v) > maxgomaxprocs)
1176                         runtime_throw("negative mcpu in scheduler");
1177
1178                 switch(gp->status){
1179                 case Grunnable:
1180                 case Gdead:
1181                         // Shouldn't have been running!
1182                         runtime_throw("bad gp->status in sched");
1183                 case Grunning:
1184                         gp->status = Grunnable;
1185                         gput(gp);
1186                         break;
1187                 case Gmoribund:
1188                         gp->status = Gdead;
1189                         if(gp->lockedm) {
1190                                 gp->lockedm = nil;
1191                                 m->lockedg = nil;
1192                         }
1193                         gp->idlem = nil;
1194                         runtime_memclr(&gp->context, sizeof gp->context);
1195                         gfput(gp);
1196                         if(--runtime_sched.gcount == 0)
1197                                 runtime_exit(0);
1198                         break;
1199                 }
1200                 if(gp->readyonstop){
1201                         gp->readyonstop = 0;
1202                         readylocked(gp);
1203                 }
1204         } else if(m->helpgc) {
1205                 // Bootstrap m or new m started by starttheworld.
1206                 // atomic { mcpu-- }
1207                 v = runtime_xadd(&runtime_sched.atomic, -1<<mcpuShift);
1208                 if(atomic_mcpu(v) > maxgomaxprocs)
1209                         runtime_throw("negative mcpu in scheduler");
1210                 // Compensate for increment in starttheworld().
1211                 runtime_sched.grunning--;
1212                 m->helpgc = 0;
1213         } else if(m->nextg != nil) {
1214                 // New m started by matchmg.
1215         } else {
1216                 runtime_throw("invalid m state in scheduler");
1217         }
1218
1219         // Find (or wait for) g to run.  Unlocks runtime_sched.
1220         gp = nextgandunlock();
1221         gp->readyonstop = 0;
1222         gp->status = Grunning;
1223         m->curg = gp;
1224         gp->m = m;
1225
1226         // Check whether the profiler needs to be turned on or off.
1227         hz = runtime_sched.profilehz;
1228         if(m->profilehz != hz)
1229                 runtime_resetcpuprofiler(hz);
1230
1231         runtime_gogo(gp);
1232 }
1233
1234 // Enter scheduler.  If g->status is Grunning,
1235 // re-queues g and runs everyone else who is waiting
1236 // before running g again.  If g->status is Gmoribund,
1237 // kills off g.
1238 void
1239 runtime_gosched(void)
1240 {
1241         if(m->locks != 0)
1242                 runtime_throw("gosched holding locks");
1243         if(g == m->g0)
1244                 runtime_throw("gosched of g0");
1245         runtime_mcall(schedule);
1246 }
1247
1248 // The goroutine g is about to enter a system call.
1249 // Record that it's not using the cpu anymore.
1250 // This is called only from the go syscall library and cgocall,
1251 // not from the low-level system calls used by the runtime.
1252 //
1253 // Entersyscall cannot split the stack: the runtime_gosave must
1254 // make g->sched refer to the caller's stack segment, because
1255 // entersyscall is going to return immediately after.
1256 // It's okay to call matchmg and notewakeup even after
1257 // decrementing mcpu, because we haven't released the
1258 // sched lock yet, so the garbage collector cannot be running.
1259
1260 void runtime_entersyscall(void) __attribute__ ((no_split_stack));
1261
1262 void
1263 runtime_entersyscall(void)
1264 {
1265         uint32 v;
1266
1267         if(m->profilehz > 0)
1268                 runtime_setprof(false);
1269
1270         // Leave SP around for gc and traceback.
1271 #ifdef USING_SPLIT_STACK
1272         g->gcstack = __splitstack_find(nil, nil, &g->gcstack_size,
1273                                        &g->gcnext_segment, &g->gcnext_sp,
1274                                        &g->gcinitial_sp);
1275 #else
1276         g->gcnext_sp = (byte *) &v;
1277 #endif
1278
1279         // Save the registers in the g structure so that any pointers
1280         // held in registers will be seen by the garbage collector.
1281         getcontext(&g->gcregs);
1282
1283         g->status = Gsyscall;
1284
1285         // Fast path.
1286         // The slow path inside the schedlock/schedunlock will get
1287         // through without stopping if it does:
1288         //      mcpu--
1289         //      gwait not true
1290         //      waitstop && mcpu <= mcpumax not true
1291         // If we can do the same with a single atomic add,
1292         // then we can skip the locks.
1293         v = runtime_xadd(&runtime_sched.atomic, -1<<mcpuShift);
1294         if(!atomic_gwaiting(v) && (!atomic_waitstop(v) || atomic_mcpu(v) > atomic_mcpumax(v)))
1295                 return;
1296
1297         schedlock();
1298         v = runtime_atomicload(&runtime_sched.atomic);
1299         if(atomic_gwaiting(v)) {
1300                 matchmg();
1301                 v = runtime_atomicload(&runtime_sched.atomic);
1302         }
1303         if(atomic_waitstop(v) && atomic_mcpu(v) <= atomic_mcpumax(v)) {
1304                 runtime_xadd(&runtime_sched.atomic, -1<<waitstopShift);
1305                 runtime_notewakeup(&runtime_sched.stopped);
1306         }
1307
1308         schedunlock();
1309 }
1310
1311 // The goroutine g exited its system call.
1312 // Arrange for it to run on a cpu again.
1313 // This is called only from the go syscall library, not
1314 // from the low-level system calls used by the runtime.
1315 void
1316 runtime_exitsyscall(void)
1317 {
1318         G *gp;
1319         uint32 v;
1320
1321         // Fast path.
1322         // If we can do the mcpu++ bookkeeping and
1323         // find that we still have mcpu <= mcpumax, then we can
1324         // start executing Go code immediately, without having to
1325         // schedlock/schedunlock.
1326         // Also do fast return if any locks are held, so that
1327         // panic code can use syscalls to open a file.
1328         gp = g;
1329         v = runtime_xadd(&runtime_sched.atomic, (1<<mcpuShift));
1330         if((m->profilehz == runtime_sched.profilehz && atomic_mcpu(v) <= atomic_mcpumax(v)) || m->locks > 0) {
1331                 // There's a cpu for us, so we can run.
1332                 gp->status = Grunning;
1333                 // Garbage collector isn't running (since we are),
1334                 // so okay to clear gcstack.
1335 #ifdef USING_SPLIT_STACK
1336                 gp->gcstack = nil;
1337 #endif
1338                 gp->gcnext_sp = nil;
1339                 runtime_memclr(&gp->gcregs, sizeof gp->gcregs);
1340
1341                 if(m->profilehz > 0)
1342                         runtime_setprof(true);
1343                 return;
1344         }
1345
1346         // Tell scheduler to put g back on the run queue:
1347         // mostly equivalent to g->status = Grunning,
1348         // but keeps the garbage collector from thinking
1349         // that g is running right now, which it's not.
1350         gp->readyonstop = 1;
1351
1352         // All the cpus are taken.
1353         // The scheduler will ready g and put this m to sleep.
1354         // When the scheduler takes g away from m,
1355         // it will undo the runtime_sched.mcpu++ above.
1356         runtime_gosched();
1357
1358         // Gosched returned, so we're allowed to run now.
1359         // Delete the gcstack information that we left for
1360         // the garbage collector during the system call.
1361         // Must wait until now because until gosched returns
1362         // we don't know for sure that the garbage collector
1363         // is not running.
1364 #ifdef USING_SPLIT_STACK
1365         gp->gcstack = nil;
1366 #endif
1367         gp->gcnext_sp = nil;
1368         runtime_memclr(&gp->gcregs, sizeof gp->gcregs);
1369 }
1370
1371 // Allocate a new g, with a stack big enough for stacksize bytes.
1372 G*
1373 runtime_malg(int32 stacksize, byte** ret_stack, size_t* ret_stacksize)
1374 {
1375         G *newg;
1376
1377         newg = runtime_malloc(sizeof(G));
1378         if(stacksize >= 0) {
1379 #if USING_SPLIT_STACK
1380                 int dont_block_signals = 0;
1381
1382                 *ret_stack = __splitstack_makecontext(stacksize,
1383                                                       &newg->stack_context[0],
1384                                                       ret_stacksize);
1385                 __splitstack_block_signals_context(&newg->stack_context[0],
1386                                                    &dont_block_signals, nil);
1387 #else
1388                 *ret_stack = runtime_mallocgc(stacksize, FlagNoProfiling|FlagNoGC, 0, 0);
1389                 *ret_stacksize = stacksize;
1390                 newg->gcinitial_sp = *ret_stack;
1391                 newg->gcstack_size = stacksize;
1392                 runtime_xadd(&runtime_stacks_sys, stacksize);
1393 #endif
1394         }
1395         return newg;
1396 }
1397
1398 /* For runtime package testing.  */
1399
1400 void runtime_testing_entersyscall(void)
1401   __asm__("runtime.entersyscall");
1402
1403 void
1404 runtime_testing_entersyscall()
1405 {
1406         runtime_entersyscall();
1407 }
1408
1409 void runtime_testing_exitsyscall(void)
1410   __asm__("runtime.exitsyscall");
1411
1412 void
1413 runtime_testing_exitsyscall()
1414 {
1415         runtime_exitsyscall();
1416 }
1417
1418 G*
1419 __go_go(void (*fn)(void*), void* arg)
1420 {
1421         byte *sp;
1422         size_t spsize;
1423         G *newg;
1424
1425         schedlock();
1426
1427         if((newg = gfget()) != nil){
1428 #ifdef USING_SPLIT_STACK
1429                 int dont_block_signals = 0;
1430
1431                 sp = __splitstack_resetcontext(&newg->stack_context[0],
1432                                                &spsize);
1433                 __splitstack_block_signals_context(&newg->stack_context[0],
1434                                                    &dont_block_signals, nil);
1435 #else
1436                 sp = newg->gcinitial_sp;
1437                 spsize = newg->gcstack_size;
1438                 if(spsize == 0)
1439                         runtime_throw("bad spsize in __go_go");
1440                 newg->gcnext_sp = sp;
1441 #endif
1442         } else {
1443                 newg = runtime_malg(StackMin, &sp, &spsize);
1444                 if(runtime_lastg == nil)
1445                         runtime_allg = newg;
1446                 else
1447                         runtime_lastg->alllink = newg;
1448                 runtime_lastg = newg;
1449         }
1450         newg->status = Gwaiting;
1451         newg->waitreason = "new goroutine";
1452
1453         newg->entry = (byte*)fn;
1454         newg->param = arg;
1455         newg->gopc = (uintptr)__builtin_return_address(0);
1456
1457         runtime_sched.gcount++;
1458         runtime_sched.goidgen++;
1459         newg->goid = runtime_sched.goidgen;
1460
1461         if(sp == nil)
1462                 runtime_throw("nil g->stack0");
1463
1464         {
1465                 // Avoid warnings about variables clobbered by
1466                 // longjmp.
1467                 byte * volatile vsp = sp;
1468                 size_t volatile vspsize = spsize;
1469                 G * volatile vnewg = newg;
1470
1471                 getcontext(&vnewg->context);
1472                 vnewg->context.uc_stack.ss_sp = vsp;
1473 #ifdef MAKECONTEXT_STACK_TOP
1474                 vnewg->context.uc_stack.ss_sp += vspsize;
1475 #endif
1476                 vnewg->context.uc_stack.ss_size = vspsize;
1477                 makecontext(&vnewg->context, kickoff, 0);
1478
1479                 newprocreadylocked(vnewg);
1480                 schedunlock();
1481
1482                 return vnewg;
1483         }
1484 }
1485
1486 // Put on gfree list.  Sched must be locked.
1487 static void
1488 gfput(G *g)
1489 {
1490         g->schedlink = runtime_sched.gfree;
1491         runtime_sched.gfree = g;
1492 }
1493
1494 // Get from gfree list.  Sched must be locked.
1495 static G*
1496 gfget(void)
1497 {
1498         G *g;
1499
1500         g = runtime_sched.gfree;
1501         if(g)
1502                 runtime_sched.gfree = g->schedlink;
1503         return g;
1504 }
1505
1506 // Run all deferred functions for the current goroutine.
1507 static void
1508 rundefer(void)
1509 {
1510         Defer *d;
1511
1512         while((d = g->defer) != nil) {
1513                 void (*pfn)(void*);
1514
1515                 pfn = d->__pfn;
1516                 d->__pfn = nil;
1517                 if (pfn != nil)
1518                         (*pfn)(d->__arg);
1519                 g->defer = d->__next;
1520                 runtime_free(d);
1521         }
1522 }
1523
1524 void runtime_Goexit (void) asm ("runtime.Goexit");
1525
1526 void
1527 runtime_Goexit(void)
1528 {
1529         rundefer();
1530         runtime_goexit();
1531 }
1532
1533 void runtime_Gosched (void) asm ("runtime.Gosched");
1534
1535 void
1536 runtime_Gosched(void)
1537 {
1538         runtime_gosched();
1539 }
1540
1541 // Implementation of runtime.GOMAXPROCS.
1542 // delete when scheduler is stronger
1543 int32
1544 runtime_gomaxprocsfunc(int32 n)
1545 {
1546         int32 ret;
1547         uint32 v;
1548
1549         schedlock();
1550         ret = runtime_gomaxprocs;
1551         if(n <= 0)
1552                 n = ret;
1553         if(n > maxgomaxprocs)
1554                 n = maxgomaxprocs;
1555         runtime_gomaxprocs = n;
1556         if(runtime_gomaxprocs > 1)
1557                 runtime_singleproc = false;
1558         if(runtime_gcwaiting != 0) {
1559                 if(atomic_mcpumax(runtime_sched.atomic) != 1)
1560                         runtime_throw("invalid mcpumax during gc");
1561                 schedunlock();
1562                 return ret;
1563         }
1564
1565         setmcpumax(n);
1566
1567         // If there are now fewer allowed procs
1568         // than procs running, stop.
1569         v = runtime_atomicload(&runtime_sched.atomic);
1570         if((int32)atomic_mcpu(v) > n) {
1571                 schedunlock();
1572                 runtime_gosched();
1573                 return ret;
1574         }
1575         // handle more procs
1576         matchmg();
1577         schedunlock();
1578         return ret;
1579 }
1580
1581 void
1582 runtime_LockOSThread(void)
1583 {
1584         if(m == &runtime_m0 && runtime_sched.init) {
1585                 runtime_sched.lockmain = true;
1586                 return;
1587         }
1588         m->lockedg = g;
1589         g->lockedm = m;
1590 }
1591
1592 void
1593 runtime_UnlockOSThread(void)
1594 {
1595         if(m == &runtime_m0 && runtime_sched.init) {
1596                 runtime_sched.lockmain = false;
1597                 return;
1598         }
1599         m->lockedg = nil;
1600         g->lockedm = nil;
1601 }
1602
1603 bool
1604 runtime_lockedOSThread(void)
1605 {
1606         return g->lockedm != nil && m->lockedg != nil;
1607 }
1608
1609 // for testing of callbacks
1610
1611 _Bool runtime_golockedOSThread(void)
1612   asm("runtime.golockedOSThread");
1613
1614 _Bool
1615 runtime_golockedOSThread(void)
1616 {
1617         return runtime_lockedOSThread();
1618 }
1619
1620 // for testing of wire, unwire
1621 uint32
1622 runtime_mid()
1623 {
1624         return m->id;
1625 }
1626
1627 int32 runtime_NumGoroutine (void)
1628   __asm__ ("runtime.NumGoroutine");
1629
1630 int32
1631 runtime_NumGoroutine()
1632 {
1633         return runtime_sched.gcount;
1634 }
1635
1636 int32
1637 runtime_gcount(void)
1638 {
1639         return runtime_sched.gcount;
1640 }
1641
1642 int32
1643 runtime_mcount(void)
1644 {
1645         return runtime_sched.mcount;
1646 }
1647
1648 static struct {
1649         Lock;
1650         void (*fn)(uintptr*, int32);
1651         int32 hz;
1652         uintptr pcbuf[100];
1653 } prof;
1654
1655 // Called if we receive a SIGPROF signal.
1656 void
1657 runtime_sigprof(uint8 *pc __attribute__ ((unused)),
1658                 uint8 *sp __attribute__ ((unused)),
1659                 uint8 *lr __attribute__ ((unused)),
1660                 G *gp __attribute__ ((unused)))
1661 {
1662         int32 n;
1663
1664         if(prof.fn == nil || prof.hz == 0)
1665                 return;
1666
1667         runtime_lock(&prof);
1668         if(prof.fn == nil) {
1669                 runtime_unlock(&prof);
1670                 return;
1671         }
1672         n = runtime_callers(0, prof.pcbuf, nelem(prof.pcbuf));
1673         if(n > 0)
1674                 prof.fn(prof.pcbuf, n);
1675         runtime_unlock(&prof);
1676 }
1677
1678 // Arrange to call fn with a traceback hz times a second.
1679 void
1680 runtime_setcpuprofilerate(void (*fn)(uintptr*, int32), int32 hz)
1681 {
1682         // Force sane arguments.
1683         if(hz < 0)
1684                 hz = 0;
1685         if(hz == 0)
1686                 fn = nil;
1687         if(fn == nil)
1688                 hz = 0;
1689
1690         // Stop profiler on this cpu so that it is safe to lock prof.
1691         // if a profiling signal came in while we had prof locked,
1692         // it would deadlock.
1693         runtime_resetcpuprofiler(0);
1694
1695         runtime_lock(&prof);
1696         prof.fn = fn;
1697         prof.hz = hz;
1698         runtime_unlock(&prof);
1699         runtime_lock(&runtime_sched);
1700         runtime_sched.profilehz = hz;
1701         runtime_unlock(&runtime_sched);
1702
1703         if(hz != 0)
1704                 runtime_resetcpuprofiler(hz);
1705 }