libgo/runtime/proc.c

   1 // Copyright 2009 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 #include <limits.h>
   6 #include <stdlib.h>
   7 #include <pthread.h>
   8 #include <unistd.h>
   9
  10 #include "config.h"
  11 #include "runtime.h"
  12 #include "arch.h"
  13 #include "defs.h"
  14 #include "malloc.h"
  15 #include "go-defer.h"
  16
  17 #ifdef USING_SPLIT_STACK
  18
  19 /* FIXME: These are not declared anywhere.  */
  20
  21 extern void __splitstack_getcontext(void *context[10]);
  22
  23 extern void __splitstack_setcontext(void *context[10]);
  24
  25 extern void *__splitstack_makecontext(size_t, void *context[10], size_t *);
  26
  27 extern void * __splitstack_resetcontext(void *context[10], size_t *);
  28
  29 extern void *__splitstack_find(void *, void *, size_t *, void **, void **,
  30                                void **);
  31
  32 extern void __splitstack_block_signals (int *, int *);
  33
  34 extern void __splitstack_block_signals_context (void *context[10], int *,
  35                                                 int *);
  36
  37 #endif
  38
  39 #if defined(USING_SPLIT_STACK) && defined(LINKER_SUPPORTS_SPLIT_STACK)
  40 # ifdef PTHREAD_STACK_MIN
  41 #  define StackMin PTHREAD_STACK_MIN
  42 # else
  43 #  define StackMin 8192
  44 # endif
  45 #else
  46 # define StackMin 2 * 1024 * 1024
  47 #endif
  48
  49 static void schedule(G*);
  50
  51 typedef struct Sched Sched;
  52
  53 M       runtime_m0;
  54 G       runtime_g0;     // idle goroutine for m0
  55
  56 #ifdef __rtems__
  57 #define __thread
  58 #endif
  59
  60 static __thread G *g;
  61 static __thread M *m;
  62
  63 #ifndef SETCONTEXT_CLOBBERS_TLS
  64
  65 static inline void
  66 initcontext(void)
  67 {
  68 }
  69
  70 static inline void
  71 fixcontext(ucontext_t *c __attribute__ ((unused)))
  72 {
  73 }
  74
  75 # else
  76
  77 # if defined(__x86_64__) && defined(__sun__)
  78
  79 // x86_64 Solaris 10 and 11 have a bug: setcontext switches the %fs
  80 // register to that of the thread which called getcontext.  The effect
  81 // is that the address of all __thread variables changes.  This bug
  82 // also affects pthread_self() and pthread_getspecific.  We work
  83 // around it by clobbering the context field directly to keep %fs the
  84 // same.
  85
  86 static __thread greg_t fs;
  87
  88 static inline void
  89 initcontext(void)
  90 {
  91         ucontext_t c;
  92
  93         getcontext(&c);
  94         fs = c.uc_mcontext.gregs[REG_FSBASE];
  95 }
  96
  97 static inline void
  98 fixcontext(ucontext_t* c)
  99 {
 100         c->uc_mcontext.gregs[REG_FSBASE] = fs;
 101 }
 102
 103 # else
 104
 105 #  error unknown case for SETCONTEXT_CLOBBERS_TLS
 106
 107 # endif
 108
 109 #endif
 110
 111 // We can not always refer to the TLS variables directly.  The
 112 // compiler will call tls_get_addr to get the address of the variable,
 113 // and it may hold it in a register across a call to schedule.  When
 114 // we get back from the call we may be running in a different thread,
 115 // in which case the register now points to the TLS variable for a
 116 // different thread.  We use non-inlinable functions to avoid this
 117 // when necessary.
 118
 119 G* runtime_g(void) __attribute__ ((noinline, no_split_stack));
 120
 121 G*
 122 runtime_g(void)
 123 {
 124         return g;
 125 }
 126
 127 M* runtime_m(void) __attribute__ ((noinline, no_split_stack));
 128
 129 M*
 130 runtime_m(void)
 131 {
 132         return m;
 133 }
 134
 135 int32   runtime_gcwaiting;
 136
 137 // Go scheduler
 138 //
 139 // The go scheduler's job is to match ready-to-run goroutines (`g's)
 140 // with waiting-for-work schedulers (`m's).  If there are ready g's
 141 // and no waiting m's, ready() will start a new m running in a new
 142 // OS thread, so that all ready g's can run simultaneously, up to a limit.
 143 // For now, m's never go away.
 144 //
 145 // By default, Go keeps only one kernel thread (m) running user code
 146 // at a single time; other threads may be blocked in the operating system.
 147 // Setting the environment variable $GOMAXPROCS or calling
 148 // runtime.GOMAXPROCS() will change the number of user threads
 149 // allowed to execute simultaneously.  $GOMAXPROCS is thus an
 150 // approximation of the maximum number of cores to use.
 151 //
 152 // Even a program that can run without deadlock in a single process
 153 // might use more m's if given the chance.  For example, the prime
 154 // sieve will use as many m's as there are primes (up to runtime_sched.mmax),
 155 // allowing different stages of the pipeline to execute in parallel.
 156 // We could revisit this choice, only kicking off new m's for blocking
 157 // system calls, but that would limit the amount of parallel computation
 158 // that go would try to do.
 159 //
 160 // In general, one could imagine all sorts of refinements to the
 161 // scheduler, but the goal now is just to get something working on
 162 // Linux and OS X.
 163
 164 struct Sched {
 165         Lock;
 166
 167         G *gfree;       // available g's (status == Gdead)
 168         int32 goidgen;
 169
 170         G *ghead;       // g's waiting to run
 171         G *gtail;
 172         int32 gwait;    // number of g's waiting to run
 173         int32 gcount;   // number of g's that are alive
 174         int32 grunning; // number of g's running on cpu or in syscall
 175
 176         M *mhead;       // m's waiting for work
 177         int32 mwait;    // number of m's waiting for work
 178         int32 mcount;   // number of m's that have been created
 179
 180         volatile uint32 atomic; // atomic scheduling word (see below)
 181
 182         int32 profilehz;        // cpu profiling rate
 183
 184         bool init;  // running initialization
 185         bool lockmain;  // init called runtime.LockOSThread
 186
 187         Note    stopped;        // one g can set waitstop and wait here for m's to stop
 188 };
 189
 190 // The atomic word in sched is an atomic uint32 that
 191 // holds these fields.
 192 //
 193 //      [15 bits] mcpu          number of m's executing on cpu
 194 //      [15 bits] mcpumax       max number of m's allowed on cpu
 195 //      [1 bit] waitstop        some g is waiting on stopped
 196 //      [1 bit] gwaiting        gwait != 0
 197 //
 198 // These fields are the information needed by entersyscall
 199 // and exitsyscall to decide whether to coordinate with the
 200 // scheduler.  Packing them into a single machine word lets
 201 // them use a fast path with a single atomic read/write and
 202 // no lock/unlock.  This greatly reduces contention in
 203 // syscall- or cgo-heavy multithreaded programs.
 204 //
 205 // Except for entersyscall and exitsyscall, the manipulations
 206 // to these fields only happen while holding the schedlock,
 207 // so the routines holding schedlock only need to worry about
 208 // what entersyscall and exitsyscall do, not the other routines
 209 // (which also use the schedlock).
 210 //
 211 // In particular, entersyscall and exitsyscall only read mcpumax,
 212 // waitstop, and gwaiting.  They never write them.  Thus, writes to those
 213 // fields can be done (holding schedlock) without fear of write conflicts.
 214 // There may still be logic conflicts: for example, the set of waitstop must
 215 // be conditioned on mcpu >= mcpumax or else the wait may be a
 216 // spurious sleep.  The Promela model in proc.p verifies these accesses.
 217 enum {
 218         mcpuWidth = 15,
 219         mcpuMask = (1<<mcpuWidth) - 1,
 220         mcpuShift = 0,
 221         mcpumaxShift = mcpuShift + mcpuWidth,
 222         waitstopShift = mcpumaxShift + mcpuWidth,
 223         gwaitingShift = waitstopShift+1,
 224
 225         // The max value of GOMAXPROCS is constrained
 226         // by the max value we can store in the bit fields
 227         // of the atomic word.  Reserve a few high values
 228         // so that we can detect accidental decrement
 229         // beyond zero.
 230         maxgomaxprocs = mcpuMask - 10,
 231 };
 232
 233 #define atomic_mcpu(v)          (((v)>>mcpuShift)&mcpuMask)
 234 #define atomic_mcpumax(v)       (((v)>>mcpumaxShift)&mcpuMask)
 235 #define atomic_waitstop(v)      (((v)>>waitstopShift)&1)
 236 #define atomic_gwaiting(v)      (((v)>>gwaitingShift)&1)
 237
 238 Sched runtime_sched;
 239 int32 runtime_gomaxprocs;
 240 bool runtime_singleproc;
 241
 242 static bool canaddmcpu(void);
 243
 244 // An m that is waiting for notewakeup(&m->havenextg).  This may
 245 // only be accessed while the scheduler lock is held.  This is used to
 246 // minimize the number of times we call notewakeup while the scheduler
 247 // lock is held, since the m will normally move quickly to lock the
 248 // scheduler itself, producing lock contention.
 249 static M* mwakeup;
 250
 251 // Scheduling helpers.  Sched must be locked.
 252 static void gput(G*);   // put/get on ghead/gtail
 253 static G* gget(void);
 254 static void mput(M*);   // put/get on mhead
 255 static M* mget(G*);
 256 static void gfput(G*);  // put/get on gfree
 257 static G* gfget(void);
 258 static void matchmg(void);      // match m's to g's
 259 static void readylocked(G*);    // ready, but sched is locked
 260 static void mnextg(M*, G*);
 261 static void mcommoninit(M*);
 262
 263 void
 264 setmcpumax(uint32 n)
 265 {
 266         uint32 v, w;
 267
 268         for(;;) {
 269                 v = runtime_sched.atomic;
 270                 w = v;
 271                 w &= ~(mcpuMask<<mcpumaxShift);
 272                 w |= n<<mcpumaxShift;
 273                 if(runtime_cas(&runtime_sched.atomic, v, w))
 274                         break;
 275         }
 276 }
 277
 278 // First function run by a new goroutine.  This replaces gogocall.
 279 static void
 280 kickoff(void)
 281 {
 282         void (*fn)(void*);
 283
 284         fn = (void (*)(void*))(g->entry);
 285         fn(g->param);
 286         runtime_goexit();
 287 }
 288
 289 // Switch context to a different goroutine.  This is like longjmp.
 290 static void runtime_gogo(G*) __attribute__ ((noinline));
 291 static void
 292 runtime_gogo(G* newg)
 293 {
 294 #ifdef USING_SPLIT_STACK
 295         __splitstack_setcontext(&newg->stack_context[0]);
 296 #endif
 297         g = newg;
 298         newg->fromgogo = true;
 299         fixcontext(&newg->context);
 300         setcontext(&newg->context);
 301         runtime_throw("gogo setcontext returned");
 302 }
 303
 304 // Save context and call fn passing g as a parameter.  This is like
 305 // setjmp.  Because getcontext always returns 0, unlike setjmp, we use
 306 // g->fromgogo as a code.  It will be true if we got here via
 307 // setcontext.  g == nil the first time this is called in a new m.
 308 static void runtime_mcall(void (*)(G*)) __attribute__ ((noinline));
 309 static void
 310 runtime_mcall(void (*pfn)(G*))
 311 {
 312 #ifndef USING_SPLIT_STACK
 313         int i;
 314 #endif
 315
 316         // Ensure that all registers are on the stack for the garbage
 317         // collector.
 318         __builtin_unwind_init();
 319
 320         if(g == m->g0)
 321                 runtime_throw("runtime: mcall called on m->g0 stack");
 322
 323         if(g != nil) {
 324
 325 #ifdef USING_SPLIT_STACK
 326                 __splitstack_getcontext(&g->stack_context[0]);
 327 #else
 328                 g->gcnext_sp = &i;
 329 #endif
 330                 g->fromgogo = false;
 331                 getcontext(&g->context);
 332         }
 333         if (g == nil || !g->fromgogo) {
 334 #ifdef USING_SPLIT_STACK
 335                 __splitstack_setcontext(&m->g0->stack_context[0]);
 336 #endif
 337                 m->g0->entry = (byte*)pfn;
 338                 m->g0->param = g;
 339                 g = m->g0;
 340                 fixcontext(&m->g0->context);
 341                 setcontext(&m->g0->context);
 342                 runtime_throw("runtime: mcall function returned");
 343         }
 344 }
 345
 346 // The bootstrap sequence is:
 347 //
 348 //      call osinit
 349 //      call schedinit
 350 //      make & queue new G
 351 //      call runtime_mstart
 352 //
 353 // The new G calls runtime_main.
 354 void
 355 runtime_schedinit(void)
 356 {
 357         int32 n;
 358         const byte *p;
 359
 360         m = &runtime_m0;
 361         g = &runtime_g0;
 362         m->g0 = g;
 363         m->curg = g;
 364         g->m = m;
 365
 366         initcontext();
 367
 368         m->nomemprof++;
 369         runtime_mallocinit();
 370         mcommoninit(m);
 371
 372         runtime_goargs();
 373         runtime_goenvs();
 374
 375         // For debugging:
 376         // Allocate internal symbol table representation now,
 377         // so that we don't need to call malloc when we crash.
 378         // runtime_findfunc(0);
 379
 380         runtime_gomaxprocs = 1;
 381         p = runtime_getenv("GOMAXPROCS");
 382         if(p != nil && (n = runtime_atoi(p)) != 0) {
 383                 if(n > maxgomaxprocs)
 384                         n = maxgomaxprocs;
 385                 runtime_gomaxprocs = n;
 386         }
 387         setmcpumax(runtime_gomaxprocs);
 388         runtime_singleproc = runtime_gomaxprocs == 1;
 389
 390         canaddmcpu();   // mcpu++ to account for bootstrap m
 391         m->helpgc = 1;  // flag to tell schedule() to mcpu--
 392         runtime_sched.grunning++;
 393
 394         // Can not enable GC until all roots are registered.
 395         // mstats.enablegc = 1;
 396         m->nomemprof--;
 397 }
 398
 399 extern void main_init(void) __asm__ ("__go_init_main");
 400 extern void main_main(void) __asm__ ("main.main");
 401
 402 // The main goroutine.
 403 void
 404 runtime_main(void)
 405 {
 406         // Lock the main goroutine onto this, the main OS thread,
 407         // during initialization.  Most programs won't care, but a few
 408         // do require certain calls to be made by the main thread.
 409         // Those can arrange for main.main to run in the main thread
 410         // by calling runtime.LockOSThread during initialization
 411         // to preserve the lock.
 412         runtime_LockOSThread();
 413         runtime_sched.init = true;
 414         main_init();
 415         runtime_sched.init = false;
 416         if(!runtime_sched.lockmain)
 417                 runtime_UnlockOSThread();
 418
 419         // For gccgo we have to wait until after main is initialized
 420         // to enable GC, because initializing main registers the GC
 421         // roots.
 422         mstats.enablegc = 1;
 423
 424         main_main();
 425         runtime_exit(0);
 426         for(;;)
 427                 *(int32*)0 = 0;
 428 }
 429
 430 // Lock the scheduler.
 431 static void
 432 schedlock(void)
 433 {
 434         runtime_lock(&runtime_sched);
 435 }
 436
 437 // Unlock the scheduler.
 438 static void
 439 schedunlock(void)
 440 {
 441         M *m;
 442
 443         m = mwakeup;
 444         mwakeup = nil;
 445         runtime_unlock(&runtime_sched);
 446         if(m != nil)
 447                 runtime_notewakeup(&m->havenextg);
 448 }
 449
 450 void
 451 runtime_goexit(void)
 452 {
 453         g->status = Gmoribund;
 454         runtime_gosched();
 455 }
 456
 457 void
 458 runtime_goroutineheader(G *g)
 459 {
 460         const char *status;
 461
 462         switch(g->status) {
 463         case Gidle:
 464                 status = "idle";
 465                 break;
 466         case Grunnable:
 467                 status = "runnable";
 468                 break;
 469         case Grunning:
 470                 status = "running";
 471                 break;
 472         case Gsyscall:
 473                 status = "syscall";
 474                 break;
 475         case Gwaiting:
 476                 if(g->waitreason)
 477                         status = g->waitreason;
 478                 else
 479                         status = "waiting";
 480                 break;
 481         case Gmoribund:
 482                 status = "moribund";
 483                 break;
 484         default:
 485                 status = "???";
 486                 break;
 487         }
 488         runtime_printf("goroutine %d [%s]:\n", g->goid, status);
 489 }
 490
 491 void
 492 runtime_tracebackothers(G *me)
 493 {
 494         G *g;
 495
 496         for(g = runtime_allg; g != nil; g = g->alllink) {
 497                 if(g == me || g->status == Gdead)
 498                         continue;
 499                 runtime_printf("\n");
 500                 runtime_goroutineheader(g);
 501                 // runtime_traceback(g->sched.pc, g->sched.sp, 0, g);
 502         }
 503 }
 504
 505 // Mark this g as m's idle goroutine.
 506 // This functionality might be used in environments where programs
 507 // are limited to a single thread, to simulate a select-driven
 508 // network server.  It is not exposed via the standard runtime API.
 509 void
 510 runtime_idlegoroutine(void)
 511 {
 512         if(g->idlem != nil)
 513                 runtime_throw("g is already an idle goroutine");
 514         g->idlem = m;
 515 }
 516
 517 static void
 518 mcommoninit(M *m)
 519 {
 520         // Add to runtime_allm so garbage collector doesn't free m
 521         // when it is just in a register or thread-local storage.
 522         m->alllink = runtime_allm;
 523         // runtime_Cgocalls() iterates over allm w/o schedlock,
 524         // so we need to publish it safely.
 525         runtime_atomicstorep((void**)&runtime_allm, m);
 526
 527         m->id = runtime_sched.mcount++;
 528         m->fastrand = 0x49f6428aUL + m->id + runtime_cputicks();
 529
 530         if(m->mcache == nil)
 531                 m->mcache = runtime_allocmcache();
 532 }
 533
 534 // Try to increment mcpu.  Report whether succeeded.
 535 static bool
 536 canaddmcpu(void)
 537 {
 538         uint32 v;
 539
 540         for(;;) {
 541                 v = runtime_sched.atomic;
 542                 if(atomic_mcpu(v) >= atomic_mcpumax(v))
 543                         return 0;
 544                 if(runtime_cas(&runtime_sched.atomic, v, v+(1<<mcpuShift)))
 545                         return 1;
 546         }
 547 }
 548
 549 // Put on `g' queue.  Sched must be locked.
 550 static void
 551 gput(G *g)
 552 {
 553         M *m;
 554
 555         // If g is wired, hand it off directly.
 556         if((m = g->lockedm) != nil && canaddmcpu()) {
 557                 mnextg(m, g);
 558                 return;
 559         }
 560
 561         // If g is the idle goroutine for an m, hand it off.
 562         if(g->idlem != nil) {
 563                 if(g->idlem->idleg != nil) {
 564                         runtime_printf("m%d idle out of sync: g%d g%d\n",
 565                                 g->idlem->id,
 566                                 g->idlem->idleg->goid, g->goid);
 567                         runtime_throw("runtime: double idle");
 568                 }
 569                 g->idlem->idleg = g;
 570                 return;
 571         }
 572
 573         g->schedlink = nil;
 574         if(runtime_sched.ghead == nil)
 575                 runtime_sched.ghead = g;
 576         else
 577                 runtime_sched.gtail->schedlink = g;
 578         runtime_sched.gtail = g;
 579
 580         // increment gwait.
 581         // if it transitions to nonzero, set atomic gwaiting bit.
 582         if(runtime_sched.gwait++ == 0)
 583                 runtime_xadd(&runtime_sched.atomic, 1<<gwaitingShift);
 584 }
 585
 586 // Report whether gget would return something.
 587 static bool
 588 haveg(void)
 589 {
 590         return runtime_sched.ghead != nil || m->idleg != nil;
 591 }
 592
 593 // Get from `g' queue.  Sched must be locked.
 594 static G*
 595 gget(void)
 596 {
 597         G *g;
 598
 599         g = runtime_sched.ghead;
 600         if(g){
 601                 runtime_sched.ghead = g->schedlink;
 602                 if(runtime_sched.ghead == nil)
 603                         runtime_sched.gtail = nil;
 604                 // decrement gwait.
 605                 // if it transitions to zero, clear atomic gwaiting bit.
 606                 if(--runtime_sched.gwait == 0)
 607                         runtime_xadd(&runtime_sched.atomic, -1<<gwaitingShift);
 608         } else if(m->idleg != nil) {
 609                 g = m->idleg;
 610                 m->idleg = nil;
 611         }
 612         return g;
 613 }
 614
 615 // Put on `m' list.  Sched must be locked.
 616 static void
 617 mput(M *m)
 618 {
 619         m->schedlink = runtime_sched.mhead;
 620         runtime_sched.mhead = m;
 621         runtime_sched.mwait++;
 622 }
 623
 624 // Get an `m' to run `g'.  Sched must be locked.
 625 static M*
 626 mget(G *g)
 627 {
 628         M *m;
 629
 630         // if g has its own m, use it.
 631         if(g && (m = g->lockedm) != nil)
 632                 return m;
 633
 634         // otherwise use general m pool.
 635         if((m = runtime_sched.mhead) != nil){
 636                 runtime_sched.mhead = m->schedlink;
 637                 runtime_sched.mwait--;
 638         }
 639         return m;
 640 }
 641
 642 // Mark g ready to run.
 643 void
 644 runtime_ready(G *g)
 645 {
 646         schedlock();
 647         readylocked(g);
 648         schedunlock();
 649 }
 650
 651 // Mark g ready to run.  Sched is already locked.
 652 // G might be running already and about to stop.
 653 // The sched lock protects g->status from changing underfoot.
 654 static void
 655 readylocked(G *g)
 656 {
 657         if(g->m){
 658                 // Running on another machine.
 659                 // Ready it when it stops.
 660                 g->readyonstop = 1;
 661                 return;
 662         }
 663
 664         // Mark runnable.
 665         if(g->status == Grunnable || g->status == Grunning) {
 666                 runtime_printf("goroutine %d has status %d\n", g->goid, g->status);
 667                 runtime_throw("bad g->status in ready");
 668         }
 669         g->status = Grunnable;
 670
 671         gput(g);
 672         matchmg();
 673 }
 674
 675 // Same as readylocked but a different symbol so that
 676 // debuggers can set a breakpoint here and catch all
 677 // new goroutines.
 678 static void
 679 newprocreadylocked(G *g)
 680 {
 681         readylocked(g);
 682 }
 683
 684 // Pass g to m for running.
 685 // Caller has already incremented mcpu.
 686 static void
 687 mnextg(M *m, G *g)
 688 {
 689         runtime_sched.grunning++;
 690         m->nextg = g;
 691         if(m->waitnextg) {
 692                 m->waitnextg = 0;
 693                 if(mwakeup != nil)
 694                         runtime_notewakeup(&mwakeup->havenextg);
 695                 mwakeup = m;
 696         }
 697 }
 698
 699 // Get the next goroutine that m should run.
 700 // Sched must be locked on entry, is unlocked on exit.
 701 // Makes sure that at most $GOMAXPROCS g's are
 702 // running on cpus (not in system calls) at any given time.
 703 static G*
 704 nextgandunlock(void)
 705 {
 706         G *gp;
 707         uint32 v;
 708
 709 top:
 710         if(atomic_mcpu(runtime_sched.atomic) >= maxgomaxprocs)
 711                 runtime_throw("negative mcpu");
 712
 713         // If there is a g waiting as m->nextg, the mcpu++
 714         // happened before it was passed to mnextg.
 715         if(m->nextg != nil) {
 716                 gp = m->nextg;
 717                 m->nextg = nil;
 718                 schedunlock();
 719                 return gp;
 720         }
 721
 722         if(m->lockedg != nil) {
 723                 // We can only run one g, and it's not available.
 724                 // Make sure some other cpu is running to handle
 725                 // the ordinary run queue.
 726                 if(runtime_sched.gwait != 0) {
 727                         matchmg();
 728                         // m->lockedg might have been on the queue.
 729                         if(m->nextg != nil) {
 730                                 gp = m->nextg;
 731                                 m->nextg = nil;
 732                                 schedunlock();
 733                                 return gp;
 734                         }
 735                 }
 736         } else {
 737                 // Look for work on global queue.
 738                 while(haveg() && canaddmcpu()) {
 739                         gp = gget();
 740                         if(gp == nil)
 741                                 runtime_throw("gget inconsistency");
 742
 743                         if(gp->lockedm) {
 744                                 mnextg(gp->lockedm, gp);
 745                                 continue;
 746                         }
 747                         runtime_sched.grunning++;
 748                         schedunlock();
 749                         return gp;
 750                 }
 751
 752                 // The while loop ended either because the g queue is empty
 753                 // or because we have maxed out our m procs running go
 754                 // code (mcpu >= mcpumax).  We need to check that
 755                 // concurrent actions by entersyscall/exitsyscall cannot
 756                 // invalidate the decision to end the loop.
 757                 //
 758                 // We hold the sched lock, so no one else is manipulating the
 759                 // g queue or changing mcpumax.  Entersyscall can decrement
 760                 // mcpu, but if does so when there is something on the g queue,
 761                 // the gwait bit will be set, so entersyscall will take the slow path
 762                 // and use the sched lock.  So it cannot invalidate our decision.
 763                 //
 764                 // Wait on global m queue.
 765                 mput(m);
 766         }
 767
 768         v = runtime_atomicload(&runtime_sched.atomic);
 769         if(runtime_sched.grunning == 0)
 770                 runtime_throw("all goroutines are asleep - deadlock!");
 771         m->nextg = nil;
 772         m->waitnextg = 1;
 773         runtime_noteclear(&m->havenextg);
 774
 775         // Stoptheworld is waiting for all but its cpu to go to stop.
 776         // Entersyscall might have decremented mcpu too, but if so
 777         // it will see the waitstop and take the slow path.
 778         // Exitsyscall never increments mcpu beyond mcpumax.
 779         if(atomic_waitstop(v) && atomic_mcpu(v) <= atomic_mcpumax(v)) {
 780                 // set waitstop = 0 (known to be 1)
 781                 runtime_xadd(&runtime_sched.atomic, -1<<waitstopShift);
 782                 runtime_notewakeup(&runtime_sched.stopped);
 783         }
 784         schedunlock();
 785
 786         runtime_notesleep(&m->havenextg);
 787         if(m->helpgc) {
 788                 runtime_gchelper();
 789                 m->helpgc = 0;
 790                 runtime_lock(&runtime_sched);
 791                 goto top;
 792         }
 793         if((gp = m->nextg) == nil)
 794                 runtime_throw("bad m->nextg in nextgoroutine");
 795         m->nextg = nil;
 796         return gp;
 797 }
 798
 799 int32
 800 runtime_helpgc(bool *extra)
 801 {
 802         M *mp;
 803         int32 n, max;
 804
 805         // Figure out how many CPUs to use.
 806         // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc.
 807         max = runtime_gomaxprocs;
 808         if(max > runtime_ncpu)
 809                 max = runtime_ncpu > 0 ? runtime_ncpu : 1;
 810         if(max > MaxGcproc)
 811                 max = MaxGcproc;
 812
 813         // We're going to use one CPU no matter what.
 814         // Figure out the max number of additional CPUs.
 815         max--;
 816
 817         runtime_lock(&runtime_sched);
 818         n = 0;
 819         while(n < max && (mp = mget(nil)) != nil) {
 820                 n++;
 821                 mp->helpgc = 1;
 822                 mp->waitnextg = 0;
 823                 runtime_notewakeup(&mp->havenextg);
 824         }
 825         runtime_unlock(&runtime_sched);
 826         if(extra)
 827                 *extra = n != max;
 828         return n;
 829 }
 830
 831 void
 832 runtime_stoptheworld(void)
 833 {
 834         uint32 v;
 835
 836         schedlock();
 837         runtime_gcwaiting = 1;
 838
 839         setmcpumax(1);
 840
 841         // while mcpu > 1
 842         for(;;) {
 843                 v = runtime_sched.atomic;
 844                 if(atomic_mcpu(v) <= 1)
 845                         break;
 846
 847                 // It would be unsafe for multiple threads to be using
 848                 // the stopped note at once, but there is only
 849                 // ever one thread doing garbage collection.
 850                 runtime_noteclear(&runtime_sched.stopped);
 851                 if(atomic_waitstop(v))
 852                         runtime_throw("invalid waitstop");
 853
 854                 // atomic { waitstop = 1 }, predicated on mcpu <= 1 check above
 855                 // still being true.
 856                 if(!runtime_cas(&runtime_sched.atomic, v, v+(1<<waitstopShift)))
 857                         continue;
 858
 859                 schedunlock();
 860                 runtime_notesleep(&runtime_sched.stopped);
 861                 schedlock();
 862         }
 863         runtime_singleproc = runtime_gomaxprocs == 1;
 864         schedunlock();
 865 }
 866
 867 void
 868 runtime_starttheworld(bool extra)
 869 {
 870         M *m;
 871
 872         schedlock();
 873         runtime_gcwaiting = 0;
 874         setmcpumax(runtime_gomaxprocs);
 875         matchmg();
 876         if(extra && canaddmcpu()) {
 877                 // Start a new m that will (we hope) be idle
 878                 // and so available to help when the next
 879                 // garbage collection happens.
 880                 // canaddmcpu above did mcpu++
 881                 // (necessary, because m will be doing various
 882                 // initialization work so is definitely running),
 883                 // but m is not running a specific goroutine,
 884                 // so set the helpgc flag as a signal to m's
 885                 // first schedule(nil) to mcpu-- and grunning--.
 886                 m = runtime_newm();
 887                 m->helpgc = 1;
 888                 runtime_sched.grunning++;
 889         }
 890         schedunlock();
 891 }
 892
 893 // Called to start an M.
 894 void*
 895 runtime_mstart(void* mp)
 896 {
 897         m = (M*)mp;
 898         g = m->g0;
 899
 900         initcontext();
 901
 902         g->entry = nil;
 903         g->param = nil;
 904
 905         // Record top of stack for use by mcall.
 906         // Once we call schedule we're never coming back,
 907         // so other calls can reuse this stack space.
 908 #ifdef USING_SPLIT_STACK
 909         __splitstack_getcontext(&g->stack_context[0]);
 910 #else
 911         g->gcinitial_sp = &mp;
 912         g->gcstack_size = StackMin;
 913         g->gcnext_sp = &mp;
 914 #endif
 915         getcontext(&g->context);
 916
 917         if(g->entry != nil) {
 918                 // Got here from mcall.
 919                 void (*pfn)(G*) = (void (*)(G*))g->entry;
 920                 G* gp = (G*)g->param;
 921                 pfn(gp);
 922                 *(int*)0x21 = 0x21;
 923         }
 924         runtime_minit();
 925
 926 #ifdef USING_SPLIT_STACK
 927         {
 928           int dont_block_signals = 0;
 929           __splitstack_block_signals(&dont_block_signals, nil);
 930         }
 931 #endif
 932
 933         schedule(nil);
 934         return nil;
 935 }
 936
 937 typedef struct CgoThreadStart CgoThreadStart;
 938 struct CgoThreadStart
 939 {
 940         M *m;
 941         G *g;
 942         void (*fn)(void);
 943 };
 944
 945 // Kick off new m's as needed (up to mcpumax).
 946 // Sched is locked.
 947 static void
 948 matchmg(void)
 949 {
 950         G *gp;
 951         M *mp;
 952
 953         if(m->mallocing || m->gcing)
 954                 return;
 955
 956         while(haveg() && canaddmcpu()) {
 957                 gp = gget();
 958                 if(gp == nil)
 959                         runtime_throw("gget inconsistency");
 960
 961                 // Find the m that will run gp.
 962                 if((mp = mget(gp)) == nil)
 963                         mp = runtime_newm();
 964                 mnextg(mp, gp);
 965         }
 966 }
 967
 968 // Create a new m.  It will start off with a call to runtime_mstart.
 969 M*
 970 runtime_newm(void)
 971 {
 972         M *m;
 973         pthread_attr_t attr;
 974         pthread_t tid;
 975
 976         m = runtime_malloc(sizeof(M));
 977         mcommoninit(m);
 978         m->g0 = runtime_malg(-1, nil, nil);
 979
 980         if(pthread_attr_init(&attr) != 0)
 981                 runtime_throw("pthread_attr_init");
 982         if(pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0)
 983                 runtime_throw("pthread_attr_setdetachstate");
 984
 985 #ifndef PTHREAD_STACK_MIN
 986 #define PTHREAD_STACK_MIN 8192
 987 #endif
 988         if(pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN) != 0)
 989                 runtime_throw("pthread_attr_setstacksize");
 990
 991         if(pthread_create(&tid, &attr, runtime_mstart, m) != 0)
 992                 runtime_throw("pthread_create");
 993
 994         return m;
 995 }
 996
 997 // One round of scheduler: find a goroutine and run it.
 998 // The argument is the goroutine that was running before
 999 // schedule was called, or nil if this is the first call.
1000 // Never returns.
1001 static void
1002 schedule(G *gp)
1003 {
1004         int32 hz;
1005         uint32 v;
1006
1007         schedlock();
1008         if(gp != nil) {
1009                 // Just finished running gp.
1010                 gp->m = nil;
1011                 runtime_sched.grunning--;
1012
1013                 // atomic { mcpu-- }
1014                 v = runtime_xadd(&runtime_sched.atomic, -1<<mcpuShift);
1015                 if(atomic_mcpu(v) > maxgomaxprocs)
1016                         runtime_throw("negative mcpu in scheduler");
1017
1018                 switch(gp->status){
1019                 case Grunnable:
1020                 case Gdead:
1021                         // Shouldn't have been running!
1022                         runtime_throw("bad gp->status in sched");
1023                 case Grunning:
1024                         gp->status = Grunnable;
1025                         gput(gp);
1026                         break;
1027                 case Gmoribund:
1028                         gp->status = Gdead;
1029                         if(gp->lockedm) {
1030                                 gp->lockedm = nil;
1031                                 m->lockedg = nil;
1032                         }
1033                         gp->idlem = nil;
1034                         gfput(gp);
1035                         if(--runtime_sched.gcount == 0)
1036                                 runtime_exit(0);
1037                         break;
1038                 }
1039                 if(gp->readyonstop){
1040                         gp->readyonstop = 0;
1041                         readylocked(gp);
1042                 }
1043         } else if(m->helpgc) {
1044                 // Bootstrap m or new m started by starttheworld.
1045                 // atomic { mcpu-- }
1046                 v = runtime_xadd(&runtime_sched.atomic, -1<<mcpuShift);
1047                 if(atomic_mcpu(v) > maxgomaxprocs)
1048                         runtime_throw("negative mcpu in scheduler");
1049                 // Compensate for increment in starttheworld().
1050                 runtime_sched.grunning--;
1051                 m->helpgc = 0;
1052         } else if(m->nextg != nil) {
1053                 // New m started by matchmg.
1054         } else {
1055                 runtime_throw("invalid m state in scheduler");
1056         }
1057
1058         // Find (or wait for) g to run.  Unlocks runtime_sched.
1059         gp = nextgandunlock();
1060         gp->readyonstop = 0;
1061         gp->status = Grunning;
1062         m->curg = gp;
1063         gp->m = m;
1064
1065         // Check whether the profiler needs to be turned on or off.
1066         hz = runtime_sched.profilehz;
1067         if(m->profilehz != hz)
1068                 runtime_resetcpuprofiler(hz);
1069
1070         runtime_gogo(gp);
1071 }
1072
1073 // Enter scheduler.  If g->status is Grunning,
1074 // re-queues g and runs everyone else who is waiting
1075 // before running g again.  If g->status is Gmoribund,
1076 // kills off g.
1077 void
1078 runtime_gosched(void)
1079 {
1080         if(m->locks != 0)
1081                 runtime_throw("gosched holding locks");
1082         if(g == m->g0)
1083                 runtime_throw("gosched of g0");
1084         runtime_mcall(schedule);
1085 }
1086
1087 // The goroutine g is about to enter a system call.
1088 // Record that it's not using the cpu anymore.
1089 // This is called only from the go syscall library and cgocall,
1090 // not from the low-level system calls used by the runtime.
1091 //
1092 // Entersyscall cannot split the stack: the runtime_gosave must
1093 // make g->sched refer to the caller's stack segment, because
1094 // entersyscall is going to return immediately after.
1095 // It's okay to call matchmg and notewakeup even after
1096 // decrementing mcpu, because we haven't released the
1097 // sched lock yet, so the garbage collector cannot be running.
1098
1099 void runtime_entersyscall(void) __attribute__ ((no_split_stack));
1100
1101 void
1102 runtime_entersyscall(void)
1103 {
1104         uint32 v;
1105
1106         // Leave SP around for gc and traceback.
1107 #ifdef USING_SPLIT_STACK
1108         g->gcstack = __splitstack_find(NULL, NULL, &g->gcstack_size,
1109                                        &g->gcnext_segment, &g->gcnext_sp,
1110                                        &g->gcinitial_sp);
1111 #else
1112         g->gcnext_sp = (byte *) &v;
1113 #endif
1114
1115         // Save the registers in the g structure so that any pointers
1116         // held in registers will be seen by the garbage collector.
1117         // We could use getcontext here, but setjmp is more efficient
1118         // because it doesn't need to save the signal mask.
1119         setjmp(g->gcregs);
1120
1121         g->status = Gsyscall;
1122
1123         // Fast path.
1124         // The slow path inside the schedlock/schedunlock will get
1125         // through without stopping if it does:
1126         //      mcpu--
1127         //      gwait not true
1128         //      waitstop && mcpu <= mcpumax not true
1129         // If we can do the same with a single atomic add,
1130         // then we can skip the locks.
1131         v = runtime_xadd(&runtime_sched.atomic, -1<<mcpuShift);
1132         if(!atomic_gwaiting(v) && (!atomic_waitstop(v) || atomic_mcpu(v) > atomic_mcpumax(v)))
1133                 return;
1134
1135         schedlock();
1136         v = runtime_atomicload(&runtime_sched.atomic);
1137         if(atomic_gwaiting(v)) {
1138                 matchmg();
1139                 v = runtime_atomicload(&runtime_sched.atomic);
1140         }
1141         if(atomic_waitstop(v) && atomic_mcpu(v) <= atomic_mcpumax(v)) {
1142                 runtime_xadd(&runtime_sched.atomic, -1<<waitstopShift);
1143                 runtime_notewakeup(&runtime_sched.stopped);
1144         }
1145
1146         schedunlock();
1147 }
1148
1149 // The goroutine g exited its system call.
1150 // Arrange for it to run on a cpu again.
1151 // This is called only from the go syscall library, not
1152 // from the low-level system calls used by the runtime.
1153 void
1154 runtime_exitsyscall(void)
1155 {
1156         G *gp;
1157         uint32 v;
1158
1159         // Fast path.
1160         // If we can do the mcpu++ bookkeeping and
1161         // find that we still have mcpu <= mcpumax, then we can
1162         // start executing Go code immediately, without having to
1163         // schedlock/schedunlock.
1164         gp = g;
1165         v = runtime_xadd(&runtime_sched.atomic, (1<<mcpuShift));
1166         if(m->profilehz == runtime_sched.profilehz && atomic_mcpu(v) <= atomic_mcpumax(v)) {
1167                 // There's a cpu for us, so we can run.
1168                 gp->status = Grunning;
1169                 // Garbage collector isn't running (since we are),
1170                 // so okay to clear gcstack.
1171 #ifdef USING_SPLIT_STACK
1172                 gp->gcstack = nil;
1173 #endif
1174                 gp->gcnext_sp = nil;
1175                 runtime_memclr(gp->gcregs, sizeof gp->gcregs);
1176                 return;
1177         }
1178
1179         // Tell scheduler to put g back on the run queue:
1180         // mostly equivalent to g->status = Grunning,
1181         // but keeps the garbage collector from thinking
1182         // that g is running right now, which it's not.
1183         gp->readyonstop = 1;
1184
1185         // All the cpus are taken.
1186         // The scheduler will ready g and put this m to sleep.
1187         // When the scheduler takes g away from m,
1188         // it will undo the runtime_sched.mcpu++ above.
1189         runtime_gosched();
1190
1191         // Gosched returned, so we're allowed to run now.
1192         // Delete the gcstack information that we left for
1193         // the garbage collector during the system call.
1194         // Must wait until now because until gosched returns
1195         // we don't know for sure that the garbage collector
1196         // is not running.
1197 #ifdef USING_SPLIT_STACK
1198         gp->gcstack = nil;
1199 #endif
1200         gp->gcnext_sp = nil;
1201         runtime_memclr(gp->gcregs, sizeof gp->gcregs);
1202 }
1203
1204 // Allocate a new g, with a stack big enough for stacksize bytes.
1205 G*
1206 runtime_malg(int32 stacksize, byte** ret_stack, size_t* ret_stacksize)
1207 {
1208         G *newg;
1209
1210         newg = runtime_malloc(sizeof(G));
1211         if(stacksize >= 0) {
1212 #if USING_SPLIT_STACK
1213                 int dont_block_signals = 0;
1214
1215                 *ret_stack = __splitstack_makecontext(stacksize,
1216                                                       &newg->stack_context[0],
1217                                                       ret_stacksize);
1218                 __splitstack_block_signals_context(&newg->stack_context[0],
1219                                                    &dont_block_signals, nil);
1220 #else
1221                 *ret_stack = runtime_mallocgc(stacksize, FlagNoProfiling|FlagNoGC, 0, 0);
1222                 *ret_stacksize = stacksize;
1223                 newg->gcinitial_sp = *ret_stack;
1224                 newg->gcstack_size = stacksize;
1225 #endif
1226         }
1227         return newg;
1228 }
1229
1230 /* For runtime package testing.  */
1231
1232 void runtime_testing_entersyscall(void)
1233   __asm__("libgo_runtime.runtime.entersyscall");
1234
1235 void
1236 runtime_testing_entersyscall()
1237 {
1238         runtime_entersyscall();
1239 }
1240
1241 void runtime_testing_exitsyscall(void)
1242   __asm__("libgo_runtime.runtime.exitsyscall");
1243
1244 void
1245 runtime_testing_exitsyscall()
1246 {
1247         runtime_exitsyscall();
1248 }
1249
1250 G*
1251 __go_go(void (*fn)(void*), void* arg)
1252 {
1253         byte *sp;
1254         size_t spsize;
1255         G * volatile newg;      // volatile to avoid longjmp warning
1256
1257         schedlock();
1258
1259         if((newg = gfget()) != nil){
1260 #ifdef USING_SPLIT_STACK
1261                 int dont_block_signals = 0;
1262
1263                 sp = __splitstack_resetcontext(&newg->stack_context[0],
1264                                                &spsize);
1265                 __splitstack_block_signals_context(&newg->stack_context[0],
1266                                                    &dont_block_signals, nil);
1267 #else
1268                 sp = newg->gcinitial_sp;
1269                 spsize = newg->gcstack_size;
1270                 newg->gcnext_sp = sp;
1271 #endif
1272         } else {
1273                 newg = runtime_malg(StackMin, &sp, &spsize);
1274                 if(runtime_lastg == nil)
1275                         runtime_allg = newg;
1276                 else
1277                         runtime_lastg->alllink = newg;
1278                 runtime_lastg = newg;
1279         }
1280         newg->status = Gwaiting;
1281         newg->waitreason = "new goroutine";
1282
1283         newg->entry = (byte*)fn;
1284         newg->param = arg;
1285         newg->gopc = (uintptr)__builtin_return_address(0);
1286
1287         runtime_sched.gcount++;
1288         runtime_sched.goidgen++;
1289         newg->goid = runtime_sched.goidgen;
1290
1291         if(sp == nil)
1292                 runtime_throw("nil g->stack0");
1293
1294         getcontext(&newg->context);
1295         newg->context.uc_stack.ss_sp = sp;
1296         newg->context.uc_stack.ss_size = spsize;
1297         makecontext(&newg->context, kickoff, 0);
1298
1299         newprocreadylocked(newg);
1300         schedunlock();
1301
1302         return newg;
1303 //printf(" goid=%d\n", newg->goid);
1304 }
1305
1306 // Put on gfree list.  Sched must be locked.
1307 static void
1308 gfput(G *g)
1309 {
1310         g->schedlink = runtime_sched.gfree;
1311         runtime_sched.gfree = g;
1312 }
1313
1314 // Get from gfree list.  Sched must be locked.
1315 static G*
1316 gfget(void)
1317 {
1318         G *g;
1319
1320         g = runtime_sched.gfree;
1321         if(g)
1322                 runtime_sched.gfree = g->schedlink;
1323         return g;
1324 }
1325
1326 // Run all deferred functions for the current goroutine.
1327 static void
1328 rundefer(void)
1329 {
1330         Defer *d;
1331
1332         while((d = g->defer) != nil) {
1333                 void (*pfn)(void*);
1334
1335                 pfn = d->__pfn;
1336                 d->__pfn = nil;
1337                 if (pfn != nil)
1338                         (*pfn)(d->__arg);
1339                 g->defer = d->__next;
1340                 runtime_free(d);
1341         }
1342 }
1343
1344 void runtime_Goexit (void) asm ("libgo_runtime.runtime.Goexit");
1345
1346 void
1347 runtime_Goexit(void)
1348 {
1349         rundefer();
1350         runtime_goexit();
1351 }
1352
1353 void runtime_Gosched (void) asm ("libgo_runtime.runtime.Gosched");
1354
1355 void
1356 runtime_Gosched(void)
1357 {
1358         runtime_gosched();
1359 }
1360
1361 // Implementation of runtime.GOMAXPROCS.
1362 // delete when scheduler is stronger
1363 int32
1364 runtime_gomaxprocsfunc(int32 n)
1365 {
1366         int32 ret;
1367         uint32 v;
1368
1369         schedlock();
1370         ret = runtime_gomaxprocs;
1371         if(n <= 0)
1372                 n = ret;
1373         if(n > maxgomaxprocs)
1374                 n = maxgomaxprocs;
1375         runtime_gomaxprocs = n;
1376         if(runtime_gomaxprocs > 1)
1377                 runtime_singleproc = false;
1378         if(runtime_gcwaiting != 0) {
1379                 if(atomic_mcpumax(runtime_sched.atomic) != 1)
1380                         runtime_throw("invalid mcpumax during gc");
1381                 schedunlock();
1382                 return ret;
1383         }
1384
1385         setmcpumax(n);
1386
1387         // If there are now fewer allowed procs
1388         // than procs running, stop.
1389         v = runtime_atomicload(&runtime_sched.atomic);
1390         if((int32)atomic_mcpu(v) > n) {
1391                 schedunlock();
1392                 runtime_gosched();
1393                 return ret;
1394         }
1395         // handle more procs
1396         matchmg();
1397         schedunlock();
1398         return ret;
1399 }
1400
1401 void
1402 runtime_LockOSThread(void)
1403 {
1404         if(m == &runtime_m0 && runtime_sched.init) {
1405                 runtime_sched.lockmain = true;
1406                 return;
1407         }
1408         m->lockedg = g;
1409         g->lockedm = m;
1410 }
1411
1412 void
1413 runtime_UnlockOSThread(void)
1414 {
1415         if(m == &runtime_m0 && runtime_sched.init) {
1416                 runtime_sched.lockmain = false;
1417                 return;
1418         }
1419         m->lockedg = nil;
1420         g->lockedm = nil;
1421 }
1422
1423 bool
1424 runtime_lockedOSThread(void)
1425 {
1426         return g->lockedm != nil && m->lockedg != nil;
1427 }
1428
1429 // for testing of callbacks
1430
1431 _Bool runtime_golockedOSThread(void)
1432   asm("libgo_runtime.runtime.golockedOSThread");
1433
1434 _Bool
1435 runtime_golockedOSThread(void)
1436 {
1437         return runtime_lockedOSThread();
1438 }
1439
1440 // for testing of wire, unwire
1441 uint32
1442 runtime_mid()
1443 {
1444         return m->id;
1445 }
1446
1447 int32 runtime_Goroutines (void)
1448   __asm__ ("libgo_runtime.runtime.Goroutines");
1449
1450 int32
1451 runtime_Goroutines()
1452 {
1453         return runtime_sched.gcount;
1454 }
1455
1456 int32
1457 runtime_mcount(void)
1458 {
1459         return runtime_sched.mcount;
1460 }
1461
1462 static struct {
1463         Lock;
1464         void (*fn)(uintptr*, int32);
1465         int32 hz;
1466         uintptr pcbuf[100];
1467 } prof;
1468
1469 // Called if we receive a SIGPROF signal.
1470 void
1471 runtime_sigprof(uint8 *pc __attribute__ ((unused)),
1472                 uint8 *sp __attribute__ ((unused)),
1473                 uint8 *lr __attribute__ ((unused)),
1474                 G *gp __attribute__ ((unused)))
1475 {
1476         // int32 n;
1477
1478         if(prof.fn == nil || prof.hz == 0)
1479                 return;
1480
1481         runtime_lock(&prof);
1482         if(prof.fn == nil) {
1483                 runtime_unlock(&prof);
1484                 return;
1485         }
1486         // n = runtime_gentraceback(pc, sp, lr, gp, 0, prof.pcbuf, nelem(prof.pcbuf));
1487         // if(n > 0)
1488         //      prof.fn(prof.pcbuf, n);
1489         runtime_unlock(&prof);
1490 }
1491
1492 // Arrange to call fn with a traceback hz times a second.
1493 void
1494 runtime_setcpuprofilerate(void (*fn)(uintptr*, int32), int32 hz)
1495 {
1496         // Force sane arguments.
1497         if(hz < 0)
1498                 hz = 0;
1499         if(hz == 0)
1500                 fn = nil;
1501         if(fn == nil)
1502                 hz = 0;
1503
1504         // Stop profiler on this cpu so that it is safe to lock prof.
1505         // if a profiling signal came in while we had prof locked,
1506         // it would deadlock.
1507         runtime_resetcpuprofiler(0);
1508
1509         runtime_lock(&prof);
1510         prof.fn = fn;
1511         prof.hz = hz;
1512         runtime_unlock(&prof);
1513         runtime_lock(&runtime_sched);
1514         runtime_sched.profilehz = hz;
1515         runtime_unlock(&runtime_sched);
1516
1517         if(hz != 0)
1518                 runtime_resetcpuprofiler(hz);
1519 }