github.com/rohankumardubey/syslog-redirector-golang@v0.0.0-20140320174030-4859f03d829a/src/pkg/runtime/proc.c (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 #include "runtime.h" 6 #include "arch_GOARCH.h" 7 #include "zaexperiment.h" 8 #include "malloc.h" 9 #include "stack.h" 10 #include "race.h" 11 #include "type.h" 12 #include "../../cmd/ld/textflag.h" 13 14 // Goroutine scheduler 15 // The scheduler's job is to distribute ready-to-run goroutines over worker threads. 16 // 17 // The main concepts are: 18 // G - goroutine. 19 // M - worker thread, or machine. 20 // P - processor, a resource that is required to execute Go code. 21 // M must have an associated P to execute Go code, however it can be 22 // blocked or in a syscall w/o an associated P. 23 // 24 // Design doc at http://golang.org/s/go11sched. 25 26 typedef struct Sched Sched; 27 struct Sched { 28 Lock; 29 30 uint64 goidgen; 31 32 M* midle; // idle m's waiting for work 33 int32 nmidle; // number of idle m's waiting for work 34 int32 nmidlelocked; // number of locked m's waiting for work 35 int32 mcount; // number of m's that have been created 36 int32 maxmcount; // maximum number of m's allowed (or die) 37 38 P* pidle; // idle P's 39 uint32 npidle; 40 uint32 nmspinning; 41 42 // Global runnable queue. 43 G* runqhead; 44 G* runqtail; 45 int32 runqsize; 46 47 // Global cache of dead G's. 48 Lock gflock; 49 G* gfree; 50 51 uint32 gcwaiting; // gc is waiting to run 52 int32 stopwait; 53 Note stopnote; 54 uint32 sysmonwait; 55 Note sysmonnote; 56 uint64 lastpoll; 57 58 int32 profilehz; // cpu profiling rate 59 }; 60 61 // The max value of GOMAXPROCS. 62 // There are no fundamental restrictions on the value. 63 enum { MaxGomaxprocs = 1<<8 }; 64 65 Sched runtime·sched; 66 int32 runtime·gomaxprocs; 67 uint32 runtime·needextram; 68 bool runtime·iscgo; 69 M runtime·m0; 70 G runtime·g0; // idle goroutine for m0 71 G* runtime·allg; 72 G* runtime·lastg; 73 M* runtime·allm; 74 M* runtime·extram; 75 int8* runtime·goos; 76 int32 runtime·ncpu; 77 static int32 newprocs; 78 79 void runtime·mstart(void); 80 static void runqput(P*, G*); 81 static G* runqget(P*); 82 static void runqgrow(P*); 83 static G* runqsteal(P*, P*); 84 static void mput(M*); 85 static M* mget(void); 86 static void mcommoninit(M*); 87 static void schedule(void); 88 static void procresize(int32); 89 static void acquirep(P*); 90 static P* releasep(void); 91 static void newm(void(*)(void), P*); 92 static void stopm(void); 93 static void startm(P*, bool); 94 static void handoffp(P*); 95 static void wakep(void); 96 static void stoplockedm(void); 97 static void startlockedm(G*); 98 static void sysmon(void); 99 static uint32 retake(int64); 100 static void incidlelocked(int32); 101 static void checkdead(void); 102 static void exitsyscall0(G*); 103 static void park0(G*); 104 static void goexit0(G*); 105 static void gfput(P*, G*); 106 static G* gfget(P*); 107 static void gfpurge(P*); 108 static void globrunqput(G*); 109 static G* globrunqget(P*, int32); 110 static P* pidleget(void); 111 static void pidleput(P*); 112 static void injectglist(G*); 113 static bool preemptall(void); 114 static bool preemptone(P*); 115 static bool exitsyscallfast(void); 116 static bool haveexperiment(int8*); 117 118 // The bootstrap sequence is: 119 // 120 // call osinit 121 // call schedinit 122 // make & queue new G 123 // call runtime·mstart 124 // 125 // The new G calls runtime·main. 126 void 127 runtime·schedinit(void) 128 { 129 int32 n, procs; 130 byte *p; 131 Eface i; 132 133 runtime·sched.maxmcount = 10000; 134 runtime·precisestack = haveexperiment("precisestack"); 135 136 runtime·mprofinit(); 137 runtime·mallocinit(); 138 mcommoninit(m); 139 140 // Initialize the itable value for newErrorCString, 141 // so that the next time it gets called, possibly 142 // in a fault during a garbage collection, it will not 143 // need to allocated memory. 144 runtime·newErrorCString(0, &i); 145 146 runtime·goargs(); 147 runtime·goenvs(); 148 runtime·parsedebugvars(); 149 150 // Allocate internal symbol table representation now, we need it for GC anyway. 151 runtime·symtabinit(); 152 153 runtime·sched.lastpoll = runtime·nanotime(); 154 procs = 1; 155 p = runtime·getenv("GOMAXPROCS"); 156 if(p != nil && (n = runtime·atoi(p)) > 0) { 157 if(n > MaxGomaxprocs) 158 n = MaxGomaxprocs; 159 procs = n; 160 } 161 runtime·allp = runtime·malloc((MaxGomaxprocs+1)*sizeof(runtime·allp[0])); 162 procresize(procs); 163 164 mstats.enablegc = 1; 165 166 if(raceenabled) 167 g->racectx = runtime·raceinit(); 168 } 169 170 extern void main·init(void); 171 extern void main·main(void); 172 173 static FuncVal scavenger = {runtime·MHeap_Scavenger}; 174 175 static FuncVal initDone = { runtime·unlockOSThread }; 176 177 // The main goroutine. 178 void 179 runtime·main(void) 180 { 181 Defer d; 182 183 // Max stack size is 1 GB on 64-bit, 250 MB on 32-bit. 184 // Using decimal instead of binary GB and MB because 185 // they look nicer in the stack overflow failure message. 186 if(sizeof(void*) == 8) 187 runtime·maxstacksize = 1000000000; 188 else 189 runtime·maxstacksize = 250000000; 190 191 newm(sysmon, nil); 192 193 // Lock the main goroutine onto this, the main OS thread, 194 // during initialization. Most programs won't care, but a few 195 // do require certain calls to be made by the main thread. 196 // Those can arrange for main.main to run in the main thread 197 // by calling runtime.LockOSThread during initialization 198 // to preserve the lock. 199 runtime·lockOSThread(); 200 201 // Defer unlock so that runtime.Goexit during init does the unlock too. 202 d.fn = &initDone; 203 d.siz = 0; 204 d.link = g->defer; 205 d.argp = (void*)-1; 206 d.special = true; 207 d.free = false; 208 g->defer = &d; 209 210 if(m != &runtime·m0) 211 runtime·throw("runtime·main not on m0"); 212 runtime·newproc1(&scavenger, nil, 0, 0, runtime·main); 213 main·init(); 214 215 if(g->defer != &d || d.fn != &initDone) 216 runtime·throw("runtime: bad defer entry after init"); 217 g->defer = d.link; 218 runtime·unlockOSThread(); 219 220 main·main(); 221 if(raceenabled) 222 runtime·racefini(); 223 224 // Make racy client program work: if panicking on 225 // another goroutine at the same time as main returns, 226 // let the other goroutine finish printing the panic trace. 227 // Once it does, it will exit. See issue 3934. 228 if(runtime·panicking) 229 runtime·park(nil, nil, "panicwait"); 230 231 runtime·exit(0); 232 for(;;) 233 *(int32*)runtime·main = 0; 234 } 235 236 void 237 runtime·goroutineheader(G *gp) 238 { 239 int8 *status; 240 241 switch(gp->status) { 242 case Gidle: 243 status = "idle"; 244 break; 245 case Grunnable: 246 status = "runnable"; 247 break; 248 case Grunning: 249 status = "running"; 250 break; 251 case Gsyscall: 252 status = "syscall"; 253 break; 254 case Gwaiting: 255 if(gp->waitreason) 256 status = gp->waitreason; 257 else 258 status = "waiting"; 259 break; 260 default: 261 status = "???"; 262 break; 263 } 264 runtime·printf("goroutine %D [%s]:\n", gp->goid, status); 265 } 266 267 void 268 runtime·tracebackothers(G *me) 269 { 270 G *gp; 271 int32 traceback; 272 273 traceback = runtime·gotraceback(nil); 274 275 // Show the current goroutine first, if we haven't already. 276 if((gp = m->curg) != nil && gp != me) { 277 runtime·printf("\n"); 278 runtime·goroutineheader(gp); 279 runtime·traceback(~(uintptr)0, ~(uintptr)0, 0, gp); 280 } 281 282 for(gp = runtime·allg; gp != nil; gp = gp->alllink) { 283 if(gp == me || gp == m->curg || gp->status == Gdead) 284 continue; 285 if(gp->issystem && traceback < 2) 286 continue; 287 runtime·printf("\n"); 288 runtime·goroutineheader(gp); 289 if(gp->status == Grunning) { 290 runtime·printf("\tgoroutine running on other thread; stack unavailable\n"); 291 runtime·printcreatedby(gp); 292 } else 293 runtime·traceback(~(uintptr)0, ~(uintptr)0, 0, gp); 294 } 295 } 296 297 static void 298 checkmcount(void) 299 { 300 // sched lock is held 301 if(runtime·sched.mcount > runtime·sched.maxmcount) { 302 runtime·printf("runtime: program exceeds %d-thread limit\n", runtime·sched.maxmcount); 303 runtime·throw("thread exhaustion"); 304 } 305 } 306 307 static void 308 mcommoninit(M *mp) 309 { 310 // If there is no mcache runtime·callers() will crash, 311 // and we are most likely in sysmon thread so the stack is senseless anyway. 312 if(m->mcache) 313 runtime·callers(1, mp->createstack, nelem(mp->createstack)); 314 315 mp->fastrand = 0x49f6428aUL + mp->id + runtime·cputicks(); 316 317 runtime·lock(&runtime·sched); 318 mp->id = runtime·sched.mcount++; 319 checkmcount(); 320 runtime·mpreinit(mp); 321 322 // Add to runtime·allm so garbage collector doesn't free m 323 // when it is just in a register or thread-local storage. 324 mp->alllink = runtime·allm; 325 // runtime·NumCgoCall() iterates over allm w/o schedlock, 326 // so we need to publish it safely. 327 runtime·atomicstorep(&runtime·allm, mp); 328 runtime·unlock(&runtime·sched); 329 } 330 331 // Mark gp ready to run. 332 void 333 runtime·ready(G *gp) 334 { 335 // Mark runnable. 336 m->locks++; // disable preemption because it can be holding p in a local var 337 if(gp->status != Gwaiting) { 338 runtime·printf("goroutine %D has status %d\n", gp->goid, gp->status); 339 runtime·throw("bad g->status in ready"); 340 } 341 gp->status = Grunnable; 342 runqput(m->p, gp); 343 if(runtime·atomicload(&runtime·sched.npidle) != 0 && runtime·atomicload(&runtime·sched.nmspinning) == 0) // TODO: fast atomic 344 wakep(); 345 m->locks--; 346 if(m->locks == 0 && g->preempt) // restore the preemption request in case we've cleared it in newstack 347 g->stackguard0 = StackPreempt; 348 } 349 350 int32 351 runtime·gcprocs(void) 352 { 353 int32 n; 354 355 // Figure out how many CPUs to use during GC. 356 // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc. 357 runtime·lock(&runtime·sched); 358 n = runtime·gomaxprocs; 359 if(n > runtime·ncpu) 360 n = runtime·ncpu; 361 if(n > MaxGcproc) 362 n = MaxGcproc; 363 if(n > runtime·sched.nmidle+1) // one M is currently running 364 n = runtime·sched.nmidle+1; 365 runtime·unlock(&runtime·sched); 366 return n; 367 } 368 369 static bool 370 needaddgcproc(void) 371 { 372 int32 n; 373 374 runtime·lock(&runtime·sched); 375 n = runtime·gomaxprocs; 376 if(n > runtime·ncpu) 377 n = runtime·ncpu; 378 if(n > MaxGcproc) 379 n = MaxGcproc; 380 n -= runtime·sched.nmidle+1; // one M is currently running 381 runtime·unlock(&runtime·sched); 382 return n > 0; 383 } 384 385 void 386 runtime·helpgc(int32 nproc) 387 { 388 M *mp; 389 int32 n, pos; 390 391 runtime·lock(&runtime·sched); 392 pos = 0; 393 for(n = 1; n < nproc; n++) { // one M is currently running 394 if(runtime·allp[pos]->mcache == m->mcache) 395 pos++; 396 mp = mget(); 397 if(mp == nil) 398 runtime·throw("runtime·gcprocs inconsistency"); 399 mp->helpgc = n; 400 mp->mcache = runtime·allp[pos]->mcache; 401 pos++; 402 runtime·notewakeup(&mp->park); 403 } 404 runtime·unlock(&runtime·sched); 405 } 406 407 // Similar to stoptheworld but best-effort and can be called several times. 408 // There is no reverse operation, used during crashing. 409 // This function must not lock any mutexes. 410 void 411 runtime·freezetheworld(void) 412 { 413 int32 i; 414 415 if(runtime·gomaxprocs == 1) 416 return; 417 // stopwait and preemption requests can be lost 418 // due to races with concurrently executing threads, 419 // so try several times 420 for(i = 0; i < 5; i++) { 421 // this should tell the scheduler to not start any new goroutines 422 runtime·sched.stopwait = 0x7fffffff; 423 runtime·atomicstore((uint32*)&runtime·sched.gcwaiting, 1); 424 // this should stop running goroutines 425 if(!preemptall()) 426 break; // no running goroutines 427 runtime·usleep(1000); 428 } 429 // to be sure 430 runtime·usleep(1000); 431 preemptall(); 432 runtime·usleep(1000); 433 } 434 435 void 436 runtime·stoptheworld(void) 437 { 438 int32 i; 439 uint32 s; 440 P *p; 441 bool wait; 442 443 runtime·lock(&runtime·sched); 444 runtime·sched.stopwait = runtime·gomaxprocs; 445 runtime·atomicstore((uint32*)&runtime·sched.gcwaiting, 1); 446 preemptall(); 447 // stop current P 448 m->p->status = Pgcstop; 449 runtime·sched.stopwait--; 450 // try to retake all P's in Psyscall status 451 for(i = 0; i < runtime·gomaxprocs; i++) { 452 p = runtime·allp[i]; 453 s = p->status; 454 if(s == Psyscall && runtime·cas(&p->status, s, Pgcstop)) 455 runtime·sched.stopwait--; 456 } 457 // stop idle P's 458 while(p = pidleget()) { 459 p->status = Pgcstop; 460 runtime·sched.stopwait--; 461 } 462 wait = runtime·sched.stopwait > 0; 463 runtime·unlock(&runtime·sched); 464 465 // wait for remaining P's to stop voluntarily 466 if(wait) { 467 for(;;) { 468 // wait for 100us, then try to re-preempt in case of any races 469 if(runtime·notetsleep(&runtime·sched.stopnote, 100*1000)) { 470 runtime·noteclear(&runtime·sched.stopnote); 471 break; 472 } 473 preemptall(); 474 } 475 } 476 if(runtime·sched.stopwait) 477 runtime·throw("stoptheworld: not stopped"); 478 for(i = 0; i < runtime·gomaxprocs; i++) { 479 p = runtime·allp[i]; 480 if(p->status != Pgcstop) 481 runtime·throw("stoptheworld: not stopped"); 482 } 483 } 484 485 static void 486 mhelpgc(void) 487 { 488 m->helpgc = -1; 489 } 490 491 void 492 runtime·starttheworld(void) 493 { 494 P *p, *p1; 495 M *mp; 496 G *gp; 497 bool add; 498 499 m->locks++; // disable preemption because it can be holding p in a local var 500 gp = runtime·netpoll(false); // non-blocking 501 injectglist(gp); 502 add = needaddgcproc(); 503 runtime·lock(&runtime·sched); 504 if(newprocs) { 505 procresize(newprocs); 506 newprocs = 0; 507 } else 508 procresize(runtime·gomaxprocs); 509 runtime·sched.gcwaiting = 0; 510 511 p1 = nil; 512 while(p = pidleget()) { 513 // procresize() puts p's with work at the beginning of the list. 514 // Once we reach a p without a run queue, the rest don't have one either. 515 if(p->runqhead == p->runqtail) { 516 pidleput(p); 517 break; 518 } 519 p->m = mget(); 520 p->link = p1; 521 p1 = p; 522 } 523 if(runtime·sched.sysmonwait) { 524 runtime·sched.sysmonwait = false; 525 runtime·notewakeup(&runtime·sched.sysmonnote); 526 } 527 runtime·unlock(&runtime·sched); 528 529 while(p1) { 530 p = p1; 531 p1 = p1->link; 532 if(p->m) { 533 mp = p->m; 534 p->m = nil; 535 if(mp->nextp) 536 runtime·throw("starttheworld: inconsistent mp->nextp"); 537 mp->nextp = p; 538 runtime·notewakeup(&mp->park); 539 } else { 540 // Start M to run P. Do not start another M below. 541 newm(nil, p); 542 add = false; 543 } 544 } 545 546 if(add) { 547 // If GC could have used another helper proc, start one now, 548 // in the hope that it will be available next time. 549 // It would have been even better to start it before the collection, 550 // but doing so requires allocating memory, so it's tricky to 551 // coordinate. This lazy approach works out in practice: 552 // we don't mind if the first couple gc rounds don't have quite 553 // the maximum number of procs. 554 newm(mhelpgc, nil); 555 } 556 m->locks--; 557 if(m->locks == 0 && g->preempt) // restore the preemption request in case we've cleared it in newstack 558 g->stackguard0 = StackPreempt; 559 } 560 561 // Called to start an M. 562 void 563 runtime·mstart(void) 564 { 565 #ifdef GOOS_windows 566 #ifdef GOARCH_386 567 // It is used by windows-386 only. Unfortunately, seh needs 568 // to be located on os stack, and mstart runs on os stack 569 // for both m0 and m. 570 SEH seh; 571 #endif 572 #endif 573 574 if(g != m->g0) 575 runtime·throw("bad runtime·mstart"); 576 577 // Record top of stack for use by mcall. 578 // Once we call schedule we're never coming back, 579 // so other calls can reuse this stack space. 580 runtime·gosave(&m->g0->sched); 581 m->g0->sched.pc = (uintptr)-1; // make sure it is never used 582 m->g0->stackguard = m->g0->stackguard0; // cgo sets only stackguard0, copy it to stackguard 583 #ifdef GOOS_windows 584 #ifdef GOARCH_386 585 m->seh = &seh; 586 #endif 587 #endif 588 runtime·asminit(); 589 runtime·minit(); 590 591 // Install signal handlers; after minit so that minit can 592 // prepare the thread to be able to handle the signals. 593 if(m == &runtime·m0) 594 runtime·initsig(); 595 596 if(m->mstartfn) 597 m->mstartfn(); 598 599 if(m->helpgc) { 600 m->helpgc = 0; 601 stopm(); 602 } else if(m != &runtime·m0) { 603 acquirep(m->nextp); 604 m->nextp = nil; 605 } 606 schedule(); 607 608 // TODO(brainman): This point is never reached, because scheduler 609 // does not release os threads at the moment. But once this path 610 // is enabled, we must remove our seh here. 611 } 612 613 // When running with cgo, we call _cgo_thread_start 614 // to start threads for us so that we can play nicely with 615 // foreign code. 616 void (*_cgo_thread_start)(void*); 617 618 typedef struct CgoThreadStart CgoThreadStart; 619 struct CgoThreadStart 620 { 621 M *m; 622 G *g; 623 void (*fn)(void); 624 }; 625 626 // Allocate a new m unassociated with any thread. 627 // Can use p for allocation context if needed. 628 M* 629 runtime·allocm(P *p) 630 { 631 M *mp; 632 static Type *mtype; // The Go type M 633 634 m->locks++; // disable GC because it can be called from sysmon 635 if(m->p == nil) 636 acquirep(p); // temporarily borrow p for mallocs in this function 637 if(mtype == nil) { 638 Eface e; 639 runtime·gc_m_ptr(&e); 640 mtype = ((PtrType*)e.type)->elem; 641 } 642 643 mp = runtime·cnew(mtype); 644 mcommoninit(mp); 645 646 // In case of cgo, pthread_create will make us a stack. 647 // Windows will layout sched stack on OS stack. 648 if(runtime·iscgo || Windows) 649 mp->g0 = runtime·malg(-1); 650 else 651 mp->g0 = runtime·malg(8192); 652 653 if(p == m->p) 654 releasep(); 655 m->locks--; 656 if(m->locks == 0 && g->preempt) // restore the preemption request in case we've cleared it in newstack 657 g->stackguard0 = StackPreempt; 658 659 return mp; 660 } 661 662 static M* lockextra(bool nilokay); 663 static void unlockextra(M*); 664 665 // needm is called when a cgo callback happens on a 666 // thread without an m (a thread not created by Go). 667 // In this case, needm is expected to find an m to use 668 // and return with m, g initialized correctly. 669 // Since m and g are not set now (likely nil, but see below) 670 // needm is limited in what routines it can call. In particular 671 // it can only call nosplit functions (textflag 7) and cannot 672 // do any scheduling that requires an m. 673 // 674 // In order to avoid needing heavy lifting here, we adopt 675 // the following strategy: there is a stack of available m's 676 // that can be stolen. Using compare-and-swap 677 // to pop from the stack has ABA races, so we simulate 678 // a lock by doing an exchange (via casp) to steal the stack 679 // head and replace the top pointer with MLOCKED (1). 680 // This serves as a simple spin lock that we can use even 681 // without an m. The thread that locks the stack in this way 682 // unlocks the stack by storing a valid stack head pointer. 683 // 684 // In order to make sure that there is always an m structure 685 // available to be stolen, we maintain the invariant that there 686 // is always one more than needed. At the beginning of the 687 // program (if cgo is in use) the list is seeded with a single m. 688 // If needm finds that it has taken the last m off the list, its job 689 // is - once it has installed its own m so that it can do things like 690 // allocate memory - to create a spare m and put it on the list. 691 // 692 // Each of these extra m's also has a g0 and a curg that are 693 // pressed into service as the scheduling stack and current 694 // goroutine for the duration of the cgo callback. 695 // 696 // When the callback is done with the m, it calls dropm to 697 // put the m back on the list. 698 #pragma textflag NOSPLIT 699 void 700 runtime·needm(byte x) 701 { 702 M *mp; 703 704 if(runtime·needextram) { 705 // Can happen if C/C++ code calls Go from a global ctor. 706 // Can not throw, because scheduler is not initialized yet. 707 runtime·write(2, "fatal error: cgo callback before cgo call\n", 708 sizeof("fatal error: cgo callback before cgo call\n")-1); 709 runtime·exit(1); 710 } 711 712 // Lock extra list, take head, unlock popped list. 713 // nilokay=false is safe here because of the invariant above, 714 // that the extra list always contains or will soon contain 715 // at least one m. 716 mp = lockextra(false); 717 718 // Set needextram when we've just emptied the list, 719 // so that the eventual call into cgocallbackg will 720 // allocate a new m for the extra list. We delay the 721 // allocation until then so that it can be done 722 // after exitsyscall makes sure it is okay to be 723 // running at all (that is, there's no garbage collection 724 // running right now). 725 mp->needextram = mp->schedlink == nil; 726 unlockextra(mp->schedlink); 727 728 // Install m and g (= m->g0) and set the stack bounds 729 // to match the current stack. We don't actually know 730 // how big the stack is, like we don't know how big any 731 // scheduling stack is, but we assume there's at least 32 kB, 732 // which is more than enough for us. 733 runtime·setmg(mp, mp->g0); 734 g->stackbase = (uintptr)(&x + 1024); 735 g->stackguard = (uintptr)(&x - 32*1024); 736 g->stackguard0 = g->stackguard; 737 738 #ifdef GOOS_windows 739 #ifdef GOARCH_386 740 // On windows/386, we need to put an SEH frame (two words) 741 // somewhere on the current stack. We are called from cgocallback_gofunc 742 // and we know that it will leave two unused words below m->curg->sched.sp. 743 // Use those. 744 m->seh = (SEH*)((uintptr*)&x + 1); 745 #endif 746 #endif 747 748 // Initialize this thread to use the m. 749 runtime·asminit(); 750 runtime·minit(); 751 } 752 753 // newextram allocates an m and puts it on the extra list. 754 // It is called with a working local m, so that it can do things 755 // like call schedlock and allocate. 756 void 757 runtime·newextram(void) 758 { 759 M *mp, *mnext; 760 G *gp; 761 762 // Create extra goroutine locked to extra m. 763 // The goroutine is the context in which the cgo callback will run. 764 // The sched.pc will never be returned to, but setting it to 765 // runtime.goexit makes clear to the traceback routines where 766 // the goroutine stack ends. 767 mp = runtime·allocm(nil); 768 gp = runtime·malg(4096); 769 gp->sched.pc = (uintptr)runtime·goexit; 770 gp->sched.sp = gp->stackbase; 771 gp->sched.lr = 0; 772 gp->sched.g = gp; 773 gp->syscallpc = gp->sched.pc; 774 gp->syscallsp = gp->sched.sp; 775 gp->syscallstack = gp->stackbase; 776 gp->syscallguard = gp->stackguard; 777 gp->status = Gsyscall; 778 mp->curg = gp; 779 mp->locked = LockInternal; 780 mp->lockedg = gp; 781 gp->lockedm = mp; 782 gp->goid = runtime·xadd64(&runtime·sched.goidgen, 1); 783 if(raceenabled) 784 gp->racectx = runtime·racegostart(runtime·newextram); 785 // put on allg for garbage collector 786 runtime·lock(&runtime·sched); 787 if(runtime·lastg == nil) 788 runtime·allg = gp; 789 else 790 runtime·lastg->alllink = gp; 791 runtime·lastg = gp; 792 runtime·unlock(&runtime·sched); 793 794 // Add m to the extra list. 795 mnext = lockextra(true); 796 mp->schedlink = mnext; 797 unlockextra(mp); 798 } 799 800 // dropm is called when a cgo callback has called needm but is now 801 // done with the callback and returning back into the non-Go thread. 802 // It puts the current m back onto the extra list. 803 // 804 // The main expense here is the call to signalstack to release the 805 // m's signal stack, and then the call to needm on the next callback 806 // from this thread. It is tempting to try to save the m for next time, 807 // which would eliminate both these costs, but there might not be 808 // a next time: the current thread (which Go does not control) might exit. 809 // If we saved the m for that thread, there would be an m leak each time 810 // such a thread exited. Instead, we acquire and release an m on each 811 // call. These should typically not be scheduling operations, just a few 812 // atomics, so the cost should be small. 813 // 814 // TODO(rsc): An alternative would be to allocate a dummy pthread per-thread 815 // variable using pthread_key_create. Unlike the pthread keys we already use 816 // on OS X, this dummy key would never be read by Go code. It would exist 817 // only so that we could register at thread-exit-time destructor. 818 // That destructor would put the m back onto the extra list. 819 // This is purely a performance optimization. The current version, 820 // in which dropm happens on each cgo call, is still correct too. 821 // We may have to keep the current version on systems with cgo 822 // but without pthreads, like Windows. 823 void 824 runtime·dropm(void) 825 { 826 M *mp, *mnext; 827 828 // Undo whatever initialization minit did during needm. 829 runtime·unminit(); 830 831 #ifdef GOOS_windows 832 #ifdef GOARCH_386 833 m->seh = nil; // reset dangling typed pointer 834 #endif 835 #endif 836 837 // Clear m and g, and return m to the extra list. 838 // After the call to setmg we can only call nosplit functions. 839 mp = m; 840 runtime·setmg(nil, nil); 841 842 mnext = lockextra(true); 843 mp->schedlink = mnext; 844 unlockextra(mp); 845 } 846 847 #define MLOCKED ((M*)1) 848 849 // lockextra locks the extra list and returns the list head. 850 // The caller must unlock the list by storing a new list head 851 // to runtime.extram. If nilokay is true, then lockextra will 852 // return a nil list head if that's what it finds. If nilokay is false, 853 // lockextra will keep waiting until the list head is no longer nil. 854 #pragma textflag NOSPLIT 855 static M* 856 lockextra(bool nilokay) 857 { 858 M *mp; 859 void (*yield)(void); 860 861 for(;;) { 862 mp = runtime·atomicloadp(&runtime·extram); 863 if(mp == MLOCKED) { 864 yield = runtime·osyield; 865 yield(); 866 continue; 867 } 868 if(mp == nil && !nilokay) { 869 runtime·usleep(1); 870 continue; 871 } 872 if(!runtime·casp(&runtime·extram, mp, MLOCKED)) { 873 yield = runtime·osyield; 874 yield(); 875 continue; 876 } 877 break; 878 } 879 return mp; 880 } 881 882 #pragma textflag NOSPLIT 883 static void 884 unlockextra(M *mp) 885 { 886 runtime·atomicstorep(&runtime·extram, mp); 887 } 888 889 890 // Create a new m. It will start off with a call to fn, or else the scheduler. 891 static void 892 newm(void(*fn)(void), P *p) 893 { 894 M *mp; 895 896 mp = runtime·allocm(p); 897 mp->nextp = p; 898 mp->mstartfn = fn; 899 900 if(runtime·iscgo) { 901 CgoThreadStart ts; 902 903 if(_cgo_thread_start == nil) 904 runtime·throw("_cgo_thread_start missing"); 905 ts.m = mp; 906 ts.g = mp->g0; 907 ts.fn = runtime·mstart; 908 runtime·asmcgocall(_cgo_thread_start, &ts); 909 return; 910 } 911 runtime·newosproc(mp, (byte*)mp->g0->stackbase); 912 } 913 914 // Stops execution of the current m until new work is available. 915 // Returns with acquired P. 916 static void 917 stopm(void) 918 { 919 if(m->locks) 920 runtime·throw("stopm holding locks"); 921 if(m->p) 922 runtime·throw("stopm holding p"); 923 if(m->spinning) { 924 m->spinning = false; 925 runtime·xadd(&runtime·sched.nmspinning, -1); 926 } 927 928 retry: 929 runtime·lock(&runtime·sched); 930 mput(m); 931 runtime·unlock(&runtime·sched); 932 runtime·notesleep(&m->park); 933 runtime·noteclear(&m->park); 934 if(m->helpgc) { 935 runtime·gchelper(); 936 m->helpgc = 0; 937 m->mcache = nil; 938 goto retry; 939 } 940 acquirep(m->nextp); 941 m->nextp = nil; 942 } 943 944 static void 945 mspinning(void) 946 { 947 m->spinning = true; 948 } 949 950 // Schedules some M to run the p (creates an M if necessary). 951 // If p==nil, tries to get an idle P, if no idle P's returns false. 952 static void 953 startm(P *p, bool spinning) 954 { 955 M *mp; 956 void (*fn)(void); 957 958 runtime·lock(&runtime·sched); 959 if(p == nil) { 960 p = pidleget(); 961 if(p == nil) { 962 runtime·unlock(&runtime·sched); 963 if(spinning) 964 runtime·xadd(&runtime·sched.nmspinning, -1); 965 return; 966 } 967 } 968 mp = mget(); 969 runtime·unlock(&runtime·sched); 970 if(mp == nil) { 971 fn = nil; 972 if(spinning) 973 fn = mspinning; 974 newm(fn, p); 975 return; 976 } 977 if(mp->spinning) 978 runtime·throw("startm: m is spinning"); 979 if(mp->nextp) 980 runtime·throw("startm: m has p"); 981 mp->spinning = spinning; 982 mp->nextp = p; 983 runtime·notewakeup(&mp->park); 984 } 985 986 // Hands off P from syscall or locked M. 987 static void 988 handoffp(P *p) 989 { 990 // if it has local work, start it straight away 991 if(p->runqhead != p->runqtail || runtime·sched.runqsize) { 992 startm(p, false); 993 return; 994 } 995 // no local work, check that there are no spinning/idle M's, 996 // otherwise our help is not required 997 if(runtime·atomicload(&runtime·sched.nmspinning) + runtime·atomicload(&runtime·sched.npidle) == 0 && // TODO: fast atomic 998 runtime·cas(&runtime·sched.nmspinning, 0, 1)) { 999 startm(p, true); 1000 return; 1001 } 1002 runtime·lock(&runtime·sched); 1003 if(runtime·sched.gcwaiting) { 1004 p->status = Pgcstop; 1005 if(--runtime·sched.stopwait == 0) 1006 runtime·notewakeup(&runtime·sched.stopnote); 1007 runtime·unlock(&runtime·sched); 1008 return; 1009 } 1010 if(runtime·sched.runqsize) { 1011 runtime·unlock(&runtime·sched); 1012 startm(p, false); 1013 return; 1014 } 1015 // If this is the last running P and nobody is polling network, 1016 // need to wakeup another M to poll network. 1017 if(runtime·sched.npidle == runtime·gomaxprocs-1 && runtime·atomicload64(&runtime·sched.lastpoll) != 0) { 1018 runtime·unlock(&runtime·sched); 1019 startm(p, false); 1020 return; 1021 } 1022 pidleput(p); 1023 runtime·unlock(&runtime·sched); 1024 } 1025 1026 // Tries to add one more P to execute G's. 1027 // Called when a G is made runnable (newproc, ready). 1028 static void 1029 wakep(void) 1030 { 1031 // be conservative about spinning threads 1032 if(!runtime·cas(&runtime·sched.nmspinning, 0, 1)) 1033 return; 1034 startm(nil, true); 1035 } 1036 1037 // Stops execution of the current m that is locked to a g until the g is runnable again. 1038 // Returns with acquired P. 1039 static void 1040 stoplockedm(void) 1041 { 1042 P *p; 1043 1044 if(m->lockedg == nil || m->lockedg->lockedm != m) 1045 runtime·throw("stoplockedm: inconsistent locking"); 1046 if(m->p) { 1047 // Schedule another M to run this p. 1048 p = releasep(); 1049 handoffp(p); 1050 } 1051 incidlelocked(1); 1052 // Wait until another thread schedules lockedg again. 1053 runtime·notesleep(&m->park); 1054 runtime·noteclear(&m->park); 1055 if(m->lockedg->status != Grunnable) 1056 runtime·throw("stoplockedm: not runnable"); 1057 acquirep(m->nextp); 1058 m->nextp = nil; 1059 } 1060 1061 // Schedules the locked m to run the locked gp. 1062 static void 1063 startlockedm(G *gp) 1064 { 1065 M *mp; 1066 P *p; 1067 1068 mp = gp->lockedm; 1069 if(mp == m) 1070 runtime·throw("startlockedm: locked to me"); 1071 if(mp->nextp) 1072 runtime·throw("startlockedm: m has p"); 1073 // directly handoff current P to the locked m 1074 incidlelocked(-1); 1075 p = releasep(); 1076 mp->nextp = p; 1077 runtime·notewakeup(&mp->park); 1078 stopm(); 1079 } 1080 1081 // Stops the current m for stoptheworld. 1082 // Returns when the world is restarted. 1083 static void 1084 gcstopm(void) 1085 { 1086 P *p; 1087 1088 if(!runtime·sched.gcwaiting) 1089 runtime·throw("gcstopm: not waiting for gc"); 1090 if(m->spinning) { 1091 m->spinning = false; 1092 runtime·xadd(&runtime·sched.nmspinning, -1); 1093 } 1094 p = releasep(); 1095 runtime·lock(&runtime·sched); 1096 p->status = Pgcstop; 1097 if(--runtime·sched.stopwait == 0) 1098 runtime·notewakeup(&runtime·sched.stopnote); 1099 runtime·unlock(&runtime·sched); 1100 stopm(); 1101 } 1102 1103 // Schedules gp to run on the current M. 1104 // Never returns. 1105 static void 1106 execute(G *gp) 1107 { 1108 int32 hz; 1109 1110 if(gp->status != Grunnable) { 1111 runtime·printf("execute: bad g status %d\n", gp->status); 1112 runtime·throw("execute: bad g status"); 1113 } 1114 gp->status = Grunning; 1115 gp->preempt = false; 1116 gp->stackguard0 = gp->stackguard; 1117 m->p->schedtick++; 1118 m->curg = gp; 1119 gp->m = m; 1120 1121 // Check whether the profiler needs to be turned on or off. 1122 hz = runtime·sched.profilehz; 1123 if(m->profilehz != hz) 1124 runtime·resetcpuprofiler(hz); 1125 1126 runtime·gogo(&gp->sched); 1127 } 1128 1129 // Finds a runnable goroutine to execute. 1130 // Tries to steal from other P's, get g from global queue, poll network. 1131 static G* 1132 findrunnable(void) 1133 { 1134 G *gp; 1135 P *p; 1136 int32 i; 1137 1138 top: 1139 if(runtime·sched.gcwaiting) { 1140 gcstopm(); 1141 goto top; 1142 } 1143 // local runq 1144 gp = runqget(m->p); 1145 if(gp) 1146 return gp; 1147 // global runq 1148 if(runtime·sched.runqsize) { 1149 runtime·lock(&runtime·sched); 1150 gp = globrunqget(m->p, 0); 1151 runtime·unlock(&runtime·sched); 1152 if(gp) 1153 return gp; 1154 } 1155 // poll network 1156 gp = runtime·netpoll(false); // non-blocking 1157 if(gp) { 1158 injectglist(gp->schedlink); 1159 gp->status = Grunnable; 1160 return gp; 1161 } 1162 // If number of spinning M's >= number of busy P's, block. 1163 // This is necessary to prevent excessive CPU consumption 1164 // when GOMAXPROCS>>1 but the program parallelism is low. 1165 if(!m->spinning && 2 * runtime·atomicload(&runtime·sched.nmspinning) >= runtime·gomaxprocs - runtime·atomicload(&runtime·sched.npidle)) // TODO: fast atomic 1166 goto stop; 1167 if(!m->spinning) { 1168 m->spinning = true; 1169 runtime·xadd(&runtime·sched.nmspinning, 1); 1170 } 1171 // random steal from other P's 1172 for(i = 0; i < 2*runtime·gomaxprocs; i++) { 1173 if(runtime·sched.gcwaiting) 1174 goto top; 1175 p = runtime·allp[runtime·fastrand1()%runtime·gomaxprocs]; 1176 if(p == m->p) 1177 gp = runqget(p); 1178 else 1179 gp = runqsteal(m->p, p); 1180 if(gp) 1181 return gp; 1182 } 1183 stop: 1184 // return P and block 1185 runtime·lock(&runtime·sched); 1186 if(runtime·sched.gcwaiting) { 1187 runtime·unlock(&runtime·sched); 1188 goto top; 1189 } 1190 if(runtime·sched.runqsize) { 1191 gp = globrunqget(m->p, 0); 1192 runtime·unlock(&runtime·sched); 1193 return gp; 1194 } 1195 p = releasep(); 1196 pidleput(p); 1197 runtime·unlock(&runtime·sched); 1198 if(m->spinning) { 1199 m->spinning = false; 1200 runtime·xadd(&runtime·sched.nmspinning, -1); 1201 } 1202 // check all runqueues once again 1203 for(i = 0; i < runtime·gomaxprocs; i++) { 1204 p = runtime·allp[i]; 1205 if(p && p->runqhead != p->runqtail) { 1206 runtime·lock(&runtime·sched); 1207 p = pidleget(); 1208 runtime·unlock(&runtime·sched); 1209 if(p) { 1210 acquirep(p); 1211 goto top; 1212 } 1213 break; 1214 } 1215 } 1216 // poll network 1217 if(runtime·xchg64(&runtime·sched.lastpoll, 0) != 0) { 1218 if(m->p) 1219 runtime·throw("findrunnable: netpoll with p"); 1220 if(m->spinning) 1221 runtime·throw("findrunnable: netpoll with spinning"); 1222 gp = runtime·netpoll(true); // block until new work is available 1223 runtime·atomicstore64(&runtime·sched.lastpoll, runtime·nanotime()); 1224 if(gp) { 1225 runtime·lock(&runtime·sched); 1226 p = pidleget(); 1227 runtime·unlock(&runtime·sched); 1228 if(p) { 1229 acquirep(p); 1230 injectglist(gp->schedlink); 1231 gp->status = Grunnable; 1232 return gp; 1233 } 1234 injectglist(gp); 1235 } 1236 } 1237 stopm(); 1238 goto top; 1239 } 1240 1241 static void 1242 resetspinning(void) 1243 { 1244 int32 nmspinning; 1245 1246 if(m->spinning) { 1247 m->spinning = false; 1248 nmspinning = runtime·xadd(&runtime·sched.nmspinning, -1); 1249 if(nmspinning < 0) 1250 runtime·throw("findrunnable: negative nmspinning"); 1251 } else 1252 nmspinning = runtime·atomicload(&runtime·sched.nmspinning); 1253 1254 // M wakeup policy is deliberately somewhat conservative (see nmspinning handling), 1255 // so see if we need to wakeup another P here. 1256 if (nmspinning == 0 && runtime·atomicload(&runtime·sched.npidle) > 0) 1257 wakep(); 1258 } 1259 1260 // Injects the list of runnable G's into the scheduler. 1261 // Can run concurrently with GC. 1262 static void 1263 injectglist(G *glist) 1264 { 1265 int32 n; 1266 G *gp; 1267 1268 if(glist == nil) 1269 return; 1270 runtime·lock(&runtime·sched); 1271 for(n = 0; glist; n++) { 1272 gp = glist; 1273 glist = gp->schedlink; 1274 gp->status = Grunnable; 1275 globrunqput(gp); 1276 } 1277 runtime·unlock(&runtime·sched); 1278 1279 for(; n && runtime·sched.npidle; n--) 1280 startm(nil, false); 1281 } 1282 1283 // One round of scheduler: find a runnable goroutine and execute it. 1284 // Never returns. 1285 static void 1286 schedule(void) 1287 { 1288 G *gp; 1289 uint32 tick; 1290 1291 if(m->locks) 1292 runtime·throw("schedule: holding locks"); 1293 1294 top: 1295 if(runtime·sched.gcwaiting) { 1296 gcstopm(); 1297 goto top; 1298 } 1299 1300 gp = nil; 1301 // Check the global runnable queue once in a while to ensure fairness. 1302 // Otherwise two goroutines can completely occupy the local runqueue 1303 // by constantly respawning each other. 1304 tick = m->p->schedtick; 1305 // This is a fancy way to say tick%61==0, 1306 // it uses 2 MUL instructions instead of a single DIV and so is faster on modern processors. 1307 if(tick - (((uint64)tick*0x4325c53fu)>>36)*61 == 0 && runtime·sched.runqsize > 0) { 1308 runtime·lock(&runtime·sched); 1309 gp = globrunqget(m->p, 1); 1310 runtime·unlock(&runtime·sched); 1311 if(gp) 1312 resetspinning(); 1313 } 1314 if(gp == nil) { 1315 gp = runqget(m->p); 1316 if(gp && m->spinning) 1317 runtime·throw("schedule: spinning with local work"); 1318 } 1319 if(gp == nil) { 1320 gp = findrunnable(); // blocks until work is available 1321 resetspinning(); 1322 } 1323 1324 if(gp->lockedm) { 1325 // Hands off own p to the locked m, 1326 // then blocks waiting for a new p. 1327 startlockedm(gp); 1328 goto top; 1329 } 1330 1331 execute(gp); 1332 } 1333 1334 // Puts the current goroutine into a waiting state and unlocks the lock. 1335 // The goroutine can be made runnable again by calling runtime·ready(gp). 1336 void 1337 runtime·park(void(*unlockf)(Lock*), Lock *lock, int8 *reason) 1338 { 1339 m->waitlock = lock; 1340 m->waitunlockf = unlockf; 1341 g->waitreason = reason; 1342 runtime·mcall(park0); 1343 } 1344 1345 // runtime·park continuation on g0. 1346 static void 1347 park0(G *gp) 1348 { 1349 gp->status = Gwaiting; 1350 gp->m = nil; 1351 m->curg = nil; 1352 if(m->waitunlockf) { 1353 m->waitunlockf(m->waitlock); 1354 m->waitunlockf = nil; 1355 m->waitlock = nil; 1356 } 1357 if(m->lockedg) { 1358 stoplockedm(); 1359 execute(gp); // Never returns. 1360 } 1361 schedule(); 1362 } 1363 1364 // Scheduler yield. 1365 void 1366 runtime·gosched(void) 1367 { 1368 runtime·mcall(runtime·gosched0); 1369 } 1370 1371 // runtime·gosched continuation on g0. 1372 void 1373 runtime·gosched0(G *gp) 1374 { 1375 gp->status = Grunnable; 1376 gp->m = nil; 1377 m->curg = nil; 1378 runtime·lock(&runtime·sched); 1379 globrunqput(gp); 1380 runtime·unlock(&runtime·sched); 1381 if(m->lockedg) { 1382 stoplockedm(); 1383 execute(gp); // Never returns. 1384 } 1385 schedule(); 1386 } 1387 1388 // Finishes execution of the current goroutine. 1389 // Need to mark it as nosplit, because it runs with sp > stackbase (as runtime·lessstack). 1390 // Since it does not return it does not matter. But if it is preempted 1391 // at the split stack check, GC will complain about inconsistent sp. 1392 #pragma textflag NOSPLIT 1393 void 1394 runtime·goexit(void) 1395 { 1396 if(raceenabled) 1397 runtime·racegoend(); 1398 runtime·mcall(goexit0); 1399 } 1400 1401 // runtime·goexit continuation on g0. 1402 static void 1403 goexit0(G *gp) 1404 { 1405 gp->status = Gdead; 1406 gp->m = nil; 1407 gp->lockedm = nil; 1408 m->curg = nil; 1409 m->lockedg = nil; 1410 if(m->locked & ~LockExternal) { 1411 runtime·printf("invalid m->locked = %d\n", m->locked); 1412 runtime·throw("internal lockOSThread error"); 1413 } 1414 m->locked = 0; 1415 runtime·unwindstack(gp, nil); 1416 gfput(m->p, gp); 1417 schedule(); 1418 } 1419 1420 #pragma textflag NOSPLIT 1421 static void 1422 save(void *pc, uintptr sp) 1423 { 1424 g->sched.pc = (uintptr)pc; 1425 g->sched.sp = sp; 1426 g->sched.lr = 0; 1427 g->sched.ret = 0; 1428 g->sched.ctxt = 0; 1429 g->sched.g = g; 1430 } 1431 1432 // The goroutine g is about to enter a system call. 1433 // Record that it's not using the cpu anymore. 1434 // This is called only from the go syscall library and cgocall, 1435 // not from the low-level system calls used by the runtime. 1436 // 1437 // Entersyscall cannot split the stack: the runtime·gosave must 1438 // make g->sched refer to the caller's stack segment, because 1439 // entersyscall is going to return immediately after. 1440 #pragma textflag NOSPLIT 1441 void 1442 ·entersyscall(int32 dummy) 1443 { 1444 // Disable preemption because during this function g is in Gsyscall status, 1445 // but can have inconsistent g->sched, do not let GC observe it. 1446 m->locks++; 1447 1448 // Leave SP around for GC and traceback. 1449 save(runtime·getcallerpc(&dummy), runtime·getcallersp(&dummy)); 1450 g->syscallsp = g->sched.sp; 1451 g->syscallpc = g->sched.pc; 1452 g->syscallstack = g->stackbase; 1453 g->syscallguard = g->stackguard; 1454 g->status = Gsyscall; 1455 if(g->syscallsp < g->syscallguard-StackGuard || g->syscallstack < g->syscallsp) { 1456 // runtime·printf("entersyscall inconsistent %p [%p,%p]\n", 1457 // g->syscallsp, g->syscallguard-StackGuard, g->syscallstack); 1458 runtime·throw("entersyscall"); 1459 } 1460 1461 if(runtime·atomicload(&runtime·sched.sysmonwait)) { // TODO: fast atomic 1462 runtime·lock(&runtime·sched); 1463 if(runtime·atomicload(&runtime·sched.sysmonwait)) { 1464 runtime·atomicstore(&runtime·sched.sysmonwait, 0); 1465 runtime·notewakeup(&runtime·sched.sysmonnote); 1466 } 1467 runtime·unlock(&runtime·sched); 1468 save(runtime·getcallerpc(&dummy), runtime·getcallersp(&dummy)); 1469 } 1470 1471 m->mcache = nil; 1472 m->p->m = nil; 1473 runtime·atomicstore(&m->p->status, Psyscall); 1474 if(runtime·sched.gcwaiting) { 1475 runtime·lock(&runtime·sched); 1476 if (runtime·sched.stopwait > 0 && runtime·cas(&m->p->status, Psyscall, Pgcstop)) { 1477 if(--runtime·sched.stopwait == 0) 1478 runtime·notewakeup(&runtime·sched.stopnote); 1479 } 1480 runtime·unlock(&runtime·sched); 1481 save(runtime·getcallerpc(&dummy), runtime·getcallersp(&dummy)); 1482 } 1483 1484 // Goroutines must not split stacks in Gsyscall status (it would corrupt g->sched). 1485 // We set stackguard to StackPreempt so that first split stack check calls morestack. 1486 // Morestack detects this case and throws. 1487 g->stackguard0 = StackPreempt; 1488 m->locks--; 1489 } 1490 1491 // The same as runtime·entersyscall(), but with a hint that the syscall is blocking. 1492 #pragma textflag NOSPLIT 1493 void 1494 ·entersyscallblock(int32 dummy) 1495 { 1496 P *p; 1497 1498 m->locks++; // see comment in entersyscall 1499 1500 // Leave SP around for GC and traceback. 1501 save(runtime·getcallerpc(&dummy), runtime·getcallersp(&dummy)); 1502 g->syscallsp = g->sched.sp; 1503 g->syscallpc = g->sched.pc; 1504 g->syscallstack = g->stackbase; 1505 g->syscallguard = g->stackguard; 1506 g->status = Gsyscall; 1507 if(g->syscallsp < g->syscallguard-StackGuard || g->syscallstack < g->syscallsp) { 1508 // runtime·printf("entersyscall inconsistent %p [%p,%p]\n", 1509 // g->syscallsp, g->syscallguard-StackGuard, g->syscallstack); 1510 runtime·throw("entersyscallblock"); 1511 } 1512 1513 p = releasep(); 1514 handoffp(p); 1515 if(g->isbackground) // do not consider blocked scavenger for deadlock detection 1516 incidlelocked(1); 1517 1518 // Resave for traceback during blocked call. 1519 save(runtime·getcallerpc(&dummy), runtime·getcallersp(&dummy)); 1520 1521 g->stackguard0 = StackPreempt; // see comment in entersyscall 1522 m->locks--; 1523 } 1524 1525 // The goroutine g exited its system call. 1526 // Arrange for it to run on a cpu again. 1527 // This is called only from the go syscall library, not 1528 // from the low-level system calls used by the runtime. 1529 #pragma textflag NOSPLIT 1530 void 1531 runtime·exitsyscall(void) 1532 { 1533 m->locks++; // see comment in entersyscall 1534 1535 if(g->isbackground) // do not consider blocked scavenger for deadlock detection 1536 incidlelocked(-1); 1537 1538 if(exitsyscallfast()) { 1539 // There's a cpu for us, so we can run. 1540 m->p->syscalltick++; 1541 g->status = Grunning; 1542 // Garbage collector isn't running (since we are), 1543 // so okay to clear gcstack and gcsp. 1544 g->syscallstack = (uintptr)nil; 1545 g->syscallsp = (uintptr)nil; 1546 m->locks--; 1547 if(g->preempt) { 1548 // restore the preemption request in case we've cleared it in newstack 1549 g->stackguard0 = StackPreempt; 1550 } else { 1551 // otherwise restore the real stackguard, we've spoiled it in entersyscall/entersyscallblock 1552 g->stackguard0 = g->stackguard; 1553 } 1554 return; 1555 } 1556 1557 m->locks--; 1558 1559 // Call the scheduler. 1560 runtime·mcall(exitsyscall0); 1561 1562 // Scheduler returned, so we're allowed to run now. 1563 // Delete the gcstack information that we left for 1564 // the garbage collector during the system call. 1565 // Must wait until now because until gosched returns 1566 // we don't know for sure that the garbage collector 1567 // is not running. 1568 g->syscallstack = (uintptr)nil; 1569 g->syscallsp = (uintptr)nil; 1570 m->p->syscalltick++; 1571 } 1572 1573 #pragma textflag NOSPLIT 1574 static bool 1575 exitsyscallfast(void) 1576 { 1577 P *p; 1578 1579 // Freezetheworld sets stopwait but does not retake P's. 1580 if(runtime·sched.stopwait) { 1581 m->p = nil; 1582 return false; 1583 } 1584 1585 // Try to re-acquire the last P. 1586 if(m->p && m->p->status == Psyscall && runtime·cas(&m->p->status, Psyscall, Prunning)) { 1587 // There's a cpu for us, so we can run. 1588 m->mcache = m->p->mcache; 1589 m->p->m = m; 1590 return true; 1591 } 1592 // Try to get any other idle P. 1593 m->p = nil; 1594 if(runtime·sched.pidle) { 1595 runtime·lock(&runtime·sched); 1596 p = pidleget(); 1597 if(p && runtime·atomicload(&runtime·sched.sysmonwait)) { 1598 runtime·atomicstore(&runtime·sched.sysmonwait, 0); 1599 runtime·notewakeup(&runtime·sched.sysmonnote); 1600 } 1601 runtime·unlock(&runtime·sched); 1602 if(p) { 1603 acquirep(p); 1604 return true; 1605 } 1606 } 1607 return false; 1608 } 1609 1610 // runtime·exitsyscall slow path on g0. 1611 // Failed to acquire P, enqueue gp as runnable. 1612 static void 1613 exitsyscall0(G *gp) 1614 { 1615 P *p; 1616 1617 gp->status = Grunnable; 1618 gp->m = nil; 1619 m->curg = nil; 1620 runtime·lock(&runtime·sched); 1621 p = pidleget(); 1622 if(p == nil) 1623 globrunqput(gp); 1624 else if(runtime·atomicload(&runtime·sched.sysmonwait)) { 1625 runtime·atomicstore(&runtime·sched.sysmonwait, 0); 1626 runtime·notewakeup(&runtime·sched.sysmonnote); 1627 } 1628 runtime·unlock(&runtime·sched); 1629 if(p) { 1630 acquirep(p); 1631 execute(gp); // Never returns. 1632 } 1633 if(m->lockedg) { 1634 // Wait until another thread schedules gp and so m again. 1635 stoplockedm(); 1636 execute(gp); // Never returns. 1637 } 1638 stopm(); 1639 schedule(); // Never returns. 1640 } 1641 1642 // Called from syscall package before fork. 1643 void 1644 syscall·runtime_BeforeFork(void) 1645 { 1646 // Fork can hang if preempted with signals frequently enough (see issue 5517). 1647 // Ensure that we stay on the same M where we disable profiling. 1648 m->locks++; 1649 if(m->profilehz != 0) 1650 runtime·resetcpuprofiler(0); 1651 } 1652 1653 // Called from syscall package after fork in parent. 1654 void 1655 syscall·runtime_AfterFork(void) 1656 { 1657 int32 hz; 1658 1659 hz = runtime·sched.profilehz; 1660 if(hz != 0) 1661 runtime·resetcpuprofiler(hz); 1662 m->locks--; 1663 } 1664 1665 // Hook used by runtime·malg to call runtime·stackalloc on the 1666 // scheduler stack. This exists because runtime·stackalloc insists 1667 // on being called on the scheduler stack, to avoid trying to grow 1668 // the stack while allocating a new stack segment. 1669 static void 1670 mstackalloc(G *gp) 1671 { 1672 gp->param = runtime·stackalloc((uintptr)gp->param); 1673 runtime·gogo(&gp->sched); 1674 } 1675 1676 // Allocate a new g, with a stack big enough for stacksize bytes. 1677 G* 1678 runtime·malg(int32 stacksize) 1679 { 1680 G *newg; 1681 byte *stk; 1682 1683 if(StackTop < sizeof(Stktop)) { 1684 runtime·printf("runtime: SizeofStktop=%d, should be >=%d\n", (int32)StackTop, (int32)sizeof(Stktop)); 1685 runtime·throw("runtime: bad stack.h"); 1686 } 1687 1688 newg = runtime·malloc(sizeof(G)); 1689 if(stacksize >= 0) { 1690 if(g == m->g0) { 1691 // running on scheduler stack already. 1692 stk = runtime·stackalloc(StackSystem + stacksize); 1693 } else { 1694 // have to call stackalloc on scheduler stack. 1695 g->param = (void*)(StackSystem + stacksize); 1696 runtime·mcall(mstackalloc); 1697 stk = g->param; 1698 g->param = nil; 1699 } 1700 newg->stacksize = StackSystem + stacksize; 1701 newg->stack0 = (uintptr)stk; 1702 newg->stackguard = (uintptr)stk + StackGuard; 1703 newg->stackguard0 = newg->stackguard; 1704 newg->stackbase = (uintptr)stk + StackSystem + stacksize - sizeof(Stktop); 1705 runtime·memclr((byte*)newg->stackbase, sizeof(Stktop)); 1706 } 1707 return newg; 1708 } 1709 1710 // Create a new g running fn with siz bytes of arguments. 1711 // Put it on the queue of g's waiting to run. 1712 // The compiler turns a go statement into a call to this. 1713 // Cannot split the stack because it assumes that the arguments 1714 // are available sequentially after &fn; they would not be 1715 // copied if a stack split occurred. It's OK for this to call 1716 // functions that split the stack. 1717 #pragma textflag NOSPLIT 1718 void 1719 runtime·newproc(int32 siz, FuncVal* fn, ...) 1720 { 1721 byte *argp; 1722 1723 if(thechar == '5') 1724 argp = (byte*)(&fn+2); // skip caller's saved LR 1725 else 1726 argp = (byte*)(&fn+1); 1727 runtime·newproc1(fn, argp, siz, 0, runtime·getcallerpc(&siz)); 1728 } 1729 1730 // Create a new g running fn with narg bytes of arguments starting 1731 // at argp and returning nret bytes of results. callerpc is the 1732 // address of the go statement that created this. The new g is put 1733 // on the queue of g's waiting to run. 1734 G* 1735 runtime·newproc1(FuncVal *fn, byte *argp, int32 narg, int32 nret, void *callerpc) 1736 { 1737 byte *sp; 1738 G *newg; 1739 int32 siz; 1740 1741 //runtime·printf("newproc1 %p %p narg=%d nret=%d\n", fn->fn, argp, narg, nret); 1742 m->locks++; // disable preemption because it can be holding p in a local var 1743 siz = narg + nret; 1744 siz = (siz+7) & ~7; 1745 1746 // We could instead create a secondary stack frame 1747 // and make it look like goexit was on the original but 1748 // the call to the actual goroutine function was split. 1749 // Not worth it: this is almost always an error. 1750 if(siz > StackMin - 1024) 1751 runtime·throw("runtime.newproc: function arguments too large for new goroutine"); 1752 1753 if((newg = gfget(m->p)) != nil) { 1754 if(newg->stackguard - StackGuard != newg->stack0) 1755 runtime·throw("invalid stack in newg"); 1756 } else { 1757 newg = runtime·malg(StackMin); 1758 runtime·lock(&runtime·sched); 1759 if(runtime·lastg == nil) 1760 runtime·allg = newg; 1761 else 1762 runtime·lastg->alllink = newg; 1763 runtime·lastg = newg; 1764 runtime·unlock(&runtime·sched); 1765 } 1766 1767 sp = (byte*)newg->stackbase; 1768 sp -= siz; 1769 runtime·memmove(sp, argp, narg); 1770 if(thechar == '5') { 1771 // caller's LR 1772 sp -= sizeof(void*); 1773 *(void**)sp = nil; 1774 } 1775 1776 runtime·memclr((byte*)&newg->sched, sizeof newg->sched); 1777 newg->sched.sp = (uintptr)sp; 1778 newg->sched.pc = (uintptr)runtime·goexit; 1779 newg->sched.g = newg; 1780 runtime·gostartcallfn(&newg->sched, fn); 1781 newg->gopc = (uintptr)callerpc; 1782 newg->status = Grunnable; 1783 newg->goid = runtime·xadd64(&runtime·sched.goidgen, 1); 1784 newg->panicwrap = 0; 1785 if(raceenabled) 1786 newg->racectx = runtime·racegostart((void*)callerpc); 1787 runqput(m->p, newg); 1788 1789 if(runtime·atomicload(&runtime·sched.npidle) != 0 && runtime·atomicload(&runtime·sched.nmspinning) == 0 && fn->fn != runtime·main) // TODO: fast atomic 1790 wakep(); 1791 m->locks--; 1792 if(m->locks == 0 && g->preempt) // restore the preemption request in case we've cleared it in newstack 1793 g->stackguard0 = StackPreempt; 1794 return newg; 1795 } 1796 1797 // Put on gfree list. 1798 // If local list is too long, transfer a batch to the global list. 1799 static void 1800 gfput(P *p, G *gp) 1801 { 1802 if(gp->stackguard - StackGuard != gp->stack0) 1803 runtime·throw("invalid stack in gfput"); 1804 gp->schedlink = p->gfree; 1805 p->gfree = gp; 1806 p->gfreecnt++; 1807 if(p->gfreecnt >= 64) { 1808 runtime·lock(&runtime·sched.gflock); 1809 while(p->gfreecnt >= 32) { 1810 p->gfreecnt--; 1811 gp = p->gfree; 1812 p->gfree = gp->schedlink; 1813 gp->schedlink = runtime·sched.gfree; 1814 runtime·sched.gfree = gp; 1815 } 1816 runtime·unlock(&runtime·sched.gflock); 1817 } 1818 } 1819 1820 // Get from gfree list. 1821 // If local list is empty, grab a batch from global list. 1822 static G* 1823 gfget(P *p) 1824 { 1825 G *gp; 1826 1827 retry: 1828 gp = p->gfree; 1829 if(gp == nil && runtime·sched.gfree) { 1830 runtime·lock(&runtime·sched.gflock); 1831 while(p->gfreecnt < 32 && runtime·sched.gfree) { 1832 p->gfreecnt++; 1833 gp = runtime·sched.gfree; 1834 runtime·sched.gfree = gp->schedlink; 1835 gp->schedlink = p->gfree; 1836 p->gfree = gp; 1837 } 1838 runtime·unlock(&runtime·sched.gflock); 1839 goto retry; 1840 } 1841 if(gp) { 1842 p->gfree = gp->schedlink; 1843 p->gfreecnt--; 1844 } 1845 return gp; 1846 } 1847 1848 // Purge all cached G's from gfree list to the global list. 1849 static void 1850 gfpurge(P *p) 1851 { 1852 G *gp; 1853 1854 runtime·lock(&runtime·sched.gflock); 1855 while(p->gfreecnt) { 1856 p->gfreecnt--; 1857 gp = p->gfree; 1858 p->gfree = gp->schedlink; 1859 gp->schedlink = runtime·sched.gfree; 1860 runtime·sched.gfree = gp; 1861 } 1862 runtime·unlock(&runtime·sched.gflock); 1863 } 1864 1865 void 1866 runtime·Breakpoint(void) 1867 { 1868 runtime·breakpoint(); 1869 } 1870 1871 void 1872 runtime·Gosched(void) 1873 { 1874 runtime·gosched(); 1875 } 1876 1877 // Implementation of runtime.GOMAXPROCS. 1878 // delete when scheduler is even stronger 1879 int32 1880 runtime·gomaxprocsfunc(int32 n) 1881 { 1882 int32 ret; 1883 1884 if(n > MaxGomaxprocs) 1885 n = MaxGomaxprocs; 1886 runtime·lock(&runtime·sched); 1887 ret = runtime·gomaxprocs; 1888 if(n <= 0 || n == ret) { 1889 runtime·unlock(&runtime·sched); 1890 return ret; 1891 } 1892 runtime·unlock(&runtime·sched); 1893 1894 runtime·semacquire(&runtime·worldsema, false); 1895 m->gcing = 1; 1896 runtime·stoptheworld(); 1897 newprocs = n; 1898 m->gcing = 0; 1899 runtime·semrelease(&runtime·worldsema); 1900 runtime·starttheworld(); 1901 1902 return ret; 1903 } 1904 1905 // lockOSThread is called by runtime.LockOSThread and runtime.lockOSThread below 1906 // after they modify m->locked. Do not allow preemption during this call, 1907 // or else the m might be different in this function than in the caller. 1908 #pragma textflag NOSPLIT 1909 static void 1910 lockOSThread(void) 1911 { 1912 m->lockedg = g; 1913 g->lockedm = m; 1914 } 1915 1916 void 1917 runtime·LockOSThread(void) 1918 { 1919 m->locked |= LockExternal; 1920 lockOSThread(); 1921 } 1922 1923 void 1924 runtime·lockOSThread(void) 1925 { 1926 m->locked += LockInternal; 1927 lockOSThread(); 1928 } 1929 1930 1931 // unlockOSThread is called by runtime.UnlockOSThread and runtime.unlockOSThread below 1932 // after they update m->locked. Do not allow preemption during this call, 1933 // or else the m might be in different in this function than in the caller. 1934 #pragma textflag NOSPLIT 1935 static void 1936 unlockOSThread(void) 1937 { 1938 if(m->locked != 0) 1939 return; 1940 m->lockedg = nil; 1941 g->lockedm = nil; 1942 } 1943 1944 void 1945 runtime·UnlockOSThread(void) 1946 { 1947 m->locked &= ~LockExternal; 1948 unlockOSThread(); 1949 } 1950 1951 void 1952 runtime·unlockOSThread(void) 1953 { 1954 if(m->locked < LockInternal) 1955 runtime·throw("runtime: internal error: misuse of lockOSThread/unlockOSThread"); 1956 m->locked -= LockInternal; 1957 unlockOSThread(); 1958 } 1959 1960 bool 1961 runtime·lockedOSThread(void) 1962 { 1963 return g->lockedm != nil && m->lockedg != nil; 1964 } 1965 1966 // for testing of callbacks 1967 void 1968 runtime·golockedOSThread(bool ret) 1969 { 1970 ret = runtime·lockedOSThread(); 1971 FLUSH(&ret); 1972 } 1973 1974 void 1975 runtime·NumGoroutine(intgo ret) 1976 { 1977 ret = runtime·gcount(); 1978 FLUSH(&ret); 1979 } 1980 1981 int32 1982 runtime·gcount(void) 1983 { 1984 G *gp; 1985 int32 n, s; 1986 1987 n = 0; 1988 runtime·lock(&runtime·sched); 1989 // TODO(dvyukov): runtime.NumGoroutine() is O(N). 1990 // We do not want to increment/decrement centralized counter in newproc/goexit, 1991 // just to make runtime.NumGoroutine() faster. 1992 // Compromise solution is to introduce per-P counters of active goroutines. 1993 for(gp = runtime·allg; gp; gp = gp->alllink) { 1994 s = gp->status; 1995 if(s == Grunnable || s == Grunning || s == Gsyscall || s == Gwaiting) 1996 n++; 1997 } 1998 runtime·unlock(&runtime·sched); 1999 return n; 2000 } 2001 2002 int32 2003 runtime·mcount(void) 2004 { 2005 return runtime·sched.mcount; 2006 } 2007 2008 void 2009 runtime·badmcall(void (*fn)(G*)) // called from assembly 2010 { 2011 USED(fn); // TODO: print fn? 2012 runtime·throw("runtime: mcall called on m->g0 stack"); 2013 } 2014 2015 void 2016 runtime·badmcall2(void (*fn)(G*)) // called from assembly 2017 { 2018 USED(fn); 2019 runtime·throw("runtime: mcall function returned"); 2020 } 2021 2022 void 2023 runtime·badreflectcall(void) // called from assembly 2024 { 2025 runtime·panicstring("runtime: arg size to reflect.call more than 1GB"); 2026 } 2027 2028 static struct { 2029 Lock; 2030 void (*fn)(uintptr*, int32); 2031 int32 hz; 2032 uintptr pcbuf[100]; 2033 } prof; 2034 2035 static void 2036 System(void) 2037 { 2038 } 2039 2040 // Called if we receive a SIGPROF signal. 2041 void 2042 runtime·sigprof(uint8 *pc, uint8 *sp, uint8 *lr, G *gp) 2043 { 2044 int32 n; 2045 bool traceback; 2046 2047 if(prof.fn == nil || prof.hz == 0) 2048 return; 2049 traceback = true; 2050 // Windows does profiling in a dedicated thread w/o m. 2051 if(!Windows && (m == nil || m->mcache == nil)) 2052 traceback = false; 2053 2054 // Define that a "user g" is a user-created goroutine, and a "system g" 2055 // is one that is m->g0 or m->gsignal. We've only made sure that we 2056 // can unwind user g's, so exclude the system g's. 2057 // 2058 // It is not quite as easy as testing gp == m->curg (the current user g) 2059 // because we might be interrupted for profiling halfway through a 2060 // goroutine switch. The switch involves updating three (or four) values: 2061 // g, PC, SP, and (on arm) LR. The PC must be the last to be updated, 2062 // because once it gets updated the new g is running. 2063 // 2064 // When switching from a user g to a system g, LR is not considered live, 2065 // so the update only affects g, SP, and PC. Since PC must be last, there 2066 // the possible partial transitions in ordinary execution are (1) g alone is updated, 2067 // (2) both g and SP are updated, and (3) SP alone is updated. 2068 // If g is updated, we'll see a system g and not look closer. 2069 // If SP alone is updated, we can detect the partial transition by checking 2070 // whether the SP is within g's stack bounds. (We could also require that SP 2071 // be changed only after g, but the stack bounds check is needed by other 2072 // cases, so there is no need to impose an additional requirement.) 2073 // 2074 // There is one exceptional transition to a system g, not in ordinary execution. 2075 // When a signal arrives, the operating system starts the signal handler running 2076 // with an updated PC and SP. The g is updated last, at the beginning of the 2077 // handler. There are two reasons this is okay. First, until g is updated the 2078 // g and SP do not match, so the stack bounds check detects the partial transition. 2079 // Second, signal handlers currently run with signals disabled, so a profiling 2080 // signal cannot arrive during the handler. 2081 // 2082 // When switching from a system g to a user g, there are three possibilities. 2083 // 2084 // First, it may be that the g switch has no PC update, because the SP 2085 // either corresponds to a user g throughout (as in runtime.asmcgocall) 2086 // or because it has been arranged to look like a user g frame 2087 // (as in runtime.cgocallback_gofunc). In this case, since the entire 2088 // transition is a g+SP update, a partial transition updating just one of 2089 // those will be detected by the stack bounds check. 2090 // 2091 // Second, when returning from a signal handler, the PC and SP updates 2092 // are performed by the operating system in an atomic update, so the g 2093 // update must be done before them. The stack bounds check detects 2094 // the partial transition here, and (again) signal handlers run with signals 2095 // disabled, so a profiling signal cannot arrive then anyway. 2096 // 2097 // Third, the common case: it may be that the switch updates g, SP, and PC 2098 // separately, as in runtime.gogo. 2099 // 2100 // Because runtime.gogo is the only instance, we check whether the PC lies 2101 // within that function, and if so, not ask for a traceback. This approach 2102 // requires knowing the size of the runtime.gogo function, which we 2103 // record in arch_*.h and check in runtime_test.go. 2104 // 2105 // There is another apparently viable approach, recorded here in case 2106 // the "PC within runtime.gogo" check turns out not to be usable. 2107 // It would be possible to delay the update of either g or SP until immediately 2108 // before the PC update instruction. Then, because of the stack bounds check, 2109 // the only problematic interrupt point is just before that PC update instruction, 2110 // and the sigprof handler can detect that instruction and simulate stepping past 2111 // it in order to reach a consistent state. On ARM, the update of g must be made 2112 // in two places (in R10 and also in a TLS slot), so the delayed update would 2113 // need to be the SP update. The sigprof handler must read the instruction at 2114 // the current PC and if it was the known instruction (for example, JMP BX or 2115 // MOV R2, PC), use that other register in place of the PC value. 2116 // The biggest drawback to this solution is that it requires that we can tell 2117 // whether it's safe to read from the memory pointed at by PC. 2118 // In a correct program, we can test PC == nil and otherwise read, 2119 // but if a profiling signal happens at the instant that a program executes 2120 // a bad jump (before the program manages to handle the resulting fault) 2121 // the profiling handler could fault trying to read nonexistent memory. 2122 // 2123 // To recap, there are no constraints on the assembly being used for the 2124 // transition. We simply require that g and SP match and that the PC is not 2125 // in runtime.gogo. 2126 // 2127 // On Windows, one m is sending reports about all the g's, so gp == m->curg 2128 // is not a useful comparison. The profilem function in os_windows.c has 2129 // already checked that gp is a user g. 2130 if(gp == nil || 2131 (!Windows && gp != m->curg) || 2132 (uintptr)sp < gp->stackguard - StackGuard || gp->stackbase < (uintptr)sp || 2133 ((uint8*)runtime·gogo <= pc && pc < (uint8*)runtime·gogo + RuntimeGogoBytes)) 2134 traceback = false; 2135 2136 // Race detector calls asmcgocall w/o entersyscall/exitsyscall, 2137 // we can not currently unwind through asmcgocall. 2138 if(m != nil && m->racecall) 2139 traceback = false; 2140 2141 runtime·lock(&prof); 2142 if(prof.fn == nil) { 2143 runtime·unlock(&prof); 2144 return; 2145 } 2146 n = 0; 2147 if(traceback) 2148 n = runtime·gentraceback((uintptr)pc, (uintptr)sp, (uintptr)lr, gp, 0, prof.pcbuf, nelem(prof.pcbuf), nil, nil, false); 2149 if(!traceback || n <= 0) { 2150 n = 2; 2151 prof.pcbuf[0] = (uintptr)pc; 2152 prof.pcbuf[1] = (uintptr)System + 1; 2153 } 2154 prof.fn(prof.pcbuf, n); 2155 runtime·unlock(&prof); 2156 } 2157 2158 // Arrange to call fn with a traceback hz times a second. 2159 void 2160 runtime·setcpuprofilerate(void (*fn)(uintptr*, int32), int32 hz) 2161 { 2162 // Force sane arguments. 2163 if(hz < 0) 2164 hz = 0; 2165 if(hz == 0) 2166 fn = nil; 2167 if(fn == nil) 2168 hz = 0; 2169 2170 // Disable preemption, otherwise we can be rescheduled to another thread 2171 // that has profiling enabled. 2172 m->locks++; 2173 2174 // Stop profiler on this thread so that it is safe to lock prof. 2175 // if a profiling signal came in while we had prof locked, 2176 // it would deadlock. 2177 runtime·resetcpuprofiler(0); 2178 2179 runtime·lock(&prof); 2180 prof.fn = fn; 2181 prof.hz = hz; 2182 runtime·unlock(&prof); 2183 runtime·lock(&runtime·sched); 2184 runtime·sched.profilehz = hz; 2185 runtime·unlock(&runtime·sched); 2186 2187 if(hz != 0) 2188 runtime·resetcpuprofiler(hz); 2189 2190 m->locks--; 2191 } 2192 2193 // Change number of processors. The world is stopped, sched is locked. 2194 static void 2195 procresize(int32 new) 2196 { 2197 int32 i, old; 2198 G *gp; 2199 P *p; 2200 2201 old = runtime·gomaxprocs; 2202 if(old < 0 || old > MaxGomaxprocs || new <= 0 || new >MaxGomaxprocs) 2203 runtime·throw("procresize: invalid arg"); 2204 // initialize new P's 2205 for(i = 0; i < new; i++) { 2206 p = runtime·allp[i]; 2207 if(p == nil) { 2208 p = (P*)runtime·mallocgc(sizeof(*p), 0, FlagNoInvokeGC); 2209 p->id = i; 2210 p->status = Pgcstop; 2211 runtime·atomicstorep(&runtime·allp[i], p); 2212 } 2213 if(p->mcache == nil) { 2214 if(old==0 && i==0) 2215 p->mcache = m->mcache; // bootstrap 2216 else 2217 p->mcache = runtime·allocmcache(); 2218 } 2219 if(p->runq == nil) { 2220 p->runqsize = 128; 2221 p->runq = (G**)runtime·mallocgc(p->runqsize*sizeof(G*), 0, FlagNoInvokeGC); 2222 } 2223 } 2224 2225 // redistribute runnable G's evenly 2226 for(i = 0; i < old; i++) { 2227 p = runtime·allp[i]; 2228 while(gp = runqget(p)) 2229 globrunqput(gp); 2230 } 2231 // start at 1 because current M already executes some G and will acquire allp[0] below, 2232 // so if we have a spare G we want to put it into allp[1]. 2233 for(i = 1; runtime·sched.runqhead; i++) { 2234 gp = runtime·sched.runqhead; 2235 runtime·sched.runqhead = gp->schedlink; 2236 runqput(runtime·allp[i%new], gp); 2237 } 2238 runtime·sched.runqtail = nil; 2239 runtime·sched.runqsize = 0; 2240 2241 // free unused P's 2242 for(i = new; i < old; i++) { 2243 p = runtime·allp[i]; 2244 runtime·freemcache(p->mcache); 2245 p->mcache = nil; 2246 gfpurge(p); 2247 p->status = Pdead; 2248 // can't free P itself because it can be referenced by an M in syscall 2249 } 2250 2251 if(m->p) 2252 m->p->m = nil; 2253 m->p = nil; 2254 m->mcache = nil; 2255 p = runtime·allp[0]; 2256 p->m = nil; 2257 p->status = Pidle; 2258 acquirep(p); 2259 for(i = new-1; i > 0; i--) { 2260 p = runtime·allp[i]; 2261 p->status = Pidle; 2262 pidleput(p); 2263 } 2264 runtime·atomicstore((uint32*)&runtime·gomaxprocs, new); 2265 } 2266 2267 // Associate p and the current m. 2268 static void 2269 acquirep(P *p) 2270 { 2271 if(m->p || m->mcache) 2272 runtime·throw("acquirep: already in go"); 2273 if(p->m || p->status != Pidle) { 2274 runtime·printf("acquirep: p->m=%p(%d) p->status=%d\n", p->m, p->m ? p->m->id : 0, p->status); 2275 runtime·throw("acquirep: invalid p state"); 2276 } 2277 m->mcache = p->mcache; 2278 m->p = p; 2279 p->m = m; 2280 p->status = Prunning; 2281 } 2282 2283 // Disassociate p and the current m. 2284 static P* 2285 releasep(void) 2286 { 2287 P *p; 2288 2289 if(m->p == nil || m->mcache == nil) 2290 runtime·throw("releasep: invalid arg"); 2291 p = m->p; 2292 if(p->m != m || p->mcache != m->mcache || p->status != Prunning) { 2293 runtime·printf("releasep: m=%p m->p=%p p->m=%p m->mcache=%p p->mcache=%p p->status=%d\n", 2294 m, m->p, p->m, m->mcache, p->mcache, p->status); 2295 runtime·throw("releasep: invalid p state"); 2296 } 2297 m->p = nil; 2298 m->mcache = nil; 2299 p->m = nil; 2300 p->status = Pidle; 2301 return p; 2302 } 2303 2304 static void 2305 incidlelocked(int32 v) 2306 { 2307 runtime·lock(&runtime·sched); 2308 runtime·sched.nmidlelocked += v; 2309 if(v > 0) 2310 checkdead(); 2311 runtime·unlock(&runtime·sched); 2312 } 2313 2314 // Check for deadlock situation. 2315 // The check is based on number of running M's, if 0 -> deadlock. 2316 static void 2317 checkdead(void) 2318 { 2319 G *gp; 2320 int32 run, grunning, s; 2321 2322 // -1 for sysmon 2323 run = runtime·sched.mcount - runtime·sched.nmidle - runtime·sched.nmidlelocked - 1; 2324 if(run > 0) 2325 return; 2326 if(run < 0) { 2327 runtime·printf("checkdead: nmidle=%d nmidlelocked=%d mcount=%d\n", 2328 runtime·sched.nmidle, runtime·sched.nmidlelocked, runtime·sched.mcount); 2329 runtime·throw("checkdead: inconsistent counts"); 2330 } 2331 grunning = 0; 2332 for(gp = runtime·allg; gp; gp = gp->alllink) { 2333 if(gp->isbackground) 2334 continue; 2335 s = gp->status; 2336 if(s == Gwaiting) 2337 grunning++; 2338 else if(s == Grunnable || s == Grunning || s == Gsyscall) { 2339 runtime·printf("checkdead: find g %D in status %d\n", gp->goid, s); 2340 runtime·throw("checkdead: runnable g"); 2341 } 2342 } 2343 if(grunning == 0) // possible if main goroutine calls runtime·Goexit() 2344 runtime·exit(0); 2345 m->throwing = -1; // do not dump full stacks 2346 runtime·throw("all goroutines are asleep - deadlock!"); 2347 } 2348 2349 static void 2350 sysmon(void) 2351 { 2352 uint32 idle, delay; 2353 int64 now, lastpoll, lasttrace; 2354 G *gp; 2355 2356 lasttrace = 0; 2357 idle = 0; // how many cycles in succession we had not wokeup somebody 2358 delay = 0; 2359 for(;;) { 2360 if(idle == 0) // start with 20us sleep... 2361 delay = 20; 2362 else if(idle > 50) // start doubling the sleep after 1ms... 2363 delay *= 2; 2364 if(delay > 10*1000) // up to 10ms 2365 delay = 10*1000; 2366 runtime·usleep(delay); 2367 if(runtime·debug.schedtrace <= 0 && 2368 (runtime·sched.gcwaiting || runtime·atomicload(&runtime·sched.npidle) == runtime·gomaxprocs)) { // TODO: fast atomic 2369 runtime·lock(&runtime·sched); 2370 if(runtime·atomicload(&runtime·sched.gcwaiting) || runtime·atomicload(&runtime·sched.npidle) == runtime·gomaxprocs) { 2371 runtime·atomicstore(&runtime·sched.sysmonwait, 1); 2372 runtime·unlock(&runtime·sched); 2373 runtime·notesleep(&runtime·sched.sysmonnote); 2374 runtime·noteclear(&runtime·sched.sysmonnote); 2375 idle = 0; 2376 delay = 20; 2377 } else 2378 runtime·unlock(&runtime·sched); 2379 } 2380 // poll network if not polled for more than 10ms 2381 lastpoll = runtime·atomicload64(&runtime·sched.lastpoll); 2382 now = runtime·nanotime(); 2383 if(lastpoll != 0 && lastpoll + 10*1000*1000 < now) { 2384 runtime·cas64(&runtime·sched.lastpoll, lastpoll, now); 2385 gp = runtime·netpoll(false); // non-blocking 2386 if(gp) { 2387 // Need to decrement number of idle locked M's 2388 // (pretending that one more is running) before injectglist. 2389 // Otherwise it can lead to the following situation: 2390 // injectglist grabs all P's but before it starts M's to run the P's, 2391 // another M returns from syscall, finishes running its G, 2392 // observes that there is no work to do and no other running M's 2393 // and reports deadlock. 2394 incidlelocked(-1); 2395 injectglist(gp); 2396 incidlelocked(1); 2397 } 2398 } 2399 // retake P's blocked in syscalls 2400 // and preempt long running G's 2401 if(retake(now)) 2402 idle = 0; 2403 else 2404 idle++; 2405 2406 if(runtime·debug.schedtrace > 0 && lasttrace + runtime·debug.schedtrace*1000000ll <= now) { 2407 lasttrace = now; 2408 runtime·schedtrace(runtime·debug.scheddetail); 2409 } 2410 } 2411 } 2412 2413 typedef struct Pdesc Pdesc; 2414 struct Pdesc 2415 { 2416 uint32 schedtick; 2417 int64 schedwhen; 2418 uint32 syscalltick; 2419 int64 syscallwhen; 2420 }; 2421 static Pdesc pdesc[MaxGomaxprocs]; 2422 2423 static uint32 2424 retake(int64 now) 2425 { 2426 uint32 i, s, n; 2427 int64 t; 2428 P *p; 2429 Pdesc *pd; 2430 2431 n = 0; 2432 for(i = 0; i < runtime·gomaxprocs; i++) { 2433 p = runtime·allp[i]; 2434 if(p==nil) 2435 continue; 2436 pd = &pdesc[i]; 2437 s = p->status; 2438 if(s == Psyscall) { 2439 // Retake P from syscall if it's there for more than 1 sysmon tick (20us). 2440 // But only if there is other work to do. 2441 t = p->syscalltick; 2442 if(pd->syscalltick != t) { 2443 pd->syscalltick = t; 2444 pd->syscallwhen = now; 2445 continue; 2446 } 2447 if(p->runqhead == p->runqtail && 2448 runtime·atomicload(&runtime·sched.nmspinning) + runtime·atomicload(&runtime·sched.npidle) > 0) 2449 continue; 2450 // Need to decrement number of idle locked M's 2451 // (pretending that one more is running) before the CAS. 2452 // Otherwise the M from which we retake can exit the syscall, 2453 // increment nmidle and report deadlock. 2454 incidlelocked(-1); 2455 if(runtime·cas(&p->status, s, Pidle)) { 2456 n++; 2457 handoffp(p); 2458 } 2459 incidlelocked(1); 2460 } else if(s == Prunning) { 2461 // Preempt G if it's running for more than 10ms. 2462 t = p->schedtick; 2463 if(pd->schedtick != t) { 2464 pd->schedtick = t; 2465 pd->schedwhen = now; 2466 continue; 2467 } 2468 if(pd->schedwhen + 10*1000*1000 > now) 2469 continue; 2470 preemptone(p); 2471 } 2472 } 2473 return n; 2474 } 2475 2476 // Tell all goroutines that they have been preempted and they should stop. 2477 // This function is purely best-effort. It can fail to inform a goroutine if a 2478 // processor just started running it. 2479 // No locks need to be held. 2480 // Returns true if preemption request was issued to at least one goroutine. 2481 static bool 2482 preemptall(void) 2483 { 2484 P *p; 2485 int32 i; 2486 bool res; 2487 2488 res = false; 2489 for(i = 0; i < runtime·gomaxprocs; i++) { 2490 p = runtime·allp[i]; 2491 if(p == nil || p->status != Prunning) 2492 continue; 2493 res |= preemptone(p); 2494 } 2495 return res; 2496 } 2497 2498 // Tell the goroutine running on processor P to stop. 2499 // This function is purely best-effort. It can incorrectly fail to inform the 2500 // goroutine. It can send inform the wrong goroutine. Even if it informs the 2501 // correct goroutine, that goroutine might ignore the request if it is 2502 // simultaneously executing runtime·newstack. 2503 // No lock needs to be held. 2504 // Returns true if preemption request was issued. 2505 static bool 2506 preemptone(P *p) 2507 { 2508 M *mp; 2509 G *gp; 2510 2511 mp = p->m; 2512 if(mp == nil || mp == m) 2513 return false; 2514 gp = mp->curg; 2515 if(gp == nil || gp == mp->g0) 2516 return false; 2517 gp->preempt = true; 2518 gp->stackguard0 = StackPreempt; 2519 return true; 2520 } 2521 2522 void 2523 runtime·schedtrace(bool detailed) 2524 { 2525 static int64 starttime; 2526 int64 now; 2527 int64 id1, id2, id3; 2528 int32 i, q, t, h, s; 2529 int8 *fmt; 2530 M *mp, *lockedm; 2531 G *gp, *lockedg; 2532 P *p; 2533 2534 now = runtime·nanotime(); 2535 if(starttime == 0) 2536 starttime = now; 2537 2538 runtime·lock(&runtime·sched); 2539 runtime·printf("SCHED %Dms: gomaxprocs=%d idleprocs=%d threads=%d idlethreads=%d runqueue=%d", 2540 (now-starttime)/1000000, runtime·gomaxprocs, runtime·sched.npidle, runtime·sched.mcount, 2541 runtime·sched.nmidle, runtime·sched.runqsize); 2542 if(detailed) { 2543 runtime·printf(" gcwaiting=%d nmidlelocked=%d nmspinning=%d stopwait=%d sysmonwait=%d\n", 2544 runtime·sched.gcwaiting, runtime·sched.nmidlelocked, runtime·sched.nmspinning, 2545 runtime·sched.stopwait, runtime·sched.sysmonwait); 2546 } 2547 // We must be careful while reading data from P's, M's and G's. 2548 // Even if we hold schedlock, most data can be changed concurrently. 2549 // E.g. (p->m ? p->m->id : -1) can crash if p->m changes from non-nil to nil. 2550 for(i = 0; i < runtime·gomaxprocs; i++) { 2551 p = runtime·allp[i]; 2552 if(p == nil) 2553 continue; 2554 mp = p->m; 2555 t = p->runqtail; 2556 h = p->runqhead; 2557 s = p->runqsize; 2558 q = t - h; 2559 if(q < 0) 2560 q += s; 2561 if(detailed) 2562 runtime·printf(" P%d: status=%d schedtick=%d syscalltick=%d m=%d runqsize=%d/%d gfreecnt=%d\n", 2563 i, p->status, p->schedtick, p->syscalltick, mp ? mp->id : -1, q, s, p->gfreecnt); 2564 else { 2565 // In non-detailed mode format lengths of per-P run queues as: 2566 // [len1 len2 len3 len4] 2567 fmt = " %d"; 2568 if(runtime·gomaxprocs == 1) 2569 fmt = " [%d]\n"; 2570 else if(i == 0) 2571 fmt = " [%d"; 2572 else if(i == runtime·gomaxprocs-1) 2573 fmt = " %d]\n"; 2574 runtime·printf(fmt, q); 2575 } 2576 } 2577 if(!detailed) { 2578 runtime·unlock(&runtime·sched); 2579 return; 2580 } 2581 for(mp = runtime·allm; mp; mp = mp->alllink) { 2582 p = mp->p; 2583 gp = mp->curg; 2584 lockedg = mp->lockedg; 2585 id1 = -1; 2586 if(p) 2587 id1 = p->id; 2588 id2 = -1; 2589 if(gp) 2590 id2 = gp->goid; 2591 id3 = -1; 2592 if(lockedg) 2593 id3 = lockedg->goid; 2594 runtime·printf(" M%d: p=%D curg=%D mallocing=%d throwing=%d gcing=%d" 2595 " locks=%d dying=%d helpgc=%d spinning=%d lockedg=%D\n", 2596 mp->id, id1, id2, 2597 mp->mallocing, mp->throwing, mp->gcing, mp->locks, mp->dying, mp->helpgc, 2598 mp->spinning, id3); 2599 } 2600 for(gp = runtime·allg; gp; gp = gp->alllink) { 2601 mp = gp->m; 2602 lockedm = gp->lockedm; 2603 runtime·printf(" G%D: status=%d(%s) m=%d lockedm=%d\n", 2604 gp->goid, gp->status, gp->waitreason, mp ? mp->id : -1, 2605 lockedm ? lockedm->id : -1); 2606 } 2607 runtime·unlock(&runtime·sched); 2608 } 2609 2610 // Put mp on midle list. 2611 // Sched must be locked. 2612 static void 2613 mput(M *mp) 2614 { 2615 mp->schedlink = runtime·sched.midle; 2616 runtime·sched.midle = mp; 2617 runtime·sched.nmidle++; 2618 checkdead(); 2619 } 2620 2621 // Try to get an m from midle list. 2622 // Sched must be locked. 2623 static M* 2624 mget(void) 2625 { 2626 M *mp; 2627 2628 if((mp = runtime·sched.midle) != nil){ 2629 runtime·sched.midle = mp->schedlink; 2630 runtime·sched.nmidle--; 2631 } 2632 return mp; 2633 } 2634 2635 // Put gp on the global runnable queue. 2636 // Sched must be locked. 2637 static void 2638 globrunqput(G *gp) 2639 { 2640 gp->schedlink = nil; 2641 if(runtime·sched.runqtail) 2642 runtime·sched.runqtail->schedlink = gp; 2643 else 2644 runtime·sched.runqhead = gp; 2645 runtime·sched.runqtail = gp; 2646 runtime·sched.runqsize++; 2647 } 2648 2649 // Try get a batch of G's from the global runnable queue. 2650 // Sched must be locked. 2651 static G* 2652 globrunqget(P *p, int32 max) 2653 { 2654 G *gp, *gp1; 2655 int32 n; 2656 2657 if(runtime·sched.runqsize == 0) 2658 return nil; 2659 n = runtime·sched.runqsize/runtime·gomaxprocs+1; 2660 if(n > runtime·sched.runqsize) 2661 n = runtime·sched.runqsize; 2662 if(max > 0 && n > max) 2663 n = max; 2664 runtime·sched.runqsize -= n; 2665 if(runtime·sched.runqsize == 0) 2666 runtime·sched.runqtail = nil; 2667 gp = runtime·sched.runqhead; 2668 runtime·sched.runqhead = gp->schedlink; 2669 n--; 2670 while(n--) { 2671 gp1 = runtime·sched.runqhead; 2672 runtime·sched.runqhead = gp1->schedlink; 2673 runqput(p, gp1); 2674 } 2675 return gp; 2676 } 2677 2678 // Put p to on pidle list. 2679 // Sched must be locked. 2680 static void 2681 pidleput(P *p) 2682 { 2683 p->link = runtime·sched.pidle; 2684 runtime·sched.pidle = p; 2685 runtime·xadd(&runtime·sched.npidle, 1); // TODO: fast atomic 2686 } 2687 2688 // Try get a p from pidle list. 2689 // Sched must be locked. 2690 static P* 2691 pidleget(void) 2692 { 2693 P *p; 2694 2695 p = runtime·sched.pidle; 2696 if(p) { 2697 runtime·sched.pidle = p->link; 2698 runtime·xadd(&runtime·sched.npidle, -1); // TODO: fast atomic 2699 } 2700 return p; 2701 } 2702 2703 // Put g on local runnable queue. 2704 // TODO(dvyukov): consider using lock-free queue. 2705 static void 2706 runqput(P *p, G *gp) 2707 { 2708 int32 h, t, s; 2709 2710 runtime·lock(p); 2711 retry: 2712 h = p->runqhead; 2713 t = p->runqtail; 2714 s = p->runqsize; 2715 if(t == h-1 || (h == 0 && t == s-1)) { 2716 runqgrow(p); 2717 goto retry; 2718 } 2719 p->runq[t++] = gp; 2720 if(t == s) 2721 t = 0; 2722 p->runqtail = t; 2723 runtime·unlock(p); 2724 } 2725 2726 // Get g from local runnable queue. 2727 static G* 2728 runqget(P *p) 2729 { 2730 G *gp; 2731 int32 t, h, s; 2732 2733 if(p->runqhead == p->runqtail) 2734 return nil; 2735 runtime·lock(p); 2736 h = p->runqhead; 2737 t = p->runqtail; 2738 s = p->runqsize; 2739 if(t == h) { 2740 runtime·unlock(p); 2741 return nil; 2742 } 2743 gp = p->runq[h++]; 2744 if(h == s) 2745 h = 0; 2746 p->runqhead = h; 2747 runtime·unlock(p); 2748 return gp; 2749 } 2750 2751 // Grow local runnable queue. 2752 // TODO(dvyukov): consider using fixed-size array 2753 // and transfer excess to the global list (local queue can grow way too big). 2754 static void 2755 runqgrow(P *p) 2756 { 2757 G **q; 2758 int32 s, t, h, t2; 2759 2760 h = p->runqhead; 2761 t = p->runqtail; 2762 s = p->runqsize; 2763 t2 = 0; 2764 q = runtime·malloc(2*s*sizeof(*q)); 2765 while(t != h) { 2766 q[t2++] = p->runq[h++]; 2767 if(h == s) 2768 h = 0; 2769 } 2770 runtime·free(p->runq); 2771 p->runq = q; 2772 p->runqhead = 0; 2773 p->runqtail = t2; 2774 p->runqsize = 2*s; 2775 } 2776 2777 // Steal half of elements from local runnable queue of p2 2778 // and put onto local runnable queue of p. 2779 // Returns one of the stolen elements (or nil if failed). 2780 static G* 2781 runqsteal(P *p, P *p2) 2782 { 2783 G *gp, *gp1; 2784 int32 t, h, s, t2, h2, s2, c, i; 2785 2786 if(p2->runqhead == p2->runqtail) 2787 return nil; 2788 // sort locks to prevent deadlocks 2789 if(p < p2) 2790 runtime·lock(p); 2791 runtime·lock(p2); 2792 if(p2->runqhead == p2->runqtail) { 2793 runtime·unlock(p2); 2794 if(p < p2) 2795 runtime·unlock(p); 2796 return nil; 2797 } 2798 if(p >= p2) 2799 runtime·lock(p); 2800 // now we've locked both queues and know the victim is not empty 2801 h = p->runqhead; 2802 t = p->runqtail; 2803 s = p->runqsize; 2804 h2 = p2->runqhead; 2805 t2 = p2->runqtail; 2806 s2 = p2->runqsize; 2807 gp = p2->runq[h2++]; // return value 2808 if(h2 == s2) 2809 h2 = 0; 2810 // steal roughly half 2811 if(t2 > h2) 2812 c = (t2 - h2) / 2; 2813 else 2814 c = (s2 - h2 + t2) / 2; 2815 // copy 2816 for(i = 0; i != c; i++) { 2817 // the target queue is full? 2818 if(t == h-1 || (h == 0 && t == s-1)) 2819 break; 2820 // the victim queue is empty? 2821 if(t2 == h2) 2822 break; 2823 gp1 = p2->runq[h2++]; 2824 if(h2 == s2) 2825 h2 = 0; 2826 p->runq[t++] = gp1; 2827 if(t == s) 2828 t = 0; 2829 } 2830 p->runqtail = t; 2831 p2->runqhead = h2; 2832 runtime·unlock(p2); 2833 runtime·unlock(p); 2834 return gp; 2835 } 2836 2837 void 2838 runtime·testSchedLocalQueue(void) 2839 { 2840 P p; 2841 G gs[1000]; 2842 int32 i, j; 2843 2844 runtime·memclr((byte*)&p, sizeof(p)); 2845 p.runqsize = 1; 2846 p.runqhead = 0; 2847 p.runqtail = 0; 2848 p.runq = runtime·malloc(p.runqsize*sizeof(*p.runq)); 2849 2850 for(i = 0; i < nelem(gs); i++) { 2851 if(runqget(&p) != nil) 2852 runtime·throw("runq is not empty initially"); 2853 for(j = 0; j < i; j++) 2854 runqput(&p, &gs[i]); 2855 for(j = 0; j < i; j++) { 2856 if(runqget(&p) != &gs[i]) { 2857 runtime·printf("bad element at iter %d/%d\n", i, j); 2858 runtime·throw("bad element"); 2859 } 2860 } 2861 if(runqget(&p) != nil) 2862 runtime·throw("runq is not empty afterwards"); 2863 } 2864 } 2865 2866 void 2867 runtime·testSchedLocalQueueSteal(void) 2868 { 2869 P p1, p2; 2870 G gs[1000], *gp; 2871 int32 i, j, s; 2872 2873 runtime·memclr((byte*)&p1, sizeof(p1)); 2874 p1.runqsize = 1; 2875 p1.runqhead = 0; 2876 p1.runqtail = 0; 2877 p1.runq = runtime·malloc(p1.runqsize*sizeof(*p1.runq)); 2878 2879 runtime·memclr((byte*)&p2, sizeof(p2)); 2880 p2.runqsize = nelem(gs); 2881 p2.runqhead = 0; 2882 p2.runqtail = 0; 2883 p2.runq = runtime·malloc(p2.runqsize*sizeof(*p2.runq)); 2884 2885 for(i = 0; i < nelem(gs); i++) { 2886 for(j = 0; j < i; j++) { 2887 gs[j].sig = 0; 2888 runqput(&p1, &gs[j]); 2889 } 2890 gp = runqsteal(&p2, &p1); 2891 s = 0; 2892 if(gp) { 2893 s++; 2894 gp->sig++; 2895 } 2896 while(gp = runqget(&p2)) { 2897 s++; 2898 gp->sig++; 2899 } 2900 while(gp = runqget(&p1)) 2901 gp->sig++; 2902 for(j = 0; j < i; j++) { 2903 if(gs[j].sig != 1) { 2904 runtime·printf("bad element %d(%d) at iter %d\n", j, gs[j].sig, i); 2905 runtime·throw("bad element"); 2906 } 2907 } 2908 if(s != i/2 && s != i/2+1) { 2909 runtime·printf("bad steal %d, want %d or %d, iter %d\n", 2910 s, i/2, i/2+1, i); 2911 runtime·throw("bad steal"); 2912 } 2913 } 2914 } 2915 2916 extern void runtime·morestack(void); 2917 2918 // Does f mark the top of a goroutine stack? 2919 bool 2920 runtime·topofstack(Func *f) 2921 { 2922 return f->entry == (uintptr)runtime·goexit || 2923 f->entry == (uintptr)runtime·mstart || 2924 f->entry == (uintptr)runtime·mcall || 2925 f->entry == (uintptr)runtime·morestack || 2926 f->entry == (uintptr)runtime·lessstack || 2927 f->entry == (uintptr)_rt0_go; 2928 } 2929 2930 void 2931 runtime∕debug·setMaxThreads(intgo in, intgo out) 2932 { 2933 runtime·lock(&runtime·sched); 2934 out = runtime·sched.maxmcount; 2935 runtime·sched.maxmcount = in; 2936 checkmcount(); 2937 runtime·unlock(&runtime·sched); 2938 FLUSH(&out); 2939 } 2940 2941 static int8 experiment[] = GOEXPERIMENT; // defined in zaexperiment.h 2942 2943 static bool 2944 haveexperiment(int8 *name) 2945 { 2946 int32 i, j; 2947 2948 for(i=0; i<sizeof(experiment); i++) { 2949 if((i == 0 || experiment[i-1] == ',') && experiment[i] == name[0]) { 2950 for(j=0; name[j]; j++) 2951 if(experiment[i+j] != name[j]) 2952 goto nomatch; 2953 if(experiment[i+j] != '\0' && experiment[i+j] != ',') 2954 goto nomatch; 2955 return 1; 2956 } 2957 nomatch:; 2958 } 2959 return 0; 2960 }