github.com/zach-klippenstein/go@v0.0.0-20150108044943-fcfbeb3adf58/src/runtime/asm_ppc64x.s (about) 1 // Copyright 2014 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // +build ppc64 ppc64le 6 7 #include "go_asm.h" 8 #include "go_tls.h" 9 #include "funcdata.h" 10 #include "textflag.h" 11 12 TEXT runtime·rt0_go(SB),NOSPLIT,$0 13 // R1 = stack; R3 = argc; R4 = argv; R13 = C TLS base pointer 14 15 // initialize essential registers 16 BL runtime·reginit(SB) 17 18 SUB $24, R1 19 MOVW R3, 8(R1) // argc 20 MOVD R4, 16(R1) // argv 21 22 // create istack out of the given (operating system) stack. 23 // _cgo_init may update stackguard. 24 MOVD $runtime·g0(SB), g 25 MOVD $(-64*1024), R31 26 ADD R31, R1, R3 27 MOVD R3, g_stackguard0(g) 28 MOVD R3, g_stackguard1(g) 29 MOVD R3, (g_stack+stack_lo)(g) 30 MOVD R1, (g_stack+stack_hi)(g) 31 32 // if there is a _cgo_init, call it using the gcc ABI. 33 MOVD _cgo_init(SB), R12 34 CMP R0, R12 35 BEQ nocgo 36 MOVD R12, CTR // r12 = "global function entry point" 37 MOVD R13, R5 // arg 2: TLS base pointer 38 MOVD $setg_gcc<>(SB), R4 // arg 1: setg 39 MOVD g, R3 // arg 0: G 40 // C functions expect 32 bytes of space on caller stack frame 41 // and a 16-byte aligned R1 42 MOVD R1, R14 // save current stack 43 SUB $32, R1 // reserve 32 bytes 44 RLDCR $0, R1, $~15, R1 // 16-byte align 45 BL (CTR) // may clobber R0, R3-R12 46 MOVD R14, R1 // restore stack 47 XOR R0, R0 // fix R0 48 49 nocgo: 50 // update stackguard after _cgo_init 51 MOVD (g_stack+stack_lo)(g), R3 52 ADD $const__StackGuard, R3 53 MOVD R3, g_stackguard0(g) 54 MOVD R3, g_stackguard1(g) 55 56 // set the per-goroutine and per-mach "registers" 57 MOVD $runtime·m0(SB), R3 58 59 // save m->g0 = g0 60 MOVD g, m_g0(R3) 61 // save m0 to g0->m 62 MOVD R3, g_m(g) 63 64 BL runtime·check(SB) 65 66 // args are already prepared 67 BL runtime·args(SB) 68 BL runtime·osinit(SB) 69 BL runtime·schedinit(SB) 70 71 // create a new goroutine to start program 72 MOVD $runtime·main·f(SB), R3 // entry 73 MOVDU R3, -8(R1) 74 MOVDU R0, -8(R1) 75 MOVDU R0, -8(R1) 76 BL runtime·newproc(SB) 77 ADD $24, R1 78 79 // start this M 80 BL runtime·mstart(SB) 81 82 MOVD R0, 1(R0) 83 RETURN 84 85 DATA runtime·main·f+0(SB)/8,$runtime·main(SB) 86 GLOBL runtime·main·f(SB),RODATA,$8 87 88 TEXT runtime·breakpoint(SB),NOSPLIT,$-8-0 89 MOVD R0, 2(R0) // TODO: TD 90 RETURN 91 92 TEXT runtime·asminit(SB),NOSPLIT,$-8-0 93 RETURN 94 95 TEXT _cgo_reginit(SB),NOSPLIT,$-8-0 96 // crosscall_ppc64 and crosscall2 need to reginit, but can't 97 // get at the 'runtime.reginit' symbol. 98 BR runtime·reginit(SB) 99 100 TEXT runtime·reginit(SB),NOSPLIT,$-8-0 101 // set R0 to zero, it's expected by the toolchain 102 XOR R0, R0 103 // initialize essential FP registers 104 FMOVD $4503601774854144.0, F27 105 FMOVD $0.5, F29 106 FSUB F29, F29, F28 107 FADD F29, F29, F30 108 FADD F30, F30, F31 109 RETURN 110 111 /* 112 * go-routine 113 */ 114 115 // void gosave(Gobuf*) 116 // save state in Gobuf; setjmp 117 TEXT runtime·gosave(SB), NOSPLIT, $-8-8 118 MOVD buf+0(FP), R3 119 MOVD R1, gobuf_sp(R3) 120 MOVD LR, R31 121 MOVD R31, gobuf_pc(R3) 122 MOVD g, gobuf_g(R3) 123 MOVD R0, gobuf_lr(R3) 124 MOVD R0, gobuf_ret(R3) 125 MOVD R0, gobuf_ctxt(R3) 126 RETURN 127 128 // void gogo(Gobuf*) 129 // restore state from Gobuf; longjmp 130 TEXT runtime·gogo(SB), NOSPLIT, $-8-8 131 MOVD buf+0(FP), R5 132 MOVD gobuf_g(R5), g // make sure g is not nil 133 BL runtime·save_g(SB) 134 135 MOVD 0(g), R4 136 MOVD gobuf_sp(R5), R1 137 MOVD gobuf_lr(R5), R31 138 MOVD R31, LR 139 MOVD gobuf_ret(R5), R3 140 MOVD gobuf_ctxt(R5), R11 141 MOVD R0, gobuf_sp(R5) 142 MOVD R0, gobuf_ret(R5) 143 MOVD R0, gobuf_lr(R5) 144 MOVD R0, gobuf_ctxt(R5) 145 CMP R0, R0 // set condition codes for == test, needed by stack split 146 MOVD gobuf_pc(R5), R31 147 MOVD R31, CTR 148 BR (CTR) 149 150 // void mcall(fn func(*g)) 151 // Switch to m->g0's stack, call fn(g). 152 // Fn must never return. It should gogo(&g->sched) 153 // to keep running g. 154 TEXT runtime·mcall(SB), NOSPLIT, $-8-8 155 // Save caller state in g->sched 156 MOVD R1, (g_sched+gobuf_sp)(g) 157 MOVD LR, R31 158 MOVD R31, (g_sched+gobuf_pc)(g) 159 MOVD R0, (g_sched+gobuf_lr)(g) 160 MOVD g, (g_sched+gobuf_g)(g) 161 162 // Switch to m->g0 & its stack, call fn. 163 MOVD g, R3 164 MOVD g_m(g), R8 165 MOVD m_g0(R8), g 166 BL runtime·save_g(SB) 167 CMP g, R3 168 BNE 2(PC) 169 BR runtime·badmcall(SB) 170 MOVD fn+0(FP), R11 // context 171 MOVD 0(R11), R4 // code pointer 172 MOVD R4, CTR 173 MOVD (g_sched+gobuf_sp)(g), R1 // sp = m->g0->sched.sp 174 MOVDU R3, -8(R1) 175 MOVDU R0, -8(R1) 176 BL (CTR) 177 BR runtime·badmcall2(SB) 178 179 // systemstack_switch is a dummy routine that systemstack leaves at the bottom 180 // of the G stack. We need to distinguish the routine that 181 // lives at the bottom of the G stack from the one that lives 182 // at the top of the system stack because the one at the top of 183 // the system stack terminates the stack walk (see topofstack()). 184 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0 185 UNDEF 186 BL (LR) // make sure this function is not leaf 187 RETURN 188 189 // func systemstack(fn func()) 190 TEXT runtime·systemstack(SB), NOSPLIT, $0-8 191 MOVD fn+0(FP), R3 // R3 = fn 192 MOVD R3, R11 // context 193 MOVD g_m(g), R4 // R4 = m 194 195 MOVD m_gsignal(R4), R5 // R5 = gsignal 196 CMP g, R5 197 BEQ noswitch 198 199 MOVD m_g0(R4), R5 // R5 = g0 200 CMP g, R5 201 BEQ noswitch 202 203 MOVD m_curg(R4), R6 204 CMP g, R6 205 BEQ switch 206 207 // Bad: g is not gsignal, not g0, not curg. What is it? 208 // Hide call from linker nosplit analysis. 209 MOVD $runtime·badsystemstack(SB), R3 210 MOVD R3, CTR 211 BL (CTR) 212 213 switch: 214 // save our state in g->sched. Pretend to 215 // be systemstack_switch if the G stack is scanned. 216 MOVD $runtime·systemstack_switch(SB), R6 217 ADD $8, R6 // get past prologue 218 MOVD R6, (g_sched+gobuf_pc)(g) 219 MOVD R1, (g_sched+gobuf_sp)(g) 220 MOVD R0, (g_sched+gobuf_lr)(g) 221 MOVD g, (g_sched+gobuf_g)(g) 222 223 // switch to g0 224 MOVD R5, g 225 BL runtime·save_g(SB) 226 MOVD (g_sched+gobuf_sp)(g), R3 227 // make it look like mstart called systemstack on g0, to stop traceback 228 SUB $8, R3 229 MOVD $runtime·mstart(SB), R4 230 MOVD R4, 0(R3) 231 MOVD R3, R1 232 233 // call target function 234 MOVD 0(R11), R3 // code pointer 235 MOVD R3, CTR 236 BL (CTR) 237 238 // switch back to g 239 MOVD g_m(g), R3 240 MOVD m_curg(R3), g 241 BL runtime·save_g(SB) 242 MOVD (g_sched+gobuf_sp)(g), R1 243 MOVD R0, (g_sched+gobuf_sp)(g) 244 RETURN 245 246 noswitch: 247 // already on m stack, just call directly 248 MOVD 0(R11), R3 // code pointer 249 MOVD R3, CTR 250 BL (CTR) 251 RETURN 252 253 /* 254 * support for morestack 255 */ 256 257 // Called during function prolog when more stack is needed. 258 // Caller has already loaded: 259 // R3: framesize, R4: argsize, R5: LR 260 // 261 // The traceback routines see morestack on a g0 as being 262 // the top of a stack (for example, morestack calling newstack 263 // calling the scheduler calling newm calling gc), so we must 264 // record an argument size. For that purpose, it has no arguments. 265 TEXT runtime·morestack(SB),NOSPLIT,$-8-0 266 // Cannot grow scheduler stack (m->g0). 267 MOVD g_m(g), R7 268 MOVD m_g0(R7), R8 269 CMP g, R8 270 BNE 2(PC) 271 BL runtime·abort(SB) 272 273 // Cannot grow signal stack (m->gsignal). 274 MOVD m_gsignal(R7), R8 275 CMP g, R8 276 BNE 2(PC) 277 BL runtime·abort(SB) 278 279 // Called from f. 280 // Set g->sched to context in f. 281 MOVD R11, (g_sched+gobuf_ctxt)(g) 282 MOVD R1, (g_sched+gobuf_sp)(g) 283 MOVD LR, R8 284 MOVD R8, (g_sched+gobuf_pc)(g) 285 MOVD R5, (g_sched+gobuf_lr)(g) 286 287 // Called from f. 288 // Set m->morebuf to f's caller. 289 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC 290 MOVD R1, (m_morebuf+gobuf_sp)(R7) // f's caller's SP 291 MOVD g, (m_morebuf+gobuf_g)(R7) 292 293 // Call newstack on m->g0's stack. 294 MOVD m_g0(R7), g 295 BL runtime·save_g(SB) 296 MOVD (g_sched+gobuf_sp)(g), R1 297 BL runtime·newstack(SB) 298 299 // Not reached, but make sure the return PC from the call to newstack 300 // is still in this function, and not the beginning of the next. 301 UNDEF 302 303 TEXT runtime·morestack_noctxt(SB),NOSPLIT,$-8-0 304 MOVD R0, R11 305 BR runtime·morestack(SB) 306 307 // reflectcall: call a function with the given argument list 308 // func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32). 309 // we don't have variable-sized frames, so we use a small number 310 // of constant-sized-frame functions to encode a few bits of size in the pc. 311 // Caution: ugly multiline assembly macros in your future! 312 313 #define DISPATCH(NAME,MAXSIZE) \ 314 MOVD $MAXSIZE, R31; \ 315 CMP R3, R31; \ 316 BGT 4(PC); \ 317 MOVD $NAME(SB), R31; \ 318 MOVD R31, CTR; \ 319 BR (CTR) 320 // Note: can't just "BR NAME(SB)" - bad inlining results. 321 322 TEXT reflect·call(SB), NOSPLIT, $0-0 323 BR ·reflectcall(SB) 324 325 TEXT ·reflectcall(SB), NOSPLIT, $-8-32 326 MOVWZ argsize+24(FP), R3 327 // NOTE(rsc): No call16, because CALLFN needs four words 328 // of argument space to invoke callwritebarrier. 329 DISPATCH(runtime·call32, 32) 330 DISPATCH(runtime·call64, 64) 331 DISPATCH(runtime·call128, 128) 332 DISPATCH(runtime·call256, 256) 333 DISPATCH(runtime·call512, 512) 334 DISPATCH(runtime·call1024, 1024) 335 DISPATCH(runtime·call2048, 2048) 336 DISPATCH(runtime·call4096, 4096) 337 DISPATCH(runtime·call8192, 8192) 338 DISPATCH(runtime·call16384, 16384) 339 DISPATCH(runtime·call32768, 32768) 340 DISPATCH(runtime·call65536, 65536) 341 DISPATCH(runtime·call131072, 131072) 342 DISPATCH(runtime·call262144, 262144) 343 DISPATCH(runtime·call524288, 524288) 344 DISPATCH(runtime·call1048576, 1048576) 345 DISPATCH(runtime·call2097152, 2097152) 346 DISPATCH(runtime·call4194304, 4194304) 347 DISPATCH(runtime·call8388608, 8388608) 348 DISPATCH(runtime·call16777216, 16777216) 349 DISPATCH(runtime·call33554432, 33554432) 350 DISPATCH(runtime·call67108864, 67108864) 351 DISPATCH(runtime·call134217728, 134217728) 352 DISPATCH(runtime·call268435456, 268435456) 353 DISPATCH(runtime·call536870912, 536870912) 354 DISPATCH(runtime·call1073741824, 1073741824) 355 MOVD $runtime·badreflectcall(SB), R31 356 MOVD R31, CTR 357 BR (CTR) 358 359 #define CALLFN(NAME,MAXSIZE) \ 360 TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \ 361 NO_LOCAL_POINTERS; \ 362 /* copy arguments to stack */ \ 363 MOVD arg+16(FP), R3; \ 364 MOVWZ argsize+24(FP), R4; \ 365 MOVD R1, R5; \ 366 ADD $(8-1), R5; \ 367 SUB $1, R3; \ 368 ADD R5, R4; \ 369 CMP R5, R4; \ 370 BEQ 4(PC); \ 371 MOVBZU 1(R3), R6; \ 372 MOVBZU R6, 1(R5); \ 373 BR -4(PC); \ 374 /* call function */ \ 375 MOVD f+8(FP), R11; \ 376 MOVD (R11), R31; \ 377 MOVD R31, CTR; \ 378 PCDATA $PCDATA_StackMapIndex, $0; \ 379 BL (CTR); \ 380 /* copy return values back */ \ 381 MOVD arg+16(FP), R3; \ 382 MOVWZ n+24(FP), R4; \ 383 MOVWZ retoffset+28(FP), R6; \ 384 MOVD R1, R5; \ 385 ADD R6, R5; \ 386 ADD R6, R3; \ 387 SUB R6, R4; \ 388 ADD $(8-1), R5; \ 389 SUB $1, R3; \ 390 ADD R5, R4; \ 391 loop: \ 392 CMP R5, R4; \ 393 BEQ end; \ 394 MOVBZU 1(R5), R6; \ 395 MOVBZU R6, 1(R3); \ 396 BR loop; \ 397 end: \ 398 /* execute write barrier updates */ \ 399 MOVD argtype+0(FP), R7; \ 400 MOVD arg+16(FP), R3; \ 401 MOVWZ n+24(FP), R4; \ 402 MOVWZ retoffset+28(FP), R6; \ 403 MOVD R7, 8(R1); \ 404 MOVD R3, 16(R1); \ 405 MOVD R4, 24(R1); \ 406 MOVD R6, 32(R1); \ 407 BL runtime·callwritebarrier(SB); \ 408 RETURN 409 410 CALLFN(·call16, 16) 411 CALLFN(·call32, 32) 412 CALLFN(·call64, 64) 413 CALLFN(·call128, 128) 414 CALLFN(·call256, 256) 415 CALLFN(·call512, 512) 416 CALLFN(·call1024, 1024) 417 CALLFN(·call2048, 2048) 418 CALLFN(·call4096, 4096) 419 CALLFN(·call8192, 8192) 420 CALLFN(·call16384, 16384) 421 CALLFN(·call32768, 32768) 422 CALLFN(·call65536, 65536) 423 CALLFN(·call131072, 131072) 424 CALLFN(·call262144, 262144) 425 CALLFN(·call524288, 524288) 426 CALLFN(·call1048576, 1048576) 427 CALLFN(·call2097152, 2097152) 428 CALLFN(·call4194304, 4194304) 429 CALLFN(·call8388608, 8388608) 430 CALLFN(·call16777216, 16777216) 431 CALLFN(·call33554432, 33554432) 432 CALLFN(·call67108864, 67108864) 433 CALLFN(·call134217728, 134217728) 434 CALLFN(·call268435456, 268435456) 435 CALLFN(·call536870912, 536870912) 436 CALLFN(·call1073741824, 1073741824) 437 438 // bool cas(uint32 *ptr, uint32 old, uint32 new) 439 // Atomically: 440 // if(*val == old){ 441 // *val = new; 442 // return 1; 443 // } else 444 // return 0; 445 TEXT runtime·cas(SB), NOSPLIT, $0-17 446 MOVD ptr+0(FP), R3 447 MOVWZ old+8(FP), R4 448 MOVWZ new+12(FP), R5 449 cas_again: 450 SYNC 451 LWAR (R3), R6 452 CMPW R6, R4 453 BNE cas_fail 454 STWCCC R5, (R3) 455 BNE cas_again 456 MOVD $1, R3 457 SYNC 458 ISYNC 459 MOVB R3, ret+16(FP) 460 RETURN 461 cas_fail: 462 MOVD $0, R3 463 BR -5(PC) 464 465 // bool runtime·cas64(uint64 *ptr, uint64 old, uint64 new) 466 // Atomically: 467 // if(*val == *old){ 468 // *val = new; 469 // return 1; 470 // } else { 471 // return 0; 472 // } 473 TEXT runtime·cas64(SB), NOSPLIT, $0-25 474 MOVD ptr+0(FP), R3 475 MOVD old+8(FP), R4 476 MOVD new+16(FP), R5 477 cas64_again: 478 SYNC 479 LDAR (R3), R6 480 CMP R6, R4 481 BNE cas64_fail 482 STDCCC R5, (R3) 483 BNE cas64_again 484 MOVD $1, R3 485 SYNC 486 ISYNC 487 MOVB R3, ret+24(FP) 488 RETURN 489 cas64_fail: 490 MOVD $0, R3 491 BR -5(PC) 492 493 TEXT runtime·casuintptr(SB), NOSPLIT, $0-25 494 BR runtime·cas64(SB) 495 496 TEXT runtime·atomicloaduintptr(SB), NOSPLIT, $-8-16 497 BR runtime·atomicload64(SB) 498 499 TEXT runtime·atomicloaduint(SB), NOSPLIT, $-8-16 500 BR runtime·atomicload64(SB) 501 502 TEXT runtime·atomicstoreuintptr(SB), NOSPLIT, $0-16 503 BR runtime·atomicstore64(SB) 504 505 // bool casp(void **val, void *old, void *new) 506 // Atomically: 507 // if(*val == old){ 508 // *val = new; 509 // return 1; 510 // } else 511 // return 0; 512 TEXT runtime·casp1(SB), NOSPLIT, $0-25 513 BR runtime·cas64(SB) 514 515 // uint32 xadd(uint32 volatile *ptr, int32 delta) 516 // Atomically: 517 // *val += delta; 518 // return *val; 519 TEXT runtime·xadd(SB), NOSPLIT, $0-20 520 MOVD ptr+0(FP), R4 521 MOVW delta+8(FP), R5 522 SYNC 523 LWAR (R4), R3 524 ADD R5, R3 525 STWCCC R3, (R4) 526 BNE -4(PC) 527 SYNC 528 ISYNC 529 MOVW R3, ret+16(FP) 530 RETURN 531 532 TEXT runtime·xadd64(SB), NOSPLIT, $0-24 533 MOVD ptr+0(FP), R4 534 MOVD delta+8(FP), R5 535 SYNC 536 LDAR (R4), R3 537 ADD R5, R3 538 STDCCC R3, (R4) 539 BNE -4(PC) 540 SYNC 541 ISYNC 542 MOVD R3, ret+16(FP) 543 RETURN 544 545 TEXT runtime·xchg(SB), NOSPLIT, $0-20 546 MOVD ptr+0(FP), R4 547 MOVW new+8(FP), R5 548 SYNC 549 LWAR (R4), R3 550 STWCCC R5, (R4) 551 BNE -3(PC) 552 SYNC 553 ISYNC 554 MOVW R3, ret+16(FP) 555 RETURN 556 557 TEXT runtime·xchg64(SB), NOSPLIT, $0-24 558 MOVD ptr+0(FP), R4 559 MOVD new+8(FP), R5 560 SYNC 561 LDAR (R4), R3 562 STDCCC R5, (R4) 563 BNE -3(PC) 564 SYNC 565 ISYNC 566 MOVD R3, ret+16(FP) 567 RETURN 568 569 TEXT runtime·xchgp1(SB), NOSPLIT, $0-24 570 BR runtime·xchg64(SB) 571 572 TEXT runtime·xchguintptr(SB), NOSPLIT, $0-24 573 BR runtime·xchg64(SB) 574 575 TEXT runtime·procyield(SB),NOSPLIT,$0-0 576 RETURN 577 578 TEXT runtime·atomicstorep1(SB), NOSPLIT, $0-16 579 BR runtime·atomicstore64(SB) 580 581 TEXT runtime·atomicstore(SB), NOSPLIT, $0-12 582 MOVD ptr+0(FP), R3 583 MOVW val+8(FP), R4 584 SYNC 585 MOVW R4, 0(R3) 586 RETURN 587 588 TEXT runtime·atomicstore64(SB), NOSPLIT, $0-16 589 MOVD ptr+0(FP), R3 590 MOVD val+8(FP), R4 591 SYNC 592 MOVD R4, 0(R3) 593 RETURN 594 595 // void runtime·atomicor8(byte volatile*, byte); 596 TEXT runtime·atomicor8(SB), NOSPLIT, $0-9 597 MOVD ptr+0(FP), R3 598 MOVBZ val+8(FP), R4 599 // Align ptr down to 4 bytes so we can use 32-bit load/store. 600 // R5 = (R3 << 0) & ~3 601 RLDCR $0, R3, $~3, R5 602 // Compute val shift. 603 #ifdef GOARCH_ppc64 604 // Big endian. ptr = ptr ^ 3 605 XOR $3, R3 606 #endif 607 // R6 = ((ptr & 3) * 8) = (ptr << 3) & (3*8) 608 RLDC $3, R3, $(3*8), R6 609 // Shift val for aligned ptr. R4 = val << R6 610 SLD R6, R4, R4 611 612 atomicor8_again: 613 SYNC 614 LWAR (R5), R6 615 OR R4, R6 616 STWCCC R6, (R5) 617 BNE atomicor8_again 618 SYNC 619 ISYNC 620 RETURN 621 622 // void jmpdefer(fv, sp); 623 // called from deferreturn. 624 // 1. grab stored LR for caller 625 // 2. sub 4 bytes to get back to BL deferreturn 626 // 3. BR to fn 627 TEXT runtime·jmpdefer(SB), NOSPLIT, $-8-16 628 MOVD 0(R1), R31 629 SUB $4, R31 630 MOVD R31, LR 631 632 MOVD fv+0(FP), R11 633 MOVD argp+8(FP), R1 634 SUB $8, R1 635 MOVD 0(R11), R3 636 MOVD R3, CTR 637 BR (CTR) 638 639 // Save state of caller into g->sched. Smashes R31. 640 TEXT gosave<>(SB),NOSPLIT,$-8 641 MOVD LR, R31 642 MOVD R31, (g_sched+gobuf_pc)(g) 643 MOVD R1, (g_sched+gobuf_sp)(g) 644 MOVD R0, (g_sched+gobuf_lr)(g) 645 MOVD R0, (g_sched+gobuf_ret)(g) 646 MOVD R0, (g_sched+gobuf_ctxt)(g) 647 RETURN 648 649 // asmcgocall(void(*fn)(void*), void *arg) 650 // Call fn(arg) on the scheduler stack, 651 // aligned appropriately for the gcc ABI. 652 // See cgocall.c for more details. 653 TEXT ·asmcgocall(SB),NOSPLIT,$0-16 654 MOVD fn+0(FP), R3 655 MOVD arg+8(FP), R4 656 BL asmcgocall<>(SB) 657 RET 658 659 TEXT ·asmcgocall_errno(SB),NOSPLIT,$0-24 660 MOVD fn+0(FP), R3 661 MOVD arg+8(FP), R4 662 BL asmcgocall<>(SB) 663 MOVD R3, ret+16(FP) 664 RET 665 666 // asmcgocall common code. fn in R3, arg in R4. returns errno in R3. 667 TEXT asmcgocall<>(SB),NOSPLIT,$0-0 668 MOVD R1, R2 // save original stack pointer 669 MOVD g, R5 670 671 // Figure out if we need to switch to m->g0 stack. 672 // We get called to create new OS threads too, and those 673 // come in on the m->g0 stack already. 674 MOVD g_m(g), R6 675 MOVD m_g0(R6), R6 676 CMP R6, g 677 BEQ g0 678 BL gosave<>(SB) 679 MOVD R6, g 680 BL runtime·save_g(SB) 681 MOVD (g_sched+gobuf_sp)(g), R1 682 683 // Now on a scheduling stack (a pthread-created stack). 684 g0: 685 // Save room for two of our pointers, plus 32 bytes of callee 686 // save area that lives on the caller stack. 687 SUB $48, R1 688 RLDCR $0, R1, $~15, R1 // 16-byte alignment for gcc ABI 689 MOVD R5, 40(R1) // save old g on stack 690 MOVD (g_stack+stack_hi)(R5), R5 691 SUB R2, R5 692 MOVD R5, 32(R1) // save depth in old g stack (can't just save SP, as stack might be copied during a callback) 693 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?) 694 // This is a "global call", so put the global entry point in r12 695 MOVD R3, R12 696 MOVD R12, CTR 697 MOVD R4, R3 // arg in r3 698 BL (CTR) 699 700 // C code can clobber R0, so set it back to 0. F27-F31 are 701 // callee save, so we don't need to recover those. 702 XOR R0, R0 703 // Restore g, stack pointer. R3 is errno, so don't touch it 704 MOVD 40(R1), g 705 BL runtime·save_g(SB) 706 MOVD (g_stack+stack_hi)(g), R5 707 MOVD 32(R1), R6 708 SUB R6, R5 709 MOVD R5, R1 710 RET 711 712 // cgocallback(void (*fn)(void*), void *frame, uintptr framesize) 713 // Turn the fn into a Go func (by taking its address) and call 714 // cgocallback_gofunc. 715 TEXT runtime·cgocallback(SB),NOSPLIT,$24-24 716 MOVD $fn+0(FP), R3 717 MOVD R3, 8(R1) 718 MOVD frame+8(FP), R3 719 MOVD R3, 16(R1) 720 MOVD framesize+16(FP), R3 721 MOVD R3, 24(R1) 722 MOVD $runtime·cgocallback_gofunc(SB), R3 723 MOVD R3, CTR 724 BL (CTR) 725 RET 726 727 // cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize) 728 // See cgocall.c for more details. 729 TEXT ·cgocallback_gofunc(SB),NOSPLIT,$16-24 730 NO_LOCAL_POINTERS 731 732 // Load m and g from thread-local storage. 733 MOVB runtime·iscgo(SB), R3 734 CMP R3, $0 735 BEQ nocgo 736 BL runtime·load_g(SB) 737 nocgo: 738 739 // If g is nil, Go did not create the current thread. 740 // Call needm to obtain one for temporary use. 741 // In this case, we're running on the thread stack, so there's 742 // lots of space, but the linker doesn't know. Hide the call from 743 // the linker analysis by using an indirect call. 744 CMP g, $0 745 BNE havem 746 MOVD g, savedm-8(SP) // g is zero, so is m. 747 MOVD $runtime·needm(SB), R3 748 MOVD R3, CTR 749 BL (CTR) 750 751 // Set m->sched.sp = SP, so that if a panic happens 752 // during the function we are about to execute, it will 753 // have a valid SP to run on the g0 stack. 754 // The next few lines (after the havem label) 755 // will save this SP onto the stack and then write 756 // the same SP back to m->sched.sp. That seems redundant, 757 // but if an unrecovered panic happens, unwindm will 758 // restore the g->sched.sp from the stack location 759 // and then systemstack will try to use it. If we don't set it here, 760 // that restored SP will be uninitialized (typically 0) and 761 // will not be usable. 762 MOVD g_m(g), R3 763 MOVD m_g0(R3), R3 764 MOVD R1, (g_sched+gobuf_sp)(R3) 765 766 havem: 767 MOVD g_m(g), R8 768 MOVD R8, savedm-8(SP) 769 // Now there's a valid m, and we're running on its m->g0. 770 // Save current m->g0->sched.sp on stack and then set it to SP. 771 // Save current sp in m->g0->sched.sp in preparation for 772 // switch back to m->curg stack. 773 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP). 774 MOVD m_g0(R8), R3 775 MOVD (g_sched+gobuf_sp)(R3), R4 776 MOVD R4, savedsp-16(SP) 777 MOVD R1, (g_sched+gobuf_sp)(R3) 778 779 // Switch to m->curg stack and call runtime.cgocallbackg. 780 // Because we are taking over the execution of m->curg 781 // but *not* resuming what had been running, we need to 782 // save that information (m->curg->sched) so we can restore it. 783 // We can restore m->curg->sched.sp easily, because calling 784 // runtime.cgocallbackg leaves SP unchanged upon return. 785 // To save m->curg->sched.pc, we push it onto the stack. 786 // This has the added benefit that it looks to the traceback 787 // routine like cgocallbackg is going to return to that 788 // PC (because the frame we allocate below has the same 789 // size as cgocallback_gofunc's frame declared above) 790 // so that the traceback will seamlessly trace back into 791 // the earlier calls. 792 // 793 // In the new goroutine, -16(SP) and -8(SP) are unused. 794 MOVD m_curg(R8), g 795 BL runtime·save_g(SB) 796 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4 797 MOVD (g_sched+gobuf_pc)(g), R5 798 MOVD R5, -24(R4) 799 MOVD $-24(R4), R1 800 BL runtime·cgocallbackg(SB) 801 802 // Restore g->sched (== m->curg->sched) from saved values. 803 MOVD 0(R1), R5 804 MOVD R5, (g_sched+gobuf_pc)(g) 805 MOVD $24(R1), R4 806 MOVD R4, (g_sched+gobuf_sp)(g) 807 808 // Switch back to m->g0's stack and restore m->g0->sched.sp. 809 // (Unlike m->curg, the g0 goroutine never uses sched.pc, 810 // so we do not have to restore it.) 811 MOVD g_m(g), R8 812 MOVD m_g0(R8), g 813 BL runtime·save_g(SB) 814 MOVD (g_sched+gobuf_sp)(g), R1 815 MOVD savedsp-16(SP), R4 816 MOVD R4, (g_sched+gobuf_sp)(g) 817 818 // If the m on entry was nil, we called needm above to borrow an m 819 // for the duration of the call. Since the call is over, return it with dropm. 820 MOVD savedm-8(SP), R6 821 CMP R6, $0 822 BNE droppedm 823 MOVD $runtime·dropm(SB), R3 824 MOVD R3, CTR 825 BL (CTR) 826 droppedm: 827 828 // Done! 829 RET 830 831 // void setg(G*); set g. for use by needm. 832 TEXT runtime·setg(SB), NOSPLIT, $0-8 833 MOVD gg+0(FP), g 834 // This only happens if iscgo, so jump straight to save_g 835 BL runtime·save_g(SB) 836 RET 837 838 // void setg_gcc(G*); set g in C TLS. 839 // Must obey the gcc calling convention. 840 TEXT setg_gcc<>(SB),NOSPLIT,$-8-0 841 // The standard prologue clobbers R31, which is callee-save in 842 // the C ABI, so we have to use $-8-0 and save LR ourselves. 843 MOVD LR, R4 844 // Also save g and R31, since they're callee-save in C ABI 845 MOVD R31, R5 846 MOVD g, R6 847 848 MOVD R3, g 849 BL runtime·save_g(SB) 850 851 MOVD R6, g 852 MOVD R5, R31 853 MOVD R4, LR 854 RET 855 856 TEXT runtime·getcallerpc(SB),NOSPLIT,$-8-16 857 MOVD 0(R1), R3 858 MOVD R3, ret+8(FP) 859 RETURN 860 861 TEXT runtime·gogetcallerpc(SB),NOSPLIT,$-8-16 862 MOVD 0(R1), R3 863 MOVD R3,ret+8(FP) 864 RETURN 865 866 TEXT runtime·setcallerpc(SB),NOSPLIT,$-8-16 867 MOVD pc+8(FP), R3 868 MOVD R3, 0(R1) // set calling pc 869 RETURN 870 871 TEXT runtime·getcallersp(SB),NOSPLIT,$0-16 872 MOVD argp+0(FP), R3 873 SUB $8, R3 874 MOVD R3, ret+8(FP) 875 RETURN 876 877 // func gogetcallersp(p unsafe.Pointer) uintptr 878 TEXT runtime·gogetcallersp(SB),NOSPLIT,$0-16 879 MOVD sp+0(FP), R3 880 SUB $8, R3 881 MOVD R3,ret+8(FP) 882 RETURN 883 884 TEXT runtime·abort(SB),NOSPLIT,$-8-0 885 MOVW (R0), R0 886 UNDEF 887 888 #define TBRL 268 889 #define TBRU 269 /* Time base Upper/Lower */ 890 891 // int64 runtime·cputicks(void) 892 TEXT runtime·cputicks(SB),NOSPLIT,$0-8 893 MOVW SPR(TBRU), R4 894 MOVW SPR(TBRL), R3 895 MOVW SPR(TBRU), R5 896 CMPW R4, R5 897 BNE -4(PC) 898 SLD $32, R5 899 OR R5, R3 900 MOVD R3, ret+0(FP) 901 RETURN 902 903 // memhash_varlen(p unsafe.Pointer, h seed) uintptr 904 // redirects to memhash(p, h, size) using the size 905 // stored in the closure. 906 TEXT runtime·memhash_varlen(SB),NOSPLIT,$40-24 907 GO_ARGS 908 NO_LOCAL_POINTERS 909 MOVD p+0(FP), R3 910 MOVD h+8(FP), R4 911 MOVD 8(R11), R5 912 MOVD R3, 8(R1) 913 MOVD R4, 16(R1) 914 MOVD R5, 24(R1) 915 BL runtime·memhash(SB) 916 MOVD 32(R1), R3 917 MOVD R3, ret+16(FP) 918 RETURN 919 920 // AES hashing not implemented for ppc64 921 TEXT runtime·aeshash(SB),NOSPLIT,$-8-0 922 MOVW (R0), R1 923 TEXT runtime·aeshash32(SB),NOSPLIT,$-8-0 924 MOVW (R0), R1 925 TEXT runtime·aeshash64(SB),NOSPLIT,$-8-0 926 MOVW (R0), R1 927 TEXT runtime·aeshashstr(SB),NOSPLIT,$-8-0 928 MOVW (R0), R1 929 930 TEXT runtime·memeq(SB),NOSPLIT,$-8-25 931 MOVD a+0(FP), R3 932 MOVD b+8(FP), R4 933 MOVD size+16(FP), R5 934 SUB $1, R3 935 SUB $1, R4 936 ADD R3, R5, R8 937 loop: 938 CMP R3, R8 939 BNE test 940 MOVD $1, R3 941 MOVB R3, ret+24(FP) 942 RETURN 943 test: 944 MOVBZU 1(R3), R6 945 MOVBZU 1(R4), R7 946 CMP R6, R7 947 BEQ loop 948 949 MOVB R0, ret+24(FP) 950 RETURN 951 952 // memequal_varlen(a, b unsafe.Pointer) bool 953 TEXT runtime·memequal_varlen(SB),NOSPLIT,$40-17 954 MOVD a+0(FP), R3 955 MOVD b+8(FP), R4 956 CMP R3, R4 957 BEQ eq 958 MOVD 8(R11), R5 // compiler stores size at offset 8 in the closure 959 MOVD R3, 8(R1) 960 MOVD R4, 16(R1) 961 MOVD R5, 24(R1) 962 BL runtime·memeq(SB) 963 MOVBZ 32(R1), R3 964 MOVB R3, ret+16(FP) 965 RETURN 966 eq: 967 MOVD $1, R3 968 MOVB R3, ret+16(FP) 969 RETURN 970 971 // eqstring tests whether two strings are equal. 972 // See runtime_test.go:eqstring_generic for 973 // equivalent Go code. 974 TEXT runtime·eqstring(SB),NOSPLIT,$0-33 975 MOVD s1len+8(FP), R4 976 MOVD s2len+24(FP), R5 977 CMP R4, R5 978 BNE noteq 979 980 MOVD s1str+0(FP), R3 981 MOVD s2str+16(FP), R4 982 SUB $1, R3 983 SUB $1, R4 984 ADD R3, R5, R8 985 loop: 986 CMP R3, R8 987 BNE 4(PC) 988 MOVD $1, R3 989 MOVB R3, ret+32(FP) 990 RETURN 991 MOVBZU 1(R3), R6 992 MOVBZU 1(R4), R7 993 CMP R6, R7 994 BEQ loop 995 noteq: 996 MOVB R0, ret+32(FP) 997 RETURN 998 999 // TODO: share code with memeq? 1000 TEXT bytes·Equal(SB),NOSPLIT,$0-49 1001 MOVD a_len+8(FP), R3 1002 MOVD b_len+32(FP), R4 1003 1004 CMP R3, R4 // unequal lengths are not equal 1005 BNE noteq 1006 1007 MOVD a+0(FP), R5 1008 MOVD b+24(FP), R6 1009 SUB $1, R5 1010 SUB $1, R6 1011 ADD R5, R3 // end-1 1012 1013 loop: 1014 CMP R5, R3 1015 BEQ equal // reached the end 1016 MOVBZU 1(R5), R4 1017 MOVBZU 1(R6), R7 1018 CMP R4, R7 1019 BEQ loop 1020 1021 noteq: 1022 MOVBZ R0, ret+48(FP) 1023 RETURN 1024 1025 equal: 1026 MOVD $1, R3 1027 MOVBZ R3, ret+48(FP) 1028 RETURN 1029 1030 TEXT bytes·IndexByte(SB),NOSPLIT,$0-40 1031 MOVD s+0(FP), R3 1032 MOVD s_len+8(FP), R4 1033 MOVBZ c+24(FP), R5 // byte to find 1034 MOVD R3, R6 // store base for later 1035 SUB $1, R3 1036 ADD R3, R4 // end-1 1037 1038 loop: 1039 CMP R3, R4 1040 BEQ notfound 1041 MOVBZU 1(R3), R7 1042 CMP R7, R5 1043 BNE loop 1044 1045 SUB R6, R3 // remove base 1046 MOVD R3, ret+32(FP) 1047 RETURN 1048 1049 notfound: 1050 MOVD $-1, R3 1051 MOVD R3, ret+32(FP) 1052 RETURN 1053 1054 TEXT strings·IndexByte(SB),NOSPLIT,$0 1055 MOVD p+0(FP), R3 1056 MOVD b_len+8(FP), R4 1057 MOVBZ c+16(FP), R5 // byte to find 1058 MOVD R3, R6 // store base for later 1059 SUB $1, R3 1060 ADD R3, R4 // end-1 1061 1062 loop: 1063 CMP R3, R4 1064 BEQ notfound 1065 MOVBZU 1(R3), R7 1066 CMP R7, R5 1067 BNE loop 1068 1069 SUB R6, R3 // remove base 1070 MOVD R3, ret+24(FP) 1071 RETURN 1072 1073 notfound: 1074 MOVD $-1, R3 1075 MOVD R3, ret+24(FP) 1076 RETURN 1077 1078 1079 // A Duff's device for zeroing memory. 1080 // The compiler jumps to computed addresses within 1081 // this routine to zero chunks of memory. Do not 1082 // change this code without also changing the code 1083 // in ../../cmd/9g/ggen.c:/^clearfat. 1084 // R0: always zero 1085 // R3 (aka REGRT1): ptr to memory to be zeroed - 8 1086 // On return, R3 points to the last zeroed dword. 1087 TEXT runtime·duffzero(SB), NOSPLIT, $-8-0 1088 MOVDU R0, 8(R3) 1089 MOVDU R0, 8(R3) 1090 MOVDU R0, 8(R3) 1091 MOVDU R0, 8(R3) 1092 MOVDU R0, 8(R3) 1093 MOVDU R0, 8(R3) 1094 MOVDU R0, 8(R3) 1095 MOVDU R0, 8(R3) 1096 MOVDU R0, 8(R3) 1097 MOVDU R0, 8(R3) 1098 MOVDU R0, 8(R3) 1099 MOVDU R0, 8(R3) 1100 MOVDU R0, 8(R3) 1101 MOVDU R0, 8(R3) 1102 MOVDU R0, 8(R3) 1103 MOVDU R0, 8(R3) 1104 MOVDU R0, 8(R3) 1105 MOVDU R0, 8(R3) 1106 MOVDU R0, 8(R3) 1107 MOVDU R0, 8(R3) 1108 MOVDU R0, 8(R3) 1109 MOVDU R0, 8(R3) 1110 MOVDU R0, 8(R3) 1111 MOVDU R0, 8(R3) 1112 MOVDU R0, 8(R3) 1113 MOVDU R0, 8(R3) 1114 MOVDU R0, 8(R3) 1115 MOVDU R0, 8(R3) 1116 MOVDU R0, 8(R3) 1117 MOVDU R0, 8(R3) 1118 MOVDU R0, 8(R3) 1119 MOVDU R0, 8(R3) 1120 MOVDU R0, 8(R3) 1121 MOVDU R0, 8(R3) 1122 MOVDU R0, 8(R3) 1123 MOVDU R0, 8(R3) 1124 MOVDU R0, 8(R3) 1125 MOVDU R0, 8(R3) 1126 MOVDU R0, 8(R3) 1127 MOVDU R0, 8(R3) 1128 MOVDU R0, 8(R3) 1129 MOVDU R0, 8(R3) 1130 MOVDU R0, 8(R3) 1131 MOVDU R0, 8(R3) 1132 MOVDU R0, 8(R3) 1133 MOVDU R0, 8(R3) 1134 MOVDU R0, 8(R3) 1135 MOVDU R0, 8(R3) 1136 MOVDU R0, 8(R3) 1137 MOVDU R0, 8(R3) 1138 MOVDU R0, 8(R3) 1139 MOVDU R0, 8(R3) 1140 MOVDU R0, 8(R3) 1141 MOVDU R0, 8(R3) 1142 MOVDU R0, 8(R3) 1143 MOVDU R0, 8(R3) 1144 MOVDU R0, 8(R3) 1145 MOVDU R0, 8(R3) 1146 MOVDU R0, 8(R3) 1147 MOVDU R0, 8(R3) 1148 MOVDU R0, 8(R3) 1149 MOVDU R0, 8(R3) 1150 MOVDU R0, 8(R3) 1151 MOVDU R0, 8(R3) 1152 MOVDU R0, 8(R3) 1153 MOVDU R0, 8(R3) 1154 MOVDU R0, 8(R3) 1155 MOVDU R0, 8(R3) 1156 MOVDU R0, 8(R3) 1157 MOVDU R0, 8(R3) 1158 MOVDU R0, 8(R3) 1159 MOVDU R0, 8(R3) 1160 MOVDU R0, 8(R3) 1161 MOVDU R0, 8(R3) 1162 MOVDU R0, 8(R3) 1163 MOVDU R0, 8(R3) 1164 MOVDU R0, 8(R3) 1165 MOVDU R0, 8(R3) 1166 MOVDU R0, 8(R3) 1167 MOVDU R0, 8(R3) 1168 MOVDU R0, 8(R3) 1169 MOVDU R0, 8(R3) 1170 MOVDU R0, 8(R3) 1171 MOVDU R0, 8(R3) 1172 MOVDU R0, 8(R3) 1173 MOVDU R0, 8(R3) 1174 MOVDU R0, 8(R3) 1175 MOVDU R0, 8(R3) 1176 MOVDU R0, 8(R3) 1177 MOVDU R0, 8(R3) 1178 MOVDU R0, 8(R3) 1179 MOVDU R0, 8(R3) 1180 MOVDU R0, 8(R3) 1181 MOVDU R0, 8(R3) 1182 MOVDU R0, 8(R3) 1183 MOVDU R0, 8(R3) 1184 MOVDU R0, 8(R3) 1185 MOVDU R0, 8(R3) 1186 MOVDU R0, 8(R3) 1187 MOVDU R0, 8(R3) 1188 MOVDU R0, 8(R3) 1189 MOVDU R0, 8(R3) 1190 MOVDU R0, 8(R3) 1191 MOVDU R0, 8(R3) 1192 MOVDU R0, 8(R3) 1193 MOVDU R0, 8(R3) 1194 MOVDU R0, 8(R3) 1195 MOVDU R0, 8(R3) 1196 MOVDU R0, 8(R3) 1197 MOVDU R0, 8(R3) 1198 MOVDU R0, 8(R3) 1199 MOVDU R0, 8(R3) 1200 MOVDU R0, 8(R3) 1201 MOVDU R0, 8(R3) 1202 MOVDU R0, 8(R3) 1203 MOVDU R0, 8(R3) 1204 MOVDU R0, 8(R3) 1205 MOVDU R0, 8(R3) 1206 MOVDU R0, 8(R3) 1207 MOVDU R0, 8(R3) 1208 MOVDU R0, 8(R3) 1209 MOVDU R0, 8(R3) 1210 MOVDU R0, 8(R3) 1211 MOVDU R0, 8(R3) 1212 MOVDU R0, 8(R3) 1213 MOVDU R0, 8(R3) 1214 MOVDU R0, 8(R3) 1215 MOVDU R0, 8(R3) 1216 RETURN 1217 1218 TEXT runtime·fastrand1(SB), NOSPLIT, $0-4 1219 MOVD g_m(g), R4 1220 MOVWZ m_fastrand(R4), R3 1221 ADD R3, R3 1222 CMPW R3, $0 1223 BGE 2(PC) 1224 XOR $0x88888eef, R3 1225 MOVW R3, m_fastrand(R4) 1226 MOVW R3, ret+0(FP) 1227 RETURN 1228 1229 TEXT runtime·return0(SB), NOSPLIT, $0 1230 MOVW $0, R3 1231 RETURN 1232 1233 // Called from cgo wrappers, this function returns g->m->curg.stack.hi. 1234 // Must obey the gcc calling convention. 1235 TEXT _cgo_topofstack(SB),NOSPLIT,$-8 1236 // g (R30) and R31 are callee-save in the C ABI, so save them 1237 MOVD g, R4 1238 MOVD R31, R5 1239 MOVD LR, R6 1240 1241 BL runtime·load_g(SB) // clobbers g (R30), R31 1242 MOVD g_m(g), R3 1243 MOVD m_curg(R3), R3 1244 MOVD (g_stack+stack_hi)(R3), R3 1245 1246 MOVD R4, g 1247 MOVD R5, R31 1248 MOVD R6, LR 1249 RET 1250 1251 // The top-most function running on a goroutine 1252 // returns to goexit+PCQuantum. 1253 TEXT runtime·goexit(SB),NOSPLIT,$-8-0 1254 MOVD R0, R0 // NOP 1255 BL runtime·goexit1(SB) // does not return 1256 1257 TEXT runtime·getg(SB),NOSPLIT,$-8-8 1258 MOVD g, ret+0(FP) 1259 RETURN 1260 1261 TEXT runtime·prefetcht0(SB),NOSPLIT,$0-8 1262 RETURN 1263 1264 TEXT runtime·prefetcht1(SB),NOSPLIT,$0-8 1265 RETURN 1266 1267 TEXT runtime·prefetcht2(SB),NOSPLIT,$0-8 1268 RETURN 1269 1270 TEXT runtime·prefetchnta(SB),NOSPLIT,$0-8 1271 RETURN