github.com/4ad/go@v0.0.0-20161219182952-69a12818b605/src/runtime/asm_ppc64x.s (about) 1 // Copyright 2014 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // +build ppc64 ppc64le 6 7 #include "go_asm.h" 8 #include "go_tls.h" 9 #include "funcdata.h" 10 #include "textflag.h" 11 #include "asm_ppc64x.h" 12 13 TEXT runtime·rt0_go(SB),NOSPLIT,$0 14 // R1 = stack; R3 = argc; R4 = argv; R13 = C TLS base pointer 15 16 // initialize essential registers 17 BL runtime·reginit(SB) 18 19 SUB $(FIXED_FRAME+16), R1 20 MOVD R2, 24(R1) // stash the TOC pointer away again now we've created a new frame 21 MOVW R3, FIXED_FRAME+0(R1) // argc 22 MOVD R4, FIXED_FRAME+8(R1) // argv 23 24 // create istack out of the given (operating system) stack. 25 // _cgo_init may update stackguard. 26 MOVD $runtime·g0(SB), g 27 MOVD $(-64*1024), R31 28 ADD R31, R1, R3 29 MOVD R3, g_stackguard0(g) 30 MOVD R3, g_stackguard1(g) 31 MOVD R3, (g_stack+stack_lo)(g) 32 MOVD R1, (g_stack+stack_hi)(g) 33 34 // if there is a _cgo_init, call it using the gcc ABI. 35 MOVD _cgo_init(SB), R12 36 CMP R0, R12 37 BEQ nocgo 38 MOVD R12, CTR // r12 = "global function entry point" 39 MOVD R13, R5 // arg 2: TLS base pointer 40 MOVD $setg_gcc<>(SB), R4 // arg 1: setg 41 MOVD g, R3 // arg 0: G 42 // C functions expect 32 bytes of space on caller stack frame 43 // and a 16-byte aligned R1 44 MOVD R1, R14 // save current stack 45 SUB $32, R1 // reserve 32 bytes 46 RLDCR $0, R1, $~15, R1 // 16-byte align 47 BL (CTR) // may clobber R0, R3-R12 48 MOVD R14, R1 // restore stack 49 MOVD 24(R1), R2 50 XOR R0, R0 // fix R0 51 52 nocgo: 53 // update stackguard after _cgo_init 54 MOVD (g_stack+stack_lo)(g), R3 55 ADD $const__StackGuard, R3 56 MOVD R3, g_stackguard0(g) 57 MOVD R3, g_stackguard1(g) 58 59 // set the per-goroutine and per-mach "registers" 60 MOVD $runtime·m0(SB), R3 61 62 // save m->g0 = g0 63 MOVD g, m_g0(R3) 64 // save m0 to g0->m 65 MOVD R3, g_m(g) 66 67 BL runtime·check(SB) 68 69 // args are already prepared 70 BL runtime·args(SB) 71 BL runtime·osinit(SB) 72 BL runtime·schedinit(SB) 73 74 // create a new goroutine to start program 75 MOVD $runtime·mainPC(SB), R3 // entry 76 MOVDU R3, -8(R1) 77 MOVDU R0, -8(R1) 78 MOVDU R0, -8(R1) 79 MOVDU R0, -8(R1) 80 MOVDU R0, -8(R1) 81 MOVDU R0, -8(R1) 82 BL runtime·newproc(SB) 83 ADD $(16+FIXED_FRAME), R1 84 85 // start this M 86 BL runtime·mstart(SB) 87 88 MOVD R0, 1(R0) 89 RET 90 91 DATA runtime·mainPC+0(SB)/8,$runtime·main(SB) 92 GLOBL runtime·mainPC(SB),RODATA,$8 93 94 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0 95 MOVD R0, 2(R0) // TODO: TD 96 RET 97 98 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0 99 RET 100 101 TEXT _cgo_reginit(SB),NOSPLIT|NOFRAME,$0-0 102 // crosscall_ppc64 and crosscall2 need to reginit, but can't 103 // get at the 'runtime.reginit' symbol. 104 BR runtime·reginit(SB) 105 106 TEXT runtime·reginit(SB),NOSPLIT|NOFRAME,$0-0 107 // set R0 to zero, it's expected by the toolchain 108 XOR R0, R0 109 // initialize essential FP registers 110 FMOVD $4503601774854144.0, F27 111 FMOVD $0.5, F29 112 FSUB F29, F29, F28 113 FADD F29, F29, F30 114 FADD F30, F30, F31 115 RET 116 117 /* 118 * go-routine 119 */ 120 121 // void gosave(Gobuf*) 122 // save state in Gobuf; setjmp 123 TEXT runtime·gosave(SB), NOSPLIT|NOFRAME, $0-8 124 MOVD buf+0(FP), R3 125 MOVD R1, gobuf_sp(R3) 126 MOVD LR, R31 127 MOVD R31, gobuf_pc(R3) 128 MOVD g, gobuf_g(R3) 129 MOVD R0, gobuf_lr(R3) 130 MOVD R0, gobuf_ret(R3) 131 MOVD R0, gobuf_ctxt(R3) 132 RET 133 134 // void gogo(Gobuf*) 135 // restore state from Gobuf; longjmp 136 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8 137 MOVD buf+0(FP), R5 138 MOVD gobuf_g(R5), g // make sure g is not nil 139 BL runtime·save_g(SB) 140 141 MOVD 0(g), R4 142 MOVD gobuf_sp(R5), R1 143 MOVD gobuf_lr(R5), R31 144 MOVD R31, LR 145 MOVD gobuf_ret(R5), R3 146 MOVD gobuf_ctxt(R5), R11 147 MOVD R0, gobuf_sp(R5) 148 MOVD R0, gobuf_ret(R5) 149 MOVD R0, gobuf_lr(R5) 150 MOVD R0, gobuf_ctxt(R5) 151 CMP R0, R0 // set condition codes for == test, needed by stack split 152 MOVD gobuf_pc(R5), R12 153 MOVD R12, CTR 154 BR (CTR) 155 156 // void mcall(fn func(*g)) 157 // Switch to m->g0's stack, call fn(g). 158 // Fn must never return. It should gogo(&g->sched) 159 // to keep running g. 160 TEXT runtime·mcall(SB), NOSPLIT|NOFRAME, $0-8 161 // Save caller state in g->sched 162 MOVD R1, (g_sched+gobuf_sp)(g) 163 MOVD LR, R31 164 MOVD R31, (g_sched+gobuf_pc)(g) 165 MOVD R0, (g_sched+gobuf_lr)(g) 166 MOVD g, (g_sched+gobuf_g)(g) 167 168 // Switch to m->g0 & its stack, call fn. 169 MOVD g, R3 170 MOVD g_m(g), R8 171 MOVD m_g0(R8), g 172 BL runtime·save_g(SB) 173 CMP g, R3 174 BNE 2(PC) 175 BR runtime·badmcall(SB) 176 MOVD fn+0(FP), R11 // context 177 MOVD 0(R11), R12 // code pointer 178 MOVD R12, CTR 179 MOVD (g_sched+gobuf_sp)(g), R1 // sp = m->g0->sched.sp 180 MOVDU R3, -8(R1) 181 MOVDU R0, -8(R1) 182 MOVDU R0, -8(R1) 183 MOVDU R0, -8(R1) 184 MOVDU R0, -8(R1) 185 BL (CTR) 186 MOVD 24(R1), R2 187 BR runtime·badmcall2(SB) 188 189 // systemstack_switch is a dummy routine that systemstack leaves at the bottom 190 // of the G stack. We need to distinguish the routine that 191 // lives at the bottom of the G stack from the one that lives 192 // at the top of the system stack because the one at the top of 193 // the system stack terminates the stack walk (see topofstack()). 194 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0 195 // We have several undefs here so that 16 bytes past 196 // $runtime·systemstack_switch lies within them whether or not the 197 // instructions that derive r2 from r12 are there. 198 UNDEF 199 UNDEF 200 UNDEF 201 BL (LR) // make sure this function is not leaf 202 RET 203 204 // func systemstack(fn func()) 205 TEXT runtime·systemstack(SB), NOSPLIT, $0-8 206 MOVD fn+0(FP), R3 // R3 = fn 207 MOVD R3, R11 // context 208 MOVD g_m(g), R4 // R4 = m 209 210 MOVD m_gsignal(R4), R5 // R5 = gsignal 211 CMP g, R5 212 BEQ noswitch 213 214 MOVD m_g0(R4), R5 // R5 = g0 215 CMP g, R5 216 BEQ noswitch 217 218 MOVD m_curg(R4), R6 219 CMP g, R6 220 BEQ switch 221 222 // Bad: g is not gsignal, not g0, not curg. What is it? 223 // Hide call from linker nosplit analysis. 224 MOVD $runtime·badsystemstack(SB), R12 225 MOVD R12, CTR 226 BL (CTR) 227 228 switch: 229 // save our state in g->sched. Pretend to 230 // be systemstack_switch if the G stack is scanned. 231 MOVD $runtime·systemstack_switch(SB), R6 232 ADD $16, R6 // get past prologue (including r2-setting instructions when they're there) 233 MOVD R6, (g_sched+gobuf_pc)(g) 234 MOVD R1, (g_sched+gobuf_sp)(g) 235 MOVD R0, (g_sched+gobuf_lr)(g) 236 MOVD g, (g_sched+gobuf_g)(g) 237 238 // switch to g0 239 MOVD R5, g 240 BL runtime·save_g(SB) 241 MOVD (g_sched+gobuf_sp)(g), R3 242 // make it look like mstart called systemstack on g0, to stop traceback 243 SUB $FIXED_FRAME, R3 244 MOVD $runtime·mstart(SB), R4 245 MOVD R4, 0(R3) 246 MOVD R3, R1 247 248 // call target function 249 MOVD 0(R11), R12 // code pointer 250 MOVD R12, CTR 251 BL (CTR) 252 253 // restore TOC pointer. It seems unlikely that we will use systemstack 254 // to call a function defined in another module, but the results of 255 // doing so would be so confusing that it's worth doing this. 256 MOVD g_m(g), R3 257 MOVD m_curg(R3), g 258 MOVD (g_sched+gobuf_sp)(g), R3 259 MOVD 24(R3), R2 260 // switch back to g 261 MOVD g_m(g), R3 262 MOVD m_curg(R3), g 263 BL runtime·save_g(SB) 264 MOVD (g_sched+gobuf_sp)(g), R1 265 MOVD R0, (g_sched+gobuf_sp)(g) 266 RET 267 268 noswitch: 269 // already on m stack, just call directly 270 MOVD 0(R11), R12 // code pointer 271 MOVD R12, CTR 272 BL (CTR) 273 MOVD 24(R1), R2 274 RET 275 276 /* 277 * support for morestack 278 */ 279 280 // Called during function prolog when more stack is needed. 281 // Caller has already loaded: 282 // R3: framesize, R4: argsize, R5: LR 283 // 284 // The traceback routines see morestack on a g0 as being 285 // the top of a stack (for example, morestack calling newstack 286 // calling the scheduler calling newm calling gc), so we must 287 // record an argument size. For that purpose, it has no arguments. 288 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0 289 // Cannot grow scheduler stack (m->g0). 290 MOVD g_m(g), R7 291 MOVD m_g0(R7), R8 292 CMP g, R8 293 BNE 2(PC) 294 BL runtime·abort(SB) 295 296 // Cannot grow signal stack (m->gsignal). 297 MOVD m_gsignal(R7), R8 298 CMP g, R8 299 BNE 2(PC) 300 BL runtime·abort(SB) 301 302 // Called from f. 303 // Set g->sched to context in f. 304 MOVD R11, (g_sched+gobuf_ctxt)(g) 305 MOVD R1, (g_sched+gobuf_sp)(g) 306 MOVD LR, R8 307 MOVD R8, (g_sched+gobuf_pc)(g) 308 MOVD R5, (g_sched+gobuf_lr)(g) 309 310 // Called from f. 311 // Set m->morebuf to f's caller. 312 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC 313 MOVD R1, (m_morebuf+gobuf_sp)(R7) // f's caller's SP 314 MOVD g, (m_morebuf+gobuf_g)(R7) 315 316 // Call newstack on m->g0's stack. 317 MOVD m_g0(R7), g 318 BL runtime·save_g(SB) 319 MOVD (g_sched+gobuf_sp)(g), R1 320 BL runtime·newstack(SB) 321 322 // Not reached, but make sure the return PC from the call to newstack 323 // is still in this function, and not the beginning of the next. 324 UNDEF 325 326 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0 327 MOVD R0, R11 328 BR runtime·morestack(SB) 329 330 TEXT runtime·stackBarrier(SB),NOSPLIT,$0 331 // We came here via a RET to an overwritten LR. 332 // R3 may be live. Other registers are available. 333 334 // Get the original return PC, g.stkbar[g.stkbarPos].savedLRVal. 335 MOVD (g_stkbar+slice_array)(g), R4 336 MOVD g_stkbarPos(g), R5 337 MOVD $stkbar__size, R6 338 MULLD R5, R6 339 ADD R4, R6 340 MOVD stkbar_savedLRVal(R6), R6 341 // Record that this stack barrier was hit. 342 ADD $1, R5 343 MOVD R5, g_stkbarPos(g) 344 // Jump to the original return PC. 345 MOVD R6, CTR 346 BR (CTR) 347 348 // reflectcall: call a function with the given argument list 349 // func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32). 350 // we don't have variable-sized frames, so we use a small number 351 // of constant-sized-frame functions to encode a few bits of size in the pc. 352 // Caution: ugly multiline assembly macros in your future! 353 354 #define DISPATCH(NAME,MAXSIZE) \ 355 MOVD $MAXSIZE, R31; \ 356 CMP R3, R31; \ 357 BGT 4(PC); \ 358 MOVD $NAME(SB), R12; \ 359 MOVD R12, CTR; \ 360 BR (CTR) 361 // Note: can't just "BR NAME(SB)" - bad inlining results. 362 363 TEXT reflect·call(SB), NOSPLIT, $0-0 364 BR ·reflectcall(SB) 365 366 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-32 367 MOVWZ argsize+24(FP), R3 368 // NOTE(rsc): No call16, because CALLFN needs four words 369 // of argument space to invoke callwritebarrier. 370 DISPATCH(runtime·call32, 32) 371 DISPATCH(runtime·call64, 64) 372 DISPATCH(runtime·call128, 128) 373 DISPATCH(runtime·call256, 256) 374 DISPATCH(runtime·call512, 512) 375 DISPATCH(runtime·call1024, 1024) 376 DISPATCH(runtime·call2048, 2048) 377 DISPATCH(runtime·call4096, 4096) 378 DISPATCH(runtime·call8192, 8192) 379 DISPATCH(runtime·call16384, 16384) 380 DISPATCH(runtime·call32768, 32768) 381 DISPATCH(runtime·call65536, 65536) 382 DISPATCH(runtime·call131072, 131072) 383 DISPATCH(runtime·call262144, 262144) 384 DISPATCH(runtime·call524288, 524288) 385 DISPATCH(runtime·call1048576, 1048576) 386 DISPATCH(runtime·call2097152, 2097152) 387 DISPATCH(runtime·call4194304, 4194304) 388 DISPATCH(runtime·call8388608, 8388608) 389 DISPATCH(runtime·call16777216, 16777216) 390 DISPATCH(runtime·call33554432, 33554432) 391 DISPATCH(runtime·call67108864, 67108864) 392 DISPATCH(runtime·call134217728, 134217728) 393 DISPATCH(runtime·call268435456, 268435456) 394 DISPATCH(runtime·call536870912, 536870912) 395 DISPATCH(runtime·call1073741824, 1073741824) 396 MOVD $runtime·badreflectcall(SB), R12 397 MOVD R12, CTR 398 BR (CTR) 399 400 #define CALLFN(NAME,MAXSIZE) \ 401 TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \ 402 NO_LOCAL_POINTERS; \ 403 /* copy arguments to stack */ \ 404 MOVD arg+16(FP), R3; \ 405 MOVWZ argsize+24(FP), R4; \ 406 MOVD R1, R5; \ 407 ADD $(FIXED_FRAME-1), R5; \ 408 SUB $1, R3; \ 409 ADD R5, R4; \ 410 CMP R5, R4; \ 411 BEQ 4(PC); \ 412 MOVBZU 1(R3), R6; \ 413 MOVBZU R6, 1(R5); \ 414 BR -4(PC); \ 415 /* call function */ \ 416 MOVD f+8(FP), R11; \ 417 MOVD (R11), R12; \ 418 MOVD R12, CTR; \ 419 PCDATA $PCDATA_StackMapIndex, $0; \ 420 BL (CTR); \ 421 MOVD 24(R1), R2; \ 422 /* copy return values back */ \ 423 MOVD arg+16(FP), R3; \ 424 MOVWZ n+24(FP), R4; \ 425 MOVWZ retoffset+28(FP), R6; \ 426 MOVD R1, R5; \ 427 ADD R6, R5; \ 428 ADD R6, R3; \ 429 SUB R6, R4; \ 430 ADD $(FIXED_FRAME-1), R5; \ 431 SUB $1, R3; \ 432 ADD R5, R4; \ 433 loop: \ 434 CMP R5, R4; \ 435 BEQ end; \ 436 MOVBZU 1(R5), R6; \ 437 MOVBZU R6, 1(R3); \ 438 BR loop; \ 439 end: \ 440 /* execute write barrier updates */ \ 441 MOVD argtype+0(FP), R7; \ 442 MOVD arg+16(FP), R3; \ 443 MOVWZ n+24(FP), R4; \ 444 MOVWZ retoffset+28(FP), R6; \ 445 MOVD R7, FIXED_FRAME+0(R1); \ 446 MOVD R3, FIXED_FRAME+8(R1); \ 447 MOVD R4, FIXED_FRAME+16(R1); \ 448 MOVD R6, FIXED_FRAME+24(R1); \ 449 BL runtime·callwritebarrier(SB); \ 450 RET 451 452 CALLFN(·call32, 32) 453 CALLFN(·call64, 64) 454 CALLFN(·call128, 128) 455 CALLFN(·call256, 256) 456 CALLFN(·call512, 512) 457 CALLFN(·call1024, 1024) 458 CALLFN(·call2048, 2048) 459 CALLFN(·call4096, 4096) 460 CALLFN(·call8192, 8192) 461 CALLFN(·call16384, 16384) 462 CALLFN(·call32768, 32768) 463 CALLFN(·call65536, 65536) 464 CALLFN(·call131072, 131072) 465 CALLFN(·call262144, 262144) 466 CALLFN(·call524288, 524288) 467 CALLFN(·call1048576, 1048576) 468 CALLFN(·call2097152, 2097152) 469 CALLFN(·call4194304, 4194304) 470 CALLFN(·call8388608, 8388608) 471 CALLFN(·call16777216, 16777216) 472 CALLFN(·call33554432, 33554432) 473 CALLFN(·call67108864, 67108864) 474 CALLFN(·call134217728, 134217728) 475 CALLFN(·call268435456, 268435456) 476 CALLFN(·call536870912, 536870912) 477 CALLFN(·call1073741824, 1073741824) 478 479 TEXT runtime·procyield(SB),NOSPLIT,$0-0 480 RET 481 482 // void jmpdefer(fv, sp); 483 // called from deferreturn. 484 // 1. grab stored LR for caller 485 // 2. sub 8 bytes to get back to either nop or toc reload before deferreturn 486 // 3. BR to fn 487 // When dynamically linking Go, it is not sufficient to rewind to the BL 488 // deferreturn -- we might be jumping between modules and so we need to reset 489 // the TOC pointer in r2. To do this, codegen inserts MOVD 24(R1), R2 *before* 490 // the BL deferreturn and jmpdefer rewinds to that. 491 TEXT runtime·jmpdefer(SB), NOSPLIT|NOFRAME, $0-16 492 MOVD 0(R1), R31 493 SUB $8, R31 494 MOVD R31, LR 495 496 MOVD fv+0(FP), R11 497 MOVD argp+8(FP), R1 498 SUB $FIXED_FRAME, R1 499 MOVD 0(R11), R12 500 MOVD R12, CTR 501 BR (CTR) 502 503 // Save state of caller into g->sched. Smashes R31. 504 TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0 505 MOVD LR, R31 506 MOVD R31, (g_sched+gobuf_pc)(g) 507 MOVD R1, (g_sched+gobuf_sp)(g) 508 MOVD R0, (g_sched+gobuf_lr)(g) 509 MOVD R0, (g_sched+gobuf_ret)(g) 510 MOVD R0, (g_sched+gobuf_ctxt)(g) 511 RET 512 513 // func asmcgocall(fn, arg unsafe.Pointer) int32 514 // Call fn(arg) on the scheduler stack, 515 // aligned appropriately for the gcc ABI. 516 // See cgocall.go for more details. 517 TEXT ·asmcgocall(SB),NOSPLIT,$0-20 518 MOVD fn+0(FP), R3 519 MOVD arg+8(FP), R4 520 521 MOVD R1, R7 // save original stack pointer 522 MOVD g, R5 523 524 // Figure out if we need to switch to m->g0 stack. 525 // We get called to create new OS threads too, and those 526 // come in on the m->g0 stack already. 527 MOVD g_m(g), R6 528 MOVD m_g0(R6), R6 529 CMP R6, g 530 BEQ g0 531 BL gosave<>(SB) 532 MOVD R6, g 533 BL runtime·save_g(SB) 534 MOVD (g_sched+gobuf_sp)(g), R1 535 536 // Now on a scheduling stack (a pthread-created stack). 537 g0: 538 // Save room for two of our pointers, plus 32 bytes of callee 539 // save area that lives on the caller stack. 540 SUB $48, R1 541 RLDCR $0, R1, $~15, R1 // 16-byte alignment for gcc ABI 542 MOVD R5, 40(R1) // save old g on stack 543 MOVD (g_stack+stack_hi)(R5), R5 544 SUB R7, R5 545 MOVD R5, 32(R1) // save depth in old g stack (can't just save SP, as stack might be copied during a callback) 546 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?) 547 // This is a "global call", so put the global entry point in r12 548 MOVD R3, R12 549 MOVD R12, CTR 550 MOVD R4, R3 // arg in r3 551 BL (CTR) 552 553 // C code can clobber R0, so set it back to 0. F27-F31 are 554 // callee save, so we don't need to recover those. 555 XOR R0, R0 556 // Restore g, stack pointer, toc pointer. 557 // R3 is errno, so don't touch it 558 MOVD 40(R1), g 559 MOVD (g_stack+stack_hi)(g), R5 560 MOVD 32(R1), R6 561 SUB R6, R5 562 MOVD 24(R5), R2 563 BL runtime·save_g(SB) 564 MOVD (g_stack+stack_hi)(g), R5 565 MOVD 32(R1), R6 566 SUB R6, R5 567 MOVD R5, R1 568 569 MOVW R3, ret+16(FP) 570 RET 571 572 // cgocallback(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt) 573 // Turn the fn into a Go func (by taking its address) and call 574 // cgocallback_gofunc. 575 TEXT runtime·cgocallback(SB),NOSPLIT,$32-32 576 MOVD $fn+0(FP), R3 577 MOVD R3, FIXED_FRAME+0(R1) 578 MOVD frame+8(FP), R3 579 MOVD R3, FIXED_FRAME+8(R1) 580 MOVD framesize+16(FP), R3 581 MOVD R3, FIXED_FRAME+16(R1) 582 MOVD ctxt+24(FP), R3 583 MOVD R3, FIXED_FRAME+24(R1) 584 MOVD $runtime·cgocallback_gofunc(SB), R12 585 MOVD R12, CTR 586 BL (CTR) 587 RET 588 589 // cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize, uintptr ctxt) 590 // See cgocall.go for more details. 591 TEXT ·cgocallback_gofunc(SB),NOSPLIT,$16-32 592 NO_LOCAL_POINTERS 593 594 // Load m and g from thread-local storage. 595 MOVB runtime·iscgo(SB), R3 596 CMP R3, $0 597 BEQ nocgo 598 BL runtime·load_g(SB) 599 nocgo: 600 601 // If g is nil, Go did not create the current thread. 602 // Call needm to obtain one for temporary use. 603 // In this case, we're running on the thread stack, so there's 604 // lots of space, but the linker doesn't know. Hide the call from 605 // the linker analysis by using an indirect call. 606 CMP g, $0 607 BEQ needm 608 609 MOVD g_m(g), R8 610 MOVD R8, savedm-8(SP) 611 BR havem 612 613 needm: 614 MOVD g, savedm-8(SP) // g is zero, so is m. 615 MOVD $runtime·needm(SB), R12 616 MOVD R12, CTR 617 BL (CTR) 618 619 // Set m->sched.sp = SP, so that if a panic happens 620 // during the function we are about to execute, it will 621 // have a valid SP to run on the g0 stack. 622 // The next few lines (after the havem label) 623 // will save this SP onto the stack and then write 624 // the same SP back to m->sched.sp. That seems redundant, 625 // but if an unrecovered panic happens, unwindm will 626 // restore the g->sched.sp from the stack location 627 // and then systemstack will try to use it. If we don't set it here, 628 // that restored SP will be uninitialized (typically 0) and 629 // will not be usable. 630 MOVD g_m(g), R8 631 MOVD m_g0(R8), R3 632 MOVD R1, (g_sched+gobuf_sp)(R3) 633 634 havem: 635 // Now there's a valid m, and we're running on its m->g0. 636 // Save current m->g0->sched.sp on stack and then set it to SP. 637 // Save current sp in m->g0->sched.sp in preparation for 638 // switch back to m->curg stack. 639 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP). 640 MOVD m_g0(R8), R3 641 MOVD (g_sched+gobuf_sp)(R3), R4 642 MOVD R4, savedsp-16(SP) 643 MOVD R1, (g_sched+gobuf_sp)(R3) 644 645 // Switch to m->curg stack and call runtime.cgocallbackg. 646 // Because we are taking over the execution of m->curg 647 // but *not* resuming what had been running, we need to 648 // save that information (m->curg->sched) so we can restore it. 649 // We can restore m->curg->sched.sp easily, because calling 650 // runtime.cgocallbackg leaves SP unchanged upon return. 651 // To save m->curg->sched.pc, we push it onto the stack. 652 // This has the added benefit that it looks to the traceback 653 // routine like cgocallbackg is going to return to that 654 // PC (because the frame we allocate below has the same 655 // size as cgocallback_gofunc's frame declared above) 656 // so that the traceback will seamlessly trace back into 657 // the earlier calls. 658 // 659 // In the new goroutine, -8(SP) is unused (where SP refers to 660 // m->curg's SP while we're setting it up, before we've adjusted it). 661 MOVD m_curg(R8), g 662 BL runtime·save_g(SB) 663 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4 664 MOVD (g_sched+gobuf_pc)(g), R5 665 MOVD R5, -(FIXED_FRAME+16)(R4) 666 MOVD ctxt+24(FP), R3 667 MOVD R3, -16(R4) 668 MOVD $-(FIXED_FRAME+16)(R4), R1 669 BL runtime·cgocallbackg(SB) 670 671 // Restore g->sched (== m->curg->sched) from saved values. 672 MOVD 0(R1), R5 673 MOVD R5, (g_sched+gobuf_pc)(g) 674 MOVD $(FIXED_FRAME+16)(R1), R4 675 MOVD R4, (g_sched+gobuf_sp)(g) 676 677 // Switch back to m->g0's stack and restore m->g0->sched.sp. 678 // (Unlike m->curg, the g0 goroutine never uses sched.pc, 679 // so we do not have to restore it.) 680 MOVD g_m(g), R8 681 MOVD m_g0(R8), g 682 BL runtime·save_g(SB) 683 MOVD (g_sched+gobuf_sp)(g), R1 684 MOVD savedsp-16(SP), R4 685 MOVD R4, (g_sched+gobuf_sp)(g) 686 687 // If the m on entry was nil, we called needm above to borrow an m 688 // for the duration of the call. Since the call is over, return it with dropm. 689 MOVD savedm-8(SP), R6 690 CMP R6, $0 691 BNE droppedm 692 MOVD $runtime·dropm(SB), R12 693 MOVD R12, CTR 694 BL (CTR) 695 droppedm: 696 697 // Done! 698 RET 699 700 // void setg(G*); set g. for use by needm. 701 TEXT runtime·setg(SB), NOSPLIT, $0-8 702 MOVD gg+0(FP), g 703 // This only happens if iscgo, so jump straight to save_g 704 BL runtime·save_g(SB) 705 RET 706 707 // void setg_gcc(G*); set g in C TLS. 708 // Must obey the gcc calling convention. 709 TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0 710 // The standard prologue clobbers R31, which is callee-save in 711 // the C ABI, so we have to use $-8-0 and save LR ourselves. 712 MOVD LR, R4 713 // Also save g and R31, since they're callee-save in C ABI 714 MOVD R31, R5 715 MOVD g, R6 716 717 MOVD R3, g 718 BL runtime·save_g(SB) 719 720 MOVD R6, g 721 MOVD R5, R31 722 MOVD R4, LR 723 RET 724 725 TEXT runtime·getcallerpc(SB),NOSPLIT,$8-16 726 MOVD FIXED_FRAME+8(R1), R3 // LR saved by caller 727 MOVD runtime·stackBarrierPC(SB), R4 728 CMP R3, R4 729 BNE nobar 730 // Get original return PC. 731 BL runtime·nextBarrierPC(SB) 732 MOVD FIXED_FRAME+0(R1), R3 733 nobar: 734 MOVD R3, ret+8(FP) 735 RET 736 737 TEXT runtime·setcallerpc(SB),NOSPLIT,$8-16 738 MOVD pc+8(FP), R3 739 MOVD FIXED_FRAME+8(R1), R4 740 MOVD runtime·stackBarrierPC(SB), R5 741 CMP R4, R5 742 BEQ setbar 743 MOVD R3, FIXED_FRAME+8(R1) // set LR in caller 744 RET 745 setbar: 746 // Set the stack barrier return PC. 747 MOVD R3, FIXED_FRAME+0(R1) 748 BL runtime·setNextBarrierPC(SB) 749 RET 750 751 TEXT runtime·getcallersp(SB),NOSPLIT,$0-16 752 MOVD argp+0(FP), R3 753 SUB $FIXED_FRAME, R3 754 MOVD R3, ret+8(FP) 755 RET 756 757 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0 758 MOVW (R0), R0 759 UNDEF 760 761 #define TBRL 268 762 #define TBRU 269 /* Time base Upper/Lower */ 763 764 // int64 runtime·cputicks(void) 765 TEXT runtime·cputicks(SB),NOSPLIT,$0-8 766 MOVW SPR(TBRU), R4 767 MOVW SPR(TBRL), R3 768 MOVW SPR(TBRU), R5 769 CMPW R4, R5 770 BNE -4(PC) 771 SLD $32, R5 772 OR R5, R3 773 MOVD R3, ret+0(FP) 774 RET 775 776 // memhash_varlen(p unsafe.Pointer, h seed) uintptr 777 // redirects to memhash(p, h, size) using the size 778 // stored in the closure. 779 TEXT runtime·memhash_varlen(SB),NOSPLIT,$40-24 780 GO_ARGS 781 NO_LOCAL_POINTERS 782 MOVD p+0(FP), R3 783 MOVD h+8(FP), R4 784 MOVD 8(R11), R5 785 MOVD R3, FIXED_FRAME+0(R1) 786 MOVD R4, FIXED_FRAME+8(R1) 787 MOVD R5, FIXED_FRAME+16(R1) 788 BL runtime·memhash(SB) 789 MOVD FIXED_FRAME+24(R1), R3 790 MOVD R3, ret+16(FP) 791 RET 792 793 // AES hashing not implemented for ppc64 794 TEXT runtime·aeshash(SB),NOSPLIT|NOFRAME,$0-0 795 MOVW (R0), R1 796 TEXT runtime·aeshash32(SB),NOSPLIT|NOFRAME,$0-0 797 MOVW (R0), R1 798 TEXT runtime·aeshash64(SB),NOSPLIT|NOFRAME,$0-0 799 MOVW (R0), R1 800 TEXT runtime·aeshashstr(SB),NOSPLIT|NOFRAME,$0-0 801 MOVW (R0), R1 802 803 TEXT runtime·memequal(SB),NOSPLIT,$0-25 804 MOVD a+0(FP), R3 805 MOVD b+8(FP), R4 806 MOVD size+16(FP), R5 807 808 BL runtime·memeqbody(SB) 809 MOVB R9, ret+24(FP) 810 RET 811 812 // memequal_varlen(a, b unsafe.Pointer) bool 813 TEXT runtime·memequal_varlen(SB),NOSPLIT,$40-17 814 MOVD a+0(FP), R3 815 MOVD b+8(FP), R4 816 CMP R3, R4 817 BEQ eq 818 MOVD 8(R11), R5 // compiler stores size at offset 8 in the closure 819 BL runtime·memeqbody(SB) 820 MOVB R9, ret+16(FP) 821 RET 822 eq: 823 MOVD $1, R3 824 MOVB R3, ret+16(FP) 825 RET 826 827 // Do an efficieint memequal for ppc64 828 // for reuse where possible. 829 // R3 = s1 830 // R4 = s2 831 // R5 = len 832 // R9 = return value 833 // R6, R7 clobbered 834 TEXT runtime·memeqbody(SB),NOSPLIT|NOFRAME,$0-0 835 MOVD R5,CTR 836 CMP R5,$8 // only optimize >=8 837 BLT simplecheck 838 DCBT (R3) // cache hint 839 DCBT (R4) 840 CMP R5,$32 // optimize >= 32 841 MOVD R5,R6 // needed if setup8a branch 842 BLT setup8a // 8 byte moves only 843 setup32a: // 8 byte aligned, >= 32 bytes 844 SRADCC $5,R5,R6 // number of 32 byte chunks to compare 845 MOVD R6,CTR 846 loop32a: 847 MOVD 0(R3),R6 // doublewords to compare 848 MOVD 0(R4),R7 849 MOVD 8(R3),R8 // 850 MOVD 8(R4),R9 851 CMP R6,R7 // bytes batch? 852 BNE noteq 853 MOVD 16(R3),R6 854 MOVD 16(R4),R7 855 CMP R8,R9 // bytes match? 856 MOVD 24(R3),R8 857 MOVD 24(R4),R9 858 BNE noteq 859 CMP R6,R7 // bytes match? 860 BNE noteq 861 ADD $32,R3 // bump up to next 32 862 ADD $32,R4 863 CMP R8,R9 // bytes match? 864 BC 8,2,loop32a // br ctr and cr 865 BNE noteq 866 ANDCC $24,R5,R6 // Any 8 byte chunks? 867 BEQ leftover // and result is 0 868 setup8a: 869 SRADCC $3,R6,R6 // get the 8 byte count 870 BEQ leftover // shifted value is 0 871 MOVD R6,CTR 872 loop8: 873 MOVD 0(R3),R6 // doublewords to compare 874 ADD $8,R3 875 MOVD 0(R4),R7 876 ADD $8,R4 877 CMP R6,R7 // match? 878 BC 8,2,loop8 // bt ctr <> 0 && cr 879 BNE noteq 880 leftover: 881 ANDCC $7,R5,R6 // check for leftover bytes 882 BEQ equal 883 MOVD R6,CTR 884 BR simple 885 simplecheck: 886 CMP R5,$0 887 BEQ equal 888 simple: 889 MOVBZ 0(R3), R6 890 ADD $1,R3 891 MOVBZ 0(R4), R7 892 ADD $1,R4 893 CMP R6, R7 894 BNE noteq 895 BC 8,2,simple 896 BNE noteq 897 BR equal 898 noteq: 899 MOVD $0, R9 900 RET 901 equal: 902 MOVD $1, R9 903 RET 904 905 // eqstring tests whether two strings are equal. 906 // The compiler guarantees that strings passed 907 // to eqstring have equal length. 908 // See runtime_test.go:eqstring_generic for 909 // equivalent Go code. 910 TEXT runtime·eqstring(SB),NOSPLIT,$0-33 911 MOVD s1str+0(FP), R3 912 MOVD s2str+16(FP), R4 913 MOVD $1, R5 914 MOVB R5, ret+32(FP) 915 CMP R3, R4 916 BNE 2(PC) 917 RET 918 MOVD s1len+8(FP), R5 919 BL runtime·memeqbody(SB) 920 MOVB R9, ret+32(FP) 921 RET 922 923 TEXT bytes·Equal(SB),NOSPLIT,$0-49 924 MOVD a_len+8(FP), R4 925 MOVD b_len+32(FP), R5 926 CMP R5, R4 // unequal lengths are not equal 927 BNE noteq 928 MOVD a+0(FP), R3 929 MOVD b+24(FP), R4 930 BL runtime·memeqbody(SB) 931 932 MOVBZ R9,ret+48(FP) 933 RET 934 935 noteq: 936 MOVBZ $0,ret+48(FP) 937 RET 938 939 equal: 940 MOVD $1,R3 941 MOVBZ R3,ret+48(FP) 942 RET 943 944 TEXT bytes·IndexByte(SB),NOSPLIT,$0-40 945 MOVD s+0(FP), R3 946 MOVD s_len+8(FP), R4 947 MOVBZ c+24(FP), R5 // byte to find 948 MOVD R3, R6 // store base for later 949 SUB $1, R3 950 ADD R3, R4 // end-1 951 952 loop: 953 CMP R3, R4 954 BEQ notfound 955 MOVBZU 1(R3), R7 956 CMP R7, R5 957 BNE loop 958 959 SUB R6, R3 // remove base 960 MOVD R3, ret+32(FP) 961 RET 962 963 notfound: 964 MOVD $-1, R3 965 MOVD R3, ret+32(FP) 966 RET 967 968 TEXT strings·IndexByte(SB),NOSPLIT,$0-32 969 MOVD p+0(FP), R3 970 MOVD b_len+8(FP), R4 971 MOVBZ c+16(FP), R5 // byte to find 972 MOVD R3, R6 // store base for later 973 SUB $1, R3 974 ADD R3, R4 // end-1 975 976 loop: 977 CMP R3, R4 978 BEQ notfound 979 MOVBZU 1(R3), R7 980 CMP R7, R5 981 BNE loop 982 983 SUB R6, R3 // remove base 984 MOVD R3, ret+24(FP) 985 RET 986 987 notfound: 988 MOVD $-1, R3 989 MOVD R3, ret+24(FP) 990 RET 991 992 TEXT runtime·cmpstring(SB),NOSPLIT|NOFRAME,$0-40 993 MOVD s1_base+0(FP), R5 994 MOVD s1_len+8(FP), R3 995 MOVD s2_base+16(FP), R6 996 MOVD s2_len+24(FP), R4 997 MOVD $ret+32(FP), R7 998 BR runtime·cmpbody<>(SB) 999 1000 TEXT bytes·Compare(SB),NOSPLIT|NOFRAME,$0-56 1001 MOVD s1+0(FP), R5 1002 MOVD s1+8(FP), R3 1003 MOVD s2+24(FP), R6 1004 MOVD s2+32(FP), R4 1005 MOVD $ret+48(FP), R7 1006 BR runtime·cmpbody<>(SB) 1007 1008 // On entry: 1009 // R3 is the length of s1 1010 // R4 is the length of s2 1011 // R5 points to the start of s1 1012 // R6 points to the start of s2 1013 // R7 points to return value (-1/0/1 will be written here) 1014 // 1015 // On exit: 1016 // R5, R6, R8, R9 and R10 are clobbered 1017 TEXT runtime·cmpbody<>(SB),NOSPLIT|NOFRAME,$0-0 1018 CMP R5, R6 1019 BEQ samebytes // same starting pointers; compare lengths 1020 SUB $1, R5 1021 SUB $1, R6 1022 MOVD R4, R8 1023 CMP R3, R4 1024 BGE 2(PC) 1025 MOVD R3, R8 // R8 is min(R3, R4) 1026 ADD R5, R8 // R5 is current byte in s1, R8 is last byte in s1 to compare 1027 loop: 1028 CMP R5, R8 1029 BEQ samebytes // all compared bytes were the same; compare lengths 1030 MOVBZU 1(R5), R9 1031 MOVBZU 1(R6), R10 1032 CMP R9, R10 1033 BEQ loop 1034 // bytes differed 1035 MOVD $1, R4 1036 BGT 2(PC) 1037 NEG R4 1038 MOVD R4, (R7) 1039 RET 1040 samebytes: 1041 MOVD $1, R8 1042 CMP R3, R4 1043 BNE 3(PC) 1044 MOVD R0, (R7) 1045 RET 1046 BGT 2(PC) 1047 NEG R8 1048 MOVD R8, (R7) 1049 RET 1050 1051 TEXT runtime·fastrand1(SB), NOSPLIT, $0-4 1052 MOVD g_m(g), R4 1053 MOVWZ m_fastrand(R4), R3 1054 ADD R3, R3 1055 CMPW R3, $0 1056 BGE 2(PC) 1057 XOR $0x88888eef, R3 1058 MOVW R3, m_fastrand(R4) 1059 MOVW R3, ret+0(FP) 1060 RET 1061 1062 TEXT runtime·return0(SB), NOSPLIT, $0 1063 MOVW $0, R3 1064 RET 1065 1066 // Called from cgo wrappers, this function returns g->m->curg.stack.hi. 1067 // Must obey the gcc calling convention. 1068 TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0 1069 // g (R30) and R31 are callee-save in the C ABI, so save them 1070 MOVD g, R4 1071 MOVD R31, R5 1072 MOVD LR, R6 1073 1074 BL runtime·load_g(SB) // clobbers g (R30), R31 1075 MOVD g_m(g), R3 1076 MOVD m_curg(R3), R3 1077 MOVD (g_stack+stack_hi)(R3), R3 1078 1079 MOVD R4, g 1080 MOVD R5, R31 1081 MOVD R6, LR 1082 RET 1083 1084 // The top-most function running on a goroutine 1085 // returns to goexit+PCQuantum. 1086 // 1087 // When dynamically linking Go, it can be returned to from a function 1088 // implemented in a different module and so needs to reload the TOC pointer 1089 // from the stack (although this function declares that it does not set up x-a 1090 // frame, newproc1 does in fact allocate one for goexit and saves the TOC 1091 // pointer in the correct place). 1092 // goexit+_PCQuantum is halfway through the usual global entry point prologue 1093 // that derives r2 from r12 which is a bit silly, but not harmful. 1094 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME,$0-0 1095 MOVD 24(R1), R2 1096 BL runtime·goexit1(SB) // does not return 1097 // traceback from goexit1 must hit code range of goexit 1098 MOVD R0, R0 // NOP 1099 1100 TEXT runtime·prefetcht0(SB),NOSPLIT,$0-8 1101 RET 1102 1103 TEXT runtime·prefetcht1(SB),NOSPLIT,$0-8 1104 RET 1105 1106 TEXT runtime·prefetcht2(SB),NOSPLIT,$0-8 1107 RET 1108 1109 TEXT runtime·prefetchnta(SB),NOSPLIT,$0-8 1110 RET 1111 1112 TEXT runtime·sigreturn(SB),NOSPLIT,$0-8 1113 RET 1114 1115 // prepGoExitFrame saves the current TOC pointer (i.e. the TOC pointer for the 1116 // module containing runtime) to the frame that goexit will execute in when 1117 // the goroutine exits. It's implemented in assembly mainly because that's the 1118 // easiest way to get access to R2. 1119 TEXT runtime·prepGoExitFrame(SB),NOSPLIT,$0-8 1120 MOVD sp+0(FP), R3 1121 MOVD R2, 24(R3) 1122 RET 1123 1124 TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0 1125 ADD $-8, R1 1126 MOVD R31, 0(R1) 1127 MOVD runtime·lastmoduledatap(SB), R4 1128 MOVD R3, moduledata_next(R4) 1129 MOVD R3, runtime·lastmoduledatap(SB) 1130 MOVD 0(R1), R31 1131 ADD $8, R1 1132 RET 1133 1134 TEXT ·checkASM(SB),NOSPLIT,$0-1 1135 MOVW $1, R3 1136 MOVB R3, ret+0(FP) 1137 RET