github.com/comwrg/go/src@v0.0.0-20220319063731-c238d0440370/runtime/asm_ppc64x.s (about) 1 // Copyright 2014 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 //go:build ppc64 || ppc64le 6 // +build ppc64 ppc64le 7 8 #include "go_asm.h" 9 #include "go_tls.h" 10 #include "funcdata.h" 11 #include "textflag.h" 12 #include "asm_ppc64x.h" 13 14 #ifdef GOOS_aix 15 #define cgoCalleeStackSize 48 16 #else 17 #define cgoCalleeStackSize 32 18 #endif 19 20 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0 21 // R1 = stack; R3 = argc; R4 = argv; R13 = C TLS base pointer 22 23 // initialize essential registers 24 BL runtime·reginit(SB) 25 26 SUB $(FIXED_FRAME+16), R1 27 MOVD R2, 24(R1) // stash the TOC pointer away again now we've created a new frame 28 MOVW R3, FIXED_FRAME+0(R1) // argc 29 MOVD R4, FIXED_FRAME+8(R1) // argv 30 31 // create istack out of the given (operating system) stack. 32 // _cgo_init may update stackguard. 33 MOVD $runtime·g0(SB), g 34 BL runtime·save_g(SB) 35 MOVD $(-64*1024), R31 36 ADD R31, R1, R3 37 MOVD R3, g_stackguard0(g) 38 MOVD R3, g_stackguard1(g) 39 MOVD R3, (g_stack+stack_lo)(g) 40 MOVD R1, (g_stack+stack_hi)(g) 41 42 // if there is a _cgo_init, call it using the gcc ABI. 43 MOVD _cgo_init(SB), R12 44 CMP R0, R12 45 BEQ nocgo 46 #ifdef GOARCH_ppc64 47 // ppc64 use elf ABI v1. we must get the real entry address from 48 // first slot of the function descriptor before call. 49 MOVD 8(R12), R2 50 MOVD (R12), R12 51 #endif 52 MOVD R12, CTR // r12 = "global function entry point" 53 MOVD R13, R5 // arg 2: TLS base pointer 54 MOVD $setg_gcc<>(SB), R4 // arg 1: setg 55 MOVD g, R3 // arg 0: G 56 // C functions expect 32 (48 for AIX) bytes of space on caller 57 // stack frame and a 16-byte aligned R1 58 MOVD R1, R14 // save current stack 59 SUB $cgoCalleeStackSize, R1 // reserve the callee area 60 RLDCR $0, R1, $~15, R1 // 16-byte align 61 BL (CTR) // may clobber R0, R3-R12 62 MOVD R14, R1 // restore stack 63 #ifndef GOOS_aix 64 MOVD 24(R1), R2 65 #endif 66 XOR R0, R0 // fix R0 67 68 nocgo: 69 // update stackguard after _cgo_init 70 MOVD (g_stack+stack_lo)(g), R3 71 ADD $const__StackGuard, R3 72 MOVD R3, g_stackguard0(g) 73 MOVD R3, g_stackguard1(g) 74 75 // set the per-goroutine and per-mach "registers" 76 MOVD $runtime·m0(SB), R3 77 78 // save m->g0 = g0 79 MOVD g, m_g0(R3) 80 // save m0 to g0->m 81 MOVD R3, g_m(g) 82 83 BL runtime·check(SB) 84 85 // args are already prepared 86 BL runtime·args(SB) 87 BL runtime·osinit(SB) 88 BL runtime·schedinit(SB) 89 90 // create a new goroutine to start program 91 MOVD $runtime·mainPC(SB), R3 // entry 92 MOVDU R3, -8(R1) 93 MOVDU R0, -8(R1) 94 MOVDU R0, -8(R1) 95 MOVDU R0, -8(R1) 96 MOVDU R0, -8(R1) 97 MOVDU R0, -8(R1) 98 BL runtime·newproc(SB) 99 ADD $(16+FIXED_FRAME), R1 100 101 // start this M 102 BL runtime·mstart(SB) 103 104 MOVD R0, 0(R0) 105 RET 106 107 DATA runtime·mainPC+0(SB)/8,$runtime·main(SB) 108 GLOBL runtime·mainPC(SB),RODATA,$8 109 110 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0 111 MOVD R0, 0(R0) // TODO: TD 112 RET 113 114 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0 115 RET 116 117 // Any changes must be reflected to runtime/cgo/gcc_aix_ppc64.S:.crosscall_ppc64 118 TEXT _cgo_reginit(SB),NOSPLIT|NOFRAME,$0-0 119 // crosscall_ppc64 and crosscall2 need to reginit, but can't 120 // get at the 'runtime.reginit' symbol. 121 BR runtime·reginit(SB) 122 123 TEXT runtime·reginit(SB),NOSPLIT|NOFRAME,$0-0 124 // set R0 to zero, it's expected by the toolchain 125 XOR R0, R0 126 RET 127 128 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0 129 BL runtime·mstart0(SB) 130 RET // not reached 131 132 /* 133 * go-routine 134 */ 135 136 // void gogo(Gobuf*) 137 // restore state from Gobuf; longjmp 138 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8 139 MOVD buf+0(FP), R5 140 MOVD gobuf_g(R5), R6 141 MOVD 0(R6), R4 // make sure g != nil 142 BR gogo<>(SB) 143 144 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0 145 MOVD R6, g 146 BL runtime·save_g(SB) 147 148 MOVD gobuf_sp(R5), R1 149 MOVD gobuf_lr(R5), R31 150 #ifndef GOOS_aix 151 MOVD 24(R1), R2 // restore R2 152 #endif 153 MOVD R31, LR 154 MOVD gobuf_ret(R5), R3 155 MOVD gobuf_ctxt(R5), R11 156 MOVD R0, gobuf_sp(R5) 157 MOVD R0, gobuf_ret(R5) 158 MOVD R0, gobuf_lr(R5) 159 MOVD R0, gobuf_ctxt(R5) 160 CMP R0, R0 // set condition codes for == test, needed by stack split 161 MOVD gobuf_pc(R5), R12 162 MOVD R12, CTR 163 BR (CTR) 164 165 // void mcall(fn func(*g)) 166 // Switch to m->g0's stack, call fn(g). 167 // Fn must never return. It should gogo(&g->sched) 168 // to keep running g. 169 TEXT runtime·mcall(SB), NOSPLIT|NOFRAME, $0-8 170 // Save caller state in g->sched 171 MOVD R1, (g_sched+gobuf_sp)(g) 172 MOVD LR, R31 173 MOVD R31, (g_sched+gobuf_pc)(g) 174 MOVD R0, (g_sched+gobuf_lr)(g) 175 176 // Switch to m->g0 & its stack, call fn. 177 MOVD g, R3 178 MOVD g_m(g), R8 179 MOVD m_g0(R8), g 180 BL runtime·save_g(SB) 181 CMP g, R3 182 BNE 2(PC) 183 BR runtime·badmcall(SB) 184 MOVD fn+0(FP), R11 // context 185 MOVD 0(R11), R12 // code pointer 186 MOVD R12, CTR 187 MOVD (g_sched+gobuf_sp)(g), R1 // sp = m->g0->sched.sp 188 MOVDU R3, -8(R1) 189 MOVDU R0, -8(R1) 190 MOVDU R0, -8(R1) 191 MOVDU R0, -8(R1) 192 MOVDU R0, -8(R1) 193 BL (CTR) 194 MOVD 24(R1), R2 195 BR runtime·badmcall2(SB) 196 197 // systemstack_switch is a dummy routine that systemstack leaves at the bottom 198 // of the G stack. We need to distinguish the routine that 199 // lives at the bottom of the G stack from the one that lives 200 // at the top of the system stack because the one at the top of 201 // the system stack terminates the stack walk (see topofstack()). 202 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0 203 // We have several undefs here so that 16 bytes past 204 // $runtime·systemstack_switch lies within them whether or not the 205 // instructions that derive r2 from r12 are there. 206 UNDEF 207 UNDEF 208 UNDEF 209 BL (LR) // make sure this function is not leaf 210 RET 211 212 // func systemstack(fn func()) 213 TEXT runtime·systemstack(SB), NOSPLIT, $0-8 214 MOVD fn+0(FP), R3 // R3 = fn 215 MOVD R3, R11 // context 216 MOVD g_m(g), R4 // R4 = m 217 218 MOVD m_gsignal(R4), R5 // R5 = gsignal 219 CMP g, R5 220 BEQ noswitch 221 222 MOVD m_g0(R4), R5 // R5 = g0 223 CMP g, R5 224 BEQ noswitch 225 226 MOVD m_curg(R4), R6 227 CMP g, R6 228 BEQ switch 229 230 // Bad: g is not gsignal, not g0, not curg. What is it? 231 // Hide call from linker nosplit analysis. 232 MOVD $runtime·badsystemstack(SB), R12 233 MOVD R12, CTR 234 BL (CTR) 235 BL runtime·abort(SB) 236 237 switch: 238 // save our state in g->sched. Pretend to 239 // be systemstack_switch if the G stack is scanned. 240 BL gosave_systemstack_switch<>(SB) 241 242 // switch to g0 243 MOVD R5, g 244 BL runtime·save_g(SB) 245 MOVD (g_sched+gobuf_sp)(g), R1 246 247 // call target function 248 MOVD 0(R11), R12 // code pointer 249 MOVD R12, CTR 250 BL (CTR) 251 252 // restore TOC pointer. It seems unlikely that we will use systemstack 253 // to call a function defined in another module, but the results of 254 // doing so would be so confusing that it's worth doing this. 255 MOVD g_m(g), R3 256 MOVD m_curg(R3), g 257 MOVD (g_sched+gobuf_sp)(g), R3 258 #ifndef GOOS_aix 259 MOVD 24(R3), R2 260 #endif 261 // switch back to g 262 MOVD g_m(g), R3 263 MOVD m_curg(R3), g 264 BL runtime·save_g(SB) 265 MOVD (g_sched+gobuf_sp)(g), R1 266 MOVD R0, (g_sched+gobuf_sp)(g) 267 RET 268 269 noswitch: 270 // already on m stack, just call directly 271 // On other arches we do a tail call here, but it appears to be 272 // impossible to tail call a function pointer in shared mode on 273 // ppc64 because the caller is responsible for restoring the TOC. 274 MOVD 0(R11), R12 // code pointer 275 MOVD R12, CTR 276 BL (CTR) 277 #ifndef GOOS_aix 278 MOVD 24(R1), R2 279 #endif 280 RET 281 282 /* 283 * support for morestack 284 */ 285 286 // Called during function prolog when more stack is needed. 287 // Caller has already loaded: 288 // R3: framesize, R4: argsize, R5: LR 289 // 290 // The traceback routines see morestack on a g0 as being 291 // the top of a stack (for example, morestack calling newstack 292 // calling the scheduler calling newm calling gc), so we must 293 // record an argument size. For that purpose, it has no arguments. 294 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0 295 // Cannot grow scheduler stack (m->g0). 296 MOVD g_m(g), R7 297 MOVD m_g0(R7), R8 298 CMP g, R8 299 BNE 3(PC) 300 BL runtime·badmorestackg0(SB) 301 BL runtime·abort(SB) 302 303 // Cannot grow signal stack (m->gsignal). 304 MOVD m_gsignal(R7), R8 305 CMP g, R8 306 BNE 3(PC) 307 BL runtime·badmorestackgsignal(SB) 308 BL runtime·abort(SB) 309 310 // Called from f. 311 // Set g->sched to context in f. 312 MOVD R1, (g_sched+gobuf_sp)(g) 313 MOVD LR, R8 314 MOVD R8, (g_sched+gobuf_pc)(g) 315 MOVD R5, (g_sched+gobuf_lr)(g) 316 MOVD R11, (g_sched+gobuf_ctxt)(g) 317 318 // Called from f. 319 // Set m->morebuf to f's caller. 320 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC 321 MOVD R1, (m_morebuf+gobuf_sp)(R7) // f's caller's SP 322 MOVD g, (m_morebuf+gobuf_g)(R7) 323 324 // Call newstack on m->g0's stack. 325 MOVD m_g0(R7), g 326 BL runtime·save_g(SB) 327 MOVD (g_sched+gobuf_sp)(g), R1 328 MOVDU R0, -(FIXED_FRAME+0)(R1) // create a call frame on g0 329 BL runtime·newstack(SB) 330 331 // Not reached, but make sure the return PC from the call to newstack 332 // is still in this function, and not the beginning of the next. 333 UNDEF 334 335 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0 336 MOVD R0, R11 337 BR runtime·morestack(SB) 338 339 // reflectcall: call a function with the given argument list 340 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs). 341 // we don't have variable-sized frames, so we use a small number 342 // of constant-sized-frame functions to encode a few bits of size in the pc. 343 // Caution: ugly multiline assembly macros in your future! 344 345 #define DISPATCH(NAME,MAXSIZE) \ 346 MOVD $MAXSIZE, R31; \ 347 CMP R3, R31; \ 348 BGT 4(PC); \ 349 MOVD $NAME(SB), R12; \ 350 MOVD R12, CTR; \ 351 BR (CTR) 352 // Note: can't just "BR NAME(SB)" - bad inlining results. 353 354 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48 355 MOVWZ frameSize+32(FP), R3 356 DISPATCH(runtime·call16, 16) 357 DISPATCH(runtime·call32, 32) 358 DISPATCH(runtime·call64, 64) 359 DISPATCH(runtime·call128, 128) 360 DISPATCH(runtime·call256, 256) 361 DISPATCH(runtime·call512, 512) 362 DISPATCH(runtime·call1024, 1024) 363 DISPATCH(runtime·call2048, 2048) 364 DISPATCH(runtime·call4096, 4096) 365 DISPATCH(runtime·call8192, 8192) 366 DISPATCH(runtime·call16384, 16384) 367 DISPATCH(runtime·call32768, 32768) 368 DISPATCH(runtime·call65536, 65536) 369 DISPATCH(runtime·call131072, 131072) 370 DISPATCH(runtime·call262144, 262144) 371 DISPATCH(runtime·call524288, 524288) 372 DISPATCH(runtime·call1048576, 1048576) 373 DISPATCH(runtime·call2097152, 2097152) 374 DISPATCH(runtime·call4194304, 4194304) 375 DISPATCH(runtime·call8388608, 8388608) 376 DISPATCH(runtime·call16777216, 16777216) 377 DISPATCH(runtime·call33554432, 33554432) 378 DISPATCH(runtime·call67108864, 67108864) 379 DISPATCH(runtime·call134217728, 134217728) 380 DISPATCH(runtime·call268435456, 268435456) 381 DISPATCH(runtime·call536870912, 536870912) 382 DISPATCH(runtime·call1073741824, 1073741824) 383 MOVD $runtime·badreflectcall(SB), R12 384 MOVD R12, CTR 385 BR (CTR) 386 387 #define CALLFN(NAME,MAXSIZE) \ 388 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \ 389 NO_LOCAL_POINTERS; \ 390 /* copy arguments to stack */ \ 391 MOVD stackArgs+16(FP), R3; \ 392 MOVWZ stackArgsSize+24(FP), R4; \ 393 MOVD R1, R5; \ 394 CMP R4, $8; \ 395 BLT tailsetup; \ 396 /* copy 8 at a time if possible */ \ 397 ADD $(FIXED_FRAME-8), R5; \ 398 SUB $8, R3; \ 399 top: \ 400 MOVDU 8(R3), R7; \ 401 MOVDU R7, 8(R5); \ 402 SUB $8, R4; \ 403 CMP R4, $8; \ 404 BGE top; \ 405 /* handle remaining bytes */ \ 406 CMP $0, R4; \ 407 BEQ callfn; \ 408 ADD $7, R3; \ 409 ADD $7, R5; \ 410 BR tail; \ 411 tailsetup: \ 412 CMP $0, R4; \ 413 BEQ callfn; \ 414 ADD $(FIXED_FRAME-1), R5; \ 415 SUB $1, R3; \ 416 tail: \ 417 MOVBU 1(R3), R6; \ 418 MOVBU R6, 1(R5); \ 419 SUB $1, R4; \ 420 CMP $0, R4; \ 421 BGT tail; \ 422 callfn: \ 423 /* call function */ \ 424 MOVD f+8(FP), R11; \ 425 #ifdef GOOS_aix \ 426 /* AIX won't trigger a SIGSEGV if R11 = nil */ \ 427 /* So it manually triggers it */ \ 428 CMP R0, R11 \ 429 BNE 2(PC) \ 430 MOVD R0, 0(R0) \ 431 #endif \ 432 MOVD (R11), R12; \ 433 MOVD R12, CTR; \ 434 PCDATA $PCDATA_StackMapIndex, $0; \ 435 BL (CTR); \ 436 #ifndef GOOS_aix \ 437 MOVD 24(R1), R2; \ 438 #endif \ 439 /* copy return values back */ \ 440 MOVD stackArgsType+0(FP), R7; \ 441 MOVD stackArgs+16(FP), R3; \ 442 MOVWZ stackArgsSize+24(FP), R4; \ 443 MOVWZ stackRetOffset+28(FP), R6; \ 444 ADD $FIXED_FRAME, R1, R5; \ 445 ADD R6, R5; \ 446 ADD R6, R3; \ 447 SUB R6, R4; \ 448 BL callRet<>(SB); \ 449 RET 450 451 // callRet copies return values back at the end of call*. This is a 452 // separate function so it can allocate stack space for the arguments 453 // to reflectcallmove. It does not follow the Go ABI; it expects its 454 // arguments in registers. 455 TEXT callRet<>(SB), NOSPLIT, $40-0 456 MOVD R7, FIXED_FRAME+0(R1) 457 MOVD R3, FIXED_FRAME+8(R1) 458 MOVD R5, FIXED_FRAME+16(R1) 459 MOVD R4, FIXED_FRAME+24(R1) 460 MOVD $0, FIXED_FRAME+32(R1) 461 BL runtime·reflectcallmove(SB) 462 RET 463 464 CALLFN(·call16, 16) 465 CALLFN(·call32, 32) 466 CALLFN(·call64, 64) 467 CALLFN(·call128, 128) 468 CALLFN(·call256, 256) 469 CALLFN(·call512, 512) 470 CALLFN(·call1024, 1024) 471 CALLFN(·call2048, 2048) 472 CALLFN(·call4096, 4096) 473 CALLFN(·call8192, 8192) 474 CALLFN(·call16384, 16384) 475 CALLFN(·call32768, 32768) 476 CALLFN(·call65536, 65536) 477 CALLFN(·call131072, 131072) 478 CALLFN(·call262144, 262144) 479 CALLFN(·call524288, 524288) 480 CALLFN(·call1048576, 1048576) 481 CALLFN(·call2097152, 2097152) 482 CALLFN(·call4194304, 4194304) 483 CALLFN(·call8388608, 8388608) 484 CALLFN(·call16777216, 16777216) 485 CALLFN(·call33554432, 33554432) 486 CALLFN(·call67108864, 67108864) 487 CALLFN(·call134217728, 134217728) 488 CALLFN(·call268435456, 268435456) 489 CALLFN(·call536870912, 536870912) 490 CALLFN(·call1073741824, 1073741824) 491 492 TEXT runtime·procyield(SB),NOSPLIT|NOFRAME,$0-4 493 MOVW cycles+0(FP), R7 494 // POWER does not have a pause/yield instruction equivalent. 495 // Instead, we can lower the program priority by setting the 496 // Program Priority Register prior to the wait loop and set it 497 // back to default afterwards. On Linux, the default priority is 498 // medium-low. For details, see page 837 of the ISA 3.0. 499 OR R1, R1, R1 // Set PPR priority to low 500 again: 501 SUB $1, R7 502 CMP $0, R7 503 BNE again 504 OR R6, R6, R6 // Set PPR priority back to medium-low 505 RET 506 507 // void jmpdefer(fv, sp); 508 // called from deferreturn. 509 // 1. grab stored LR for caller 510 // 2. sub 8 bytes to get back to either nop or toc reload before deferreturn 511 // 3. BR to fn 512 // When dynamically linking Go, it is not sufficient to rewind to the BL 513 // deferreturn -- we might be jumping between modules and so we need to reset 514 // the TOC pointer in r2. To do this, codegen inserts MOVD 24(R1), R2 *before* 515 // the BL deferreturn and jmpdefer rewinds to that. 516 TEXT runtime·jmpdefer(SB), NOSPLIT|NOFRAME, $0-16 517 MOVD 0(R1), R31 518 SUB $8, R31 519 MOVD R31, LR 520 521 MOVD fv+0(FP), R11 522 MOVD argp+8(FP), R1 523 SUB $FIXED_FRAME, R1 524 #ifdef GOOS_aix 525 // AIX won't trigger a SIGSEGV if R11 = nil 526 // So it manually triggers it 527 CMP R0, R11 528 BNE 2(PC) 529 MOVD R0, 0(R0) 530 #endif 531 MOVD 0(R11), R12 532 MOVD R12, CTR 533 BR (CTR) 534 535 // Save state of caller into g->sched, 536 // but using fake PC from systemstack_switch. 537 // Must only be called from functions with no locals ($0) 538 // or else unwinding from systemstack_switch is incorrect. 539 // Smashes R31. 540 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 541 MOVD $runtime·systemstack_switch(SB), R31 542 ADD $16, R31 // get past prologue (including r2-setting instructions when they're there) 543 MOVD R31, (g_sched+gobuf_pc)(g) 544 MOVD R1, (g_sched+gobuf_sp)(g) 545 MOVD R0, (g_sched+gobuf_lr)(g) 546 MOVD R0, (g_sched+gobuf_ret)(g) 547 // Assert ctxt is zero. See func save. 548 MOVD (g_sched+gobuf_ctxt)(g), R31 549 CMP R0, R31 550 BEQ 2(PC) 551 BL runtime·abort(SB) 552 RET 553 554 #ifdef GOOS_aix 555 #define asmcgocallSaveOffset cgoCalleeStackSize + 8 556 #else 557 #define asmcgocallSaveOffset cgoCalleeStackSize 558 #endif 559 560 // func asmcgocall(fn, arg unsafe.Pointer) int32 561 // Call fn(arg) on the scheduler stack, 562 // aligned appropriately for the gcc ABI. 563 // See cgocall.go for more details. 564 TEXT ·asmcgocall(SB),NOSPLIT,$0-20 565 MOVD fn+0(FP), R3 566 MOVD arg+8(FP), R4 567 568 MOVD R1, R7 // save original stack pointer 569 MOVD g, R5 570 571 // Figure out if we need to switch to m->g0 stack. 572 // We get called to create new OS threads too, and those 573 // come in on the m->g0 stack already. 574 // Moreover, if it's called inside the signal handler, it must not switch 575 // to g0 as it can be in use by another syscall. 576 MOVD g_m(g), R8 577 MOVD m_gsignal(R8), R6 578 CMP R6, g 579 BEQ g0 580 MOVD m_g0(R8), R6 581 CMP R6, g 582 BEQ g0 583 BL gosave_systemstack_switch<>(SB) 584 MOVD R6, g 585 BL runtime·save_g(SB) 586 MOVD (g_sched+gobuf_sp)(g), R1 587 588 // Now on a scheduling stack (a pthread-created stack). 589 g0: 590 #ifdef GOOS_aix 591 // Create a fake LR to improve backtrace. 592 MOVD $runtime·asmcgocall(SB), R6 593 MOVD R6, 16(R1) 594 // AIX also save one argument on the stack. 595 SUB $8, R1 596 #endif 597 // Save room for two of our pointers, plus the callee 598 // save area that lives on the caller stack. 599 SUB $(asmcgocallSaveOffset+16), R1 600 RLDCR $0, R1, $~15, R1 // 16-byte alignment for gcc ABI 601 MOVD R5, (asmcgocallSaveOffset+8)(R1)// save old g on stack 602 MOVD (g_stack+stack_hi)(R5), R5 603 SUB R7, R5 604 MOVD R5, asmcgocallSaveOffset(R1) // save depth in old g stack (can't just save SP, as stack might be copied during a callback) 605 #ifdef GOOS_aix 606 MOVD R7, 0(R1) // Save frame pointer to allow manual backtrace with gdb 607 #else 608 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?) 609 #endif 610 // This is a "global call", so put the global entry point in r12 611 MOVD R3, R12 612 613 #ifdef GOARCH_ppc64 614 // ppc64 use elf ABI v1. we must get the real entry address from 615 // first slot of the function descriptor before call. 616 // Same for AIX. 617 MOVD 8(R12), R2 618 MOVD (R12), R12 619 #endif 620 MOVD R12, CTR 621 MOVD R4, R3 // arg in r3 622 BL (CTR) 623 // C code can clobber R0, so set it back to 0. F27-F31 are 624 // callee save, so we don't need to recover those. 625 XOR R0, R0 626 // Restore g, stack pointer, toc pointer. 627 // R3 is errno, so don't touch it 628 MOVD (asmcgocallSaveOffset+8)(R1), g 629 MOVD (g_stack+stack_hi)(g), R5 630 MOVD asmcgocallSaveOffset(R1), R6 631 SUB R6, R5 632 #ifndef GOOS_aix 633 MOVD 24(R5), R2 634 #endif 635 MOVD R5, R1 636 BL runtime·save_g(SB) 637 638 MOVW R3, ret+16(FP) 639 RET 640 641 // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) 642 // See cgocall.go for more details. 643 TEXT ·cgocallback(SB),NOSPLIT,$24-24 644 NO_LOCAL_POINTERS 645 646 // Load m and g from thread-local storage. 647 MOVBZ runtime·iscgo(SB), R3 648 CMP R3, $0 649 BEQ nocgo 650 BL runtime·load_g(SB) 651 nocgo: 652 653 // If g is nil, Go did not create the current thread. 654 // Call needm to obtain one for temporary use. 655 // In this case, we're running on the thread stack, so there's 656 // lots of space, but the linker doesn't know. Hide the call from 657 // the linker analysis by using an indirect call. 658 CMP g, $0 659 BEQ needm 660 661 MOVD g_m(g), R8 662 MOVD R8, savedm-8(SP) 663 BR havem 664 665 needm: 666 MOVD g, savedm-8(SP) // g is zero, so is m. 667 MOVD $runtime·needm(SB), R12 668 MOVD R12, CTR 669 BL (CTR) 670 671 // Set m->sched.sp = SP, so that if a panic happens 672 // during the function we are about to execute, it will 673 // have a valid SP to run on the g0 stack. 674 // The next few lines (after the havem label) 675 // will save this SP onto the stack and then write 676 // the same SP back to m->sched.sp. That seems redundant, 677 // but if an unrecovered panic happens, unwindm will 678 // restore the g->sched.sp from the stack location 679 // and then systemstack will try to use it. If we don't set it here, 680 // that restored SP will be uninitialized (typically 0) and 681 // will not be usable. 682 MOVD g_m(g), R8 683 MOVD m_g0(R8), R3 684 MOVD R1, (g_sched+gobuf_sp)(R3) 685 686 havem: 687 // Now there's a valid m, and we're running on its m->g0. 688 // Save current m->g0->sched.sp on stack and then set it to SP. 689 // Save current sp in m->g0->sched.sp in preparation for 690 // switch back to m->curg stack. 691 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP). 692 MOVD m_g0(R8), R3 693 MOVD (g_sched+gobuf_sp)(R3), R4 694 MOVD R4, savedsp-24(SP) // must match frame size 695 MOVD R1, (g_sched+gobuf_sp)(R3) 696 697 // Switch to m->curg stack and call runtime.cgocallbackg. 698 // Because we are taking over the execution of m->curg 699 // but *not* resuming what had been running, we need to 700 // save that information (m->curg->sched) so we can restore it. 701 // We can restore m->curg->sched.sp easily, because calling 702 // runtime.cgocallbackg leaves SP unchanged upon return. 703 // To save m->curg->sched.pc, we push it onto the curg stack and 704 // open a frame the same size as cgocallback's g0 frame. 705 // Once we switch to the curg stack, the pushed PC will appear 706 // to be the return PC of cgocallback, so that the traceback 707 // will seamlessly trace back into the earlier calls. 708 MOVD m_curg(R8), g 709 BL runtime·save_g(SB) 710 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4 711 MOVD (g_sched+gobuf_pc)(g), R5 712 MOVD R5, -(24+FIXED_FRAME)(R4) // "saved LR"; must match frame size 713 // Gather our arguments into registers. 714 MOVD fn+0(FP), R5 715 MOVD frame+8(FP), R6 716 MOVD ctxt+16(FP), R7 717 MOVD $-(24+FIXED_FRAME)(R4), R1 // switch stack; must match frame size 718 MOVD R5, FIXED_FRAME+0(R1) 719 MOVD R6, FIXED_FRAME+8(R1) 720 MOVD R7, FIXED_FRAME+16(R1) 721 BL runtime·cgocallbackg(SB) 722 723 // Restore g->sched (== m->curg->sched) from saved values. 724 MOVD 0(R1), R5 725 MOVD R5, (g_sched+gobuf_pc)(g) 726 MOVD $(24+FIXED_FRAME)(R1), R4 // must match frame size 727 MOVD R4, (g_sched+gobuf_sp)(g) 728 729 // Switch back to m->g0's stack and restore m->g0->sched.sp. 730 // (Unlike m->curg, the g0 goroutine never uses sched.pc, 731 // so we do not have to restore it.) 732 MOVD g_m(g), R8 733 MOVD m_g0(R8), g 734 BL runtime·save_g(SB) 735 MOVD (g_sched+gobuf_sp)(g), R1 736 MOVD savedsp-24(SP), R4 // must match frame size 737 MOVD R4, (g_sched+gobuf_sp)(g) 738 739 // If the m on entry was nil, we called needm above to borrow an m 740 // for the duration of the call. Since the call is over, return it with dropm. 741 MOVD savedm-8(SP), R6 742 CMP R6, $0 743 BNE droppedm 744 MOVD $runtime·dropm(SB), R12 745 MOVD R12, CTR 746 BL (CTR) 747 droppedm: 748 749 // Done! 750 RET 751 752 // void setg(G*); set g. for use by needm. 753 TEXT runtime·setg(SB), NOSPLIT, $0-8 754 MOVD gg+0(FP), g 755 // This only happens if iscgo, so jump straight to save_g 756 BL runtime·save_g(SB) 757 RET 758 759 #ifdef GOARCH_ppc64 760 #ifdef GOOS_aix 761 DATA setg_gcc<>+0(SB)/8, $_setg_gcc<>(SB) 762 DATA setg_gcc<>+8(SB)/8, $TOC(SB) 763 DATA setg_gcc<>+16(SB)/8, $0 764 GLOBL setg_gcc<>(SB), NOPTR, $24 765 #else 766 TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0 767 DWORD $_setg_gcc<>(SB) 768 DWORD $0 769 DWORD $0 770 #endif 771 #endif 772 773 // void setg_gcc(G*); set g in C TLS. 774 // Must obey the gcc calling convention. 775 #ifdef GOARCH_ppc64le 776 TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0 777 #else 778 TEXT _setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0 779 #endif 780 // The standard prologue clobbers R31, which is callee-save in 781 // the C ABI, so we have to use $-8-0 and save LR ourselves. 782 MOVD LR, R4 783 // Also save g and R31, since they're callee-save in C ABI 784 MOVD R31, R5 785 MOVD g, R6 786 787 MOVD R3, g 788 BL runtime·save_g(SB) 789 790 MOVD R6, g 791 MOVD R5, R31 792 MOVD R4, LR 793 RET 794 795 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0 796 MOVW (R0), R0 797 UNDEF 798 799 #define TBR 268 800 801 // int64 runtime·cputicks(void) 802 TEXT runtime·cputicks(SB),NOSPLIT,$0-8 803 MOVD SPR(TBR), R3 804 MOVD R3, ret+0(FP) 805 RET 806 807 // AES hashing not implemented for ppc64 808 TEXT runtime·memhash(SB),NOSPLIT|NOFRAME,$0-32 809 JMP runtime·memhashFallback(SB) 810 TEXT runtime·strhash(SB),NOSPLIT|NOFRAME,$0-24 811 JMP runtime·strhashFallback(SB) 812 TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-24 813 JMP runtime·memhash32Fallback(SB) 814 TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-24 815 JMP runtime·memhash64Fallback(SB) 816 817 TEXT runtime·return0(SB), NOSPLIT, $0 818 MOVW $0, R3 819 RET 820 821 // Called from cgo wrappers, this function returns g->m->curg.stack.hi. 822 // Must obey the gcc calling convention. 823 #ifdef GOOS_aix 824 // On AIX, _cgo_topofstack is defined in runtime/cgo, because it must 825 // be a longcall in order to prevent trampolines from ld. 826 TEXT __cgo_topofstack(SB),NOSPLIT|NOFRAME,$0 827 #else 828 TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0 829 #endif 830 // g (R30) and R31 are callee-save in the C ABI, so save them 831 MOVD g, R4 832 MOVD R31, R5 833 MOVD LR, R6 834 835 BL runtime·load_g(SB) // clobbers g (R30), R31 836 MOVD g_m(g), R3 837 MOVD m_curg(R3), R3 838 MOVD (g_stack+stack_hi)(R3), R3 839 840 MOVD R4, g 841 MOVD R5, R31 842 MOVD R6, LR 843 RET 844 845 // The top-most function running on a goroutine 846 // returns to goexit+PCQuantum. 847 // 848 // When dynamically linking Go, it can be returned to from a function 849 // implemented in a different module and so needs to reload the TOC pointer 850 // from the stack (although this function declares that it does not set up x-a 851 // frame, newproc1 does in fact allocate one for goexit and saves the TOC 852 // pointer in the correct place). 853 // goexit+_PCQuantum is halfway through the usual global entry point prologue 854 // that derives r2 from r12 which is a bit silly, but not harmful. 855 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0 856 MOVD 24(R1), R2 857 BL runtime·goexit1(SB) // does not return 858 // traceback from goexit1 must hit code range of goexit 859 MOVD R0, R0 // NOP 860 861 // prepGoExitFrame saves the current TOC pointer (i.e. the TOC pointer for the 862 // module containing runtime) to the frame that goexit will execute in when 863 // the goroutine exits. It's implemented in assembly mainly because that's the 864 // easiest way to get access to R2. 865 TEXT runtime·prepGoExitFrame(SB),NOSPLIT,$0-8 866 MOVD sp+0(FP), R3 867 MOVD R2, 24(R3) 868 RET 869 870 TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0 871 ADD $-8, R1 872 MOVD R31, 0(R1) 873 MOVD runtime·lastmoduledatap(SB), R4 874 MOVD R3, moduledata_next(R4) 875 MOVD R3, runtime·lastmoduledatap(SB) 876 MOVD 0(R1), R31 877 ADD $8, R1 878 RET 879 880 TEXT ·checkASM(SB),NOSPLIT,$0-1 881 MOVW $1, R3 882 MOVB R3, ret+0(FP) 883 RET 884 885 // gcWriteBarrier performs a heap pointer write and informs the GC. 886 // 887 // gcWriteBarrier does NOT follow the Go ABI. It takes two arguments: 888 // - R20 is the destination of the write 889 // - R21 is the value being written at R20. 890 // It clobbers condition codes. 891 // It does not clobber R0 through R17 (except special registers), 892 // but may clobber any other register, *including* R31. 893 TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$112 894 // The standard prologue clobbers R31. 895 // We use R18 and R19 as scratch registers. 896 MOVD g_m(g), R18 897 MOVD m_p(R18), R18 898 MOVD (p_wbBuf+wbBuf_next)(R18), R19 899 // Increment wbBuf.next position. 900 ADD $16, R19 901 MOVD R19, (p_wbBuf+wbBuf_next)(R18) 902 MOVD (p_wbBuf+wbBuf_end)(R18), R18 903 CMP R18, R19 904 // Record the write. 905 MOVD R21, -16(R19) // Record value 906 MOVD (R20), R18 // TODO: This turns bad writes into bad reads. 907 MOVD R18, -8(R19) // Record *slot 908 // Is the buffer full? (flags set in CMP above) 909 BEQ flush 910 ret: 911 // Do the write. 912 MOVD R21, (R20) 913 RET 914 915 flush: 916 // Save registers R0 through R15 since these were not saved by the caller. 917 // We don't save all registers on ppc64 because it takes too much space. 918 MOVD R20, (FIXED_FRAME+0)(R1) // Also first argument to wbBufFlush 919 MOVD R21, (FIXED_FRAME+8)(R1) // Also second argument to wbBufFlush 920 // R0 is always 0, so no need to spill. 921 // R1 is SP. 922 // R2 is SB. 923 MOVD R3, (FIXED_FRAME+16)(R1) 924 MOVD R4, (FIXED_FRAME+24)(R1) 925 MOVD R5, (FIXED_FRAME+32)(R1) 926 MOVD R6, (FIXED_FRAME+40)(R1) 927 MOVD R7, (FIXED_FRAME+48)(R1) 928 MOVD R8, (FIXED_FRAME+56)(R1) 929 MOVD R9, (FIXED_FRAME+64)(R1) 930 MOVD R10, (FIXED_FRAME+72)(R1) 931 // R11, R12 may be clobbered by external-linker-inserted trampoline 932 // R13 is REGTLS 933 MOVD R14, (FIXED_FRAME+80)(R1) 934 MOVD R15, (FIXED_FRAME+88)(R1) 935 MOVD R16, (FIXED_FRAME+96)(R1) 936 MOVD R17, (FIXED_FRAME+104)(R1) 937 938 // This takes arguments R20 and R21. 939 CALL runtime·wbBufFlush(SB) 940 941 MOVD (FIXED_FRAME+0)(R1), R20 942 MOVD (FIXED_FRAME+8)(R1), R21 943 MOVD (FIXED_FRAME+16)(R1), R3 944 MOVD (FIXED_FRAME+24)(R1), R4 945 MOVD (FIXED_FRAME+32)(R1), R5 946 MOVD (FIXED_FRAME+40)(R1), R6 947 MOVD (FIXED_FRAME+48)(R1), R7 948 MOVD (FIXED_FRAME+56)(R1), R8 949 MOVD (FIXED_FRAME+64)(R1), R9 950 MOVD (FIXED_FRAME+72)(R1), R10 951 MOVD (FIXED_FRAME+80)(R1), R14 952 MOVD (FIXED_FRAME+88)(R1), R15 953 MOVD (FIXED_FRAME+96)(R1), R16 954 MOVD (FIXED_FRAME+104)(R1), R17 955 JMP ret 956 957 // Note: these functions use a special calling convention to save generated code space. 958 // Arguments are passed in registers, but the space for those arguments are allocated 959 // in the caller's stack frame. These stubs write the args into that stack space and 960 // then tail call to the corresponding runtime handler. 961 // The tail call makes these stubs disappear in backtraces. 962 TEXT runtime·panicIndex(SB),NOSPLIT,$0-16 963 MOVD R3, x+0(FP) 964 MOVD R4, y+8(FP) 965 JMP runtime·goPanicIndex(SB) 966 TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16 967 MOVD R3, x+0(FP) 968 MOVD R4, y+8(FP) 969 JMP runtime·goPanicIndexU(SB) 970 TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16 971 MOVD R4, x+0(FP) 972 MOVD R5, y+8(FP) 973 JMP runtime·goPanicSliceAlen(SB) 974 TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16 975 MOVD R4, x+0(FP) 976 MOVD R5, y+8(FP) 977 JMP runtime·goPanicSliceAlenU(SB) 978 TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16 979 MOVD R4, x+0(FP) 980 MOVD R5, y+8(FP) 981 JMP runtime·goPanicSliceAcap(SB) 982 TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16 983 MOVD R4, x+0(FP) 984 MOVD R5, y+8(FP) 985 JMP runtime·goPanicSliceAcapU(SB) 986 TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16 987 MOVD R3, x+0(FP) 988 MOVD R4, y+8(FP) 989 JMP runtime·goPanicSliceB(SB) 990 TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16 991 MOVD R3, x+0(FP) 992 MOVD R4, y+8(FP) 993 JMP runtime·goPanicSliceBU(SB) 994 TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16 995 MOVD R5, x+0(FP) 996 MOVD R6, y+8(FP) 997 JMP runtime·goPanicSlice3Alen(SB) 998 TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16 999 MOVD R5, x+0(FP) 1000 MOVD R6, y+8(FP) 1001 JMP runtime·goPanicSlice3AlenU(SB) 1002 TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16 1003 MOVD R5, x+0(FP) 1004 MOVD R6, y+8(FP) 1005 JMP runtime·goPanicSlice3Acap(SB) 1006 TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16 1007 MOVD R5, x+0(FP) 1008 MOVD R6, y+8(FP) 1009 JMP runtime·goPanicSlice3AcapU(SB) 1010 TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16 1011 MOVD R4, x+0(FP) 1012 MOVD R5, y+8(FP) 1013 JMP runtime·goPanicSlice3B(SB) 1014 TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16 1015 MOVD R4, x+0(FP) 1016 MOVD R5, y+8(FP) 1017 JMP runtime·goPanicSlice3BU(SB) 1018 TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16 1019 MOVD R3, x+0(FP) 1020 MOVD R4, y+8(FP) 1021 JMP runtime·goPanicSlice3C(SB) 1022 TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16 1023 MOVD R3, x+0(FP) 1024 MOVD R4, y+8(FP) 1025 JMP runtime·goPanicSlice3CU(SB) 1026 TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-16 1027 MOVD R5, x+0(FP) 1028 MOVD R6, y+8(FP) 1029 JMP runtime·goPanicSliceConvert(SB)