github.com/4ad/go@v0.0.0-20161219182952-69a12818b605/src/runtime/asm_s390x.s (about) 1 // Copyright 2016 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 #include "go_asm.h" 6 #include "go_tls.h" 7 #include "funcdata.h" 8 #include "textflag.h" 9 10 // Indicate the status of vector facility 11 // -1: init value 12 // 0: vector not installed 13 // 1: vector installed and enabled 14 // 2: vector installed but not enabled 15 16 DATA runtime·vectorfacility+0x00(SB)/4, $-1 17 GLOBL runtime·vectorfacility(SB), NOPTR, $4 18 19 TEXT runtime·checkvectorfacility(SB),NOSPLIT,$32-0 20 MOVD $2, R0 21 MOVD R1, tmp-32(SP) 22 MOVD $x-24(SP), R1 23 XC $24, 0(R1), 0(R1) 24 // STFLE 0(R1) 25 WORD $0xB2B01000 26 MOVBZ z-8(SP), R1 27 AND $0x40, R1 28 BNE vectorinstalled 29 MOVB $0, runtime·vectorfacility(SB) //Vector not installed 30 MOVD tmp-32(SP), R1 31 MOVD $0, R0 32 RET 33 vectorinstalled: 34 // check if the vector instruction has been enabled 35 VLEIB $0, $0xF, V16 36 VLGVB $0, V16, R0 37 CMPBEQ R0, $0xF, vectorenabled 38 MOVB $2, runtime·vectorfacility(SB) //Vector installed but not enabled 39 MOVD tmp-32(SP), R1 40 MOVD $0, R0 41 RET 42 vectorenabled: 43 MOVB $1, runtime·vectorfacility(SB) //Vector installed and enabled 44 MOVD tmp-32(SP), R1 45 MOVD $0, R0 46 RET 47 48 TEXT runtime·rt0_go(SB),NOSPLIT,$0 49 // R2 = argc; R3 = argv; R11 = temp; R13 = g; R15 = stack pointer 50 // C TLS base pointer in AR0:AR1 51 52 // initialize essential registers 53 XOR R0, R0 54 55 SUB $24, R15 56 MOVW R2, 8(R15) // argc 57 MOVD R3, 16(R15) // argv 58 59 // create istack out of the given (operating system) stack. 60 // _cgo_init may update stackguard. 61 MOVD $runtime·g0(SB), g 62 MOVD R15, R11 63 SUB $(64*1024), R11 64 MOVD R11, g_stackguard0(g) 65 MOVD R11, g_stackguard1(g) 66 MOVD R11, (g_stack+stack_lo)(g) 67 MOVD R15, (g_stack+stack_hi)(g) 68 69 // if there is a _cgo_init, call it using the gcc ABI. 70 MOVD _cgo_init(SB), R11 71 CMPBEQ R11, $0, nocgo 72 MOVW AR0, R4 // (AR0 << 32 | AR1) is the TLS base pointer; MOVD is translated to EAR 73 SLD $32, R4, R4 74 MOVW AR1, R4 // arg 2: TLS base pointer 75 MOVD $setg_gcc<>(SB), R3 // arg 1: setg 76 MOVD g, R2 // arg 0: G 77 // C functions expect 160 bytes of space on caller stack frame 78 // and an 8-byte aligned stack pointer 79 MOVD R15, R9 // save current stack (R9 is preserved in the Linux ABI) 80 SUB $160, R15 // reserve 160 bytes 81 MOVD $~7, R6 82 AND R6, R15 // 8-byte align 83 BL R11 // this call clobbers volatile registers according to Linux ABI (R0-R5, R14) 84 MOVD R9, R15 // restore stack 85 XOR R0, R0 // zero R0 86 87 nocgo: 88 // update stackguard after _cgo_init 89 MOVD (g_stack+stack_lo)(g), R2 90 ADD $const__StackGuard, R2 91 MOVD R2, g_stackguard0(g) 92 MOVD R2, g_stackguard1(g) 93 94 // set the per-goroutine and per-mach "registers" 95 MOVD $runtime·m0(SB), R2 96 97 // save m->g0 = g0 98 MOVD g, m_g0(R2) 99 // save m0 to g0->m 100 MOVD R2, g_m(g) 101 102 BL runtime·check(SB) 103 104 // argc/argv are already prepared on stack 105 BL runtime·args(SB) 106 BL runtime·osinit(SB) 107 BL runtime·schedinit(SB) 108 109 // create a new goroutine to start program 110 MOVD $runtime·mainPC(SB), R2 // entry 111 SUB $24, R15 112 MOVD R2, 16(R15) 113 MOVD R0, 8(R15) 114 MOVD R0, 0(R15) 115 BL runtime·newproc(SB) 116 ADD $24, R15 117 118 // start this M 119 BL runtime·mstart(SB) 120 121 MOVD R0, 1(R0) 122 RET 123 124 DATA runtime·mainPC+0(SB)/8,$runtime·main(SB) 125 GLOBL runtime·mainPC(SB),RODATA,$8 126 127 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0 128 MOVD R0, 2(R0) 129 RET 130 131 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0 132 RET 133 134 /* 135 * go-routine 136 */ 137 138 // void gosave(Gobuf*) 139 // save state in Gobuf; setjmp 140 TEXT runtime·gosave(SB), NOSPLIT, $-8-8 141 MOVD buf+0(FP), R3 142 MOVD R15, gobuf_sp(R3) 143 MOVD LR, gobuf_pc(R3) 144 MOVD g, gobuf_g(R3) 145 MOVD $0, gobuf_lr(R3) 146 MOVD $0, gobuf_ret(R3) 147 MOVD $0, gobuf_ctxt(R3) 148 RET 149 150 // void gogo(Gobuf*) 151 // restore state from Gobuf; longjmp 152 TEXT runtime·gogo(SB), NOSPLIT, $-8-8 153 MOVD buf+0(FP), R5 154 MOVD gobuf_g(R5), g // make sure g is not nil 155 BL runtime·save_g(SB) 156 157 MOVD 0(g), R4 158 MOVD gobuf_sp(R5), R15 159 MOVD gobuf_lr(R5), LR 160 MOVD gobuf_ret(R5), R3 161 MOVD gobuf_ctxt(R5), R12 162 MOVD $0, gobuf_sp(R5) 163 MOVD $0, gobuf_ret(R5) 164 MOVD $0, gobuf_lr(R5) 165 MOVD $0, gobuf_ctxt(R5) 166 CMP R0, R0 // set condition codes for == test, needed by stack split 167 MOVD gobuf_pc(R5), R6 168 BR (R6) 169 170 // void mcall(fn func(*g)) 171 // Switch to m->g0's stack, call fn(g). 172 // Fn must never return. It should gogo(&g->sched) 173 // to keep running g. 174 TEXT runtime·mcall(SB), NOSPLIT, $-8-8 175 // Save caller state in g->sched 176 MOVD R15, (g_sched+gobuf_sp)(g) 177 MOVD LR, (g_sched+gobuf_pc)(g) 178 MOVD R0, (g_sched+gobuf_lr)(g) 179 MOVD g, (g_sched+gobuf_g)(g) 180 181 // Switch to m->g0 & its stack, call fn. 182 MOVD g, R3 183 MOVD g_m(g), R8 184 MOVD m_g0(R8), g 185 BL runtime·save_g(SB) 186 CMP g, R3 187 BNE 2(PC) 188 BR runtime·badmcall(SB) 189 MOVD fn+0(FP), R12 // context 190 MOVD 0(R12), R4 // code pointer 191 MOVD (g_sched+gobuf_sp)(g), R15 // sp = m->g0->sched.sp 192 SUB $16, R15 193 MOVD R3, 8(R15) 194 MOVD $0, 0(R15) 195 BL (R4) 196 BR runtime·badmcall2(SB) 197 198 // systemstack_switch is a dummy routine that systemstack leaves at the bottom 199 // of the G stack. We need to distinguish the routine that 200 // lives at the bottom of the G stack from the one that lives 201 // at the top of the system stack because the one at the top of 202 // the system stack terminates the stack walk (see topofstack()). 203 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0 204 UNDEF 205 BL (LR) // make sure this function is not leaf 206 RET 207 208 // func systemstack(fn func()) 209 TEXT runtime·systemstack(SB), NOSPLIT, $0-8 210 MOVD fn+0(FP), R3 // R3 = fn 211 MOVD R3, R12 // context 212 MOVD g_m(g), R4 // R4 = m 213 214 MOVD m_gsignal(R4), R5 // R5 = gsignal 215 CMPBEQ g, R5, noswitch 216 217 MOVD m_g0(R4), R5 // R5 = g0 218 CMPBEQ g, R5, noswitch 219 220 MOVD m_curg(R4), R6 221 CMPBEQ g, R6, switch 222 223 // Bad: g is not gsignal, not g0, not curg. What is it? 224 // Hide call from linker nosplit analysis. 225 MOVD $runtime·badsystemstack(SB), R3 226 BL (R3) 227 228 switch: 229 // save our state in g->sched. Pretend to 230 // be systemstack_switch if the G stack is scanned. 231 MOVD $runtime·systemstack_switch(SB), R6 232 ADD $16, R6 // get past prologue 233 MOVD R6, (g_sched+gobuf_pc)(g) 234 MOVD R15, (g_sched+gobuf_sp)(g) 235 MOVD R0, (g_sched+gobuf_lr)(g) 236 MOVD g, (g_sched+gobuf_g)(g) 237 238 // switch to g0 239 MOVD R5, g 240 BL runtime·save_g(SB) 241 MOVD (g_sched+gobuf_sp)(g), R3 242 // make it look like mstart called systemstack on g0, to stop traceback 243 SUB $8, R3 244 MOVD $runtime·mstart(SB), R4 245 MOVD R4, 0(R3) 246 MOVD R3, R15 247 248 // call target function 249 MOVD 0(R12), R3 // code pointer 250 BL (R3) 251 252 // switch back to g 253 MOVD g_m(g), R3 254 MOVD m_curg(R3), g 255 BL runtime·save_g(SB) 256 MOVD (g_sched+gobuf_sp)(g), R15 257 MOVD $0, (g_sched+gobuf_sp)(g) 258 RET 259 260 noswitch: 261 // already on m stack, just call directly 262 MOVD 0(R12), R3 // code pointer 263 BL (R3) 264 RET 265 266 /* 267 * support for morestack 268 */ 269 270 // Called during function prolog when more stack is needed. 271 // Caller has already loaded: 272 // R3: framesize, R4: argsize, R5: LR 273 // 274 // The traceback routines see morestack on a g0 as being 275 // the top of a stack (for example, morestack calling newstack 276 // calling the scheduler calling newm calling gc), so we must 277 // record an argument size. For that purpose, it has no arguments. 278 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0 279 // Cannot grow scheduler stack (m->g0). 280 MOVD g_m(g), R7 281 MOVD m_g0(R7), R8 282 CMPBNE g, R8, 2(PC) 283 BL runtime·abort(SB) 284 285 // Cannot grow signal stack (m->gsignal). 286 MOVD m_gsignal(R7), R8 287 CMP g, R8 288 BNE 2(PC) 289 BL runtime·abort(SB) 290 291 // Called from f. 292 // Set g->sched to context in f. 293 MOVD R12, (g_sched+gobuf_ctxt)(g) 294 MOVD R15, (g_sched+gobuf_sp)(g) 295 MOVD LR, R8 296 MOVD R8, (g_sched+gobuf_pc)(g) 297 MOVD R5, (g_sched+gobuf_lr)(g) 298 299 // Called from f. 300 // Set m->morebuf to f's caller. 301 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC 302 MOVD R15, (m_morebuf+gobuf_sp)(R7) // f's caller's SP 303 MOVD g, (m_morebuf+gobuf_g)(R7) 304 305 // Call newstack on m->g0's stack. 306 MOVD m_g0(R7), g 307 BL runtime·save_g(SB) 308 MOVD (g_sched+gobuf_sp)(g), R15 309 BL runtime·newstack(SB) 310 311 // Not reached, but make sure the return PC from the call to newstack 312 // is still in this function, and not the beginning of the next. 313 UNDEF 314 315 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0 316 MOVD $0, R12 317 BR runtime·morestack(SB) 318 319 TEXT runtime·stackBarrier(SB),NOSPLIT,$0 320 // We came here via a RET to an overwritten LR. 321 // R3 may be live. Other registers are available. 322 323 // Get the original return PC, g.stkbar[g.stkbarPos].savedLRVal. 324 MOVD (g_stkbar+slice_array)(g), R4 325 MOVD g_stkbarPos(g), R5 326 MOVD $stkbar__size, R6 327 MULLD R5, R6 328 ADD R4, R6 329 MOVD stkbar_savedLRVal(R6), R6 330 // Record that this stack barrier was hit. 331 ADD $1, R5 332 MOVD R5, g_stkbarPos(g) 333 // Jump to the original return PC. 334 BR (R6) 335 336 // reflectcall: call a function with the given argument list 337 // func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32). 338 // we don't have variable-sized frames, so we use a small number 339 // of constant-sized-frame functions to encode a few bits of size in the pc. 340 // Caution: ugly multiline assembly macros in your future! 341 342 #define DISPATCH(NAME,MAXSIZE) \ 343 MOVD $MAXSIZE, R4; \ 344 CMP R3, R4; \ 345 BGT 3(PC); \ 346 MOVD $NAME(SB), R5; \ 347 BR (R5) 348 // Note: can't just "BR NAME(SB)" - bad inlining results. 349 350 TEXT reflect·call(SB), NOSPLIT, $0-0 351 BR ·reflectcall(SB) 352 353 TEXT ·reflectcall(SB), NOSPLIT, $-8-32 354 MOVWZ argsize+24(FP), R3 355 // NOTE(rsc): No call16, because CALLFN needs four words 356 // of argument space to invoke callwritebarrier. 357 DISPATCH(runtime·call32, 32) 358 DISPATCH(runtime·call64, 64) 359 DISPATCH(runtime·call128, 128) 360 DISPATCH(runtime·call256, 256) 361 DISPATCH(runtime·call512, 512) 362 DISPATCH(runtime·call1024, 1024) 363 DISPATCH(runtime·call2048, 2048) 364 DISPATCH(runtime·call4096, 4096) 365 DISPATCH(runtime·call8192, 8192) 366 DISPATCH(runtime·call16384, 16384) 367 DISPATCH(runtime·call32768, 32768) 368 DISPATCH(runtime·call65536, 65536) 369 DISPATCH(runtime·call131072, 131072) 370 DISPATCH(runtime·call262144, 262144) 371 DISPATCH(runtime·call524288, 524288) 372 DISPATCH(runtime·call1048576, 1048576) 373 DISPATCH(runtime·call2097152, 2097152) 374 DISPATCH(runtime·call4194304, 4194304) 375 DISPATCH(runtime·call8388608, 8388608) 376 DISPATCH(runtime·call16777216, 16777216) 377 DISPATCH(runtime·call33554432, 33554432) 378 DISPATCH(runtime·call67108864, 67108864) 379 DISPATCH(runtime·call134217728, 134217728) 380 DISPATCH(runtime·call268435456, 268435456) 381 DISPATCH(runtime·call536870912, 536870912) 382 DISPATCH(runtime·call1073741824, 1073741824) 383 MOVD $runtime·badreflectcall(SB), R5 384 BR (R5) 385 386 #define CALLFN(NAME,MAXSIZE) \ 387 TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \ 388 NO_LOCAL_POINTERS; \ 389 /* copy arguments to stack */ \ 390 MOVD arg+16(FP), R3; \ 391 MOVWZ argsize+24(FP), R4; \ 392 MOVD R15, R5; \ 393 ADD $(8-1), R5; \ 394 SUB $1, R3; \ 395 ADD R5, R4; \ 396 CMP R5, R4; \ 397 BEQ 6(PC); \ 398 ADD $1, R3; \ 399 ADD $1, R5; \ 400 MOVBZ 0(R3), R6; \ 401 MOVBZ R6, 0(R5); \ 402 BR -6(PC); \ 403 /* call function */ \ 404 MOVD f+8(FP), R12; \ 405 MOVD (R12), R8; \ 406 PCDATA $PCDATA_StackMapIndex, $0; \ 407 BL (R8); \ 408 /* copy return values back */ \ 409 MOVD arg+16(FP), R3; \ 410 MOVWZ n+24(FP), R4; \ 411 MOVWZ retoffset+28(FP), R6; \ 412 MOVD R15, R5; \ 413 ADD R6, R5; \ 414 ADD R6, R3; \ 415 SUB R6, R4; \ 416 ADD $(8-1), R5; \ 417 SUB $1, R3; \ 418 ADD R5, R4; \ 419 loop: \ 420 CMP R5, R4; \ 421 BEQ end; \ 422 ADD $1, R5; \ 423 ADD $1, R3; \ 424 MOVBZ 0(R5), R6; \ 425 MOVBZ R6, 0(R3); \ 426 BR loop; \ 427 end: \ 428 /* execute write barrier updates */ \ 429 MOVD argtype+0(FP), R7; \ 430 MOVD arg+16(FP), R3; \ 431 MOVWZ n+24(FP), R4; \ 432 MOVWZ retoffset+28(FP), R6; \ 433 MOVD R7, 8(R15); \ 434 MOVD R3, 16(R15); \ 435 MOVD R4, 24(R15); \ 436 MOVD R6, 32(R15); \ 437 BL runtime·callwritebarrier(SB); \ 438 RET 439 440 CALLFN(·call32, 32) 441 CALLFN(·call64, 64) 442 CALLFN(·call128, 128) 443 CALLFN(·call256, 256) 444 CALLFN(·call512, 512) 445 CALLFN(·call1024, 1024) 446 CALLFN(·call2048, 2048) 447 CALLFN(·call4096, 4096) 448 CALLFN(·call8192, 8192) 449 CALLFN(·call16384, 16384) 450 CALLFN(·call32768, 32768) 451 CALLFN(·call65536, 65536) 452 CALLFN(·call131072, 131072) 453 CALLFN(·call262144, 262144) 454 CALLFN(·call524288, 524288) 455 CALLFN(·call1048576, 1048576) 456 CALLFN(·call2097152, 2097152) 457 CALLFN(·call4194304, 4194304) 458 CALLFN(·call8388608, 8388608) 459 CALLFN(·call16777216, 16777216) 460 CALLFN(·call33554432, 33554432) 461 CALLFN(·call67108864, 67108864) 462 CALLFN(·call134217728, 134217728) 463 CALLFN(·call268435456, 268435456) 464 CALLFN(·call536870912, 536870912) 465 CALLFN(·call1073741824, 1073741824) 466 467 TEXT runtime·procyield(SB),NOSPLIT,$0-0 468 RET 469 470 // void jmpdefer(fv, sp); 471 // called from deferreturn. 472 // 1. grab stored LR for caller 473 // 2. sub 6 bytes to get back to BL deferreturn (size of BRASL instruction) 474 // 3. BR to fn 475 TEXT runtime·jmpdefer(SB),NOSPLIT|NOFRAME,$0-16 476 MOVD 0(R15), R1 477 SUB $6, R1, LR 478 479 MOVD fv+0(FP), R12 480 MOVD argp+8(FP), R15 481 SUB $8, R15 482 MOVD 0(R12), R3 483 BR (R3) 484 485 // Save state of caller into g->sched. Smashes R31. 486 TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0 487 MOVD LR, (g_sched+gobuf_pc)(g) 488 MOVD R15, (g_sched+gobuf_sp)(g) 489 MOVD $0, (g_sched+gobuf_lr)(g) 490 MOVD $0, (g_sched+gobuf_ret)(g) 491 MOVD $0, (g_sched+gobuf_ctxt)(g) 492 RET 493 494 // func asmcgocall(fn, arg unsafe.Pointer) int32 495 // Call fn(arg) on the scheduler stack, 496 // aligned appropriately for the gcc ABI. 497 // See cgocall.go for more details. 498 TEXT ·asmcgocall(SB),NOSPLIT,$0-20 499 // R2 = argc; R3 = argv; R11 = temp; R13 = g; R15 = stack pointer 500 // C TLS base pointer in AR0:AR1 501 MOVD fn+0(FP), R3 502 MOVD arg+8(FP), R4 503 504 MOVD R15, R2 // save original stack pointer 505 MOVD g, R5 506 507 // Figure out if we need to switch to m->g0 stack. 508 // We get called to create new OS threads too, and those 509 // come in on the m->g0 stack already. 510 MOVD g_m(g), R6 511 MOVD m_g0(R6), R6 512 CMPBEQ R6, g, g0 513 BL gosave<>(SB) 514 MOVD R6, g 515 BL runtime·save_g(SB) 516 MOVD (g_sched+gobuf_sp)(g), R15 517 518 // Now on a scheduling stack (a pthread-created stack). 519 g0: 520 // Save room for two of our pointers, plus 160 bytes of callee 521 // save area that lives on the caller stack. 522 SUB $176, R15 523 MOVD $~7, R6 524 AND R6, R15 // 8-byte alignment for gcc ABI 525 MOVD R5, 168(R15) // save old g on stack 526 MOVD (g_stack+stack_hi)(R5), R5 527 SUB R2, R5 528 MOVD R5, 160(R15) // save depth in old g stack (can't just save SP, as stack might be copied during a callback) 529 MOVD R0, 0(R15) // clear back chain pointer (TODO can we give it real back trace information?) 530 MOVD R4, R2 // arg in R2 531 BL R3 // can clobber: R0-R5, R14, F0-F3, F5, F7-F15 532 533 XOR R0, R0 // set R0 back to 0. 534 // Restore g, stack pointer. 535 MOVD 168(R15), g 536 BL runtime·save_g(SB) 537 MOVD (g_stack+stack_hi)(g), R5 538 MOVD 160(R15), R6 539 SUB R6, R5 540 MOVD R5, R15 541 542 MOVW R2, ret+16(FP) 543 RET 544 545 // cgocallback(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt) 546 // Turn the fn into a Go func (by taking its address) and call 547 // cgocallback_gofunc. 548 TEXT runtime·cgocallback(SB),NOSPLIT,$32-32 549 MOVD $fn+0(FP), R3 550 MOVD R3, 8(R15) 551 MOVD frame+8(FP), R3 552 MOVD R3, 16(R15) 553 MOVD framesize+16(FP), R3 554 MOVD R3, 24(R15) 555 MOVD ctxt+24(FP), R3 556 MOVD R3, 32(R15) 557 MOVD $runtime·cgocallback_gofunc(SB), R3 558 BL (R3) 559 RET 560 561 // cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize, uintptr ctxt) 562 // See cgocall.go for more details. 563 TEXT ·cgocallback_gofunc(SB),NOSPLIT,$16-32 564 NO_LOCAL_POINTERS 565 566 // Load m and g from thread-local storage. 567 MOVB runtime·iscgo(SB), R3 568 CMPBEQ R3, $0, nocgo 569 BL runtime·load_g(SB) 570 571 nocgo: 572 // If g is nil, Go did not create the current thread. 573 // Call needm to obtain one for temporary use. 574 // In this case, we're running on the thread stack, so there's 575 // lots of space, but the linker doesn't know. Hide the call from 576 // the linker analysis by using an indirect call. 577 CMPBEQ g, $0, needm 578 579 MOVD g_m(g), R8 580 MOVD R8, savedm-8(SP) 581 BR havem 582 583 needm: 584 MOVD g, savedm-8(SP) // g is zero, so is m. 585 MOVD $runtime·needm(SB), R3 586 BL (R3) 587 588 // Set m->sched.sp = SP, so that if a panic happens 589 // during the function we are about to execute, it will 590 // have a valid SP to run on the g0 stack. 591 // The next few lines (after the havem label) 592 // will save this SP onto the stack and then write 593 // the same SP back to m->sched.sp. That seems redundant, 594 // but if an unrecovered panic happens, unwindm will 595 // restore the g->sched.sp from the stack location 596 // and then systemstack will try to use it. If we don't set it here, 597 // that restored SP will be uninitialized (typically 0) and 598 // will not be usable. 599 MOVD g_m(g), R8 600 MOVD m_g0(R8), R3 601 MOVD R15, (g_sched+gobuf_sp)(R3) 602 603 havem: 604 // Now there's a valid m, and we're running on its m->g0. 605 // Save current m->g0->sched.sp on stack and then set it to SP. 606 // Save current sp in m->g0->sched.sp in preparation for 607 // switch back to m->curg stack. 608 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP). 609 MOVD m_g0(R8), R3 610 MOVD (g_sched+gobuf_sp)(R3), R4 611 MOVD R4, savedsp-16(SP) 612 MOVD R15, (g_sched+gobuf_sp)(R3) 613 614 // Switch to m->curg stack and call runtime.cgocallbackg. 615 // Because we are taking over the execution of m->curg 616 // but *not* resuming what had been running, we need to 617 // save that information (m->curg->sched) so we can restore it. 618 // We can restore m->curg->sched.sp easily, because calling 619 // runtime.cgocallbackg leaves SP unchanged upon return. 620 // To save m->curg->sched.pc, we push it onto the stack. 621 // This has the added benefit that it looks to the traceback 622 // routine like cgocallbackg is going to return to that 623 // PC (because the frame we allocate below has the same 624 // size as cgocallback_gofunc's frame declared above) 625 // so that the traceback will seamlessly trace back into 626 // the earlier calls. 627 // 628 // In the new goroutine, -8(SP) is unused (where SP refers to 629 // m->curg's SP while we're setting it up, before we've adjusted it). 630 MOVD m_curg(R8), g 631 BL runtime·save_g(SB) 632 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4 633 MOVD (g_sched+gobuf_pc)(g), R5 634 MOVD R5, -24(R4) 635 MOVD ctxt+24(FP), R5 636 MOVD R5, -16(R4) 637 MOVD $-24(R4), R15 638 BL runtime·cgocallbackg(SB) 639 640 // Restore g->sched (== m->curg->sched) from saved values. 641 MOVD 0(R15), R5 642 MOVD R5, (g_sched+gobuf_pc)(g) 643 MOVD $24(R15), R4 644 MOVD R4, (g_sched+gobuf_sp)(g) 645 646 // Switch back to m->g0's stack and restore m->g0->sched.sp. 647 // (Unlike m->curg, the g0 goroutine never uses sched.pc, 648 // so we do not have to restore it.) 649 MOVD g_m(g), R8 650 MOVD m_g0(R8), g 651 BL runtime·save_g(SB) 652 MOVD (g_sched+gobuf_sp)(g), R15 653 MOVD savedsp-16(SP), R4 654 MOVD R4, (g_sched+gobuf_sp)(g) 655 656 // If the m on entry was nil, we called needm above to borrow an m 657 // for the duration of the call. Since the call is over, return it with dropm. 658 MOVD savedm-8(SP), R6 659 CMPBNE R6, $0, droppedm 660 MOVD $runtime·dropm(SB), R3 661 BL (R3) 662 droppedm: 663 664 // Done! 665 RET 666 667 // void setg(G*); set g. for use by needm. 668 TEXT runtime·setg(SB), NOSPLIT, $0-8 669 MOVD gg+0(FP), g 670 // This only happens if iscgo, so jump straight to save_g 671 BL runtime·save_g(SB) 672 RET 673 674 // void setg_gcc(G*); set g in C TLS. 675 // Must obey the gcc calling convention. 676 TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0 677 // The standard prologue clobbers LR (R14), which is callee-save in 678 // the C ABI, so we have to use NOFRAME and save LR ourselves. 679 MOVD LR, R1 680 // Also save g, R10, and R11 since they're callee-save in C ABI 681 MOVD R10, R3 682 MOVD g, R4 683 MOVD R11, R5 684 685 MOVD R2, g 686 BL runtime·save_g(SB) 687 688 MOVD R5, R11 689 MOVD R4, g 690 MOVD R3, R10 691 MOVD R1, LR 692 RET 693 694 TEXT runtime·getcallerpc(SB),NOSPLIT,$8-16 695 MOVD 16(R15), R3 // LR saved by caller 696 MOVD runtime·stackBarrierPC(SB), R4 697 CMPBNE R3, R4, nobar 698 // Get original return PC. 699 BL runtime·nextBarrierPC(SB) 700 MOVD 8(R15), R3 701 nobar: 702 MOVD R3, ret+8(FP) 703 RET 704 705 TEXT runtime·setcallerpc(SB),NOSPLIT,$8-16 706 MOVD pc+8(FP), R3 707 MOVD 16(R15), R4 708 MOVD runtime·stackBarrierPC(SB), R5 709 CMPBEQ R4, R5, setbar 710 MOVD R3, 16(R15) // set LR in caller 711 RET 712 setbar: 713 // Set the stack barrier return PC. 714 MOVD R3, 8(R15) 715 BL runtime·setNextBarrierPC(SB) 716 RET 717 718 TEXT runtime·getcallersp(SB),NOSPLIT,$0-16 719 MOVD argp+0(FP), R3 720 SUB $8, R3 721 MOVD R3, ret+8(FP) 722 RET 723 724 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0 725 MOVW (R0), R0 726 UNDEF 727 728 // int64 runtime·cputicks(void) 729 TEXT runtime·cputicks(SB),NOSPLIT,$0-8 730 // The TOD clock on s390 counts from the year 1900 in ~250ps intervals. 731 // This means that since about 1972 the msb has been set, making the 732 // result of a call to STORE CLOCK (stck) a negative number. 733 // We clear the msb to make it positive. 734 STCK ret+0(FP) // serialises before and after call 735 MOVD ret+0(FP), R3 // R3 will wrap to 0 in the year 2043 736 SLD $1, R3 737 SRD $1, R3 738 MOVD R3, ret+0(FP) 739 RET 740 741 // memhash_varlen(p unsafe.Pointer, h seed) uintptr 742 // redirects to memhash(p, h, size) using the size 743 // stored in the closure. 744 TEXT runtime·memhash_varlen(SB),NOSPLIT,$40-24 745 GO_ARGS 746 NO_LOCAL_POINTERS 747 MOVD p+0(FP), R3 748 MOVD h+8(FP), R4 749 MOVD 8(R12), R5 750 MOVD R3, 8(R15) 751 MOVD R4, 16(R15) 752 MOVD R5, 24(R15) 753 BL runtime·memhash(SB) 754 MOVD 32(R15), R3 755 MOVD R3, ret+16(FP) 756 RET 757 758 // AES hashing not implemented for s390x 759 TEXT runtime·aeshash(SB),NOSPLIT|NOFRAME,$0-0 760 MOVW (R0), R15 761 TEXT runtime·aeshash32(SB),NOSPLIT|NOFRAME,$0-0 762 MOVW (R0), R15 763 TEXT runtime·aeshash64(SB),NOSPLIT|NOFRAME,$0-0 764 MOVW (R0), R15 765 TEXT runtime·aeshashstr(SB),NOSPLIT|NOFRAME,$0-0 766 MOVW (R0), R15 767 768 // memequal(p, q unsafe.Pointer, size uintptr) bool 769 TEXT runtime·memequal(SB),NOSPLIT|NOFRAME,$0-25 770 MOVD p+0(FP), R3 771 MOVD q+8(FP), R5 772 MOVD size+16(FP), R6 773 LA ret+24(FP), R7 774 BR runtime·memeqbody(SB) 775 776 // memequal_varlen(a, b unsafe.Pointer) bool 777 TEXT runtime·memequal_varlen(SB),NOSPLIT|NOFRAME,$0-17 778 MOVD a+0(FP), R3 779 MOVD b+8(FP), R5 780 MOVD 8(R12), R6 // compiler stores size at offset 8 in the closure 781 LA ret+16(FP), R7 782 BR runtime·memeqbody(SB) 783 784 // eqstring tests whether two strings are equal. 785 // The compiler guarantees that strings passed 786 // to eqstring have equal length. 787 // See runtime_test.go:eqstring_generic for 788 // equivalent Go code. 789 TEXT runtime·eqstring(SB),NOSPLIT|NOFRAME,$0-33 790 MOVD s1str+0(FP), R3 791 MOVD s1len+8(FP), R6 792 MOVD s2str+16(FP), R5 793 LA ret+32(FP), R7 794 BR runtime·memeqbody(SB) 795 796 TEXT bytes·Equal(SB),NOSPLIT|NOFRAME,$0-49 797 MOVD a_len+8(FP), R2 798 MOVD b_len+32(FP), R6 799 MOVD a+0(FP), R3 800 MOVD b+24(FP), R5 801 LA ret+48(FP), R7 802 CMPBNE R2, R6, notequal 803 BR runtime·memeqbody(SB) 804 notequal: 805 MOVB $0, ret+48(FP) 806 RET 807 808 // input: 809 // R3 = a 810 // R5 = b 811 // R6 = len 812 // R7 = address of output byte (stores 0 or 1 here) 813 // a and b have the same length 814 TEXT runtime·memeqbody(SB),NOSPLIT|NOFRAME,$0-0 815 CMPBEQ R3, R5, equal 816 loop: 817 CMPBEQ R6, $0, equal 818 CMPBLT R6, $32, tiny 819 CMP R6, $256 820 BLT tail 821 CLC $256, 0(R3), 0(R5) 822 BNE notequal 823 SUB $256, R6 824 LA 256(R3), R3 825 LA 256(R5), R5 826 BR loop 827 tail: 828 SUB $1, R6, R8 829 EXRL $runtime·memeqbodyclc(SB), R8 830 BEQ equal 831 notequal: 832 MOVB $0, 0(R7) 833 RET 834 equal: 835 MOVB $1, 0(R7) 836 RET 837 tiny: 838 MOVD $0, R2 839 CMPBLT R6, $16, lt16 840 MOVD 0(R3), R8 841 MOVD 0(R5), R9 842 CMPBNE R8, R9, notequal 843 MOVD 8(R3), R8 844 MOVD 8(R5), R9 845 CMPBNE R8, R9, notequal 846 LA 16(R2), R2 847 SUB $16, R6 848 lt16: 849 CMPBLT R6, $8, lt8 850 MOVD 0(R3)(R2*1), R8 851 MOVD 0(R5)(R2*1), R9 852 CMPBNE R8, R9, notequal 853 LA 8(R2), R2 854 SUB $8, R6 855 lt8: 856 CMPBLT R6, $4, lt4 857 MOVWZ 0(R3)(R2*1), R8 858 MOVWZ 0(R5)(R2*1), R9 859 CMPBNE R8, R9, notequal 860 LA 4(R2), R2 861 SUB $4, R6 862 lt4: 863 #define CHECK(n) \ 864 CMPBEQ R6, $n, equal \ 865 MOVB n(R3)(R2*1), R8 \ 866 MOVB n(R5)(R2*1), R9 \ 867 CMPBNE R8, R9, notequal 868 CHECK(0) 869 CHECK(1) 870 CHECK(2) 871 CHECK(3) 872 BR equal 873 874 TEXT runtime·memeqbodyclc(SB),NOSPLIT|NOFRAME,$0-0 875 CLC $1, 0(R3), 0(R5) 876 RET 877 878 TEXT runtime·fastrand1(SB), NOSPLIT, $0-4 879 MOVD g_m(g), R4 880 MOVWZ m_fastrand(R4), R3 881 ADD R3, R3 882 CMPW R3, $0 883 BGE 2(PC) 884 XOR $0x88888eef, R3 885 MOVW R3, m_fastrand(R4) 886 MOVW R3, ret+0(FP) 887 RET 888 889 TEXT bytes·IndexByte(SB),NOSPLIT,$0-40 890 MOVD s+0(FP), R3 // s => R3 891 MOVD s_len+8(FP), R4 // s_len => R4 892 MOVBZ c+24(FP), R5 // c => R5 893 MOVD $ret+32(FP), R2 // &ret => R9 894 BR runtime·indexbytebody(SB) 895 896 TEXT strings·IndexByte(SB),NOSPLIT,$0-32 897 MOVD s+0(FP), R3 // s => R3 898 MOVD s_len+8(FP), R4 // s_len => R4 899 MOVBZ c+16(FP), R5 // c => R5 900 MOVD $ret+24(FP), R2 // &ret => R9 901 BR runtime·indexbytebody(SB) 902 903 // input: 904 // R3: s 905 // R4: s_len 906 // R5: c -- byte sought 907 // R2: &ret -- address to put index into 908 TEXT runtime·indexbytebody(SB),NOSPLIT,$0 909 CMPBEQ R4, $0, notfound 910 MOVD R3, R6 // store base for later 911 ADD R3, R4, R8 // the address after the end of the string 912 //if the length is small, use loop; otherwise, use vector or srst search 913 CMPBGE R4, $16, large 914 915 residual: 916 CMPBEQ R3, R8, notfound 917 MOVBZ 0(R3), R7 918 LA 1(R3), R3 919 CMPBNE R7, R5, residual 920 921 found: 922 SUB R6, R3 923 SUB $1, R3 924 MOVD R3, 0(R2) 925 RET 926 927 notfound: 928 MOVD $-1, 0(R2) 929 RET 930 931 large: 932 MOVB runtime·vectorfacility(SB), R1 933 CMPBEQ R1, $-1, checkvector // vectorfacility = -1, vector not checked yet 934 vectorchecked: 935 CMPBEQ R1, $1, vectorimpl // vectorfacility = 1, vector supported 936 937 srstimpl: // vectorfacility != 1, not support or enable vector 938 MOVBZ R5, R0 // c needs to be in R0, leave until last minute as currently R0 is expected to be 0 939 srstloop: 940 WORD $0xB25E0083 // srst %r8, %r3 (search the range [R3, R8)) 941 BVS srstloop // interrupted - continue 942 BGT notfoundr0 943 foundr0: 944 XOR R0, R0 // reset R0 945 SUB R6, R8 // remove base 946 MOVD R8, 0(R2) 947 RET 948 notfoundr0: 949 XOR R0, R0 // reset R0 950 MOVD $-1, 0(R2) 951 RET 952 953 vectorimpl: 954 //if the address is not 16byte aligned, use loop for the header 955 AND $15, R3, R8 956 CMPBGT R8, $0, notaligned 957 958 aligned: 959 ADD R6, R4, R8 960 AND $-16, R8, R7 961 // replicate c across V17 962 VLVGB $0, R5, V19 963 VREPB $0, V19, V17 964 965 vectorloop: 966 CMPBGE R3, R7, residual 967 VL 0(R3), V16 // load string to be searched into V16 968 ADD $16, R3 969 VFEEBS V16, V17, V18 // search V17 in V16 and set conditional code accordingly 970 BVS vectorloop 971 972 // when vector search found c in the string 973 VLGVB $7, V18, R7 // load 7th element of V18 containing index into R7 974 SUB $16, R3 975 SUB R6, R3 976 ADD R3, R7 977 MOVD R7, 0(R2) 978 RET 979 980 notaligned: 981 AND $-16, R3, R8 982 ADD $16, R8 983 notalignedloop: 984 CMPBEQ R3, R8, aligned 985 MOVBZ 0(R3), R7 986 LA 1(R3), R3 987 CMPBNE R7, R5, notalignedloop 988 BR found 989 990 checkvector: 991 CALL runtime·checkvectorfacility(SB) 992 MOVB runtime·vectorfacility(SB), R1 993 BR vectorchecked 994 995 TEXT runtime·return0(SB), NOSPLIT, $0 996 MOVW $0, R3 997 RET 998 999 // Called from cgo wrappers, this function returns g->m->curg.stack.hi. 1000 // Must obey the gcc calling convention. 1001 TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0 1002 // g (R13), R10, R11 and LR (R14) are callee-save in the C ABI, so save them 1003 MOVD g, R1 1004 MOVD R10, R3 1005 MOVD LR, R4 1006 MOVD R11, R5 1007 1008 BL runtime·load_g(SB) // clobbers g (R13), R10, R11 1009 MOVD g_m(g), R2 1010 MOVD m_curg(R2), R2 1011 MOVD (g_stack+stack_hi)(R2), R2 1012 1013 MOVD R1, g 1014 MOVD R3, R10 1015 MOVD R4, LR 1016 MOVD R5, R11 1017 RET 1018 1019 // The top-most function running on a goroutine 1020 // returns to goexit+PCQuantum. 1021 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME,$0-0 1022 BYTE $0x07; BYTE $0x00; // 2-byte nop 1023 BL runtime·goexit1(SB) // does not return 1024 // traceback from goexit1 must hit code range of goexit 1025 BYTE $0x07; BYTE $0x00; // 2-byte nop 1026 1027 TEXT runtime·prefetcht0(SB),NOSPLIT,$0-8 1028 RET 1029 1030 TEXT runtime·prefetcht1(SB),NOSPLIT,$0-8 1031 RET 1032 1033 TEXT runtime·prefetcht2(SB),NOSPLIT,$0-8 1034 RET 1035 1036 TEXT runtime·prefetchnta(SB),NOSPLIT,$0-8 1037 RET 1038 1039 TEXT runtime·sigreturn(SB),NOSPLIT,$0-8 1040 RET 1041 1042 TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0 1043 SYNC 1044 RET 1045 1046 TEXT runtime·cmpstring(SB),NOSPLIT|NOFRAME,$0-40 1047 MOVD s1_base+0(FP), R3 1048 MOVD s1_len+8(FP), R4 1049 MOVD s2_base+16(FP), R5 1050 MOVD s2_len+24(FP), R6 1051 LA ret+32(FP), R7 1052 BR runtime·cmpbody(SB) 1053 1054 TEXT bytes·Compare(SB),NOSPLIT|NOFRAME,$0-56 1055 MOVD s1+0(FP), R3 1056 MOVD s1+8(FP), R4 1057 MOVD s2+24(FP), R5 1058 MOVD s2+32(FP), R6 1059 LA res+48(FP), R7 1060 BR runtime·cmpbody(SB) 1061 1062 // input: 1063 // R3 = a 1064 // R4 = alen 1065 // R5 = b 1066 // R6 = blen 1067 // R7 = address of output word (stores -1/0/1 here) 1068 TEXT runtime·cmpbody(SB),NOSPLIT|NOFRAME,$0-0 1069 CMPBEQ R3, R5, cmplengths 1070 MOVD R4, R8 1071 CMPBLE R4, R6, amin 1072 MOVD R6, R8 1073 amin: 1074 CMPBEQ R8, $0, cmplengths 1075 CMP R8, $256 1076 BLE tail 1077 loop: 1078 CLC $256, 0(R3), 0(R5) 1079 BGT gt 1080 BLT lt 1081 SUB $256, R8 1082 CMP R8, $256 1083 BGT loop 1084 tail: 1085 SUB $1, R8 1086 EXRL $runtime·cmpbodyclc(SB), R8 1087 BGT gt 1088 BLT lt 1089 cmplengths: 1090 CMP R4, R6 1091 BEQ eq 1092 BLT lt 1093 gt: 1094 MOVD $1, 0(R7) 1095 RET 1096 lt: 1097 MOVD $-1, 0(R7) 1098 RET 1099 eq: 1100 MOVD $0, 0(R7) 1101 RET 1102 1103 TEXT runtime·cmpbodyclc(SB),NOSPLIT|NOFRAME,$0-0 1104 CLC $1, 0(R3), 0(R5) 1105 RET 1106 1107 // This is called from .init_array and follows the platform, not Go, ABI. 1108 // We are overly conservative. We could only save the registers we use. 1109 // However, since this function is only called once per loaded module 1110 // performance is unimportant. 1111 TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0 1112 // Save R6-R15, F0, F2, F4 and F6 in the 1113 // register save area of the calling function 1114 STMG R6, R15, 48(R15) 1115 FMOVD F0, 128(R15) 1116 FMOVD F2, 136(R15) 1117 FMOVD F4, 144(R15) 1118 FMOVD F6, 152(R15) 1119 1120 // append the argument (passed in R2, as per the ELF ABI) to the 1121 // moduledata linked list. 1122 MOVD runtime·lastmoduledatap(SB), R1 1123 MOVD R2, moduledata_next(R1) 1124 MOVD R2, runtime·lastmoduledatap(SB) 1125 1126 // Restore R6-R15, F0, F2, F4 and F6 1127 LMG 48(R15), R6, R15 1128 FMOVD F0, 128(R15) 1129 FMOVD F2, 136(R15) 1130 FMOVD F4, 144(R15) 1131 FMOVD F6, 152(R15) 1132 RET 1133 1134 TEXT ·checkASM(SB),NOSPLIT,$0-1 1135 MOVB $1, ret+0(FP) 1136 RET