github.com/4ad/go@v0.0.0-20161219182952-69a12818b605/src/runtime/asm_arm64.s (about) 1 // Copyright 2015 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 #include "go_asm.h" 6 #include "go_tls.h" 7 #include "tls_arm64.h" 8 #include "funcdata.h" 9 #include "textflag.h" 10 11 TEXT runtime·rt0_go(SB),NOSPLIT,$0 12 // SP = stack; R0 = argc; R1 = argv 13 14 // initialize essential registers 15 BL runtime·reginit(SB) 16 17 SUB $32, RSP 18 MOVW R0, 8(RSP) // argc 19 MOVD R1, 16(RSP) // argv 20 21 // create istack out of the given (operating system) stack. 22 // _cgo_init may update stackguard. 23 MOVD $runtime·g0(SB), g 24 MOVD RSP, R7 25 MOVD $(-64*1024)(R7), R0 26 MOVD R0, g_stackguard0(g) 27 MOVD R0, g_stackguard1(g) 28 MOVD R0, (g_stack+stack_lo)(g) 29 MOVD R7, (g_stack+stack_hi)(g) 30 31 // if there is a _cgo_init, call it using the gcc ABI. 32 MOVD _cgo_init(SB), R12 33 CMP $0, R12 34 BEQ nocgo 35 36 MRS_TPIDR_R0 // load TLS base pointer 37 MOVD R0, R3 // arg 3: TLS base pointer 38 #ifdef TLSG_IS_VARIABLE 39 MOVD $runtime·tls_g(SB), R2 // arg 2: &tls_g 40 #else 41 MOVD $0, R2 // arg 2: not used when using platform's TLS 42 #endif 43 MOVD $setg_gcc<>(SB), R1 // arg 1: setg 44 MOVD g, R0 // arg 0: G 45 BL (R12) 46 MOVD _cgo_init(SB), R12 47 CMP $0, R12 48 BEQ nocgo 49 50 nocgo: 51 // update stackguard after _cgo_init 52 MOVD (g_stack+stack_lo)(g), R0 53 ADD $const__StackGuard, R0 54 MOVD R0, g_stackguard0(g) 55 MOVD R0, g_stackguard1(g) 56 57 // set the per-goroutine and per-mach "registers" 58 MOVD $runtime·m0(SB), R0 59 60 // save m->g0 = g0 61 MOVD g, m_g0(R0) 62 // save m0 to g0->m 63 MOVD R0, g_m(g) 64 65 BL runtime·check(SB) 66 67 MOVW 8(RSP), R0 // copy argc 68 MOVW R0, -8(RSP) 69 MOVD 16(RSP), R0 // copy argv 70 MOVD R0, 0(RSP) 71 BL runtime·args(SB) 72 BL runtime·osinit(SB) 73 BL runtime·schedinit(SB) 74 75 // create a new goroutine to start program 76 MOVD $runtime·mainPC(SB), R0 // entry 77 MOVD RSP, R7 78 MOVD.W $0, -8(R7) 79 MOVD.W R0, -8(R7) 80 MOVD.W $0, -8(R7) 81 MOVD.W $0, -8(R7) 82 MOVD R7, RSP 83 BL runtime·newproc(SB) 84 ADD $32, RSP 85 86 // start this M 87 BL runtime·mstart(SB) 88 89 MOVD $0, R0 90 MOVD R0, (R0) // boom 91 UNDEF 92 93 DATA runtime·mainPC+0(SB)/8,$runtime·main(SB) 94 GLOBL runtime·mainPC(SB),RODATA,$8 95 96 TEXT runtime·breakpoint(SB),NOSPLIT,$-8-0 97 BRK 98 RET 99 100 TEXT runtime·asminit(SB),NOSPLIT,$-8-0 101 RET 102 103 TEXT runtime·reginit(SB),NOSPLIT,$-8-0 104 // initialize essential FP registers 105 FMOVD $4503601774854144.0, F27 106 FMOVD $0.5, F29 107 FSUBD F29, F29, F28 108 FADDD F29, F29, F30 109 FADDD F30, F30, F31 110 RET 111 112 /* 113 * go-routine 114 */ 115 116 // void gosave(Gobuf*) 117 // save state in Gobuf; setjmp 118 TEXT runtime·gosave(SB), NOSPLIT, $-8-8 119 MOVD buf+0(FP), R3 120 MOVD RSP, R0 121 MOVD R0, gobuf_sp(R3) 122 MOVD LR, gobuf_pc(R3) 123 MOVD g, gobuf_g(R3) 124 MOVD ZR, gobuf_lr(R3) 125 MOVD ZR, gobuf_ret(R3) 126 MOVD ZR, gobuf_ctxt(R3) 127 RET 128 129 // void gogo(Gobuf*) 130 // restore state from Gobuf; longjmp 131 TEXT runtime·gogo(SB), NOSPLIT, $-8-8 132 MOVD buf+0(FP), R5 133 MOVD gobuf_g(R5), g 134 BL runtime·save_g(SB) 135 136 MOVD 0(g), R4 // make sure g is not nil 137 MOVD gobuf_sp(R5), R0 138 MOVD R0, RSP 139 MOVD gobuf_lr(R5), LR 140 MOVD gobuf_ret(R5), R0 141 MOVD gobuf_ctxt(R5), R26 142 MOVD $0, gobuf_sp(R5) 143 MOVD $0, gobuf_ret(R5) 144 MOVD $0, gobuf_lr(R5) 145 MOVD $0, gobuf_ctxt(R5) 146 CMP ZR, ZR // set condition codes for == test, needed by stack split 147 MOVD gobuf_pc(R5), R6 148 B (R6) 149 150 // void mcall(fn func(*g)) 151 // Switch to m->g0's stack, call fn(g). 152 // Fn must never return. It should gogo(&g->sched) 153 // to keep running g. 154 TEXT runtime·mcall(SB), NOSPLIT, $-8-8 155 // Save caller state in g->sched 156 MOVD RSP, R0 157 MOVD R0, (g_sched+gobuf_sp)(g) 158 MOVD LR, (g_sched+gobuf_pc)(g) 159 MOVD $0, (g_sched+gobuf_lr)(g) 160 MOVD g, (g_sched+gobuf_g)(g) 161 162 // Switch to m->g0 & its stack, call fn. 163 MOVD g, R3 164 MOVD g_m(g), R8 165 MOVD m_g0(R8), g 166 BL runtime·save_g(SB) 167 CMP g, R3 168 BNE 2(PC) 169 B runtime·badmcall(SB) 170 MOVD fn+0(FP), R26 // context 171 MOVD 0(R26), R4 // code pointer 172 MOVD (g_sched+gobuf_sp)(g), R0 173 MOVD R0, RSP // sp = m->g0->sched.sp 174 MOVD R3, -8(RSP) 175 MOVD $0, -16(RSP) 176 SUB $16, RSP 177 BL (R4) 178 B runtime·badmcall2(SB) 179 180 // systemstack_switch is a dummy routine that systemstack leaves at the bottom 181 // of the G stack. We need to distinguish the routine that 182 // lives at the bottom of the G stack from the one that lives 183 // at the top of the system stack because the one at the top of 184 // the system stack terminates the stack walk (see topofstack()). 185 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0 186 UNDEF 187 BL (LR) // make sure this function is not leaf 188 RET 189 190 // func systemstack(fn func()) 191 TEXT runtime·systemstack(SB), NOSPLIT, $0-8 192 MOVD fn+0(FP), R3 // R3 = fn 193 MOVD R3, R26 // context 194 MOVD g_m(g), R4 // R4 = m 195 196 MOVD m_gsignal(R4), R5 // R5 = gsignal 197 CMP g, R5 198 BEQ noswitch 199 200 MOVD m_g0(R4), R5 // R5 = g0 201 CMP g, R5 202 BEQ noswitch 203 204 MOVD m_curg(R4), R6 205 CMP g, R6 206 BEQ switch 207 208 // Bad: g is not gsignal, not g0, not curg. What is it? 209 // Hide call from linker nosplit analysis. 210 MOVD $runtime·badsystemstack(SB), R3 211 BL (R3) 212 213 switch: 214 // save our state in g->sched. Pretend to 215 // be systemstack_switch if the G stack is scanned. 216 MOVD $runtime·systemstack_switch(SB), R6 217 ADD $8, R6 // get past prologue 218 MOVD R6, (g_sched+gobuf_pc)(g) 219 MOVD RSP, R0 220 MOVD R0, (g_sched+gobuf_sp)(g) 221 MOVD $0, (g_sched+gobuf_lr)(g) 222 MOVD g, (g_sched+gobuf_g)(g) 223 224 // switch to g0 225 MOVD R5, g 226 BL runtime·save_g(SB) 227 MOVD (g_sched+gobuf_sp)(g), R3 228 // make it look like mstart called systemstack on g0, to stop traceback 229 SUB $16, R3 230 AND $~15, R3 231 MOVD $runtime·mstart(SB), R4 232 MOVD R4, 0(R3) 233 MOVD R3, RSP 234 235 // call target function 236 MOVD 0(R26), R3 // code pointer 237 BL (R3) 238 239 // switch back to g 240 MOVD g_m(g), R3 241 MOVD m_curg(R3), g 242 BL runtime·save_g(SB) 243 MOVD (g_sched+gobuf_sp)(g), R0 244 MOVD R0, RSP 245 MOVD $0, (g_sched+gobuf_sp)(g) 246 RET 247 248 noswitch: 249 // already on m stack, just call directly 250 MOVD 0(R26), R3 // code pointer 251 BL (R3) 252 RET 253 254 /* 255 * support for morestack 256 */ 257 258 // Called during function prolog when more stack is needed. 259 // Caller has already loaded: 260 // R3 prolog's LR (R30) 261 // 262 // The traceback routines see morestack on a g0 as being 263 // the top of a stack (for example, morestack calling newstack 264 // calling the scheduler calling newm calling gc), so we must 265 // record an argument size. For that purpose, it has no arguments. 266 TEXT runtime·morestack(SB),NOSPLIT,$-8-0 267 // Cannot grow scheduler stack (m->g0). 268 MOVD g_m(g), R8 269 MOVD m_g0(R8), R4 270 CMP g, R4 271 BNE 2(PC) 272 B runtime·abort(SB) 273 274 // Cannot grow signal stack (m->gsignal). 275 MOVD m_gsignal(R8), R4 276 CMP g, R4 277 BNE 2(PC) 278 B runtime·abort(SB) 279 280 // Called from f. 281 // Set g->sched to context in f 282 MOVD R26, (g_sched+gobuf_ctxt)(g) 283 MOVD RSP, R0 284 MOVD R0, (g_sched+gobuf_sp)(g) 285 MOVD LR, (g_sched+gobuf_pc)(g) 286 MOVD R3, (g_sched+gobuf_lr)(g) 287 288 // Called from f. 289 // Set m->morebuf to f's callers. 290 MOVD R3, (m_morebuf+gobuf_pc)(R8) // f's caller's PC 291 MOVD RSP, R0 292 MOVD R0, (m_morebuf+gobuf_sp)(R8) // f's caller's RSP 293 MOVD g, (m_morebuf+gobuf_g)(R8) 294 295 // Call newstack on m->g0's stack. 296 MOVD m_g0(R8), g 297 BL runtime·save_g(SB) 298 MOVD (g_sched+gobuf_sp)(g), R0 299 MOVD R0, RSP 300 BL runtime·newstack(SB) 301 302 // Not reached, but make sure the return PC from the call to newstack 303 // is still in this function, and not the beginning of the next. 304 UNDEF 305 306 TEXT runtime·morestack_noctxt(SB),NOSPLIT,$-4-0 307 MOVW $0, R26 308 B runtime·morestack(SB) 309 310 TEXT runtime·stackBarrier(SB),NOSPLIT,$0 311 // We came here via a RET to an overwritten LR. 312 // R0 may be live (see return0). Other registers are available. 313 314 // Get the original return PC, g.stkbar[g.stkbarPos].savedLRVal. 315 MOVD (g_stkbar+slice_array)(g), R4 316 MOVD g_stkbarPos(g), R5 317 MOVD $stkbar__size, R6 318 MUL R5, R6 319 ADD R4, R6 320 MOVD stkbar_savedLRVal(R6), R6 321 // Record that this stack barrier was hit. 322 ADD $1, R5 323 MOVD R5, g_stkbarPos(g) 324 // Jump to the original return PC. 325 B (R6) 326 327 // reflectcall: call a function with the given argument list 328 // func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32). 329 // we don't have variable-sized frames, so we use a small number 330 // of constant-sized-frame functions to encode a few bits of size in the pc. 331 // Caution: ugly multiline assembly macros in your future! 332 333 #define DISPATCH(NAME,MAXSIZE) \ 334 MOVD $MAXSIZE, R27; \ 335 CMP R27, R16; \ 336 BGT 3(PC); \ 337 MOVD $NAME(SB), R27; \ 338 B (R27) 339 // Note: can't just "B NAME(SB)" - bad inlining results. 340 341 TEXT reflect·call(SB), NOSPLIT, $0-0 342 B ·reflectcall(SB) 343 344 TEXT ·reflectcall(SB), NOSPLIT, $-8-32 345 MOVWU argsize+24(FP), R16 346 // NOTE(rsc): No call16, because CALLFN needs four words 347 // of argument space to invoke callwritebarrier. 348 DISPATCH(runtime·call32, 32) 349 DISPATCH(runtime·call64, 64) 350 DISPATCH(runtime·call128, 128) 351 DISPATCH(runtime·call256, 256) 352 DISPATCH(runtime·call512, 512) 353 DISPATCH(runtime·call1024, 1024) 354 DISPATCH(runtime·call2048, 2048) 355 DISPATCH(runtime·call4096, 4096) 356 DISPATCH(runtime·call8192, 8192) 357 DISPATCH(runtime·call16384, 16384) 358 DISPATCH(runtime·call32768, 32768) 359 DISPATCH(runtime·call65536, 65536) 360 DISPATCH(runtime·call131072, 131072) 361 DISPATCH(runtime·call262144, 262144) 362 DISPATCH(runtime·call524288, 524288) 363 DISPATCH(runtime·call1048576, 1048576) 364 DISPATCH(runtime·call2097152, 2097152) 365 DISPATCH(runtime·call4194304, 4194304) 366 DISPATCH(runtime·call8388608, 8388608) 367 DISPATCH(runtime·call16777216, 16777216) 368 DISPATCH(runtime·call33554432, 33554432) 369 DISPATCH(runtime·call67108864, 67108864) 370 DISPATCH(runtime·call134217728, 134217728) 371 DISPATCH(runtime·call268435456, 268435456) 372 DISPATCH(runtime·call536870912, 536870912) 373 DISPATCH(runtime·call1073741824, 1073741824) 374 MOVD $runtime·badreflectcall(SB), R0 375 B (R0) 376 377 #define CALLFN(NAME,MAXSIZE) \ 378 TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \ 379 NO_LOCAL_POINTERS; \ 380 /* copy arguments to stack */ \ 381 MOVD arg+16(FP), R3; \ 382 MOVWU argsize+24(FP), R4; \ 383 MOVD RSP, R5; \ 384 ADD $(8-1), R5; \ 385 SUB $1, R3; \ 386 ADD R5, R4; \ 387 CMP R5, R4; \ 388 BEQ 4(PC); \ 389 MOVBU.W 1(R3), R6; \ 390 MOVBU.W R6, 1(R5); \ 391 B -4(PC); \ 392 /* call function */ \ 393 MOVD f+8(FP), R26; \ 394 MOVD (R26), R0; \ 395 PCDATA $PCDATA_StackMapIndex, $0; \ 396 BL (R0); \ 397 /* copy return values back */ \ 398 MOVD arg+16(FP), R3; \ 399 MOVWU n+24(FP), R4; \ 400 MOVWU retoffset+28(FP), R6; \ 401 MOVD RSP, R5; \ 402 ADD R6, R5; \ 403 ADD R6, R3; \ 404 SUB R6, R4; \ 405 ADD $(8-1), R5; \ 406 SUB $1, R3; \ 407 ADD R5, R4; \ 408 loop: \ 409 CMP R5, R4; \ 410 BEQ end; \ 411 MOVBU.W 1(R5), R6; \ 412 MOVBU.W R6, 1(R3); \ 413 B loop; \ 414 end: \ 415 /* execute write barrier updates */ \ 416 MOVD argtype+0(FP), R7; \ 417 MOVD arg+16(FP), R3; \ 418 MOVWU n+24(FP), R4; \ 419 MOVWU retoffset+28(FP), R6; \ 420 MOVD R7, 8(RSP); \ 421 MOVD R3, 16(RSP); \ 422 MOVD R4, 24(RSP); \ 423 MOVD R6, 32(RSP); \ 424 BL runtime·callwritebarrier(SB); \ 425 RET 426 427 // These have 8 added to make the overall frame size a multiple of 16, 428 // as required by the ABI. (There is another +8 for the saved LR.) 429 CALLFN(·call32, 40 ) 430 CALLFN(·call64, 72 ) 431 CALLFN(·call128, 136 ) 432 CALLFN(·call256, 264 ) 433 CALLFN(·call512, 520 ) 434 CALLFN(·call1024, 1032 ) 435 CALLFN(·call2048, 2056 ) 436 CALLFN(·call4096, 4104 ) 437 CALLFN(·call8192, 8200 ) 438 CALLFN(·call16384, 16392 ) 439 CALLFN(·call32768, 32776 ) 440 CALLFN(·call65536, 65544 ) 441 CALLFN(·call131072, 131080 ) 442 CALLFN(·call262144, 262152 ) 443 CALLFN(·call524288, 524296 ) 444 CALLFN(·call1048576, 1048584 ) 445 CALLFN(·call2097152, 2097160 ) 446 CALLFN(·call4194304, 4194312 ) 447 CALLFN(·call8388608, 8388616 ) 448 CALLFN(·call16777216, 16777224 ) 449 CALLFN(·call33554432, 33554440 ) 450 CALLFN(·call67108864, 67108872 ) 451 CALLFN(·call134217728, 134217736 ) 452 CALLFN(·call268435456, 268435464 ) 453 CALLFN(·call536870912, 536870920 ) 454 CALLFN(·call1073741824, 1073741832 ) 455 456 // AES hashing not implemented for ARM64, issue #10109. 457 TEXT runtime·aeshash(SB),NOSPLIT,$-8-0 458 MOVW $0, R0 459 MOVW (R0), R1 460 TEXT runtime·aeshash32(SB),NOSPLIT,$-8-0 461 MOVW $0, R0 462 MOVW (R0), R1 463 TEXT runtime·aeshash64(SB),NOSPLIT,$-8-0 464 MOVW $0, R0 465 MOVW (R0), R1 466 TEXT runtime·aeshashstr(SB),NOSPLIT,$-8-0 467 MOVW $0, R0 468 MOVW (R0), R1 469 470 TEXT runtime·procyield(SB),NOSPLIT,$0-0 471 MOVWU cycles+0(FP), R0 472 again: 473 YIELD 474 SUBW $1, R0 475 CBNZ R0, again 476 RET 477 478 // void jmpdefer(fv, sp); 479 // called from deferreturn. 480 // 1. grab stored LR for caller 481 // 2. sub 4 bytes to get back to BL deferreturn 482 // 3. BR to fn 483 TEXT runtime·jmpdefer(SB), NOSPLIT, $-8-16 484 MOVD 0(RSP), R0 485 SUB $4, R0 486 MOVD R0, LR 487 488 MOVD fv+0(FP), R26 489 MOVD argp+8(FP), R0 490 MOVD R0, RSP 491 SUB $8, RSP 492 MOVD 0(R26), R3 493 B (R3) 494 495 // Save state of caller into g->sched. Smashes R0. 496 TEXT gosave<>(SB),NOSPLIT,$-8 497 MOVD LR, (g_sched+gobuf_pc)(g) 498 MOVD RSP, R0 499 MOVD R0, (g_sched+gobuf_sp)(g) 500 MOVD $0, (g_sched+gobuf_lr)(g) 501 MOVD $0, (g_sched+gobuf_ret)(g) 502 MOVD $0, (g_sched+gobuf_ctxt)(g) 503 RET 504 505 // func asmcgocall(fn, arg unsafe.Pointer) int32 506 // Call fn(arg) on the scheduler stack, 507 // aligned appropriately for the gcc ABI. 508 // See cgocall.go for more details. 509 TEXT ·asmcgocall(SB),NOSPLIT,$0-20 510 MOVD fn+0(FP), R1 511 MOVD arg+8(FP), R0 512 513 MOVD RSP, R2 // save original stack pointer 514 MOVD g, R4 515 516 // Figure out if we need to switch to m->g0 stack. 517 // We get called to create new OS threads too, and those 518 // come in on the m->g0 stack already. 519 MOVD g_m(g), R8 520 MOVD m_g0(R8), R3 521 CMP R3, g 522 BEQ g0 523 MOVD R0, R9 // gosave<> and save_g might clobber R0 524 BL gosave<>(SB) 525 MOVD R3, g 526 BL runtime·save_g(SB) 527 MOVD (g_sched+gobuf_sp)(g), R0 528 MOVD R0, RSP 529 MOVD R9, R0 530 531 // Now on a scheduling stack (a pthread-created stack). 532 g0: 533 // Save room for two of our pointers /*, plus 32 bytes of callee 534 // save area that lives on the caller stack. */ 535 MOVD RSP, R13 536 SUB $16, R13 537 MOVD R13, RSP 538 MOVD R4, 0(RSP) // save old g on stack 539 MOVD (g_stack+stack_hi)(R4), R4 540 SUB R2, R4 541 MOVD R4, 8(RSP) // save depth in old g stack (can't just save SP, as stack might be copied during a callback) 542 BL (R1) 543 MOVD R0, R9 544 545 // Restore g, stack pointer. R0 is errno, so don't touch it 546 MOVD 0(RSP), g 547 BL runtime·save_g(SB) 548 MOVD (g_stack+stack_hi)(g), R5 549 MOVD 8(RSP), R6 550 SUB R6, R5 551 MOVD R9, R0 552 MOVD R5, RSP 553 554 MOVW R0, ret+16(FP) 555 RET 556 557 // cgocallback(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt) 558 // Turn the fn into a Go func (by taking its address) and call 559 // cgocallback_gofunc. 560 TEXT runtime·cgocallback(SB),NOSPLIT,$40-32 561 MOVD $fn+0(FP), R0 562 MOVD R0, 8(RSP) 563 MOVD frame+8(FP), R0 564 MOVD R0, 16(RSP) 565 MOVD framesize+16(FP), R0 566 MOVD R0, 24(RSP) 567 MOVD ctxt+24(FP), R0 568 MOVD R0, 32(RSP) 569 MOVD $runtime·cgocallback_gofunc(SB), R0 570 BL (R0) 571 RET 572 573 // cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize, uintptr ctxt) 574 // See cgocall.go for more details. 575 TEXT ·cgocallback_gofunc(SB),NOSPLIT,$24-32 576 NO_LOCAL_POINTERS 577 578 // Load g from thread-local storage. 579 MOVB runtime·iscgo(SB), R3 580 CMP $0, R3 581 BEQ nocgo 582 BL runtime·load_g(SB) 583 nocgo: 584 585 // If g is nil, Go did not create the current thread. 586 // Call needm to obtain one for temporary use. 587 // In this case, we're running on the thread stack, so there's 588 // lots of space, but the linker doesn't know. Hide the call from 589 // the linker analysis by using an indirect call. 590 CMP $0, g 591 BEQ needm 592 593 MOVD g_m(g), R8 594 MOVD R8, savedm-8(SP) 595 B havem 596 597 needm: 598 MOVD g, savedm-8(SP) // g is zero, so is m. 599 MOVD $runtime·needm(SB), R0 600 BL (R0) 601 602 // Set m->sched.sp = SP, so that if a panic happens 603 // during the function we are about to execute, it will 604 // have a valid SP to run on the g0 stack. 605 // The next few lines (after the havem label) 606 // will save this SP onto the stack and then write 607 // the same SP back to m->sched.sp. That seems redundant, 608 // but if an unrecovered panic happens, unwindm will 609 // restore the g->sched.sp from the stack location 610 // and then systemstack will try to use it. If we don't set it here, 611 // that restored SP will be uninitialized (typically 0) and 612 // will not be usable. 613 MOVD g_m(g), R8 614 MOVD m_g0(R8), R3 615 MOVD RSP, R0 616 MOVD R0, (g_sched+gobuf_sp)(R3) 617 618 havem: 619 // Now there's a valid m, and we're running on its m->g0. 620 // Save current m->g0->sched.sp on stack and then set it to SP. 621 // Save current sp in m->g0->sched.sp in preparation for 622 // switch back to m->curg stack. 623 // NOTE: unwindm knows that the saved g->sched.sp is at 16(RSP) aka savedsp-16(SP). 624 // Beware that the frame size is actually 32. 625 MOVD m_g0(R8), R3 626 MOVD (g_sched+gobuf_sp)(R3), R4 627 MOVD R4, savedsp-16(SP) 628 MOVD RSP, R0 629 MOVD R0, (g_sched+gobuf_sp)(R3) 630 631 // Switch to m->curg stack and call runtime.cgocallbackg. 632 // Because we are taking over the execution of m->curg 633 // but *not* resuming what had been running, we need to 634 // save that information (m->curg->sched) so we can restore it. 635 // We can restore m->curg->sched.sp easily, because calling 636 // runtime.cgocallbackg leaves SP unchanged upon return. 637 // To save m->curg->sched.pc, we push it onto the stack. 638 // This has the added benefit that it looks to the traceback 639 // routine like cgocallbackg is going to return to that 640 // PC (because the frame we allocate below has the same 641 // size as cgocallback_gofunc's frame declared above) 642 // so that the traceback will seamlessly trace back into 643 // the earlier calls. 644 // 645 // In the new goroutine, -8(SP) is unused (where SP refers to 646 // m->curg's SP while we're setting it up, before we've adjusted it). 647 MOVD m_curg(R8), g 648 BL runtime·save_g(SB) 649 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4 650 MOVD (g_sched+gobuf_pc)(g), R5 651 MOVD R5, -(24+8)(R4) 652 MOVD ctxt+24(FP), R0 653 MOVD R0, -(16+8)(R4) 654 MOVD $-(24+8)(R4), R0 // maintain 16-byte SP alignment 655 MOVD R0, RSP 656 BL runtime·cgocallbackg(SB) 657 658 // Restore g->sched (== m->curg->sched) from saved values. 659 MOVD 0(RSP), R5 660 MOVD R5, (g_sched+gobuf_pc)(g) 661 MOVD RSP, R4 662 ADD $(24+8), R4, R4 663 MOVD R4, (g_sched+gobuf_sp)(g) 664 665 // Switch back to m->g0's stack and restore m->g0->sched.sp. 666 // (Unlike m->curg, the g0 goroutine never uses sched.pc, 667 // so we do not have to restore it.) 668 MOVD g_m(g), R8 669 MOVD m_g0(R8), g 670 BL runtime·save_g(SB) 671 MOVD (g_sched+gobuf_sp)(g), R0 672 MOVD R0, RSP 673 MOVD savedsp-16(SP), R4 674 MOVD R4, (g_sched+gobuf_sp)(g) 675 676 // If the m on entry was nil, we called needm above to borrow an m 677 // for the duration of the call. Since the call is over, return it with dropm. 678 MOVD savedm-8(SP), R6 679 CMP $0, R6 680 BNE droppedm 681 MOVD $runtime·dropm(SB), R0 682 BL (R0) 683 droppedm: 684 685 // Done! 686 RET 687 688 // Called from cgo wrappers, this function returns g->m->curg.stack.hi. 689 // Must obey the gcc calling convention. 690 TEXT _cgo_topofstack(SB),NOSPLIT,$24 691 // g (R28) and REGTMP (R27) might be clobbered by load_g. They 692 // are callee-save in the gcc calling convention, so save them. 693 MOVD R27, savedR27-8(SP) 694 MOVD g, saveG-16(SP) 695 696 BL runtime·load_g(SB) 697 MOVD g_m(g), R0 698 MOVD m_curg(R0), R0 699 MOVD (g_stack+stack_hi)(R0), R0 700 701 MOVD saveG-16(SP), g 702 MOVD savedR28-8(SP), R27 703 RET 704 705 // void setg(G*); set g. for use by needm. 706 TEXT runtime·setg(SB), NOSPLIT, $0-8 707 MOVD gg+0(FP), g 708 // This only happens if iscgo, so jump straight to save_g 709 BL runtime·save_g(SB) 710 RET 711 712 // void setg_gcc(G*); set g called from gcc 713 TEXT setg_gcc<>(SB),NOSPLIT,$8 714 MOVD R0, g 715 MOVD R27, savedR27-8(SP) 716 BL runtime·save_g(SB) 717 MOVD savedR27-8(SP), R27 718 RET 719 720 TEXT runtime·getcallerpc(SB),NOSPLIT,$8-16 721 MOVD 16(RSP), R0 // LR saved by caller 722 MOVD runtime·stackBarrierPC(SB), R1 723 CMP R0, R1 724 BNE nobar 725 // Get original return PC. 726 BL runtime·nextBarrierPC(SB) 727 MOVD 8(RSP), R0 728 nobar: 729 MOVD R0, ret+8(FP) 730 RET 731 732 TEXT runtime·setcallerpc(SB),NOSPLIT,$8-16 733 MOVD pc+8(FP), R0 734 MOVD 16(RSP), R1 735 MOVD runtime·stackBarrierPC(SB), R2 736 CMP R1, R2 737 BEQ setbar 738 MOVD R0, 16(RSP) // set LR in caller 739 RET 740 setbar: 741 // Set the stack barrier return PC. 742 MOVD R0, 8(RSP) 743 BL runtime·setNextBarrierPC(SB) 744 RET 745 746 TEXT runtime·getcallersp(SB),NOSPLIT,$0-16 747 MOVD argp+0(FP), R0 748 SUB $8, R0 749 MOVD R0, ret+8(FP) 750 RET 751 752 TEXT runtime·abort(SB),NOSPLIT,$-8-0 753 B (ZR) 754 UNDEF 755 756 // memhash_varlen(p unsafe.Pointer, h seed) uintptr 757 // redirects to memhash(p, h, size) using the size 758 // stored in the closure. 759 TEXT runtime·memhash_varlen(SB),NOSPLIT,$40-24 760 GO_ARGS 761 NO_LOCAL_POINTERS 762 MOVD p+0(FP), R3 763 MOVD h+8(FP), R4 764 MOVD 8(R26), R5 765 MOVD R3, 8(RSP) 766 MOVD R4, 16(RSP) 767 MOVD R5, 24(RSP) 768 BL runtime·memhash(SB) 769 MOVD 32(RSP), R3 770 MOVD R3, ret+16(FP) 771 RET 772 773 // memequal(p, q unsafe.Pointer, size uintptr) bool 774 TEXT runtime·memequal(SB),NOSPLIT,$-8-25 775 MOVD a+0(FP), R1 776 MOVD b+8(FP), R2 777 MOVD size+16(FP), R3 778 ADD R1, R3, R6 779 MOVD $1, R0 780 MOVB R0, ret+24(FP) 781 CMP R1, R2 782 BEQ done 783 loop: 784 CMP R1, R6 785 BEQ done 786 MOVBU.P 1(R1), R4 787 MOVBU.P 1(R2), R5 788 CMP R4, R5 789 BEQ loop 790 791 MOVB $0, ret+24(FP) 792 done: 793 RET 794 795 // memequal_varlen(a, b unsafe.Pointer) bool 796 TEXT runtime·memequal_varlen(SB),NOSPLIT,$40-17 797 MOVD a+0(FP), R3 798 MOVD b+8(FP), R4 799 CMP R3, R4 800 BEQ eq 801 MOVD 8(R26), R5 // compiler stores size at offset 8 in the closure 802 MOVD R3, 8(RSP) 803 MOVD R4, 16(RSP) 804 MOVD R5, 24(RSP) 805 BL runtime·memequal(SB) 806 MOVBU 32(RSP), R3 807 MOVB R3, ret+16(FP) 808 RET 809 eq: 810 MOVD $1, R3 811 MOVB R3, ret+16(FP) 812 RET 813 814 TEXT runtime·cmpstring(SB),NOSPLIT,$-4-40 815 MOVD s1_base+0(FP), R2 816 MOVD s1_len+8(FP), R0 817 MOVD s2_base+16(FP), R3 818 MOVD s2_len+24(FP), R1 819 ADD $40, RSP, R7 820 B runtime·cmpbody<>(SB) 821 822 TEXT bytes·Compare(SB),NOSPLIT,$-4-56 823 MOVD s1+0(FP), R2 824 MOVD s1+8(FP), R0 825 MOVD s2+24(FP), R3 826 MOVD s2+32(FP), R1 827 ADD $56, RSP, R7 828 B runtime·cmpbody<>(SB) 829 830 // On entry: 831 // R0 is the length of s1 832 // R1 is the length of s2 833 // R2 points to the start of s1 834 // R3 points to the start of s2 835 // R7 points to return value (-1/0/1 will be written here) 836 // 837 // On exit: 838 // R4, R5, and R6 are clobbered 839 TEXT runtime·cmpbody<>(SB),NOSPLIT,$-4-0 840 CMP R2, R3 841 BEQ samebytes // same starting pointers; compare lengths 842 CMP R0, R1 843 CSEL LT, R1, R0, R6 // R6 is min(R0, R1) 844 845 ADD R2, R6 // R2 is current byte in s1, R6 is last byte in s1 to compare 846 loop: 847 CMP R2, R6 848 BEQ samebytes // all compared bytes were the same; compare lengths 849 MOVBU.P 1(R2), R4 850 MOVBU.P 1(R3), R5 851 CMP R4, R5 852 BEQ loop 853 // bytes differed 854 MOVD $1, R4 855 CSNEG LT, R4, R4, R4 856 MOVD R4, (R7) 857 RET 858 samebytes: 859 MOVD $1, R4 860 CMP R0, R1 861 CSNEG LT, R4, R4, R4 862 CSEL EQ, ZR, R4, R4 863 MOVD R4, (R7) 864 RET 865 866 // eqstring tests whether two strings are equal. 867 // The compiler guarantees that strings passed 868 // to eqstring have equal length. 869 // See runtime_test.go:eqstring_generic for 870 // equivalent Go code. 871 TEXT runtime·eqstring(SB),NOSPLIT,$0-33 872 MOVD s1str+0(FP), R0 873 MOVD s1len+8(FP), R1 874 MOVD s2str+16(FP), R2 875 ADD R0, R1 // end 876 loop: 877 CMP R0, R1 878 BEQ equal // reaches the end 879 MOVBU.P 1(R0), R4 880 MOVBU.P 1(R2), R5 881 CMP R4, R5 882 BEQ loop 883 notequal: 884 MOVB ZR, ret+32(FP) 885 RET 886 equal: 887 MOVD $1, R0 888 MOVB R0, ret+32(FP) 889 RET 890 891 // 892 // functions for other packages 893 // 894 TEXT bytes·IndexByte(SB),NOSPLIT,$0-40 895 MOVD b+0(FP), R0 896 MOVD b_len+8(FP), R1 897 MOVBU c+24(FP), R2 // byte to find 898 MOVD R0, R4 // store base for later 899 ADD R0, R1 // end 900 loop: 901 CMP R0, R1 902 BEQ notfound 903 MOVBU.P 1(R0), R3 904 CMP R2, R3 905 BNE loop 906 907 SUB $1, R0 // R0 will be one beyond the position we want 908 SUB R4, R0 // remove base 909 MOVD R0, ret+32(FP) 910 RET 911 912 notfound: 913 MOVD $-1, R0 914 MOVD R0, ret+32(FP) 915 RET 916 917 TEXT strings·IndexByte(SB),NOSPLIT,$0-32 918 MOVD s+0(FP), R0 919 MOVD s_len+8(FP), R1 920 MOVBU c+16(FP), R2 // byte to find 921 MOVD R0, R4 // store base for later 922 ADD R0, R1 // end 923 loop: 924 CMP R0, R1 925 BEQ notfound 926 MOVBU.P 1(R0), R3 927 CMP R2, R3 928 BNE loop 929 930 SUB $1, R0 // R0 will be one beyond the position we want 931 SUB R4, R0 // remove base 932 MOVD R0, ret+24(FP) 933 RET 934 935 notfound: 936 MOVD $-1, R0 937 MOVD R0, ret+24(FP) 938 RET 939 940 // TODO: share code with memequal? 941 TEXT bytes·Equal(SB),NOSPLIT,$0-49 942 MOVD a_len+8(FP), R1 943 MOVD b_len+32(FP), R3 944 CMP R1, R3 // unequal lengths are not equal 945 BNE notequal 946 MOVD a+0(FP), R0 947 MOVD b+24(FP), R2 948 ADD R0, R1 // end 949 loop: 950 CMP R0, R1 951 BEQ equal // reaches the end 952 MOVBU.P 1(R0), R4 953 MOVBU.P 1(R2), R5 954 CMP R4, R5 955 BEQ loop 956 notequal: 957 MOVB ZR, ret+48(FP) 958 RET 959 equal: 960 MOVD $1, R0 961 MOVB R0, ret+48(FP) 962 RET 963 964 TEXT runtime·fastrand1(SB),NOSPLIT,$-8-4 965 MOVD g_m(g), R1 966 MOVWU m_fastrand(R1), R0 967 ADD R0, R0 968 CMPW $0, R0 969 BGE notneg 970 EOR $0x88888eef, R0 971 notneg: 972 MOVW R0, m_fastrand(R1) 973 MOVW R0, ret+0(FP) 974 RET 975 976 TEXT runtime·return0(SB), NOSPLIT, $0 977 MOVW $0, R0 978 RET 979 980 // The top-most function running on a goroutine 981 // returns to goexit+PCQuantum. 982 TEXT runtime·goexit(SB),NOSPLIT,$-8-0 983 MOVD R0, R0 // NOP 984 BL runtime·goexit1(SB) // does not return 985 986 // TODO(aram): use PRFM here. 987 TEXT runtime·prefetcht0(SB),NOSPLIT,$0-8 988 RET 989 990 TEXT runtime·prefetcht1(SB),NOSPLIT,$0-8 991 RET 992 993 TEXT runtime·prefetcht2(SB),NOSPLIT,$0-8 994 RET 995 996 TEXT runtime·prefetchnta(SB),NOSPLIT,$0-8 997 RET 998 999 TEXT runtime·sigreturn(SB),NOSPLIT,$0-8 1000 RET 1001 1002 // This is called from .init_array and follows the platform, not Go, ABI. 1003 TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0 1004 SUB $0x10, RSP 1005 MOVD R27, 8(RSP) // The access to global variables below implicitly uses R27, which is callee-save 1006 MOVD runtime·lastmoduledatap(SB), R1 1007 MOVD R0, moduledata_next(R1) 1008 MOVD R0, runtime·lastmoduledatap(SB) 1009 MOVD 8(RSP), R27 1010 ADD $0x10, RSP 1011 RET 1012 1013 TEXT ·checkASM(SB),NOSPLIT,$0-1 1014 MOVW $1, R3 1015 MOVB R3, ret+0(FP) 1016 RET