github.com/miolini/go@v0.0.0-20160405192216-fca68c8cb408/src/runtime/asm_arm64.s (about) 1 // Copyright 2015 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 #include "go_asm.h" 6 #include "go_tls.h" 7 #include "tls_arm64.h" 8 #include "funcdata.h" 9 #include "textflag.h" 10 11 TEXT runtime·rt0_go(SB),NOSPLIT,$0 12 // SP = stack; R0 = argc; R1 = argv 13 14 // initialize essential registers 15 BL runtime·reginit(SB) 16 17 SUB $32, RSP 18 MOVW R0, 8(RSP) // argc 19 MOVD R1, 16(RSP) // argv 20 21 // create istack out of the given (operating system) stack. 22 // _cgo_init may update stackguard. 23 MOVD $runtime·g0(SB), g 24 MOVD RSP, R7 25 MOVD $(-64*1024)(R7), R0 26 MOVD R0, g_stackguard0(g) 27 MOVD R0, g_stackguard1(g) 28 MOVD R0, (g_stack+stack_lo)(g) 29 MOVD R7, (g_stack+stack_hi)(g) 30 31 // if there is a _cgo_init, call it using the gcc ABI. 32 MOVD _cgo_init(SB), R12 33 CMP $0, R12 34 BEQ nocgo 35 36 MRS_TPIDR_R0 // load TLS base pointer 37 MOVD R0, R3 // arg 3: TLS base pointer 38 #ifdef TLSG_IS_VARIABLE 39 MOVD $runtime·tls_g(SB), R2 // arg 2: &tls_g 40 #else 41 MOVD $0, R2 // arg 2: not used when using platform's TLS 42 #endif 43 MOVD $setg_gcc<>(SB), R1 // arg 1: setg 44 MOVD g, R0 // arg 0: G 45 BL (R12) 46 MOVD _cgo_init(SB), R12 47 CMP $0, R12 48 BEQ nocgo 49 50 nocgo: 51 // update stackguard after _cgo_init 52 MOVD (g_stack+stack_lo)(g), R0 53 ADD $const__StackGuard, R0 54 MOVD R0, g_stackguard0(g) 55 MOVD R0, g_stackguard1(g) 56 57 // set the per-goroutine and per-mach "registers" 58 MOVD $runtime·m0(SB), R0 59 60 // save m->g0 = g0 61 MOVD g, m_g0(R0) 62 // save m0 to g0->m 63 MOVD R0, g_m(g) 64 65 BL runtime·check(SB) 66 67 MOVW 8(RSP), R0 // copy argc 68 MOVW R0, -8(RSP) 69 MOVD 16(RSP), R0 // copy argv 70 MOVD R0, 0(RSP) 71 BL runtime·args(SB) 72 BL runtime·osinit(SB) 73 BL runtime·schedinit(SB) 74 75 // create a new goroutine to start program 76 MOVD $runtime·mainPC(SB), R0 // entry 77 MOVD RSP, R7 78 MOVD.W $0, -8(R7) 79 MOVD.W R0, -8(R7) 80 MOVD.W $0, -8(R7) 81 MOVD.W $0, -8(R7) 82 MOVD R7, RSP 83 BL runtime·newproc(SB) 84 ADD $32, RSP 85 86 // start this M 87 BL runtime·mstart(SB) 88 89 MOVD $0, R0 90 MOVD R0, (R0) // boom 91 UNDEF 92 93 DATA runtime·mainPC+0(SB)/8,$runtime·main(SB) 94 GLOBL runtime·mainPC(SB),RODATA,$8 95 96 TEXT runtime·breakpoint(SB),NOSPLIT,$-8-0 97 BRK 98 RET 99 100 TEXT runtime·asminit(SB),NOSPLIT,$-8-0 101 RET 102 103 TEXT runtime·reginit(SB),NOSPLIT,$-8-0 104 // initialize essential FP registers 105 FMOVD $4503601774854144.0, F27 106 FMOVD $0.5, F29 107 FSUBD F29, F29, F28 108 FADDD F29, F29, F30 109 FADDD F30, F30, F31 110 RET 111 112 /* 113 * go-routine 114 */ 115 116 // void gosave(Gobuf*) 117 // save state in Gobuf; setjmp 118 TEXT runtime·gosave(SB), NOSPLIT, $-8-8 119 MOVD buf+0(FP), R3 120 MOVD RSP, R0 121 MOVD R0, gobuf_sp(R3) 122 MOVD LR, gobuf_pc(R3) 123 MOVD g, gobuf_g(R3) 124 MOVD ZR, gobuf_lr(R3) 125 MOVD ZR, gobuf_ret(R3) 126 MOVD ZR, gobuf_ctxt(R3) 127 RET 128 129 // void gogo(Gobuf*) 130 // restore state from Gobuf; longjmp 131 TEXT runtime·gogo(SB), NOSPLIT, $-8-8 132 MOVD buf+0(FP), R5 133 MOVD gobuf_g(R5), g 134 BL runtime·save_g(SB) 135 136 MOVD 0(g), R4 // make sure g is not nil 137 MOVD gobuf_sp(R5), R0 138 MOVD R0, RSP 139 MOVD gobuf_lr(R5), LR 140 MOVD gobuf_ret(R5), R0 141 MOVD gobuf_ctxt(R5), R26 142 MOVD $0, gobuf_sp(R5) 143 MOVD $0, gobuf_ret(R5) 144 MOVD $0, gobuf_lr(R5) 145 MOVD $0, gobuf_ctxt(R5) 146 CMP ZR, ZR // set condition codes for == test, needed by stack split 147 MOVD gobuf_pc(R5), R6 148 B (R6) 149 150 // void mcall(fn func(*g)) 151 // Switch to m->g0's stack, call fn(g). 152 // Fn must never return. It should gogo(&g->sched) 153 // to keep running g. 154 TEXT runtime·mcall(SB), NOSPLIT, $-8-8 155 // Save caller state in g->sched 156 MOVD RSP, R0 157 MOVD R0, (g_sched+gobuf_sp)(g) 158 MOVD LR, (g_sched+gobuf_pc)(g) 159 MOVD $0, (g_sched+gobuf_lr)(g) 160 MOVD g, (g_sched+gobuf_g)(g) 161 162 // Switch to m->g0 & its stack, call fn. 163 MOVD g, R3 164 MOVD g_m(g), R8 165 MOVD m_g0(R8), g 166 BL runtime·save_g(SB) 167 CMP g, R3 168 BNE 2(PC) 169 B runtime·badmcall(SB) 170 MOVD fn+0(FP), R26 // context 171 MOVD 0(R26), R4 // code pointer 172 MOVD (g_sched+gobuf_sp)(g), R0 173 MOVD R0, RSP // sp = m->g0->sched.sp 174 MOVD R3, -8(RSP) 175 MOVD $0, -16(RSP) 176 SUB $16, RSP 177 BL (R4) 178 B runtime·badmcall2(SB) 179 180 // systemstack_switch is a dummy routine that systemstack leaves at the bottom 181 // of the G stack. We need to distinguish the routine that 182 // lives at the bottom of the G stack from the one that lives 183 // at the top of the system stack because the one at the top of 184 // the system stack terminates the stack walk (see topofstack()). 185 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0 186 UNDEF 187 BL (LR) // make sure this function is not leaf 188 RET 189 190 // func systemstack(fn func()) 191 TEXT runtime·systemstack(SB), NOSPLIT, $0-8 192 MOVD fn+0(FP), R3 // R3 = fn 193 MOVD R3, R26 // context 194 MOVD g_m(g), R4 // R4 = m 195 196 MOVD m_gsignal(R4), R5 // R5 = gsignal 197 CMP g, R5 198 BEQ noswitch 199 200 MOVD m_g0(R4), R5 // R5 = g0 201 CMP g, R5 202 BEQ noswitch 203 204 MOVD m_curg(R4), R6 205 CMP g, R6 206 BEQ switch 207 208 // Bad: g is not gsignal, not g0, not curg. What is it? 209 // Hide call from linker nosplit analysis. 210 MOVD $runtime·badsystemstack(SB), R3 211 BL (R3) 212 213 switch: 214 // save our state in g->sched. Pretend to 215 // be systemstack_switch if the G stack is scanned. 216 MOVD $runtime·systemstack_switch(SB), R6 217 ADD $8, R6 // get past prologue 218 MOVD R6, (g_sched+gobuf_pc)(g) 219 MOVD RSP, R0 220 MOVD R0, (g_sched+gobuf_sp)(g) 221 MOVD $0, (g_sched+gobuf_lr)(g) 222 MOVD g, (g_sched+gobuf_g)(g) 223 224 // switch to g0 225 MOVD R5, g 226 BL runtime·save_g(SB) 227 MOVD (g_sched+gobuf_sp)(g), R3 228 // make it look like mstart called systemstack on g0, to stop traceback 229 SUB $16, R3 230 AND $~15, R3 231 MOVD $runtime·mstart(SB), R4 232 MOVD R4, 0(R3) 233 MOVD R3, RSP 234 235 // call target function 236 MOVD 0(R26), R3 // code pointer 237 BL (R3) 238 239 // switch back to g 240 MOVD g_m(g), R3 241 MOVD m_curg(R3), g 242 BL runtime·save_g(SB) 243 MOVD (g_sched+gobuf_sp)(g), R0 244 MOVD R0, RSP 245 MOVD $0, (g_sched+gobuf_sp)(g) 246 RET 247 248 noswitch: 249 // already on m stack, just call directly 250 MOVD 0(R26), R3 // code pointer 251 BL (R3) 252 RET 253 254 /* 255 * support for morestack 256 */ 257 258 // Called during function prolog when more stack is needed. 259 // Caller has already loaded: 260 // R3 prolog's LR (R30) 261 // 262 // The traceback routines see morestack on a g0 as being 263 // the top of a stack (for example, morestack calling newstack 264 // calling the scheduler calling newm calling gc), so we must 265 // record an argument size. For that purpose, it has no arguments. 266 TEXT runtime·morestack(SB),NOSPLIT,$-8-0 267 // Cannot grow scheduler stack (m->g0). 268 MOVD g_m(g), R8 269 MOVD m_g0(R8), R4 270 CMP g, R4 271 BNE 2(PC) 272 B runtime·abort(SB) 273 274 // Cannot grow signal stack (m->gsignal). 275 MOVD m_gsignal(R8), R4 276 CMP g, R4 277 BNE 2(PC) 278 B runtime·abort(SB) 279 280 // Called from f. 281 // Set g->sched to context in f 282 MOVD R26, (g_sched+gobuf_ctxt)(g) 283 MOVD RSP, R0 284 MOVD R0, (g_sched+gobuf_sp)(g) 285 MOVD LR, (g_sched+gobuf_pc)(g) 286 MOVD R3, (g_sched+gobuf_lr)(g) 287 288 // Called from f. 289 // Set m->morebuf to f's callers. 290 MOVD R3, (m_morebuf+gobuf_pc)(R8) // f's caller's PC 291 MOVD RSP, R0 292 MOVD R0, (m_morebuf+gobuf_sp)(R8) // f's caller's RSP 293 MOVD g, (m_morebuf+gobuf_g)(R8) 294 295 // Call newstack on m->g0's stack. 296 MOVD m_g0(R8), g 297 BL runtime·save_g(SB) 298 MOVD (g_sched+gobuf_sp)(g), R0 299 MOVD R0, RSP 300 BL runtime·newstack(SB) 301 302 // Not reached, but make sure the return PC from the call to newstack 303 // is still in this function, and not the beginning of the next. 304 UNDEF 305 306 TEXT runtime·morestack_noctxt(SB),NOSPLIT,$-4-0 307 MOVW $0, R26 308 B runtime·morestack(SB) 309 310 TEXT runtime·stackBarrier(SB),NOSPLIT,$0 311 // We came here via a RET to an overwritten LR. 312 // R0 may be live (see return0). Other registers are available. 313 314 // Get the original return PC, g.stkbar[g.stkbarPos].savedLRVal. 315 MOVD (g_stkbar+slice_array)(g), R4 316 MOVD g_stkbarPos(g), R5 317 MOVD $stkbar__size, R6 318 MUL R5, R6 319 ADD R4, R6 320 MOVD stkbar_savedLRVal(R6), R6 321 // Record that this stack barrier was hit. 322 ADD $1, R5 323 MOVD R5, g_stkbarPos(g) 324 // Jump to the original return PC. 325 B (R6) 326 327 // reflectcall: call a function with the given argument list 328 // func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32). 329 // we don't have variable-sized frames, so we use a small number 330 // of constant-sized-frame functions to encode a few bits of size in the pc. 331 // Caution: ugly multiline assembly macros in your future! 332 333 #define DISPATCH(NAME,MAXSIZE) \ 334 MOVD $MAXSIZE, R27; \ 335 CMP R27, R16; \ 336 BGT 3(PC); \ 337 MOVD $NAME(SB), R27; \ 338 B (R27) 339 // Note: can't just "B NAME(SB)" - bad inlining results. 340 341 TEXT reflect·call(SB), NOSPLIT, $0-0 342 B ·reflectcall(SB) 343 344 TEXT ·reflectcall(SB), NOSPLIT, $-8-32 345 MOVWU argsize+24(FP), R16 346 // NOTE(rsc): No call16, because CALLFN needs four words 347 // of argument space to invoke callwritebarrier. 348 DISPATCH(runtime·call32, 32) 349 DISPATCH(runtime·call64, 64) 350 DISPATCH(runtime·call128, 128) 351 DISPATCH(runtime·call256, 256) 352 DISPATCH(runtime·call512, 512) 353 DISPATCH(runtime·call1024, 1024) 354 DISPATCH(runtime·call2048, 2048) 355 DISPATCH(runtime·call4096, 4096) 356 DISPATCH(runtime·call8192, 8192) 357 DISPATCH(runtime·call16384, 16384) 358 DISPATCH(runtime·call32768, 32768) 359 DISPATCH(runtime·call65536, 65536) 360 DISPATCH(runtime·call131072, 131072) 361 DISPATCH(runtime·call262144, 262144) 362 DISPATCH(runtime·call524288, 524288) 363 DISPATCH(runtime·call1048576, 1048576) 364 DISPATCH(runtime·call2097152, 2097152) 365 DISPATCH(runtime·call4194304, 4194304) 366 DISPATCH(runtime·call8388608, 8388608) 367 DISPATCH(runtime·call16777216, 16777216) 368 DISPATCH(runtime·call33554432, 33554432) 369 DISPATCH(runtime·call67108864, 67108864) 370 DISPATCH(runtime·call134217728, 134217728) 371 DISPATCH(runtime·call268435456, 268435456) 372 DISPATCH(runtime·call536870912, 536870912) 373 DISPATCH(runtime·call1073741824, 1073741824) 374 MOVD $runtime·badreflectcall(SB), R0 375 B (R0) 376 377 #define CALLFN(NAME,MAXSIZE) \ 378 TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \ 379 NO_LOCAL_POINTERS; \ 380 /* copy arguments to stack */ \ 381 MOVD arg+16(FP), R3; \ 382 MOVWU argsize+24(FP), R4; \ 383 MOVD RSP, R5; \ 384 ADD $(8-1), R5; \ 385 SUB $1, R3; \ 386 ADD R5, R4; \ 387 CMP R5, R4; \ 388 BEQ 4(PC); \ 389 MOVBU.W 1(R3), R6; \ 390 MOVBU.W R6, 1(R5); \ 391 B -4(PC); \ 392 /* call function */ \ 393 MOVD f+8(FP), R26; \ 394 MOVD (R26), R0; \ 395 PCDATA $PCDATA_StackMapIndex, $0; \ 396 BL (R0); \ 397 /* copy return values back */ \ 398 MOVD arg+16(FP), R3; \ 399 MOVWU n+24(FP), R4; \ 400 MOVWU retoffset+28(FP), R6; \ 401 MOVD RSP, R5; \ 402 ADD R6, R5; \ 403 ADD R6, R3; \ 404 SUB R6, R4; \ 405 ADD $(8-1), R5; \ 406 SUB $1, R3; \ 407 ADD R5, R4; \ 408 loop: \ 409 CMP R5, R4; \ 410 BEQ end; \ 411 MOVBU.W 1(R5), R6; \ 412 MOVBU.W R6, 1(R3); \ 413 B loop; \ 414 end: \ 415 /* execute write barrier updates */ \ 416 MOVD argtype+0(FP), R7; \ 417 MOVD arg+16(FP), R3; \ 418 MOVWU n+24(FP), R4; \ 419 MOVWU retoffset+28(FP), R6; \ 420 MOVD R7, 8(RSP); \ 421 MOVD R3, 16(RSP); \ 422 MOVD R4, 24(RSP); \ 423 MOVD R6, 32(RSP); \ 424 BL runtime·callwritebarrier(SB); \ 425 RET 426 427 // These have 8 added to make the overall frame size a multiple of 16, 428 // as required by the ABI. (There is another +8 for the saved LR.) 429 CALLFN(·call32, 40 ) 430 CALLFN(·call64, 72 ) 431 CALLFN(·call128, 136 ) 432 CALLFN(·call256, 264 ) 433 CALLFN(·call512, 520 ) 434 CALLFN(·call1024, 1032 ) 435 CALLFN(·call2048, 2056 ) 436 CALLFN(·call4096, 4104 ) 437 CALLFN(·call8192, 8200 ) 438 CALLFN(·call16384, 16392 ) 439 CALLFN(·call32768, 32776 ) 440 CALLFN(·call65536, 65544 ) 441 CALLFN(·call131072, 131080 ) 442 CALLFN(·call262144, 262152 ) 443 CALLFN(·call524288, 524296 ) 444 CALLFN(·call1048576, 1048584 ) 445 CALLFN(·call2097152, 2097160 ) 446 CALLFN(·call4194304, 4194312 ) 447 CALLFN(·call8388608, 8388616 ) 448 CALLFN(·call16777216, 16777224 ) 449 CALLFN(·call33554432, 33554440 ) 450 CALLFN(·call67108864, 67108872 ) 451 CALLFN(·call134217728, 134217736 ) 452 CALLFN(·call268435456, 268435464 ) 453 CALLFN(·call536870912, 536870920 ) 454 CALLFN(·call1073741824, 1073741832 ) 455 456 // AES hashing not implemented for ARM64, issue #10109. 457 TEXT runtime·aeshash(SB),NOSPLIT,$-8-0 458 MOVW $0, R0 459 MOVW (R0), R1 460 TEXT runtime·aeshash32(SB),NOSPLIT,$-8-0 461 MOVW $0, R0 462 MOVW (R0), R1 463 TEXT runtime·aeshash64(SB),NOSPLIT,$-8-0 464 MOVW $0, R0 465 MOVW (R0), R1 466 TEXT runtime·aeshashstr(SB),NOSPLIT,$-8-0 467 MOVW $0, R0 468 MOVW (R0), R1 469 470 TEXT runtime·procyield(SB),NOSPLIT,$0-0 471 MOVWU cycles+0(FP), R0 472 again: 473 YIELD 474 SUBW $1, R0 475 CBNZ R0, again 476 RET 477 478 // void jmpdefer(fv, sp); 479 // called from deferreturn. 480 // 1. grab stored LR for caller 481 // 2. sub 4 bytes to get back to BL deferreturn 482 // 3. BR to fn 483 TEXT runtime·jmpdefer(SB), NOSPLIT, $-8-16 484 MOVD 0(RSP), R0 485 SUB $4, R0 486 MOVD R0, LR 487 488 MOVD fv+0(FP), R26 489 MOVD argp+8(FP), R0 490 MOVD R0, RSP 491 SUB $8, RSP 492 MOVD 0(R26), R3 493 B (R3) 494 495 // Save state of caller into g->sched. Smashes R0. 496 TEXT gosave<>(SB),NOSPLIT,$-8 497 MOVD LR, (g_sched+gobuf_pc)(g) 498 MOVD RSP, R0 499 MOVD R0, (g_sched+gobuf_sp)(g) 500 MOVD $0, (g_sched+gobuf_lr)(g) 501 MOVD $0, (g_sched+gobuf_ret)(g) 502 MOVD $0, (g_sched+gobuf_ctxt)(g) 503 RET 504 505 // func asmcgocall(fn, arg unsafe.Pointer) int32 506 // Call fn(arg) on the scheduler stack, 507 // aligned appropriately for the gcc ABI. 508 // See cgocall.go for more details. 509 TEXT ·asmcgocall(SB),NOSPLIT,$0-20 510 MOVD fn+0(FP), R1 511 MOVD arg+8(FP), R0 512 513 MOVD RSP, R2 // save original stack pointer 514 MOVD g, R4 515 516 // Figure out if we need to switch to m->g0 stack. 517 // We get called to create new OS threads too, and those 518 // come in on the m->g0 stack already. 519 MOVD g_m(g), R8 520 MOVD m_g0(R8), R3 521 CMP R3, g 522 BEQ g0 523 MOVD R0, R9 // gosave<> and save_g might clobber R0 524 BL gosave<>(SB) 525 MOVD R3, g 526 BL runtime·save_g(SB) 527 MOVD (g_sched+gobuf_sp)(g), R0 528 MOVD R0, RSP 529 MOVD R9, R0 530 531 // Now on a scheduling stack (a pthread-created stack). 532 g0: 533 // Save room for two of our pointers /*, plus 32 bytes of callee 534 // save area that lives on the caller stack. */ 535 MOVD RSP, R13 536 SUB $16, R13 537 MOVD R13, RSP 538 MOVD R4, 0(RSP) // save old g on stack 539 MOVD (g_stack+stack_hi)(R4), R4 540 SUB R2, R4 541 MOVD R4, 8(RSP) // save depth in old g stack (can't just save SP, as stack might be copied during a callback) 542 BL (R1) 543 MOVD R0, R9 544 545 // Restore g, stack pointer. R0 is errno, so don't touch it 546 MOVD 0(RSP), g 547 BL runtime·save_g(SB) 548 MOVD (g_stack+stack_hi)(g), R5 549 MOVD 8(RSP), R6 550 SUB R6, R5 551 MOVD R9, R0 552 MOVD R5, RSP 553 554 MOVW R0, ret+16(FP) 555 RET 556 557 // cgocallback(void (*fn)(void*), void *frame, uintptr framesize) 558 // Turn the fn into a Go func (by taking its address) and call 559 // cgocallback_gofunc. 560 TEXT runtime·cgocallback(SB),NOSPLIT,$24-24 561 MOVD $fn+0(FP), R0 562 MOVD R0, 8(RSP) 563 MOVD frame+8(FP), R0 564 MOVD R0, 16(RSP) 565 MOVD framesize+16(FP), R0 566 MOVD R0, 24(RSP) 567 MOVD $runtime·cgocallback_gofunc(SB), R0 568 BL (R0) 569 RET 570 571 // cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize) 572 // See cgocall.go for more details. 573 TEXT ·cgocallback_gofunc(SB),NOSPLIT,$24-24 574 NO_LOCAL_POINTERS 575 576 // Load g from thread-local storage. 577 MOVB runtime·iscgo(SB), R3 578 CMP $0, R3 579 BEQ nocgo 580 BL runtime·load_g(SB) 581 nocgo: 582 583 // If g is nil, Go did not create the current thread. 584 // Call needm to obtain one for temporary use. 585 // In this case, we're running on the thread stack, so there's 586 // lots of space, but the linker doesn't know. Hide the call from 587 // the linker analysis by using an indirect call. 588 CMP $0, g 589 BEQ needm 590 591 MOVD g_m(g), R8 592 MOVD R8, savedm-8(SP) 593 B havem 594 595 needm: 596 MOVD g, savedm-8(SP) // g is zero, so is m. 597 MOVD $runtime·needm(SB), R0 598 BL (R0) 599 600 // Set m->sched.sp = SP, so that if a panic happens 601 // during the function we are about to execute, it will 602 // have a valid SP to run on the g0 stack. 603 // The next few lines (after the havem label) 604 // will save this SP onto the stack and then write 605 // the same SP back to m->sched.sp. That seems redundant, 606 // but if an unrecovered panic happens, unwindm will 607 // restore the g->sched.sp from the stack location 608 // and then systemstack will try to use it. If we don't set it here, 609 // that restored SP will be uninitialized (typically 0) and 610 // will not be usable. 611 MOVD g_m(g), R8 612 MOVD m_g0(R8), R3 613 MOVD RSP, R0 614 MOVD R0, (g_sched+gobuf_sp)(R3) 615 616 havem: 617 // Now there's a valid m, and we're running on its m->g0. 618 // Save current m->g0->sched.sp on stack and then set it to SP. 619 // Save current sp in m->g0->sched.sp in preparation for 620 // switch back to m->curg stack. 621 // NOTE: unwindm knows that the saved g->sched.sp is at 16(RSP) aka savedsp-16(SP). 622 // Beware that the frame size is actually 32. 623 MOVD m_g0(R8), R3 624 MOVD (g_sched+gobuf_sp)(R3), R4 625 MOVD R4, savedsp-16(SP) 626 MOVD RSP, R0 627 MOVD R0, (g_sched+gobuf_sp)(R3) 628 629 // Switch to m->curg stack and call runtime.cgocallbackg. 630 // Because we are taking over the execution of m->curg 631 // but *not* resuming what had been running, we need to 632 // save that information (m->curg->sched) so we can restore it. 633 // We can restore m->curg->sched.sp easily, because calling 634 // runtime.cgocallbackg leaves SP unchanged upon return. 635 // To save m->curg->sched.pc, we push it onto the stack. 636 // This has the added benefit that it looks to the traceback 637 // routine like cgocallbackg is going to return to that 638 // PC (because the frame we allocate below has the same 639 // size as cgocallback_gofunc's frame declared above) 640 // so that the traceback will seamlessly trace back into 641 // the earlier calls. 642 // 643 // In the new goroutine, -16(SP) and -8(SP) are unused. 644 MOVD m_curg(R8), g 645 BL runtime·save_g(SB) 646 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4 647 MOVD (g_sched+gobuf_pc)(g), R5 648 MOVD R5, -(24+8)(R4) // maintain 16-byte SP alignment 649 MOVD $-(24+8)(R4), R0 650 MOVD R0, RSP 651 BL runtime·cgocallbackg(SB) 652 653 // Restore g->sched (== m->curg->sched) from saved values. 654 MOVD 0(RSP), R5 655 MOVD R5, (g_sched+gobuf_pc)(g) 656 MOVD RSP, R4 657 ADD $(24+8), R4, R4 658 MOVD R4, (g_sched+gobuf_sp)(g) 659 660 // Switch back to m->g0's stack and restore m->g0->sched.sp. 661 // (Unlike m->curg, the g0 goroutine never uses sched.pc, 662 // so we do not have to restore it.) 663 MOVD g_m(g), R8 664 MOVD m_g0(R8), g 665 BL runtime·save_g(SB) 666 MOVD (g_sched+gobuf_sp)(g), R0 667 MOVD R0, RSP 668 MOVD savedsp-16(SP), R4 669 MOVD R4, (g_sched+gobuf_sp)(g) 670 671 // If the m on entry was nil, we called needm above to borrow an m 672 // for the duration of the call. Since the call is over, return it with dropm. 673 MOVD savedm-8(SP), R6 674 CMP $0, R6 675 BNE droppedm 676 MOVD $runtime·dropm(SB), R0 677 BL (R0) 678 droppedm: 679 680 // Done! 681 RET 682 683 // Called from cgo wrappers, this function returns g->m->curg.stack.hi. 684 // Must obey the gcc calling convention. 685 TEXT _cgo_topofstack(SB),NOSPLIT,$24 686 // g (R28) and REGTMP (R27) might be clobbered by load_g. They 687 // are callee-save in the gcc calling convention, so save them. 688 MOVD R27, savedR27-8(SP) 689 MOVD g, saveG-16(SP) 690 691 BL runtime·load_g(SB) 692 MOVD g_m(g), R0 693 MOVD m_curg(R0), R0 694 MOVD (g_stack+stack_hi)(R0), R0 695 696 MOVD saveG-16(SP), g 697 MOVD savedR28-8(SP), R27 698 RET 699 700 // void setg(G*); set g. for use by needm. 701 TEXT runtime·setg(SB), NOSPLIT, $0-8 702 MOVD gg+0(FP), g 703 // This only happens if iscgo, so jump straight to save_g 704 BL runtime·save_g(SB) 705 RET 706 707 // void setg_gcc(G*); set g called from gcc 708 TEXT setg_gcc<>(SB),NOSPLIT,$8 709 MOVD R0, g 710 MOVD R27, savedR27-8(SP) 711 BL runtime·save_g(SB) 712 MOVD savedR27-8(SP), R27 713 RET 714 715 TEXT runtime·getcallerpc(SB),NOSPLIT,$8-16 716 MOVD 16(RSP), R0 // LR saved by caller 717 MOVD runtime·stackBarrierPC(SB), R1 718 CMP R0, R1 719 BNE nobar 720 // Get original return PC. 721 BL runtime·nextBarrierPC(SB) 722 MOVD 8(RSP), R0 723 nobar: 724 MOVD R0, ret+8(FP) 725 RET 726 727 TEXT runtime·setcallerpc(SB),NOSPLIT,$8-16 728 MOVD pc+8(FP), R0 729 MOVD 16(RSP), R1 730 MOVD runtime·stackBarrierPC(SB), R2 731 CMP R1, R2 732 BEQ setbar 733 MOVD R0, 16(RSP) // set LR in caller 734 RET 735 setbar: 736 // Set the stack barrier return PC. 737 MOVD R0, 8(RSP) 738 BL runtime·setNextBarrierPC(SB) 739 RET 740 741 TEXT runtime·getcallersp(SB),NOSPLIT,$0-16 742 MOVD argp+0(FP), R0 743 SUB $8, R0 744 MOVD R0, ret+8(FP) 745 RET 746 747 TEXT runtime·abort(SB),NOSPLIT,$-8-0 748 B (ZR) 749 UNDEF 750 751 // memhash_varlen(p unsafe.Pointer, h seed) uintptr 752 // redirects to memhash(p, h, size) using the size 753 // stored in the closure. 754 TEXT runtime·memhash_varlen(SB),NOSPLIT,$40-24 755 GO_ARGS 756 NO_LOCAL_POINTERS 757 MOVD p+0(FP), R3 758 MOVD h+8(FP), R4 759 MOVD 8(R26), R5 760 MOVD R3, 8(RSP) 761 MOVD R4, 16(RSP) 762 MOVD R5, 24(RSP) 763 BL runtime·memhash(SB) 764 MOVD 32(RSP), R3 765 MOVD R3, ret+16(FP) 766 RET 767 768 // memequal(p, q unsafe.Pointer, size uintptr) bool 769 TEXT runtime·memequal(SB),NOSPLIT,$-8-25 770 MOVD a+0(FP), R1 771 MOVD b+8(FP), R2 772 MOVD size+16(FP), R3 773 ADD R1, R3, R6 774 MOVD $1, R0 775 MOVB R0, ret+24(FP) 776 CMP R1, R2 777 BEQ done 778 loop: 779 CMP R1, R6 780 BEQ done 781 MOVBU.P 1(R1), R4 782 MOVBU.P 1(R2), R5 783 CMP R4, R5 784 BEQ loop 785 786 MOVB $0, ret+24(FP) 787 done: 788 RET 789 790 // memequal_varlen(a, b unsafe.Pointer) bool 791 TEXT runtime·memequal_varlen(SB),NOSPLIT,$40-17 792 MOVD a+0(FP), R3 793 MOVD b+8(FP), R4 794 CMP R3, R4 795 BEQ eq 796 MOVD 8(R26), R5 // compiler stores size at offset 8 in the closure 797 MOVD R3, 8(RSP) 798 MOVD R4, 16(RSP) 799 MOVD R5, 24(RSP) 800 BL runtime·memequal(SB) 801 MOVBU 32(RSP), R3 802 MOVB R3, ret+16(FP) 803 RET 804 eq: 805 MOVD $1, R3 806 MOVB R3, ret+16(FP) 807 RET 808 809 TEXT runtime·cmpstring(SB),NOSPLIT,$-4-40 810 MOVD s1_base+0(FP), R2 811 MOVD s1_len+8(FP), R0 812 MOVD s2_base+16(FP), R3 813 MOVD s2_len+24(FP), R1 814 ADD $40, RSP, R7 815 B runtime·cmpbody<>(SB) 816 817 TEXT bytes·Compare(SB),NOSPLIT,$-4-56 818 MOVD s1+0(FP), R2 819 MOVD s1+8(FP), R0 820 MOVD s2+24(FP), R3 821 MOVD s2+32(FP), R1 822 ADD $56, RSP, R7 823 B runtime·cmpbody<>(SB) 824 825 // On entry: 826 // R0 is the length of s1 827 // R1 is the length of s2 828 // R2 points to the start of s1 829 // R3 points to the start of s2 830 // R7 points to return value (-1/0/1 will be written here) 831 // 832 // On exit: 833 // R4, R5, and R6 are clobbered 834 TEXT runtime·cmpbody<>(SB),NOSPLIT,$-4-0 835 CMP R2, R3 836 BEQ samebytes // same starting pointers; compare lengths 837 CMP R0, R1 838 CSEL LT, R1, R0, R6 // R6 is min(R0, R1) 839 840 ADD R2, R6 // R2 is current byte in s1, R6 is last byte in s1 to compare 841 loop: 842 CMP R2, R6 843 BEQ samebytes // all compared bytes were the same; compare lengths 844 MOVBU.P 1(R2), R4 845 MOVBU.P 1(R3), R5 846 CMP R4, R5 847 BEQ loop 848 // bytes differed 849 MOVD $1, R4 850 CSNEG LT, R4, R4, R4 851 MOVD R4, (R7) 852 RET 853 samebytes: 854 MOVD $1, R4 855 CMP R0, R1 856 CSNEG LT, R4, R4, R4 857 CSEL EQ, ZR, R4, R4 858 MOVD R4, (R7) 859 RET 860 861 // eqstring tests whether two strings are equal. 862 // The compiler guarantees that strings passed 863 // to eqstring have equal length. 864 // See runtime_test.go:eqstring_generic for 865 // equivalent Go code. 866 TEXT runtime·eqstring(SB),NOSPLIT,$0-33 867 MOVD s1str+0(FP), R0 868 MOVD s1len+8(FP), R1 869 MOVD s2str+16(FP), R2 870 ADD R0, R1 // end 871 loop: 872 CMP R0, R1 873 BEQ equal // reaches the end 874 MOVBU.P 1(R0), R4 875 MOVBU.P 1(R2), R5 876 CMP R4, R5 877 BEQ loop 878 notequal: 879 MOVB ZR, ret+32(FP) 880 RET 881 equal: 882 MOVD $1, R0 883 MOVB R0, ret+32(FP) 884 RET 885 886 // 887 // functions for other packages 888 // 889 TEXT bytes·IndexByte(SB),NOSPLIT,$0-40 890 MOVD b+0(FP), R0 891 MOVD b_len+8(FP), R1 892 MOVBU c+24(FP), R2 // byte to find 893 MOVD R0, R4 // store base for later 894 ADD R0, R1 // end 895 loop: 896 CMP R0, R1 897 BEQ notfound 898 MOVBU.P 1(R0), R3 899 CMP R2, R3 900 BNE loop 901 902 SUB $1, R0 // R0 will be one beyond the position we want 903 SUB R4, R0 // remove base 904 MOVD R0, ret+32(FP) 905 RET 906 907 notfound: 908 MOVD $-1, R0 909 MOVD R0, ret+32(FP) 910 RET 911 912 TEXT strings·IndexByte(SB),NOSPLIT,$0-32 913 MOVD s+0(FP), R0 914 MOVD s_len+8(FP), R1 915 MOVBU c+16(FP), R2 // byte to find 916 MOVD R0, R4 // store base for later 917 ADD R0, R1 // end 918 loop: 919 CMP R0, R1 920 BEQ notfound 921 MOVBU.P 1(R0), R3 922 CMP R2, R3 923 BNE loop 924 925 SUB $1, R0 // R0 will be one beyond the position we want 926 SUB R4, R0 // remove base 927 MOVD R0, ret+24(FP) 928 RET 929 930 notfound: 931 MOVD $-1, R0 932 MOVD R0, ret+24(FP) 933 RET 934 935 // TODO: share code with memequal? 936 TEXT bytes·Equal(SB),NOSPLIT,$0-49 937 MOVD a_len+8(FP), R1 938 MOVD b_len+32(FP), R3 939 CMP R1, R3 // unequal lengths are not equal 940 BNE notequal 941 MOVD a+0(FP), R0 942 MOVD b+24(FP), R2 943 ADD R0, R1 // end 944 loop: 945 CMP R0, R1 946 BEQ equal // reaches the end 947 MOVBU.P 1(R0), R4 948 MOVBU.P 1(R2), R5 949 CMP R4, R5 950 BEQ loop 951 notequal: 952 MOVB ZR, ret+48(FP) 953 RET 954 equal: 955 MOVD $1, R0 956 MOVB R0, ret+48(FP) 957 RET 958 959 TEXT runtime·fastrand1(SB),NOSPLIT,$-8-4 960 MOVD g_m(g), R1 961 MOVWU m_fastrand(R1), R0 962 ADD R0, R0 963 CMPW $0, R0 964 BGE notneg 965 EOR $0x88888eef, R0 966 notneg: 967 MOVW R0, m_fastrand(R1) 968 MOVW R0, ret+0(FP) 969 RET 970 971 TEXT runtime·return0(SB), NOSPLIT, $0 972 MOVW $0, R0 973 RET 974 975 // The top-most function running on a goroutine 976 // returns to goexit+PCQuantum. 977 TEXT runtime·goexit(SB),NOSPLIT,$-8-0 978 MOVD R0, R0 // NOP 979 BL runtime·goexit1(SB) // does not return 980 981 // TODO(aram): use PRFM here. 982 TEXT runtime·prefetcht0(SB),NOSPLIT,$0-8 983 RET 984 985 TEXT runtime·prefetcht1(SB),NOSPLIT,$0-8 986 RET 987 988 TEXT runtime·prefetcht2(SB),NOSPLIT,$0-8 989 RET 990 991 TEXT runtime·prefetchnta(SB),NOSPLIT,$0-8 992 RET 993 994 TEXT runtime·sigreturn(SB),NOSPLIT,$0-8 995 RET 996 997 // This is called from .init_array and follows the platform, not Go, ABI. 998 TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0 999 SUB $0x10, RSP 1000 MOVD R27, 8(RSP) // The access to global variables below implicitly uses R27, which is callee-save 1001 MOVD runtime·lastmoduledatap(SB), R1 1002 MOVD R0, moduledata_next(R1) 1003 MOVD R0, runtime·lastmoduledatap(SB) 1004 MOVD 8(RSP), R27 1005 ADD $0x10, RSP 1006 RET 1007 1008 TEXT ·checkASM(SB),NOSPLIT,$0-1 1009 MOVW $1, R3 1010 MOVB R3, ret+0(FP) 1011 RET