github.com/karrick/go@v0.0.0-20170817181416-d5b0ec858b37/src/runtime/asm_arm64.s (about) 1 // Copyright 2015 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 #include "go_asm.h" 6 #include "go_tls.h" 7 #include "tls_arm64.h" 8 #include "funcdata.h" 9 #include "textflag.h" 10 11 TEXT runtime·rt0_go(SB),NOSPLIT,$0 12 // SP = stack; R0 = argc; R1 = argv 13 14 SUB $32, RSP 15 MOVW R0, 8(RSP) // argc 16 MOVD R1, 16(RSP) // argv 17 18 // create istack out of the given (operating system) stack. 19 // _cgo_init may update stackguard. 20 MOVD $runtime·g0(SB), g 21 MOVD RSP, R7 22 MOVD $(-64*1024)(R7), R0 23 MOVD R0, g_stackguard0(g) 24 MOVD R0, g_stackguard1(g) 25 MOVD R0, (g_stack+stack_lo)(g) 26 MOVD R7, (g_stack+stack_hi)(g) 27 28 // if there is a _cgo_init, call it using the gcc ABI. 29 MOVD _cgo_init(SB), R12 30 CMP $0, R12 31 BEQ nocgo 32 33 MRS_TPIDR_R0 // load TLS base pointer 34 MOVD R0, R3 // arg 3: TLS base pointer 35 #ifdef TLSG_IS_VARIABLE 36 MOVD $runtime·tls_g(SB), R2 // arg 2: &tls_g 37 #else 38 MOVD $0, R2 // arg 2: not used when using platform's TLS 39 #endif 40 MOVD $setg_gcc<>(SB), R1 // arg 1: setg 41 MOVD g, R0 // arg 0: G 42 BL (R12) 43 MOVD _cgo_init(SB), R12 44 CMP $0, R12 45 BEQ nocgo 46 47 nocgo: 48 // update stackguard after _cgo_init 49 MOVD (g_stack+stack_lo)(g), R0 50 ADD $const__StackGuard, R0 51 MOVD R0, g_stackguard0(g) 52 MOVD R0, g_stackguard1(g) 53 54 // set the per-goroutine and per-mach "registers" 55 MOVD $runtime·m0(SB), R0 56 57 // save m->g0 = g0 58 MOVD g, m_g0(R0) 59 // save m0 to g0->m 60 MOVD R0, g_m(g) 61 62 BL runtime·check(SB) 63 64 MOVW 8(RSP), R0 // copy argc 65 MOVW R0, -8(RSP) 66 MOVD 16(RSP), R0 // copy argv 67 MOVD R0, 0(RSP) 68 BL runtime·args(SB) 69 BL runtime·osinit(SB) 70 BL runtime·schedinit(SB) 71 72 // create a new goroutine to start program 73 MOVD $runtime·mainPC(SB), R0 // entry 74 MOVD RSP, R7 75 MOVD.W $0, -8(R7) 76 MOVD.W R0, -8(R7) 77 MOVD.W $0, -8(R7) 78 MOVD.W $0, -8(R7) 79 MOVD R7, RSP 80 BL runtime·newproc(SB) 81 ADD $32, RSP 82 83 // start this M 84 BL runtime·mstart(SB) 85 86 MOVD $0, R0 87 MOVD R0, (R0) // boom 88 UNDEF 89 90 DATA runtime·mainPC+0(SB)/8,$runtime·main(SB) 91 GLOBL runtime·mainPC(SB),RODATA,$8 92 93 TEXT runtime·breakpoint(SB),NOSPLIT,$-8-0 94 BRK 95 RET 96 97 TEXT runtime·asminit(SB),NOSPLIT,$-8-0 98 RET 99 100 /* 101 * go-routine 102 */ 103 104 // void gosave(Gobuf*) 105 // save state in Gobuf; setjmp 106 TEXT runtime·gosave(SB), NOSPLIT, $-8-8 107 MOVD buf+0(FP), R3 108 MOVD RSP, R0 109 MOVD R0, gobuf_sp(R3) 110 MOVD LR, gobuf_pc(R3) 111 MOVD g, gobuf_g(R3) 112 MOVD ZR, gobuf_lr(R3) 113 MOVD ZR, gobuf_ret(R3) 114 // Assert ctxt is zero. See func save. 115 MOVD gobuf_ctxt(R3), R0 116 CMP $0, R0 117 BEQ 2(PC) 118 CALL runtime·badctxt(SB) 119 RET 120 121 // void gogo(Gobuf*) 122 // restore state from Gobuf; longjmp 123 TEXT runtime·gogo(SB), NOSPLIT, $24-8 124 MOVD buf+0(FP), R5 125 126 // If ctxt is not nil, invoke deletion barrier before overwriting. 127 MOVD gobuf_ctxt(R5), R0 128 CMP $0, R0 129 BEQ nilctxt 130 MOVD $gobuf_ctxt(R5), R0 131 MOVD R0, 8(RSP) 132 MOVD ZR, 16(RSP) 133 BL runtime·writebarrierptr_prewrite(SB) 134 MOVD buf+0(FP), R5 135 136 nilctxt: 137 MOVD gobuf_g(R5), g 138 BL runtime·save_g(SB) 139 140 MOVD 0(g), R4 // make sure g is not nil 141 MOVD gobuf_sp(R5), R0 142 MOVD R0, RSP 143 MOVD gobuf_lr(R5), LR 144 MOVD gobuf_ret(R5), R0 145 MOVD gobuf_ctxt(R5), R26 146 MOVD $0, gobuf_sp(R5) 147 MOVD $0, gobuf_ret(R5) 148 MOVD $0, gobuf_lr(R5) 149 MOVD $0, gobuf_ctxt(R5) 150 CMP ZR, ZR // set condition codes for == test, needed by stack split 151 MOVD gobuf_pc(R5), R6 152 B (R6) 153 154 // void mcall(fn func(*g)) 155 // Switch to m->g0's stack, call fn(g). 156 // Fn must never return. It should gogo(&g->sched) 157 // to keep running g. 158 TEXT runtime·mcall(SB), NOSPLIT, $-8-8 159 // Save caller state in g->sched 160 MOVD RSP, R0 161 MOVD R0, (g_sched+gobuf_sp)(g) 162 MOVD LR, (g_sched+gobuf_pc)(g) 163 MOVD $0, (g_sched+gobuf_lr)(g) 164 MOVD g, (g_sched+gobuf_g)(g) 165 166 // Switch to m->g0 & its stack, call fn. 167 MOVD g, R3 168 MOVD g_m(g), R8 169 MOVD m_g0(R8), g 170 BL runtime·save_g(SB) 171 CMP g, R3 172 BNE 2(PC) 173 B runtime·badmcall(SB) 174 MOVD fn+0(FP), R26 // context 175 MOVD 0(R26), R4 // code pointer 176 MOVD (g_sched+gobuf_sp)(g), R0 177 MOVD R0, RSP // sp = m->g0->sched.sp 178 MOVD R3, -8(RSP) 179 MOVD $0, -16(RSP) 180 SUB $16, RSP 181 BL (R4) 182 B runtime·badmcall2(SB) 183 184 // systemstack_switch is a dummy routine that systemstack leaves at the bottom 185 // of the G stack. We need to distinguish the routine that 186 // lives at the bottom of the G stack from the one that lives 187 // at the top of the system stack because the one at the top of 188 // the system stack terminates the stack walk (see topofstack()). 189 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0 190 UNDEF 191 BL (LR) // make sure this function is not leaf 192 RET 193 194 // func systemstack(fn func()) 195 TEXT runtime·systemstack(SB), NOSPLIT, $0-8 196 MOVD fn+0(FP), R3 // R3 = fn 197 MOVD R3, R26 // context 198 MOVD g_m(g), R4 // R4 = m 199 200 MOVD m_gsignal(R4), R5 // R5 = gsignal 201 CMP g, R5 202 BEQ noswitch 203 204 MOVD m_g0(R4), R5 // R5 = g0 205 CMP g, R5 206 BEQ noswitch 207 208 MOVD m_curg(R4), R6 209 CMP g, R6 210 BEQ switch 211 212 // Bad: g is not gsignal, not g0, not curg. What is it? 213 // Hide call from linker nosplit analysis. 214 MOVD $runtime·badsystemstack(SB), R3 215 BL (R3) 216 217 switch: 218 // save our state in g->sched. Pretend to 219 // be systemstack_switch if the G stack is scanned. 220 MOVD $runtime·systemstack_switch(SB), R6 221 ADD $8, R6 // get past prologue 222 MOVD R6, (g_sched+gobuf_pc)(g) 223 MOVD RSP, R0 224 MOVD R0, (g_sched+gobuf_sp)(g) 225 MOVD $0, (g_sched+gobuf_lr)(g) 226 MOVD g, (g_sched+gobuf_g)(g) 227 228 // switch to g0 229 MOVD R5, g 230 BL runtime·save_g(SB) 231 MOVD (g_sched+gobuf_sp)(g), R3 232 // make it look like mstart called systemstack on g0, to stop traceback 233 SUB $16, R3 234 AND $~15, R3 235 MOVD $runtime·mstart(SB), R4 236 MOVD R4, 0(R3) 237 MOVD R3, RSP 238 239 // call target function 240 MOVD 0(R26), R3 // code pointer 241 BL (R3) 242 243 // switch back to g 244 MOVD g_m(g), R3 245 MOVD m_curg(R3), g 246 BL runtime·save_g(SB) 247 MOVD (g_sched+gobuf_sp)(g), R0 248 MOVD R0, RSP 249 MOVD $0, (g_sched+gobuf_sp)(g) 250 RET 251 252 noswitch: 253 // already on m stack, just call directly 254 MOVD 0(R26), R3 // code pointer 255 BL (R3) 256 RET 257 258 /* 259 * support for morestack 260 */ 261 262 // Called during function prolog when more stack is needed. 263 // Caller has already loaded: 264 // R3 prolog's LR (R30) 265 // 266 // The traceback routines see morestack on a g0 as being 267 // the top of a stack (for example, morestack calling newstack 268 // calling the scheduler calling newm calling gc), so we must 269 // record an argument size. For that purpose, it has no arguments. 270 TEXT runtime·morestack(SB),NOSPLIT,$-8-0 271 // Cannot grow scheduler stack (m->g0). 272 MOVD g_m(g), R8 273 MOVD m_g0(R8), R4 274 CMP g, R4 275 BNE 3(PC) 276 BL runtime·badmorestackg0(SB) 277 B runtime·abort(SB) 278 279 // Cannot grow signal stack (m->gsignal). 280 MOVD m_gsignal(R8), R4 281 CMP g, R4 282 BNE 3(PC) 283 BL runtime·badmorestackgsignal(SB) 284 B runtime·abort(SB) 285 286 // Called from f. 287 // Set g->sched to context in f 288 MOVD RSP, R0 289 MOVD R0, (g_sched+gobuf_sp)(g) 290 MOVD LR, (g_sched+gobuf_pc)(g) 291 MOVD R3, (g_sched+gobuf_lr)(g) 292 // newstack will fill gobuf.ctxt. 293 294 // Called from f. 295 // Set m->morebuf to f's callers. 296 MOVD R3, (m_morebuf+gobuf_pc)(R8) // f's caller's PC 297 MOVD RSP, R0 298 MOVD R0, (m_morebuf+gobuf_sp)(R8) // f's caller's RSP 299 MOVD g, (m_morebuf+gobuf_g)(R8) 300 301 // Call newstack on m->g0's stack. 302 MOVD m_g0(R8), g 303 BL runtime·save_g(SB) 304 MOVD (g_sched+gobuf_sp)(g), R0 305 MOVD R0, RSP 306 MOVD.W $0, -16(RSP) // create a call frame on g0 307 MOVD R26, 8(RSP) // ctxt argument 308 BL runtime·newstack(SB) 309 310 // Not reached, but make sure the return PC from the call to newstack 311 // is still in this function, and not the beginning of the next. 312 UNDEF 313 314 TEXT runtime·morestack_noctxt(SB),NOSPLIT,$-4-0 315 MOVW $0, R26 316 B runtime·morestack(SB) 317 318 // reflectcall: call a function with the given argument list 319 // func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32). 320 // we don't have variable-sized frames, so we use a small number 321 // of constant-sized-frame functions to encode a few bits of size in the pc. 322 // Caution: ugly multiline assembly macros in your future! 323 324 #define DISPATCH(NAME,MAXSIZE) \ 325 MOVD $MAXSIZE, R27; \ 326 CMP R27, R16; \ 327 BGT 3(PC); \ 328 MOVD $NAME(SB), R27; \ 329 B (R27) 330 // Note: can't just "B NAME(SB)" - bad inlining results. 331 332 TEXT reflect·call(SB), NOSPLIT, $0-0 333 B ·reflectcall(SB) 334 335 TEXT ·reflectcall(SB), NOSPLIT, $-8-32 336 MOVWU argsize+24(FP), R16 337 DISPATCH(runtime·call32, 32) 338 DISPATCH(runtime·call64, 64) 339 DISPATCH(runtime·call128, 128) 340 DISPATCH(runtime·call256, 256) 341 DISPATCH(runtime·call512, 512) 342 DISPATCH(runtime·call1024, 1024) 343 DISPATCH(runtime·call2048, 2048) 344 DISPATCH(runtime·call4096, 4096) 345 DISPATCH(runtime·call8192, 8192) 346 DISPATCH(runtime·call16384, 16384) 347 DISPATCH(runtime·call32768, 32768) 348 DISPATCH(runtime·call65536, 65536) 349 DISPATCH(runtime·call131072, 131072) 350 DISPATCH(runtime·call262144, 262144) 351 DISPATCH(runtime·call524288, 524288) 352 DISPATCH(runtime·call1048576, 1048576) 353 DISPATCH(runtime·call2097152, 2097152) 354 DISPATCH(runtime·call4194304, 4194304) 355 DISPATCH(runtime·call8388608, 8388608) 356 DISPATCH(runtime·call16777216, 16777216) 357 DISPATCH(runtime·call33554432, 33554432) 358 DISPATCH(runtime·call67108864, 67108864) 359 DISPATCH(runtime·call134217728, 134217728) 360 DISPATCH(runtime·call268435456, 268435456) 361 DISPATCH(runtime·call536870912, 536870912) 362 DISPATCH(runtime·call1073741824, 1073741824) 363 MOVD $runtime·badreflectcall(SB), R0 364 B (R0) 365 366 #define CALLFN(NAME,MAXSIZE) \ 367 TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \ 368 NO_LOCAL_POINTERS; \ 369 /* copy arguments to stack */ \ 370 MOVD arg+16(FP), R3; \ 371 MOVWU argsize+24(FP), R4; \ 372 MOVD RSP, R5; \ 373 ADD $(8-1), R5; \ 374 SUB $1, R3; \ 375 ADD R5, R4; \ 376 CMP R5, R4; \ 377 BEQ 4(PC); \ 378 MOVBU.W 1(R3), R6; \ 379 MOVBU.W R6, 1(R5); \ 380 B -4(PC); \ 381 /* call function */ \ 382 MOVD f+8(FP), R26; \ 383 MOVD (R26), R0; \ 384 PCDATA $PCDATA_StackMapIndex, $0; \ 385 BL (R0); \ 386 /* copy return values back */ \ 387 MOVD argtype+0(FP), R7; \ 388 MOVD arg+16(FP), R3; \ 389 MOVWU n+24(FP), R4; \ 390 MOVWU retoffset+28(FP), R6; \ 391 ADD $8, RSP, R5; \ 392 ADD R6, R5; \ 393 ADD R6, R3; \ 394 SUB R6, R4; \ 395 BL callRet<>(SB); \ 396 RET 397 398 // callRet copies return values back at the end of call*. This is a 399 // separate function so it can allocate stack space for the arguments 400 // to reflectcallmove. It does not follow the Go ABI; it expects its 401 // arguments in registers. 402 TEXT callRet<>(SB), NOSPLIT, $40-0 403 MOVD R7, 8(RSP) 404 MOVD R3, 16(RSP) 405 MOVD R5, 24(RSP) 406 MOVD R4, 32(RSP) 407 BL runtime·reflectcallmove(SB) 408 RET 409 410 // These have 8 added to make the overall frame size a multiple of 16, 411 // as required by the ABI. (There is another +8 for the saved LR.) 412 CALLFN(·call32, 40 ) 413 CALLFN(·call64, 72 ) 414 CALLFN(·call128, 136 ) 415 CALLFN(·call256, 264 ) 416 CALLFN(·call512, 520 ) 417 CALLFN(·call1024, 1032 ) 418 CALLFN(·call2048, 2056 ) 419 CALLFN(·call4096, 4104 ) 420 CALLFN(·call8192, 8200 ) 421 CALLFN(·call16384, 16392 ) 422 CALLFN(·call32768, 32776 ) 423 CALLFN(·call65536, 65544 ) 424 CALLFN(·call131072, 131080 ) 425 CALLFN(·call262144, 262152 ) 426 CALLFN(·call524288, 524296 ) 427 CALLFN(·call1048576, 1048584 ) 428 CALLFN(·call2097152, 2097160 ) 429 CALLFN(·call4194304, 4194312 ) 430 CALLFN(·call8388608, 8388616 ) 431 CALLFN(·call16777216, 16777224 ) 432 CALLFN(·call33554432, 33554440 ) 433 CALLFN(·call67108864, 67108872 ) 434 CALLFN(·call134217728, 134217736 ) 435 CALLFN(·call268435456, 268435464 ) 436 CALLFN(·call536870912, 536870920 ) 437 CALLFN(·call1073741824, 1073741832 ) 438 439 // AES hashing not implemented for ARM64, issue #10109. 440 TEXT runtime·aeshash(SB),NOSPLIT,$-8-0 441 MOVW $0, R0 442 MOVW (R0), R1 443 TEXT runtime·aeshash32(SB),NOSPLIT,$-8-0 444 MOVW $0, R0 445 MOVW (R0), R1 446 TEXT runtime·aeshash64(SB),NOSPLIT,$-8-0 447 MOVW $0, R0 448 MOVW (R0), R1 449 TEXT runtime·aeshashstr(SB),NOSPLIT,$-8-0 450 MOVW $0, R0 451 MOVW (R0), R1 452 453 TEXT runtime·procyield(SB),NOSPLIT,$0-0 454 MOVWU cycles+0(FP), R0 455 again: 456 YIELD 457 SUBW $1, R0 458 CBNZ R0, again 459 RET 460 461 // void jmpdefer(fv, sp); 462 // called from deferreturn. 463 // 1. grab stored LR for caller 464 // 2. sub 4 bytes to get back to BL deferreturn 465 // 3. BR to fn 466 TEXT runtime·jmpdefer(SB), NOSPLIT, $-8-16 467 MOVD 0(RSP), R0 468 SUB $4, R0 469 MOVD R0, LR 470 471 MOVD fv+0(FP), R26 472 MOVD argp+8(FP), R0 473 MOVD R0, RSP 474 SUB $8, RSP 475 MOVD 0(R26), R3 476 B (R3) 477 478 // Save state of caller into g->sched. Smashes R0. 479 TEXT gosave<>(SB),NOSPLIT,$-8 480 MOVD LR, (g_sched+gobuf_pc)(g) 481 MOVD RSP, R0 482 MOVD R0, (g_sched+gobuf_sp)(g) 483 MOVD $0, (g_sched+gobuf_lr)(g) 484 MOVD $0, (g_sched+gobuf_ret)(g) 485 // Assert ctxt is zero. See func save. 486 MOVD (g_sched+gobuf_ctxt)(g), R0 487 CMP $0, R0 488 BEQ 2(PC) 489 CALL runtime·badctxt(SB) 490 RET 491 492 // func asmcgocall(fn, arg unsafe.Pointer) int32 493 // Call fn(arg) on the scheduler stack, 494 // aligned appropriately for the gcc ABI. 495 // See cgocall.go for more details. 496 TEXT ·asmcgocall(SB),NOSPLIT,$0-20 497 MOVD fn+0(FP), R1 498 MOVD arg+8(FP), R0 499 500 MOVD RSP, R2 // save original stack pointer 501 MOVD g, R4 502 503 // Figure out if we need to switch to m->g0 stack. 504 // We get called to create new OS threads too, and those 505 // come in on the m->g0 stack already. 506 MOVD g_m(g), R8 507 MOVD m_g0(R8), R3 508 CMP R3, g 509 BEQ g0 510 MOVD R0, R9 // gosave<> and save_g might clobber R0 511 BL gosave<>(SB) 512 MOVD R3, g 513 BL runtime·save_g(SB) 514 MOVD (g_sched+gobuf_sp)(g), R0 515 MOVD R0, RSP 516 MOVD R9, R0 517 518 // Now on a scheduling stack (a pthread-created stack). 519 g0: 520 // Save room for two of our pointers /*, plus 32 bytes of callee 521 // save area that lives on the caller stack. */ 522 MOVD RSP, R13 523 SUB $16, R13 524 MOVD R13, RSP 525 MOVD R4, 0(RSP) // save old g on stack 526 MOVD (g_stack+stack_hi)(R4), R4 527 SUB R2, R4 528 MOVD R4, 8(RSP) // save depth in old g stack (can't just save SP, as stack might be copied during a callback) 529 BL (R1) 530 MOVD R0, R9 531 532 // Restore g, stack pointer. R0 is errno, so don't touch it 533 MOVD 0(RSP), g 534 BL runtime·save_g(SB) 535 MOVD (g_stack+stack_hi)(g), R5 536 MOVD 8(RSP), R6 537 SUB R6, R5 538 MOVD R9, R0 539 MOVD R5, RSP 540 541 MOVW R0, ret+16(FP) 542 RET 543 544 // cgocallback(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt) 545 // Turn the fn into a Go func (by taking its address) and call 546 // cgocallback_gofunc. 547 TEXT runtime·cgocallback(SB),NOSPLIT,$40-32 548 MOVD $fn+0(FP), R0 549 MOVD R0, 8(RSP) 550 MOVD frame+8(FP), R0 551 MOVD R0, 16(RSP) 552 MOVD framesize+16(FP), R0 553 MOVD R0, 24(RSP) 554 MOVD ctxt+24(FP), R0 555 MOVD R0, 32(RSP) 556 MOVD $runtime·cgocallback_gofunc(SB), R0 557 BL (R0) 558 RET 559 560 // cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize, uintptr ctxt) 561 // See cgocall.go for more details. 562 TEXT ·cgocallback_gofunc(SB),NOSPLIT,$24-32 563 NO_LOCAL_POINTERS 564 565 // Load g from thread-local storage. 566 MOVB runtime·iscgo(SB), R3 567 CMP $0, R3 568 BEQ nocgo 569 BL runtime·load_g(SB) 570 nocgo: 571 572 // If g is nil, Go did not create the current thread. 573 // Call needm to obtain one for temporary use. 574 // In this case, we're running on the thread stack, so there's 575 // lots of space, but the linker doesn't know. Hide the call from 576 // the linker analysis by using an indirect call. 577 CMP $0, g 578 BEQ needm 579 580 MOVD g_m(g), R8 581 MOVD R8, savedm-8(SP) 582 B havem 583 584 needm: 585 MOVD g, savedm-8(SP) // g is zero, so is m. 586 MOVD $runtime·needm(SB), R0 587 BL (R0) 588 589 // Set m->sched.sp = SP, so that if a panic happens 590 // during the function we are about to execute, it will 591 // have a valid SP to run on the g0 stack. 592 // The next few lines (after the havem label) 593 // will save this SP onto the stack and then write 594 // the same SP back to m->sched.sp. That seems redundant, 595 // but if an unrecovered panic happens, unwindm will 596 // restore the g->sched.sp from the stack location 597 // and then systemstack will try to use it. If we don't set it here, 598 // that restored SP will be uninitialized (typically 0) and 599 // will not be usable. 600 MOVD g_m(g), R8 601 MOVD m_g0(R8), R3 602 MOVD RSP, R0 603 MOVD R0, (g_sched+gobuf_sp)(R3) 604 605 havem: 606 // Now there's a valid m, and we're running on its m->g0. 607 // Save current m->g0->sched.sp on stack and then set it to SP. 608 // Save current sp in m->g0->sched.sp in preparation for 609 // switch back to m->curg stack. 610 // NOTE: unwindm knows that the saved g->sched.sp is at 16(RSP) aka savedsp-16(SP). 611 // Beware that the frame size is actually 32. 612 MOVD m_g0(R8), R3 613 MOVD (g_sched+gobuf_sp)(R3), R4 614 MOVD R4, savedsp-16(SP) 615 MOVD RSP, R0 616 MOVD R0, (g_sched+gobuf_sp)(R3) 617 618 // Switch to m->curg stack and call runtime.cgocallbackg. 619 // Because we are taking over the execution of m->curg 620 // but *not* resuming what had been running, we need to 621 // save that information (m->curg->sched) so we can restore it. 622 // We can restore m->curg->sched.sp easily, because calling 623 // runtime.cgocallbackg leaves SP unchanged upon return. 624 // To save m->curg->sched.pc, we push it onto the stack. 625 // This has the added benefit that it looks to the traceback 626 // routine like cgocallbackg is going to return to that 627 // PC (because the frame we allocate below has the same 628 // size as cgocallback_gofunc's frame declared above) 629 // so that the traceback will seamlessly trace back into 630 // the earlier calls. 631 // 632 // In the new goroutine, -8(SP) is unused (where SP refers to 633 // m->curg's SP while we're setting it up, before we've adjusted it). 634 MOVD m_curg(R8), g 635 BL runtime·save_g(SB) 636 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4 637 MOVD (g_sched+gobuf_pc)(g), R5 638 MOVD R5, -(24+8)(R4) 639 MOVD ctxt+24(FP), R0 640 MOVD R0, -(16+8)(R4) 641 MOVD $-(24+8)(R4), R0 // maintain 16-byte SP alignment 642 MOVD R0, RSP 643 BL runtime·cgocallbackg(SB) 644 645 // Restore g->sched (== m->curg->sched) from saved values. 646 MOVD 0(RSP), R5 647 MOVD R5, (g_sched+gobuf_pc)(g) 648 MOVD RSP, R4 649 ADD $(24+8), R4, R4 650 MOVD R4, (g_sched+gobuf_sp)(g) 651 652 // Switch back to m->g0's stack and restore m->g0->sched.sp. 653 // (Unlike m->curg, the g0 goroutine never uses sched.pc, 654 // so we do not have to restore it.) 655 MOVD g_m(g), R8 656 MOVD m_g0(R8), g 657 BL runtime·save_g(SB) 658 MOVD (g_sched+gobuf_sp)(g), R0 659 MOVD R0, RSP 660 MOVD savedsp-16(SP), R4 661 MOVD R4, (g_sched+gobuf_sp)(g) 662 663 // If the m on entry was nil, we called needm above to borrow an m 664 // for the duration of the call. Since the call is over, return it with dropm. 665 MOVD savedm-8(SP), R6 666 CMP $0, R6 667 BNE droppedm 668 MOVD $runtime·dropm(SB), R0 669 BL (R0) 670 droppedm: 671 672 // Done! 673 RET 674 675 // Called from cgo wrappers, this function returns g->m->curg.stack.hi. 676 // Must obey the gcc calling convention. 677 TEXT _cgo_topofstack(SB),NOSPLIT,$24 678 // g (R28) and REGTMP (R27) might be clobbered by load_g. They 679 // are callee-save in the gcc calling convention, so save them. 680 MOVD R27, savedR27-8(SP) 681 MOVD g, saveG-16(SP) 682 683 BL runtime·load_g(SB) 684 MOVD g_m(g), R0 685 MOVD m_curg(R0), R0 686 MOVD (g_stack+stack_hi)(R0), R0 687 688 MOVD saveG-16(SP), g 689 MOVD savedR28-8(SP), R27 690 RET 691 692 // void setg(G*); set g. for use by needm. 693 TEXT runtime·setg(SB), NOSPLIT, $0-8 694 MOVD gg+0(FP), g 695 // This only happens if iscgo, so jump straight to save_g 696 BL runtime·save_g(SB) 697 RET 698 699 // void setg_gcc(G*); set g called from gcc 700 TEXT setg_gcc<>(SB),NOSPLIT,$8 701 MOVD R0, g 702 MOVD R27, savedR27-8(SP) 703 BL runtime·save_g(SB) 704 MOVD savedR27-8(SP), R27 705 RET 706 707 TEXT runtime·getcallerpc(SB),NOSPLIT,$8-16 708 MOVD 16(RSP), R0 // LR saved by caller 709 MOVD R0, ret+8(FP) 710 RET 711 712 TEXT runtime·abort(SB),NOSPLIT,$-8-0 713 B (ZR) 714 UNDEF 715 716 // memequal(p, q unsafe.Pointer, size uintptr) bool 717 TEXT runtime·memequal(SB),NOSPLIT,$-8-25 718 MOVD a+0(FP), R1 719 MOVD b+8(FP), R2 720 MOVD size+16(FP), R3 721 ADD R1, R3, R6 722 MOVD $1, R0 723 MOVB R0, ret+24(FP) 724 CMP R1, R2 725 BEQ done 726 loop: 727 CMP R1, R6 728 BEQ done 729 MOVBU.P 1(R1), R4 730 MOVBU.P 1(R2), R5 731 CMP R4, R5 732 BEQ loop 733 734 MOVB $0, ret+24(FP) 735 done: 736 RET 737 738 // memequal_varlen(a, b unsafe.Pointer) bool 739 TEXT runtime·memequal_varlen(SB),NOSPLIT,$40-17 740 MOVD a+0(FP), R3 741 MOVD b+8(FP), R4 742 CMP R3, R4 743 BEQ eq 744 MOVD 8(R26), R5 // compiler stores size at offset 8 in the closure 745 MOVD R3, 8(RSP) 746 MOVD R4, 16(RSP) 747 MOVD R5, 24(RSP) 748 BL runtime·memequal(SB) 749 MOVBU 32(RSP), R3 750 MOVB R3, ret+16(FP) 751 RET 752 eq: 753 MOVD $1, R3 754 MOVB R3, ret+16(FP) 755 RET 756 757 TEXT runtime·cmpstring(SB),NOSPLIT,$-4-40 758 MOVD s1_base+0(FP), R2 759 MOVD s1_len+8(FP), R0 760 MOVD s2_base+16(FP), R3 761 MOVD s2_len+24(FP), R1 762 ADD $40, RSP, R7 763 B runtime·cmpbody<>(SB) 764 765 TEXT bytes·Compare(SB),NOSPLIT,$-4-56 766 MOVD s1+0(FP), R2 767 MOVD s1+8(FP), R0 768 MOVD s2+24(FP), R3 769 MOVD s2+32(FP), R1 770 ADD $56, RSP, R7 771 B runtime·cmpbody<>(SB) 772 773 // On entry: 774 // R0 is the length of s1 775 // R1 is the length of s2 776 // R2 points to the start of s1 777 // R3 points to the start of s2 778 // R7 points to return value (-1/0/1 will be written here) 779 // 780 // On exit: 781 // R4, R5, and R6 are clobbered 782 TEXT runtime·cmpbody<>(SB),NOSPLIT,$-4-0 783 CMP R2, R3 784 BEQ samebytes // same starting pointers; compare lengths 785 CMP R0, R1 786 CSEL LT, R1, R0, R6 // R6 is min(R0, R1) 787 788 ADD R2, R6 // R2 is current byte in s1, R6 is last byte in s1 to compare 789 loop: 790 CMP R2, R6 791 BEQ samebytes // all compared bytes were the same; compare lengths 792 MOVBU.P 1(R2), R4 793 MOVBU.P 1(R3), R5 794 CMP R4, R5 795 BEQ loop 796 // bytes differed 797 MOVD $1, R4 798 CSNEG LT, R4, R4, R4 799 MOVD R4, (R7) 800 RET 801 samebytes: 802 MOVD $1, R4 803 CMP R0, R1 804 CSNEG LT, R4, R4, R4 805 CSEL EQ, ZR, R4, R4 806 MOVD R4, (R7) 807 RET 808 809 // eqstring tests whether two strings are equal. 810 // The compiler guarantees that strings passed 811 // to eqstring have equal length. 812 // See runtime_test.go:eqstring_generic for 813 // equivalent Go code. 814 TEXT runtime·eqstring(SB),NOSPLIT,$0-33 815 MOVD s1_base+0(FP), R0 816 MOVD s1_len+8(FP), R1 817 MOVD s2_base+16(FP), R2 818 ADD R0, R1 // end 819 loop: 820 CMP R0, R1 821 BEQ equal // reaches the end 822 MOVBU.P 1(R0), R4 823 MOVBU.P 1(R2), R5 824 CMP R4, R5 825 BEQ loop 826 notequal: 827 MOVB ZR, ret+32(FP) 828 RET 829 equal: 830 MOVD $1, R0 831 MOVB R0, ret+32(FP) 832 RET 833 834 // 835 // functions for other packages 836 // 837 TEXT bytes·IndexByte(SB),NOSPLIT,$0-40 838 MOVD b+0(FP), R0 839 MOVD b_len+8(FP), R1 840 MOVBU c+24(FP), R2 // byte to find 841 MOVD R0, R4 // store base for later 842 ADD R0, R1 // end 843 loop: 844 CMP R0, R1 845 BEQ notfound 846 MOVBU.P 1(R0), R3 847 CMP R2, R3 848 BNE loop 849 850 SUB $1, R0 // R0 will be one beyond the position we want 851 SUB R4, R0 // remove base 852 MOVD R0, ret+32(FP) 853 RET 854 855 notfound: 856 MOVD $-1, R0 857 MOVD R0, ret+32(FP) 858 RET 859 860 TEXT strings·IndexByte(SB),NOSPLIT,$0-32 861 MOVD s+0(FP), R0 862 MOVD s_len+8(FP), R1 863 MOVBU c+16(FP), R2 // byte to find 864 MOVD R0, R4 // store base for later 865 ADD R0, R1 // end 866 loop: 867 CMP R0, R1 868 BEQ notfound 869 MOVBU.P 1(R0), R3 870 CMP R2, R3 871 BNE loop 872 873 SUB $1, R0 // R0 will be one beyond the position we want 874 SUB R4, R0 // remove base 875 MOVD R0, ret+24(FP) 876 RET 877 878 notfound: 879 MOVD $-1, R0 880 MOVD R0, ret+24(FP) 881 RET 882 883 // TODO: share code with memequal? 884 TEXT bytes·Equal(SB),NOSPLIT,$0-49 885 MOVD a_len+8(FP), R1 886 MOVD b_len+32(FP), R3 887 CMP R1, R3 // unequal lengths are not equal 888 BNE notequal 889 MOVD a+0(FP), R0 890 MOVD b+24(FP), R2 891 ADD R0, R1 // end 892 loop: 893 CMP R0, R1 894 BEQ equal // reaches the end 895 MOVBU.P 1(R0), R4 896 MOVBU.P 1(R2), R5 897 CMP R4, R5 898 BEQ loop 899 notequal: 900 MOVB ZR, ret+48(FP) 901 RET 902 equal: 903 MOVD $1, R0 904 MOVB R0, ret+48(FP) 905 RET 906 907 TEXT runtime·return0(SB), NOSPLIT, $0 908 MOVW $0, R0 909 RET 910 911 // The top-most function running on a goroutine 912 // returns to goexit+PCQuantum. 913 TEXT runtime·goexit(SB),NOSPLIT,$-8-0 914 MOVD R0, R0 // NOP 915 BL runtime·goexit1(SB) // does not return 916 917 TEXT runtime·sigreturn(SB),NOSPLIT,$0-0 918 RET 919 920 // This is called from .init_array and follows the platform, not Go, ABI. 921 TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0 922 SUB $0x10, RSP 923 MOVD R27, 8(RSP) // The access to global variables below implicitly uses R27, which is callee-save 924 MOVD runtime·lastmoduledatap(SB), R1 925 MOVD R0, moduledata_next(R1) 926 MOVD R0, runtime·lastmoduledatap(SB) 927 MOVD 8(RSP), R27 928 ADD $0x10, RSP 929 RET 930 931 TEXT ·checkASM(SB),NOSPLIT,$0-1 932 MOVW $1, R3 933 MOVB R3, ret+0(FP) 934 RET