github.com/MangoDowner/go-gm@v0.0.0-20180818020936-8baa2bd4408c/src/runtime/asm_arm64.s (about) 1 // Copyright 2015 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 #include "go_asm.h" 6 #include "go_tls.h" 7 #include "tls_arm64.h" 8 #include "funcdata.h" 9 #include "textflag.h" 10 11 TEXT runtime·rt0_go(SB),NOSPLIT,$0 12 // SP = stack; R0 = argc; R1 = argv 13 14 SUB $32, RSP 15 MOVW R0, 8(RSP) // argc 16 MOVD R1, 16(RSP) // argv 17 18 // create istack out of the given (operating system) stack. 19 // _cgo_init may update stackguard. 20 MOVD $runtime·g0(SB), g 21 MOVD RSP, R7 22 MOVD $(-64*1024)(R7), R0 23 MOVD R0, g_stackguard0(g) 24 MOVD R0, g_stackguard1(g) 25 MOVD R0, (g_stack+stack_lo)(g) 26 MOVD R7, (g_stack+stack_hi)(g) 27 28 // if there is a _cgo_init, call it using the gcc ABI. 29 MOVD _cgo_init(SB), R12 30 CMP $0, R12 31 BEQ nocgo 32 33 MRS_TPIDR_R0 // load TLS base pointer 34 MOVD R0, R3 // arg 3: TLS base pointer 35 #ifdef TLSG_IS_VARIABLE 36 MOVD $runtime·tls_g(SB), R2 // arg 2: &tls_g 37 #else 38 MOVD $0, R2 // arg 2: not used when using platform's TLS 39 #endif 40 MOVD $setg_gcc<>(SB), R1 // arg 1: setg 41 MOVD g, R0 // arg 0: G 42 BL (R12) 43 MOVD _cgo_init(SB), R12 44 CMP $0, R12 45 BEQ nocgo 46 47 nocgo: 48 // update stackguard after _cgo_init 49 MOVD (g_stack+stack_lo)(g), R0 50 ADD $const__StackGuard, R0 51 MOVD R0, g_stackguard0(g) 52 MOVD R0, g_stackguard1(g) 53 54 // set the per-goroutine and per-mach "registers" 55 MOVD $runtime·m0(SB), R0 56 57 // save m->g0 = g0 58 MOVD g, m_g0(R0) 59 // save m0 to g0->m 60 MOVD R0, g_m(g) 61 62 BL runtime·check(SB) 63 64 MOVW 8(RSP), R0 // copy argc 65 MOVW R0, -8(RSP) 66 MOVD 16(RSP), R0 // copy argv 67 MOVD R0, 0(RSP) 68 BL runtime·args(SB) 69 BL runtime·osinit(SB) 70 BL runtime·schedinit(SB) 71 72 // create a new goroutine to start program 73 MOVD $runtime·mainPC(SB), R0 // entry 74 MOVD RSP, R7 75 MOVD.W $0, -8(R7) 76 MOVD.W R0, -8(R7) 77 MOVD.W $0, -8(R7) 78 MOVD.W $0, -8(R7) 79 MOVD R7, RSP 80 BL runtime·newproc(SB) 81 ADD $32, RSP 82 83 // start this M 84 BL runtime·mstart(SB) 85 86 MOVD $0, R0 87 MOVD R0, (R0) // boom 88 UNDEF 89 90 DATA runtime·mainPC+0(SB)/8,$runtime·main(SB) 91 GLOBL runtime·mainPC(SB),RODATA,$8 92 93 TEXT runtime·breakpoint(SB),NOSPLIT,$-8-0 94 BRK 95 RET 96 97 TEXT runtime·asminit(SB),NOSPLIT,$-8-0 98 RET 99 100 /* 101 * go-routine 102 */ 103 104 // void gosave(Gobuf*) 105 // save state in Gobuf; setjmp 106 TEXT runtime·gosave(SB), NOSPLIT, $-8-8 107 MOVD buf+0(FP), R3 108 MOVD RSP, R0 109 MOVD R0, gobuf_sp(R3) 110 MOVD LR, gobuf_pc(R3) 111 MOVD g, gobuf_g(R3) 112 MOVD ZR, gobuf_lr(R3) 113 MOVD ZR, gobuf_ret(R3) 114 // Assert ctxt is zero. See func save. 115 MOVD gobuf_ctxt(R3), R0 116 CMP $0, R0 117 BEQ 2(PC) 118 CALL runtime·badctxt(SB) 119 RET 120 121 // void gogo(Gobuf*) 122 // restore state from Gobuf; longjmp 123 TEXT runtime·gogo(SB), NOSPLIT, $24-8 124 MOVD buf+0(FP), R5 125 126 // If ctxt is not nil, invoke deletion barrier before overwriting. 127 MOVD gobuf_ctxt(R5), R0 128 CMP $0, R0 129 BEQ nilctxt 130 MOVD $gobuf_ctxt(R5), R0 131 MOVD R0, 8(RSP) 132 MOVD ZR, 16(RSP) 133 BL runtime·writebarrierptr_prewrite(SB) 134 MOVD buf+0(FP), R5 135 136 nilctxt: 137 MOVD gobuf_g(R5), g 138 BL runtime·save_g(SB) 139 140 MOVD 0(g), R4 // make sure g is not nil 141 MOVD gobuf_sp(R5), R0 142 MOVD R0, RSP 143 MOVD gobuf_lr(R5), LR 144 MOVD gobuf_ret(R5), R0 145 MOVD gobuf_ctxt(R5), R26 146 MOVD $0, gobuf_sp(R5) 147 MOVD $0, gobuf_ret(R5) 148 MOVD $0, gobuf_lr(R5) 149 MOVD $0, gobuf_ctxt(R5) 150 CMP ZR, ZR // set condition codes for == test, needed by stack split 151 MOVD gobuf_pc(R5), R6 152 B (R6) 153 154 // void mcall(fn func(*g)) 155 // Switch to m->g0's stack, call fn(g). 156 // Fn must never return. It should gogo(&g->sched) 157 // to keep running g. 158 TEXT runtime·mcall(SB), NOSPLIT, $-8-8 159 // Save caller state in g->sched 160 MOVD RSP, R0 161 MOVD R0, (g_sched+gobuf_sp)(g) 162 MOVD LR, (g_sched+gobuf_pc)(g) 163 MOVD $0, (g_sched+gobuf_lr)(g) 164 MOVD g, (g_sched+gobuf_g)(g) 165 166 // Switch to m->g0 & its stack, call fn. 167 MOVD g, R3 168 MOVD g_m(g), R8 169 MOVD m_g0(R8), g 170 BL runtime·save_g(SB) 171 CMP g, R3 172 BNE 2(PC) 173 B runtime·badmcall(SB) 174 MOVD fn+0(FP), R26 // context 175 MOVD 0(R26), R4 // code pointer 176 MOVD (g_sched+gobuf_sp)(g), R0 177 MOVD R0, RSP // sp = m->g0->sched.sp 178 MOVD R3, -8(RSP) 179 MOVD $0, -16(RSP) 180 SUB $16, RSP 181 BL (R4) 182 B runtime·badmcall2(SB) 183 184 // systemstack_switch is a dummy routine that systemstack leaves at the bottom 185 // of the G stack. We need to distinguish the routine that 186 // lives at the bottom of the G stack from the one that lives 187 // at the top of the system stack because the one at the top of 188 // the system stack terminates the stack walk (see topofstack()). 189 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0 190 UNDEF 191 BL (LR) // make sure this function is not leaf 192 RET 193 194 // func systemstack(fn func()) 195 TEXT runtime·systemstack(SB), NOSPLIT, $0-8 196 MOVD fn+0(FP), R3 // R3 = fn 197 MOVD R3, R26 // context 198 MOVD g_m(g), R4 // R4 = m 199 200 MOVD m_gsignal(R4), R5 // R5 = gsignal 201 CMP g, R5 202 BEQ noswitch 203 204 MOVD m_g0(R4), R5 // R5 = g0 205 CMP g, R5 206 BEQ noswitch 207 208 MOVD m_curg(R4), R6 209 CMP g, R6 210 BEQ switch 211 212 // Bad: g is not gsignal, not g0, not curg. What is it? 213 // Hide call from linker nosplit analysis. 214 MOVD $runtime·badsystemstack(SB), R3 215 BL (R3) 216 217 switch: 218 // save our state in g->sched. Pretend to 219 // be systemstack_switch if the G stack is scanned. 220 MOVD $runtime·systemstack_switch(SB), R6 221 ADD $8, R6 // get past prologue 222 MOVD R6, (g_sched+gobuf_pc)(g) 223 MOVD RSP, R0 224 MOVD R0, (g_sched+gobuf_sp)(g) 225 MOVD $0, (g_sched+gobuf_lr)(g) 226 MOVD g, (g_sched+gobuf_g)(g) 227 228 // switch to g0 229 MOVD R5, g 230 BL runtime·save_g(SB) 231 MOVD (g_sched+gobuf_sp)(g), R3 232 // make it look like mstart called systemstack on g0, to stop traceback 233 SUB $16, R3 234 AND $~15, R3 235 MOVD $runtime·mstart(SB), R4 236 MOVD R4, 0(R3) 237 MOVD R3, RSP 238 239 // call target function 240 MOVD 0(R26), R3 // code pointer 241 BL (R3) 242 243 // switch back to g 244 MOVD g_m(g), R3 245 MOVD m_curg(R3), g 246 BL runtime·save_g(SB) 247 MOVD (g_sched+gobuf_sp)(g), R0 248 MOVD R0, RSP 249 MOVD $0, (g_sched+gobuf_sp)(g) 250 RET 251 252 noswitch: 253 // already on m stack, just call directly 254 MOVD 0(R26), R3 // code pointer 255 BL (R3) 256 RET 257 258 /* 259 * support for morestack 260 */ 261 262 // Called during function prolog when more stack is needed. 263 // Caller has already loaded: 264 // R3 prolog's LR (R30) 265 // 266 // The traceback routines see morestack on a g0 as being 267 // the top of a stack (for example, morestack calling newstack 268 // calling the scheduler calling newm calling gc), so we must 269 // record an argument size. For that purpose, it has no arguments. 270 TEXT runtime·morestack(SB),NOSPLIT,$-8-0 271 // Cannot grow scheduler stack (m->g0). 272 MOVD g_m(g), R8 273 MOVD m_g0(R8), R4 274 CMP g, R4 275 BNE 3(PC) 276 BL runtime·badmorestackg0(SB) 277 B runtime·abort(SB) 278 279 // Cannot grow signal stack (m->gsignal). 280 MOVD m_gsignal(R8), R4 281 CMP g, R4 282 BNE 3(PC) 283 BL runtime·badmorestackgsignal(SB) 284 B runtime·abort(SB) 285 286 // Called from f. 287 // Set g->sched to context in f 288 MOVD RSP, R0 289 MOVD R0, (g_sched+gobuf_sp)(g) 290 MOVD LR, (g_sched+gobuf_pc)(g) 291 MOVD R3, (g_sched+gobuf_lr)(g) 292 // newstack will fill gobuf.ctxt. 293 294 // Called from f. 295 // Set m->morebuf to f's callers. 296 MOVD R3, (m_morebuf+gobuf_pc)(R8) // f's caller's PC 297 MOVD RSP, R0 298 MOVD R0, (m_morebuf+gobuf_sp)(R8) // f's caller's RSP 299 MOVD g, (m_morebuf+gobuf_g)(R8) 300 301 // Call newstack on m->g0's stack. 302 MOVD m_g0(R8), g 303 BL runtime·save_g(SB) 304 MOVD (g_sched+gobuf_sp)(g), R0 305 MOVD R0, RSP 306 MOVD.W $0, -16(RSP) // create a call frame on g0 307 MOVD R26, 8(RSP) // ctxt argument 308 BL runtime·newstack(SB) 309 310 // Not reached, but make sure the return PC from the call to newstack 311 // is still in this function, and not the beginning of the next. 312 UNDEF 313 314 TEXT runtime·morestack_noctxt(SB),NOSPLIT,$-4-0 315 MOVW $0, R26 316 B runtime·morestack(SB) 317 318 // reflectcall: call a function with the given argument list 319 // func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32). 320 // we don't have variable-sized frames, so we use a small number 321 // of constant-sized-frame functions to encode a few bits of size in the pc. 322 // Caution: ugly multiline assembly macros in your future! 323 324 #define DISPATCH(NAME,MAXSIZE) \ 325 MOVD $MAXSIZE, R27; \ 326 CMP R27, R16; \ 327 BGT 3(PC); \ 328 MOVD $NAME(SB), R27; \ 329 B (R27) 330 // Note: can't just "B NAME(SB)" - bad inlining results. 331 332 TEXT reflect·call(SB), NOSPLIT, $0-0 333 B ·reflectcall(SB) 334 335 TEXT ·reflectcall(SB), NOSPLIT, $-8-32 336 MOVWU argsize+24(FP), R16 337 DISPATCH(runtime·call32, 32) 338 DISPATCH(runtime·call64, 64) 339 DISPATCH(runtime·call128, 128) 340 DISPATCH(runtime·call256, 256) 341 DISPATCH(runtime·call512, 512) 342 DISPATCH(runtime·call1024, 1024) 343 DISPATCH(runtime·call2048, 2048) 344 DISPATCH(runtime·call4096, 4096) 345 DISPATCH(runtime·call8192, 8192) 346 DISPATCH(runtime·call16384, 16384) 347 DISPATCH(runtime·call32768, 32768) 348 DISPATCH(runtime·call65536, 65536) 349 DISPATCH(runtime·call131072, 131072) 350 DISPATCH(runtime·call262144, 262144) 351 DISPATCH(runtime·call524288, 524288) 352 DISPATCH(runtime·call1048576, 1048576) 353 DISPATCH(runtime·call2097152, 2097152) 354 DISPATCH(runtime·call4194304, 4194304) 355 DISPATCH(runtime·call8388608, 8388608) 356 DISPATCH(runtime·call16777216, 16777216) 357 DISPATCH(runtime·call33554432, 33554432) 358 DISPATCH(runtime·call67108864, 67108864) 359 DISPATCH(runtime·call134217728, 134217728) 360 DISPATCH(runtime·call268435456, 268435456) 361 DISPATCH(runtime·call536870912, 536870912) 362 DISPATCH(runtime·call1073741824, 1073741824) 363 MOVD $runtime·badreflectcall(SB), R0 364 B (R0) 365 366 #define CALLFN(NAME,MAXSIZE) \ 367 TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \ 368 NO_LOCAL_POINTERS; \ 369 /* copy arguments to stack */ \ 370 MOVD arg+16(FP), R3; \ 371 MOVWU argsize+24(FP), R4; \ 372 MOVD RSP, R5; \ 373 ADD $(8-1), R5; \ 374 SUB $1, R3; \ 375 ADD R5, R4; \ 376 CMP R5, R4; \ 377 BEQ 4(PC); \ 378 MOVBU.W 1(R3), R6; \ 379 MOVBU.W R6, 1(R5); \ 380 B -4(PC); \ 381 /* call function */ \ 382 MOVD f+8(FP), R26; \ 383 MOVD (R26), R0; \ 384 PCDATA $PCDATA_StackMapIndex, $0; \ 385 BL (R0); \ 386 /* copy return values back */ \ 387 MOVD argtype+0(FP), R7; \ 388 MOVD arg+16(FP), R3; \ 389 MOVWU n+24(FP), R4; \ 390 MOVWU retoffset+28(FP), R6; \ 391 ADD $8, RSP, R5; \ 392 ADD R6, R5; \ 393 ADD R6, R3; \ 394 SUB R6, R4; \ 395 BL callRet<>(SB); \ 396 RET 397 398 // callRet copies return values back at the end of call*. This is a 399 // separate function so it can allocate stack space for the arguments 400 // to reflectcallmove. It does not follow the Go ABI; it expects its 401 // arguments in registers. 402 TEXT callRet<>(SB), NOSPLIT, $40-0 403 MOVD R7, 8(RSP) 404 MOVD R3, 16(RSP) 405 MOVD R5, 24(RSP) 406 MOVD R4, 32(RSP) 407 BL runtime·reflectcallmove(SB) 408 RET 409 410 // These have 8 added to make the overall frame size a multiple of 16, 411 // as required by the ABI. (There is another +8 for the saved LR.) 412 CALLFN(·call32, 40 ) 413 CALLFN(·call64, 72 ) 414 CALLFN(·call128, 136 ) 415 CALLFN(·call256, 264 ) 416 CALLFN(·call512, 520 ) 417 CALLFN(·call1024, 1032 ) 418 CALLFN(·call2048, 2056 ) 419 CALLFN(·call4096, 4104 ) 420 CALLFN(·call8192, 8200 ) 421 CALLFN(·call16384, 16392 ) 422 CALLFN(·call32768, 32776 ) 423 CALLFN(·call65536, 65544 ) 424 CALLFN(·call131072, 131080 ) 425 CALLFN(·call262144, 262152 ) 426 CALLFN(·call524288, 524296 ) 427 CALLFN(·call1048576, 1048584 ) 428 CALLFN(·call2097152, 2097160 ) 429 CALLFN(·call4194304, 4194312 ) 430 CALLFN(·call8388608, 8388616 ) 431 CALLFN(·call16777216, 16777224 ) 432 CALLFN(·call33554432, 33554440 ) 433 CALLFN(·call67108864, 67108872 ) 434 CALLFN(·call134217728, 134217736 ) 435 CALLFN(·call268435456, 268435464 ) 436 CALLFN(·call536870912, 536870920 ) 437 CALLFN(·call1073741824, 1073741832 ) 438 439 // AES hashing not implemented for ARM64, issue #10109. 440 TEXT runtime·aeshash(SB),NOSPLIT,$-8-0 441 MOVW $0, R0 442 MOVW (R0), R1 443 TEXT runtime·aeshash32(SB),NOSPLIT,$-8-0 444 MOVW $0, R0 445 MOVW (R0), R1 446 TEXT runtime·aeshash64(SB),NOSPLIT,$-8-0 447 MOVW $0, R0 448 MOVW (R0), R1 449 TEXT runtime·aeshashstr(SB),NOSPLIT,$-8-0 450 MOVW $0, R0 451 MOVW (R0), R1 452 453 TEXT runtime·procyield(SB),NOSPLIT,$0-0 454 MOVWU cycles+0(FP), R0 455 again: 456 YIELD 457 SUBW $1, R0 458 CBNZ R0, again 459 RET 460 461 // void jmpdefer(fv, sp); 462 // called from deferreturn. 463 // 1. grab stored LR for caller 464 // 2. sub 4 bytes to get back to BL deferreturn 465 // 3. BR to fn 466 TEXT runtime·jmpdefer(SB), NOSPLIT, $-8-16 467 MOVD 0(RSP), R0 468 SUB $4, R0 469 MOVD R0, LR 470 471 MOVD fv+0(FP), R26 472 MOVD argp+8(FP), R0 473 MOVD R0, RSP 474 SUB $8, RSP 475 MOVD 0(R26), R3 476 B (R3) 477 478 // Save state of caller into g->sched. Smashes R0. 479 TEXT gosave<>(SB),NOSPLIT,$-8 480 MOVD LR, (g_sched+gobuf_pc)(g) 481 MOVD RSP, R0 482 MOVD R0, (g_sched+gobuf_sp)(g) 483 MOVD $0, (g_sched+gobuf_lr)(g) 484 MOVD $0, (g_sched+gobuf_ret)(g) 485 // Assert ctxt is zero. See func save. 486 MOVD (g_sched+gobuf_ctxt)(g), R0 487 CMP $0, R0 488 BEQ 2(PC) 489 CALL runtime·badctxt(SB) 490 RET 491 492 // func asmcgocall(fn, arg unsafe.Pointer) int32 493 // Call fn(arg) on the scheduler stack, 494 // aligned appropriately for the gcc ABI. 495 // See cgocall.go for more details. 496 TEXT ·asmcgocall(SB),NOSPLIT,$0-20 497 MOVD fn+0(FP), R1 498 MOVD arg+8(FP), R0 499 500 MOVD RSP, R2 // save original stack pointer 501 MOVD g, R4 502 503 // Figure out if we need to switch to m->g0 stack. 504 // We get called to create new OS threads too, and those 505 // come in on the m->g0 stack already. 506 MOVD g_m(g), R8 507 MOVD m_g0(R8), R3 508 CMP R3, g 509 BEQ g0 510 MOVD R0, R9 // gosave<> and save_g might clobber R0 511 BL gosave<>(SB) 512 MOVD R3, g 513 BL runtime·save_g(SB) 514 MOVD (g_sched+gobuf_sp)(g), R0 515 MOVD R0, RSP 516 MOVD R9, R0 517 518 // Now on a scheduling stack (a pthread-created stack). 519 g0: 520 // Save room for two of our pointers /*, plus 32 bytes of callee 521 // save area that lives on the caller stack. */ 522 MOVD RSP, R13 523 SUB $16, R13 524 MOVD R13, RSP 525 MOVD R4, 0(RSP) // save old g on stack 526 MOVD (g_stack+stack_hi)(R4), R4 527 SUB R2, R4 528 MOVD R4, 8(RSP) // save depth in old g stack (can't just save SP, as stack might be copied during a callback) 529 BL (R1) 530 MOVD R0, R9 531 532 // Restore g, stack pointer. R0 is errno, so don't touch it 533 MOVD 0(RSP), g 534 BL runtime·save_g(SB) 535 MOVD (g_stack+stack_hi)(g), R5 536 MOVD 8(RSP), R6 537 SUB R6, R5 538 MOVD R9, R0 539 MOVD R5, RSP 540 541 MOVW R0, ret+16(FP) 542 RET 543 544 // cgocallback(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt) 545 // Turn the fn into a Go func (by taking its address) and call 546 // cgocallback_gofunc. 547 TEXT runtime·cgocallback(SB),NOSPLIT,$40-32 548 MOVD $fn+0(FP), R0 549 MOVD R0, 8(RSP) 550 MOVD frame+8(FP), R0 551 MOVD R0, 16(RSP) 552 MOVD framesize+16(FP), R0 553 MOVD R0, 24(RSP) 554 MOVD ctxt+24(FP), R0 555 MOVD R0, 32(RSP) 556 MOVD $runtime·cgocallback_gofunc(SB), R0 557 BL (R0) 558 RET 559 560 // cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize, uintptr ctxt) 561 // See cgocall.go for more details. 562 TEXT ·cgocallback_gofunc(SB),NOSPLIT,$24-32 563 NO_LOCAL_POINTERS 564 565 // Load g from thread-local storage. 566 MOVB runtime·iscgo(SB), R3 567 CMP $0, R3 568 BEQ nocgo 569 BL runtime·load_g(SB) 570 nocgo: 571 572 // If g is nil, Go did not create the current thread. 573 // Call needm to obtain one for temporary use. 574 // In this case, we're running on the thread stack, so there's 575 // lots of space, but the linker doesn't know. Hide the call from 576 // the linker analysis by using an indirect call. 577 CMP $0, g 578 BEQ needm 579 580 MOVD g_m(g), R8 581 MOVD R8, savedm-8(SP) 582 B havem 583 584 needm: 585 MOVD g, savedm-8(SP) // g is zero, so is m. 586 MOVD $runtime·needm(SB), R0 587 BL (R0) 588 589 // Set m->sched.sp = SP, so that if a panic happens 590 // during the function we are about to execute, it will 591 // have a valid SP to run on the g0 stack. 592 // The next few lines (after the havem label) 593 // will save this SP onto the stack and then write 594 // the same SP back to m->sched.sp. That seems redundant, 595 // but if an unrecovered panic happens, unwindm will 596 // restore the g->sched.sp from the stack location 597 // and then systemstack will try to use it. If we don't set it here, 598 // that restored SP will be uninitialized (typically 0) and 599 // will not be usable. 600 MOVD g_m(g), R8 601 MOVD m_g0(R8), R3 602 MOVD RSP, R0 603 MOVD R0, (g_sched+gobuf_sp)(R3) 604 605 havem: 606 // Now there's a valid m, and we're running on its m->g0. 607 // Save current m->g0->sched.sp on stack and then set it to SP. 608 // Save current sp in m->g0->sched.sp in preparation for 609 // switch back to m->curg stack. 610 // NOTE: unwindm knows that the saved g->sched.sp is at 16(RSP) aka savedsp-16(SP). 611 // Beware that the frame size is actually 32. 612 MOVD m_g0(R8), R3 613 MOVD (g_sched+gobuf_sp)(R3), R4 614 MOVD R4, savedsp-16(SP) 615 MOVD RSP, R0 616 MOVD R0, (g_sched+gobuf_sp)(R3) 617 618 // Switch to m->curg stack and call runtime.cgocallbackg. 619 // Because we are taking over the execution of m->curg 620 // but *not* resuming what had been running, we need to 621 // save that information (m->curg->sched) so we can restore it. 622 // We can restore m->curg->sched.sp easily, because calling 623 // runtime.cgocallbackg leaves SP unchanged upon return. 624 // To save m->curg->sched.pc, we push it onto the stack. 625 // This has the added benefit that it looks to the traceback 626 // routine like cgocallbackg is going to return to that 627 // PC (because the frame we allocate below has the same 628 // size as cgocallback_gofunc's frame declared above) 629 // so that the traceback will seamlessly trace back into 630 // the earlier calls. 631 // 632 // In the new goroutine, -8(SP) is unused (where SP refers to 633 // m->curg's SP while we're setting it up, before we've adjusted it). 634 MOVD m_curg(R8), g 635 BL runtime·save_g(SB) 636 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4 637 MOVD (g_sched+gobuf_pc)(g), R5 638 MOVD R5, -(24+8)(R4) 639 MOVD ctxt+24(FP), R0 640 MOVD R0, -(16+8)(R4) 641 MOVD $-(24+8)(R4), R0 // maintain 16-byte SP alignment 642 MOVD R0, RSP 643 BL runtime·cgocallbackg(SB) 644 645 // Restore g->sched (== m->curg->sched) from saved values. 646 MOVD 0(RSP), R5 647 MOVD R5, (g_sched+gobuf_pc)(g) 648 MOVD RSP, R4 649 ADD $(24+8), R4, R4 650 MOVD R4, (g_sched+gobuf_sp)(g) 651 652 // Switch back to m->g0's stack and restore m->g0->sched.sp. 653 // (Unlike m->curg, the g0 goroutine never uses sched.pc, 654 // so we do not have to restore it.) 655 MOVD g_m(g), R8 656 MOVD m_g0(R8), g 657 BL runtime·save_g(SB) 658 MOVD (g_sched+gobuf_sp)(g), R0 659 MOVD R0, RSP 660 MOVD savedsp-16(SP), R4 661 MOVD R4, (g_sched+gobuf_sp)(g) 662 663 // If the m on entry was nil, we called needm above to borrow an m 664 // for the duration of the call. Since the call is over, return it with dropm. 665 MOVD savedm-8(SP), R6 666 CMP $0, R6 667 BNE droppedm 668 MOVD $runtime·dropm(SB), R0 669 BL (R0) 670 droppedm: 671 672 // Done! 673 RET 674 675 // Called from cgo wrappers, this function returns g->m->curg.stack.hi. 676 // Must obey the gcc calling convention. 677 TEXT _cgo_topofstack(SB),NOSPLIT,$24 678 // g (R28) and REGTMP (R27) might be clobbered by load_g. They 679 // are callee-save in the gcc calling convention, so save them. 680 MOVD R27, savedR27-8(SP) 681 MOVD g, saveG-16(SP) 682 683 BL runtime·load_g(SB) 684 MOVD g_m(g), R0 685 MOVD m_curg(R0), R0 686 MOVD (g_stack+stack_hi)(R0), R0 687 688 MOVD saveG-16(SP), g 689 MOVD savedR28-8(SP), R27 690 RET 691 692 // void setg(G*); set g. for use by needm. 693 TEXT runtime·setg(SB), NOSPLIT, $0-8 694 MOVD gg+0(FP), g 695 // This only happens if iscgo, so jump straight to save_g 696 BL runtime·save_g(SB) 697 RET 698 699 // void setg_gcc(G*); set g called from gcc 700 TEXT setg_gcc<>(SB),NOSPLIT,$8 701 MOVD R0, g 702 MOVD R27, savedR27-8(SP) 703 BL runtime·save_g(SB) 704 MOVD savedR27-8(SP), R27 705 RET 706 707 TEXT runtime·getcallerpc(SB),NOSPLIT,$8-16 708 MOVD 16(RSP), R0 // LR saved by caller 709 MOVD R0, ret+8(FP) 710 RET 711 712 TEXT runtime·abort(SB),NOSPLIT,$-8-0 713 B (ZR) 714 UNDEF 715 716 // memhash_varlen(p unsafe.Pointer, h seed) uintptr 717 // redirects to memhash(p, h, size) using the size 718 // stored in the closure. 719 TEXT runtime·memhash_varlen(SB),NOSPLIT,$40-24 720 GO_ARGS 721 NO_LOCAL_POINTERS 722 MOVD p+0(FP), R3 723 MOVD h+8(FP), R4 724 MOVD 8(R26), R5 725 MOVD R3, 8(RSP) 726 MOVD R4, 16(RSP) 727 MOVD R5, 24(RSP) 728 BL runtime·memhash(SB) 729 MOVD 32(RSP), R3 730 MOVD R3, ret+16(FP) 731 RET 732 733 // memequal(p, q unsafe.Pointer, size uintptr) bool 734 TEXT runtime·memequal(SB),NOSPLIT,$-8-25 735 MOVD a+0(FP), R1 736 MOVD b+8(FP), R2 737 MOVD size+16(FP), R3 738 ADD R1, R3, R6 739 MOVD $1, R0 740 MOVB R0, ret+24(FP) 741 CMP R1, R2 742 BEQ done 743 loop: 744 CMP R1, R6 745 BEQ done 746 MOVBU.P 1(R1), R4 747 MOVBU.P 1(R2), R5 748 CMP R4, R5 749 BEQ loop 750 751 MOVB $0, ret+24(FP) 752 done: 753 RET 754 755 // memequal_varlen(a, b unsafe.Pointer) bool 756 TEXT runtime·memequal_varlen(SB),NOSPLIT,$40-17 757 MOVD a+0(FP), R3 758 MOVD b+8(FP), R4 759 CMP R3, R4 760 BEQ eq 761 MOVD 8(R26), R5 // compiler stores size at offset 8 in the closure 762 MOVD R3, 8(RSP) 763 MOVD R4, 16(RSP) 764 MOVD R5, 24(RSP) 765 BL runtime·memequal(SB) 766 MOVBU 32(RSP), R3 767 MOVB R3, ret+16(FP) 768 RET 769 eq: 770 MOVD $1, R3 771 MOVB R3, ret+16(FP) 772 RET 773 774 TEXT runtime·cmpstring(SB),NOSPLIT,$-4-40 775 MOVD s1_base+0(FP), R2 776 MOVD s1_len+8(FP), R0 777 MOVD s2_base+16(FP), R3 778 MOVD s2_len+24(FP), R1 779 ADD $40, RSP, R7 780 B runtime·cmpbody<>(SB) 781 782 TEXT bytes·Compare(SB),NOSPLIT,$-4-56 783 MOVD s1+0(FP), R2 784 MOVD s1+8(FP), R0 785 MOVD s2+24(FP), R3 786 MOVD s2+32(FP), R1 787 ADD $56, RSP, R7 788 B runtime·cmpbody<>(SB) 789 790 // On entry: 791 // R0 is the length of s1 792 // R1 is the length of s2 793 // R2 points to the start of s1 794 // R3 points to the start of s2 795 // R7 points to return value (-1/0/1 will be written here) 796 // 797 // On exit: 798 // R4, R5, and R6 are clobbered 799 TEXT runtime·cmpbody<>(SB),NOSPLIT,$-4-0 800 CMP R2, R3 801 BEQ samebytes // same starting pointers; compare lengths 802 CMP R0, R1 803 CSEL LT, R1, R0, R6 // R6 is min(R0, R1) 804 805 ADD R2, R6 // R2 is current byte in s1, R6 is last byte in s1 to compare 806 loop: 807 CMP R2, R6 808 BEQ samebytes // all compared bytes were the same; compare lengths 809 MOVBU.P 1(R2), R4 810 MOVBU.P 1(R3), R5 811 CMP R4, R5 812 BEQ loop 813 // bytes differed 814 MOVD $1, R4 815 CSNEG LT, R4, R4, R4 816 MOVD R4, (R7) 817 RET 818 samebytes: 819 MOVD $1, R4 820 CMP R0, R1 821 CSNEG LT, R4, R4, R4 822 CSEL EQ, ZR, R4, R4 823 MOVD R4, (R7) 824 RET 825 826 // eqstring tests whether two strings are equal. 827 // The compiler guarantees that strings passed 828 // to eqstring have equal length. 829 // See runtime_test.go:eqstring_generic for 830 // equivalent Go code. 831 TEXT runtime·eqstring(SB),NOSPLIT,$0-33 832 MOVD s1_base+0(FP), R0 833 MOVD s1_len+8(FP), R1 834 MOVD s2_base+16(FP), R2 835 ADD R0, R1 // end 836 loop: 837 CMP R0, R1 838 BEQ equal // reaches the end 839 MOVBU.P 1(R0), R4 840 MOVBU.P 1(R2), R5 841 CMP R4, R5 842 BEQ loop 843 notequal: 844 MOVB ZR, ret+32(FP) 845 RET 846 equal: 847 MOVD $1, R0 848 MOVB R0, ret+32(FP) 849 RET 850 851 // 852 // functions for other packages 853 // 854 TEXT bytes·IndexByte(SB),NOSPLIT,$0-40 855 MOVD b+0(FP), R0 856 MOVD b_len+8(FP), R1 857 MOVBU c+24(FP), R2 // byte to find 858 MOVD R0, R4 // store base for later 859 ADD R0, R1 // end 860 loop: 861 CMP R0, R1 862 BEQ notfound 863 MOVBU.P 1(R0), R3 864 CMP R2, R3 865 BNE loop 866 867 SUB $1, R0 // R0 will be one beyond the position we want 868 SUB R4, R0 // remove base 869 MOVD R0, ret+32(FP) 870 RET 871 872 notfound: 873 MOVD $-1, R0 874 MOVD R0, ret+32(FP) 875 RET 876 877 TEXT strings·IndexByte(SB),NOSPLIT,$0-32 878 MOVD s+0(FP), R0 879 MOVD s_len+8(FP), R1 880 MOVBU c+16(FP), R2 // byte to find 881 MOVD R0, R4 // store base for later 882 ADD R0, R1 // end 883 loop: 884 CMP R0, R1 885 BEQ notfound 886 MOVBU.P 1(R0), R3 887 CMP R2, R3 888 BNE loop 889 890 SUB $1, R0 // R0 will be one beyond the position we want 891 SUB R4, R0 // remove base 892 MOVD R0, ret+24(FP) 893 RET 894 895 notfound: 896 MOVD $-1, R0 897 MOVD R0, ret+24(FP) 898 RET 899 900 // TODO: share code with memequal? 901 TEXT bytes·Equal(SB),NOSPLIT,$0-49 902 MOVD a_len+8(FP), R1 903 MOVD b_len+32(FP), R3 904 CMP R1, R3 // unequal lengths are not equal 905 BNE notequal 906 MOVD a+0(FP), R0 907 MOVD b+24(FP), R2 908 ADD R0, R1 // end 909 loop: 910 CMP R0, R1 911 BEQ equal // reaches the end 912 MOVBU.P 1(R0), R4 913 MOVBU.P 1(R2), R5 914 CMP R4, R5 915 BEQ loop 916 notequal: 917 MOVB ZR, ret+48(FP) 918 RET 919 equal: 920 MOVD $1, R0 921 MOVB R0, ret+48(FP) 922 RET 923 924 TEXT runtime·return0(SB), NOSPLIT, $0 925 MOVW $0, R0 926 RET 927 928 // The top-most function running on a goroutine 929 // returns to goexit+PCQuantum. 930 TEXT runtime·goexit(SB),NOSPLIT,$-8-0 931 MOVD R0, R0 // NOP 932 BL runtime·goexit1(SB) // does not return 933 934 // TODO(aram): use PRFM here. 935 TEXT runtime·prefetcht0(SB),NOSPLIT,$0-8 936 RET 937 938 TEXT runtime·prefetcht1(SB),NOSPLIT,$0-8 939 RET 940 941 TEXT runtime·prefetcht2(SB),NOSPLIT,$0-8 942 RET 943 944 TEXT runtime·prefetchnta(SB),NOSPLIT,$0-8 945 RET 946 947 TEXT runtime·sigreturn(SB),NOSPLIT,$0-0 948 RET 949 950 // This is called from .init_array and follows the platform, not Go, ABI. 951 TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0 952 SUB $0x10, RSP 953 MOVD R27, 8(RSP) // The access to global variables below implicitly uses R27, which is callee-save 954 MOVD runtime·lastmoduledatap(SB), R1 955 MOVD R0, moduledata_next(R1) 956 MOVD R0, runtime·lastmoduledatap(SB) 957 MOVD 8(RSP), R27 958 ADD $0x10, RSP 959 RET 960 961 TEXT ·checkASM(SB),NOSPLIT,$0-1 962 MOVW $1, R3 963 MOVB R3, ret+0(FP) 964 RET