github.com/AESNooper/go/src@v0.0.0-20220218095104-b56a4ab1bbbb/runtime/asm_arm64.s (about) 1 // Copyright 2015 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 #include "go_asm.h" 6 #include "go_tls.h" 7 #include "tls_arm64.h" 8 #include "funcdata.h" 9 #include "textflag.h" 10 11 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0 12 // SP = stack; R0 = argc; R1 = argv 13 14 SUB $32, RSP 15 MOVW R0, 8(RSP) // argc 16 MOVD R1, 16(RSP) // argv 17 18 #ifdef TLS_darwin 19 // Initialize TLS. 20 MOVD ZR, g // clear g, make sure it's not junk. 21 SUB $32, RSP 22 MRS_TPIDR_R0 23 AND $~7, R0 24 MOVD R0, 16(RSP) // arg2: TLS base 25 MOVD $runtime·tls_g(SB), R2 26 MOVD R2, 8(RSP) // arg1: &tlsg 27 BL ·tlsinit(SB) 28 ADD $32, RSP 29 #endif 30 31 // create istack out of the given (operating system) stack. 32 // _cgo_init may update stackguard. 33 MOVD $runtime·g0(SB), g 34 MOVD RSP, R7 35 MOVD $(-64*1024)(R7), R0 36 MOVD R0, g_stackguard0(g) 37 MOVD R0, g_stackguard1(g) 38 MOVD R0, (g_stack+stack_lo)(g) 39 MOVD R7, (g_stack+stack_hi)(g) 40 41 // if there is a _cgo_init, call it using the gcc ABI. 42 MOVD _cgo_init(SB), R12 43 CBZ R12, nocgo 44 45 #ifdef GOOS_android 46 MRS_TPIDR_R0 // load TLS base pointer 47 MOVD R0, R3 // arg 3: TLS base pointer 48 MOVD $runtime·tls_g(SB), R2 // arg 2: &tls_g 49 #else 50 MOVD $0, R2 // arg 2: not used when using platform's TLS 51 #endif 52 MOVD $setg_gcc<>(SB), R1 // arg 1: setg 53 MOVD g, R0 // arg 0: G 54 SUB $16, RSP // reserve 16 bytes for sp-8 where fp may be saved. 55 BL (R12) 56 ADD $16, RSP 57 58 nocgo: 59 BL runtime·save_g(SB) 60 // update stackguard after _cgo_init 61 MOVD (g_stack+stack_lo)(g), R0 62 ADD $const__StackGuard, R0 63 MOVD R0, g_stackguard0(g) 64 MOVD R0, g_stackguard1(g) 65 66 // set the per-goroutine and per-mach "registers" 67 MOVD $runtime·m0(SB), R0 68 69 // save m->g0 = g0 70 MOVD g, m_g0(R0) 71 // save m0 to g0->m 72 MOVD R0, g_m(g) 73 74 BL runtime·check(SB) 75 76 #ifdef GOOS_windows 77 BL runtime·wintls(SB) 78 #endif 79 80 MOVW 8(RSP), R0 // copy argc 81 MOVW R0, -8(RSP) 82 MOVD 16(RSP), R0 // copy argv 83 MOVD R0, 0(RSP) 84 BL runtime·args(SB) 85 BL runtime·osinit(SB) 86 BL runtime·schedinit(SB) 87 88 // create a new goroutine to start program 89 MOVD $runtime·mainPC(SB), R0 // entry 90 SUB $16, RSP 91 MOVD R0, 8(RSP) // arg 92 MOVD $0, 0(RSP) // dummy LR 93 BL runtime·newproc(SB) 94 ADD $16, RSP 95 96 // start this M 97 BL runtime·mstart(SB) 98 99 MOVD $0, R0 100 MOVD R0, (R0) // boom 101 UNDEF 102 103 DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB) 104 GLOBL runtime·mainPC(SB),RODATA,$8 105 106 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0 107 BRK 108 RET 109 110 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0 111 RET 112 113 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0 114 BL runtime·mstart0(SB) 115 RET // not reached 116 117 /* 118 * go-routine 119 */ 120 121 // void gogo(Gobuf*) 122 // restore state from Gobuf; longjmp 123 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8 124 MOVD buf+0(FP), R5 125 MOVD gobuf_g(R5), R6 126 MOVD 0(R6), R4 // make sure g != nil 127 B gogo<>(SB) 128 129 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0 130 MOVD R6, g 131 BL runtime·save_g(SB) 132 133 MOVD gobuf_sp(R5), R0 134 MOVD R0, RSP 135 MOVD gobuf_bp(R5), R29 136 MOVD gobuf_lr(R5), LR 137 MOVD gobuf_ret(R5), R0 138 MOVD gobuf_ctxt(R5), R26 139 MOVD $0, gobuf_sp(R5) 140 MOVD $0, gobuf_bp(R5) 141 MOVD $0, gobuf_ret(R5) 142 MOVD $0, gobuf_lr(R5) 143 MOVD $0, gobuf_ctxt(R5) 144 CMP ZR, ZR // set condition codes for == test, needed by stack split 145 MOVD gobuf_pc(R5), R6 146 B (R6) 147 148 // void mcall(fn func(*g)) 149 // Switch to m->g0's stack, call fn(g). 150 // Fn must never return. It should gogo(&g->sched) 151 // to keep running g. 152 TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8 153 #ifdef GOEXPERIMENT_regabiargs 154 MOVD R0, R26 // context 155 #else 156 MOVD fn+0(FP), R26 // context 157 #endif 158 159 // Save caller state in g->sched 160 MOVD RSP, R0 161 MOVD R0, (g_sched+gobuf_sp)(g) 162 MOVD R29, (g_sched+gobuf_bp)(g) 163 MOVD LR, (g_sched+gobuf_pc)(g) 164 MOVD $0, (g_sched+gobuf_lr)(g) 165 166 // Switch to m->g0 & its stack, call fn. 167 MOVD g, R3 168 MOVD g_m(g), R8 169 MOVD m_g0(R8), g 170 BL runtime·save_g(SB) 171 CMP g, R3 172 BNE 2(PC) 173 B runtime·badmcall(SB) 174 175 MOVD (g_sched+gobuf_sp)(g), R0 176 MOVD R0, RSP // sp = m->g0->sched.sp 177 MOVD (g_sched+gobuf_bp)(g), R29 178 #ifdef GOEXPERIMENT_regabiargs 179 MOVD R3, R0 // arg = g 180 #else 181 MOVD R3, -8(RSP) // arg = g 182 #endif 183 MOVD $0, -16(RSP) // dummy LR 184 SUB $16, RSP 185 MOVD 0(R26), R4 // code pointer 186 BL (R4) 187 B runtime·badmcall2(SB) 188 189 // systemstack_switch is a dummy routine that systemstack leaves at the bottom 190 // of the G stack. We need to distinguish the routine that 191 // lives at the bottom of the G stack from the one that lives 192 // at the top of the system stack because the one at the top of 193 // the system stack terminates the stack walk (see topofstack()). 194 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0 195 UNDEF 196 BL (LR) // make sure this function is not leaf 197 RET 198 199 // func systemstack(fn func()) 200 TEXT runtime·systemstack(SB), NOSPLIT, $0-8 201 MOVD fn+0(FP), R3 // R3 = fn 202 MOVD R3, R26 // context 203 MOVD g_m(g), R4 // R4 = m 204 205 MOVD m_gsignal(R4), R5 // R5 = gsignal 206 CMP g, R5 207 BEQ noswitch 208 209 MOVD m_g0(R4), R5 // R5 = g0 210 CMP g, R5 211 BEQ noswitch 212 213 MOVD m_curg(R4), R6 214 CMP g, R6 215 BEQ switch 216 217 // Bad: g is not gsignal, not g0, not curg. What is it? 218 // Hide call from linker nosplit analysis. 219 MOVD $runtime·badsystemstack(SB), R3 220 BL (R3) 221 B runtime·abort(SB) 222 223 switch: 224 // save our state in g->sched. Pretend to 225 // be systemstack_switch if the G stack is scanned. 226 BL gosave_systemstack_switch<>(SB) 227 228 // switch to g0 229 MOVD R5, g 230 BL runtime·save_g(SB) 231 MOVD (g_sched+gobuf_sp)(g), R3 232 MOVD R3, RSP 233 MOVD (g_sched+gobuf_bp)(g), R29 234 235 // call target function 236 MOVD 0(R26), R3 // code pointer 237 BL (R3) 238 239 // switch back to g 240 MOVD g_m(g), R3 241 MOVD m_curg(R3), g 242 BL runtime·save_g(SB) 243 MOVD (g_sched+gobuf_sp)(g), R0 244 MOVD R0, RSP 245 MOVD (g_sched+gobuf_bp)(g), R29 246 MOVD $0, (g_sched+gobuf_sp)(g) 247 MOVD $0, (g_sched+gobuf_bp)(g) 248 RET 249 250 noswitch: 251 // already on m stack, just call directly 252 // Using a tail call here cleans up tracebacks since we won't stop 253 // at an intermediate systemstack. 254 MOVD 0(R26), R3 // code pointer 255 MOVD.P 16(RSP), R30 // restore LR 256 SUB $8, RSP, R29 // restore FP 257 B (R3) 258 259 /* 260 * support for morestack 261 */ 262 263 // Called during function prolog when more stack is needed. 264 // Caller has already loaded: 265 // R3 prolog's LR (R30) 266 // 267 // The traceback routines see morestack on a g0 as being 268 // the top of a stack (for example, morestack calling newstack 269 // calling the scheduler calling newm calling gc), so we must 270 // record an argument size. For that purpose, it has no arguments. 271 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0 272 // Cannot grow scheduler stack (m->g0). 273 MOVD g_m(g), R8 274 MOVD m_g0(R8), R4 275 CMP g, R4 276 BNE 3(PC) 277 BL runtime·badmorestackg0(SB) 278 B runtime·abort(SB) 279 280 // Cannot grow signal stack (m->gsignal). 281 MOVD m_gsignal(R8), R4 282 CMP g, R4 283 BNE 3(PC) 284 BL runtime·badmorestackgsignal(SB) 285 B runtime·abort(SB) 286 287 // Called from f. 288 // Set g->sched to context in f 289 MOVD RSP, R0 290 MOVD R0, (g_sched+gobuf_sp)(g) 291 MOVD R29, (g_sched+gobuf_bp)(g) 292 MOVD LR, (g_sched+gobuf_pc)(g) 293 MOVD R3, (g_sched+gobuf_lr)(g) 294 MOVD R26, (g_sched+gobuf_ctxt)(g) 295 296 // Called from f. 297 // Set m->morebuf to f's callers. 298 MOVD R3, (m_morebuf+gobuf_pc)(R8) // f's caller's PC 299 MOVD RSP, R0 300 MOVD R0, (m_morebuf+gobuf_sp)(R8) // f's caller's RSP 301 MOVD g, (m_morebuf+gobuf_g)(R8) 302 303 // Call newstack on m->g0's stack. 304 MOVD m_g0(R8), g 305 BL runtime·save_g(SB) 306 MOVD (g_sched+gobuf_sp)(g), R0 307 MOVD R0, RSP 308 MOVD (g_sched+gobuf_bp)(g), R29 309 MOVD.W $0, -16(RSP) // create a call frame on g0 (saved LR; keep 16-aligned) 310 BL runtime·newstack(SB) 311 312 // Not reached, but make sure the return PC from the call to newstack 313 // is still in this function, and not the beginning of the next. 314 UNDEF 315 316 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0 317 MOVW $0, R26 318 B runtime·morestack(SB) 319 320 #ifdef GOEXPERIMENT_regabireflect 321 // spillArgs stores return values from registers to a *internal/abi.RegArgs in R20. 322 TEXT ·spillArgs(SB),NOSPLIT,$0-0 323 MOVD R0, (0*8)(R20) 324 MOVD R1, (1*8)(R20) 325 MOVD R2, (2*8)(R20) 326 MOVD R3, (3*8)(R20) 327 MOVD R4, (4*8)(R20) 328 MOVD R5, (5*8)(R20) 329 MOVD R6, (6*8)(R20) 330 MOVD R7, (7*8)(R20) 331 MOVD R8, (8*8)(R20) 332 MOVD R9, (9*8)(R20) 333 MOVD R10, (10*8)(R20) 334 MOVD R11, (11*8)(R20) 335 MOVD R12, (12*8)(R20) 336 MOVD R13, (13*8)(R20) 337 MOVD R14, (14*8)(R20) 338 MOVD R15, (15*8)(R20) 339 FMOVD F0, (16*8)(R20) 340 FMOVD F1, (17*8)(R20) 341 FMOVD F2, (18*8)(R20) 342 FMOVD F3, (19*8)(R20) 343 FMOVD F4, (20*8)(R20) 344 FMOVD F5, (21*8)(R20) 345 FMOVD F6, (22*8)(R20) 346 FMOVD F7, (23*8)(R20) 347 FMOVD F8, (24*8)(R20) 348 FMOVD F9, (25*8)(R20) 349 FMOVD F10, (26*8)(R20) 350 FMOVD F11, (27*8)(R20) 351 FMOVD F12, (28*8)(R20) 352 FMOVD F13, (29*8)(R20) 353 FMOVD F14, (30*8)(R20) 354 FMOVD F15, (31*8)(R20) 355 RET 356 357 // unspillArgs loads args into registers from a *internal/abi.RegArgs in R20. 358 TEXT ·unspillArgs(SB),NOSPLIT,$0-0 359 MOVD (0*8)(R20), R0 360 MOVD (1*8)(R20), R1 361 MOVD (2*8)(R20), R2 362 MOVD (3*8)(R20), R3 363 MOVD (4*8)(R20), R4 364 MOVD (5*8)(R20), R5 365 MOVD (6*8)(R20), R6 366 MOVD (7*8)(R20), R7 367 MOVD (8*8)(R20), R8 368 MOVD (9*8)(R20), R9 369 MOVD (10*8)(R20), R10 370 MOVD (11*8)(R20), R11 371 MOVD (12*8)(R20), R12 372 MOVD (13*8)(R20), R13 373 MOVD (14*8)(R20), R14 374 MOVD (15*8)(R20), R15 375 FMOVD (16*8)(R20), F0 376 FMOVD (17*8)(R20), F1 377 FMOVD (18*8)(R20), F2 378 FMOVD (19*8)(R20), F3 379 FMOVD (20*8)(R20), F4 380 FMOVD (21*8)(R20), F5 381 FMOVD (22*8)(R20), F6 382 FMOVD (23*8)(R20), F7 383 FMOVD (24*8)(R20), F8 384 FMOVD (25*8)(R20), F9 385 FMOVD (26*8)(R20), F10 386 FMOVD (27*8)(R20), F11 387 FMOVD (28*8)(R20), F12 388 FMOVD (29*8)(R20), F13 389 FMOVD (30*8)(R20), F14 390 FMOVD (31*8)(R20), F15 391 RET 392 #else 393 TEXT ·spillArgs(SB),NOSPLIT,$0-0 394 RET 395 396 TEXT ·unspillArgs(SB),NOSPLIT,$0-0 397 RET 398 #endif 399 400 // reflectcall: call a function with the given argument list 401 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs). 402 // we don't have variable-sized frames, so we use a small number 403 // of constant-sized-frame functions to encode a few bits of size in the pc. 404 // Caution: ugly multiline assembly macros in your future! 405 406 #define DISPATCH(NAME,MAXSIZE) \ 407 MOVD $MAXSIZE, R27; \ 408 CMP R27, R16; \ 409 BGT 3(PC); \ 410 MOVD $NAME(SB), R27; \ 411 B (R27) 412 // Note: can't just "B NAME(SB)" - bad inlining results. 413 414 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48 415 MOVWU frameSize+32(FP), R16 416 DISPATCH(runtime·call16, 16) 417 DISPATCH(runtime·call32, 32) 418 DISPATCH(runtime·call64, 64) 419 DISPATCH(runtime·call128, 128) 420 DISPATCH(runtime·call256, 256) 421 DISPATCH(runtime·call512, 512) 422 DISPATCH(runtime·call1024, 1024) 423 DISPATCH(runtime·call2048, 2048) 424 DISPATCH(runtime·call4096, 4096) 425 DISPATCH(runtime·call8192, 8192) 426 DISPATCH(runtime·call16384, 16384) 427 DISPATCH(runtime·call32768, 32768) 428 DISPATCH(runtime·call65536, 65536) 429 DISPATCH(runtime·call131072, 131072) 430 DISPATCH(runtime·call262144, 262144) 431 DISPATCH(runtime·call524288, 524288) 432 DISPATCH(runtime·call1048576, 1048576) 433 DISPATCH(runtime·call2097152, 2097152) 434 DISPATCH(runtime·call4194304, 4194304) 435 DISPATCH(runtime·call8388608, 8388608) 436 DISPATCH(runtime·call16777216, 16777216) 437 DISPATCH(runtime·call33554432, 33554432) 438 DISPATCH(runtime·call67108864, 67108864) 439 DISPATCH(runtime·call134217728, 134217728) 440 DISPATCH(runtime·call268435456, 268435456) 441 DISPATCH(runtime·call536870912, 536870912) 442 DISPATCH(runtime·call1073741824, 1073741824) 443 MOVD $runtime·badreflectcall(SB), R0 444 B (R0) 445 446 #define CALLFN(NAME,MAXSIZE) \ 447 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \ 448 NO_LOCAL_POINTERS; \ 449 /* copy arguments to stack */ \ 450 MOVD stackArgs+16(FP), R3; \ 451 MOVWU stackArgsSize+24(FP), R4; \ 452 ADD $8, RSP, R5; \ 453 BIC $0xf, R4, R6; \ 454 CBZ R6, 6(PC); \ 455 /* if R6=(argsize&~15) != 0 */ \ 456 ADD R6, R5, R6; \ 457 /* copy 16 bytes a time */ \ 458 LDP.P 16(R3), (R7, R8); \ 459 STP.P (R7, R8), 16(R5); \ 460 CMP R5, R6; \ 461 BNE -3(PC); \ 462 AND $0xf, R4, R6; \ 463 CBZ R6, 6(PC); \ 464 /* if R6=(argsize&15) != 0 */ \ 465 ADD R6, R5, R6; \ 466 /* copy 1 byte a time for the rest */ \ 467 MOVBU.P 1(R3), R7; \ 468 MOVBU.P R7, 1(R5); \ 469 CMP R5, R6; \ 470 BNE -3(PC); \ 471 /* set up argument registers */ \ 472 MOVD regArgs+40(FP), R20; \ 473 CALL ·unspillArgs(SB); \ 474 /* call function */ \ 475 MOVD f+8(FP), R26; \ 476 MOVD (R26), R20; \ 477 PCDATA $PCDATA_StackMapIndex, $0; \ 478 BL (R20); \ 479 /* copy return values back */ \ 480 MOVD regArgs+40(FP), R20; \ 481 CALL ·spillArgs(SB); \ 482 MOVD stackArgsType+0(FP), R7; \ 483 MOVD stackArgs+16(FP), R3; \ 484 MOVWU stackArgsSize+24(FP), R4; \ 485 MOVWU stackRetOffset+28(FP), R6; \ 486 ADD $8, RSP, R5; \ 487 ADD R6, R5; \ 488 ADD R6, R3; \ 489 SUB R6, R4; \ 490 BL callRet<>(SB); \ 491 RET 492 493 // callRet copies return values back at the end of call*. This is a 494 // separate function so it can allocate stack space for the arguments 495 // to reflectcallmove. It does not follow the Go ABI; it expects its 496 // arguments in registers. 497 TEXT callRet<>(SB), NOSPLIT, $48-0 498 NO_LOCAL_POINTERS 499 MOVD R7, 8(RSP) 500 MOVD R3, 16(RSP) 501 MOVD R5, 24(RSP) 502 MOVD R4, 32(RSP) 503 MOVD R20, 40(RSP) 504 BL runtime·reflectcallmove(SB) 505 RET 506 507 CALLFN(·call16, 16) 508 CALLFN(·call32, 32) 509 CALLFN(·call64, 64) 510 CALLFN(·call128, 128) 511 CALLFN(·call256, 256) 512 CALLFN(·call512, 512) 513 CALLFN(·call1024, 1024) 514 CALLFN(·call2048, 2048) 515 CALLFN(·call4096, 4096) 516 CALLFN(·call8192, 8192) 517 CALLFN(·call16384, 16384) 518 CALLFN(·call32768, 32768) 519 CALLFN(·call65536, 65536) 520 CALLFN(·call131072, 131072) 521 CALLFN(·call262144, 262144) 522 CALLFN(·call524288, 524288) 523 CALLFN(·call1048576, 1048576) 524 CALLFN(·call2097152, 2097152) 525 CALLFN(·call4194304, 4194304) 526 CALLFN(·call8388608, 8388608) 527 CALLFN(·call16777216, 16777216) 528 CALLFN(·call33554432, 33554432) 529 CALLFN(·call67108864, 67108864) 530 CALLFN(·call134217728, 134217728) 531 CALLFN(·call268435456, 268435456) 532 CALLFN(·call536870912, 536870912) 533 CALLFN(·call1073741824, 1073741824) 534 535 // func memhash32(p unsafe.Pointer, h uintptr) uintptr 536 TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24 537 MOVB runtime·useAeshash(SB), R10 538 CBZ R10, noaes 539 #ifndef GOEXPERIMENT_regabiargs 540 MOVD p+0(FP), R0 541 MOVD h+8(FP), R1 542 MOVD $ret+16(FP), R2 543 #endif 544 MOVD $runtime·aeskeysched+0(SB), R3 545 546 VEOR V0.B16, V0.B16, V0.B16 547 VLD1 (R3), [V2.B16] 548 VLD1 (R0), V0.S[1] 549 VMOV R1, V0.S[0] 550 551 AESE V2.B16, V0.B16 552 AESMC V0.B16, V0.B16 553 AESE V2.B16, V0.B16 554 AESMC V0.B16, V0.B16 555 AESE V2.B16, V0.B16 556 557 #ifdef GOEXPERIMENT_regabiargs 558 VMOV V0.D[0], R0 559 #else 560 VST1 [V0.D1], (R2) 561 #endif 562 RET 563 noaes: 564 B runtime·memhash32Fallback<ABIInternal>(SB) 565 566 // func memhash64(p unsafe.Pointer, h uintptr) uintptr 567 TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24 568 MOVB runtime·useAeshash(SB), R10 569 CBZ R10, noaes 570 #ifndef GOEXPERIMENT_regabiargs 571 MOVD p+0(FP), R0 572 MOVD h+8(FP), R1 573 MOVD $ret+16(FP), R2 574 #endif 575 MOVD $runtime·aeskeysched+0(SB), R3 576 577 VEOR V0.B16, V0.B16, V0.B16 578 VLD1 (R3), [V2.B16] 579 VLD1 (R0), V0.D[1] 580 VMOV R1, V0.D[0] 581 582 AESE V2.B16, V0.B16 583 AESMC V0.B16, V0.B16 584 AESE V2.B16, V0.B16 585 AESMC V0.B16, V0.B16 586 AESE V2.B16, V0.B16 587 588 #ifdef GOEXPERIMENT_regabiargs 589 VMOV V0.D[0], R0 590 #else 591 VST1 [V0.D1], (R2) 592 #endif 593 RET 594 noaes: 595 B runtime·memhash64Fallback<ABIInternal>(SB) 596 597 // func memhash(p unsafe.Pointer, h, size uintptr) uintptr 598 TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32 599 MOVB runtime·useAeshash(SB), R10 600 CBZ R10, noaes 601 #ifndef GOEXPERIMENT_regabiargs 602 MOVD p+0(FP), R0 603 MOVD h+8(FP), R1 604 MOVD s+16(FP), R2 605 MOVD $ret+24(FP), R8 606 #endif 607 B aeshashbody<>(SB) 608 noaes: 609 B runtime·memhashFallback<ABIInternal>(SB) 610 611 // func strhash(p unsafe.Pointer, h uintptr) uintptr 612 TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24 613 MOVB runtime·useAeshash(SB), R10 614 CBZ R10, noaes 615 #ifdef GOEXPERIMENT_regabiargs 616 LDP (R0), (R0, R2) // string data / length 617 #else 618 MOVD p+0(FP), R10 // string pointer 619 LDP (R10), (R0, R2) // string data / length 620 MOVD h+8(FP), R1 621 MOVD $ret+16(FP), R8 // return adddress 622 #endif 623 B aeshashbody<>(SB) 624 noaes: 625 B runtime·strhashFallback<ABIInternal>(SB) 626 627 // R0: data 628 // R1: seed data 629 // R2: length 630 #ifdef GOEXPERIMENT_regabiargs 631 // At return, R0 = return value 632 #else 633 // R8: address to put return value 634 #endif 635 TEXT aeshashbody<>(SB),NOSPLIT|NOFRAME,$0 636 VEOR V30.B16, V30.B16, V30.B16 637 VMOV R1, V30.D[0] 638 VMOV R2, V30.D[1] // load length into seed 639 640 MOVD $runtime·aeskeysched+0(SB), R4 641 VLD1.P 16(R4), [V0.B16] 642 AESE V30.B16, V0.B16 643 AESMC V0.B16, V0.B16 644 CMP $16, R2 645 BLO aes0to15 646 BEQ aes16 647 CMP $32, R2 648 BLS aes17to32 649 CMP $64, R2 650 BLS aes33to64 651 CMP $128, R2 652 BLS aes65to128 653 B aes129plus 654 655 aes0to15: 656 CBZ R2, aes0 657 VEOR V2.B16, V2.B16, V2.B16 658 TBZ $3, R2, less_than_8 659 VLD1.P 8(R0), V2.D[0] 660 661 less_than_8: 662 TBZ $2, R2, less_than_4 663 VLD1.P 4(R0), V2.S[2] 664 665 less_than_4: 666 TBZ $1, R2, less_than_2 667 VLD1.P 2(R0), V2.H[6] 668 669 less_than_2: 670 TBZ $0, R2, done 671 VLD1 (R0), V2.B[14] 672 done: 673 AESE V0.B16, V2.B16 674 AESMC V2.B16, V2.B16 675 AESE V0.B16, V2.B16 676 AESMC V2.B16, V2.B16 677 AESE V0.B16, V2.B16 678 679 #ifdef GOEXPERIMENT_regabiargs 680 VMOV V2.D[0], R0 681 #else 682 VST1 [V2.D1], (R8) 683 #endif 684 RET 685 686 aes0: 687 #ifdef GOEXPERIMENT_regabiargs 688 VMOV V0.D[0], R0 689 #else 690 VST1 [V0.D1], (R8) 691 #endif 692 RET 693 694 aes16: 695 VLD1 (R0), [V2.B16] 696 B done 697 698 aes17to32: 699 // make second seed 700 VLD1 (R4), [V1.B16] 701 AESE V30.B16, V1.B16 702 AESMC V1.B16, V1.B16 703 SUB $16, R2, R10 704 VLD1.P (R0)(R10), [V2.B16] 705 VLD1 (R0), [V3.B16] 706 707 AESE V0.B16, V2.B16 708 AESMC V2.B16, V2.B16 709 AESE V1.B16, V3.B16 710 AESMC V3.B16, V3.B16 711 712 AESE V0.B16, V2.B16 713 AESMC V2.B16, V2.B16 714 AESE V1.B16, V3.B16 715 AESMC V3.B16, V3.B16 716 717 AESE V0.B16, V2.B16 718 AESE V1.B16, V3.B16 719 720 VEOR V3.B16, V2.B16, V2.B16 721 #ifdef GOEXPERIMENT_regabiargs 722 VMOV V2.D[0], R0 723 #else 724 VST1 [V2.D1], (R8) 725 #endif 726 RET 727 728 aes33to64: 729 VLD1 (R4), [V1.B16, V2.B16, V3.B16] 730 AESE V30.B16, V1.B16 731 AESMC V1.B16, V1.B16 732 AESE V30.B16, V2.B16 733 AESMC V2.B16, V2.B16 734 AESE V30.B16, V3.B16 735 AESMC V3.B16, V3.B16 736 SUB $32, R2, R10 737 738 VLD1.P (R0)(R10), [V4.B16, V5.B16] 739 VLD1 (R0), [V6.B16, V7.B16] 740 741 AESE V0.B16, V4.B16 742 AESMC V4.B16, V4.B16 743 AESE V1.B16, V5.B16 744 AESMC V5.B16, V5.B16 745 AESE V2.B16, V6.B16 746 AESMC V6.B16, V6.B16 747 AESE V3.B16, V7.B16 748 AESMC V7.B16, V7.B16 749 750 AESE V0.B16, V4.B16 751 AESMC V4.B16, V4.B16 752 AESE V1.B16, V5.B16 753 AESMC V5.B16, V5.B16 754 AESE V2.B16, V6.B16 755 AESMC V6.B16, V6.B16 756 AESE V3.B16, V7.B16 757 AESMC V7.B16, V7.B16 758 759 AESE V0.B16, V4.B16 760 AESE V1.B16, V5.B16 761 AESE V2.B16, V6.B16 762 AESE V3.B16, V7.B16 763 764 VEOR V6.B16, V4.B16, V4.B16 765 VEOR V7.B16, V5.B16, V5.B16 766 VEOR V5.B16, V4.B16, V4.B16 767 768 #ifdef GOEXPERIMENT_regabiargs 769 VMOV V4.D[0], R0 770 #else 771 VST1 [V4.D1], (R8) 772 #endif 773 RET 774 775 aes65to128: 776 VLD1.P 64(R4), [V1.B16, V2.B16, V3.B16, V4.B16] 777 VLD1 (R4), [V5.B16, V6.B16, V7.B16] 778 AESE V30.B16, V1.B16 779 AESMC V1.B16, V1.B16 780 AESE V30.B16, V2.B16 781 AESMC V2.B16, V2.B16 782 AESE V30.B16, V3.B16 783 AESMC V3.B16, V3.B16 784 AESE V30.B16, V4.B16 785 AESMC V4.B16, V4.B16 786 AESE V30.B16, V5.B16 787 AESMC V5.B16, V5.B16 788 AESE V30.B16, V6.B16 789 AESMC V6.B16, V6.B16 790 AESE V30.B16, V7.B16 791 AESMC V7.B16, V7.B16 792 793 SUB $64, R2, R10 794 VLD1.P (R0)(R10), [V8.B16, V9.B16, V10.B16, V11.B16] 795 VLD1 (R0), [V12.B16, V13.B16, V14.B16, V15.B16] 796 AESE V0.B16, V8.B16 797 AESMC V8.B16, V8.B16 798 AESE V1.B16, V9.B16 799 AESMC V9.B16, V9.B16 800 AESE V2.B16, V10.B16 801 AESMC V10.B16, V10.B16 802 AESE V3.B16, V11.B16 803 AESMC V11.B16, V11.B16 804 AESE V4.B16, V12.B16 805 AESMC V12.B16, V12.B16 806 AESE V5.B16, V13.B16 807 AESMC V13.B16, V13.B16 808 AESE V6.B16, V14.B16 809 AESMC V14.B16, V14.B16 810 AESE V7.B16, V15.B16 811 AESMC V15.B16, V15.B16 812 813 AESE V0.B16, V8.B16 814 AESMC V8.B16, V8.B16 815 AESE V1.B16, V9.B16 816 AESMC V9.B16, V9.B16 817 AESE V2.B16, V10.B16 818 AESMC V10.B16, V10.B16 819 AESE V3.B16, V11.B16 820 AESMC V11.B16, V11.B16 821 AESE V4.B16, V12.B16 822 AESMC V12.B16, V12.B16 823 AESE V5.B16, V13.B16 824 AESMC V13.B16, V13.B16 825 AESE V6.B16, V14.B16 826 AESMC V14.B16, V14.B16 827 AESE V7.B16, V15.B16 828 AESMC V15.B16, V15.B16 829 830 AESE V0.B16, V8.B16 831 AESE V1.B16, V9.B16 832 AESE V2.B16, V10.B16 833 AESE V3.B16, V11.B16 834 AESE V4.B16, V12.B16 835 AESE V5.B16, V13.B16 836 AESE V6.B16, V14.B16 837 AESE V7.B16, V15.B16 838 839 VEOR V12.B16, V8.B16, V8.B16 840 VEOR V13.B16, V9.B16, V9.B16 841 VEOR V14.B16, V10.B16, V10.B16 842 VEOR V15.B16, V11.B16, V11.B16 843 VEOR V10.B16, V8.B16, V8.B16 844 VEOR V11.B16, V9.B16, V9.B16 845 VEOR V9.B16, V8.B16, V8.B16 846 847 #ifdef GOEXPERIMENT_regabiargs 848 VMOV V8.D[0], R0 849 #else 850 VST1 [V8.D1], (R8) 851 #endif 852 RET 853 854 aes129plus: 855 PRFM (R0), PLDL1KEEP 856 VLD1.P 64(R4), [V1.B16, V2.B16, V3.B16, V4.B16] 857 VLD1 (R4), [V5.B16, V6.B16, V7.B16] 858 AESE V30.B16, V1.B16 859 AESMC V1.B16, V1.B16 860 AESE V30.B16, V2.B16 861 AESMC V2.B16, V2.B16 862 AESE V30.B16, V3.B16 863 AESMC V3.B16, V3.B16 864 AESE V30.B16, V4.B16 865 AESMC V4.B16, V4.B16 866 AESE V30.B16, V5.B16 867 AESMC V5.B16, V5.B16 868 AESE V30.B16, V6.B16 869 AESMC V6.B16, V6.B16 870 AESE V30.B16, V7.B16 871 AESMC V7.B16, V7.B16 872 ADD R0, R2, R10 873 SUB $128, R10, R10 874 VLD1.P 64(R10), [V8.B16, V9.B16, V10.B16, V11.B16] 875 VLD1 (R10), [V12.B16, V13.B16, V14.B16, V15.B16] 876 SUB $1, R2, R2 877 LSR $7, R2, R2 878 879 aesloop: 880 AESE V8.B16, V0.B16 881 AESMC V0.B16, V0.B16 882 AESE V9.B16, V1.B16 883 AESMC V1.B16, V1.B16 884 AESE V10.B16, V2.B16 885 AESMC V2.B16, V2.B16 886 AESE V11.B16, V3.B16 887 AESMC V3.B16, V3.B16 888 AESE V12.B16, V4.B16 889 AESMC V4.B16, V4.B16 890 AESE V13.B16, V5.B16 891 AESMC V5.B16, V5.B16 892 AESE V14.B16, V6.B16 893 AESMC V6.B16, V6.B16 894 AESE V15.B16, V7.B16 895 AESMC V7.B16, V7.B16 896 897 VLD1.P 64(R0), [V8.B16, V9.B16, V10.B16, V11.B16] 898 AESE V8.B16, V0.B16 899 AESMC V0.B16, V0.B16 900 AESE V9.B16, V1.B16 901 AESMC V1.B16, V1.B16 902 AESE V10.B16, V2.B16 903 AESMC V2.B16, V2.B16 904 AESE V11.B16, V3.B16 905 AESMC V3.B16, V3.B16 906 907 VLD1.P 64(R0), [V12.B16, V13.B16, V14.B16, V15.B16] 908 AESE V12.B16, V4.B16 909 AESMC V4.B16, V4.B16 910 AESE V13.B16, V5.B16 911 AESMC V5.B16, V5.B16 912 AESE V14.B16, V6.B16 913 AESMC V6.B16, V6.B16 914 AESE V15.B16, V7.B16 915 AESMC V7.B16, V7.B16 916 SUB $1, R2, R2 917 CBNZ R2, aesloop 918 919 AESE V8.B16, V0.B16 920 AESMC V0.B16, V0.B16 921 AESE V9.B16, V1.B16 922 AESMC V1.B16, V1.B16 923 AESE V10.B16, V2.B16 924 AESMC V2.B16, V2.B16 925 AESE V11.B16, V3.B16 926 AESMC V3.B16, V3.B16 927 AESE V12.B16, V4.B16 928 AESMC V4.B16, V4.B16 929 AESE V13.B16, V5.B16 930 AESMC V5.B16, V5.B16 931 AESE V14.B16, V6.B16 932 AESMC V6.B16, V6.B16 933 AESE V15.B16, V7.B16 934 AESMC V7.B16, V7.B16 935 936 AESE V8.B16, V0.B16 937 AESMC V0.B16, V0.B16 938 AESE V9.B16, V1.B16 939 AESMC V1.B16, V1.B16 940 AESE V10.B16, V2.B16 941 AESMC V2.B16, V2.B16 942 AESE V11.B16, V3.B16 943 AESMC V3.B16, V3.B16 944 AESE V12.B16, V4.B16 945 AESMC V4.B16, V4.B16 946 AESE V13.B16, V5.B16 947 AESMC V5.B16, V5.B16 948 AESE V14.B16, V6.B16 949 AESMC V6.B16, V6.B16 950 AESE V15.B16, V7.B16 951 AESMC V7.B16, V7.B16 952 953 AESE V8.B16, V0.B16 954 AESE V9.B16, V1.B16 955 AESE V10.B16, V2.B16 956 AESE V11.B16, V3.B16 957 AESE V12.B16, V4.B16 958 AESE V13.B16, V5.B16 959 AESE V14.B16, V6.B16 960 AESE V15.B16, V7.B16 961 962 VEOR V0.B16, V1.B16, V0.B16 963 VEOR V2.B16, V3.B16, V2.B16 964 VEOR V4.B16, V5.B16, V4.B16 965 VEOR V6.B16, V7.B16, V6.B16 966 VEOR V0.B16, V2.B16, V0.B16 967 VEOR V4.B16, V6.B16, V4.B16 968 VEOR V4.B16, V0.B16, V0.B16 969 970 #ifdef GOEXPERIMENT_regabiargs 971 VMOV V0.D[0], R0 972 #else 973 VST1 [V0.D1], (R8) 974 #endif 975 RET 976 977 TEXT runtime·procyield(SB),NOSPLIT,$0-0 978 MOVWU cycles+0(FP), R0 979 again: 980 YIELD 981 SUBW $1, R0 982 CBNZ R0, again 983 RET 984 985 // Save state of caller into g->sched, 986 // but using fake PC from systemstack_switch. 987 // Must only be called from functions with no locals ($0) 988 // or else unwinding from systemstack_switch is incorrect. 989 // Smashes R0. 990 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 991 MOVD $runtime·systemstack_switch(SB), R0 992 ADD $8, R0 // get past prologue 993 MOVD R0, (g_sched+gobuf_pc)(g) 994 MOVD RSP, R0 995 MOVD R0, (g_sched+gobuf_sp)(g) 996 MOVD R29, (g_sched+gobuf_bp)(g) 997 MOVD $0, (g_sched+gobuf_lr)(g) 998 MOVD $0, (g_sched+gobuf_ret)(g) 999 // Assert ctxt is zero. See func save. 1000 MOVD (g_sched+gobuf_ctxt)(g), R0 1001 CBZ R0, 2(PC) 1002 CALL runtime·abort(SB) 1003 RET 1004 1005 // func asmcgocall_no_g(fn, arg unsafe.Pointer) 1006 // Call fn(arg) aligned appropriately for the gcc ABI. 1007 // Called on a system stack, and there may be no g yet (during needm). 1008 TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16 1009 MOVD fn+0(FP), R1 1010 MOVD arg+8(FP), R0 1011 SUB $16, RSP // skip over saved frame pointer below RSP 1012 BL (R1) 1013 ADD $16, RSP // skip over saved frame pointer below RSP 1014 RET 1015 1016 // func asmcgocall(fn, arg unsafe.Pointer) int32 1017 // Call fn(arg) on the scheduler stack, 1018 // aligned appropriately for the gcc ABI. 1019 // See cgocall.go for more details. 1020 TEXT ·asmcgocall(SB),NOSPLIT,$0-20 1021 MOVD fn+0(FP), R1 1022 MOVD arg+8(FP), R0 1023 1024 MOVD RSP, R2 // save original stack pointer 1025 CBZ g, nosave 1026 MOVD g, R4 1027 1028 // Figure out if we need to switch to m->g0 stack. 1029 // We get called to create new OS threads too, and those 1030 // come in on the m->g0 stack already. Or we might already 1031 // be on the m->gsignal stack. 1032 MOVD g_m(g), R8 1033 MOVD m_gsignal(R8), R3 1034 CMP R3, g 1035 BEQ nosave 1036 MOVD m_g0(R8), R3 1037 CMP R3, g 1038 BEQ nosave 1039 1040 // Switch to system stack. 1041 MOVD R0, R9 // gosave_systemstack_switch<> and save_g might clobber R0 1042 BL gosave_systemstack_switch<>(SB) 1043 MOVD R3, g 1044 BL runtime·save_g(SB) 1045 MOVD (g_sched+gobuf_sp)(g), R0 1046 MOVD R0, RSP 1047 MOVD (g_sched+gobuf_bp)(g), R29 1048 MOVD R9, R0 1049 1050 // Now on a scheduling stack (a pthread-created stack). 1051 // Save room for two of our pointers /*, plus 32 bytes of callee 1052 // save area that lives on the caller stack. */ 1053 MOVD RSP, R13 1054 SUB $16, R13 1055 MOVD R13, RSP 1056 MOVD R4, 0(RSP) // save old g on stack 1057 MOVD (g_stack+stack_hi)(R4), R4 1058 SUB R2, R4 1059 MOVD R4, 8(RSP) // save depth in old g stack (can't just save SP, as stack might be copied during a callback) 1060 BL (R1) 1061 MOVD R0, R9 1062 1063 // Restore g, stack pointer. R0 is errno, so don't touch it 1064 MOVD 0(RSP), g 1065 BL runtime·save_g(SB) 1066 MOVD (g_stack+stack_hi)(g), R5 1067 MOVD 8(RSP), R6 1068 SUB R6, R5 1069 MOVD R9, R0 1070 MOVD R5, RSP 1071 1072 MOVW R0, ret+16(FP) 1073 RET 1074 1075 nosave: 1076 // Running on a system stack, perhaps even without a g. 1077 // Having no g can happen during thread creation or thread teardown 1078 // (see needm/dropm on Solaris, for example). 1079 // This code is like the above sequence but without saving/restoring g 1080 // and without worrying about the stack moving out from under us 1081 // (because we're on a system stack, not a goroutine stack). 1082 // The above code could be used directly if already on a system stack, 1083 // but then the only path through this code would be a rare case on Solaris. 1084 // Using this code for all "already on system stack" calls exercises it more, 1085 // which should help keep it correct. 1086 MOVD RSP, R13 1087 SUB $16, R13 1088 MOVD R13, RSP 1089 MOVD $0, R4 1090 MOVD R4, 0(RSP) // Where above code stores g, in case someone looks during debugging. 1091 MOVD R2, 8(RSP) // Save original stack pointer. 1092 BL (R1) 1093 // Restore stack pointer. 1094 MOVD 8(RSP), R2 1095 MOVD R2, RSP 1096 MOVD R0, ret+16(FP) 1097 RET 1098 1099 // cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) 1100 // See cgocall.go for more details. 1101 TEXT ·cgocallback(SB),NOSPLIT,$24-24 1102 NO_LOCAL_POINTERS 1103 1104 // Load g from thread-local storage. 1105 BL runtime·load_g(SB) 1106 1107 // If g is nil, Go did not create the current thread. 1108 // Call needm to obtain one for temporary use. 1109 // In this case, we're running on the thread stack, so there's 1110 // lots of space, but the linker doesn't know. Hide the call from 1111 // the linker analysis by using an indirect call. 1112 CBZ g, needm 1113 1114 MOVD g_m(g), R8 1115 MOVD R8, savedm-8(SP) 1116 B havem 1117 1118 needm: 1119 MOVD g, savedm-8(SP) // g is zero, so is m. 1120 MOVD $runtime·needm(SB), R0 1121 BL (R0) 1122 1123 // Set m->g0->sched.sp = SP, so that if a panic happens 1124 // during the function we are about to execute, it will 1125 // have a valid SP to run on the g0 stack. 1126 // The next few lines (after the havem label) 1127 // will save this SP onto the stack and then write 1128 // the same SP back to m->sched.sp. That seems redundant, 1129 // but if an unrecovered panic happens, unwindm will 1130 // restore the g->sched.sp from the stack location 1131 // and then systemstack will try to use it. If we don't set it here, 1132 // that restored SP will be uninitialized (typically 0) and 1133 // will not be usable. 1134 MOVD g_m(g), R8 1135 MOVD m_g0(R8), R3 1136 MOVD RSP, R0 1137 MOVD R0, (g_sched+gobuf_sp)(R3) 1138 MOVD R29, (g_sched+gobuf_bp)(R3) 1139 1140 havem: 1141 // Now there's a valid m, and we're running on its m->g0. 1142 // Save current m->g0->sched.sp on stack and then set it to SP. 1143 // Save current sp in m->g0->sched.sp in preparation for 1144 // switch back to m->curg stack. 1145 // NOTE: unwindm knows that the saved g->sched.sp is at 16(RSP) aka savedsp-16(SP). 1146 // Beware that the frame size is actually 32+16. 1147 MOVD m_g0(R8), R3 1148 MOVD (g_sched+gobuf_sp)(R3), R4 1149 MOVD R4, savedsp-16(SP) 1150 MOVD RSP, R0 1151 MOVD R0, (g_sched+gobuf_sp)(R3) 1152 1153 // Switch to m->curg stack and call runtime.cgocallbackg. 1154 // Because we are taking over the execution of m->curg 1155 // but *not* resuming what had been running, we need to 1156 // save that information (m->curg->sched) so we can restore it. 1157 // We can restore m->curg->sched.sp easily, because calling 1158 // runtime.cgocallbackg leaves SP unchanged upon return. 1159 // To save m->curg->sched.pc, we push it onto the curg stack and 1160 // open a frame the same size as cgocallback's g0 frame. 1161 // Once we switch to the curg stack, the pushed PC will appear 1162 // to be the return PC of cgocallback, so that the traceback 1163 // will seamlessly trace back into the earlier calls. 1164 MOVD m_curg(R8), g 1165 BL runtime·save_g(SB) 1166 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4 1167 MOVD (g_sched+gobuf_pc)(g), R5 1168 MOVD R5, -48(R4) 1169 MOVD (g_sched+gobuf_bp)(g), R5 1170 MOVD R5, -56(R4) 1171 // Gather our arguments into registers. 1172 MOVD fn+0(FP), R1 1173 MOVD frame+8(FP), R2 1174 MOVD ctxt+16(FP), R3 1175 MOVD $-48(R4), R0 // maintain 16-byte SP alignment 1176 MOVD R0, RSP // switch stack 1177 MOVD R1, 8(RSP) 1178 MOVD R2, 16(RSP) 1179 MOVD R3, 24(RSP) 1180 MOVD $runtime·cgocallbackg(SB), R0 1181 CALL (R0) // indirect call to bypass nosplit check. We're on a different stack now. 1182 1183 // Restore g->sched (== m->curg->sched) from saved values. 1184 MOVD 0(RSP), R5 1185 MOVD R5, (g_sched+gobuf_pc)(g) 1186 MOVD RSP, R4 1187 ADD $48, R4, R4 1188 MOVD R4, (g_sched+gobuf_sp)(g) 1189 1190 // Switch back to m->g0's stack and restore m->g0->sched.sp. 1191 // (Unlike m->curg, the g0 goroutine never uses sched.pc, 1192 // so we do not have to restore it.) 1193 MOVD g_m(g), R8 1194 MOVD m_g0(R8), g 1195 BL runtime·save_g(SB) 1196 MOVD (g_sched+gobuf_sp)(g), R0 1197 MOVD R0, RSP 1198 MOVD savedsp-16(SP), R4 1199 MOVD R4, (g_sched+gobuf_sp)(g) 1200 1201 // If the m on entry was nil, we called needm above to borrow an m 1202 // for the duration of the call. Since the call is over, return it with dropm. 1203 MOVD savedm-8(SP), R6 1204 CBNZ R6, droppedm 1205 MOVD $runtime·dropm(SB), R0 1206 BL (R0) 1207 droppedm: 1208 1209 // Done! 1210 RET 1211 1212 // Called from cgo wrappers, this function returns g->m->curg.stack.hi. 1213 // Must obey the gcc calling convention. 1214 TEXT _cgo_topofstack(SB),NOSPLIT,$24 1215 // g (R28) and REGTMP (R27) might be clobbered by load_g. They 1216 // are callee-save in the gcc calling convention, so save them. 1217 MOVD R27, savedR27-8(SP) 1218 MOVD g, saveG-16(SP) 1219 1220 BL runtime·load_g(SB) 1221 MOVD g_m(g), R0 1222 MOVD m_curg(R0), R0 1223 MOVD (g_stack+stack_hi)(R0), R0 1224 1225 MOVD saveG-16(SP), g 1226 MOVD savedR28-8(SP), R27 1227 RET 1228 1229 // void setg(G*); set g. for use by needm. 1230 TEXT runtime·setg(SB), NOSPLIT, $0-8 1231 MOVD gg+0(FP), g 1232 // This only happens if iscgo, so jump straight to save_g 1233 BL runtime·save_g(SB) 1234 RET 1235 1236 // void setg_gcc(G*); set g called from gcc 1237 TEXT setg_gcc<>(SB),NOSPLIT,$8 1238 MOVD R0, g 1239 MOVD R27, savedR27-8(SP) 1240 BL runtime·save_g(SB) 1241 MOVD savedR27-8(SP), R27 1242 RET 1243 1244 TEXT runtime·emptyfunc(SB),0,$0-0 1245 RET 1246 1247 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0 1248 MOVD ZR, R0 1249 MOVD (R0), R0 1250 UNDEF 1251 1252 TEXT runtime·return0(SB), NOSPLIT, $0 1253 MOVW $0, R0 1254 RET 1255 1256 // The top-most function running on a goroutine 1257 // returns to goexit+PCQuantum. 1258 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0 1259 MOVD R0, R0 // NOP 1260 BL runtime·goexit1(SB) // does not return 1261 1262 // This is called from .init_array and follows the platform, not Go, ABI. 1263 TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0 1264 SUB $0x10, RSP 1265 MOVD R27, 8(RSP) // The access to global variables below implicitly uses R27, which is callee-save 1266 MOVD runtime·lastmoduledatap(SB), R1 1267 MOVD R0, moduledata_next(R1) 1268 MOVD R0, runtime·lastmoduledatap(SB) 1269 MOVD 8(RSP), R27 1270 ADD $0x10, RSP 1271 RET 1272 1273 TEXT ·checkASM(SB),NOSPLIT,$0-1 1274 MOVW $1, R3 1275 MOVB R3, ret+0(FP) 1276 RET 1277 1278 // gcWriteBarrier performs a heap pointer write and informs the GC. 1279 // 1280 // gcWriteBarrier does NOT follow the Go ABI. It takes two arguments: 1281 // - R2 is the destination of the write 1282 // - R3 is the value being written at R2 1283 // It clobbers condition codes. 1284 // It does not clobber any general-purpose registers, 1285 // but may clobber others (e.g., floating point registers) 1286 // The act of CALLing gcWriteBarrier will clobber R30 (LR). 1287 // 1288 // Defined as ABIInternal since the compiler generates ABIInternal 1289 // calls to it directly and it does not use the stack-based Go ABI. 1290 TEXT runtime·gcWriteBarrier<ABIInternal>(SB),NOSPLIT,$200 1291 // Save the registers clobbered by the fast path. 1292 MOVD R0, 184(RSP) 1293 MOVD R1, 192(RSP) 1294 MOVD g_m(g), R0 1295 MOVD m_p(R0), R0 1296 MOVD (p_wbBuf+wbBuf_next)(R0), R1 1297 // Increment wbBuf.next position. 1298 ADD $16, R1 1299 MOVD R1, (p_wbBuf+wbBuf_next)(R0) 1300 MOVD (p_wbBuf+wbBuf_end)(R0), R0 1301 CMP R1, R0 1302 // Record the write. 1303 MOVD R3, -16(R1) // Record value 1304 MOVD (R2), R0 // TODO: This turns bad writes into bad reads. 1305 MOVD R0, -8(R1) // Record *slot 1306 // Is the buffer full? (flags set in CMP above) 1307 BEQ flush 1308 ret: 1309 MOVD 184(RSP), R0 1310 MOVD 192(RSP), R1 1311 // Do the write. 1312 MOVD R3, (R2) 1313 RET 1314 1315 flush: 1316 // Save all general purpose registers since these could be 1317 // clobbered by wbBufFlush and were not saved by the caller. 1318 MOVD R2, 8(RSP) // Also first argument to wbBufFlush 1319 MOVD R3, 16(RSP) // Also second argument to wbBufFlush 1320 // R0 already saved 1321 // R1 already saved 1322 MOVD R4, 24(RSP) 1323 MOVD R5, 32(RSP) 1324 MOVD R6, 40(RSP) 1325 MOVD R7, 48(RSP) 1326 MOVD R8, 56(RSP) 1327 MOVD R9, 64(RSP) 1328 MOVD R10, 72(RSP) 1329 MOVD R11, 80(RSP) 1330 MOVD R12, 88(RSP) 1331 MOVD R13, 96(RSP) 1332 MOVD R14, 104(RSP) 1333 MOVD R15, 112(RSP) 1334 // R16, R17 may be clobbered by linker trampoline 1335 // R18 is unused. 1336 MOVD R19, 120(RSP) 1337 MOVD R20, 128(RSP) 1338 MOVD R21, 136(RSP) 1339 MOVD R22, 144(RSP) 1340 MOVD R23, 152(RSP) 1341 MOVD R24, 160(RSP) 1342 MOVD R25, 168(RSP) 1343 MOVD R26, 176(RSP) 1344 // R27 is temp register. 1345 // R28 is g. 1346 // R29 is frame pointer (unused). 1347 // R30 is LR, which was saved by the prologue. 1348 // R31 is SP. 1349 1350 // This takes arguments R2 and R3. 1351 CALL runtime·wbBufFlush(SB) 1352 1353 MOVD 8(RSP), R2 1354 MOVD 16(RSP), R3 1355 MOVD 24(RSP), R4 1356 MOVD 32(RSP), R5 1357 MOVD 40(RSP), R6 1358 MOVD 48(RSP), R7 1359 MOVD 56(RSP), R8 1360 MOVD 64(RSP), R9 1361 MOVD 72(RSP), R10 1362 MOVD 80(RSP), R11 1363 MOVD 88(RSP), R12 1364 MOVD 96(RSP), R13 1365 MOVD 104(RSP), R14 1366 MOVD 112(RSP), R15 1367 MOVD 120(RSP), R19 1368 MOVD 128(RSP), R20 1369 MOVD 136(RSP), R21 1370 MOVD 144(RSP), R22 1371 MOVD 152(RSP), R23 1372 MOVD 160(RSP), R24 1373 MOVD 168(RSP), R25 1374 MOVD 176(RSP), R26 1375 JMP ret 1376 1377 // Note: these functions use a special calling convention to save generated code space. 1378 // Arguments are passed in registers, but the space for those arguments are allocated 1379 // in the caller's stack frame. These stubs write the args into that stack space and 1380 // then tail call to the corresponding runtime handler. 1381 // The tail call makes these stubs disappear in backtraces. 1382 // 1383 // Defined as ABIInternal since the compiler generates ABIInternal 1384 // calls to it directly and it does not use the stack-based Go ABI. 1385 TEXT runtime·panicIndex<ABIInternal>(SB),NOSPLIT,$0-16 1386 #ifndef GOEXPERIMENT_regabiargs 1387 MOVD R0, x+0(FP) 1388 MOVD R1, y+8(FP) 1389 #endif 1390 JMP runtime·goPanicIndex<ABIInternal>(SB) 1391 TEXT runtime·panicIndexU<ABIInternal>(SB),NOSPLIT,$0-16 1392 #ifndef GOEXPERIMENT_regabiargs 1393 MOVD R0, x+0(FP) 1394 MOVD R1, y+8(FP) 1395 #endif 1396 JMP runtime·goPanicIndexU<ABIInternal>(SB) 1397 TEXT runtime·panicSliceAlen<ABIInternal>(SB),NOSPLIT,$0-16 1398 #ifdef GOEXPERIMENT_regabiargs 1399 MOVD R1, R0 1400 MOVD R2, R1 1401 #else 1402 MOVD R1, x+0(FP) 1403 MOVD R2, y+8(FP) 1404 #endif 1405 JMP runtime·goPanicSliceAlen<ABIInternal>(SB) 1406 TEXT runtime·panicSliceAlenU<ABIInternal>(SB),NOSPLIT,$0-16 1407 #ifdef GOEXPERIMENT_regabiargs 1408 MOVD R1, R0 1409 MOVD R2, R1 1410 #else 1411 MOVD R1, x+0(FP) 1412 MOVD R2, y+8(FP) 1413 #endif 1414 JMP runtime·goPanicSliceAlenU<ABIInternal>(SB) 1415 TEXT runtime·panicSliceAcap<ABIInternal>(SB),NOSPLIT,$0-16 1416 #ifdef GOEXPERIMENT_regabiargs 1417 MOVD R1, R0 1418 MOVD R2, R1 1419 #else 1420 MOVD R1, x+0(FP) 1421 MOVD R2, y+8(FP) 1422 #endif 1423 JMP runtime·goPanicSliceAcap<ABIInternal>(SB) 1424 TEXT runtime·panicSliceAcapU<ABIInternal>(SB),NOSPLIT,$0-16 1425 #ifdef GOEXPERIMENT_regabiargs 1426 MOVD R1, R0 1427 MOVD R2, R1 1428 #else 1429 MOVD R1, x+0(FP) 1430 MOVD R2, y+8(FP) 1431 #endif 1432 JMP runtime·goPanicSliceAcapU<ABIInternal>(SB) 1433 TEXT runtime·panicSliceB<ABIInternal>(SB),NOSPLIT,$0-16 1434 #ifndef GOEXPERIMENT_regabiargs 1435 MOVD R0, x+0(FP) 1436 MOVD R1, y+8(FP) 1437 #endif 1438 JMP runtime·goPanicSliceB<ABIInternal>(SB) 1439 TEXT runtime·panicSliceBU<ABIInternal>(SB),NOSPLIT,$0-16 1440 #ifndef GOEXPERIMENT_regabiargs 1441 MOVD R0, x+0(FP) 1442 MOVD R1, y+8(FP) 1443 #endif 1444 JMP runtime·goPanicSliceBU<ABIInternal>(SB) 1445 TEXT runtime·panicSlice3Alen<ABIInternal>(SB),NOSPLIT,$0-16 1446 #ifdef GOEXPERIMENT_regabiargs 1447 MOVD R2, R0 1448 MOVD R3, R1 1449 #else 1450 MOVD R2, x+0(FP) 1451 MOVD R3, y+8(FP) 1452 #endif 1453 JMP runtime·goPanicSlice3Alen<ABIInternal>(SB) 1454 TEXT runtime·panicSlice3AlenU<ABIInternal>(SB),NOSPLIT,$0-16 1455 #ifdef GOEXPERIMENT_regabiargs 1456 MOVD R2, R0 1457 MOVD R3, R1 1458 #else 1459 MOVD R2, x+0(FP) 1460 MOVD R3, y+8(FP) 1461 #endif 1462 JMP runtime·goPanicSlice3AlenU<ABIInternal>(SB) 1463 TEXT runtime·panicSlice3Acap<ABIInternal>(SB),NOSPLIT,$0-16 1464 #ifdef GOEXPERIMENT_regabiargs 1465 MOVD R2, R0 1466 MOVD R3, R1 1467 #else 1468 MOVD R2, x+0(FP) 1469 MOVD R3, y+8(FP) 1470 #endif 1471 JMP runtime·goPanicSlice3Acap<ABIInternal>(SB) 1472 TEXT runtime·panicSlice3AcapU<ABIInternal>(SB),NOSPLIT,$0-16 1473 #ifdef GOEXPERIMENT_regabiargs 1474 MOVD R2, R0 1475 MOVD R3, R1 1476 #else 1477 MOVD R2, x+0(FP) 1478 MOVD R3, y+8(FP) 1479 #endif 1480 JMP runtime·goPanicSlice3AcapU<ABIInternal>(SB) 1481 TEXT runtime·panicSlice3B<ABIInternal>(SB),NOSPLIT,$0-16 1482 #ifdef GOEXPERIMENT_regabiargs 1483 MOVD R1, R0 1484 MOVD R2, R1 1485 #else 1486 MOVD R1, x+0(FP) 1487 MOVD R2, y+8(FP) 1488 #endif 1489 JMP runtime·goPanicSlice3B<ABIInternal>(SB) 1490 TEXT runtime·panicSlice3BU<ABIInternal>(SB),NOSPLIT,$0-16 1491 #ifdef GOEXPERIMENT_regabiargs 1492 MOVD R1, R0 1493 MOVD R2, R1 1494 #else 1495 MOVD R1, x+0(FP) 1496 MOVD R2, y+8(FP) 1497 #endif 1498 JMP runtime·goPanicSlice3BU<ABIInternal>(SB) 1499 TEXT runtime·panicSlice3C<ABIInternal>(SB),NOSPLIT,$0-16 1500 #ifndef GOEXPERIMENT_regabiargs 1501 MOVD R0, x+0(FP) 1502 MOVD R1, y+8(FP) 1503 #endif 1504 JMP runtime·goPanicSlice3C<ABIInternal>(SB) 1505 TEXT runtime·panicSlice3CU<ABIInternal>(SB),NOSPLIT,$0-16 1506 #ifndef GOEXPERIMENT_regabiargs 1507 MOVD R0, x+0(FP) 1508 MOVD R1, y+8(FP) 1509 #endif 1510 JMP runtime·goPanicSlice3CU<ABIInternal>(SB) 1511 TEXT runtime·panicSliceConvert<ABIInternal>(SB),NOSPLIT,$0-16 1512 #ifdef GOEXPERIMENT_regabiargs 1513 MOVD R2, R0 1514 MOVD R3, R1 1515 #else 1516 MOVD R2, x+0(FP) 1517 MOVD R3, y+8(FP) 1518 #endif 1519 JMP runtime·goPanicSliceConvert<ABIInternal>(SB)