github.com/comwrg/go/src@v0.0.0-20220319063731-c238d0440370/runtime/asm_arm.s (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 #include "go_asm.h" 6 #include "go_tls.h" 7 #include "funcdata.h" 8 #include "textflag.h" 9 10 // _rt0_arm is common startup code for most ARM systems when using 11 // internal linking. This is the entry point for the program from the 12 // kernel for an ordinary -buildmode=exe program. The stack holds the 13 // number of arguments and the C-style argv. 14 TEXT _rt0_arm(SB),NOSPLIT|NOFRAME,$0 15 MOVW (R13), R0 // argc 16 MOVW $4(R13), R1 // argv 17 B runtime·rt0_go(SB) 18 19 // main is common startup code for most ARM systems when using 20 // external linking. The C startup code will call the symbol "main" 21 // passing argc and argv in the usual C ABI registers R0 and R1. 22 TEXT main(SB),NOSPLIT|NOFRAME,$0 23 B runtime·rt0_go(SB) 24 25 // _rt0_arm_lib is common startup code for most ARM systems when 26 // using -buildmode=c-archive or -buildmode=c-shared. The linker will 27 // arrange to invoke this function as a global constructor (for 28 // c-archive) or when the shared library is loaded (for c-shared). 29 // We expect argc and argv to be passed in the usual C ABI registers 30 // R0 and R1. 31 TEXT _rt0_arm_lib(SB),NOSPLIT,$104 32 // Preserve callee-save registers. Raspberry Pi's dlopen(), for example, 33 // actually cares that R11 is preserved. 34 MOVW R4, 12(R13) 35 MOVW R5, 16(R13) 36 MOVW R6, 20(R13) 37 MOVW R7, 24(R13) 38 MOVW R8, 28(R13) 39 MOVW g, 32(R13) 40 MOVW R11, 36(R13) 41 42 // Skip floating point registers on GOARM < 6. 43 MOVB runtime·goarm(SB), R11 44 CMP $6, R11 45 BLT skipfpsave 46 MOVD F8, (40+8*0)(R13) 47 MOVD F9, (40+8*1)(R13) 48 MOVD F10, (40+8*2)(R13) 49 MOVD F11, (40+8*3)(R13) 50 MOVD F12, (40+8*4)(R13) 51 MOVD F13, (40+8*5)(R13) 52 MOVD F14, (40+8*6)(R13) 53 MOVD F15, (40+8*7)(R13) 54 skipfpsave: 55 // Save argc/argv. 56 MOVW R0, _rt0_arm_lib_argc<>(SB) 57 MOVW R1, _rt0_arm_lib_argv<>(SB) 58 59 MOVW $0, g // Initialize g. 60 61 // Synchronous initialization. 62 CALL runtime·libpreinit(SB) 63 64 // Create a new thread to do the runtime initialization. 65 MOVW _cgo_sys_thread_create(SB), R2 66 CMP $0, R2 67 BEQ nocgo 68 MOVW $_rt0_arm_lib_go<>(SB), R0 69 MOVW $0, R1 70 BL (R2) 71 B rr 72 nocgo: 73 MOVW $0x800000, R0 // stacksize = 8192KB 74 MOVW $_rt0_arm_lib_go<>(SB), R1 // fn 75 MOVW R0, 4(R13) 76 MOVW R1, 8(R13) 77 BL runtime·newosproc0(SB) 78 rr: 79 // Restore callee-save registers and return. 80 MOVB runtime·goarm(SB), R11 81 CMP $6, R11 82 BLT skipfprest 83 MOVD (40+8*0)(R13), F8 84 MOVD (40+8*1)(R13), F9 85 MOVD (40+8*2)(R13), F10 86 MOVD (40+8*3)(R13), F11 87 MOVD (40+8*4)(R13), F12 88 MOVD (40+8*5)(R13), F13 89 MOVD (40+8*6)(R13), F14 90 MOVD (40+8*7)(R13), F15 91 skipfprest: 92 MOVW 12(R13), R4 93 MOVW 16(R13), R5 94 MOVW 20(R13), R6 95 MOVW 24(R13), R7 96 MOVW 28(R13), R8 97 MOVW 32(R13), g 98 MOVW 36(R13), R11 99 RET 100 101 // _rt0_arm_lib_go initializes the Go runtime. 102 // This is started in a separate thread by _rt0_arm_lib. 103 TEXT _rt0_arm_lib_go<>(SB),NOSPLIT,$8 104 MOVW _rt0_arm_lib_argc<>(SB), R0 105 MOVW _rt0_arm_lib_argv<>(SB), R1 106 B runtime·rt0_go(SB) 107 108 DATA _rt0_arm_lib_argc<>(SB)/4,$0 109 GLOBL _rt0_arm_lib_argc<>(SB),NOPTR,$4 110 DATA _rt0_arm_lib_argv<>(SB)/4,$0 111 GLOBL _rt0_arm_lib_argv<>(SB),NOPTR,$4 112 113 // using NOFRAME means do not save LR on stack. 114 // argc is in R0, argv is in R1. 115 TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 116 MOVW $0xcafebabe, R12 117 118 // copy arguments forward on an even stack 119 // use R13 instead of SP to avoid linker rewriting the offsets 120 SUB $64, R13 // plenty of scratch 121 AND $~7, R13 122 MOVW R0, 60(R13) // save argc, argv away 123 MOVW R1, 64(R13) 124 125 // set up g register 126 // g is R10 127 MOVW $runtime·g0(SB), g 128 MOVW $runtime·m0(SB), R8 129 130 // save m->g0 = g0 131 MOVW g, m_g0(R8) 132 // save g->m = m0 133 MOVW R8, g_m(g) 134 135 // create istack out of the OS stack 136 // (1MB of system stack is available on iOS and Android) 137 MOVW $(-64*1024+104)(R13), R0 138 MOVW R0, g_stackguard0(g) 139 MOVW R0, g_stackguard1(g) 140 MOVW R0, (g_stack+stack_lo)(g) 141 MOVW R13, (g_stack+stack_hi)(g) 142 143 BL runtime·emptyfunc(SB) // fault if stack check is wrong 144 145 #ifdef GOOS_openbsd 146 // Save g to TLS so that it is available from signal trampoline. 147 BL runtime·save_g(SB) 148 #endif 149 150 BL runtime·_initcgo(SB) // will clobber R0-R3 151 152 // update stackguard after _cgo_init 153 MOVW (g_stack+stack_lo)(g), R0 154 ADD $const__StackGuard, R0 155 MOVW R0, g_stackguard0(g) 156 MOVW R0, g_stackguard1(g) 157 158 BL runtime·check(SB) 159 160 // saved argc, argv 161 MOVW 60(R13), R0 162 MOVW R0, 4(R13) 163 MOVW 64(R13), R1 164 MOVW R1, 8(R13) 165 BL runtime·args(SB) 166 BL runtime·checkgoarm(SB) 167 BL runtime·osinit(SB) 168 BL runtime·schedinit(SB) 169 170 // create a new goroutine to start program 171 MOVW $runtime·mainPC(SB), R0 172 MOVW.W R0, -4(R13) 173 MOVW $8, R0 174 MOVW.W R0, -4(R13) 175 MOVW $0, R0 176 MOVW.W R0, -4(R13) // push $0 as guard 177 BL runtime·newproc(SB) 178 MOVW $12(R13), R13 // pop args and LR 179 180 // start this M 181 BL runtime·mstart(SB) 182 183 MOVW $1234, R0 184 MOVW $1000, R1 185 MOVW R0, (R1) // fail hard 186 187 DATA runtime·mainPC+0(SB)/4,$runtime·main(SB) 188 GLOBL runtime·mainPC(SB),RODATA,$4 189 190 TEXT runtime·breakpoint(SB),NOSPLIT,$0-0 191 // gdb won't skip this breakpoint instruction automatically, 192 // so you must manually "set $pc+=4" to skip it and continue. 193 #ifdef GOOS_plan9 194 WORD $0xD1200070 // undefined instruction used as armv5 breakpoint in Plan 9 195 #else 196 WORD $0xe7f001f0 // undefined instruction that gdb understands is a software breakpoint 197 #endif 198 RET 199 200 TEXT runtime·asminit(SB),NOSPLIT,$0-0 201 // disable runfast (flush-to-zero) mode of vfp if runtime.goarm > 5 202 MOVB runtime·goarm(SB), R11 203 CMP $5, R11 204 BLE 4(PC) 205 WORD $0xeef1ba10 // vmrs r11, fpscr 206 BIC $(1<<24), R11 207 WORD $0xeee1ba10 // vmsr fpscr, r11 208 RET 209 210 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0 211 BL runtime·mstart0(SB) 212 RET // not reached 213 214 /* 215 * go-routine 216 */ 217 218 // void gogo(Gobuf*) 219 // restore state from Gobuf; longjmp 220 TEXT runtime·gogo(SB),NOSPLIT|NOFRAME,$0-4 221 MOVW buf+0(FP), R1 222 MOVW gobuf_g(R1), R0 223 MOVW 0(R0), R2 // make sure g != nil 224 B gogo<>(SB) 225 226 TEXT gogo<>(SB),NOSPLIT|NOFRAME,$0 227 BL setg<>(SB) 228 MOVW gobuf_sp(R1), R13 // restore SP==R13 229 MOVW gobuf_lr(R1), LR 230 MOVW gobuf_ret(R1), R0 231 MOVW gobuf_ctxt(R1), R7 232 MOVW $0, R11 233 MOVW R11, gobuf_sp(R1) // clear to help garbage collector 234 MOVW R11, gobuf_ret(R1) 235 MOVW R11, gobuf_lr(R1) 236 MOVW R11, gobuf_ctxt(R1) 237 MOVW gobuf_pc(R1), R11 238 CMP R11, R11 // set condition codes for == test, needed by stack split 239 B (R11) 240 241 // func mcall(fn func(*g)) 242 // Switch to m->g0's stack, call fn(g). 243 // Fn must never return. It should gogo(&g->sched) 244 // to keep running g. 245 TEXT runtime·mcall(SB),NOSPLIT|NOFRAME,$0-4 246 // Save caller state in g->sched. 247 MOVW R13, (g_sched+gobuf_sp)(g) 248 MOVW LR, (g_sched+gobuf_pc)(g) 249 MOVW $0, R11 250 MOVW R11, (g_sched+gobuf_lr)(g) 251 252 // Switch to m->g0 & its stack, call fn. 253 MOVW g, R1 254 MOVW g_m(g), R8 255 MOVW m_g0(R8), R0 256 BL setg<>(SB) 257 CMP g, R1 258 B.NE 2(PC) 259 B runtime·badmcall(SB) 260 MOVW fn+0(FP), R0 261 MOVW (g_sched+gobuf_sp)(g), R13 262 SUB $8, R13 263 MOVW R1, 4(R13) 264 MOVW R0, R7 265 MOVW 0(R0), R0 266 BL (R0) 267 B runtime·badmcall2(SB) 268 RET 269 270 // systemstack_switch is a dummy routine that systemstack leaves at the bottom 271 // of the G stack. We need to distinguish the routine that 272 // lives at the bottom of the G stack from the one that lives 273 // at the top of the system stack because the one at the top of 274 // the system stack terminates the stack walk (see topofstack()). 275 TEXT runtime·systemstack_switch(SB),NOSPLIT,$0-0 276 MOVW $0, R0 277 BL (R0) // clobber lr to ensure push {lr} is kept 278 RET 279 280 // func systemstack(fn func()) 281 TEXT runtime·systemstack(SB),NOSPLIT,$0-4 282 MOVW fn+0(FP), R0 // R0 = fn 283 MOVW g_m(g), R1 // R1 = m 284 285 MOVW m_gsignal(R1), R2 // R2 = gsignal 286 CMP g, R2 287 B.EQ noswitch 288 289 MOVW m_g0(R1), R2 // R2 = g0 290 CMP g, R2 291 B.EQ noswitch 292 293 MOVW m_curg(R1), R3 294 CMP g, R3 295 B.EQ switch 296 297 // Bad: g is not gsignal, not g0, not curg. What is it? 298 // Hide call from linker nosplit analysis. 299 MOVW $runtime·badsystemstack(SB), R0 300 BL (R0) 301 B runtime·abort(SB) 302 303 switch: 304 // save our state in g->sched. Pretend to 305 // be systemstack_switch if the G stack is scanned. 306 BL gosave_systemstack_switch<>(SB) 307 308 // switch to g0 309 MOVW R0, R5 310 MOVW R2, R0 311 BL setg<>(SB) 312 MOVW R5, R0 313 MOVW (g_sched+gobuf_sp)(R2), R13 314 315 // call target function 316 MOVW R0, R7 317 MOVW 0(R0), R0 318 BL (R0) 319 320 // switch back to g 321 MOVW g_m(g), R1 322 MOVW m_curg(R1), R0 323 BL setg<>(SB) 324 MOVW (g_sched+gobuf_sp)(g), R13 325 MOVW $0, R3 326 MOVW R3, (g_sched+gobuf_sp)(g) 327 RET 328 329 noswitch: 330 // Using a tail call here cleans up tracebacks since we won't stop 331 // at an intermediate systemstack. 332 MOVW R0, R7 333 MOVW 0(R0), R0 334 MOVW.P 4(R13), R14 // restore LR 335 B (R0) 336 337 /* 338 * support for morestack 339 */ 340 341 // Called during function prolog when more stack is needed. 342 // R3 prolog's LR 343 // using NOFRAME means do not save LR on stack. 344 // 345 // The traceback routines see morestack on a g0 as being 346 // the top of a stack (for example, morestack calling newstack 347 // calling the scheduler calling newm calling gc), so we must 348 // record an argument size. For that purpose, it has no arguments. 349 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0 350 // Cannot grow scheduler stack (m->g0). 351 MOVW g_m(g), R8 352 MOVW m_g0(R8), R4 353 CMP g, R4 354 BNE 3(PC) 355 BL runtime·badmorestackg0(SB) 356 B runtime·abort(SB) 357 358 // Cannot grow signal stack (m->gsignal). 359 MOVW m_gsignal(R8), R4 360 CMP g, R4 361 BNE 3(PC) 362 BL runtime·badmorestackgsignal(SB) 363 B runtime·abort(SB) 364 365 // Called from f. 366 // Set g->sched to context in f. 367 MOVW R13, (g_sched+gobuf_sp)(g) 368 MOVW LR, (g_sched+gobuf_pc)(g) 369 MOVW R3, (g_sched+gobuf_lr)(g) 370 MOVW R7, (g_sched+gobuf_ctxt)(g) 371 372 // Called from f. 373 // Set m->morebuf to f's caller. 374 MOVW R3, (m_morebuf+gobuf_pc)(R8) // f's caller's PC 375 MOVW R13, (m_morebuf+gobuf_sp)(R8) // f's caller's SP 376 MOVW g, (m_morebuf+gobuf_g)(R8) 377 378 // Call newstack on m->g0's stack. 379 MOVW m_g0(R8), R0 380 BL setg<>(SB) 381 MOVW (g_sched+gobuf_sp)(g), R13 382 MOVW $0, R0 383 MOVW.W R0, -4(R13) // create a call frame on g0 (saved LR) 384 BL runtime·newstack(SB) 385 386 // Not reached, but make sure the return PC from the call to newstack 387 // is still in this function, and not the beginning of the next. 388 RET 389 390 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0 391 MOVW $0, R7 392 B runtime·morestack(SB) 393 394 // reflectcall: call a function with the given argument list 395 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs). 396 // we don't have variable-sized frames, so we use a small number 397 // of constant-sized-frame functions to encode a few bits of size in the pc. 398 // Caution: ugly multiline assembly macros in your future! 399 400 #define DISPATCH(NAME,MAXSIZE) \ 401 CMP $MAXSIZE, R0; \ 402 B.HI 3(PC); \ 403 MOVW $NAME(SB), R1; \ 404 B (R1) 405 406 TEXT ·reflectcall(SB),NOSPLIT|NOFRAME,$0-28 407 MOVW frameSize+20(FP), R0 408 DISPATCH(runtime·call16, 16) 409 DISPATCH(runtime·call32, 32) 410 DISPATCH(runtime·call64, 64) 411 DISPATCH(runtime·call128, 128) 412 DISPATCH(runtime·call256, 256) 413 DISPATCH(runtime·call512, 512) 414 DISPATCH(runtime·call1024, 1024) 415 DISPATCH(runtime·call2048, 2048) 416 DISPATCH(runtime·call4096, 4096) 417 DISPATCH(runtime·call8192, 8192) 418 DISPATCH(runtime·call16384, 16384) 419 DISPATCH(runtime·call32768, 32768) 420 DISPATCH(runtime·call65536, 65536) 421 DISPATCH(runtime·call131072, 131072) 422 DISPATCH(runtime·call262144, 262144) 423 DISPATCH(runtime·call524288, 524288) 424 DISPATCH(runtime·call1048576, 1048576) 425 DISPATCH(runtime·call2097152, 2097152) 426 DISPATCH(runtime·call4194304, 4194304) 427 DISPATCH(runtime·call8388608, 8388608) 428 DISPATCH(runtime·call16777216, 16777216) 429 DISPATCH(runtime·call33554432, 33554432) 430 DISPATCH(runtime·call67108864, 67108864) 431 DISPATCH(runtime·call134217728, 134217728) 432 DISPATCH(runtime·call268435456, 268435456) 433 DISPATCH(runtime·call536870912, 536870912) 434 DISPATCH(runtime·call1073741824, 1073741824) 435 MOVW $runtime·badreflectcall(SB), R1 436 B (R1) 437 438 #define CALLFN(NAME,MAXSIZE) \ 439 TEXT NAME(SB), WRAPPER, $MAXSIZE-28; \ 440 NO_LOCAL_POINTERS; \ 441 /* copy arguments to stack */ \ 442 MOVW stackArgs+8(FP), R0; \ 443 MOVW stackArgsSize+12(FP), R2; \ 444 ADD $4, R13, R1; \ 445 CMP $0, R2; \ 446 B.EQ 5(PC); \ 447 MOVBU.P 1(R0), R5; \ 448 MOVBU.P R5, 1(R1); \ 449 SUB $1, R2, R2; \ 450 B -5(PC); \ 451 /* call function */ \ 452 MOVW f+4(FP), R7; \ 453 MOVW (R7), R0; \ 454 PCDATA $PCDATA_StackMapIndex, $0; \ 455 BL (R0); \ 456 /* copy return values back */ \ 457 MOVW stackArgsType+0(FP), R4; \ 458 MOVW stackArgs+8(FP), R0; \ 459 MOVW stackArgsSize+12(FP), R2; \ 460 MOVW stackArgsRetOffset+16(FP), R3; \ 461 ADD $4, R13, R1; \ 462 ADD R3, R1; \ 463 ADD R3, R0; \ 464 SUB R3, R2; \ 465 BL callRet<>(SB); \ 466 RET 467 468 // callRet copies return values back at the end of call*. This is a 469 // separate function so it can allocate stack space for the arguments 470 // to reflectcallmove. It does not follow the Go ABI; it expects its 471 // arguments in registers. 472 TEXT callRet<>(SB), NOSPLIT, $20-0 473 MOVW R4, 4(R13) 474 MOVW R0, 8(R13) 475 MOVW R1, 12(R13) 476 MOVW R2, 16(R13) 477 MOVW $0, R7 478 MOVW R7, 20(R13) 479 BL runtime·reflectcallmove(SB) 480 RET 481 482 CALLFN(·call16, 16) 483 CALLFN(·call32, 32) 484 CALLFN(·call64, 64) 485 CALLFN(·call128, 128) 486 CALLFN(·call256, 256) 487 CALLFN(·call512, 512) 488 CALLFN(·call1024, 1024) 489 CALLFN(·call2048, 2048) 490 CALLFN(·call4096, 4096) 491 CALLFN(·call8192, 8192) 492 CALLFN(·call16384, 16384) 493 CALLFN(·call32768, 32768) 494 CALLFN(·call65536, 65536) 495 CALLFN(·call131072, 131072) 496 CALLFN(·call262144, 262144) 497 CALLFN(·call524288, 524288) 498 CALLFN(·call1048576, 1048576) 499 CALLFN(·call2097152, 2097152) 500 CALLFN(·call4194304, 4194304) 501 CALLFN(·call8388608, 8388608) 502 CALLFN(·call16777216, 16777216) 503 CALLFN(·call33554432, 33554432) 504 CALLFN(·call67108864, 67108864) 505 CALLFN(·call134217728, 134217728) 506 CALLFN(·call268435456, 268435456) 507 CALLFN(·call536870912, 536870912) 508 CALLFN(·call1073741824, 1073741824) 509 510 // void jmpdefer(fn, sp); 511 // called from deferreturn. 512 // 1. grab stored LR for caller 513 // 2. sub 4 bytes to get back to BL deferreturn 514 // 3. B to fn 515 TEXT runtime·jmpdefer(SB),NOSPLIT,$0-8 516 MOVW 0(R13), LR 517 MOVW $-4(LR), LR // BL deferreturn 518 MOVW fv+0(FP), R7 519 MOVW argp+4(FP), R13 520 MOVW $-4(R13), R13 // SP is 4 below argp, due to saved LR 521 MOVW 0(R7), R1 522 B (R1) 523 524 // Save state of caller into g->sched, 525 // but using fake PC from systemstack_switch. 526 // Must only be called from functions with no locals ($0) 527 // or else unwinding from systemstack_switch is incorrect. 528 // Smashes R11. 529 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 530 MOVW $runtime·systemstack_switch(SB), R11 531 ADD $4, R11 // get past push {lr} 532 MOVW R11, (g_sched+gobuf_pc)(g) 533 MOVW R13, (g_sched+gobuf_sp)(g) 534 MOVW $0, R11 535 MOVW R11, (g_sched+gobuf_lr)(g) 536 MOVW R11, (g_sched+gobuf_ret)(g) 537 // Assert ctxt is zero. See func save. 538 MOVW (g_sched+gobuf_ctxt)(g), R11 539 TST R11, R11 540 B.EQ 2(PC) 541 BL runtime·abort(SB) 542 RET 543 544 // func asmcgocall_no_g(fn, arg unsafe.Pointer) 545 // Call fn(arg) aligned appropriately for the gcc ABI. 546 // Called on a system stack, and there may be no g yet (during needm). 547 TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-8 548 MOVW fn+0(FP), R1 549 MOVW arg+4(FP), R0 550 MOVW R13, R2 551 SUB $32, R13 552 BIC $0x7, R13 // alignment for gcc ABI 553 MOVW R2, 8(R13) 554 BL (R1) 555 MOVW 8(R13), R2 556 MOVW R2, R13 557 RET 558 559 // func asmcgocall(fn, arg unsafe.Pointer) int32 560 // Call fn(arg) on the scheduler stack, 561 // aligned appropriately for the gcc ABI. 562 // See cgocall.go for more details. 563 TEXT ·asmcgocall(SB),NOSPLIT,$0-12 564 MOVW fn+0(FP), R1 565 MOVW arg+4(FP), R0 566 567 MOVW R13, R2 568 CMP $0, g 569 BEQ nosave 570 MOVW g, R4 571 572 // Figure out if we need to switch to m->g0 stack. 573 // We get called to create new OS threads too, and those 574 // come in on the m->g0 stack already. 575 MOVW g_m(g), R8 576 MOVW m_gsignal(R8), R3 577 CMP R3, g 578 BEQ nosave 579 MOVW m_g0(R8), R3 580 CMP R3, g 581 BEQ nosave 582 BL gosave_systemstack_switch<>(SB) 583 MOVW R0, R5 584 MOVW R3, R0 585 BL setg<>(SB) 586 MOVW R5, R0 587 MOVW (g_sched+gobuf_sp)(g), R13 588 589 // Now on a scheduling stack (a pthread-created stack). 590 SUB $24, R13 591 BIC $0x7, R13 // alignment for gcc ABI 592 MOVW R4, 20(R13) // save old g 593 MOVW (g_stack+stack_hi)(R4), R4 594 SUB R2, R4 595 MOVW R4, 16(R13) // save depth in stack (can't just save SP, as stack might be copied during a callback) 596 BL (R1) 597 598 // Restore registers, g, stack pointer. 599 MOVW R0, R5 600 MOVW 20(R13), R0 601 BL setg<>(SB) 602 MOVW (g_stack+stack_hi)(g), R1 603 MOVW 16(R13), R2 604 SUB R2, R1 605 MOVW R5, R0 606 MOVW R1, R13 607 608 MOVW R0, ret+8(FP) 609 RET 610 611 nosave: 612 // Running on a system stack, perhaps even without a g. 613 // Having no g can happen during thread creation or thread teardown 614 // (see needm/dropm on Solaris, for example). 615 // This code is like the above sequence but without saving/restoring g 616 // and without worrying about the stack moving out from under us 617 // (because we're on a system stack, not a goroutine stack). 618 // The above code could be used directly if already on a system stack, 619 // but then the only path through this code would be a rare case on Solaris. 620 // Using this code for all "already on system stack" calls exercises it more, 621 // which should help keep it correct. 622 SUB $24, R13 623 BIC $0x7, R13 // alignment for gcc ABI 624 // save null g in case someone looks during debugging. 625 MOVW $0, R4 626 MOVW R4, 20(R13) 627 MOVW R2, 16(R13) // Save old stack pointer. 628 BL (R1) 629 // Restore stack pointer. 630 MOVW 16(R13), R2 631 MOVW R2, R13 632 MOVW R0, ret+8(FP) 633 RET 634 635 // cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) 636 // See cgocall.go for more details. 637 TEXT ·cgocallback(SB),NOSPLIT,$12-12 638 NO_LOCAL_POINTERS 639 640 // Load m and g from thread-local storage. 641 #ifdef GOOS_openbsd 642 BL runtime·load_g(SB) 643 #else 644 MOVB runtime·iscgo(SB), R0 645 CMP $0, R0 646 BL.NE runtime·load_g(SB) 647 #endif 648 649 // If g is nil, Go did not create the current thread. 650 // Call needm to obtain one for temporary use. 651 // In this case, we're running on the thread stack, so there's 652 // lots of space, but the linker doesn't know. Hide the call from 653 // the linker analysis by using an indirect call. 654 CMP $0, g 655 B.EQ needm 656 657 MOVW g_m(g), R8 658 MOVW R8, savedm-4(SP) 659 B havem 660 661 needm: 662 MOVW g, savedm-4(SP) // g is zero, so is m. 663 MOVW $runtime·needm(SB), R0 664 BL (R0) 665 666 // Set m->g0->sched.sp = SP, so that if a panic happens 667 // during the function we are about to execute, it will 668 // have a valid SP to run on the g0 stack. 669 // The next few lines (after the havem label) 670 // will save this SP onto the stack and then write 671 // the same SP back to m->sched.sp. That seems redundant, 672 // but if an unrecovered panic happens, unwindm will 673 // restore the g->sched.sp from the stack location 674 // and then systemstack will try to use it. If we don't set it here, 675 // that restored SP will be uninitialized (typically 0) and 676 // will not be usable. 677 MOVW g_m(g), R8 678 MOVW m_g0(R8), R3 679 MOVW R13, (g_sched+gobuf_sp)(R3) 680 681 havem: 682 // Now there's a valid m, and we're running on its m->g0. 683 // Save current m->g0->sched.sp on stack and then set it to SP. 684 // Save current sp in m->g0->sched.sp in preparation for 685 // switch back to m->curg stack. 686 // NOTE: unwindm knows that the saved g->sched.sp is at 4(R13) aka savedsp-12(SP). 687 MOVW m_g0(R8), R3 688 MOVW (g_sched+gobuf_sp)(R3), R4 689 MOVW R4, savedsp-12(SP) // must match frame size 690 MOVW R13, (g_sched+gobuf_sp)(R3) 691 692 // Switch to m->curg stack and call runtime.cgocallbackg. 693 // Because we are taking over the execution of m->curg 694 // but *not* resuming what had been running, we need to 695 // save that information (m->curg->sched) so we can restore it. 696 // We can restore m->curg->sched.sp easily, because calling 697 // runtime.cgocallbackg leaves SP unchanged upon return. 698 // To save m->curg->sched.pc, we push it onto the curg stack and 699 // open a frame the same size as cgocallback's g0 frame. 700 // Once we switch to the curg stack, the pushed PC will appear 701 // to be the return PC of cgocallback, so that the traceback 702 // will seamlessly trace back into the earlier calls. 703 MOVW m_curg(R8), R0 704 BL setg<>(SB) 705 MOVW (g_sched+gobuf_sp)(g), R4 // prepare stack as R4 706 MOVW (g_sched+gobuf_pc)(g), R5 707 MOVW R5, -(12+4)(R4) // "saved LR"; must match frame size 708 // Gather our arguments into registers. 709 MOVW fn+0(FP), R1 710 MOVW frame+4(FP), R2 711 MOVW ctxt+8(FP), R3 712 MOVW $-(12+4)(R4), R13 // switch stack; must match frame size 713 MOVW R1, 4(R13) 714 MOVW R2, 8(R13) 715 MOVW R3, 12(R13) 716 BL runtime·cgocallbackg(SB) 717 718 // Restore g->sched (== m->curg->sched) from saved values. 719 MOVW 0(R13), R5 720 MOVW R5, (g_sched+gobuf_pc)(g) 721 MOVW $(12+4)(R13), R4 // must match frame size 722 MOVW R4, (g_sched+gobuf_sp)(g) 723 724 // Switch back to m->g0's stack and restore m->g0->sched.sp. 725 // (Unlike m->curg, the g0 goroutine never uses sched.pc, 726 // so we do not have to restore it.) 727 MOVW g_m(g), R8 728 MOVW m_g0(R8), R0 729 BL setg<>(SB) 730 MOVW (g_sched+gobuf_sp)(g), R13 731 MOVW savedsp-12(SP), R4 // must match frame size 732 MOVW R4, (g_sched+gobuf_sp)(g) 733 734 // If the m on entry was nil, we called needm above to borrow an m 735 // for the duration of the call. Since the call is over, return it with dropm. 736 MOVW savedm-4(SP), R6 737 CMP $0, R6 738 B.NE 3(PC) 739 MOVW $runtime·dropm(SB), R0 740 BL (R0) 741 742 // Done! 743 RET 744 745 // void setg(G*); set g. for use by needm. 746 TEXT runtime·setg(SB),NOSPLIT|NOFRAME,$0-4 747 MOVW gg+0(FP), R0 748 B setg<>(SB) 749 750 TEXT setg<>(SB),NOSPLIT|NOFRAME,$0-0 751 MOVW R0, g 752 753 // Save g to thread-local storage. 754 #ifdef GOOS_windows 755 B runtime·save_g(SB) 756 #else 757 #ifdef GOOS_openbsd 758 B runtime·save_g(SB) 759 #else 760 MOVB runtime·iscgo(SB), R0 761 CMP $0, R0 762 B.EQ 2(PC) 763 B runtime·save_g(SB) 764 765 MOVW g, R0 766 RET 767 #endif 768 #endif 769 770 TEXT runtime·emptyfunc(SB),0,$0-0 771 RET 772 773 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0 774 MOVW $0, R0 775 MOVW (R0), R1 776 777 // armPublicationBarrier is a native store/store barrier for ARMv7+. 778 // On earlier ARM revisions, armPublicationBarrier is a no-op. 779 // This will not work on SMP ARMv6 machines, if any are in use. 780 // To implement publicationBarrier in sys_$GOOS_arm.s using the native 781 // instructions, use: 782 // 783 // TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0 784 // B runtime·armPublicationBarrier(SB) 785 // 786 TEXT runtime·armPublicationBarrier(SB),NOSPLIT|NOFRAME,$0-0 787 MOVB runtime·goarm(SB), R11 788 CMP $7, R11 789 BLT 2(PC) 790 DMB MB_ST 791 RET 792 793 // AES hashing not implemented for ARM 794 TEXT runtime·memhash(SB),NOSPLIT|NOFRAME,$0-16 795 JMP runtime·memhashFallback(SB) 796 TEXT runtime·strhash(SB),NOSPLIT|NOFRAME,$0-12 797 JMP runtime·strhashFallback(SB) 798 TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-12 799 JMP runtime·memhash32Fallback(SB) 800 TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-12 801 JMP runtime·memhash64Fallback(SB) 802 803 TEXT runtime·return0(SB),NOSPLIT,$0 804 MOVW $0, R0 805 RET 806 807 TEXT runtime·procyield(SB),NOSPLIT|NOFRAME,$0 808 MOVW cycles+0(FP), R1 809 MOVW $0, R0 810 yieldloop: 811 WORD $0xe320f001 // YIELD (NOP pre-ARMv6K) 812 CMP R0, R1 813 B.NE 2(PC) 814 RET 815 SUB $1, R1 816 B yieldloop 817 818 // Called from cgo wrappers, this function returns g->m->curg.stack.hi. 819 // Must obey the gcc calling convention. 820 TEXT _cgo_topofstack(SB),NOSPLIT,$8 821 // R11 and g register are clobbered by load_g. They are 822 // callee-save in the gcc calling convention, so save them here. 823 MOVW R11, saveR11-4(SP) 824 MOVW g, saveG-8(SP) 825 826 BL runtime·load_g(SB) 827 MOVW g_m(g), R0 828 MOVW m_curg(R0), R0 829 MOVW (g_stack+stack_hi)(R0), R0 830 831 MOVW saveG-8(SP), g 832 MOVW saveR11-4(SP), R11 833 RET 834 835 // The top-most function running on a goroutine 836 // returns to goexit+PCQuantum. 837 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0 838 MOVW R0, R0 // NOP 839 BL runtime·goexit1(SB) // does not return 840 // traceback from goexit1 must hit code range of goexit 841 MOVW R0, R0 // NOP 842 843 // x -> x/1000000, x%1000000, called from Go with args, results on stack. 844 TEXT runtime·usplit(SB),NOSPLIT,$0-12 845 MOVW x+0(FP), R0 846 CALL runtime·usplitR0(SB) 847 MOVW R0, q+4(FP) 848 MOVW R1, r+8(FP) 849 RET 850 851 // R0, R1 = R0/1000000, R0%1000000 852 TEXT runtime·usplitR0(SB),NOSPLIT,$0 853 // magic multiply to avoid software divide without available m. 854 // see output of go tool compile -S for x/1000000. 855 MOVW R0, R3 856 MOVW $1125899907, R1 857 MULLU R1, R0, (R0, R1) 858 MOVW R0>>18, R0 859 MOVW $1000000, R1 860 MULU R0, R1 861 SUB R1, R3, R1 862 RET 863 864 // This is called from .init_array and follows the platform, not Go, ABI. 865 TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0 866 MOVW R9, saver9-4(SP) // The access to global variables below implicitly uses R9, which is callee-save 867 MOVW R11, saver11-8(SP) // Likewise, R11 is the temp register, but callee-save in C ABI 868 MOVW runtime·lastmoduledatap(SB), R1 869 MOVW R0, moduledata_next(R1) 870 MOVW R0, runtime·lastmoduledatap(SB) 871 MOVW saver11-8(SP), R11 872 MOVW saver9-4(SP), R9 873 RET 874 875 TEXT ·checkASM(SB),NOSPLIT,$0-1 876 MOVW $1, R3 877 MOVB R3, ret+0(FP) 878 RET 879 880 // gcWriteBarrier performs a heap pointer write and informs the GC. 881 // 882 // gcWriteBarrier does NOT follow the Go ABI. It takes two arguments: 883 // - R2 is the destination of the write 884 // - R3 is the value being written at R2 885 // It clobbers condition codes. 886 // It does not clobber any other general-purpose registers, 887 // but may clobber others (e.g., floating point registers). 888 // The act of CALLing gcWriteBarrier will clobber R14 (LR). 889 TEXT runtime·gcWriteBarrier(SB),NOSPLIT|NOFRAME,$0 890 // Save the registers clobbered by the fast path. 891 MOVM.DB.W [R0,R1], (R13) 892 MOVW g_m(g), R0 893 MOVW m_p(R0), R0 894 MOVW (p_wbBuf+wbBuf_next)(R0), R1 895 // Increment wbBuf.next position. 896 ADD $8, R1 897 MOVW R1, (p_wbBuf+wbBuf_next)(R0) 898 MOVW (p_wbBuf+wbBuf_end)(R0), R0 899 CMP R1, R0 900 // Record the write. 901 MOVW R3, -8(R1) // Record value 902 MOVW (R2), R0 // TODO: This turns bad writes into bad reads. 903 MOVW R0, -4(R1) // Record *slot 904 // Is the buffer full? (flags set in CMP above) 905 B.EQ flush 906 ret: 907 MOVM.IA.W (R13), [R0,R1] 908 // Do the write. 909 MOVW R3, (R2) 910 RET 911 912 flush: 913 // Save all general purpose registers since these could be 914 // clobbered by wbBufFlush and were not saved by the caller. 915 // 916 // R0 and R1 were saved at entry. 917 // R10 is g, so preserved. 918 // R11 is linker temp, so no need to save. 919 // R13 is stack pointer. 920 // R15 is PC. 921 // 922 // This also sets up R2 and R3 as the arguments to wbBufFlush. 923 MOVM.DB.W [R2-R9,R12], (R13) 924 // Save R14 (LR) because the fast path above doesn't save it, 925 // but needs it to RET. This is after the MOVM so it appears below 926 // the arguments in the stack frame. 927 MOVM.DB.W [R14], (R13) 928 929 // This takes arguments R2 and R3. 930 CALL runtime·wbBufFlush(SB) 931 932 MOVM.IA.W (R13), [R14] 933 MOVM.IA.W (R13), [R2-R9,R12] 934 JMP ret 935 936 // Note: these functions use a special calling convention to save generated code space. 937 // Arguments are passed in registers, but the space for those arguments are allocated 938 // in the caller's stack frame. These stubs write the args into that stack space and 939 // then tail call to the corresponding runtime handler. 940 // The tail call makes these stubs disappear in backtraces. 941 TEXT runtime·panicIndex(SB),NOSPLIT,$0-8 942 MOVW R0, x+0(FP) 943 MOVW R1, y+4(FP) 944 JMP runtime·goPanicIndex(SB) 945 TEXT runtime·panicIndexU(SB),NOSPLIT,$0-8 946 MOVW R0, x+0(FP) 947 MOVW R1, y+4(FP) 948 JMP runtime·goPanicIndexU(SB) 949 TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-8 950 MOVW R1, x+0(FP) 951 MOVW R2, y+4(FP) 952 JMP runtime·goPanicSliceAlen(SB) 953 TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-8 954 MOVW R1, x+0(FP) 955 MOVW R2, y+4(FP) 956 JMP runtime·goPanicSliceAlenU(SB) 957 TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-8 958 MOVW R1, x+0(FP) 959 MOVW R2, y+4(FP) 960 JMP runtime·goPanicSliceAcap(SB) 961 TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-8 962 MOVW R1, x+0(FP) 963 MOVW R2, y+4(FP) 964 JMP runtime·goPanicSliceAcapU(SB) 965 TEXT runtime·panicSliceB(SB),NOSPLIT,$0-8 966 MOVW R0, x+0(FP) 967 MOVW R1, y+4(FP) 968 JMP runtime·goPanicSliceB(SB) 969 TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-8 970 MOVW R0, x+0(FP) 971 MOVW R1, y+4(FP) 972 JMP runtime·goPanicSliceBU(SB) 973 TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-8 974 MOVW R2, x+0(FP) 975 MOVW R3, y+4(FP) 976 JMP runtime·goPanicSlice3Alen(SB) 977 TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-8 978 MOVW R2, x+0(FP) 979 MOVW R3, y+4(FP) 980 JMP runtime·goPanicSlice3AlenU(SB) 981 TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-8 982 MOVW R2, x+0(FP) 983 MOVW R3, y+4(FP) 984 JMP runtime·goPanicSlice3Acap(SB) 985 TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-8 986 MOVW R2, x+0(FP) 987 MOVW R3, y+4(FP) 988 JMP runtime·goPanicSlice3AcapU(SB) 989 TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-8 990 MOVW R1, x+0(FP) 991 MOVW R2, y+4(FP) 992 JMP runtime·goPanicSlice3B(SB) 993 TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-8 994 MOVW R1, x+0(FP) 995 MOVW R2, y+4(FP) 996 JMP runtime·goPanicSlice3BU(SB) 997 TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-8 998 MOVW R0, x+0(FP) 999 MOVW R1, y+4(FP) 1000 JMP runtime·goPanicSlice3C(SB) 1001 TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-8 1002 MOVW R0, x+0(FP) 1003 MOVW R1, y+4(FP) 1004 JMP runtime·goPanicSlice3CU(SB) 1005 TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-8 1006 MOVW R2, x+0(FP) 1007 MOVW R3, y+4(FP) 1008 JMP runtime·goPanicSliceConvert(SB) 1009 1010 // Extended versions for 64-bit indexes. 1011 TEXT runtime·panicExtendIndex(SB),NOSPLIT,$0-12 1012 MOVW R4, hi+0(FP) 1013 MOVW R0, lo+4(FP) 1014 MOVW R1, y+8(FP) 1015 JMP runtime·goPanicExtendIndex(SB) 1016 TEXT runtime·panicExtendIndexU(SB),NOSPLIT,$0-12 1017 MOVW R4, hi+0(FP) 1018 MOVW R0, lo+4(FP) 1019 MOVW R1, y+8(FP) 1020 JMP runtime·goPanicExtendIndexU(SB) 1021 TEXT runtime·panicExtendSliceAlen(SB),NOSPLIT,$0-12 1022 MOVW R4, hi+0(FP) 1023 MOVW R1, lo+4(FP) 1024 MOVW R2, y+8(FP) 1025 JMP runtime·goPanicExtendSliceAlen(SB) 1026 TEXT runtime·panicExtendSliceAlenU(SB),NOSPLIT,$0-12 1027 MOVW R4, hi+0(FP) 1028 MOVW R1, lo+4(FP) 1029 MOVW R2, y+8(FP) 1030 JMP runtime·goPanicExtendSliceAlenU(SB) 1031 TEXT runtime·panicExtendSliceAcap(SB),NOSPLIT,$0-12 1032 MOVW R4, hi+0(FP) 1033 MOVW R1, lo+4(FP) 1034 MOVW R2, y+8(FP) 1035 JMP runtime·goPanicExtendSliceAcap(SB) 1036 TEXT runtime·panicExtendSliceAcapU(SB),NOSPLIT,$0-12 1037 MOVW R4, hi+0(FP) 1038 MOVW R1, lo+4(FP) 1039 MOVW R2, y+8(FP) 1040 JMP runtime·goPanicExtendSliceAcapU(SB) 1041 TEXT runtime·panicExtendSliceB(SB),NOSPLIT,$0-12 1042 MOVW R4, hi+0(FP) 1043 MOVW R0, lo+4(FP) 1044 MOVW R1, y+8(FP) 1045 JMP runtime·goPanicExtendSliceB(SB) 1046 TEXT runtime·panicExtendSliceBU(SB),NOSPLIT,$0-12 1047 MOVW R4, hi+0(FP) 1048 MOVW R0, lo+4(FP) 1049 MOVW R1, y+8(FP) 1050 JMP runtime·goPanicExtendSliceBU(SB) 1051 TEXT runtime·panicExtendSlice3Alen(SB),NOSPLIT,$0-12 1052 MOVW R4, hi+0(FP) 1053 MOVW R2, lo+4(FP) 1054 MOVW R3, y+8(FP) 1055 JMP runtime·goPanicExtendSlice3Alen(SB) 1056 TEXT runtime·panicExtendSlice3AlenU(SB),NOSPLIT,$0-12 1057 MOVW R4, hi+0(FP) 1058 MOVW R2, lo+4(FP) 1059 MOVW R3, y+8(FP) 1060 JMP runtime·goPanicExtendSlice3AlenU(SB) 1061 TEXT runtime·panicExtendSlice3Acap(SB),NOSPLIT,$0-12 1062 MOVW R4, hi+0(FP) 1063 MOVW R2, lo+4(FP) 1064 MOVW R3, y+8(FP) 1065 JMP runtime·goPanicExtendSlice3Acap(SB) 1066 TEXT runtime·panicExtendSlice3AcapU(SB),NOSPLIT,$0-12 1067 MOVW R4, hi+0(FP) 1068 MOVW R2, lo+4(FP) 1069 MOVW R3, y+8(FP) 1070 JMP runtime·goPanicExtendSlice3AcapU(SB) 1071 TEXT runtime·panicExtendSlice3B(SB),NOSPLIT,$0-12 1072 MOVW R4, hi+0(FP) 1073 MOVW R1, lo+4(FP) 1074 MOVW R2, y+8(FP) 1075 JMP runtime·goPanicExtendSlice3B(SB) 1076 TEXT runtime·panicExtendSlice3BU(SB),NOSPLIT,$0-12 1077 MOVW R4, hi+0(FP) 1078 MOVW R1, lo+4(FP) 1079 MOVW R2, y+8(FP) 1080 JMP runtime·goPanicExtendSlice3BU(SB) 1081 TEXT runtime·panicExtendSlice3C(SB),NOSPLIT,$0-12 1082 MOVW R4, hi+0(FP) 1083 MOVW R0, lo+4(FP) 1084 MOVW R1, y+8(FP) 1085 JMP runtime·goPanicExtendSlice3C(SB) 1086 TEXT runtime·panicExtendSlice3CU(SB),NOSPLIT,$0-12 1087 MOVW R4, hi+0(FP) 1088 MOVW R0, lo+4(FP) 1089 MOVW R1, y+8(FP) 1090 JMP runtime·goPanicExtendSlice3CU(SB)