github.com/m10x/go/src@v0.0.0-20220112094212-ba61592315da/runtime/asm_arm.s (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 #include "go_asm.h" 6 #include "go_tls.h" 7 #include "funcdata.h" 8 #include "textflag.h" 9 10 // _rt0_arm is common startup code for most ARM systems when using 11 // internal linking. This is the entry point for the program from the 12 // kernel for an ordinary -buildmode=exe program. The stack holds the 13 // number of arguments and the C-style argv. 14 TEXT _rt0_arm(SB),NOSPLIT|NOFRAME,$0 15 MOVW (R13), R0 // argc 16 MOVW $4(R13), R1 // argv 17 B runtime·rt0_go(SB) 18 19 // main is common startup code for most ARM systems when using 20 // external linking. The C startup code will call the symbol "main" 21 // passing argc and argv in the usual C ABI registers R0 and R1. 22 TEXT main(SB),NOSPLIT|NOFRAME,$0 23 B runtime·rt0_go(SB) 24 25 // _rt0_arm_lib is common startup code for most ARM systems when 26 // using -buildmode=c-archive or -buildmode=c-shared. The linker will 27 // arrange to invoke this function as a global constructor (for 28 // c-archive) or when the shared library is loaded (for c-shared). 29 // We expect argc and argv to be passed in the usual C ABI registers 30 // R0 and R1. 31 TEXT _rt0_arm_lib(SB),NOSPLIT,$104 32 // Preserve callee-save registers. Raspberry Pi's dlopen(), for example, 33 // actually cares that R11 is preserved. 34 MOVW R4, 12(R13) 35 MOVW R5, 16(R13) 36 MOVW R6, 20(R13) 37 MOVW R7, 24(R13) 38 MOVW R8, 28(R13) 39 MOVW g, 32(R13) 40 MOVW R11, 36(R13) 41 42 // Skip floating point registers on GOARM < 6. 43 MOVB runtime·goarm(SB), R11 44 CMP $6, R11 45 BLT skipfpsave 46 MOVD F8, (40+8*0)(R13) 47 MOVD F9, (40+8*1)(R13) 48 MOVD F10, (40+8*2)(R13) 49 MOVD F11, (40+8*3)(R13) 50 MOVD F12, (40+8*4)(R13) 51 MOVD F13, (40+8*5)(R13) 52 MOVD F14, (40+8*6)(R13) 53 MOVD F15, (40+8*7)(R13) 54 skipfpsave: 55 // Save argc/argv. 56 MOVW R0, _rt0_arm_lib_argc<>(SB) 57 MOVW R1, _rt0_arm_lib_argv<>(SB) 58 59 MOVW $0, g // Initialize g. 60 61 // Synchronous initialization. 62 CALL runtime·libpreinit(SB) 63 64 // Create a new thread to do the runtime initialization. 65 MOVW _cgo_sys_thread_create(SB), R2 66 CMP $0, R2 67 BEQ nocgo 68 MOVW $_rt0_arm_lib_go<>(SB), R0 69 MOVW $0, R1 70 BL (R2) 71 B rr 72 nocgo: 73 MOVW $0x800000, R0 // stacksize = 8192KB 74 MOVW $_rt0_arm_lib_go<>(SB), R1 // fn 75 MOVW R0, 4(R13) 76 MOVW R1, 8(R13) 77 BL runtime·newosproc0(SB) 78 rr: 79 // Restore callee-save registers and return. 80 MOVB runtime·goarm(SB), R11 81 CMP $6, R11 82 BLT skipfprest 83 MOVD (40+8*0)(R13), F8 84 MOVD (40+8*1)(R13), F9 85 MOVD (40+8*2)(R13), F10 86 MOVD (40+8*3)(R13), F11 87 MOVD (40+8*4)(R13), F12 88 MOVD (40+8*5)(R13), F13 89 MOVD (40+8*6)(R13), F14 90 MOVD (40+8*7)(R13), F15 91 skipfprest: 92 MOVW 12(R13), R4 93 MOVW 16(R13), R5 94 MOVW 20(R13), R6 95 MOVW 24(R13), R7 96 MOVW 28(R13), R8 97 MOVW 32(R13), g 98 MOVW 36(R13), R11 99 RET 100 101 // _rt0_arm_lib_go initializes the Go runtime. 102 // This is started in a separate thread by _rt0_arm_lib. 103 TEXT _rt0_arm_lib_go<>(SB),NOSPLIT,$8 104 MOVW _rt0_arm_lib_argc<>(SB), R0 105 MOVW _rt0_arm_lib_argv<>(SB), R1 106 B runtime·rt0_go(SB) 107 108 DATA _rt0_arm_lib_argc<>(SB)/4,$0 109 GLOBL _rt0_arm_lib_argc<>(SB),NOPTR,$4 110 DATA _rt0_arm_lib_argv<>(SB)/4,$0 111 GLOBL _rt0_arm_lib_argv<>(SB),NOPTR,$4 112 113 // using NOFRAME means do not save LR on stack. 114 // argc is in R0, argv is in R1. 115 TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 116 MOVW $0xcafebabe, R12 117 118 // copy arguments forward on an even stack 119 // use R13 instead of SP to avoid linker rewriting the offsets 120 SUB $64, R13 // plenty of scratch 121 AND $~7, R13 122 MOVW R0, 60(R13) // save argc, argv away 123 MOVW R1, 64(R13) 124 125 // set up g register 126 // g is R10 127 MOVW $runtime·g0(SB), g 128 MOVW $runtime·m0(SB), R8 129 130 // save m->g0 = g0 131 MOVW g, m_g0(R8) 132 // save g->m = m0 133 MOVW R8, g_m(g) 134 135 // create istack out of the OS stack 136 // (1MB of system stack is available on iOS and Android) 137 MOVW $(-64*1024+104)(R13), R0 138 MOVW R0, g_stackguard0(g) 139 MOVW R0, g_stackguard1(g) 140 MOVW R0, (g_stack+stack_lo)(g) 141 MOVW R13, (g_stack+stack_hi)(g) 142 143 BL runtime·emptyfunc(SB) // fault if stack check is wrong 144 145 #ifdef GOOS_openbsd 146 // Save g to TLS so that it is available from signal trampoline. 147 BL runtime·save_g(SB) 148 #endif 149 150 BL runtime·_initcgo(SB) // will clobber R0-R3 151 152 // update stackguard after _cgo_init 153 MOVW (g_stack+stack_lo)(g), R0 154 ADD $const__StackGuard, R0 155 MOVW R0, g_stackguard0(g) 156 MOVW R0, g_stackguard1(g) 157 158 BL runtime·check(SB) 159 160 // saved argc, argv 161 MOVW 60(R13), R0 162 MOVW R0, 4(R13) 163 MOVW 64(R13), R1 164 MOVW R1, 8(R13) 165 BL runtime·args(SB) 166 BL runtime·checkgoarm(SB) 167 BL runtime·osinit(SB) 168 BL runtime·schedinit(SB) 169 170 // create a new goroutine to start program 171 SUB $8, R13 172 MOVW $runtime·mainPC(SB), R0 173 MOVW R0, 4(R13) // arg 1: fn 174 MOVW $0, R0 175 MOVW R0, 0(R13) // dummy LR 176 BL runtime·newproc(SB) 177 ADD $8, R13 // pop args and LR 178 179 // start this M 180 BL runtime·mstart(SB) 181 182 MOVW $1234, R0 183 MOVW $1000, R1 184 MOVW R0, (R1) // fail hard 185 186 DATA runtime·mainPC+0(SB)/4,$runtime·main(SB) 187 GLOBL runtime·mainPC(SB),RODATA,$4 188 189 TEXT runtime·breakpoint(SB),NOSPLIT,$0-0 190 // gdb won't skip this breakpoint instruction automatically, 191 // so you must manually "set $pc+=4" to skip it and continue. 192 #ifdef GOOS_plan9 193 WORD $0xD1200070 // undefined instruction used as armv5 breakpoint in Plan 9 194 #else 195 WORD $0xe7f001f0 // undefined instruction that gdb understands is a software breakpoint 196 #endif 197 RET 198 199 TEXT runtime·asminit(SB),NOSPLIT,$0-0 200 // disable runfast (flush-to-zero) mode of vfp if runtime.goarm > 5 201 MOVB runtime·goarm(SB), R11 202 CMP $5, R11 203 BLE 4(PC) 204 WORD $0xeef1ba10 // vmrs r11, fpscr 205 BIC $(1<<24), R11 206 WORD $0xeee1ba10 // vmsr fpscr, r11 207 RET 208 209 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0 210 BL runtime·mstart0(SB) 211 RET // not reached 212 213 /* 214 * go-routine 215 */ 216 217 // void gogo(Gobuf*) 218 // restore state from Gobuf; longjmp 219 TEXT runtime·gogo(SB),NOSPLIT|NOFRAME,$0-4 220 MOVW buf+0(FP), R1 221 MOVW gobuf_g(R1), R0 222 MOVW 0(R0), R2 // make sure g != nil 223 B gogo<>(SB) 224 225 TEXT gogo<>(SB),NOSPLIT|NOFRAME,$0 226 BL setg<>(SB) 227 MOVW gobuf_sp(R1), R13 // restore SP==R13 228 MOVW gobuf_lr(R1), LR 229 MOVW gobuf_ret(R1), R0 230 MOVW gobuf_ctxt(R1), R7 231 MOVW $0, R11 232 MOVW R11, gobuf_sp(R1) // clear to help garbage collector 233 MOVW R11, gobuf_ret(R1) 234 MOVW R11, gobuf_lr(R1) 235 MOVW R11, gobuf_ctxt(R1) 236 MOVW gobuf_pc(R1), R11 237 CMP R11, R11 // set condition codes for == test, needed by stack split 238 B (R11) 239 240 // func mcall(fn func(*g)) 241 // Switch to m->g0's stack, call fn(g). 242 // Fn must never return. It should gogo(&g->sched) 243 // to keep running g. 244 TEXT runtime·mcall(SB),NOSPLIT|NOFRAME,$0-4 245 // Save caller state in g->sched. 246 MOVW R13, (g_sched+gobuf_sp)(g) 247 MOVW LR, (g_sched+gobuf_pc)(g) 248 MOVW $0, R11 249 MOVW R11, (g_sched+gobuf_lr)(g) 250 251 // Switch to m->g0 & its stack, call fn. 252 MOVW g, R1 253 MOVW g_m(g), R8 254 MOVW m_g0(R8), R0 255 BL setg<>(SB) 256 CMP g, R1 257 B.NE 2(PC) 258 B runtime·badmcall(SB) 259 MOVW fn+0(FP), R0 260 MOVW (g_sched+gobuf_sp)(g), R13 261 SUB $8, R13 262 MOVW R1, 4(R13) 263 MOVW R0, R7 264 MOVW 0(R0), R0 265 BL (R0) 266 B runtime·badmcall2(SB) 267 RET 268 269 // systemstack_switch is a dummy routine that systemstack leaves at the bottom 270 // of the G stack. We need to distinguish the routine that 271 // lives at the bottom of the G stack from the one that lives 272 // at the top of the system stack because the one at the top of 273 // the system stack terminates the stack walk (see topofstack()). 274 TEXT runtime·systemstack_switch(SB),NOSPLIT,$0-0 275 MOVW $0, R0 276 BL (R0) // clobber lr to ensure push {lr} is kept 277 RET 278 279 // func systemstack(fn func()) 280 TEXT runtime·systemstack(SB),NOSPLIT,$0-4 281 MOVW fn+0(FP), R0 // R0 = fn 282 MOVW g_m(g), R1 // R1 = m 283 284 MOVW m_gsignal(R1), R2 // R2 = gsignal 285 CMP g, R2 286 B.EQ noswitch 287 288 MOVW m_g0(R1), R2 // R2 = g0 289 CMP g, R2 290 B.EQ noswitch 291 292 MOVW m_curg(R1), R3 293 CMP g, R3 294 B.EQ switch 295 296 // Bad: g is not gsignal, not g0, not curg. What is it? 297 // Hide call from linker nosplit analysis. 298 MOVW $runtime·badsystemstack(SB), R0 299 BL (R0) 300 B runtime·abort(SB) 301 302 switch: 303 // save our state in g->sched. Pretend to 304 // be systemstack_switch if the G stack is scanned. 305 BL gosave_systemstack_switch<>(SB) 306 307 // switch to g0 308 MOVW R0, R5 309 MOVW R2, R0 310 BL setg<>(SB) 311 MOVW R5, R0 312 MOVW (g_sched+gobuf_sp)(R2), R13 313 314 // call target function 315 MOVW R0, R7 316 MOVW 0(R0), R0 317 BL (R0) 318 319 // switch back to g 320 MOVW g_m(g), R1 321 MOVW m_curg(R1), R0 322 BL setg<>(SB) 323 MOVW (g_sched+gobuf_sp)(g), R13 324 MOVW $0, R3 325 MOVW R3, (g_sched+gobuf_sp)(g) 326 RET 327 328 noswitch: 329 // Using a tail call here cleans up tracebacks since we won't stop 330 // at an intermediate systemstack. 331 MOVW R0, R7 332 MOVW 0(R0), R0 333 MOVW.P 4(R13), R14 // restore LR 334 B (R0) 335 336 /* 337 * support for morestack 338 */ 339 340 // Called during function prolog when more stack is needed. 341 // R3 prolog's LR 342 // using NOFRAME means do not save LR on stack. 343 // 344 // The traceback routines see morestack on a g0 as being 345 // the top of a stack (for example, morestack calling newstack 346 // calling the scheduler calling newm calling gc), so we must 347 // record an argument size. For that purpose, it has no arguments. 348 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0 349 // Cannot grow scheduler stack (m->g0). 350 MOVW g_m(g), R8 351 MOVW m_g0(R8), R4 352 CMP g, R4 353 BNE 3(PC) 354 BL runtime·badmorestackg0(SB) 355 B runtime·abort(SB) 356 357 // Cannot grow signal stack (m->gsignal). 358 MOVW m_gsignal(R8), R4 359 CMP g, R4 360 BNE 3(PC) 361 BL runtime·badmorestackgsignal(SB) 362 B runtime·abort(SB) 363 364 // Called from f. 365 // Set g->sched to context in f. 366 MOVW R13, (g_sched+gobuf_sp)(g) 367 MOVW LR, (g_sched+gobuf_pc)(g) 368 MOVW R3, (g_sched+gobuf_lr)(g) 369 MOVW R7, (g_sched+gobuf_ctxt)(g) 370 371 // Called from f. 372 // Set m->morebuf to f's caller. 373 MOVW R3, (m_morebuf+gobuf_pc)(R8) // f's caller's PC 374 MOVW R13, (m_morebuf+gobuf_sp)(R8) // f's caller's SP 375 MOVW g, (m_morebuf+gobuf_g)(R8) 376 377 // Call newstack on m->g0's stack. 378 MOVW m_g0(R8), R0 379 BL setg<>(SB) 380 MOVW (g_sched+gobuf_sp)(g), R13 381 MOVW $0, R0 382 MOVW.W R0, -4(R13) // create a call frame on g0 (saved LR) 383 BL runtime·newstack(SB) 384 385 // Not reached, but make sure the return PC from the call to newstack 386 // is still in this function, and not the beginning of the next. 387 RET 388 389 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0 390 MOVW $0, R7 391 B runtime·morestack(SB) 392 393 // reflectcall: call a function with the given argument list 394 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs). 395 // we don't have variable-sized frames, so we use a small number 396 // of constant-sized-frame functions to encode a few bits of size in the pc. 397 // Caution: ugly multiline assembly macros in your future! 398 399 #define DISPATCH(NAME,MAXSIZE) \ 400 CMP $MAXSIZE, R0; \ 401 B.HI 3(PC); \ 402 MOVW $NAME(SB), R1; \ 403 B (R1) 404 405 TEXT ·reflectcall(SB),NOSPLIT|NOFRAME,$0-28 406 MOVW frameSize+20(FP), R0 407 DISPATCH(runtime·call16, 16) 408 DISPATCH(runtime·call32, 32) 409 DISPATCH(runtime·call64, 64) 410 DISPATCH(runtime·call128, 128) 411 DISPATCH(runtime·call256, 256) 412 DISPATCH(runtime·call512, 512) 413 DISPATCH(runtime·call1024, 1024) 414 DISPATCH(runtime·call2048, 2048) 415 DISPATCH(runtime·call4096, 4096) 416 DISPATCH(runtime·call8192, 8192) 417 DISPATCH(runtime·call16384, 16384) 418 DISPATCH(runtime·call32768, 32768) 419 DISPATCH(runtime·call65536, 65536) 420 DISPATCH(runtime·call131072, 131072) 421 DISPATCH(runtime·call262144, 262144) 422 DISPATCH(runtime·call524288, 524288) 423 DISPATCH(runtime·call1048576, 1048576) 424 DISPATCH(runtime·call2097152, 2097152) 425 DISPATCH(runtime·call4194304, 4194304) 426 DISPATCH(runtime·call8388608, 8388608) 427 DISPATCH(runtime·call16777216, 16777216) 428 DISPATCH(runtime·call33554432, 33554432) 429 DISPATCH(runtime·call67108864, 67108864) 430 DISPATCH(runtime·call134217728, 134217728) 431 DISPATCH(runtime·call268435456, 268435456) 432 DISPATCH(runtime·call536870912, 536870912) 433 DISPATCH(runtime·call1073741824, 1073741824) 434 MOVW $runtime·badreflectcall(SB), R1 435 B (R1) 436 437 #define CALLFN(NAME,MAXSIZE) \ 438 TEXT NAME(SB), WRAPPER, $MAXSIZE-28; \ 439 NO_LOCAL_POINTERS; \ 440 /* copy arguments to stack */ \ 441 MOVW stackArgs+8(FP), R0; \ 442 MOVW stackArgsSize+12(FP), R2; \ 443 ADD $4, R13, R1; \ 444 CMP $0, R2; \ 445 B.EQ 5(PC); \ 446 MOVBU.P 1(R0), R5; \ 447 MOVBU.P R5, 1(R1); \ 448 SUB $1, R2, R2; \ 449 B -5(PC); \ 450 /* call function */ \ 451 MOVW f+4(FP), R7; \ 452 MOVW (R7), R0; \ 453 PCDATA $PCDATA_StackMapIndex, $0; \ 454 BL (R0); \ 455 /* copy return values back */ \ 456 MOVW stackArgsType+0(FP), R4; \ 457 MOVW stackArgs+8(FP), R0; \ 458 MOVW stackArgsSize+12(FP), R2; \ 459 MOVW stackArgsRetOffset+16(FP), R3; \ 460 ADD $4, R13, R1; \ 461 ADD R3, R1; \ 462 ADD R3, R0; \ 463 SUB R3, R2; \ 464 BL callRet<>(SB); \ 465 RET 466 467 // callRet copies return values back at the end of call*. This is a 468 // separate function so it can allocate stack space for the arguments 469 // to reflectcallmove. It does not follow the Go ABI; it expects its 470 // arguments in registers. 471 TEXT callRet<>(SB), NOSPLIT, $20-0 472 MOVW R4, 4(R13) 473 MOVW R0, 8(R13) 474 MOVW R1, 12(R13) 475 MOVW R2, 16(R13) 476 MOVW $0, R7 477 MOVW R7, 20(R13) 478 BL runtime·reflectcallmove(SB) 479 RET 480 481 CALLFN(·call16, 16) 482 CALLFN(·call32, 32) 483 CALLFN(·call64, 64) 484 CALLFN(·call128, 128) 485 CALLFN(·call256, 256) 486 CALLFN(·call512, 512) 487 CALLFN(·call1024, 1024) 488 CALLFN(·call2048, 2048) 489 CALLFN(·call4096, 4096) 490 CALLFN(·call8192, 8192) 491 CALLFN(·call16384, 16384) 492 CALLFN(·call32768, 32768) 493 CALLFN(·call65536, 65536) 494 CALLFN(·call131072, 131072) 495 CALLFN(·call262144, 262144) 496 CALLFN(·call524288, 524288) 497 CALLFN(·call1048576, 1048576) 498 CALLFN(·call2097152, 2097152) 499 CALLFN(·call4194304, 4194304) 500 CALLFN(·call8388608, 8388608) 501 CALLFN(·call16777216, 16777216) 502 CALLFN(·call33554432, 33554432) 503 CALLFN(·call67108864, 67108864) 504 CALLFN(·call134217728, 134217728) 505 CALLFN(·call268435456, 268435456) 506 CALLFN(·call536870912, 536870912) 507 CALLFN(·call1073741824, 1073741824) 508 509 // Save state of caller into g->sched, 510 // but using fake PC from systemstack_switch. 511 // Must only be called from functions with no locals ($0) 512 // or else unwinding from systemstack_switch is incorrect. 513 // Smashes R11. 514 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 515 MOVW $runtime·systemstack_switch(SB), R11 516 ADD $4, R11 // get past push {lr} 517 MOVW R11, (g_sched+gobuf_pc)(g) 518 MOVW R13, (g_sched+gobuf_sp)(g) 519 MOVW $0, R11 520 MOVW R11, (g_sched+gobuf_lr)(g) 521 MOVW R11, (g_sched+gobuf_ret)(g) 522 // Assert ctxt is zero. See func save. 523 MOVW (g_sched+gobuf_ctxt)(g), R11 524 TST R11, R11 525 B.EQ 2(PC) 526 BL runtime·abort(SB) 527 RET 528 529 // func asmcgocall_no_g(fn, arg unsafe.Pointer) 530 // Call fn(arg) aligned appropriately for the gcc ABI. 531 // Called on a system stack, and there may be no g yet (during needm). 532 TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-8 533 MOVW fn+0(FP), R1 534 MOVW arg+4(FP), R0 535 MOVW R13, R2 536 SUB $32, R13 537 BIC $0x7, R13 // alignment for gcc ABI 538 MOVW R2, 8(R13) 539 BL (R1) 540 MOVW 8(R13), R2 541 MOVW R2, R13 542 RET 543 544 // func asmcgocall(fn, arg unsafe.Pointer) int32 545 // Call fn(arg) on the scheduler stack, 546 // aligned appropriately for the gcc ABI. 547 // See cgocall.go for more details. 548 TEXT ·asmcgocall(SB),NOSPLIT,$0-12 549 MOVW fn+0(FP), R1 550 MOVW arg+4(FP), R0 551 552 MOVW R13, R2 553 CMP $0, g 554 BEQ nosave 555 MOVW g, R4 556 557 // Figure out if we need to switch to m->g0 stack. 558 // We get called to create new OS threads too, and those 559 // come in on the m->g0 stack already. Or we might already 560 // be on the m->gsignal stack. 561 MOVW g_m(g), R8 562 MOVW m_gsignal(R8), R3 563 CMP R3, g 564 BEQ nosave 565 MOVW m_g0(R8), R3 566 CMP R3, g 567 BEQ nosave 568 BL gosave_systemstack_switch<>(SB) 569 MOVW R0, R5 570 MOVW R3, R0 571 BL setg<>(SB) 572 MOVW R5, R0 573 MOVW (g_sched+gobuf_sp)(g), R13 574 575 // Now on a scheduling stack (a pthread-created stack). 576 SUB $24, R13 577 BIC $0x7, R13 // alignment for gcc ABI 578 MOVW R4, 20(R13) // save old g 579 MOVW (g_stack+stack_hi)(R4), R4 580 SUB R2, R4 581 MOVW R4, 16(R13) // save depth in stack (can't just save SP, as stack might be copied during a callback) 582 BL (R1) 583 584 // Restore registers, g, stack pointer. 585 MOVW R0, R5 586 MOVW 20(R13), R0 587 BL setg<>(SB) 588 MOVW (g_stack+stack_hi)(g), R1 589 MOVW 16(R13), R2 590 SUB R2, R1 591 MOVW R5, R0 592 MOVW R1, R13 593 594 MOVW R0, ret+8(FP) 595 RET 596 597 nosave: 598 // Running on a system stack, perhaps even without a g. 599 // Having no g can happen during thread creation or thread teardown 600 // (see needm/dropm on Solaris, for example). 601 // This code is like the above sequence but without saving/restoring g 602 // and without worrying about the stack moving out from under us 603 // (because we're on a system stack, not a goroutine stack). 604 // The above code could be used directly if already on a system stack, 605 // but then the only path through this code would be a rare case on Solaris. 606 // Using this code for all "already on system stack" calls exercises it more, 607 // which should help keep it correct. 608 SUB $24, R13 609 BIC $0x7, R13 // alignment for gcc ABI 610 // save null g in case someone looks during debugging. 611 MOVW $0, R4 612 MOVW R4, 20(R13) 613 MOVW R2, 16(R13) // Save old stack pointer. 614 BL (R1) 615 // Restore stack pointer. 616 MOVW 16(R13), R2 617 MOVW R2, R13 618 MOVW R0, ret+8(FP) 619 RET 620 621 // cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) 622 // See cgocall.go for more details. 623 TEXT ·cgocallback(SB),NOSPLIT,$12-12 624 NO_LOCAL_POINTERS 625 626 // Load m and g from thread-local storage. 627 #ifdef GOOS_openbsd 628 BL runtime·load_g(SB) 629 #else 630 MOVB runtime·iscgo(SB), R0 631 CMP $0, R0 632 BL.NE runtime·load_g(SB) 633 #endif 634 635 // If g is nil, Go did not create the current thread. 636 // Call needm to obtain one for temporary use. 637 // In this case, we're running on the thread stack, so there's 638 // lots of space, but the linker doesn't know. Hide the call from 639 // the linker analysis by using an indirect call. 640 CMP $0, g 641 B.EQ needm 642 643 MOVW g_m(g), R8 644 MOVW R8, savedm-4(SP) 645 B havem 646 647 needm: 648 MOVW g, savedm-4(SP) // g is zero, so is m. 649 MOVW $runtime·needm(SB), R0 650 BL (R0) 651 652 // Set m->g0->sched.sp = SP, so that if a panic happens 653 // during the function we are about to execute, it will 654 // have a valid SP to run on the g0 stack. 655 // The next few lines (after the havem label) 656 // will save this SP onto the stack and then write 657 // the same SP back to m->sched.sp. That seems redundant, 658 // but if an unrecovered panic happens, unwindm will 659 // restore the g->sched.sp from the stack location 660 // and then systemstack will try to use it. If we don't set it here, 661 // that restored SP will be uninitialized (typically 0) and 662 // will not be usable. 663 MOVW g_m(g), R8 664 MOVW m_g0(R8), R3 665 MOVW R13, (g_sched+gobuf_sp)(R3) 666 667 havem: 668 // Now there's a valid m, and we're running on its m->g0. 669 // Save current m->g0->sched.sp on stack and then set it to SP. 670 // Save current sp in m->g0->sched.sp in preparation for 671 // switch back to m->curg stack. 672 // NOTE: unwindm knows that the saved g->sched.sp is at 4(R13) aka savedsp-12(SP). 673 MOVW m_g0(R8), R3 674 MOVW (g_sched+gobuf_sp)(R3), R4 675 MOVW R4, savedsp-12(SP) // must match frame size 676 MOVW R13, (g_sched+gobuf_sp)(R3) 677 678 // Switch to m->curg stack and call runtime.cgocallbackg. 679 // Because we are taking over the execution of m->curg 680 // but *not* resuming what had been running, we need to 681 // save that information (m->curg->sched) so we can restore it. 682 // We can restore m->curg->sched.sp easily, because calling 683 // runtime.cgocallbackg leaves SP unchanged upon return. 684 // To save m->curg->sched.pc, we push it onto the curg stack and 685 // open a frame the same size as cgocallback's g0 frame. 686 // Once we switch to the curg stack, the pushed PC will appear 687 // to be the return PC of cgocallback, so that the traceback 688 // will seamlessly trace back into the earlier calls. 689 MOVW m_curg(R8), R0 690 BL setg<>(SB) 691 MOVW (g_sched+gobuf_sp)(g), R4 // prepare stack as R4 692 MOVW (g_sched+gobuf_pc)(g), R5 693 MOVW R5, -(12+4)(R4) // "saved LR"; must match frame size 694 // Gather our arguments into registers. 695 MOVW fn+0(FP), R1 696 MOVW frame+4(FP), R2 697 MOVW ctxt+8(FP), R3 698 MOVW $-(12+4)(R4), R13 // switch stack; must match frame size 699 MOVW R1, 4(R13) 700 MOVW R2, 8(R13) 701 MOVW R3, 12(R13) 702 BL runtime·cgocallbackg(SB) 703 704 // Restore g->sched (== m->curg->sched) from saved values. 705 MOVW 0(R13), R5 706 MOVW R5, (g_sched+gobuf_pc)(g) 707 MOVW $(12+4)(R13), R4 // must match frame size 708 MOVW R4, (g_sched+gobuf_sp)(g) 709 710 // Switch back to m->g0's stack and restore m->g0->sched.sp. 711 // (Unlike m->curg, the g0 goroutine never uses sched.pc, 712 // so we do not have to restore it.) 713 MOVW g_m(g), R8 714 MOVW m_g0(R8), R0 715 BL setg<>(SB) 716 MOVW (g_sched+gobuf_sp)(g), R13 717 MOVW savedsp-12(SP), R4 // must match frame size 718 MOVW R4, (g_sched+gobuf_sp)(g) 719 720 // If the m on entry was nil, we called needm above to borrow an m 721 // for the duration of the call. Since the call is over, return it with dropm. 722 MOVW savedm-4(SP), R6 723 CMP $0, R6 724 B.NE 3(PC) 725 MOVW $runtime·dropm(SB), R0 726 BL (R0) 727 728 // Done! 729 RET 730 731 // void setg(G*); set g. for use by needm. 732 TEXT runtime·setg(SB),NOSPLIT|NOFRAME,$0-4 733 MOVW gg+0(FP), R0 734 B setg<>(SB) 735 736 TEXT setg<>(SB),NOSPLIT|NOFRAME,$0-0 737 MOVW R0, g 738 739 // Save g to thread-local storage. 740 #ifdef GOOS_windows 741 B runtime·save_g(SB) 742 #else 743 #ifdef GOOS_openbsd 744 B runtime·save_g(SB) 745 #else 746 MOVB runtime·iscgo(SB), R0 747 CMP $0, R0 748 B.EQ 2(PC) 749 B runtime·save_g(SB) 750 751 MOVW g, R0 752 RET 753 #endif 754 #endif 755 756 TEXT runtime·emptyfunc(SB),0,$0-0 757 RET 758 759 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0 760 MOVW $0, R0 761 MOVW (R0), R1 762 763 // armPublicationBarrier is a native store/store barrier for ARMv7+. 764 // On earlier ARM revisions, armPublicationBarrier is a no-op. 765 // This will not work on SMP ARMv6 machines, if any are in use. 766 // To implement publicationBarrier in sys_$GOOS_arm.s using the native 767 // instructions, use: 768 // 769 // TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0 770 // B runtime·armPublicationBarrier(SB) 771 // 772 TEXT runtime·armPublicationBarrier(SB),NOSPLIT|NOFRAME,$0-0 773 MOVB runtime·goarm(SB), R11 774 CMP $7, R11 775 BLT 2(PC) 776 DMB MB_ST 777 RET 778 779 // AES hashing not implemented for ARM 780 TEXT runtime·memhash(SB),NOSPLIT|NOFRAME,$0-16 781 JMP runtime·memhashFallback(SB) 782 TEXT runtime·strhash(SB),NOSPLIT|NOFRAME,$0-12 783 JMP runtime·strhashFallback(SB) 784 TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-12 785 JMP runtime·memhash32Fallback(SB) 786 TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-12 787 JMP runtime·memhash64Fallback(SB) 788 789 TEXT runtime·return0(SB),NOSPLIT,$0 790 MOVW $0, R0 791 RET 792 793 TEXT runtime·procyield(SB),NOSPLIT|NOFRAME,$0 794 MOVW cycles+0(FP), R1 795 MOVW $0, R0 796 yieldloop: 797 WORD $0xe320f001 // YIELD (NOP pre-ARMv6K) 798 CMP R0, R1 799 B.NE 2(PC) 800 RET 801 SUB $1, R1 802 B yieldloop 803 804 // Called from cgo wrappers, this function returns g->m->curg.stack.hi. 805 // Must obey the gcc calling convention. 806 TEXT _cgo_topofstack(SB),NOSPLIT,$8 807 // R11 and g register are clobbered by load_g. They are 808 // callee-save in the gcc calling convention, so save them here. 809 MOVW R11, saveR11-4(SP) 810 MOVW g, saveG-8(SP) 811 812 BL runtime·load_g(SB) 813 MOVW g_m(g), R0 814 MOVW m_curg(R0), R0 815 MOVW (g_stack+stack_hi)(R0), R0 816 817 MOVW saveG-8(SP), g 818 MOVW saveR11-4(SP), R11 819 RET 820 821 // The top-most function running on a goroutine 822 // returns to goexit+PCQuantum. 823 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0 824 MOVW R0, R0 // NOP 825 BL runtime·goexit1(SB) // does not return 826 // traceback from goexit1 must hit code range of goexit 827 MOVW R0, R0 // NOP 828 829 // x -> x/1000000, x%1000000, called from Go with args, results on stack. 830 TEXT runtime·usplit(SB),NOSPLIT,$0-12 831 MOVW x+0(FP), R0 832 CALL runtime·usplitR0(SB) 833 MOVW R0, q+4(FP) 834 MOVW R1, r+8(FP) 835 RET 836 837 // R0, R1 = R0/1000000, R0%1000000 838 TEXT runtime·usplitR0(SB),NOSPLIT,$0 839 // magic multiply to avoid software divide without available m. 840 // see output of go tool compile -S for x/1000000. 841 MOVW R0, R3 842 MOVW $1125899907, R1 843 MULLU R1, R0, (R0, R1) 844 MOVW R0>>18, R0 845 MOVW $1000000, R1 846 MULU R0, R1 847 SUB R1, R3, R1 848 RET 849 850 // This is called from .init_array and follows the platform, not Go, ABI. 851 TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0 852 MOVW R9, saver9-4(SP) // The access to global variables below implicitly uses R9, which is callee-save 853 MOVW R11, saver11-8(SP) // Likewise, R11 is the temp register, but callee-save in C ABI 854 MOVW runtime·lastmoduledatap(SB), R1 855 MOVW R0, moduledata_next(R1) 856 MOVW R0, runtime·lastmoduledatap(SB) 857 MOVW saver11-8(SP), R11 858 MOVW saver9-4(SP), R9 859 RET 860 861 TEXT ·checkASM(SB),NOSPLIT,$0-1 862 MOVW $1, R3 863 MOVB R3, ret+0(FP) 864 RET 865 866 // gcWriteBarrier performs a heap pointer write and informs the GC. 867 // 868 // gcWriteBarrier does NOT follow the Go ABI. It takes two arguments: 869 // - R2 is the destination of the write 870 // - R3 is the value being written at R2 871 // It clobbers condition codes. 872 // It does not clobber any other general-purpose registers, 873 // but may clobber others (e.g., floating point registers). 874 // The act of CALLing gcWriteBarrier will clobber R14 (LR). 875 TEXT runtime·gcWriteBarrier(SB),NOSPLIT|NOFRAME,$0 876 // Save the registers clobbered by the fast path. 877 MOVM.DB.W [R0,R1], (R13) 878 MOVW g_m(g), R0 879 MOVW m_p(R0), R0 880 MOVW (p_wbBuf+wbBuf_next)(R0), R1 881 // Increment wbBuf.next position. 882 ADD $8, R1 883 MOVW R1, (p_wbBuf+wbBuf_next)(R0) 884 MOVW (p_wbBuf+wbBuf_end)(R0), R0 885 CMP R1, R0 886 // Record the write. 887 MOVW R3, -8(R1) // Record value 888 MOVW (R2), R0 // TODO: This turns bad writes into bad reads. 889 MOVW R0, -4(R1) // Record *slot 890 // Is the buffer full? (flags set in CMP above) 891 B.EQ flush 892 ret: 893 MOVM.IA.W (R13), [R0,R1] 894 // Do the write. 895 MOVW R3, (R2) 896 RET 897 898 flush: 899 // Save all general purpose registers since these could be 900 // clobbered by wbBufFlush and were not saved by the caller. 901 // 902 // R0 and R1 were saved at entry. 903 // R10 is g, so preserved. 904 // R11 is linker temp, so no need to save. 905 // R13 is stack pointer. 906 // R15 is PC. 907 // 908 // This also sets up R2 and R3 as the arguments to wbBufFlush. 909 MOVM.DB.W [R2-R9,R12], (R13) 910 // Save R14 (LR) because the fast path above doesn't save it, 911 // but needs it to RET. This is after the MOVM so it appears below 912 // the arguments in the stack frame. 913 MOVM.DB.W [R14], (R13) 914 915 // This takes arguments R2 and R3. 916 CALL runtime·wbBufFlush(SB) 917 918 MOVM.IA.W (R13), [R14] 919 MOVM.IA.W (R13), [R2-R9,R12] 920 JMP ret 921 922 // Note: these functions use a special calling convention to save generated code space. 923 // Arguments are passed in registers, but the space for those arguments are allocated 924 // in the caller's stack frame. These stubs write the args into that stack space and 925 // then tail call to the corresponding runtime handler. 926 // The tail call makes these stubs disappear in backtraces. 927 TEXT runtime·panicIndex(SB),NOSPLIT,$0-8 928 MOVW R0, x+0(FP) 929 MOVW R1, y+4(FP) 930 JMP runtime·goPanicIndex(SB) 931 TEXT runtime·panicIndexU(SB),NOSPLIT,$0-8 932 MOVW R0, x+0(FP) 933 MOVW R1, y+4(FP) 934 JMP runtime·goPanicIndexU(SB) 935 TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-8 936 MOVW R1, x+0(FP) 937 MOVW R2, y+4(FP) 938 JMP runtime·goPanicSliceAlen(SB) 939 TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-8 940 MOVW R1, x+0(FP) 941 MOVW R2, y+4(FP) 942 JMP runtime·goPanicSliceAlenU(SB) 943 TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-8 944 MOVW R1, x+0(FP) 945 MOVW R2, y+4(FP) 946 JMP runtime·goPanicSliceAcap(SB) 947 TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-8 948 MOVW R1, x+0(FP) 949 MOVW R2, y+4(FP) 950 JMP runtime·goPanicSliceAcapU(SB) 951 TEXT runtime·panicSliceB(SB),NOSPLIT,$0-8 952 MOVW R0, x+0(FP) 953 MOVW R1, y+4(FP) 954 JMP runtime·goPanicSliceB(SB) 955 TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-8 956 MOVW R0, x+0(FP) 957 MOVW R1, y+4(FP) 958 JMP runtime·goPanicSliceBU(SB) 959 TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-8 960 MOVW R2, x+0(FP) 961 MOVW R3, y+4(FP) 962 JMP runtime·goPanicSlice3Alen(SB) 963 TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-8 964 MOVW R2, x+0(FP) 965 MOVW R3, y+4(FP) 966 JMP runtime·goPanicSlice3AlenU(SB) 967 TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-8 968 MOVW R2, x+0(FP) 969 MOVW R3, y+4(FP) 970 JMP runtime·goPanicSlice3Acap(SB) 971 TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-8 972 MOVW R2, x+0(FP) 973 MOVW R3, y+4(FP) 974 JMP runtime·goPanicSlice3AcapU(SB) 975 TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-8 976 MOVW R1, x+0(FP) 977 MOVW R2, y+4(FP) 978 JMP runtime·goPanicSlice3B(SB) 979 TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-8 980 MOVW R1, x+0(FP) 981 MOVW R2, y+4(FP) 982 JMP runtime·goPanicSlice3BU(SB) 983 TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-8 984 MOVW R0, x+0(FP) 985 MOVW R1, y+4(FP) 986 JMP runtime·goPanicSlice3C(SB) 987 TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-8 988 MOVW R0, x+0(FP) 989 MOVW R1, y+4(FP) 990 JMP runtime·goPanicSlice3CU(SB) 991 TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-8 992 MOVW R2, x+0(FP) 993 MOVW R3, y+4(FP) 994 JMP runtime·goPanicSliceConvert(SB) 995 996 // Extended versions for 64-bit indexes. 997 TEXT runtime·panicExtendIndex(SB),NOSPLIT,$0-12 998 MOVW R4, hi+0(FP) 999 MOVW R0, lo+4(FP) 1000 MOVW R1, y+8(FP) 1001 JMP runtime·goPanicExtendIndex(SB) 1002 TEXT runtime·panicExtendIndexU(SB),NOSPLIT,$0-12 1003 MOVW R4, hi+0(FP) 1004 MOVW R0, lo+4(FP) 1005 MOVW R1, y+8(FP) 1006 JMP runtime·goPanicExtendIndexU(SB) 1007 TEXT runtime·panicExtendSliceAlen(SB),NOSPLIT,$0-12 1008 MOVW R4, hi+0(FP) 1009 MOVW R1, lo+4(FP) 1010 MOVW R2, y+8(FP) 1011 JMP runtime·goPanicExtendSliceAlen(SB) 1012 TEXT runtime·panicExtendSliceAlenU(SB),NOSPLIT,$0-12 1013 MOVW R4, hi+0(FP) 1014 MOVW R1, lo+4(FP) 1015 MOVW R2, y+8(FP) 1016 JMP runtime·goPanicExtendSliceAlenU(SB) 1017 TEXT runtime·panicExtendSliceAcap(SB),NOSPLIT,$0-12 1018 MOVW R4, hi+0(FP) 1019 MOVW R1, lo+4(FP) 1020 MOVW R2, y+8(FP) 1021 JMP runtime·goPanicExtendSliceAcap(SB) 1022 TEXT runtime·panicExtendSliceAcapU(SB),NOSPLIT,$0-12 1023 MOVW R4, hi+0(FP) 1024 MOVW R1, lo+4(FP) 1025 MOVW R2, y+8(FP) 1026 JMP runtime·goPanicExtendSliceAcapU(SB) 1027 TEXT runtime·panicExtendSliceB(SB),NOSPLIT,$0-12 1028 MOVW R4, hi+0(FP) 1029 MOVW R0, lo+4(FP) 1030 MOVW R1, y+8(FP) 1031 JMP runtime·goPanicExtendSliceB(SB) 1032 TEXT runtime·panicExtendSliceBU(SB),NOSPLIT,$0-12 1033 MOVW R4, hi+0(FP) 1034 MOVW R0, lo+4(FP) 1035 MOVW R1, y+8(FP) 1036 JMP runtime·goPanicExtendSliceBU(SB) 1037 TEXT runtime·panicExtendSlice3Alen(SB),NOSPLIT,$0-12 1038 MOVW R4, hi+0(FP) 1039 MOVW R2, lo+4(FP) 1040 MOVW R3, y+8(FP) 1041 JMP runtime·goPanicExtendSlice3Alen(SB) 1042 TEXT runtime·panicExtendSlice3AlenU(SB),NOSPLIT,$0-12 1043 MOVW R4, hi+0(FP) 1044 MOVW R2, lo+4(FP) 1045 MOVW R3, y+8(FP) 1046 JMP runtime·goPanicExtendSlice3AlenU(SB) 1047 TEXT runtime·panicExtendSlice3Acap(SB),NOSPLIT,$0-12 1048 MOVW R4, hi+0(FP) 1049 MOVW R2, lo+4(FP) 1050 MOVW R3, y+8(FP) 1051 JMP runtime·goPanicExtendSlice3Acap(SB) 1052 TEXT runtime·panicExtendSlice3AcapU(SB),NOSPLIT,$0-12 1053 MOVW R4, hi+0(FP) 1054 MOVW R2, lo+4(FP) 1055 MOVW R3, y+8(FP) 1056 JMP runtime·goPanicExtendSlice3AcapU(SB) 1057 TEXT runtime·panicExtendSlice3B(SB),NOSPLIT,$0-12 1058 MOVW R4, hi+0(FP) 1059 MOVW R1, lo+4(FP) 1060 MOVW R2, y+8(FP) 1061 JMP runtime·goPanicExtendSlice3B(SB) 1062 TEXT runtime·panicExtendSlice3BU(SB),NOSPLIT,$0-12 1063 MOVW R4, hi+0(FP) 1064 MOVW R1, lo+4(FP) 1065 MOVW R2, y+8(FP) 1066 JMP runtime·goPanicExtendSlice3BU(SB) 1067 TEXT runtime·panicExtendSlice3C(SB),NOSPLIT,$0-12 1068 MOVW R4, hi+0(FP) 1069 MOVW R0, lo+4(FP) 1070 MOVW R1, y+8(FP) 1071 JMP runtime·goPanicExtendSlice3C(SB) 1072 TEXT runtime·panicExtendSlice3CU(SB),NOSPLIT,$0-12 1073 MOVW R4, hi+0(FP) 1074 MOVW R0, lo+4(FP) 1075 MOVW R1, y+8(FP) 1076 JMP runtime·goPanicExtendSlice3CU(SB)