github.com/comwrg/go/src@v0.0.0-20220319063731-c238d0440370/runtime/mkpreempt.go (about) 1 // Copyright 2019 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 //go:build ignore 6 // +build ignore 7 8 // mkpreempt generates the asyncPreempt functions for each 9 // architecture. 10 package main 11 12 import ( 13 "flag" 14 "fmt" 15 "io" 16 "log" 17 "os" 18 "strings" 19 ) 20 21 // Copied from cmd/compile/internal/ssa/gen/*Ops.go 22 23 var regNames386 = []string{ 24 "AX", 25 "CX", 26 "DX", 27 "BX", 28 "SP", 29 "BP", 30 "SI", 31 "DI", 32 "X0", 33 "X1", 34 "X2", 35 "X3", 36 "X4", 37 "X5", 38 "X6", 39 "X7", 40 } 41 42 var regNamesAMD64 = []string{ 43 "AX", 44 "CX", 45 "DX", 46 "BX", 47 "SP", 48 "BP", 49 "SI", 50 "DI", 51 "R8", 52 "R9", 53 "R10", 54 "R11", 55 "R12", 56 "R13", 57 "R14", 58 "R15", 59 "X0", 60 "X1", 61 "X2", 62 "X3", 63 "X4", 64 "X5", 65 "X6", 66 "X7", 67 "X8", 68 "X9", 69 "X10", 70 "X11", 71 "X12", 72 "X13", 73 "X14", 74 "X15", 75 } 76 77 var out io.Writer 78 79 var arches = map[string]func(){ 80 "386": gen386, 81 "amd64": genAMD64, 82 "arm": genARM, 83 "arm64": genARM64, 84 "mips64x": func() { genMIPS(true) }, 85 "mipsx": func() { genMIPS(false) }, 86 "ppc64x": genPPC64, 87 "riscv64": genRISCV64, 88 "s390x": genS390X, 89 "wasm": genWasm, 90 } 91 var beLe = map[string]bool{"mips64x": true, "mipsx": true, "ppc64x": true} 92 93 func main() { 94 flag.Parse() 95 if flag.NArg() > 0 { 96 out = os.Stdout 97 for _, arch := range flag.Args() { 98 gen, ok := arches[arch] 99 if !ok { 100 log.Fatalf("unknown arch %s", arch) 101 } 102 header(arch) 103 gen() 104 } 105 return 106 } 107 108 for arch, gen := range arches { 109 f, err := os.Create(fmt.Sprintf("preempt_%s.s", arch)) 110 if err != nil { 111 log.Fatal(err) 112 } 113 out = f 114 header(arch) 115 gen() 116 if err := f.Close(); err != nil { 117 log.Fatal(err) 118 } 119 } 120 } 121 122 func header(arch string) { 123 fmt.Fprintf(out, "// Code generated by mkpreempt.go; DO NOT EDIT.\n\n") 124 if beLe[arch] { 125 base := arch[:len(arch)-1] 126 fmt.Fprintf(out, "//go:build %s || %sle\n", base, base) 127 fmt.Fprintf(out, "// +build %s %sle\n\n", base, base) 128 } 129 fmt.Fprintf(out, "#include \"go_asm.h\"\n") 130 fmt.Fprintf(out, "#include \"textflag.h\"\n\n") 131 fmt.Fprintf(out, "// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally.\n") 132 fmt.Fprintf(out, "TEXT ·asyncPreempt<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-0\n") 133 } 134 135 func p(f string, args ...interface{}) { 136 fmted := fmt.Sprintf(f, args...) 137 fmt.Fprintf(out, "\t%s\n", strings.ReplaceAll(fmted, "\n", "\n\t")) 138 } 139 140 func label(l string) { 141 fmt.Fprintf(out, "%s\n", l) 142 } 143 144 type layout struct { 145 stack int 146 regs []regPos 147 sp string // stack pointer register 148 } 149 150 type regPos struct { 151 pos int 152 153 op string 154 reg string 155 156 // If this register requires special save and restore, these 157 // give those operations with a %d placeholder for the stack 158 // offset. 159 save, restore string 160 } 161 162 func (l *layout) add(op, reg string, size int) { 163 l.regs = append(l.regs, regPos{op: op, reg: reg, pos: l.stack}) 164 l.stack += size 165 } 166 167 func (l *layout) addSpecial(save, restore string, size int) { 168 l.regs = append(l.regs, regPos{save: save, restore: restore, pos: l.stack}) 169 l.stack += size 170 } 171 172 func (l *layout) save() { 173 for _, reg := range l.regs { 174 if reg.save != "" { 175 p(reg.save, reg.pos) 176 } else { 177 p("%s %s, %d(%s)", reg.op, reg.reg, reg.pos, l.sp) 178 } 179 } 180 } 181 182 func (l *layout) restore() { 183 for i := len(l.regs) - 1; i >= 0; i-- { 184 reg := l.regs[i] 185 if reg.restore != "" { 186 p(reg.restore, reg.pos) 187 } else { 188 p("%s %d(%s), %s", reg.op, reg.pos, l.sp, reg.reg) 189 } 190 } 191 } 192 193 func gen386() { 194 p("PUSHFL") 195 // Save general purpose registers. 196 var l = layout{sp: "SP"} 197 for _, reg := range regNames386 { 198 if reg == "SP" || strings.HasPrefix(reg, "X") { 199 continue 200 } 201 l.add("MOVL", reg, 4) 202 } 203 204 // Save SSE state only if supported. 205 lSSE := layout{stack: l.stack, sp: "SP"} 206 for i := 0; i < 8; i++ { 207 lSSE.add("MOVUPS", fmt.Sprintf("X%d", i), 16) 208 } 209 210 p("ADJSP $%d", lSSE.stack) 211 p("NOP SP") 212 l.save() 213 p("CMPB internal∕cpu·X86+const_offsetX86HasSSE2(SB), $1\nJNE nosse") 214 lSSE.save() 215 label("nosse:") 216 p("CALL ·asyncPreempt2(SB)") 217 p("CMPB internal∕cpu·X86+const_offsetX86HasSSE2(SB), $1\nJNE nosse2") 218 lSSE.restore() 219 label("nosse2:") 220 l.restore() 221 p("ADJSP $%d", -lSSE.stack) 222 223 p("POPFL") 224 p("RET") 225 } 226 227 func genAMD64() { 228 // Assign stack offsets. 229 var l = layout{sp: "SP"} 230 for _, reg := range regNamesAMD64 { 231 if reg == "SP" || reg == "BP" { 232 continue 233 } 234 if !strings.HasPrefix(reg, "X") { 235 l.add("MOVQ", reg, 8) 236 } 237 } 238 lSSE := layout{stack: l.stack, sp: "SP"} 239 for _, reg := range regNamesAMD64 { 240 if strings.HasPrefix(reg, "X") { 241 lSSE.add("MOVUPS", reg, 16) 242 } 243 } 244 245 // TODO: MXCSR register? 246 247 p("PUSHQ BP") 248 p("MOVQ SP, BP") 249 p("// Save flags before clobbering them") 250 p("PUSHFQ") 251 p("// obj doesn't understand ADD/SUB on SP, but does understand ADJSP") 252 p("ADJSP $%d", lSSE.stack) 253 p("// But vet doesn't know ADJSP, so suppress vet stack checking") 254 p("NOP SP") 255 256 l.save() 257 258 // Apparently, the signal handling code path in darwin kernel leaves 259 // the upper bits of Y registers in a dirty state, which causes 260 // many SSE operations (128-bit and narrower) become much slower. 261 // Clear the upper bits to get to a clean state. See issue #37174. 262 // It is safe here as Go code don't use the upper bits of Y registers. 263 p("#ifdef GOOS_darwin") 264 p("CMPB internal∕cpu·X86+const_offsetX86HasAVX(SB), $0") 265 p("JE 2(PC)") 266 p("VZEROUPPER") 267 p("#endif") 268 269 lSSE.save() 270 p("CALL ·asyncPreempt2(SB)") 271 lSSE.restore() 272 l.restore() 273 p("ADJSP $%d", -lSSE.stack) 274 p("POPFQ") 275 p("POPQ BP") 276 p("RET") 277 } 278 279 func genARM() { 280 // Add integer registers R0-R12. 281 // R13 (SP), R14 (LR), R15 (PC) are special and not saved here. 282 var l = layout{sp: "R13", stack: 4} // add LR slot 283 for i := 0; i <= 12; i++ { 284 reg := fmt.Sprintf("R%d", i) 285 if i == 10 { 286 continue // R10 is g register, no need to save/restore 287 } 288 l.add("MOVW", reg, 4) 289 } 290 // Add flag register. 291 l.addSpecial( 292 "MOVW CPSR, R0\nMOVW R0, %d(R13)", 293 "MOVW %d(R13), R0\nMOVW R0, CPSR", 294 4) 295 296 // Add floating point registers F0-F15 and flag register. 297 var lfp = layout{stack: l.stack, sp: "R13"} 298 lfp.addSpecial( 299 "MOVW FPCR, R0\nMOVW R0, %d(R13)", 300 "MOVW %d(R13), R0\nMOVW R0, FPCR", 301 4) 302 for i := 0; i <= 15; i++ { 303 reg := fmt.Sprintf("F%d", i) 304 lfp.add("MOVD", reg, 8) 305 } 306 307 p("MOVW.W R14, -%d(R13)", lfp.stack) // allocate frame, save LR 308 l.save() 309 p("MOVB ·goarm(SB), R0\nCMP $6, R0\nBLT nofp") // test goarm, and skip FP registers if goarm=5. 310 lfp.save() 311 label("nofp:") 312 p("CALL ·asyncPreempt2(SB)") 313 p("MOVB ·goarm(SB), R0\nCMP $6, R0\nBLT nofp2") // test goarm, and skip FP registers if goarm=5. 314 lfp.restore() 315 label("nofp2:") 316 l.restore() 317 318 p("MOVW %d(R13), R14", lfp.stack) // sigctxt.pushCall pushes LR on stack, restore it 319 p("MOVW.P %d(R13), R15", lfp.stack+4) // load PC, pop frame (including the space pushed by sigctxt.pushCall) 320 p("UNDEF") // shouldn't get here 321 } 322 323 func genARM64() { 324 // Add integer registers R0-R26 325 // R27 (REGTMP), R28 (g), R29 (FP), R30 (LR), R31 (SP) are special 326 // and not saved here. 327 var l = layout{sp: "RSP", stack: 8} // add slot to save PC of interrupted instruction 328 for i := 0; i <= 26; i++ { 329 if i == 18 { 330 continue // R18 is not used, skip 331 } 332 reg := fmt.Sprintf("R%d", i) 333 l.add("MOVD", reg, 8) 334 } 335 // Add flag registers. 336 l.addSpecial( 337 "MOVD NZCV, R0\nMOVD R0, %d(RSP)", 338 "MOVD %d(RSP), R0\nMOVD R0, NZCV", 339 8) 340 l.addSpecial( 341 "MOVD FPSR, R0\nMOVD R0, %d(RSP)", 342 "MOVD %d(RSP), R0\nMOVD R0, FPSR", 343 8) 344 // TODO: FPCR? I don't think we'll change it, so no need to save. 345 // Add floating point registers F0-F31. 346 for i := 0; i <= 31; i++ { 347 reg := fmt.Sprintf("F%d", i) 348 l.add("FMOVD", reg, 8) 349 } 350 if l.stack%16 != 0 { 351 l.stack += 8 // SP needs 16-byte alignment 352 } 353 354 // allocate frame, save PC of interrupted instruction (in LR) 355 p("MOVD R30, %d(RSP)", -l.stack) 356 p("SUB $%d, RSP", l.stack) 357 p("#ifdef GOOS_linux") 358 p("MOVD R29, -8(RSP)") // save frame pointer (only used on Linux) 359 p("SUB $8, RSP, R29") // set up new frame pointer 360 p("#endif") 361 // On iOS, save the LR again after decrementing SP. We run the 362 // signal handler on the G stack (as it doesn't support sigaltstack), 363 // so any writes below SP may be clobbered. 364 p("#ifdef GOOS_ios") 365 p("MOVD R30, (RSP)") 366 p("#endif") 367 368 l.save() 369 p("CALL ·asyncPreempt2(SB)") 370 l.restore() 371 372 p("MOVD %d(RSP), R30", l.stack) // sigctxt.pushCall has pushed LR (at interrupt) on stack, restore it 373 p("#ifdef GOOS_linux") 374 p("MOVD -8(RSP), R29") // restore frame pointer 375 p("#endif") 376 p("MOVD (RSP), R27") // load PC to REGTMP 377 p("ADD $%d, RSP", l.stack+16) // pop frame (including the space pushed by sigctxt.pushCall) 378 p("JMP (R27)") 379 } 380 381 func genMIPS(_64bit bool) { 382 mov := "MOVW" 383 movf := "MOVF" 384 add := "ADD" 385 sub := "SUB" 386 r28 := "R28" 387 regsize := 4 388 softfloat := "GOMIPS_softfloat" 389 if _64bit { 390 mov = "MOVV" 391 movf = "MOVD" 392 add = "ADDV" 393 sub = "SUBV" 394 r28 = "RSB" 395 regsize = 8 396 softfloat = "GOMIPS64_softfloat" 397 } 398 399 // Add integer registers R1-R22, R24-R25, R28 400 // R0 (zero), R23 (REGTMP), R29 (SP), R30 (g), R31 (LR) are special, 401 // and not saved here. R26 and R27 are reserved by kernel and not used. 402 var l = layout{sp: "R29", stack: regsize} // add slot to save PC of interrupted instruction (in LR) 403 for i := 1; i <= 25; i++ { 404 if i == 23 { 405 continue // R23 is REGTMP 406 } 407 reg := fmt.Sprintf("R%d", i) 408 l.add(mov, reg, regsize) 409 } 410 l.add(mov, r28, regsize) 411 l.addSpecial( 412 mov+" HI, R1\n"+mov+" R1, %d(R29)", 413 mov+" %d(R29), R1\n"+mov+" R1, HI", 414 regsize) 415 l.addSpecial( 416 mov+" LO, R1\n"+mov+" R1, %d(R29)", 417 mov+" %d(R29), R1\n"+mov+" R1, LO", 418 regsize) 419 420 // Add floating point control/status register FCR31 (FCR0-FCR30 are irrelevant) 421 var lfp = layout{sp: "R29", stack: l.stack} 422 lfp.addSpecial( 423 mov+" FCR31, R1\n"+mov+" R1, %d(R29)", 424 mov+" %d(R29), R1\n"+mov+" R1, FCR31", 425 regsize) 426 // Add floating point registers F0-F31. 427 for i := 0; i <= 31; i++ { 428 reg := fmt.Sprintf("F%d", i) 429 lfp.add(movf, reg, regsize) 430 } 431 432 // allocate frame, save PC of interrupted instruction (in LR) 433 p(mov+" R31, -%d(R29)", lfp.stack) 434 p(sub+" $%d, R29", lfp.stack) 435 436 l.save() 437 p("#ifndef %s", softfloat) 438 lfp.save() 439 p("#endif") 440 p("CALL ·asyncPreempt2(SB)") 441 p("#ifndef %s", softfloat) 442 lfp.restore() 443 p("#endif") 444 l.restore() 445 446 p(mov+" %d(R29), R31", lfp.stack) // sigctxt.pushCall has pushed LR (at interrupt) on stack, restore it 447 p(mov + " (R29), R23") // load PC to REGTMP 448 p(add+" $%d, R29", lfp.stack+regsize) // pop frame (including the space pushed by sigctxt.pushCall) 449 p("JMP (R23)") 450 } 451 452 func genPPC64() { 453 // Add integer registers R3-R29 454 // R0 (zero), R1 (SP), R30 (g) are special and not saved here. 455 // R2 (TOC pointer in PIC mode), R12 (function entry address in PIC mode) have been saved in sigctxt.pushCall. 456 // R31 (REGTMP) will be saved manually. 457 var l = layout{sp: "R1", stack: 32 + 8} // MinFrameSize on PPC64, plus one word for saving R31 458 for i := 3; i <= 29; i++ { 459 if i == 12 || i == 13 { 460 // R12 has been saved in sigctxt.pushCall. 461 // R13 is TLS pointer, not used by Go code. we must NOT 462 // restore it, otherwise if we parked and resumed on a 463 // different thread we'll mess up TLS addresses. 464 continue 465 } 466 reg := fmt.Sprintf("R%d", i) 467 l.add("MOVD", reg, 8) 468 } 469 l.addSpecial( 470 "MOVW CR, R31\nMOVW R31, %d(R1)", 471 "MOVW %d(R1), R31\nMOVFL R31, $0xff", // this is MOVW R31, CR 472 8) // CR is 4-byte wide, but just keep the alignment 473 l.addSpecial( 474 "MOVD XER, R31\nMOVD R31, %d(R1)", 475 "MOVD %d(R1), R31\nMOVD R31, XER", 476 8) 477 // Add floating point registers F0-F31. 478 for i := 0; i <= 31; i++ { 479 reg := fmt.Sprintf("F%d", i) 480 l.add("FMOVD", reg, 8) 481 } 482 // Add floating point control/status register FPSCR. 483 l.addSpecial( 484 "MOVFL FPSCR, F0\nFMOVD F0, %d(R1)", 485 "FMOVD %d(R1), F0\nMOVFL F0, FPSCR", 486 8) 487 488 p("MOVD R31, -%d(R1)", l.stack-32) // save R31 first, we'll use R31 for saving LR 489 p("MOVD LR, R31") 490 p("MOVDU R31, -%d(R1)", l.stack) // allocate frame, save PC of interrupted instruction (in LR) 491 492 l.save() 493 p("CALL ·asyncPreempt2(SB)") 494 l.restore() 495 496 p("MOVD %d(R1), R31", l.stack) // sigctxt.pushCall has pushed LR, R2, R12 (at interrupt) on stack, restore them 497 p("MOVD R31, LR") 498 p("MOVD %d(R1), R2", l.stack+8) 499 p("MOVD %d(R1), R12", l.stack+16) 500 p("MOVD (R1), R31") // load PC to CTR 501 p("MOVD R31, CTR") 502 p("MOVD 32(R1), R31") // restore R31 503 p("ADD $%d, R1", l.stack+32) // pop frame (including the space pushed by sigctxt.pushCall) 504 p("JMP (CTR)") 505 } 506 507 func genRISCV64() { 508 // X0 (zero), X1 (LR), X2 (SP), X4 (TP), X27 (g), X31 (TMP) are special. 509 var l = layout{sp: "X2", stack: 8} 510 511 // Add integer registers (X3, X5-X26, X28-30). 512 for i := 3; i < 31; i++ { 513 if i == 4 || i == 27 { 514 continue 515 } 516 reg := fmt.Sprintf("X%d", i) 517 l.add("MOV", reg, 8) 518 } 519 520 // Add floating point registers (F0-F31). 521 for i := 0; i <= 31; i++ { 522 reg := fmt.Sprintf("F%d", i) 523 l.add("MOVD", reg, 8) 524 } 525 526 p("MOV X1, -%d(X2)", l.stack) 527 p("ADD $-%d, X2", l.stack) 528 l.save() 529 p("CALL ·asyncPreempt2(SB)") 530 l.restore() 531 p("MOV %d(X2), X1", l.stack) 532 p("MOV (X2), X31") 533 p("ADD $%d, X2", l.stack+8) 534 p("JMP (X31)") 535 } 536 537 func genS390X() { 538 // Add integer registers R0-R12 539 // R13 (g), R14 (LR), R15 (SP) are special, and not saved here. 540 // Saving R10 (REGTMP) is not necessary, but it is saved anyway. 541 var l = layout{sp: "R15", stack: 16} // add slot to save PC of interrupted instruction and flags 542 l.addSpecial( 543 "STMG R0, R12, %d(R15)", 544 "LMG %d(R15), R0, R12", 545 13*8) 546 // Add floating point registers F0-F31. 547 for i := 0; i <= 15; i++ { 548 reg := fmt.Sprintf("F%d", i) 549 l.add("FMOVD", reg, 8) 550 } 551 552 // allocate frame, save PC of interrupted instruction (in LR) and flags (condition code) 553 p("IPM R10") // save flags upfront, as ADD will clobber flags 554 p("MOVD R14, -%d(R15)", l.stack) 555 p("ADD $-%d, R15", l.stack) 556 p("MOVW R10, 8(R15)") // save flags 557 558 l.save() 559 p("CALL ·asyncPreempt2(SB)") 560 l.restore() 561 562 p("MOVD %d(R15), R14", l.stack) // sigctxt.pushCall has pushed LR (at interrupt) on stack, restore it 563 p("ADD $%d, R15", l.stack+8) // pop frame (including the space pushed by sigctxt.pushCall) 564 p("MOVWZ -%d(R15), R10", l.stack) // load flags to REGTMP 565 p("TMLH R10, $(3<<12)") // restore flags 566 p("MOVD -%d(R15), R10", l.stack+8) // load PC to REGTMP 567 p("JMP (R10)") 568 } 569 570 func genWasm() { 571 p("// No async preemption on wasm") 572 p("UNDEF") 573 } 574 575 func notImplemented() { 576 p("// Not implemented yet") 577 p("JMP ·abort(SB)") 578 }