github.com/m10x/go/src@v0.0.0-20220112094212-ba61592315da/runtime/mkpreempt.go (about) 1 // Copyright 2019 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 //go:build ignore 6 7 // mkpreempt generates the asyncPreempt functions for each 8 // architecture. 9 package main 10 11 import ( 12 "flag" 13 "fmt" 14 "io" 15 "log" 16 "os" 17 "strings" 18 ) 19 20 // Copied from cmd/compile/internal/ssa/gen/*Ops.go 21 22 var regNames386 = []string{ 23 "AX", 24 "CX", 25 "DX", 26 "BX", 27 "SP", 28 "BP", 29 "SI", 30 "DI", 31 "X0", 32 "X1", 33 "X2", 34 "X3", 35 "X4", 36 "X5", 37 "X6", 38 "X7", 39 } 40 41 var regNamesAMD64 = []string{ 42 "AX", 43 "CX", 44 "DX", 45 "BX", 46 "SP", 47 "BP", 48 "SI", 49 "DI", 50 "R8", 51 "R9", 52 "R10", 53 "R11", 54 "R12", 55 "R13", 56 "R14", 57 "R15", 58 "X0", 59 "X1", 60 "X2", 61 "X3", 62 "X4", 63 "X5", 64 "X6", 65 "X7", 66 "X8", 67 "X9", 68 "X10", 69 "X11", 70 "X12", 71 "X13", 72 "X14", 73 "X15", 74 } 75 76 var out io.Writer 77 78 var arches = map[string]func(){ 79 "386": gen386, 80 "amd64": genAMD64, 81 "arm": genARM, 82 "arm64": genARM64, 83 "mips64x": func() { genMIPS(true) }, 84 "mipsx": func() { genMIPS(false) }, 85 "ppc64x": genPPC64, 86 "riscv64": genRISCV64, 87 "s390x": genS390X, 88 "wasm": genWasm, 89 } 90 var beLe = map[string]bool{"mips64x": true, "mipsx": true, "ppc64x": true} 91 92 func main() { 93 flag.Parse() 94 if flag.NArg() > 0 { 95 out = os.Stdout 96 for _, arch := range flag.Args() { 97 gen, ok := arches[arch] 98 if !ok { 99 log.Fatalf("unknown arch %s", arch) 100 } 101 header(arch) 102 gen() 103 } 104 return 105 } 106 107 for arch, gen := range arches { 108 f, err := os.Create(fmt.Sprintf("preempt_%s.s", arch)) 109 if err != nil { 110 log.Fatal(err) 111 } 112 out = f 113 header(arch) 114 gen() 115 if err := f.Close(); err != nil { 116 log.Fatal(err) 117 } 118 } 119 } 120 121 func header(arch string) { 122 fmt.Fprintf(out, "// Code generated by mkpreempt.go; DO NOT EDIT.\n\n") 123 if beLe[arch] { 124 base := arch[:len(arch)-1] 125 fmt.Fprintf(out, "//go:build %s || %sle\n", base, base) 126 } 127 fmt.Fprintf(out, "#include \"go_asm.h\"\n") 128 fmt.Fprintf(out, "#include \"textflag.h\"\n\n") 129 fmt.Fprintf(out, "TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0\n") 130 } 131 132 func p(f string, args ...any) { 133 fmted := fmt.Sprintf(f, args...) 134 fmt.Fprintf(out, "\t%s\n", strings.ReplaceAll(fmted, "\n", "\n\t")) 135 } 136 137 func label(l string) { 138 fmt.Fprintf(out, "%s\n", l) 139 } 140 141 type layout struct { 142 stack int 143 regs []regPos 144 sp string // stack pointer register 145 } 146 147 type regPos struct { 148 pos int 149 150 op string 151 reg string 152 153 // If this register requires special save and restore, these 154 // give those operations with a %d placeholder for the stack 155 // offset. 156 save, restore string 157 } 158 159 func (l *layout) add(op, reg string, size int) { 160 l.regs = append(l.regs, regPos{op: op, reg: reg, pos: l.stack}) 161 l.stack += size 162 } 163 164 func (l *layout) addSpecial(save, restore string, size int) { 165 l.regs = append(l.regs, regPos{save: save, restore: restore, pos: l.stack}) 166 l.stack += size 167 } 168 169 func (l *layout) save() { 170 for _, reg := range l.regs { 171 if reg.save != "" { 172 p(reg.save, reg.pos) 173 } else { 174 p("%s %s, %d(%s)", reg.op, reg.reg, reg.pos, l.sp) 175 } 176 } 177 } 178 179 func (l *layout) restore() { 180 for i := len(l.regs) - 1; i >= 0; i-- { 181 reg := l.regs[i] 182 if reg.restore != "" { 183 p(reg.restore, reg.pos) 184 } else { 185 p("%s %d(%s), %s", reg.op, reg.pos, l.sp, reg.reg) 186 } 187 } 188 } 189 190 func gen386() { 191 p("PUSHFL") 192 // Save general purpose registers. 193 var l = layout{sp: "SP"} 194 for _, reg := range regNames386 { 195 if reg == "SP" || strings.HasPrefix(reg, "X") { 196 continue 197 } 198 l.add("MOVL", reg, 4) 199 } 200 201 softfloat := "GO386_softfloat" 202 203 // Save SSE state only if supported. 204 lSSE := layout{stack: l.stack, sp: "SP"} 205 for i := 0; i < 8; i++ { 206 lSSE.add("MOVUPS", fmt.Sprintf("X%d", i), 16) 207 } 208 209 p("ADJSP $%d", lSSE.stack) 210 p("NOP SP") 211 l.save() 212 p("#ifndef %s", softfloat) 213 lSSE.save() 214 p("#endif") 215 p("CALL ·asyncPreempt2(SB)") 216 p("#ifndef %s", softfloat) 217 lSSE.restore() 218 p("#endif") 219 l.restore() 220 p("ADJSP $%d", -lSSE.stack) 221 222 p("POPFL") 223 p("RET") 224 } 225 226 func genAMD64() { 227 // Assign stack offsets. 228 var l = layout{sp: "SP"} 229 for _, reg := range regNamesAMD64 { 230 if reg == "SP" || reg == "BP" { 231 continue 232 } 233 if !strings.HasPrefix(reg, "X") { 234 l.add("MOVQ", reg, 8) 235 } 236 } 237 lSSE := layout{stack: l.stack, sp: "SP"} 238 for _, reg := range regNamesAMD64 { 239 if strings.HasPrefix(reg, "X") { 240 lSSE.add("MOVUPS", reg, 16) 241 } 242 } 243 244 // TODO: MXCSR register? 245 246 p("PUSHQ BP") 247 p("MOVQ SP, BP") 248 p("// Save flags before clobbering them") 249 p("PUSHFQ") 250 p("// obj doesn't understand ADD/SUB on SP, but does understand ADJSP") 251 p("ADJSP $%d", lSSE.stack) 252 p("// But vet doesn't know ADJSP, so suppress vet stack checking") 253 p("NOP SP") 254 255 l.save() 256 257 // Apparently, the signal handling code path in darwin kernel leaves 258 // the upper bits of Y registers in a dirty state, which causes 259 // many SSE operations (128-bit and narrower) become much slower. 260 // Clear the upper bits to get to a clean state. See issue #37174. 261 // It is safe here as Go code don't use the upper bits of Y registers. 262 p("#ifdef GOOS_darwin") 263 p("CMPB internal∕cpu·X86+const_offsetX86HasAVX(SB), $0") 264 p("JE 2(PC)") 265 p("VZEROUPPER") 266 p("#endif") 267 268 lSSE.save() 269 p("CALL ·asyncPreempt2(SB)") 270 lSSE.restore() 271 l.restore() 272 p("ADJSP $%d", -lSSE.stack) 273 p("POPFQ") 274 p("POPQ BP") 275 p("RET") 276 } 277 278 func genARM() { 279 // Add integer registers R0-R12. 280 // R13 (SP), R14 (LR), R15 (PC) are special and not saved here. 281 var l = layout{sp: "R13", stack: 4} // add LR slot 282 for i := 0; i <= 12; i++ { 283 reg := fmt.Sprintf("R%d", i) 284 if i == 10 { 285 continue // R10 is g register, no need to save/restore 286 } 287 l.add("MOVW", reg, 4) 288 } 289 // Add flag register. 290 l.addSpecial( 291 "MOVW CPSR, R0\nMOVW R0, %d(R13)", 292 "MOVW %d(R13), R0\nMOVW R0, CPSR", 293 4) 294 295 // Add floating point registers F0-F15 and flag register. 296 var lfp = layout{stack: l.stack, sp: "R13"} 297 lfp.addSpecial( 298 "MOVW FPCR, R0\nMOVW R0, %d(R13)", 299 "MOVW %d(R13), R0\nMOVW R0, FPCR", 300 4) 301 for i := 0; i <= 15; i++ { 302 reg := fmt.Sprintf("F%d", i) 303 lfp.add("MOVD", reg, 8) 304 } 305 306 p("MOVW.W R14, -%d(R13)", lfp.stack) // allocate frame, save LR 307 l.save() 308 p("MOVB ·goarm(SB), R0\nCMP $6, R0\nBLT nofp") // test goarm, and skip FP registers if goarm=5. 309 lfp.save() 310 label("nofp:") 311 p("CALL ·asyncPreempt2(SB)") 312 p("MOVB ·goarm(SB), R0\nCMP $6, R0\nBLT nofp2") // test goarm, and skip FP registers if goarm=5. 313 lfp.restore() 314 label("nofp2:") 315 l.restore() 316 317 p("MOVW %d(R13), R14", lfp.stack) // sigctxt.pushCall pushes LR on stack, restore it 318 p("MOVW.P %d(R13), R15", lfp.stack+4) // load PC, pop frame (including the space pushed by sigctxt.pushCall) 319 p("UNDEF") // shouldn't get here 320 } 321 322 func genARM64() { 323 // Add integer registers R0-R26 324 // R27 (REGTMP), R28 (g), R29 (FP), R30 (LR), R31 (SP) are special 325 // and not saved here. 326 var l = layout{sp: "RSP", stack: 8} // add slot to save PC of interrupted instruction 327 for i := 0; i <= 26; i++ { 328 if i == 18 { 329 continue // R18 is not used, skip 330 } 331 reg := fmt.Sprintf("R%d", i) 332 l.add("MOVD", reg, 8) 333 } 334 // Add flag registers. 335 l.addSpecial( 336 "MOVD NZCV, R0\nMOVD R0, %d(RSP)", 337 "MOVD %d(RSP), R0\nMOVD R0, NZCV", 338 8) 339 l.addSpecial( 340 "MOVD FPSR, R0\nMOVD R0, %d(RSP)", 341 "MOVD %d(RSP), R0\nMOVD R0, FPSR", 342 8) 343 // TODO: FPCR? I don't think we'll change it, so no need to save. 344 // Add floating point registers F0-F31. 345 for i := 0; i <= 31; i++ { 346 reg := fmt.Sprintf("F%d", i) 347 l.add("FMOVD", reg, 8) 348 } 349 if l.stack%16 != 0 { 350 l.stack += 8 // SP needs 16-byte alignment 351 } 352 353 // allocate frame, save PC of interrupted instruction (in LR) 354 p("MOVD R30, %d(RSP)", -l.stack) 355 p("SUB $%d, RSP", l.stack) 356 p("#ifdef GOOS_linux") 357 p("MOVD R29, -8(RSP)") // save frame pointer (only used on Linux) 358 p("SUB $8, RSP, R29") // set up new frame pointer 359 p("#endif") 360 // On iOS, save the LR again after decrementing SP. We run the 361 // signal handler on the G stack (as it doesn't support sigaltstack), 362 // so any writes below SP may be clobbered. 363 p("#ifdef GOOS_ios") 364 p("MOVD R30, (RSP)") 365 p("#endif") 366 367 l.save() 368 p("CALL ·asyncPreempt2(SB)") 369 l.restore() 370 371 p("MOVD %d(RSP), R30", l.stack) // sigctxt.pushCall has pushed LR (at interrupt) on stack, restore it 372 p("#ifdef GOOS_linux") 373 p("MOVD -8(RSP), R29") // restore frame pointer 374 p("#endif") 375 p("MOVD (RSP), R27") // load PC to REGTMP 376 p("ADD $%d, RSP", l.stack+16) // pop frame (including the space pushed by sigctxt.pushCall) 377 p("JMP (R27)") 378 } 379 380 func genMIPS(_64bit bool) { 381 mov := "MOVW" 382 movf := "MOVF" 383 add := "ADD" 384 sub := "SUB" 385 r28 := "R28" 386 regsize := 4 387 softfloat := "GOMIPS_softfloat" 388 if _64bit { 389 mov = "MOVV" 390 movf = "MOVD" 391 add = "ADDV" 392 sub = "SUBV" 393 r28 = "RSB" 394 regsize = 8 395 softfloat = "GOMIPS64_softfloat" 396 } 397 398 // Add integer registers R1-R22, R24-R25, R28 399 // R0 (zero), R23 (REGTMP), R29 (SP), R30 (g), R31 (LR) are special, 400 // and not saved here. R26 and R27 are reserved by kernel and not used. 401 var l = layout{sp: "R29", stack: regsize} // add slot to save PC of interrupted instruction (in LR) 402 for i := 1; i <= 25; i++ { 403 if i == 23 { 404 continue // R23 is REGTMP 405 } 406 reg := fmt.Sprintf("R%d", i) 407 l.add(mov, reg, regsize) 408 } 409 l.add(mov, r28, regsize) 410 l.addSpecial( 411 mov+" HI, R1\n"+mov+" R1, %d(R29)", 412 mov+" %d(R29), R1\n"+mov+" R1, HI", 413 regsize) 414 l.addSpecial( 415 mov+" LO, R1\n"+mov+" R1, %d(R29)", 416 mov+" %d(R29), R1\n"+mov+" R1, LO", 417 regsize) 418 419 // Add floating point control/status register FCR31 (FCR0-FCR30 are irrelevant) 420 var lfp = layout{sp: "R29", stack: l.stack} 421 lfp.addSpecial( 422 mov+" FCR31, R1\n"+mov+" R1, %d(R29)", 423 mov+" %d(R29), R1\n"+mov+" R1, FCR31", 424 regsize) 425 // Add floating point registers F0-F31. 426 for i := 0; i <= 31; i++ { 427 reg := fmt.Sprintf("F%d", i) 428 lfp.add(movf, reg, regsize) 429 } 430 431 // allocate frame, save PC of interrupted instruction (in LR) 432 p(mov+" R31, -%d(R29)", lfp.stack) 433 p(sub+" $%d, R29", lfp.stack) 434 435 l.save() 436 p("#ifndef %s", softfloat) 437 lfp.save() 438 p("#endif") 439 p("CALL ·asyncPreempt2(SB)") 440 p("#ifndef %s", softfloat) 441 lfp.restore() 442 p("#endif") 443 l.restore() 444 445 p(mov+" %d(R29), R31", lfp.stack) // sigctxt.pushCall has pushed LR (at interrupt) on stack, restore it 446 p(mov + " (R29), R23") // load PC to REGTMP 447 p(add+" $%d, R29", lfp.stack+regsize) // pop frame (including the space pushed by sigctxt.pushCall) 448 p("JMP (R23)") 449 } 450 451 func genPPC64() { 452 // Add integer registers R3-R29 453 // R0 (zero), R1 (SP), R30 (g) are special and not saved here. 454 // R2 (TOC pointer in PIC mode), R12 (function entry address in PIC mode) have been saved in sigctxt.pushCall. 455 // R31 (REGTMP) will be saved manually. 456 var l = layout{sp: "R1", stack: 32 + 8} // MinFrameSize on PPC64, plus one word for saving R31 457 for i := 3; i <= 29; i++ { 458 if i == 12 || i == 13 { 459 // R12 has been saved in sigctxt.pushCall. 460 // R13 is TLS pointer, not used by Go code. we must NOT 461 // restore it, otherwise if we parked and resumed on a 462 // different thread we'll mess up TLS addresses. 463 continue 464 } 465 reg := fmt.Sprintf("R%d", i) 466 l.add("MOVD", reg, 8) 467 } 468 l.addSpecial( 469 "MOVW CR, R31\nMOVW R31, %d(R1)", 470 "MOVW %d(R1), R31\nMOVFL R31, $0xff", // this is MOVW R31, CR 471 8) // CR is 4-byte wide, but just keep the alignment 472 l.addSpecial( 473 "MOVD XER, R31\nMOVD R31, %d(R1)", 474 "MOVD %d(R1), R31\nMOVD R31, XER", 475 8) 476 // Add floating point registers F0-F31. 477 for i := 0; i <= 31; i++ { 478 reg := fmt.Sprintf("F%d", i) 479 l.add("FMOVD", reg, 8) 480 } 481 // Add floating point control/status register FPSCR. 482 l.addSpecial( 483 "MOVFL FPSCR, F0\nFMOVD F0, %d(R1)", 484 "FMOVD %d(R1), F0\nMOVFL F0, FPSCR", 485 8) 486 487 p("MOVD R31, -%d(R1)", l.stack-32) // save R31 first, we'll use R31 for saving LR 488 p("MOVD LR, R31") 489 p("MOVDU R31, -%d(R1)", l.stack) // allocate frame, save PC of interrupted instruction (in LR) 490 491 l.save() 492 p("CALL ·asyncPreempt2(SB)") 493 l.restore() 494 495 p("MOVD %d(R1), R31", l.stack) // sigctxt.pushCall has pushed LR, R2, R12 (at interrupt) on stack, restore them 496 p("MOVD R31, LR") 497 p("MOVD %d(R1), R2", l.stack+8) 498 p("MOVD %d(R1), R12", l.stack+16) 499 p("MOVD (R1), R31") // load PC to CTR 500 p("MOVD R31, CTR") 501 p("MOVD 32(R1), R31") // restore R31 502 p("ADD $%d, R1", l.stack+32) // pop frame (including the space pushed by sigctxt.pushCall) 503 p("JMP (CTR)") 504 } 505 506 func genRISCV64() { 507 // X0 (zero), X1 (LR), X2 (SP), X3 (GP), X4 (TP), X27 (g), X31 (TMP) are special. 508 var l = layout{sp: "X2", stack: 8} 509 510 // Add integer registers (X5-X26, X28-30). 511 for i := 5; i < 31; i++ { 512 if i == 27 { 513 continue 514 } 515 reg := fmt.Sprintf("X%d", i) 516 l.add("MOV", reg, 8) 517 } 518 519 // Add floating point registers (F0-F31). 520 for i := 0; i <= 31; i++ { 521 reg := fmt.Sprintf("F%d", i) 522 l.add("MOVD", reg, 8) 523 } 524 525 p("MOV X1, -%d(X2)", l.stack) 526 p("ADD $-%d, X2", l.stack) 527 l.save() 528 p("CALL ·asyncPreempt2(SB)") 529 l.restore() 530 p("MOV %d(X2), X1", l.stack) 531 p("MOV (X2), X31") 532 p("ADD $%d, X2", l.stack+8) 533 p("JMP (X31)") 534 } 535 536 func genS390X() { 537 // Add integer registers R0-R12 538 // R13 (g), R14 (LR), R15 (SP) are special, and not saved here. 539 // Saving R10 (REGTMP) is not necessary, but it is saved anyway. 540 var l = layout{sp: "R15", stack: 16} // add slot to save PC of interrupted instruction and flags 541 l.addSpecial( 542 "STMG R0, R12, %d(R15)", 543 "LMG %d(R15), R0, R12", 544 13*8) 545 // Add floating point registers F0-F31. 546 for i := 0; i <= 15; i++ { 547 reg := fmt.Sprintf("F%d", i) 548 l.add("FMOVD", reg, 8) 549 } 550 551 // allocate frame, save PC of interrupted instruction (in LR) and flags (condition code) 552 p("IPM R10") // save flags upfront, as ADD will clobber flags 553 p("MOVD R14, -%d(R15)", l.stack) 554 p("ADD $-%d, R15", l.stack) 555 p("MOVW R10, 8(R15)") // save flags 556 557 l.save() 558 p("CALL ·asyncPreempt2(SB)") 559 l.restore() 560 561 p("MOVD %d(R15), R14", l.stack) // sigctxt.pushCall has pushed LR (at interrupt) on stack, restore it 562 p("ADD $%d, R15", l.stack+8) // pop frame (including the space pushed by sigctxt.pushCall) 563 p("MOVWZ -%d(R15), R10", l.stack) // load flags to REGTMP 564 p("TMLH R10, $(3<<12)") // restore flags 565 p("MOVD -%d(R15), R10", l.stack+8) // load PC to REGTMP 566 p("JMP (R10)") 567 } 568 569 func genWasm() { 570 p("// No async preemption on wasm") 571 p("UNDEF") 572 } 573 574 func notImplemented() { 575 p("// Not implemented yet") 576 p("JMP ·abort(SB)") 577 }