github.com/megatontech/mynoteforgo@v0.0.0-20200507084910-5d0c6ea6e890/源码/cmd/compile/internal/ssa/regalloc.go (about) 1 // Copyright 2015 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Register allocation. 6 // 7 // We use a version of a linear scan register allocator. We treat the 8 // whole function as a single long basic block and run through 9 // it using a greedy register allocator. Then all merge edges 10 // (those targeting a block with len(Preds)>1) are processed to 11 // shuffle data into the place that the target of the edge expects. 12 // 13 // The greedy allocator moves values into registers just before they 14 // are used, spills registers only when necessary, and spills the 15 // value whose next use is farthest in the future. 16 // 17 // The register allocator requires that a block is not scheduled until 18 // at least one of its predecessors have been scheduled. The most recent 19 // such predecessor provides the starting register state for a block. 20 // 21 // It also requires that there are no critical edges (critical = 22 // comes from a block with >1 successor and goes to a block with >1 23 // predecessor). This makes it easy to add fixup code on merge edges - 24 // the source of a merge edge has only one successor, so we can add 25 // fixup code to the end of that block. 26 27 // Spilling 28 // 29 // During the normal course of the allocator, we might throw a still-live 30 // value out of all registers. When that value is subsequently used, we must 31 // load it from a slot on the stack. We must also issue an instruction to 32 // initialize that stack location with a copy of v. 33 // 34 // pre-regalloc: 35 // (1) v = Op ... 36 // (2) x = Op ... 37 // (3) ... = Op v ... 38 // 39 // post-regalloc: 40 // (1) v = Op ... : AX // computes v, store result in AX 41 // s = StoreReg v // spill v to a stack slot 42 // (2) x = Op ... : AX // some other op uses AX 43 // c = LoadReg s : CX // restore v from stack slot 44 // (3) ... = Op c ... // use the restored value 45 // 46 // Allocation occurs normally until we reach (3) and we realize we have 47 // a use of v and it isn't in any register. At that point, we allocate 48 // a spill (a StoreReg) for v. We can't determine the correct place for 49 // the spill at this point, so we allocate the spill as blockless initially. 50 // The restore is then generated to load v back into a register so it can 51 // be used. Subsequent uses of v will use the restored value c instead. 52 // 53 // What remains is the question of where to schedule the spill. 54 // During allocation, we keep track of the dominator of all restores of v. 55 // The spill of v must dominate that block. The spill must also be issued at 56 // a point where v is still in a register. 57 // 58 // To find the right place, start at b, the block which dominates all restores. 59 // - If b is v.Block, then issue the spill right after v. 60 // It is known to be in a register at that point, and dominates any restores. 61 // - Otherwise, if v is in a register at the start of b, 62 // put the spill of v at the start of b. 63 // - Otherwise, set b = immediate dominator of b, and repeat. 64 // 65 // Phi values are special, as always. We define two kinds of phis, those 66 // where the merge happens in a register (a "register" phi) and those where 67 // the merge happens in a stack location (a "stack" phi). 68 // 69 // A register phi must have the phi and all of its inputs allocated to the 70 // same register. Register phis are spilled similarly to regular ops. 71 // 72 // A stack phi must have the phi and all of its inputs allocated to the same 73 // stack location. Stack phis start out life already spilled - each phi 74 // input must be a store (using StoreReg) at the end of the corresponding 75 // predecessor block. 76 // b1: y = ... : AX b2: z = ... : BX 77 // y2 = StoreReg y z2 = StoreReg z 78 // goto b3 goto b3 79 // b3: x = phi(y2, z2) 80 // The stack allocator knows that StoreReg args of stack-allocated phis 81 // must be allocated to the same stack slot as the phi that uses them. 82 // x is now a spilled value and a restore must appear before its first use. 83 84 // TODO 85 86 // Use an affinity graph to mark two values which should use the 87 // same register. This affinity graph will be used to prefer certain 88 // registers for allocation. This affinity helps eliminate moves that 89 // are required for phi implementations and helps generate allocations 90 // for 2-register architectures. 91 92 // Note: regalloc generates a not-quite-SSA output. If we have: 93 // 94 // b1: x = ... : AX 95 // x2 = StoreReg x 96 // ... AX gets reused for something else ... 97 // if ... goto b3 else b4 98 // 99 // b3: x3 = LoadReg x2 : BX b4: x4 = LoadReg x2 : CX 100 // ... use x3 ... ... use x4 ... 101 // 102 // b2: ... use x3 ... 103 // 104 // If b3 is the primary predecessor of b2, then we use x3 in b2 and 105 // add a x4:CX->BX copy at the end of b4. 106 // But the definition of x3 doesn't dominate b2. We should really 107 // insert a dummy phi at the start of b2 (x5=phi(x3,x4):BX) to keep 108 // SSA form. For now, we ignore this problem as remaining in strict 109 // SSA form isn't needed after regalloc. We'll just leave the use 110 // of x3 not dominated by the definition of x3, and the CX->BX copy 111 // will have no use (so don't run deadcode after regalloc!). 112 // TODO: maybe we should introduce these extra phis? 113 114 package ssa 115 116 import ( 117 "cmd/compile/internal/types" 118 "cmd/internal/objabi" 119 "cmd/internal/src" 120 "cmd/internal/sys" 121 "fmt" 122 "math/bits" 123 "unsafe" 124 ) 125 126 const ( 127 moveSpills = iota 128 logSpills 129 regDebug 130 stackDebug 131 ) 132 133 // distance is a measure of how far into the future values are used. 134 // distance is measured in units of instructions. 135 const ( 136 likelyDistance = 1 137 normalDistance = 10 138 unlikelyDistance = 100 139 ) 140 141 // regalloc performs register allocation on f. It sets f.RegAlloc 142 // to the resulting allocation. 143 func regalloc(f *Func) { 144 var s regAllocState 145 s.init(f) 146 s.regalloc(f) 147 } 148 149 type register uint8 150 151 const noRegister register = 255 152 153 // A regMask encodes a set of machine registers. 154 // TODO: regMask -> regSet? 155 type regMask uint64 156 157 func (m regMask) String() string { 158 s := "" 159 for r := register(0); m != 0; r++ { 160 if m>>r&1 == 0 { 161 continue 162 } 163 m &^= regMask(1) << r 164 if s != "" { 165 s += " " 166 } 167 s += fmt.Sprintf("r%d", r) 168 } 169 return s 170 } 171 172 func (s *regAllocState) RegMaskString(m regMask) string { 173 str := "" 174 for r := register(0); m != 0; r++ { 175 if m>>r&1 == 0 { 176 continue 177 } 178 m &^= regMask(1) << r 179 if str != "" { 180 str += " " 181 } 182 str += s.registers[r].String() 183 } 184 return str 185 } 186 187 // countRegs returns the number of set bits in the register mask. 188 func countRegs(r regMask) int { 189 return bits.OnesCount64(uint64(r)) 190 } 191 192 // pickReg picks an arbitrary register from the register mask. 193 func pickReg(r regMask) register { 194 if r == 0 { 195 panic("can't pick a register from an empty set") 196 } 197 // pick the lowest one 198 return register(bits.TrailingZeros64(uint64(r))) 199 } 200 201 type use struct { 202 dist int32 // distance from start of the block to a use of a value 203 pos src.XPos // source position of the use 204 next *use // linked list of uses of a value in nondecreasing dist order 205 } 206 207 // A valState records the register allocation state for a (pre-regalloc) value. 208 type valState struct { 209 regs regMask // the set of registers holding a Value (usually just one) 210 uses *use // list of uses in this block 211 spill *Value // spilled copy of the Value (if any) 212 restoreMin int32 // minimum of all restores' blocks' sdom.entry 213 restoreMax int32 // maximum of all restores' blocks' sdom.exit 214 needReg bool // cached value of !v.Type.IsMemory() && !v.Type.IsVoid() && !.v.Type.IsFlags() 215 rematerializeable bool // cached value of v.rematerializeable() 216 } 217 218 type regState struct { 219 v *Value // Original (preregalloc) Value stored in this register. 220 c *Value // A Value equal to v which is currently in a register. Might be v or a copy of it. 221 // If a register is unused, v==c==nil 222 } 223 224 type regAllocState struct { 225 f *Func 226 227 sdom SparseTree 228 registers []Register 229 numRegs register 230 SPReg register 231 SBReg register 232 GReg register 233 allocatable regMask 234 235 // for each block, its primary predecessor. 236 // A predecessor of b is primary if it is the closest 237 // predecessor that appears before b in the layout order. 238 // We record the index in the Preds list where the primary predecessor sits. 239 primary []int32 240 241 // live values at the end of each block. live[b.ID] is a list of value IDs 242 // which are live at the end of b, together with a count of how many instructions 243 // forward to the next use. 244 live [][]liveInfo 245 // desired register assignments at the end of each block. 246 // Note that this is a static map computed before allocation occurs. Dynamic 247 // register desires (from partially completed allocations) will trump 248 // this information. 249 desired []desiredState 250 251 // current state of each (preregalloc) Value 252 values []valState 253 254 // ID of SP, SB values 255 sp, sb ID 256 257 // For each Value, map from its value ID back to the 258 // preregalloc Value it was derived from. 259 orig []*Value 260 261 // current state of each register 262 regs []regState 263 264 // registers that contain values which can't be kicked out 265 nospill regMask 266 267 // mask of registers currently in use 268 used regMask 269 270 // mask of registers used in the current instruction 271 tmpused regMask 272 273 // current block we're working on 274 curBlock *Block 275 276 // cache of use records 277 freeUseRecords *use 278 279 // endRegs[blockid] is the register state at the end of each block. 280 // encoded as a set of endReg records. 281 endRegs [][]endReg 282 283 // startRegs[blockid] is the register state at the start of merge blocks. 284 // saved state does not include the state of phi ops in the block. 285 startRegs [][]startReg 286 287 // spillLive[blockid] is the set of live spills at the end of each block 288 spillLive [][]ID 289 290 // a set of copies we generated to move things around, and 291 // whether it is used in shuffle. Unused copies will be deleted. 292 copies map[*Value]bool 293 294 loopnest *loopnest 295 296 // choose a good order in which to visit blocks for allocation purposes. 297 visitOrder []*Block 298 } 299 300 type endReg struct { 301 r register 302 v *Value // pre-regalloc value held in this register (TODO: can we use ID here?) 303 c *Value // cached version of the value 304 } 305 306 type startReg struct { 307 r register 308 v *Value // pre-regalloc value needed in this register 309 c *Value // cached version of the value 310 pos src.XPos // source position of use of this register 311 } 312 313 // freeReg frees up register r. Any current user of r is kicked out. 314 func (s *regAllocState) freeReg(r register) { 315 v := s.regs[r].v 316 if v == nil { 317 s.f.Fatalf("tried to free an already free register %d\n", r) 318 } 319 320 // Mark r as unused. 321 if s.f.pass.debug > regDebug { 322 fmt.Printf("freeReg %s (dump %s/%s)\n", &s.registers[r], v, s.regs[r].c) 323 } 324 s.regs[r] = regState{} 325 s.values[v.ID].regs &^= regMask(1) << r 326 s.used &^= regMask(1) << r 327 } 328 329 // freeRegs frees up all registers listed in m. 330 func (s *regAllocState) freeRegs(m regMask) { 331 for m&s.used != 0 { 332 s.freeReg(pickReg(m & s.used)) 333 } 334 } 335 336 // setOrig records that c's original value is the same as 337 // v's original value. 338 func (s *regAllocState) setOrig(c *Value, v *Value) { 339 for int(c.ID) >= len(s.orig) { 340 s.orig = append(s.orig, nil) 341 } 342 if s.orig[c.ID] != nil { 343 s.f.Fatalf("orig value set twice %s %s", c, v) 344 } 345 s.orig[c.ID] = s.orig[v.ID] 346 } 347 348 // assignReg assigns register r to hold c, a copy of v. 349 // r must be unused. 350 func (s *regAllocState) assignReg(r register, v *Value, c *Value) { 351 if s.f.pass.debug > regDebug { 352 fmt.Printf("assignReg %s %s/%s\n", &s.registers[r], v, c) 353 } 354 if s.regs[r].v != nil { 355 s.f.Fatalf("tried to assign register %d to %s/%s but it is already used by %s", r, v, c, s.regs[r].v) 356 } 357 358 // Update state. 359 s.regs[r] = regState{v, c} 360 s.values[v.ID].regs |= regMask(1) << r 361 s.used |= regMask(1) << r 362 s.f.setHome(c, &s.registers[r]) 363 } 364 365 // allocReg chooses a register from the set of registers in mask. 366 // If there is no unused register, a Value will be kicked out of 367 // a register to make room. 368 func (s *regAllocState) allocReg(mask regMask, v *Value) register { 369 if v.OnWasmStack { 370 return noRegister 371 } 372 373 mask &= s.allocatable 374 mask &^= s.nospill 375 if mask == 0 { 376 s.f.Fatalf("no register available for %s", v.LongString()) 377 } 378 379 // Pick an unused register if one is available. 380 if mask&^s.used != 0 { 381 return pickReg(mask &^ s.used) 382 } 383 384 // Pick a value to spill. Spill the value with the 385 // farthest-in-the-future use. 386 // TODO: Prefer registers with already spilled Values? 387 // TODO: Modify preference using affinity graph. 388 // TODO: if a single value is in multiple registers, spill one of them 389 // before spilling a value in just a single register. 390 391 // Find a register to spill. We spill the register containing the value 392 // whose next use is as far in the future as possible. 393 // https://en.wikipedia.org/wiki/Page_replacement_algorithm#The_theoretically_optimal_page_replacement_algorithm 394 var r register 395 maxuse := int32(-1) 396 for t := register(0); t < s.numRegs; t++ { 397 if mask>>t&1 == 0 { 398 continue 399 } 400 v := s.regs[t].v 401 if n := s.values[v.ID].uses.dist; n > maxuse { 402 // v's next use is farther in the future than any value 403 // we've seen so far. A new best spill candidate. 404 r = t 405 maxuse = n 406 } 407 } 408 if maxuse == -1 { 409 s.f.Fatalf("couldn't find register to spill") 410 } 411 412 if s.f.Config.ctxt.Arch.Arch == sys.ArchWasm { 413 // TODO(neelance): In theory this should never happen, because all wasm registers are equal. 414 // So if there is still a free register, the allocation should have picked that one in the first place insead of 415 // trying to kick some other value out. In practice, this case does happen and it breaks the stack optimization. 416 s.freeReg(r) 417 return r 418 } 419 420 // Try to move it around before kicking out, if there is a free register. 421 // We generate a Copy and record it. It will be deleted if never used. 422 v2 := s.regs[r].v 423 m := s.compatRegs(v2.Type) &^ s.used &^ s.tmpused &^ (regMask(1) << r) 424 if m != 0 && !s.values[v2.ID].rematerializeable && countRegs(s.values[v2.ID].regs) == 1 { 425 r2 := pickReg(m) 426 c := s.curBlock.NewValue1(v2.Pos, OpCopy, v2.Type, s.regs[r].c) 427 s.copies[c] = false 428 if s.f.pass.debug > regDebug { 429 fmt.Printf("copy %s to %s : %s\n", v2, c, &s.registers[r2]) 430 } 431 s.setOrig(c, v2) 432 s.assignReg(r2, v2, c) 433 } 434 s.freeReg(r) 435 return r 436 } 437 438 // makeSpill returns a Value which represents the spilled value of v. 439 // b is the block in which the spill is used. 440 func (s *regAllocState) makeSpill(v *Value, b *Block) *Value { 441 vi := &s.values[v.ID] 442 if vi.spill != nil { 443 // Final block not known - keep track of subtree where restores reside. 444 vi.restoreMin = min32(vi.restoreMin, s.sdom[b.ID].entry) 445 vi.restoreMax = max32(vi.restoreMax, s.sdom[b.ID].exit) 446 return vi.spill 447 } 448 // Make a spill for v. We don't know where we want 449 // to put it yet, so we leave it blockless for now. 450 spill := s.f.newValueNoBlock(OpStoreReg, v.Type, v.Pos) 451 // We also don't know what the spill's arg will be. 452 // Leave it argless for now. 453 s.setOrig(spill, v) 454 vi.spill = spill 455 vi.restoreMin = s.sdom[b.ID].entry 456 vi.restoreMax = s.sdom[b.ID].exit 457 return spill 458 } 459 460 // allocValToReg allocates v to a register selected from regMask and 461 // returns the register copy of v. Any previous user is kicked out and spilled 462 // (if necessary). Load code is added at the current pc. If nospill is set the 463 // allocated register is marked nospill so the assignment cannot be 464 // undone until the caller allows it by clearing nospill. Returns a 465 // *Value which is either v or a copy of v allocated to the chosen register. 466 func (s *regAllocState) allocValToReg(v *Value, mask regMask, nospill bool, pos src.XPos) *Value { 467 if s.f.Config.ctxt.Arch.Arch == sys.ArchWasm && v.rematerializeable() { 468 c := v.copyIntoWithXPos(s.curBlock, pos) 469 c.OnWasmStack = true 470 s.setOrig(c, v) 471 return c 472 } 473 if v.OnWasmStack { 474 return v 475 } 476 477 vi := &s.values[v.ID] 478 pos = pos.WithNotStmt() 479 // Check if v is already in a requested register. 480 if mask&vi.regs != 0 { 481 r := pickReg(mask & vi.regs) 482 if s.regs[r].v != v || s.regs[r].c == nil { 483 panic("bad register state") 484 } 485 if nospill { 486 s.nospill |= regMask(1) << r 487 } 488 return s.regs[r].c 489 } 490 491 var r register 492 // If nospill is set, the value is used immedately, so it can live on the WebAssembly stack. 493 onWasmStack := nospill && s.f.Config.ctxt.Arch.Arch == sys.ArchWasm 494 if !onWasmStack { 495 // Allocate a register. 496 r = s.allocReg(mask, v) 497 } 498 499 // Allocate v to the new register. 500 var c *Value 501 if vi.regs != 0 { 502 // Copy from a register that v is already in. 503 r2 := pickReg(vi.regs) 504 if s.regs[r2].v != v { 505 panic("bad register state") 506 } 507 c = s.curBlock.NewValue1(pos, OpCopy, v.Type, s.regs[r2].c) 508 } else if v.rematerializeable() { 509 // Rematerialize instead of loading from the spill location. 510 c = v.copyIntoWithXPos(s.curBlock, pos) 511 } else { 512 // Load v from its spill location. 513 spill := s.makeSpill(v, s.curBlock) 514 if s.f.pass.debug > logSpills { 515 s.f.Warnl(vi.spill.Pos, "load spill for %v from %v", v, spill) 516 } 517 c = s.curBlock.NewValue1(pos, OpLoadReg, v.Type, spill) 518 } 519 520 s.setOrig(c, v) 521 522 if onWasmStack { 523 c.OnWasmStack = true 524 return c 525 } 526 527 s.assignReg(r, v, c) 528 if c.Op == OpLoadReg && s.isGReg(r) { 529 s.f.Fatalf("allocValToReg.OpLoadReg targeting g: " + c.LongString()) 530 } 531 if nospill { 532 s.nospill |= regMask(1) << r 533 } 534 return c 535 } 536 537 // isLeaf reports whether f performs any calls. 538 func isLeaf(f *Func) bool { 539 for _, b := range f.Blocks { 540 for _, v := range b.Values { 541 if opcodeTable[v.Op].call { 542 return false 543 } 544 } 545 } 546 return true 547 } 548 549 func (s *regAllocState) init(f *Func) { 550 s.f = f 551 s.f.RegAlloc = s.f.Cache.locs[:0] 552 s.registers = f.Config.registers 553 if nr := len(s.registers); nr == 0 || nr > int(noRegister) || nr > int(unsafe.Sizeof(regMask(0))*8) { 554 s.f.Fatalf("bad number of registers: %d", nr) 555 } else { 556 s.numRegs = register(nr) 557 } 558 // Locate SP, SB, and g registers. 559 s.SPReg = noRegister 560 s.SBReg = noRegister 561 s.GReg = noRegister 562 for r := register(0); r < s.numRegs; r++ { 563 switch s.registers[r].String() { 564 case "SP": 565 s.SPReg = r 566 case "SB": 567 s.SBReg = r 568 case "g": 569 s.GReg = r 570 } 571 } 572 // Make sure we found all required registers. 573 switch noRegister { 574 case s.SPReg: 575 s.f.Fatalf("no SP register found") 576 case s.SBReg: 577 s.f.Fatalf("no SB register found") 578 case s.GReg: 579 if f.Config.hasGReg { 580 s.f.Fatalf("no g register found") 581 } 582 } 583 584 // Figure out which registers we're allowed to use. 585 s.allocatable = s.f.Config.gpRegMask | s.f.Config.fpRegMask | s.f.Config.specialRegMask 586 s.allocatable &^= 1 << s.SPReg 587 s.allocatable &^= 1 << s.SBReg 588 if s.f.Config.hasGReg { 589 s.allocatable &^= 1 << s.GReg 590 } 591 if s.f.Config.ctxt.Framepointer_enabled && s.f.Config.FPReg >= 0 { 592 s.allocatable &^= 1 << uint(s.f.Config.FPReg) 593 } 594 if s.f.Config.LinkReg != -1 { 595 if isLeaf(f) { 596 // Leaf functions don't save/restore the link register. 597 s.allocatable &^= 1 << uint(s.f.Config.LinkReg) 598 } 599 if s.f.Config.arch == "arm" && objabi.GOARM == 5 { 600 // On ARMv5 we insert softfloat calls at each FP instruction. 601 // This clobbers LR almost everywhere. Disable allocating LR 602 // on ARMv5. 603 s.allocatable &^= 1 << uint(s.f.Config.LinkReg) 604 } 605 } 606 if s.f.Config.ctxt.Flag_dynlink { 607 switch s.f.Config.arch { 608 case "amd64": 609 s.allocatable &^= 1 << 15 // R15 610 case "arm": 611 s.allocatable &^= 1 << 9 // R9 612 case "ppc64le": // R2 already reserved. 613 // nothing to do 614 case "arm64": 615 // nothing to do? 616 case "386": 617 // nothing to do. 618 // Note that for Flag_shared (position independent code) 619 // we do need to be careful, but that carefulness is hidden 620 // in the rewrite rules so we always have a free register 621 // available for global load/stores. See gen/386.rules (search for Flag_shared). 622 case "s390x": 623 s.allocatable &^= 1 << 11 // R11 624 default: 625 s.f.fe.Fatalf(src.NoXPos, "arch %s not implemented", s.f.Config.arch) 626 } 627 } 628 if s.f.Config.nacl { 629 switch s.f.Config.arch { 630 case "arm": 631 s.allocatable &^= 1 << 9 // R9 is "thread pointer" on nacl/arm 632 case "amd64p32": 633 s.allocatable &^= 1 << 5 // BP - reserved for nacl 634 s.allocatable &^= 1 << 15 // R15 - reserved for nacl 635 } 636 } 637 if s.f.Config.use387 { 638 s.allocatable &^= 1 << 15 // X7 disallowed (one 387 register is used as scratch space during SSE->387 generation in ../x86/387.go) 639 } 640 641 // Linear scan register allocation can be influenced by the order in which blocks appear. 642 // Decouple the register allocation order from the generated block order. 643 // This also creates an opportunity for experiments to find a better order. 644 s.visitOrder = layoutRegallocOrder(f) 645 646 // Compute block order. This array allows us to distinguish forward edges 647 // from backward edges and compute how far they go. 648 blockOrder := make([]int32, f.NumBlocks()) 649 for i, b := range s.visitOrder { 650 blockOrder[b.ID] = int32(i) 651 } 652 653 s.regs = make([]regState, s.numRegs) 654 s.values = make([]valState, f.NumValues()) 655 s.orig = make([]*Value, f.NumValues()) 656 s.copies = make(map[*Value]bool) 657 for _, b := range s.visitOrder { 658 for _, v := range b.Values { 659 if !v.Type.IsMemory() && !v.Type.IsVoid() && !v.Type.IsFlags() && !v.Type.IsTuple() { 660 s.values[v.ID].needReg = true 661 s.values[v.ID].rematerializeable = v.rematerializeable() 662 s.orig[v.ID] = v 663 } 664 // Note: needReg is false for values returning Tuple types. 665 // Instead, we mark the corresponding Selects as needReg. 666 } 667 } 668 s.computeLive() 669 670 // Compute primary predecessors. 671 s.primary = make([]int32, f.NumBlocks()) 672 for _, b := range s.visitOrder { 673 best := -1 674 for i, e := range b.Preds { 675 p := e.b 676 if blockOrder[p.ID] >= blockOrder[b.ID] { 677 continue // backward edge 678 } 679 if best == -1 || blockOrder[p.ID] > blockOrder[b.Preds[best].b.ID] { 680 best = i 681 } 682 } 683 s.primary[b.ID] = int32(best) 684 } 685 686 s.endRegs = make([][]endReg, f.NumBlocks()) 687 s.startRegs = make([][]startReg, f.NumBlocks()) 688 s.spillLive = make([][]ID, f.NumBlocks()) 689 s.sdom = f.sdom() 690 691 // wasm: Mark instructions that can be optimized to have their values only on the WebAssembly stack. 692 if f.Config.ctxt.Arch.Arch == sys.ArchWasm { 693 canLiveOnStack := f.newSparseSet(f.NumValues()) 694 defer f.retSparseSet(canLiveOnStack) 695 for _, b := range f.Blocks { 696 // New block. Clear candidate set. 697 canLiveOnStack.clear() 698 if b.Control != nil && b.Control.Uses == 1 && !opcodeTable[b.Control.Op].generic { 699 canLiveOnStack.add(b.Control.ID) 700 } 701 // Walking backwards. 702 for i := len(b.Values) - 1; i >= 0; i-- { 703 v := b.Values[i] 704 if canLiveOnStack.contains(v.ID) { 705 v.OnWasmStack = true 706 } else { 707 // Value can not live on stack. Values are not allowed to be reordered, so clear candidate set. 708 canLiveOnStack.clear() 709 } 710 for _, arg := range v.Args { 711 // Value can live on the stack if: 712 // - it is only used once 713 // - it is used in the same basic block 714 // - it is not a "mem" value 715 // - it is a WebAssembly op 716 if arg.Uses == 1 && arg.Block == v.Block && !arg.Type.IsMemory() && !opcodeTable[arg.Op].generic { 717 canLiveOnStack.add(arg.ID) 718 } 719 } 720 } 721 } 722 } 723 } 724 725 // Adds a use record for id at distance dist from the start of the block. 726 // All calls to addUse must happen with nonincreasing dist. 727 func (s *regAllocState) addUse(id ID, dist int32, pos src.XPos) { 728 r := s.freeUseRecords 729 if r != nil { 730 s.freeUseRecords = r.next 731 } else { 732 r = &use{} 733 } 734 r.dist = dist 735 r.pos = pos 736 r.next = s.values[id].uses 737 s.values[id].uses = r 738 if r.next != nil && dist > r.next.dist { 739 s.f.Fatalf("uses added in wrong order") 740 } 741 } 742 743 // advanceUses advances the uses of v's args from the state before v to the state after v. 744 // Any values which have no more uses are deallocated from registers. 745 func (s *regAllocState) advanceUses(v *Value) { 746 for _, a := range v.Args { 747 if !s.values[a.ID].needReg { 748 continue 749 } 750 ai := &s.values[a.ID] 751 r := ai.uses 752 ai.uses = r.next 753 if r.next == nil { 754 // Value is dead, free all registers that hold it. 755 s.freeRegs(ai.regs) 756 } 757 r.next = s.freeUseRecords 758 s.freeUseRecords = r 759 } 760 } 761 762 // liveAfterCurrentInstruction reports whether v is live after 763 // the current instruction is completed. v must be used by the 764 // current instruction. 765 func (s *regAllocState) liveAfterCurrentInstruction(v *Value) bool { 766 u := s.values[v.ID].uses 767 d := u.dist 768 for u != nil && u.dist == d { 769 u = u.next 770 } 771 return u != nil && u.dist > d 772 } 773 774 // Sets the state of the registers to that encoded in regs. 775 func (s *regAllocState) setState(regs []endReg) { 776 s.freeRegs(s.used) 777 for _, x := range regs { 778 s.assignReg(x.r, x.v, x.c) 779 } 780 } 781 782 // compatRegs returns the set of registers which can store a type t. 783 func (s *regAllocState) compatRegs(t *types.Type) regMask { 784 var m regMask 785 if t.IsTuple() || t.IsFlags() { 786 return 0 787 } 788 if t.IsFloat() || t == types.TypeInt128 { 789 m = s.f.Config.fpRegMask 790 } else { 791 m = s.f.Config.gpRegMask 792 } 793 return m & s.allocatable 794 } 795 796 // regspec returns the regInfo for operation op. 797 func (s *regAllocState) regspec(op Op) regInfo { 798 if op == OpConvert { 799 // OpConvert is a generic op, so it doesn't have a 800 // register set in the static table. It can use any 801 // allocatable integer register. 802 m := s.allocatable & s.f.Config.gpRegMask 803 return regInfo{inputs: []inputInfo{{regs: m}}, outputs: []outputInfo{{regs: m}}} 804 } 805 return opcodeTable[op].reg 806 } 807 808 func (s *regAllocState) isGReg(r register) bool { 809 return s.f.Config.hasGReg && s.GReg == r 810 } 811 812 func (s *regAllocState) regalloc(f *Func) { 813 regValLiveSet := f.newSparseSet(f.NumValues()) // set of values that may be live in register 814 defer f.retSparseSet(regValLiveSet) 815 var oldSched []*Value 816 var phis []*Value 817 var phiRegs []register 818 var args []*Value 819 820 // Data structure used for computing desired registers. 821 var desired desiredState 822 823 // Desired registers for inputs & outputs for each instruction in the block. 824 type dentry struct { 825 out [4]register // desired output registers 826 in [3][4]register // desired input registers (for inputs 0,1, and 2) 827 } 828 var dinfo []dentry 829 830 if f.Entry != f.Blocks[0] { 831 f.Fatalf("entry block must be first") 832 } 833 834 for _, b := range s.visitOrder { 835 if s.f.pass.debug > regDebug { 836 fmt.Printf("Begin processing block %v\n", b) 837 } 838 s.curBlock = b 839 840 // Initialize regValLiveSet and uses fields for this block. 841 // Walk backwards through the block doing liveness analysis. 842 regValLiveSet.clear() 843 for _, e := range s.live[b.ID] { 844 s.addUse(e.ID, int32(len(b.Values))+e.dist, e.pos) // pseudo-uses from beyond end of block 845 regValLiveSet.add(e.ID) 846 } 847 if v := b.Control; v != nil && s.values[v.ID].needReg { 848 s.addUse(v.ID, int32(len(b.Values)), b.Pos) // pseudo-use by control value 849 regValLiveSet.add(v.ID) 850 } 851 for i := len(b.Values) - 1; i >= 0; i-- { 852 v := b.Values[i] 853 regValLiveSet.remove(v.ID) 854 if v.Op == OpPhi { 855 // Remove v from the live set, but don't add 856 // any inputs. This is the state the len(b.Preds)>1 857 // case below desires; it wants to process phis specially. 858 continue 859 } 860 if opcodeTable[v.Op].call { 861 // Function call clobbers all the registers but SP and SB. 862 regValLiveSet.clear() 863 if s.sp != 0 && s.values[s.sp].uses != nil { 864 regValLiveSet.add(s.sp) 865 } 866 if s.sb != 0 && s.values[s.sb].uses != nil { 867 regValLiveSet.add(s.sb) 868 } 869 } 870 for _, a := range v.Args { 871 if !s.values[a.ID].needReg { 872 continue 873 } 874 s.addUse(a.ID, int32(i), v.Pos) 875 regValLiveSet.add(a.ID) 876 } 877 } 878 if s.f.pass.debug > regDebug { 879 fmt.Printf("use distances for %s\n", b) 880 for i := range s.values { 881 vi := &s.values[i] 882 u := vi.uses 883 if u == nil { 884 continue 885 } 886 fmt.Printf(" v%d:", i) 887 for u != nil { 888 fmt.Printf(" %d", u.dist) 889 u = u.next 890 } 891 fmt.Println() 892 } 893 } 894 895 // Make a copy of the block schedule so we can generate a new one in place. 896 // We make a separate copy for phis and regular values. 897 nphi := 0 898 for _, v := range b.Values { 899 if v.Op != OpPhi { 900 break 901 } 902 nphi++ 903 } 904 phis = append(phis[:0], b.Values[:nphi]...) 905 oldSched = append(oldSched[:0], b.Values[nphi:]...) 906 b.Values = b.Values[:0] 907 908 // Initialize start state of block. 909 if b == f.Entry { 910 // Regalloc state is empty to start. 911 if nphi > 0 { 912 f.Fatalf("phis in entry block") 913 } 914 } else if len(b.Preds) == 1 { 915 // Start regalloc state with the end state of the previous block. 916 s.setState(s.endRegs[b.Preds[0].b.ID]) 917 if nphi > 0 { 918 f.Fatalf("phis in single-predecessor block") 919 } 920 // Drop any values which are no longer live. 921 // This may happen because at the end of p, a value may be 922 // live but only used by some other successor of p. 923 for r := register(0); r < s.numRegs; r++ { 924 v := s.regs[r].v 925 if v != nil && !regValLiveSet.contains(v.ID) { 926 s.freeReg(r) 927 } 928 } 929 } else { 930 // This is the complicated case. We have more than one predecessor, 931 // which means we may have Phi ops. 932 933 // Start with the final register state of the primary predecessor 934 idx := s.primary[b.ID] 935 if idx < 0 { 936 f.Fatalf("block with no primary predecessor %s", b) 937 } 938 p := b.Preds[idx].b 939 s.setState(s.endRegs[p.ID]) 940 941 if s.f.pass.debug > regDebug { 942 fmt.Printf("starting merge block %s with end state of %s:\n", b, p) 943 for _, x := range s.endRegs[p.ID] { 944 fmt.Printf(" %s: orig:%s cache:%s\n", &s.registers[x.r], x.v, x.c) 945 } 946 } 947 948 // Decide on registers for phi ops. Use the registers determined 949 // by the primary predecessor if we can. 950 // TODO: pick best of (already processed) predecessors? 951 // Majority vote? Deepest nesting level? 952 phiRegs = phiRegs[:0] 953 var phiUsed regMask 954 955 for _, v := range phis { 956 if !s.values[v.ID].needReg { 957 phiRegs = append(phiRegs, noRegister) 958 continue 959 } 960 a := v.Args[idx] 961 // Some instructions target not-allocatable registers. 962 // They're not suitable for further (phi-function) allocation. 963 m := s.values[a.ID].regs &^ phiUsed & s.allocatable 964 if m != 0 { 965 r := pickReg(m) 966 phiUsed |= regMask(1) << r 967 phiRegs = append(phiRegs, r) 968 } else { 969 phiRegs = append(phiRegs, noRegister) 970 } 971 } 972 973 // Second pass - deallocate any phi inputs which are now dead. 974 for i, v := range phis { 975 if !s.values[v.ID].needReg { 976 continue 977 } 978 a := v.Args[idx] 979 if !regValLiveSet.contains(a.ID) { 980 // Input is dead beyond the phi, deallocate 981 // anywhere else it might live. 982 s.freeRegs(s.values[a.ID].regs) 983 } else { 984 // Input is still live. 985 // Try to move it around before kicking out, if there is a free register. 986 // We generate a Copy in the predecessor block and record it. It will be 987 // deleted if never used. 988 r := phiRegs[i] 989 if r == noRegister { 990 continue 991 } 992 // Pick a free register. At this point some registers used in the predecessor 993 // block may have been deallocated. Those are the ones used for Phis. Exclude 994 // them (and they are not going to be helpful anyway). 995 m := s.compatRegs(a.Type) &^ s.used &^ phiUsed 996 if m != 0 && !s.values[a.ID].rematerializeable && countRegs(s.values[a.ID].regs) == 1 { 997 r2 := pickReg(m) 998 c := p.NewValue1(a.Pos, OpCopy, a.Type, s.regs[r].c) 999 s.copies[c] = false 1000 if s.f.pass.debug > regDebug { 1001 fmt.Printf("copy %s to %s : %s\n", a, c, &s.registers[r2]) 1002 } 1003 s.setOrig(c, a) 1004 s.assignReg(r2, a, c) 1005 s.endRegs[p.ID] = append(s.endRegs[p.ID], endReg{r2, a, c}) 1006 } 1007 s.freeReg(r) 1008 } 1009 } 1010 1011 // Copy phi ops into new schedule. 1012 b.Values = append(b.Values, phis...) 1013 1014 // Third pass - pick registers for phis whose inputs 1015 // were not in a register. 1016 for i, v := range phis { 1017 if !s.values[v.ID].needReg { 1018 continue 1019 } 1020 if phiRegs[i] != noRegister { 1021 continue 1022 } 1023 if s.f.Config.use387 && v.Type.IsFloat() { 1024 continue // 387 can't handle floats in registers between blocks 1025 } 1026 m := s.compatRegs(v.Type) &^ phiUsed &^ s.used 1027 if m != 0 { 1028 r := pickReg(m) 1029 phiRegs[i] = r 1030 phiUsed |= regMask(1) << r 1031 } 1032 } 1033 1034 // Set registers for phis. Add phi spill code. 1035 for i, v := range phis { 1036 if !s.values[v.ID].needReg { 1037 continue 1038 } 1039 r := phiRegs[i] 1040 if r == noRegister { 1041 // stack-based phi 1042 // Spills will be inserted in all the predecessors below. 1043 s.values[v.ID].spill = v // v starts life spilled 1044 continue 1045 } 1046 // register-based phi 1047 s.assignReg(r, v, v) 1048 } 1049 1050 // Deallocate any values which are no longer live. Phis are excluded. 1051 for r := register(0); r < s.numRegs; r++ { 1052 if phiUsed>>r&1 != 0 { 1053 continue 1054 } 1055 v := s.regs[r].v 1056 if v != nil && !regValLiveSet.contains(v.ID) { 1057 s.freeReg(r) 1058 } 1059 } 1060 1061 // Save the starting state for use by merge edges. 1062 // We append to a stack allocated variable that we'll 1063 // later copy into s.startRegs in one fell swoop, to save 1064 // on allocations. 1065 regList := make([]startReg, 0, 32) 1066 for r := register(0); r < s.numRegs; r++ { 1067 v := s.regs[r].v 1068 if v == nil { 1069 continue 1070 } 1071 if phiUsed>>r&1 != 0 { 1072 // Skip registers that phis used, we'll handle those 1073 // specially during merge edge processing. 1074 continue 1075 } 1076 regList = append(regList, startReg{r, v, s.regs[r].c, s.values[v.ID].uses.pos}) 1077 } 1078 s.startRegs[b.ID] = make([]startReg, len(regList)) 1079 copy(s.startRegs[b.ID], regList) 1080 1081 if s.f.pass.debug > regDebug { 1082 fmt.Printf("after phis\n") 1083 for _, x := range s.startRegs[b.ID] { 1084 fmt.Printf(" %s: v%d\n", &s.registers[x.r], x.v.ID) 1085 } 1086 } 1087 } 1088 1089 // Allocate space to record the desired registers for each value. 1090 if l := len(oldSched); cap(dinfo) < l { 1091 dinfo = make([]dentry, l) 1092 } else { 1093 dinfo = dinfo[:l] 1094 for i := range dinfo { 1095 dinfo[i] = dentry{} 1096 } 1097 } 1098 1099 // Load static desired register info at the end of the block. 1100 desired.copy(&s.desired[b.ID]) 1101 1102 // Check actual assigned registers at the start of the next block(s). 1103 // Dynamically assigned registers will trump the static 1104 // desired registers computed during liveness analysis. 1105 // Note that we do this phase after startRegs is set above, so that 1106 // we get the right behavior for a block which branches to itself. 1107 for _, e := range b.Succs { 1108 succ := e.b 1109 // TODO: prioritize likely successor? 1110 for _, x := range s.startRegs[succ.ID] { 1111 desired.add(x.v.ID, x.r) 1112 } 1113 // Process phi ops in succ. 1114 pidx := e.i 1115 for _, v := range succ.Values { 1116 if v.Op != OpPhi { 1117 break 1118 } 1119 if !s.values[v.ID].needReg { 1120 continue 1121 } 1122 rp, ok := s.f.getHome(v.ID).(*Register) 1123 if !ok { 1124 continue 1125 } 1126 desired.add(v.Args[pidx].ID, register(rp.num)) 1127 } 1128 } 1129 // Walk values backwards computing desired register info. 1130 // See computeLive for more comments. 1131 for i := len(oldSched) - 1; i >= 0; i-- { 1132 v := oldSched[i] 1133 prefs := desired.remove(v.ID) 1134 regspec := s.regspec(v.Op) 1135 desired.clobber(regspec.clobbers) 1136 for _, j := range regspec.inputs { 1137 if countRegs(j.regs) != 1 { 1138 continue 1139 } 1140 desired.clobber(j.regs) 1141 desired.add(v.Args[j.idx].ID, pickReg(j.regs)) 1142 } 1143 if opcodeTable[v.Op].resultInArg0 { 1144 if opcodeTable[v.Op].commutative { 1145 desired.addList(v.Args[1].ID, prefs) 1146 } 1147 desired.addList(v.Args[0].ID, prefs) 1148 } 1149 // Save desired registers for this value. 1150 dinfo[i].out = prefs 1151 for j, a := range v.Args { 1152 if j >= len(dinfo[i].in) { 1153 break 1154 } 1155 dinfo[i].in[j] = desired.get(a.ID) 1156 } 1157 } 1158 1159 // Process all the non-phi values. 1160 for idx, v := range oldSched { 1161 if s.f.pass.debug > regDebug { 1162 fmt.Printf(" processing %s\n", v.LongString()) 1163 } 1164 regspec := s.regspec(v.Op) 1165 if v.Op == OpPhi { 1166 f.Fatalf("phi %s not at start of block", v) 1167 } 1168 if v.Op == OpSP { 1169 s.assignReg(s.SPReg, v, v) 1170 b.Values = append(b.Values, v) 1171 s.advanceUses(v) 1172 s.sp = v.ID 1173 continue 1174 } 1175 if v.Op == OpSB { 1176 s.assignReg(s.SBReg, v, v) 1177 b.Values = append(b.Values, v) 1178 s.advanceUses(v) 1179 s.sb = v.ID 1180 continue 1181 } 1182 if v.Op == OpSelect0 || v.Op == OpSelect1 { 1183 if s.values[v.ID].needReg { 1184 var i = 0 1185 if v.Op == OpSelect1 { 1186 i = 1 1187 } 1188 s.assignReg(register(s.f.getHome(v.Args[0].ID).(LocPair)[i].(*Register).num), v, v) 1189 } 1190 b.Values = append(b.Values, v) 1191 s.advanceUses(v) 1192 goto issueSpill 1193 } 1194 if v.Op == OpGetG && s.f.Config.hasGReg { 1195 // use hardware g register 1196 if s.regs[s.GReg].v != nil { 1197 s.freeReg(s.GReg) // kick out the old value 1198 } 1199 s.assignReg(s.GReg, v, v) 1200 b.Values = append(b.Values, v) 1201 s.advanceUses(v) 1202 goto issueSpill 1203 } 1204 if v.Op == OpArg { 1205 // Args are "pre-spilled" values. We don't allocate 1206 // any register here. We just set up the spill pointer to 1207 // point at itself and any later user will restore it to use it. 1208 s.values[v.ID].spill = v 1209 b.Values = append(b.Values, v) 1210 s.advanceUses(v) 1211 continue 1212 } 1213 if v.Op == OpKeepAlive { 1214 // Make sure the argument to v is still live here. 1215 s.advanceUses(v) 1216 a := v.Args[0] 1217 vi := &s.values[a.ID] 1218 if vi.regs == 0 && !vi.rematerializeable { 1219 // Use the spill location. 1220 // This forces later liveness analysis to make the 1221 // value live at this point. 1222 v.SetArg(0, s.makeSpill(a, b)) 1223 } else { 1224 // In-register and rematerializeable values are already live. 1225 // These are typically rematerializeable constants like nil, 1226 // or values of a variable that were modified since the last call. 1227 v.Op = OpCopy 1228 v.SetArgs1(v.Args[1]) 1229 } 1230 b.Values = append(b.Values, v) 1231 continue 1232 } 1233 if len(regspec.inputs) == 0 && len(regspec.outputs) == 0 { 1234 // No register allocation required (or none specified yet) 1235 s.freeRegs(regspec.clobbers) 1236 b.Values = append(b.Values, v) 1237 s.advanceUses(v) 1238 continue 1239 } 1240 1241 if s.values[v.ID].rematerializeable { 1242 // Value is rematerializeable, don't issue it here. 1243 // It will get issued just before each use (see 1244 // allocValueToReg). 1245 for _, a := range v.Args { 1246 a.Uses-- 1247 } 1248 s.advanceUses(v) 1249 continue 1250 } 1251 1252 if s.f.pass.debug > regDebug { 1253 fmt.Printf("value %s\n", v.LongString()) 1254 fmt.Printf(" out:") 1255 for _, r := range dinfo[idx].out { 1256 if r != noRegister { 1257 fmt.Printf(" %s", &s.registers[r]) 1258 } 1259 } 1260 fmt.Println() 1261 for i := 0; i < len(v.Args) && i < 3; i++ { 1262 fmt.Printf(" in%d:", i) 1263 for _, r := range dinfo[idx].in[i] { 1264 if r != noRegister { 1265 fmt.Printf(" %s", &s.registers[r]) 1266 } 1267 } 1268 fmt.Println() 1269 } 1270 } 1271 1272 // Move arguments to registers. Process in an ordering defined 1273 // by the register specification (most constrained first). 1274 args = append(args[:0], v.Args...) 1275 for _, i := range regspec.inputs { 1276 mask := i.regs 1277 if mask&s.values[args[i.idx].ID].regs == 0 { 1278 // Need a new register for the input. 1279 mask &= s.allocatable 1280 mask &^= s.nospill 1281 // Used desired register if available. 1282 if i.idx < 3 { 1283 for _, r := range dinfo[idx].in[i.idx] { 1284 if r != noRegister && (mask&^s.used)>>r&1 != 0 { 1285 // Desired register is allowed and unused. 1286 mask = regMask(1) << r 1287 break 1288 } 1289 } 1290 } 1291 // Avoid registers we're saving for other values. 1292 if mask&^desired.avoid != 0 { 1293 mask &^= desired.avoid 1294 } 1295 } 1296 args[i.idx] = s.allocValToReg(args[i.idx], mask, true, v.Pos) 1297 } 1298 1299 // If the output clobbers the input register, make sure we have 1300 // at least two copies of the input register so we don't 1301 // have to reload the value from the spill location. 1302 if opcodeTable[v.Op].resultInArg0 { 1303 var m regMask 1304 if !s.liveAfterCurrentInstruction(v.Args[0]) { 1305 // arg0 is dead. We can clobber its register. 1306 goto ok 1307 } 1308 if s.values[v.Args[0].ID].rematerializeable { 1309 // We can rematerialize the input, don't worry about clobbering it. 1310 goto ok 1311 } 1312 if countRegs(s.values[v.Args[0].ID].regs) >= 2 { 1313 // we have at least 2 copies of arg0. We can afford to clobber one. 1314 goto ok 1315 } 1316 if opcodeTable[v.Op].commutative { 1317 if !s.liveAfterCurrentInstruction(v.Args[1]) { 1318 args[0], args[1] = args[1], args[0] 1319 goto ok 1320 } 1321 if s.values[v.Args[1].ID].rematerializeable { 1322 args[0], args[1] = args[1], args[0] 1323 goto ok 1324 } 1325 if countRegs(s.values[v.Args[1].ID].regs) >= 2 { 1326 args[0], args[1] = args[1], args[0] 1327 goto ok 1328 } 1329 } 1330 1331 // We can't overwrite arg0 (or arg1, if commutative). So we 1332 // need to make a copy of an input so we have a register we can modify. 1333 1334 // Possible new registers to copy into. 1335 m = s.compatRegs(v.Args[0].Type) &^ s.used 1336 if m == 0 { 1337 // No free registers. In this case we'll just clobber 1338 // an input and future uses of that input must use a restore. 1339 // TODO(khr): We should really do this like allocReg does it, 1340 // spilling the value with the most distant next use. 1341 goto ok 1342 } 1343 1344 // Try to move an input to the desired output. 1345 for _, r := range dinfo[idx].out { 1346 if r != noRegister && m>>r&1 != 0 { 1347 m = regMask(1) << r 1348 args[0] = s.allocValToReg(v.Args[0], m, true, v.Pos) 1349 // Note: we update args[0] so the instruction will 1350 // use the register copy we just made. 1351 goto ok 1352 } 1353 } 1354 // Try to copy input to its desired location & use its old 1355 // location as the result register. 1356 for _, r := range dinfo[idx].in[0] { 1357 if r != noRegister && m>>r&1 != 0 { 1358 m = regMask(1) << r 1359 c := s.allocValToReg(v.Args[0], m, true, v.Pos) 1360 s.copies[c] = false 1361 // Note: no update to args[0] so the instruction will 1362 // use the original copy. 1363 goto ok 1364 } 1365 } 1366 if opcodeTable[v.Op].commutative { 1367 for _, r := range dinfo[idx].in[1] { 1368 if r != noRegister && m>>r&1 != 0 { 1369 m = regMask(1) << r 1370 c := s.allocValToReg(v.Args[1], m, true, v.Pos) 1371 s.copies[c] = false 1372 args[0], args[1] = args[1], args[0] 1373 goto ok 1374 } 1375 } 1376 } 1377 // Avoid future fixed uses if we can. 1378 if m&^desired.avoid != 0 { 1379 m &^= desired.avoid 1380 } 1381 // Save input 0 to a new register so we can clobber it. 1382 c := s.allocValToReg(v.Args[0], m, true, v.Pos) 1383 s.copies[c] = false 1384 } 1385 1386 ok: 1387 // Now that all args are in regs, we're ready to issue the value itself. 1388 // Before we pick a register for the output value, allow input registers 1389 // to be deallocated. We do this here so that the output can use the 1390 // same register as a dying input. 1391 if !opcodeTable[v.Op].resultNotInArgs { 1392 s.tmpused = s.nospill 1393 s.nospill = 0 1394 s.advanceUses(v) // frees any registers holding args that are no longer live 1395 } 1396 1397 // Dump any registers which will be clobbered 1398 s.freeRegs(regspec.clobbers) 1399 s.tmpused |= regspec.clobbers 1400 1401 // Pick registers for outputs. 1402 { 1403 outRegs := [2]register{noRegister, noRegister} 1404 var used regMask 1405 for _, out := range regspec.outputs { 1406 mask := out.regs & s.allocatable &^ used 1407 if mask == 0 { 1408 continue 1409 } 1410 if opcodeTable[v.Op].resultInArg0 && out.idx == 0 { 1411 if !opcodeTable[v.Op].commutative { 1412 // Output must use the same register as input 0. 1413 r := register(s.f.getHome(args[0].ID).(*Register).num) 1414 mask = regMask(1) << r 1415 } else { 1416 // Output must use the same register as input 0 or 1. 1417 r0 := register(s.f.getHome(args[0].ID).(*Register).num) 1418 r1 := register(s.f.getHome(args[1].ID).(*Register).num) 1419 // Check r0 and r1 for desired output register. 1420 found := false 1421 for _, r := range dinfo[idx].out { 1422 if (r == r0 || r == r1) && (mask&^s.used)>>r&1 != 0 { 1423 mask = regMask(1) << r 1424 found = true 1425 if r == r1 { 1426 args[0], args[1] = args[1], args[0] 1427 } 1428 break 1429 } 1430 } 1431 if !found { 1432 // Neither are desired, pick r0. 1433 mask = regMask(1) << r0 1434 } 1435 } 1436 } 1437 for _, r := range dinfo[idx].out { 1438 if r != noRegister && (mask&^s.used)>>r&1 != 0 { 1439 // Desired register is allowed and unused. 1440 mask = regMask(1) << r 1441 break 1442 } 1443 } 1444 // Avoid registers we're saving for other values. 1445 if mask&^desired.avoid != 0 { 1446 mask &^= desired.avoid 1447 } 1448 r := s.allocReg(mask, v) 1449 outRegs[out.idx] = r 1450 used |= regMask(1) << r 1451 s.tmpused |= regMask(1) << r 1452 } 1453 // Record register choices 1454 if v.Type.IsTuple() { 1455 var outLocs LocPair 1456 if r := outRegs[0]; r != noRegister { 1457 outLocs[0] = &s.registers[r] 1458 } 1459 if r := outRegs[1]; r != noRegister { 1460 outLocs[1] = &s.registers[r] 1461 } 1462 s.f.setHome(v, outLocs) 1463 // Note that subsequent SelectX instructions will do the assignReg calls. 1464 } else { 1465 if r := outRegs[0]; r != noRegister { 1466 s.assignReg(r, v, v) 1467 } 1468 } 1469 } 1470 1471 // deallocate dead args, if we have not done so 1472 if opcodeTable[v.Op].resultNotInArgs { 1473 s.nospill = 0 1474 s.advanceUses(v) // frees any registers holding args that are no longer live 1475 } 1476 s.tmpused = 0 1477 1478 // Issue the Value itself. 1479 for i, a := range args { 1480 v.SetArg(i, a) // use register version of arguments 1481 } 1482 b.Values = append(b.Values, v) 1483 1484 issueSpill: 1485 } 1486 1487 // Load control value into reg. 1488 if v := b.Control; v != nil && s.values[v.ID].needReg { 1489 if s.f.pass.debug > regDebug { 1490 fmt.Printf(" processing control %s\n", v.LongString()) 1491 } 1492 // We assume that a control input can be passed in any 1493 // type-compatible register. If this turns out not to be true, 1494 // we'll need to introduce a regspec for a block's control value. 1495 b.Control = s.allocValToReg(v, s.compatRegs(v.Type), false, b.Pos) 1496 if b.Control != v { 1497 v.Uses-- 1498 b.Control.Uses++ 1499 } 1500 // Remove this use from the uses list. 1501 vi := &s.values[v.ID] 1502 u := vi.uses 1503 vi.uses = u.next 1504 if u.next == nil { 1505 s.freeRegs(vi.regs) // value is dead 1506 } 1507 u.next = s.freeUseRecords 1508 s.freeUseRecords = u 1509 } 1510 1511 // Spill any values that can't live across basic block boundaries. 1512 if s.f.Config.use387 { 1513 s.freeRegs(s.f.Config.fpRegMask) 1514 } 1515 1516 // If we are approaching a merge point and we are the primary 1517 // predecessor of it, find live values that we use soon after 1518 // the merge point and promote them to registers now. 1519 if len(b.Succs) == 1 { 1520 if s.f.Config.hasGReg && s.regs[s.GReg].v != nil { 1521 s.freeReg(s.GReg) // Spill value in G register before any merge. 1522 } 1523 // For this to be worthwhile, the loop must have no calls in it. 1524 top := b.Succs[0].b 1525 loop := s.loopnest.b2l[top.ID] 1526 if loop == nil || loop.header != top || loop.containsUnavoidableCall { 1527 goto badloop 1528 } 1529 1530 // TODO: sort by distance, pick the closest ones? 1531 for _, live := range s.live[b.ID] { 1532 if live.dist >= unlikelyDistance { 1533 // Don't preload anything live after the loop. 1534 continue 1535 } 1536 vid := live.ID 1537 vi := &s.values[vid] 1538 if vi.regs != 0 { 1539 continue 1540 } 1541 if vi.rematerializeable { 1542 continue 1543 } 1544 v := s.orig[vid] 1545 if s.f.Config.use387 && v.Type.IsFloat() { 1546 continue // 387 can't handle floats in registers between blocks 1547 } 1548 m := s.compatRegs(v.Type) &^ s.used 1549 if m&^desired.avoid != 0 { 1550 m &^= desired.avoid 1551 } 1552 if m != 0 { 1553 s.allocValToReg(v, m, false, b.Pos) 1554 } 1555 } 1556 } 1557 badloop: 1558 ; 1559 1560 // Save end-of-block register state. 1561 // First count how many, this cuts allocations in half. 1562 k := 0 1563 for r := register(0); r < s.numRegs; r++ { 1564 v := s.regs[r].v 1565 if v == nil { 1566 continue 1567 } 1568 k++ 1569 } 1570 regList := make([]endReg, 0, k) 1571 for r := register(0); r < s.numRegs; r++ { 1572 v := s.regs[r].v 1573 if v == nil { 1574 continue 1575 } 1576 regList = append(regList, endReg{r, v, s.regs[r].c}) 1577 } 1578 s.endRegs[b.ID] = regList 1579 1580 if checkEnabled { 1581 regValLiveSet.clear() 1582 for _, x := range s.live[b.ID] { 1583 regValLiveSet.add(x.ID) 1584 } 1585 for r := register(0); r < s.numRegs; r++ { 1586 v := s.regs[r].v 1587 if v == nil { 1588 continue 1589 } 1590 if !regValLiveSet.contains(v.ID) { 1591 s.f.Fatalf("val %s is in reg but not live at end of %s", v, b) 1592 } 1593 } 1594 } 1595 1596 // If a value is live at the end of the block and 1597 // isn't in a register, generate a use for the spill location. 1598 // We need to remember this information so that 1599 // the liveness analysis in stackalloc is correct. 1600 for _, e := range s.live[b.ID] { 1601 vi := &s.values[e.ID] 1602 if vi.regs != 0 { 1603 // in a register, we'll use that source for the merge. 1604 continue 1605 } 1606 if vi.rematerializeable { 1607 // we'll rematerialize during the merge. 1608 continue 1609 } 1610 //fmt.Printf("live-at-end spill for %s at %s\n", s.orig[e.ID], b) 1611 spill := s.makeSpill(s.orig[e.ID], b) 1612 s.spillLive[b.ID] = append(s.spillLive[b.ID], spill.ID) 1613 } 1614 1615 // Clear any final uses. 1616 // All that is left should be the pseudo-uses added for values which 1617 // are live at the end of b. 1618 for _, e := range s.live[b.ID] { 1619 u := s.values[e.ID].uses 1620 if u == nil { 1621 f.Fatalf("live at end, no uses v%d", e.ID) 1622 } 1623 if u.next != nil { 1624 f.Fatalf("live at end, too many uses v%d", e.ID) 1625 } 1626 s.values[e.ID].uses = nil 1627 u.next = s.freeUseRecords 1628 s.freeUseRecords = u 1629 } 1630 } 1631 1632 // Decide where the spills we generated will go. 1633 s.placeSpills() 1634 1635 // Anything that didn't get a register gets a stack location here. 1636 // (StoreReg, stack-based phis, inputs, ...) 1637 stacklive := stackalloc(s.f, s.spillLive) 1638 1639 // Fix up all merge edges. 1640 s.shuffle(stacklive) 1641 1642 // Erase any copies we never used. 1643 // Also, an unused copy might be the only use of another copy, 1644 // so continue erasing until we reach a fixed point. 1645 for { 1646 progress := false 1647 for c, used := range s.copies { 1648 if !used && c.Uses == 0 { 1649 if s.f.pass.debug > regDebug { 1650 fmt.Printf("delete copied value %s\n", c.LongString()) 1651 } 1652 c.RemoveArg(0) 1653 f.freeValue(c) 1654 delete(s.copies, c) 1655 progress = true 1656 } 1657 } 1658 if !progress { 1659 break 1660 } 1661 } 1662 1663 for _, b := range s.visitOrder { 1664 i := 0 1665 for _, v := range b.Values { 1666 if v.Op == OpInvalid { 1667 continue 1668 } 1669 b.Values[i] = v 1670 i++ 1671 } 1672 b.Values = b.Values[:i] 1673 } 1674 } 1675 1676 func (s *regAllocState) placeSpills() { 1677 f := s.f 1678 1679 // Precompute some useful info. 1680 phiRegs := make([]regMask, f.NumBlocks()) 1681 for _, b := range s.visitOrder { 1682 var m regMask 1683 for _, v := range b.Values { 1684 if v.Op != OpPhi { 1685 break 1686 } 1687 if r, ok := f.getHome(v.ID).(*Register); ok { 1688 m |= regMask(1) << uint(r.num) 1689 } 1690 } 1691 phiRegs[b.ID] = m 1692 } 1693 1694 // Start maps block IDs to the list of spills 1695 // that go at the start of the block (but after any phis). 1696 start := map[ID][]*Value{} 1697 // After maps value IDs to the list of spills 1698 // that go immediately after that value ID. 1699 after := map[ID][]*Value{} 1700 1701 for i := range s.values { 1702 vi := s.values[i] 1703 spill := vi.spill 1704 if spill == nil { 1705 continue 1706 } 1707 if spill.Block != nil { 1708 // Some spills are already fully set up, 1709 // like OpArgs and stack-based phis. 1710 continue 1711 } 1712 v := s.orig[i] 1713 1714 // Walk down the dominator tree looking for a good place to 1715 // put the spill of v. At the start "best" is the best place 1716 // we have found so far. 1717 // TODO: find a way to make this O(1) without arbitrary cutoffs. 1718 best := v.Block 1719 bestArg := v 1720 var bestDepth int16 1721 if l := s.loopnest.b2l[best.ID]; l != nil { 1722 bestDepth = l.depth 1723 } 1724 b := best 1725 const maxSpillSearch = 100 1726 for i := 0; i < maxSpillSearch; i++ { 1727 // Find the child of b in the dominator tree which 1728 // dominates all restores. 1729 p := b 1730 b = nil 1731 for c := s.sdom.Child(p); c != nil && i < maxSpillSearch; c, i = s.sdom.Sibling(c), i+1 { 1732 if s.sdom[c.ID].entry <= vi.restoreMin && s.sdom[c.ID].exit >= vi.restoreMax { 1733 // c also dominates all restores. Walk down into c. 1734 b = c 1735 break 1736 } 1737 } 1738 if b == nil { 1739 // Ran out of blocks which dominate all restores. 1740 break 1741 } 1742 1743 var depth int16 1744 if l := s.loopnest.b2l[b.ID]; l != nil { 1745 depth = l.depth 1746 } 1747 if depth > bestDepth { 1748 // Don't push the spill into a deeper loop. 1749 continue 1750 } 1751 1752 // If v is in a register at the start of b, we can 1753 // place the spill here (after the phis). 1754 if len(b.Preds) == 1 { 1755 for _, e := range s.endRegs[b.Preds[0].b.ID] { 1756 if e.v == v { 1757 // Found a better spot for the spill. 1758 best = b 1759 bestArg = e.c 1760 bestDepth = depth 1761 break 1762 } 1763 } 1764 } else { 1765 for _, e := range s.startRegs[b.ID] { 1766 if e.v == v { 1767 // Found a better spot for the spill. 1768 best = b 1769 bestArg = e.c 1770 bestDepth = depth 1771 break 1772 } 1773 } 1774 } 1775 } 1776 1777 // Put the spill in the best block we found. 1778 spill.Block = best 1779 spill.AddArg(bestArg) 1780 if best == v.Block && v.Op != OpPhi { 1781 // Place immediately after v. 1782 after[v.ID] = append(after[v.ID], spill) 1783 } else { 1784 // Place at the start of best block. 1785 start[best.ID] = append(start[best.ID], spill) 1786 } 1787 } 1788 1789 // Insert spill instructions into the block schedules. 1790 var oldSched []*Value 1791 for _, b := range s.visitOrder { 1792 nphi := 0 1793 for _, v := range b.Values { 1794 if v.Op != OpPhi { 1795 break 1796 } 1797 nphi++ 1798 } 1799 oldSched = append(oldSched[:0], b.Values[nphi:]...) 1800 b.Values = b.Values[:nphi] 1801 b.Values = append(b.Values, start[b.ID]...) 1802 for _, v := range oldSched { 1803 b.Values = append(b.Values, v) 1804 b.Values = append(b.Values, after[v.ID]...) 1805 } 1806 } 1807 } 1808 1809 // shuffle fixes up all the merge edges (those going into blocks of indegree > 1). 1810 func (s *regAllocState) shuffle(stacklive [][]ID) { 1811 var e edgeState 1812 e.s = s 1813 e.cache = map[ID][]*Value{} 1814 e.contents = map[Location]contentRecord{} 1815 if s.f.pass.debug > regDebug { 1816 fmt.Printf("shuffle %s\n", s.f.Name) 1817 fmt.Println(s.f.String()) 1818 } 1819 1820 for _, b := range s.visitOrder { 1821 if len(b.Preds) <= 1 { 1822 continue 1823 } 1824 e.b = b 1825 for i, edge := range b.Preds { 1826 p := edge.b 1827 e.p = p 1828 e.setup(i, s.endRegs[p.ID], s.startRegs[b.ID], stacklive[p.ID]) 1829 e.process() 1830 } 1831 } 1832 } 1833 1834 type edgeState struct { 1835 s *regAllocState 1836 p, b *Block // edge goes from p->b. 1837 1838 // for each pre-regalloc value, a list of equivalent cached values 1839 cache map[ID][]*Value 1840 cachedVals []ID // (superset of) keys of the above map, for deterministic iteration 1841 1842 // map from location to the value it contains 1843 contents map[Location]contentRecord 1844 1845 // desired destination locations 1846 destinations []dstRecord 1847 extra []dstRecord 1848 1849 usedRegs regMask // registers currently holding something 1850 uniqueRegs regMask // registers holding the only copy of a value 1851 finalRegs regMask // registers holding final target 1852 rematerializeableRegs regMask // registers that hold rematerializeable values 1853 } 1854 1855 type contentRecord struct { 1856 vid ID // pre-regalloc value 1857 c *Value // cached value 1858 final bool // this is a satisfied destination 1859 pos src.XPos // source position of use of the value 1860 } 1861 1862 type dstRecord struct { 1863 loc Location // register or stack slot 1864 vid ID // pre-regalloc value it should contain 1865 splice **Value // place to store reference to the generating instruction 1866 pos src.XPos // source position of use of this location 1867 } 1868 1869 // setup initializes the edge state for shuffling. 1870 func (e *edgeState) setup(idx int, srcReg []endReg, dstReg []startReg, stacklive []ID) { 1871 if e.s.f.pass.debug > regDebug { 1872 fmt.Printf("edge %s->%s\n", e.p, e.b) 1873 } 1874 1875 // Clear state. 1876 for _, vid := range e.cachedVals { 1877 delete(e.cache, vid) 1878 } 1879 e.cachedVals = e.cachedVals[:0] 1880 for k := range e.contents { 1881 delete(e.contents, k) 1882 } 1883 e.usedRegs = 0 1884 e.uniqueRegs = 0 1885 e.finalRegs = 0 1886 e.rematerializeableRegs = 0 1887 1888 // Live registers can be sources. 1889 for _, x := range srcReg { 1890 e.set(&e.s.registers[x.r], x.v.ID, x.c, false, src.NoXPos) // don't care the position of the source 1891 } 1892 // So can all of the spill locations. 1893 for _, spillID := range stacklive { 1894 v := e.s.orig[spillID] 1895 spill := e.s.values[v.ID].spill 1896 if !e.s.sdom.isAncestorEq(spill.Block, e.p) { 1897 // Spills were placed that only dominate the uses found 1898 // during the first regalloc pass. The edge fixup code 1899 // can't use a spill location if the spill doesn't dominate 1900 // the edge. 1901 // We are guaranteed that if the spill doesn't dominate this edge, 1902 // then the value is available in a register (because we called 1903 // makeSpill for every value not in a register at the start 1904 // of an edge). 1905 continue 1906 } 1907 e.set(e.s.f.getHome(spillID), v.ID, spill, false, src.NoXPos) // don't care the position of the source 1908 } 1909 1910 // Figure out all the destinations we need. 1911 dsts := e.destinations[:0] 1912 for _, x := range dstReg { 1913 dsts = append(dsts, dstRecord{&e.s.registers[x.r], x.v.ID, nil, x.pos}) 1914 } 1915 // Phis need their args to end up in a specific location. 1916 for _, v := range e.b.Values { 1917 if v.Op != OpPhi { 1918 break 1919 } 1920 loc := e.s.f.getHome(v.ID) 1921 if loc == nil { 1922 continue 1923 } 1924 dsts = append(dsts, dstRecord{loc, v.Args[idx].ID, &v.Args[idx], v.Pos}) 1925 } 1926 e.destinations = dsts 1927 1928 if e.s.f.pass.debug > regDebug { 1929 for _, vid := range e.cachedVals { 1930 a := e.cache[vid] 1931 for _, c := range a { 1932 fmt.Printf("src %s: v%d cache=%s\n", e.s.f.getHome(c.ID), vid, c) 1933 } 1934 } 1935 for _, d := range e.destinations { 1936 fmt.Printf("dst %s: v%d\n", d.loc, d.vid) 1937 } 1938 } 1939 } 1940 1941 // process generates code to move all the values to the right destination locations. 1942 func (e *edgeState) process() { 1943 dsts := e.destinations 1944 1945 // Process the destinations until they are all satisfied. 1946 for len(dsts) > 0 { 1947 i := 0 1948 for _, d := range dsts { 1949 if !e.processDest(d.loc, d.vid, d.splice, d.pos) { 1950 // Failed - save for next iteration. 1951 dsts[i] = d 1952 i++ 1953 } 1954 } 1955 if i < len(dsts) { 1956 // Made some progress. Go around again. 1957 dsts = dsts[:i] 1958 1959 // Append any extras destinations we generated. 1960 dsts = append(dsts, e.extra...) 1961 e.extra = e.extra[:0] 1962 continue 1963 } 1964 1965 // We made no progress. That means that any 1966 // remaining unsatisfied moves are in simple cycles. 1967 // For example, A -> B -> C -> D -> A. 1968 // A ----> B 1969 // ^ | 1970 // | | 1971 // | v 1972 // D <---- C 1973 1974 // To break the cycle, we pick an unused register, say R, 1975 // and put a copy of B there. 1976 // A ----> B 1977 // ^ | 1978 // | | 1979 // | v 1980 // D <---- C <---- R=copyofB 1981 // When we resume the outer loop, the A->B move can now proceed, 1982 // and eventually the whole cycle completes. 1983 1984 // Copy any cycle location to a temp register. This duplicates 1985 // one of the cycle entries, allowing the just duplicated value 1986 // to be overwritten and the cycle to proceed. 1987 d := dsts[0] 1988 loc := d.loc 1989 vid := e.contents[loc].vid 1990 c := e.contents[loc].c 1991 r := e.findRegFor(c.Type) 1992 if e.s.f.pass.debug > regDebug { 1993 fmt.Printf("breaking cycle with v%d in %s:%s\n", vid, loc, c) 1994 } 1995 e.erase(r) 1996 pos := d.pos.WithNotStmt() 1997 if _, isReg := loc.(*Register); isReg { 1998 c = e.p.NewValue1(pos, OpCopy, c.Type, c) 1999 } else { 2000 c = e.p.NewValue1(pos, OpLoadReg, c.Type, c) 2001 } 2002 e.set(r, vid, c, false, pos) 2003 if c.Op == OpLoadReg && e.s.isGReg(register(r.(*Register).num)) { 2004 e.s.f.Fatalf("process.OpLoadReg targeting g: " + c.LongString()) 2005 } 2006 } 2007 } 2008 2009 // processDest generates code to put value vid into location loc. Returns true 2010 // if progress was made. 2011 func (e *edgeState) processDest(loc Location, vid ID, splice **Value, pos src.XPos) bool { 2012 pos = pos.WithNotStmt() 2013 occupant := e.contents[loc] 2014 if occupant.vid == vid { 2015 // Value is already in the correct place. 2016 e.contents[loc] = contentRecord{vid, occupant.c, true, pos} 2017 if splice != nil { 2018 (*splice).Uses-- 2019 *splice = occupant.c 2020 occupant.c.Uses++ 2021 } 2022 // Note: if splice==nil then c will appear dead. This is 2023 // non-SSA formed code, so be careful after this pass not to run 2024 // deadcode elimination. 2025 if _, ok := e.s.copies[occupant.c]; ok { 2026 // The copy at occupant.c was used to avoid spill. 2027 e.s.copies[occupant.c] = true 2028 } 2029 return true 2030 } 2031 2032 // Check if we're allowed to clobber the destination location. 2033 if len(e.cache[occupant.vid]) == 1 && !e.s.values[occupant.vid].rematerializeable { 2034 // We can't overwrite the last copy 2035 // of a value that needs to survive. 2036 return false 2037 } 2038 2039 // Copy from a source of v, register preferred. 2040 v := e.s.orig[vid] 2041 var c *Value 2042 var src Location 2043 if e.s.f.pass.debug > regDebug { 2044 fmt.Printf("moving v%d to %s\n", vid, loc) 2045 fmt.Printf("sources of v%d:", vid) 2046 } 2047 for _, w := range e.cache[vid] { 2048 h := e.s.f.getHome(w.ID) 2049 if e.s.f.pass.debug > regDebug { 2050 fmt.Printf(" %s:%s", h, w) 2051 } 2052 _, isreg := h.(*Register) 2053 if src == nil || isreg { 2054 c = w 2055 src = h 2056 } 2057 } 2058 if e.s.f.pass.debug > regDebug { 2059 if src != nil { 2060 fmt.Printf(" [use %s]\n", src) 2061 } else { 2062 fmt.Printf(" [no source]\n") 2063 } 2064 } 2065 _, dstReg := loc.(*Register) 2066 2067 // Pre-clobber destination. This avoids the 2068 // following situation: 2069 // - v is currently held in R0 and stacktmp0. 2070 // - We want to copy stacktmp1 to stacktmp0. 2071 // - We choose R0 as the temporary register. 2072 // During the copy, both R0 and stacktmp0 are 2073 // clobbered, losing both copies of v. Oops! 2074 // Erasing the destination early means R0 will not 2075 // be chosen as the temp register, as it will then 2076 // be the last copy of v. 2077 e.erase(loc) 2078 var x *Value 2079 if c == nil || e.s.values[vid].rematerializeable { 2080 if !e.s.values[vid].rematerializeable { 2081 e.s.f.Fatalf("can't find source for %s->%s: %s\n", e.p, e.b, v.LongString()) 2082 } 2083 if dstReg { 2084 x = v.copyInto(e.p) 2085 } else { 2086 // Rematerialize into stack slot. Need a free 2087 // register to accomplish this. 2088 r := e.findRegFor(v.Type) 2089 e.erase(r) 2090 x = v.copyIntoWithXPos(e.p, pos) 2091 e.set(r, vid, x, false, pos) 2092 // Make sure we spill with the size of the slot, not the 2093 // size of x (which might be wider due to our dropping 2094 // of narrowing conversions). 2095 x = e.p.NewValue1(pos, OpStoreReg, loc.(LocalSlot).Type, x) 2096 } 2097 } else { 2098 // Emit move from src to dst. 2099 _, srcReg := src.(*Register) 2100 if srcReg { 2101 if dstReg { 2102 x = e.p.NewValue1(pos, OpCopy, c.Type, c) 2103 } else { 2104 x = e.p.NewValue1(pos, OpStoreReg, loc.(LocalSlot).Type, c) 2105 } 2106 } else { 2107 if dstReg { 2108 x = e.p.NewValue1(pos, OpLoadReg, c.Type, c) 2109 } else { 2110 // mem->mem. Use temp register. 2111 r := e.findRegFor(c.Type) 2112 e.erase(r) 2113 t := e.p.NewValue1(pos, OpLoadReg, c.Type, c) 2114 e.set(r, vid, t, false, pos) 2115 x = e.p.NewValue1(pos, OpStoreReg, loc.(LocalSlot).Type, t) 2116 } 2117 } 2118 } 2119 e.set(loc, vid, x, true, pos) 2120 if x.Op == OpLoadReg && e.s.isGReg(register(loc.(*Register).num)) { 2121 e.s.f.Fatalf("processDest.OpLoadReg targeting g: " + x.LongString()) 2122 } 2123 if splice != nil { 2124 (*splice).Uses-- 2125 *splice = x 2126 x.Uses++ 2127 } 2128 return true 2129 } 2130 2131 // set changes the contents of location loc to hold the given value and its cached representative. 2132 func (e *edgeState) set(loc Location, vid ID, c *Value, final bool, pos src.XPos) { 2133 e.s.f.setHome(c, loc) 2134 e.contents[loc] = contentRecord{vid, c, final, pos} 2135 a := e.cache[vid] 2136 if len(a) == 0 { 2137 e.cachedVals = append(e.cachedVals, vid) 2138 } 2139 a = append(a, c) 2140 e.cache[vid] = a 2141 if r, ok := loc.(*Register); ok { 2142 e.usedRegs |= regMask(1) << uint(r.num) 2143 if final { 2144 e.finalRegs |= regMask(1) << uint(r.num) 2145 } 2146 if len(a) == 1 { 2147 e.uniqueRegs |= regMask(1) << uint(r.num) 2148 } 2149 if len(a) == 2 { 2150 if t, ok := e.s.f.getHome(a[0].ID).(*Register); ok { 2151 e.uniqueRegs &^= regMask(1) << uint(t.num) 2152 } 2153 } 2154 if e.s.values[vid].rematerializeable { 2155 e.rematerializeableRegs |= regMask(1) << uint(r.num) 2156 } 2157 } 2158 if e.s.f.pass.debug > regDebug { 2159 fmt.Printf("%s\n", c.LongString()) 2160 fmt.Printf("v%d now available in %s:%s\n", vid, loc, c) 2161 } 2162 } 2163 2164 // erase removes any user of loc. 2165 func (e *edgeState) erase(loc Location) { 2166 cr := e.contents[loc] 2167 if cr.c == nil { 2168 return 2169 } 2170 vid := cr.vid 2171 2172 if cr.final { 2173 // Add a destination to move this value back into place. 2174 // Make sure it gets added to the tail of the destination queue 2175 // so we make progress on other moves first. 2176 e.extra = append(e.extra, dstRecord{loc, cr.vid, nil, cr.pos}) 2177 } 2178 2179 // Remove c from the list of cached values. 2180 a := e.cache[vid] 2181 for i, c := range a { 2182 if e.s.f.getHome(c.ID) == loc { 2183 if e.s.f.pass.debug > regDebug { 2184 fmt.Printf("v%d no longer available in %s:%s\n", vid, loc, c) 2185 } 2186 a[i], a = a[len(a)-1], a[:len(a)-1] 2187 break 2188 } 2189 } 2190 e.cache[vid] = a 2191 2192 // Update register masks. 2193 if r, ok := loc.(*Register); ok { 2194 e.usedRegs &^= regMask(1) << uint(r.num) 2195 if cr.final { 2196 e.finalRegs &^= regMask(1) << uint(r.num) 2197 } 2198 e.rematerializeableRegs &^= regMask(1) << uint(r.num) 2199 } 2200 if len(a) == 1 { 2201 if r, ok := e.s.f.getHome(a[0].ID).(*Register); ok { 2202 e.uniqueRegs |= regMask(1) << uint(r.num) 2203 } 2204 } 2205 } 2206 2207 // findRegFor finds a register we can use to make a temp copy of type typ. 2208 func (e *edgeState) findRegFor(typ *types.Type) Location { 2209 // Which registers are possibilities. 2210 var m regMask 2211 types := &e.s.f.Config.Types 2212 if typ.IsFloat() { 2213 m = e.s.compatRegs(types.Float64) 2214 } else { 2215 m = e.s.compatRegs(types.Int64) 2216 } 2217 2218 // Pick a register. In priority order: 2219 // 1) an unused register 2220 // 2) a non-unique register not holding a final value 2221 // 3) a non-unique register 2222 // 4) a register holding a rematerializeable value 2223 x := m &^ e.usedRegs 2224 if x != 0 { 2225 return &e.s.registers[pickReg(x)] 2226 } 2227 x = m &^ e.uniqueRegs &^ e.finalRegs 2228 if x != 0 { 2229 return &e.s.registers[pickReg(x)] 2230 } 2231 x = m &^ e.uniqueRegs 2232 if x != 0 { 2233 return &e.s.registers[pickReg(x)] 2234 } 2235 x = m & e.rematerializeableRegs 2236 if x != 0 { 2237 return &e.s.registers[pickReg(x)] 2238 } 2239 2240 // No register is available. 2241 // Pick a register to spill. 2242 for _, vid := range e.cachedVals { 2243 a := e.cache[vid] 2244 for _, c := range a { 2245 if r, ok := e.s.f.getHome(c.ID).(*Register); ok && m>>uint(r.num)&1 != 0 { 2246 if !c.rematerializeable() { 2247 x := e.p.NewValue1(c.Pos, OpStoreReg, c.Type, c) 2248 // Allocate a temp location to spill a register to. 2249 // The type of the slot is immaterial - it will not be live across 2250 // any safepoint. Just use a type big enough to hold any register. 2251 t := LocalSlot{N: e.s.f.fe.Auto(c.Pos, types.Int64), Type: types.Int64} 2252 // TODO: reuse these slots. They'll need to be erased first. 2253 e.set(t, vid, x, false, c.Pos) 2254 if e.s.f.pass.debug > regDebug { 2255 fmt.Printf(" SPILL %s->%s %s\n", r, t, x.LongString()) 2256 } 2257 } 2258 // r will now be overwritten by the caller. At some point 2259 // later, the newly saved value will be moved back to its 2260 // final destination in processDest. 2261 return r 2262 } 2263 } 2264 } 2265 2266 fmt.Printf("m:%d unique:%d final:%d rematerializable:%d\n", m, e.uniqueRegs, e.finalRegs, e.rematerializeableRegs) 2267 for _, vid := range e.cachedVals { 2268 a := e.cache[vid] 2269 for _, c := range a { 2270 fmt.Printf("v%d: %s %s\n", vid, c, e.s.f.getHome(c.ID)) 2271 } 2272 } 2273 e.s.f.Fatalf("can't find empty register on edge %s->%s", e.p, e.b) 2274 return nil 2275 } 2276 2277 // rematerializeable reports whether the register allocator should recompute 2278 // a value instead of spilling/restoring it. 2279 func (v *Value) rematerializeable() bool { 2280 if !opcodeTable[v.Op].rematerializeable { 2281 return false 2282 } 2283 for _, a := range v.Args { 2284 // SP and SB (generated by OpSP and OpSB) are always available. 2285 if a.Op != OpSP && a.Op != OpSB { 2286 return false 2287 } 2288 } 2289 return true 2290 } 2291 2292 type liveInfo struct { 2293 ID ID // ID of value 2294 dist int32 // # of instructions before next use 2295 pos src.XPos // source position of next use 2296 } 2297 2298 // computeLive computes a map from block ID to a list of value IDs live at the end 2299 // of that block. Together with the value ID is a count of how many instructions 2300 // to the next use of that value. The resulting map is stored in s.live. 2301 // computeLive also computes the desired register information at the end of each block. 2302 // This desired register information is stored in s.desired. 2303 // TODO: this could be quadratic if lots of variables are live across lots of 2304 // basic blocks. Figure out a way to make this function (or, more precisely, the user 2305 // of this function) require only linear size & time. 2306 func (s *regAllocState) computeLive() { 2307 f := s.f 2308 s.live = make([][]liveInfo, f.NumBlocks()) 2309 s.desired = make([]desiredState, f.NumBlocks()) 2310 var phis []*Value 2311 2312 live := f.newSparseMap(f.NumValues()) 2313 defer f.retSparseMap(live) 2314 t := f.newSparseMap(f.NumValues()) 2315 defer f.retSparseMap(t) 2316 2317 // Keep track of which value we want in each register. 2318 var desired desiredState 2319 2320 // Instead of iterating over f.Blocks, iterate over their postordering. 2321 // Liveness information flows backward, so starting at the end 2322 // increases the probability that we will stabilize quickly. 2323 // TODO: Do a better job yet. Here's one possibility: 2324 // Calculate the dominator tree and locate all strongly connected components. 2325 // If a value is live in one block of an SCC, it is live in all. 2326 // Walk the dominator tree from end to beginning, just once, treating SCC 2327 // components as single blocks, duplicated calculated liveness information 2328 // out to all of them. 2329 po := f.postorder() 2330 s.loopnest = f.loopnest() 2331 s.loopnest.calculateDepths() 2332 for { 2333 changed := false 2334 2335 for _, b := range po { 2336 // Start with known live values at the end of the block. 2337 // Add len(b.Values) to adjust from end-of-block distance 2338 // to beginning-of-block distance. 2339 live.clear() 2340 for _, e := range s.live[b.ID] { 2341 live.set(e.ID, e.dist+int32(len(b.Values)), e.pos) 2342 } 2343 2344 // Mark control value as live 2345 if b.Control != nil && s.values[b.Control.ID].needReg { 2346 live.set(b.Control.ID, int32(len(b.Values)), b.Pos) 2347 } 2348 2349 // Propagate backwards to the start of the block 2350 // Assumes Values have been scheduled. 2351 phis = phis[:0] 2352 for i := len(b.Values) - 1; i >= 0; i-- { 2353 v := b.Values[i] 2354 live.remove(v.ID) 2355 if v.Op == OpPhi { 2356 // save phi ops for later 2357 phis = append(phis, v) 2358 continue 2359 } 2360 if opcodeTable[v.Op].call { 2361 c := live.contents() 2362 for i := range c { 2363 c[i].val += unlikelyDistance 2364 } 2365 } 2366 for _, a := range v.Args { 2367 if s.values[a.ID].needReg { 2368 live.set(a.ID, int32(i), v.Pos) 2369 } 2370 } 2371 } 2372 // Propagate desired registers backwards. 2373 desired.copy(&s.desired[b.ID]) 2374 for i := len(b.Values) - 1; i >= 0; i-- { 2375 v := b.Values[i] 2376 prefs := desired.remove(v.ID) 2377 if v.Op == OpPhi { 2378 // TODO: if v is a phi, save desired register for phi inputs. 2379 // For now, we just drop it and don't propagate 2380 // desired registers back though phi nodes. 2381 continue 2382 } 2383 regspec := s.regspec(v.Op) 2384 // Cancel desired registers if they get clobbered. 2385 desired.clobber(regspec.clobbers) 2386 // Update desired registers if there are any fixed register inputs. 2387 for _, j := range regspec.inputs { 2388 if countRegs(j.regs) != 1 { 2389 continue 2390 } 2391 desired.clobber(j.regs) 2392 desired.add(v.Args[j.idx].ID, pickReg(j.regs)) 2393 } 2394 // Set desired register of input 0 if this is a 2-operand instruction. 2395 if opcodeTable[v.Op].resultInArg0 { 2396 if opcodeTable[v.Op].commutative { 2397 desired.addList(v.Args[1].ID, prefs) 2398 } 2399 desired.addList(v.Args[0].ID, prefs) 2400 } 2401 } 2402 2403 // For each predecessor of b, expand its list of live-at-end values. 2404 // invariant: live contains the values live at the start of b (excluding phi inputs) 2405 for i, e := range b.Preds { 2406 p := e.b 2407 // Compute additional distance for the edge. 2408 // Note: delta must be at least 1 to distinguish the control 2409 // value use from the first user in a successor block. 2410 delta := int32(normalDistance) 2411 if len(p.Succs) == 2 { 2412 if p.Succs[0].b == b && p.Likely == BranchLikely || 2413 p.Succs[1].b == b && p.Likely == BranchUnlikely { 2414 delta = likelyDistance 2415 } 2416 if p.Succs[0].b == b && p.Likely == BranchUnlikely || 2417 p.Succs[1].b == b && p.Likely == BranchLikely { 2418 delta = unlikelyDistance 2419 } 2420 } 2421 2422 // Update any desired registers at the end of p. 2423 s.desired[p.ID].merge(&desired) 2424 2425 // Start t off with the previously known live values at the end of p. 2426 t.clear() 2427 for _, e := range s.live[p.ID] { 2428 t.set(e.ID, e.dist, e.pos) 2429 } 2430 update := false 2431 2432 // Add new live values from scanning this block. 2433 for _, e := range live.contents() { 2434 d := e.val + delta 2435 if !t.contains(e.key) || d < t.get(e.key) { 2436 update = true 2437 t.set(e.key, d, e.aux) 2438 } 2439 } 2440 // Also add the correct arg from the saved phi values. 2441 // All phis are at distance delta (we consider them 2442 // simultaneously happening at the start of the block). 2443 for _, v := range phis { 2444 id := v.Args[i].ID 2445 if s.values[id].needReg && (!t.contains(id) || delta < t.get(id)) { 2446 update = true 2447 t.set(id, delta, v.Pos) 2448 } 2449 } 2450 2451 if !update { 2452 continue 2453 } 2454 // The live set has changed, update it. 2455 l := s.live[p.ID][:0] 2456 if cap(l) < t.size() { 2457 l = make([]liveInfo, 0, t.size()) 2458 } 2459 for _, e := range t.contents() { 2460 l = append(l, liveInfo{e.key, e.val, e.aux}) 2461 } 2462 s.live[p.ID] = l 2463 changed = true 2464 } 2465 } 2466 2467 if !changed { 2468 break 2469 } 2470 } 2471 if f.pass.debug > regDebug { 2472 fmt.Println("live values at end of each block") 2473 for _, b := range f.Blocks { 2474 fmt.Printf(" %s:", b) 2475 for _, x := range s.live[b.ID] { 2476 fmt.Printf(" v%d", x.ID) 2477 for _, e := range s.desired[b.ID].entries { 2478 if e.ID != x.ID { 2479 continue 2480 } 2481 fmt.Printf("[") 2482 first := true 2483 for _, r := range e.regs { 2484 if r == noRegister { 2485 continue 2486 } 2487 if !first { 2488 fmt.Printf(",") 2489 } 2490 fmt.Print(&s.registers[r]) 2491 first = false 2492 } 2493 fmt.Printf("]") 2494 } 2495 } 2496 if avoid := s.desired[b.ID].avoid; avoid != 0 { 2497 fmt.Printf(" avoid=%v", s.RegMaskString(avoid)) 2498 } 2499 fmt.Println() 2500 } 2501 } 2502 } 2503 2504 // A desiredState represents desired register assignments. 2505 type desiredState struct { 2506 // Desired assignments will be small, so we just use a list 2507 // of valueID+registers entries. 2508 entries []desiredStateEntry 2509 // Registers that other values want to be in. This value will 2510 // contain at least the union of the regs fields of entries, but 2511 // may contain additional entries for values that were once in 2512 // this data structure but are no longer. 2513 avoid regMask 2514 } 2515 type desiredStateEntry struct { 2516 // (pre-regalloc) value 2517 ID ID 2518 // Registers it would like to be in, in priority order. 2519 // Unused slots are filled with noRegister. 2520 regs [4]register 2521 } 2522 2523 func (d *desiredState) clear() { 2524 d.entries = d.entries[:0] 2525 d.avoid = 0 2526 } 2527 2528 // get returns a list of desired registers for value vid. 2529 func (d *desiredState) get(vid ID) [4]register { 2530 for _, e := range d.entries { 2531 if e.ID == vid { 2532 return e.regs 2533 } 2534 } 2535 return [4]register{noRegister, noRegister, noRegister, noRegister} 2536 } 2537 2538 // add records that we'd like value vid to be in register r. 2539 func (d *desiredState) add(vid ID, r register) { 2540 d.avoid |= regMask(1) << r 2541 for i := range d.entries { 2542 e := &d.entries[i] 2543 if e.ID != vid { 2544 continue 2545 } 2546 if e.regs[0] == r { 2547 // Already known and highest priority 2548 return 2549 } 2550 for j := 1; j < len(e.regs); j++ { 2551 if e.regs[j] == r { 2552 // Move from lower priority to top priority 2553 copy(e.regs[1:], e.regs[:j]) 2554 e.regs[0] = r 2555 return 2556 } 2557 } 2558 copy(e.regs[1:], e.regs[:]) 2559 e.regs[0] = r 2560 return 2561 } 2562 d.entries = append(d.entries, desiredStateEntry{vid, [4]register{r, noRegister, noRegister, noRegister}}) 2563 } 2564 2565 func (d *desiredState) addList(vid ID, regs [4]register) { 2566 // regs is in priority order, so iterate in reverse order. 2567 for i := len(regs) - 1; i >= 0; i-- { 2568 r := regs[i] 2569 if r != noRegister { 2570 d.add(vid, r) 2571 } 2572 } 2573 } 2574 2575 // clobber erases any desired registers in the set m. 2576 func (d *desiredState) clobber(m regMask) { 2577 for i := 0; i < len(d.entries); { 2578 e := &d.entries[i] 2579 j := 0 2580 for _, r := range e.regs { 2581 if r != noRegister && m>>r&1 == 0 { 2582 e.regs[j] = r 2583 j++ 2584 } 2585 } 2586 if j == 0 { 2587 // No more desired registers for this value. 2588 d.entries[i] = d.entries[len(d.entries)-1] 2589 d.entries = d.entries[:len(d.entries)-1] 2590 continue 2591 } 2592 for ; j < len(e.regs); j++ { 2593 e.regs[j] = noRegister 2594 } 2595 i++ 2596 } 2597 d.avoid &^= m 2598 } 2599 2600 // copy copies a desired state from another desiredState x. 2601 func (d *desiredState) copy(x *desiredState) { 2602 d.entries = append(d.entries[:0], x.entries...) 2603 d.avoid = x.avoid 2604 } 2605 2606 // remove removes the desired registers for vid and returns them. 2607 func (d *desiredState) remove(vid ID) [4]register { 2608 for i := range d.entries { 2609 if d.entries[i].ID == vid { 2610 regs := d.entries[i].regs 2611 d.entries[i] = d.entries[len(d.entries)-1] 2612 d.entries = d.entries[:len(d.entries)-1] 2613 return regs 2614 } 2615 } 2616 return [4]register{noRegister, noRegister, noRegister, noRegister} 2617 } 2618 2619 // merge merges another desired state x into d. 2620 func (d *desiredState) merge(x *desiredState) { 2621 d.avoid |= x.avoid 2622 // There should only be a few desired registers, so 2623 // linear insert is ok. 2624 for _, e := range x.entries { 2625 d.addList(e.ID, e.regs) 2626 } 2627 } 2628 2629 func min32(x, y int32) int32 { 2630 if x < y { 2631 return x 2632 } 2633 return y 2634 } 2635 func max32(x, y int32) int32 { 2636 if x > y { 2637 return x 2638 } 2639 return y 2640 }