github.com/cloudwego/frugal@v0.1.15/internal/atm/ssa/pass_regalloc.go (about)

     1  /*
     2   * Copyright 2022 ByteDance Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package ssa
    18  
    19  import (
    20      `fmt`
    21      `sort`
    22      `strings`
    23      `unsafe`
    24  
    25      `github.com/cloudwego/frugal/internal/rt`
    26      `gonum.org/v1/gonum/graph/coloring`
    27      `gonum.org/v1/gonum/graph/simple`
    28  )
    29  
    30  type (
    31      _RegSet map[Reg]struct{}
    32  )
    33  
    34  func (self _RegSet) add(r Reg) _RegSet {
    35      self[r] = struct{}{}
    36      return self
    37  }
    38  
    39  func (self _RegSet) union(rs _RegSet) {
    40      for r := range rs {
    41          self.add(r)
    42      }
    43  }
    44  
    45  func (self _RegSet) remove(r Reg) {
    46      delete(self, r)
    47  }
    48  
    49  func (self _RegSet) subtract(rs _RegSet) {
    50      for r := range rs {
    51          self.remove(r)
    52      }
    53  }
    54  
    55  func (self _RegSet) toslice() []Reg {
    56      nb := len(self)
    57      rr := make([]Reg, 0, nb)
    58  
    59      /* extract all registers */
    60      for r := range self {
    61          rr = append(rr, r)
    62      }
    63  
    64      /* sort by register ID */
    65      sort.Slice(rr, func(i int, j int) bool { return rr[i] < rr[j] })
    66      return rr
    67  }
    68  
    69  func (self _RegSet) contains(r Reg) (ret bool) {
    70      _, ret = self[r]
    71      return
    72  }
    73  
    74  func (self _RegSet) String() string {
    75      nb := len(self)
    76      rs := make([]string, 0, nb)
    77  
    78      /* convert every register */
    79      for _, r := range self.toslice() {
    80          rs = append(rs, r.String())
    81      }
    82  
    83      /* join them together */
    84      return fmt.Sprintf(
    85          "{%s}",
    86          strings.Join(rs, ", "),
    87      )
    88  }
    89  
    90  type _RegTabPair struct {
    91      rr Reg
    92      rs _RegSet
    93  }
    94  
    95  type _RegTab struct {
    96      p []_RegSet
    97      m map[Reg]_RegSet
    98  }
    99  
   100  func mkregtab() *_RegTab {
   101      return &_RegTab {
   102          p: make([]_RegSet, 0, 16),
   103          m: make(map[Reg]_RegSet, len(ArchRegs)),
   104      }
   105  }
   106  
   107  func (self *_RegTab) reset() {
   108      for k, v := range self.m {
   109          self.p = append(self.p, v)
   110          delete(self.m, k)
   111          rt.MapClear(v)
   112      }
   113  }
   114  
   115  func (self *_RegTab) pairs() (r []_RegTabPair) {
   116      r = make([]_RegTabPair, 0, len(self.m))
   117      for rr, rs := range self.m { r = append(r, _RegTabPair { rr, rs }) }
   118      sort.Slice(r, func(i int, j int) bool { return r[i].rr < r[j].rr })
   119      return
   120  }
   121  
   122  func (self *_RegTab) alloc(n int) (rs _RegSet) {
   123      if p := len(self.p); p == 0 {
   124          rs = make(_RegSet, n)
   125          return
   126      } else {
   127          rs, self.p = self.p[p - 1], self.p[:p - 1]
   128          return
   129      }
   130  }
   131  
   132  func (self *_RegTab) clone(s _RegSet) (rs _RegSet) {
   133      rs = self.alloc(len(s))
   134      for r := range s { rs.add(r) }
   135      return
   136  }
   137  
   138  func (self *_RegTab) mkset(r ...Reg) (rs _RegSet) {
   139      rs = self.alloc(len(r))
   140      for _, v := range r { rs.add(v) }
   141      return
   142  }
   143  
   144  func (self *_RegTab) mksetp(r []*Reg) (rs _RegSet) {
   145      rs = self.alloc(len(r))
   146      for _, v := range r { rs.add(*v) }
   147      return
   148  }
   149  
   150  func (self *_RegTab) relate(k Reg, v Reg) {
   151      if p, ok := self.m[k]; ok {
   152          p.add(v)
   153      } else {
   154          self.m[k] = self.alloc(1).add(v)
   155      }
   156  }
   157  
   158  func (self *_RegTab) remove(r Reg) (rs _RegSet) {
   159      rs = self.m[r]
   160      delete(self.m, r)
   161      return
   162  }
   163  
   164  // RegAlloc performs register allocation on CFG.
   165  type RegAlloc struct{}
   166  
   167  func (self RegAlloc) livein(p *_RegTab, lr map[Pos]_RegSet, bb *BasicBlock, in map[int]_RegSet, out map[int]_RegSet) _RegSet {
   168      var ok bool
   169      var rs _RegSet
   170      var use IrUsages
   171      var def IrDefinitions
   172  
   173      /* check for cached live-in sets */
   174      if rs, ok = in[bb.Id]; ok {
   175          return p.clone(rs)
   176      }
   177  
   178      /* calculate the live-out set of current block */
   179      tr := bb.Term
   180      regs := p.clone(self.liveout(p, lr, bb, in, out))
   181  
   182      /* assume all terminators are non-definitive */
   183      if _, ok = tr.(IrDefinitions); ok {
   184          panic("regalloc: definitions within terminators")
   185      }
   186  
   187      /* add the terminator usages if any */
   188      if use, ok = tr.(IrUsages); ok {
   189          regs.union(p.mksetp(use.Usages()))
   190      }
   191  
   192      /* mark live range of the terminator */
   193      rr := p.clone(regs)
   194      lr[pos(bb, _P_term)] = rr
   195  
   196      /* live(i-1) = use(i) ∪ (live(i) - { def(i) }) */
   197      for i := len(bb.Ins) - 1; i >= 0; i-- {
   198          if def, ok = bb.Ins[i].(IrDefinitions) ; ok { regs.subtract(p.mksetp(def.Definitions())) }
   199          if use, ok = bb.Ins[i].(IrUsages)      ; ok { regs.union(p.mksetp(use.Usages())) }
   200          lr[pos(bb, i)] = p.clone(regs)
   201      }
   202  
   203      /* should not have any Phi nodes */
   204      if len(bb.Phi) != 0 {
   205          panic("regalloc: unexpected Phi nodes")
   206      }
   207  
   208      /* update the cache */
   209      in[bb.Id] = p.clone(regs)
   210      return regs
   211  }
   212  
   213  func (self RegAlloc) liveout(p *_RegTab, lr map[Pos]_RegSet, bb *BasicBlock, in map[int]_RegSet, out map[int]_RegSet) _RegSet {
   214      var ok bool
   215      var rr []Reg
   216      var rs _RegSet
   217  
   218      /* check for cached live-out sets */
   219      if rs, ok = out[bb.Id]; ok {
   220          return rs
   221      }
   222  
   223      /* check for return blocks */
   224      if rr, ok = IrTryIntoArchReturn(bb.Term); ok {
   225          rs = p.mkset(rr...)
   226          out[bb.Id] = rs
   227          return rs
   228      }
   229  
   230      /* create a new register set */
   231      rs = p.alloc(0)
   232      it := bb.Term.Successors()
   233  
   234      /* live-out(p) = ∑(live-in(succ(p))) */
   235      for out[bb.Id] = nil; it.Next(); {
   236          rs.union(self.livein(p, lr, it.Block(), in, out))
   237      }
   238  
   239      /* update cache */
   240      out[bb.Id] = rs
   241      return rs
   242  }
   243  
   244  /* try to choose a different color from reloadRegs */
   245  func (self RegAlloc) colorDiffWithReload(rig *simple.UndirectedGraph, reg Reg, reloadReg map[Reg]int, arch []Reg, colormap map[Reg]int) {
   246      sameWithReload := false
   247      for _, c := range reloadReg {
   248          if c == colormap[reg] {
   249              sameWithReload = true
   250              break
   251          }
   252      }
   253      if !sameWithReload { return }
   254  
   255      /* all possible colors */
   256      colors := make(map[int]struct{})
   257      for i := range arch {
   258          colors[i] = struct{}{}
   259      }
   260  
   261      /* choose a different color from it's neightbors */
   262      for r := rig.From(int64(reg)); r.Next(); {
   263          delete(colors, colormap[Reg(r.Node().ID())])
   264      }
   265  
   266      /* choose a different color from reloadRegs */
   267      for _, v := range reloadReg {
   268          delete(colors, v)
   269      }
   270  
   271      if len(colors) > 0 {
   272          /* there have some other colors different with reload regs */
   273          for c := range colors {
   274              colormap[reg] = c
   275              break
   276          }
   277      }
   278  }
   279  
   280  /* try to choose the same color for reloadReg as reg */
   281  func (self RegAlloc) colorSameWithReg(rig *simple.UndirectedGraph, reloadReg Reg, arch []Reg, colormap map[Reg]int, reg Reg) {
   282      /* all possible colors */
   283      colors := make(map[int]struct{})
   284      for i := range arch {
   285          colors[i] = struct{}{}
   286      }
   287  
   288      /* choose a different color from it's neightbors */
   289      for r := rig.From(int64(reloadReg)); r.Next(); {
   290          delete(colors, colormap[Reg(r.Node().ID())])
   291      }
   292  
   293      /* try to choose the same color with reg */
   294      if _, ok := colors[colormap[reg]]; ok {
   295          colormap[reloadReg] = colormap[reg]
   296      }
   297  }
   298  
   299  func (self RegAlloc) Apply(cfg *CFG) {
   300      var rig *simple.UndirectedGraph
   301      var pass int
   302      var arch []Reg
   303      var colormap map[Reg]int
   304  
   305      /* reusable state */
   306      pool := mkregtab()
   307      regmap := make(map[int]Reg)
   308      livein := make(map[int]_RegSet)
   309      liveout := make(map[int]_RegSet)
   310      liveset := make(map[Pos]_RegSet)
   311      archcolors := make(map[int64]int, len(ArchRegs))
   312      coalescemap := make(map[Reg]Reg)
   313      invcoalescemap := make(map[Reg][]Reg)
   314  
   315      /* register coalescer */
   316      coalesce := func(rr []*Reg) {
   317          for _, r := range rr {
   318              if c, ok := coalescemap[*r]; ok {
   319                  *r = c
   320              }
   321          }
   322      }
   323  
   324      /* calculate allocatable registers */
   325      for _, r := range ArchRegs {
   326          if !ArchRegReserved[r] {
   327              arch = append(arch, IrSetArch(Rz, r))
   328          }
   329      }
   330  
   331      /* allocate colors to the registers */
   332      for i, r := range arch {
   333          archcolors[int64(r)] = i
   334      }
   335  
   336      /* register precolorer */
   337      precolor := func(rr []*Reg) {
   338          for _, r := range rr {
   339              if r.Kind() == K_arch {
   340                  *r = IrSetArch(Rz, ArchRegs[r.Name()])
   341              }
   342          }
   343      }
   344  
   345      /* precolor all physical registers */
   346      cfg.PostOrder().ForEach(func(bb *BasicBlock) {
   347          ok := false
   348          use := IrUsages(nil)
   349          def := IrDefinitions(nil)
   350  
   351          /* scan all the instructions */
   352          for _, v := range bb.Ins {
   353              if use, ok = v.(IrUsages)      ; ok { precolor(use.Usages()) }
   354              if def, ok = v.(IrDefinitions) ; ok { precolor(def.Definitions()) }
   355          }
   356  
   357          /* scan the terminator */
   358          if use, ok = bb.Term.(IrUsages); ok {
   359              precolor(use.Usages())
   360          }
   361      })
   362  
   363      /* loop until no more retries */
   364      for {
   365          pool.reset()
   366          rt.MapClear(livein)
   367          rt.MapClear(liveout)
   368          rt.MapClear(liveset)
   369          rt.MapClear(coalescemap)
   370          rt.MapClear(invcoalescemap)
   371  
   372          /* expand operands but not in the first pass */
   373          if pass != 0 {
   374              new(OperandAlloc).Apply(cfg)
   375          }
   376  
   377          /* Phase 1: Calculate live ranges */
   378          lr := self.livein(pool, liveset, cfg.Root, livein, liveout)
   379          rig = simple.NewUndirectedGraph()
   380  
   381          /* sanity check: no registers live at the entry point */
   382          if len(lr) != 0 {
   383              panic("regalloc: live registers at entry: " + lr.String())
   384          }
   385  
   386          /* Phase 2: Build register interference graph */
   387          for _, rs := range liveset {
   388              rr := rs.toslice()
   389              nr := len(rr)
   390  
   391              /* special case of a single live register */
   392              if nr == 1 && rig.Node(int64(rr[0])) == nil {
   393                  p, _ := rig.NodeWithID(int64(rr[0]))
   394                  rig.AddNode(p)
   395                  continue
   396              }
   397  
   398              /* create every edge */
   399              for i := 0; i < nr - 1; i++ {
   400                  for j := i + 1; j < nr; j++ {
   401                      p, _ := rig.NodeWithID(int64(rr[i]))
   402                      q, _ := rig.NodeWithID(int64(rr[j]))
   403                      rig.SetEdge(rig.NewEdge(p, q))
   404                  }
   405              }
   406          }
   407  
   408          /* make sure all physical registers are in the RIG */
   409          for _, r := range arch {
   410              if p, ok := rig.NodeWithID(int64(r)); ok {
   411                  rig.AddNode(p)
   412              }
   413          }
   414  
   415          /* Phase 3: Coalescing registers, find out all the coalescing pairs */
   416          cfg.PostOrder().ForEach(func(bb *BasicBlock) {
   417              for _, v := range bb.Ins {
   418                  var r0 Reg
   419                  var r1 Reg
   420                  var rx Reg
   421                  var ry Reg
   422                  var ok bool
   423  
   424                  /* only interested in copy instructions */
   425                  if rx, ry, ok = IrArchTryIntoCopy(v); !ok {
   426                      continue
   427                  }
   428  
   429                  /* locate the coalescing source register */
   430                  if r0, ok = coalescemap[rx]; ok { rx = r0 }
   431                  if r1, ok = coalescemap[ry]; ok { ry = r1 }
   432  
   433                  /* check if the two registers can be coalesced */
   434                  if rx == ry || rig.HasEdgeBetween(int64(rx), int64(ry)) || (rx.Kind() == K_arch && ry.Kind() == K_arch) {
   435                      continue
   436                  }
   437  
   438                  /* make sure Y is the node with a lower degree */
   439                  if rig.From(int64(rx)).Len() < rig.From(int64(ry)).Len() {
   440                      rx, ry = ry, rx
   441                  }
   442  
   443                  /* all the adjacent nodes of Y */
   444                  p := rig.From(int64(ry))
   445                  ok = true
   446  
   447                  /* determain whether it's safe to coalesce using George's heuristic */
   448                  for p.Next() {
   449                      if t := p.Node().ID(); len(arch) <= rig.From(t).Len() && !rig.HasEdgeBetween(t, int64(rx)) {
   450                          ok = false
   451                          break
   452                      }
   453                  }
   454  
   455                  /* check if it can be coalesced */
   456                  if !ok {
   457                      continue
   458                  }
   459  
   460                  /* r0 is the target register, r1 will be coalesced */
   461                  if regorder(rx) < regorder(ry) {
   462                      r0 = rx
   463                      r1 = ry
   464                  } else {
   465                      r0 = ry
   466                      r1 = rx
   467                  }
   468  
   469                  /* invcoalescemap's key is the target reg (named r0), it's value is the reg list coalesced to r0 */
   470                  for _, r := range invcoalescemap[r1] {
   471                      coalescemap[r] = r0
   472                  }
   473  
   474                  /* also mark the inverse relationship */
   475                  invcoalescemap[r0] = append(invcoalescemap[r0], r1)
   476                  invcoalescemap[r0] = append(invcoalescemap[r0], invcoalescemap[r1]...)
   477  
   478                  /* coalesce r0 and r1 (replace r1 by r0) in the interference graph */
   479                  for p = rig.From(int64(r1)); p.Next(); {
   480                      if t := p.Node().ID(); !rig.HasEdgeBetween(t, int64(r0)) {
   481                          nt, _ := rig.NodeWithID(t)
   482                          n0, _ := rig.NodeWithID(int64(r0))
   483                          rig.SetEdge(rig.NewEdge(n0, nt))
   484                      }
   485                  }
   486  
   487                  /* mark as coalesced */
   488                  coalescemap[r1] = r0
   489                  rig.RemoveNode(int64(r1))
   490              }
   491          })
   492  
   493          /* coalesce if needed */
   494          if len(coalescemap) != 0 {
   495              cfg.PostOrder().ForEach(func(bb *BasicBlock) {
   496                  var ok bool
   497                  var use IrUsages
   498                  var def IrDefinitions
   499  
   500                  /* should not have Phi nodes */
   501                  if len(bb.Phi) != 0 {
   502                      panic("regalloc: unexpected Phi nodes")
   503                  }
   504  
   505                  /* scan instructions */
   506                  for _, v := range bb.Ins {
   507                      if use, ok = v.(IrUsages)      ; ok { coalesce(use.Usages()) }
   508                      if def, ok = v.(IrDefinitions) ; ok { coalesce(def.Definitions()) }
   509                  }
   510  
   511                  /* scan terminator */
   512                  if use, ok = bb.Term.(IrUsages); ok {
   513                      coalesce(use.Usages())
   514                  }
   515              })
   516          }
   517  
   518          /* remove copies to itself */
   519          cfg.PostOrder().ForEach(func(bb *BasicBlock) {
   520              ins := bb.Ins
   521              bb.Ins = bb.Ins[:0]
   522  
   523              /* filter the instructions */
   524              for _, p := range ins {
   525                  if rd, rs, ok := IrArchTryIntoCopy(p); !ok || rd != rs {
   526                      bb.Ins = append(bb.Ins, p)
   527                  }
   528              }
   529          })
   530  
   531          /* try again if coalesce occured */
   532          if len(coalescemap) != 0 {
   533              continue
   534          }
   535  
   536          /* Phase 4: Attempt to color the RIG */
   537          k, m, _ := coloring.WelshPowell(rig, archcolors)
   538          colormap = *(*map[Reg]int)(unsafe.Pointer(&m))
   539  
   540          /* check for len(arch)-colorablity */
   541          if k <= len(arch) {
   542              break
   543          }
   544  
   545          /* the second pass should always be colorable */
   546          if pass++; pass >= 2 {
   547              panic("regalloc: this CFG may not colorable")
   548          }
   549  
   550          /* calculate the color sets */
   551          colors := coloring.Sets(m)
   552          colorset := *(*map[int][]Reg)(unsafe.Pointer(&colors))
   553  
   554          /* Phase 4: Spill excess registers to stack */
   555          for i, r := range arch {
   556              rr := Rz
   557              ok := false
   558              rs := colorset[i]
   559  
   560              /* remove from color map */
   561              for _, rr = range rs {
   562                  if colormap[rr] != i {
   563                      panic("regalloc: color mismatch for register " + rr.String())
   564                  } else if delete(colormap, rr); rr == r {
   565                      ok = true
   566                  }
   567              }
   568  
   569              /* remove from color set */
   570              if delete(colorset, i); !ok {
   571                  panic("regalloc: invalid coloring: missing register " + r.String())
   572              }
   573          }
   574  
   575          /* spill those without a physical register */
   576          cfg.PostOrder().ForEach(func(bb *BasicBlock) {
   577              var cc int
   578              var rr Reg
   579              var ok bool
   580              var use IrUsages
   581  
   582              /* should not contain Phi nodes */
   583              if len(bb.Phi) != 0 {
   584                  panic("regalloc: unexpected Phi node")
   585              }
   586  
   587              /* allocate buffer for new instructions */
   588              ins := bb.Ins
   589              bb.Ins = make([]IrNode, 0, len(ins))
   590  
   591              /* scan every instructions */
   592              for _, v := range ins {
   593                  var s Reg
   594                  var t Reg
   595                  var r *Reg
   596                  var d IrDefinitions
   597  
   598                  /* clear the register map */
   599                  for c := range regmap {
   600                      delete(regmap, c)
   601                  }
   602  
   603                  /* reload as needed */
   604                  if use, ok = v.(IrUsages); ok {
   605                      for _, r = range use.Usages() {
   606                          if cc, ok = colormap[*r]; ok {
   607                              if rr, ok = regmap[cc]; ok {
   608                                  *r = rr
   609                              } else {
   610                                  *r = cfg.CreateRegister(r.Ptr())
   611                                  bb.Ins = append(bb.Ins, IrCreateSpill(*r, cc, IrSpillReload))
   612                                  regmap[cc] = *r
   613                              }
   614                          }
   615                      }
   616                  }
   617  
   618                  /* add the instruction itself */
   619                  i := len(bb.Ins)
   620                  bb.Ins = append(bb.Ins, v)
   621  
   622                  /* no definitions */
   623                  if d, ok = v.(IrDefinitions); !ok {
   624                      continue
   625                  }
   626  
   627                  /* spill as needed */
   628                  for _, r = range d.Definitions() {
   629                      if cc, ok = colormap[*r]; ok {
   630                          if t, s, ok = IrArchTryIntoCopy(v); !ok {
   631                              bb.Ins = append(bb.Ins, IrCreateSpill(*r, cc, IrSpillStore))
   632                          } else {
   633                              bb.Ins[i] = IrCreateSpillEx(s, t.Ptr(), cc, IrSpillStore)
   634                          }
   635                      }
   636                  }
   637              }
   638  
   639              /* clear the register map for terminator */
   640              for c := range regmap {
   641                  delete(regmap, c)
   642              }
   643  
   644              /* scan for reloads in the terminator */
   645              if use, ok = bb.Term.(IrUsages); ok {
   646                  for _, r := range use.Usages() {
   647                      if cc, ok = colormap[*r]; ok {
   648                          if rr, ok = regmap[cc]; ok {
   649                              *r = rr
   650                          } else {
   651                              *r = cfg.CreateRegister(r.Ptr())
   652                              bb.Ins = append(bb.Ins, IrCreateSpill(*r, cc, IrSpillReload))
   653                              regmap[cc] = *r
   654                          }
   655                      }
   656                  }
   657              }
   658          })
   659      }
   660  
   661      /* finetune color allocation plan */
   662      cfg.PostOrder().ForEach(func(bb *BasicBlock) {
   663          reloadRegs := make(map[Reg]int)
   664          spillSlots := make(map[int]Reg)
   665          slotReg := make(map[int]Reg)
   666  
   667          /* process instructions */
   668          for _, v := range bb.Ins {
   669              if spillIr, ok := v.(*IrSpill); ok && spillIr.Op == IrSpillReload {
   670                  if _, ok = reloadRegs[spillIr.R]; !ok {
   671                      /* try to assign the same color to reloadReg and spillReg with the same slot */
   672                      if r, ok := spillSlots[spillIr.S.ID()]; ok {
   673                          self.colorSameWithReg(rig, spillIr.R, arch, colormap, r)
   674                      } else if r, ok := slotReg[spillIr.S.ID()]; ok {
   675                          /* try to assign the same color to reloadRegs with the same slot */
   676                          self.colorSameWithReg(rig, spillIr.R, arch, colormap, r)
   677                      } else {
   678                          /* try to assign different color to reloadRegs with different slots */
   679                          self.colorDiffWithReload(rig, spillIr.R, reloadRegs, arch, colormap)
   680                      }
   681                      slotReg[spillIr.S.ID()] = spillIr.R
   682                      reloadRegs[spillIr.R] = colormap[spillIr.R]
   683                  }
   684              } else if ok && spillIr.Op == IrSpillStore {
   685                  spillSlots[spillIr.S.ID()] = spillIr.R
   686              } else {
   687                  /* try to choose color different from reload regs for defined regs */
   688                  if def, ok := v.(IrDefinitions); ok {
   689                      for _, r := range def.Definitions() {
   690                          self.colorDiffWithReload(rig, *r, reloadRegs, arch, colormap)
   691                      }
   692                  }
   693              }
   694          }
   695      })
   696  
   697      /* register substitution routine */
   698      replaceregs := func(rr []*Reg) {
   699          for _, r := range rr {
   700              if c, ok := colormap[*r]; ok {
   701                  *r = arch[c]
   702              }
   703          }
   704      }
   705  
   706      /* Phase 5: Replace all the virtual registers with physical registers */
   707      cfg.PostOrder().ForEach(func(bb *BasicBlock) {
   708          var ok bool
   709          var use IrUsages
   710          var def IrDefinitions
   711  
   712          /* should not contain Phi nodes */
   713          if len(bb.Phi) != 0 {
   714              panic("regalloc: unexpected Phi node")
   715          }
   716  
   717          /* replace instructions */
   718          for _, v := range bb.Ins {
   719              if use, ok = v.(IrUsages)      ; ok { replaceregs(use.Usages()) }
   720              if def, ok = v.(IrDefinitions) ; ok { replaceregs(def.Definitions()) }
   721          }
   722  
   723          /* replace the terminator */
   724          if use, ok = bb.Term.(IrUsages); ok {
   725              replaceregs(use.Usages())
   726          }
   727      })
   728  
   729      /* remove copies to itself */
   730      cfg.PostOrder().ForEach(func(bb *BasicBlock) {
   731          ins := bb.Ins
   732          bb.Ins = bb.Ins[:0]
   733  
   734          /* filter the instructions */
   735          for _, p := range ins {
   736              if rd, rs, ok := IrArchTryIntoCopy(p); !ok || rd != rs {
   737                  bb.Ins = append(bb.Ins, p)
   738              }
   739          }
   740      })
   741  
   742      /* remove redundant spill and reload instructions where both the register and stack aren't modified */
   743      cfg.PostOrder().ForEach(func(bb *BasicBlock) {
   744          /* register to stack slot is a one to one mapping */
   745          regSlot := make(map[Reg]int)
   746          slotReg := make(map[int]Reg)
   747          ins := bb.Ins
   748          bb.Ins = nil
   749          def := IrDefinitions(nil)
   750  
   751          /* scan every instruction */
   752          for _, v := range ins {
   753              if spillIr, ok := v.(*IrSpill); ok {
   754                  /* if there has been a one-one mapping between the register and stack slot, abandon this spill/reload instruction */
   755                  if s, ok := regSlot[spillIr.R]; ok && s == spillIr.S.ID() {
   756                      if r, ok := slotReg[spillIr.S.ID()]; ok && r == spillIr.R {
   757                          continue
   758                      }
   759                  }
   760  
   761                  /* delete old mapping relations */
   762                  delete(regSlot, slotReg[spillIr.S.ID()])
   763                  delete(slotReg, regSlot[spillIr.R])
   764  
   765                  /* establish new mapping relations */
   766                  slotReg[spillIr.S.ID()] = spillIr.R
   767                  regSlot[spillIr.R] = spillIr.S.ID()
   768                  bb.Ins = append(bb.Ins, v)
   769              } else {
   770                  if def, ok = v.(IrDefinitions); ok {
   771                      for _, r := range def.Definitions() {
   772                          /* delete old mapping relations */
   773                          delete(slotReg, regSlot[*r])
   774                          delete(regSlot, *r)
   775                      }
   776                  }
   777                  bb.Ins = append(bb.Ins, v)
   778              }
   779          }
   780      })
   781  
   782      /* remove redundant spill where the same stack slot is overwritten before loading */
   783      cfg.PostOrder().ForEach(func(bb *BasicBlock) {
   784          var redundantIrPos []int
   785          storeStack := make(map[int][]int)
   786  
   787          /* scan every instruction */
   788          for i, v := range bb.Ins {
   789              if spillIr, ok := v.(*IrSpill) ; ok && spillIr.Op == IrSpillStore {
   790                  storeStack[spillIr.S.ID()] = append(storeStack[spillIr.S.ID()], i)
   791              } else if ok && spillIr.Op == IrSpillReload {
   792                  if irPos, ok := storeStack[spillIr.S.ID()]; ok {
   793                      redundantIrPos = append(redundantIrPos, irPos[0 : len(irPos)-1]...)
   794                      delete(storeStack, spillIr.S.ID())
   795                  }
   796              }
   797          }
   798  
   799          for _, irPos := range storeStack {
   800              if len(irPos) > 1 {
   801                  redundantIrPos = append(redundantIrPos, irPos[0 : len(irPos)-1]...)
   802              }
   803          }
   804  
   805          /* abandon redundant spill instructions according to their position in the block */
   806          if len(redundantIrPos) > 0 {
   807              ins := bb.Ins
   808              bb.Ins = nil
   809              sort.Ints(redundantIrPos)
   810              startPos := 0
   811              for _, p := range redundantIrPos {
   812                  bb.Ins = append(bb.Ins, ins[startPos : p]...)
   813                  startPos = p + 1
   814              }
   815              if startPos < len(ins) {
   816                  bb.Ins = append(bb.Ins, ins[startPos : ]...)
   817              }
   818          }
   819      })
   820  
   821      regSliceToSet := func(rr []*Reg) _RegSet {
   822          rs := make(_RegSet, 0)
   823          for _, r := range rr {
   824              rs.add(*r)
   825          }
   826          return rs
   827      }
   828  
   829      /* remove redundant reload where the register isn't used after being reloaded */
   830      cfg.PostOrder().ForEach(func(bb *BasicBlock) {
   831          var ok bool
   832          var use IrUsages
   833          var def IrDefinitions
   834          var spillIr *IrSpill
   835          var removePos []int
   836          rs := make(_RegSet, 0)
   837  
   838          /* add the terminator usages if any */
   839          if use, ok = bb.Term.(IrUsages); ok { rs.union(regSliceToSet(use.Usages())) }
   840  
   841          /* live(i-1) = use(i) ∪ (live(i) - { def(i) }) */
   842          for i := len(bb.Ins) - 1; i >= 0; i-- {
   843              if def, ok = bb.Ins[i].(IrDefinitions); ok {
   844                  if spillIr, ok = bb.Ins[i].(*IrSpill); ok && spillIr.Op == IrSpillReload {
   845                      /* if the reloaded reg isn't used afterwards, record its position and then remove it */
   846                      if !rs.contains(spillIr.R) {
   847                          removePos = append(removePos, i)
   848                          continue
   849                      }
   850                  }
   851                  rs.subtract(regSliceToSet(def.Definitions()))
   852              }
   853              if use, ok = bb.Ins[i].(IrUsages); ok { rs.union(regSliceToSet(use.Usages())) }
   854          }
   855  
   856          if len(removePos) > 0 {
   857              ins := bb.Ins
   858              bb.Ins = nil
   859              sort.Ints(removePos)
   860              startPos := 0
   861              for _, p := range removePos {
   862                  bb.Ins = append(bb.Ins, ins[startPos : p]...)
   863                  startPos = p + 1
   864              }
   865              if startPos < len(ins) {
   866                  bb.Ins = append(bb.Ins, ins[startPos : ]...)
   867              }
   868          }
   869      })
   870  }