github.com/megatontech/mynoteforgo@v0.0.0-20200507084910-5d0c6ea6e890/源码/cmd/compile/internal/arm64/ssa.go (about) 1 // Copyright 2016 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package arm64 6 7 import ( 8 "math" 9 10 "cmd/compile/internal/gc" 11 "cmd/compile/internal/ssa" 12 "cmd/compile/internal/types" 13 "cmd/internal/obj" 14 "cmd/internal/obj/arm64" 15 ) 16 17 // loadByType returns the load instruction of the given type. 18 func loadByType(t *types.Type) obj.As { 19 if t.IsFloat() { 20 switch t.Size() { 21 case 4: 22 return arm64.AFMOVS 23 case 8: 24 return arm64.AFMOVD 25 } 26 } else { 27 switch t.Size() { 28 case 1: 29 if t.IsSigned() { 30 return arm64.AMOVB 31 } else { 32 return arm64.AMOVBU 33 } 34 case 2: 35 if t.IsSigned() { 36 return arm64.AMOVH 37 } else { 38 return arm64.AMOVHU 39 } 40 case 4: 41 if t.IsSigned() { 42 return arm64.AMOVW 43 } else { 44 return arm64.AMOVWU 45 } 46 case 8: 47 return arm64.AMOVD 48 } 49 } 50 panic("bad load type") 51 } 52 53 // storeByType returns the store instruction of the given type. 54 func storeByType(t *types.Type) obj.As { 55 if t.IsFloat() { 56 switch t.Size() { 57 case 4: 58 return arm64.AFMOVS 59 case 8: 60 return arm64.AFMOVD 61 } 62 } else { 63 switch t.Size() { 64 case 1: 65 return arm64.AMOVB 66 case 2: 67 return arm64.AMOVH 68 case 4: 69 return arm64.AMOVW 70 case 8: 71 return arm64.AMOVD 72 } 73 } 74 panic("bad store type") 75 } 76 77 // makeshift encodes a register shifted by a constant, used as an Offset in Prog 78 func makeshift(reg int16, typ int64, s int64) int64 { 79 return int64(reg&31)<<16 | typ | (s&63)<<10 80 } 81 82 // genshift generates a Prog for r = r0 op (r1 shifted by n) 83 func genshift(s *gc.SSAGenState, as obj.As, r0, r1, r int16, typ int64, n int64) *obj.Prog { 84 p := s.Prog(as) 85 p.From.Type = obj.TYPE_SHIFT 86 p.From.Offset = makeshift(r1, typ, n) 87 p.Reg = r0 88 if r != 0 { 89 p.To.Type = obj.TYPE_REG 90 p.To.Reg = r 91 } 92 return p 93 } 94 95 // generate the memory operand for the indexed load/store instructions 96 func genIndexedOperand(v *ssa.Value) obj.Addr { 97 // Reg: base register, Index: (shifted) index register 98 mop := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[0].Reg()} 99 switch v.Op { 100 case ssa.OpARM64MOVDloadidx8, ssa.OpARM64MOVDstoreidx8, ssa.OpARM64MOVDstorezeroidx8: 101 mop.Index = arm64.REG_LSL | 3<<5 | v.Args[1].Reg()&31 102 case ssa.OpARM64MOVWloadidx4, ssa.OpARM64MOVWUloadidx4, ssa.OpARM64MOVWstoreidx4, ssa.OpARM64MOVWstorezeroidx4: 103 mop.Index = arm64.REG_LSL | 2<<5 | v.Args[1].Reg()&31 104 case ssa.OpARM64MOVHloadidx2, ssa.OpARM64MOVHUloadidx2, ssa.OpARM64MOVHstoreidx2, ssa.OpARM64MOVHstorezeroidx2: 105 mop.Index = arm64.REG_LSL | 1<<5 | v.Args[1].Reg()&31 106 default: // not shifted 107 mop.Index = v.Args[1].Reg() 108 } 109 return mop 110 } 111 112 func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { 113 switch v.Op { 114 case ssa.OpCopy, ssa.OpARM64MOVDreg: 115 if v.Type.IsMemory() { 116 return 117 } 118 x := v.Args[0].Reg() 119 y := v.Reg() 120 if x == y { 121 return 122 } 123 as := arm64.AMOVD 124 if v.Type.IsFloat() { 125 switch v.Type.Size() { 126 case 4: 127 as = arm64.AFMOVS 128 case 8: 129 as = arm64.AFMOVD 130 default: 131 panic("bad float size") 132 } 133 } 134 p := s.Prog(as) 135 p.From.Type = obj.TYPE_REG 136 p.From.Reg = x 137 p.To.Type = obj.TYPE_REG 138 p.To.Reg = y 139 case ssa.OpARM64MOVDnop: 140 if v.Reg() != v.Args[0].Reg() { 141 v.Fatalf("input[0] and output not in same register %s", v.LongString()) 142 } 143 // nothing to do 144 case ssa.OpLoadReg: 145 if v.Type.IsFlags() { 146 v.Fatalf("load flags not implemented: %v", v.LongString()) 147 return 148 } 149 p := s.Prog(loadByType(v.Type)) 150 gc.AddrAuto(&p.From, v.Args[0]) 151 p.To.Type = obj.TYPE_REG 152 p.To.Reg = v.Reg() 153 case ssa.OpStoreReg: 154 if v.Type.IsFlags() { 155 v.Fatalf("store flags not implemented: %v", v.LongString()) 156 return 157 } 158 p := s.Prog(storeByType(v.Type)) 159 p.From.Type = obj.TYPE_REG 160 p.From.Reg = v.Args[0].Reg() 161 gc.AddrAuto(&p.To, v) 162 case ssa.OpARM64ADD, 163 ssa.OpARM64SUB, 164 ssa.OpARM64AND, 165 ssa.OpARM64OR, 166 ssa.OpARM64XOR, 167 ssa.OpARM64BIC, 168 ssa.OpARM64EON, 169 ssa.OpARM64ORN, 170 ssa.OpARM64MUL, 171 ssa.OpARM64MULW, 172 ssa.OpARM64MNEG, 173 ssa.OpARM64MNEGW, 174 ssa.OpARM64MULH, 175 ssa.OpARM64UMULH, 176 ssa.OpARM64MULL, 177 ssa.OpARM64UMULL, 178 ssa.OpARM64DIV, 179 ssa.OpARM64UDIV, 180 ssa.OpARM64DIVW, 181 ssa.OpARM64UDIVW, 182 ssa.OpARM64MOD, 183 ssa.OpARM64UMOD, 184 ssa.OpARM64MODW, 185 ssa.OpARM64UMODW, 186 ssa.OpARM64SLL, 187 ssa.OpARM64SRL, 188 ssa.OpARM64SRA, 189 ssa.OpARM64FADDS, 190 ssa.OpARM64FADDD, 191 ssa.OpARM64FSUBS, 192 ssa.OpARM64FSUBD, 193 ssa.OpARM64FMULS, 194 ssa.OpARM64FMULD, 195 ssa.OpARM64FNMULS, 196 ssa.OpARM64FNMULD, 197 ssa.OpARM64FDIVS, 198 ssa.OpARM64FDIVD, 199 ssa.OpARM64ROR, 200 ssa.OpARM64RORW: 201 r := v.Reg() 202 r1 := v.Args[0].Reg() 203 r2 := v.Args[1].Reg() 204 p := s.Prog(v.Op.Asm()) 205 p.From.Type = obj.TYPE_REG 206 p.From.Reg = r2 207 p.Reg = r1 208 p.To.Type = obj.TYPE_REG 209 p.To.Reg = r 210 case ssa.OpARM64FMADDS, 211 ssa.OpARM64FMADDD, 212 ssa.OpARM64FNMADDS, 213 ssa.OpARM64FNMADDD, 214 ssa.OpARM64FMSUBS, 215 ssa.OpARM64FMSUBD, 216 ssa.OpARM64FNMSUBS, 217 ssa.OpARM64FNMSUBD, 218 ssa.OpARM64MADD, 219 ssa.OpARM64MADDW, 220 ssa.OpARM64MSUB, 221 ssa.OpARM64MSUBW: 222 rt := v.Reg() 223 ra := v.Args[0].Reg() 224 rm := v.Args[1].Reg() 225 rn := v.Args[2].Reg() 226 p := s.Prog(v.Op.Asm()) 227 p.Reg = ra 228 p.From.Type = obj.TYPE_REG 229 p.From.Reg = rm 230 p.SetFrom3(obj.Addr{Type: obj.TYPE_REG, Reg: rn}) 231 p.To.Type = obj.TYPE_REG 232 p.To.Reg = rt 233 case ssa.OpARM64ADDconst, 234 ssa.OpARM64SUBconst, 235 ssa.OpARM64ANDconst, 236 ssa.OpARM64ORconst, 237 ssa.OpARM64XORconst, 238 ssa.OpARM64SLLconst, 239 ssa.OpARM64SRLconst, 240 ssa.OpARM64SRAconst, 241 ssa.OpARM64RORconst, 242 ssa.OpARM64RORWconst: 243 p := s.Prog(v.Op.Asm()) 244 p.From.Type = obj.TYPE_CONST 245 p.From.Offset = v.AuxInt 246 p.Reg = v.Args[0].Reg() 247 p.To.Type = obj.TYPE_REG 248 p.To.Reg = v.Reg() 249 case ssa.OpARM64EXTRconst, 250 ssa.OpARM64EXTRWconst: 251 p := s.Prog(v.Op.Asm()) 252 p.From.Type = obj.TYPE_CONST 253 p.From.Offset = v.AuxInt 254 p.SetFrom3(obj.Addr{Type: obj.TYPE_REG, Reg: v.Args[0].Reg()}) 255 p.Reg = v.Args[1].Reg() 256 p.To.Type = obj.TYPE_REG 257 p.To.Reg = v.Reg() 258 case ssa.OpARM64MVNshiftLL, ssa.OpARM64NEGshiftLL: 259 genshift(s, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm64.SHIFT_LL, v.AuxInt) 260 case ssa.OpARM64MVNshiftRL, ssa.OpARM64NEGshiftRL: 261 genshift(s, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm64.SHIFT_LR, v.AuxInt) 262 case ssa.OpARM64MVNshiftRA, ssa.OpARM64NEGshiftRA: 263 genshift(s, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm64.SHIFT_AR, v.AuxInt) 264 case ssa.OpARM64ADDshiftLL, 265 ssa.OpARM64SUBshiftLL, 266 ssa.OpARM64ANDshiftLL, 267 ssa.OpARM64ORshiftLL, 268 ssa.OpARM64XORshiftLL, 269 ssa.OpARM64EONshiftLL, 270 ssa.OpARM64ORNshiftLL, 271 ssa.OpARM64BICshiftLL: 272 genshift(s, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_LL, v.AuxInt) 273 case ssa.OpARM64ADDshiftRL, 274 ssa.OpARM64SUBshiftRL, 275 ssa.OpARM64ANDshiftRL, 276 ssa.OpARM64ORshiftRL, 277 ssa.OpARM64XORshiftRL, 278 ssa.OpARM64EONshiftRL, 279 ssa.OpARM64ORNshiftRL, 280 ssa.OpARM64BICshiftRL: 281 genshift(s, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_LR, v.AuxInt) 282 case ssa.OpARM64ADDshiftRA, 283 ssa.OpARM64SUBshiftRA, 284 ssa.OpARM64ANDshiftRA, 285 ssa.OpARM64ORshiftRA, 286 ssa.OpARM64XORshiftRA, 287 ssa.OpARM64EONshiftRA, 288 ssa.OpARM64ORNshiftRA, 289 ssa.OpARM64BICshiftRA: 290 genshift(s, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_AR, v.AuxInt) 291 case ssa.OpARM64MOVDconst: 292 p := s.Prog(v.Op.Asm()) 293 p.From.Type = obj.TYPE_CONST 294 p.From.Offset = v.AuxInt 295 p.To.Type = obj.TYPE_REG 296 p.To.Reg = v.Reg() 297 case ssa.OpARM64FMOVSconst, 298 ssa.OpARM64FMOVDconst: 299 p := s.Prog(v.Op.Asm()) 300 p.From.Type = obj.TYPE_FCONST 301 p.From.Val = math.Float64frombits(uint64(v.AuxInt)) 302 p.To.Type = obj.TYPE_REG 303 p.To.Reg = v.Reg() 304 case ssa.OpARM64CMP, 305 ssa.OpARM64CMPW, 306 ssa.OpARM64CMN, 307 ssa.OpARM64CMNW, 308 ssa.OpARM64TST, 309 ssa.OpARM64TSTW, 310 ssa.OpARM64FCMPS, 311 ssa.OpARM64FCMPD: 312 p := s.Prog(v.Op.Asm()) 313 p.From.Type = obj.TYPE_REG 314 p.From.Reg = v.Args[1].Reg() 315 p.Reg = v.Args[0].Reg() 316 case ssa.OpARM64CMPconst, 317 ssa.OpARM64CMPWconst, 318 ssa.OpARM64CMNconst, 319 ssa.OpARM64CMNWconst, 320 ssa.OpARM64TSTconst, 321 ssa.OpARM64TSTWconst: 322 p := s.Prog(v.Op.Asm()) 323 p.From.Type = obj.TYPE_CONST 324 p.From.Offset = v.AuxInt 325 p.Reg = v.Args[0].Reg() 326 case ssa.OpARM64CMPshiftLL, ssa.OpARM64CMNshiftLL, ssa.OpARM64TSTshiftLL: 327 genshift(s, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_LL, v.AuxInt) 328 case ssa.OpARM64CMPshiftRL, ssa.OpARM64CMNshiftRL, ssa.OpARM64TSTshiftRL: 329 genshift(s, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_LR, v.AuxInt) 330 case ssa.OpARM64CMPshiftRA, ssa.OpARM64CMNshiftRA, ssa.OpARM64TSTshiftRA: 331 genshift(s, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_AR, v.AuxInt) 332 case ssa.OpARM64MOVDaddr: 333 p := s.Prog(arm64.AMOVD) 334 p.From.Type = obj.TYPE_ADDR 335 p.From.Reg = v.Args[0].Reg() 336 p.To.Type = obj.TYPE_REG 337 p.To.Reg = v.Reg() 338 339 var wantreg string 340 // MOVD $sym+off(base), R 341 // the assembler expands it as the following: 342 // - base is SP: add constant offset to SP (R13) 343 // when constant is large, tmp register (R11) may be used 344 // - base is SB: load external address from constant pool (use relocation) 345 switch v.Aux.(type) { 346 default: 347 v.Fatalf("aux is of unknown type %T", v.Aux) 348 case *obj.LSym: 349 wantreg = "SB" 350 gc.AddAux(&p.From, v) 351 case *gc.Node: 352 wantreg = "SP" 353 gc.AddAux(&p.From, v) 354 case nil: 355 // No sym, just MOVD $off(SP), R 356 wantreg = "SP" 357 p.From.Offset = v.AuxInt 358 } 359 if reg := v.Args[0].RegName(); reg != wantreg { 360 v.Fatalf("bad reg %s for symbol type %T, want %s", reg, v.Aux, wantreg) 361 } 362 case ssa.OpARM64MOVBload, 363 ssa.OpARM64MOVBUload, 364 ssa.OpARM64MOVHload, 365 ssa.OpARM64MOVHUload, 366 ssa.OpARM64MOVWload, 367 ssa.OpARM64MOVWUload, 368 ssa.OpARM64MOVDload, 369 ssa.OpARM64FMOVSload, 370 ssa.OpARM64FMOVDload: 371 p := s.Prog(v.Op.Asm()) 372 p.From.Type = obj.TYPE_MEM 373 p.From.Reg = v.Args[0].Reg() 374 gc.AddAux(&p.From, v) 375 p.To.Type = obj.TYPE_REG 376 p.To.Reg = v.Reg() 377 case ssa.OpARM64MOVBloadidx, 378 ssa.OpARM64MOVBUloadidx, 379 ssa.OpARM64MOVHloadidx, 380 ssa.OpARM64MOVHUloadidx, 381 ssa.OpARM64MOVWloadidx, 382 ssa.OpARM64MOVWUloadidx, 383 ssa.OpARM64MOVDloadidx, 384 ssa.OpARM64FMOVSloadidx, 385 ssa.OpARM64FMOVDloadidx, 386 ssa.OpARM64MOVHloadidx2, 387 ssa.OpARM64MOVHUloadidx2, 388 ssa.OpARM64MOVWloadidx4, 389 ssa.OpARM64MOVWUloadidx4, 390 ssa.OpARM64MOVDloadidx8: 391 p := s.Prog(v.Op.Asm()) 392 p.From = genIndexedOperand(v) 393 p.To.Type = obj.TYPE_REG 394 p.To.Reg = v.Reg() 395 case ssa.OpARM64LDAR, 396 ssa.OpARM64LDARW: 397 p := s.Prog(v.Op.Asm()) 398 p.From.Type = obj.TYPE_MEM 399 p.From.Reg = v.Args[0].Reg() 400 gc.AddAux(&p.From, v) 401 p.To.Type = obj.TYPE_REG 402 p.To.Reg = v.Reg0() 403 case ssa.OpARM64MOVBstore, 404 ssa.OpARM64MOVHstore, 405 ssa.OpARM64MOVWstore, 406 ssa.OpARM64MOVDstore, 407 ssa.OpARM64FMOVSstore, 408 ssa.OpARM64FMOVDstore, 409 ssa.OpARM64STLR, 410 ssa.OpARM64STLRW: 411 p := s.Prog(v.Op.Asm()) 412 p.From.Type = obj.TYPE_REG 413 p.From.Reg = v.Args[1].Reg() 414 p.To.Type = obj.TYPE_MEM 415 p.To.Reg = v.Args[0].Reg() 416 gc.AddAux(&p.To, v) 417 case ssa.OpARM64MOVBstoreidx, 418 ssa.OpARM64MOVHstoreidx, 419 ssa.OpARM64MOVWstoreidx, 420 ssa.OpARM64MOVDstoreidx, 421 ssa.OpARM64FMOVSstoreidx, 422 ssa.OpARM64FMOVDstoreidx, 423 ssa.OpARM64MOVHstoreidx2, 424 ssa.OpARM64MOVWstoreidx4, 425 ssa.OpARM64MOVDstoreidx8: 426 p := s.Prog(v.Op.Asm()) 427 p.To = genIndexedOperand(v) 428 p.From.Type = obj.TYPE_REG 429 p.From.Reg = v.Args[2].Reg() 430 case ssa.OpARM64STP: 431 p := s.Prog(v.Op.Asm()) 432 p.From.Type = obj.TYPE_REGREG 433 p.From.Reg = v.Args[1].Reg() 434 p.From.Offset = int64(v.Args[2].Reg()) 435 p.To.Type = obj.TYPE_MEM 436 p.To.Reg = v.Args[0].Reg() 437 gc.AddAux(&p.To, v) 438 case ssa.OpARM64MOVBstorezero, 439 ssa.OpARM64MOVHstorezero, 440 ssa.OpARM64MOVWstorezero, 441 ssa.OpARM64MOVDstorezero: 442 p := s.Prog(v.Op.Asm()) 443 p.From.Type = obj.TYPE_REG 444 p.From.Reg = arm64.REGZERO 445 p.To.Type = obj.TYPE_MEM 446 p.To.Reg = v.Args[0].Reg() 447 gc.AddAux(&p.To, v) 448 case ssa.OpARM64MOVBstorezeroidx, 449 ssa.OpARM64MOVHstorezeroidx, 450 ssa.OpARM64MOVWstorezeroidx, 451 ssa.OpARM64MOVDstorezeroidx, 452 ssa.OpARM64MOVHstorezeroidx2, 453 ssa.OpARM64MOVWstorezeroidx4, 454 ssa.OpARM64MOVDstorezeroidx8: 455 p := s.Prog(v.Op.Asm()) 456 p.To = genIndexedOperand(v) 457 p.From.Type = obj.TYPE_REG 458 p.From.Reg = arm64.REGZERO 459 case ssa.OpARM64MOVQstorezero: 460 p := s.Prog(v.Op.Asm()) 461 p.From.Type = obj.TYPE_REGREG 462 p.From.Reg = arm64.REGZERO 463 p.From.Offset = int64(arm64.REGZERO) 464 p.To.Type = obj.TYPE_MEM 465 p.To.Reg = v.Args[0].Reg() 466 gc.AddAux(&p.To, v) 467 case ssa.OpARM64BFI, 468 ssa.OpARM64BFXIL: 469 r := v.Reg() 470 if r != v.Args[0].Reg() { 471 v.Fatalf("input[0] and output not in same register %s", v.LongString()) 472 } 473 p := s.Prog(v.Op.Asm()) 474 p.From.Type = obj.TYPE_CONST 475 p.From.Offset = v.AuxInt >> 8 476 p.SetFrom3(obj.Addr{Type: obj.TYPE_CONST, Offset: v.AuxInt & 0xff}) 477 p.Reg = v.Args[1].Reg() 478 p.To.Type = obj.TYPE_REG 479 p.To.Reg = r 480 case ssa.OpARM64SBFIZ, 481 ssa.OpARM64SBFX, 482 ssa.OpARM64UBFIZ, 483 ssa.OpARM64UBFX: 484 p := s.Prog(v.Op.Asm()) 485 p.From.Type = obj.TYPE_CONST 486 p.From.Offset = v.AuxInt >> 8 487 p.SetFrom3(obj.Addr{Type: obj.TYPE_CONST, Offset: v.AuxInt & 0xff}) 488 p.Reg = v.Args[0].Reg() 489 p.To.Type = obj.TYPE_REG 490 p.To.Reg = v.Reg() 491 case ssa.OpARM64LoweredMuluhilo: 492 r0 := v.Args[0].Reg() 493 r1 := v.Args[1].Reg() 494 p := s.Prog(arm64.AUMULH) 495 p.From.Type = obj.TYPE_REG 496 p.From.Reg = r1 497 p.Reg = r0 498 p.To.Type = obj.TYPE_REG 499 p.To.Reg = v.Reg0() 500 p1 := s.Prog(arm64.AMUL) 501 p1.From.Type = obj.TYPE_REG 502 p1.From.Reg = r1 503 p1.Reg = r0 504 p1.To.Type = obj.TYPE_REG 505 p1.To.Reg = v.Reg1() 506 case ssa.OpARM64LoweredAtomicExchange64, 507 ssa.OpARM64LoweredAtomicExchange32: 508 // LDAXR (Rarg0), Rout 509 // STLXR Rarg1, (Rarg0), Rtmp 510 // CBNZ Rtmp, -2(PC) 511 ld := arm64.ALDAXR 512 st := arm64.ASTLXR 513 if v.Op == ssa.OpARM64LoweredAtomicExchange32 { 514 ld = arm64.ALDAXRW 515 st = arm64.ASTLXRW 516 } 517 r0 := v.Args[0].Reg() 518 r1 := v.Args[1].Reg() 519 out := v.Reg0() 520 p := s.Prog(ld) 521 p.From.Type = obj.TYPE_MEM 522 p.From.Reg = r0 523 p.To.Type = obj.TYPE_REG 524 p.To.Reg = out 525 p1 := s.Prog(st) 526 p1.From.Type = obj.TYPE_REG 527 p1.From.Reg = r1 528 p1.To.Type = obj.TYPE_MEM 529 p1.To.Reg = r0 530 p1.RegTo2 = arm64.REGTMP 531 p2 := s.Prog(arm64.ACBNZ) 532 p2.From.Type = obj.TYPE_REG 533 p2.From.Reg = arm64.REGTMP 534 p2.To.Type = obj.TYPE_BRANCH 535 gc.Patch(p2, p) 536 case ssa.OpARM64LoweredAtomicAdd64, 537 ssa.OpARM64LoweredAtomicAdd32: 538 // LDAXR (Rarg0), Rout 539 // ADD Rarg1, Rout 540 // STLXR Rout, (Rarg0), Rtmp 541 // CBNZ Rtmp, -3(PC) 542 ld := arm64.ALDAXR 543 st := arm64.ASTLXR 544 if v.Op == ssa.OpARM64LoweredAtomicAdd32 { 545 ld = arm64.ALDAXRW 546 st = arm64.ASTLXRW 547 } 548 r0 := v.Args[0].Reg() 549 r1 := v.Args[1].Reg() 550 out := v.Reg0() 551 p := s.Prog(ld) 552 p.From.Type = obj.TYPE_MEM 553 p.From.Reg = r0 554 p.To.Type = obj.TYPE_REG 555 p.To.Reg = out 556 p1 := s.Prog(arm64.AADD) 557 p1.From.Type = obj.TYPE_REG 558 p1.From.Reg = r1 559 p1.To.Type = obj.TYPE_REG 560 p1.To.Reg = out 561 p2 := s.Prog(st) 562 p2.From.Type = obj.TYPE_REG 563 p2.From.Reg = out 564 p2.To.Type = obj.TYPE_MEM 565 p2.To.Reg = r0 566 p2.RegTo2 = arm64.REGTMP 567 p3 := s.Prog(arm64.ACBNZ) 568 p3.From.Type = obj.TYPE_REG 569 p3.From.Reg = arm64.REGTMP 570 p3.To.Type = obj.TYPE_BRANCH 571 gc.Patch(p3, p) 572 case ssa.OpARM64LoweredAtomicAdd64Variant, 573 ssa.OpARM64LoweredAtomicAdd32Variant: 574 // LDADDAL Rarg1, (Rarg0), Rout 575 // ADD Rarg1, Rout 576 op := arm64.ALDADDALD 577 if v.Op == ssa.OpARM64LoweredAtomicAdd32Variant { 578 op = arm64.ALDADDALW 579 } 580 r0 := v.Args[0].Reg() 581 r1 := v.Args[1].Reg() 582 out := v.Reg0() 583 p := s.Prog(op) 584 p.From.Type = obj.TYPE_REG 585 p.From.Reg = r1 586 p.To.Type = obj.TYPE_MEM 587 p.To.Reg = r0 588 p.RegTo2 = out 589 p1 := s.Prog(arm64.AADD) 590 p1.From.Type = obj.TYPE_REG 591 p1.From.Reg = r1 592 p1.To.Type = obj.TYPE_REG 593 p1.To.Reg = out 594 case ssa.OpARM64LoweredAtomicCas64, 595 ssa.OpARM64LoweredAtomicCas32: 596 // LDAXR (Rarg0), Rtmp 597 // CMP Rarg1, Rtmp 598 // BNE 3(PC) 599 // STLXR Rarg2, (Rarg0), Rtmp 600 // CBNZ Rtmp, -4(PC) 601 // CSET EQ, Rout 602 ld := arm64.ALDAXR 603 st := arm64.ASTLXR 604 cmp := arm64.ACMP 605 if v.Op == ssa.OpARM64LoweredAtomicCas32 { 606 ld = arm64.ALDAXRW 607 st = arm64.ASTLXRW 608 cmp = arm64.ACMPW 609 } 610 r0 := v.Args[0].Reg() 611 r1 := v.Args[1].Reg() 612 r2 := v.Args[2].Reg() 613 out := v.Reg0() 614 p := s.Prog(ld) 615 p.From.Type = obj.TYPE_MEM 616 p.From.Reg = r0 617 p.To.Type = obj.TYPE_REG 618 p.To.Reg = arm64.REGTMP 619 p1 := s.Prog(cmp) 620 p1.From.Type = obj.TYPE_REG 621 p1.From.Reg = r1 622 p1.Reg = arm64.REGTMP 623 p2 := s.Prog(arm64.ABNE) 624 p2.To.Type = obj.TYPE_BRANCH 625 p3 := s.Prog(st) 626 p3.From.Type = obj.TYPE_REG 627 p3.From.Reg = r2 628 p3.To.Type = obj.TYPE_MEM 629 p3.To.Reg = r0 630 p3.RegTo2 = arm64.REGTMP 631 p4 := s.Prog(arm64.ACBNZ) 632 p4.From.Type = obj.TYPE_REG 633 p4.From.Reg = arm64.REGTMP 634 p4.To.Type = obj.TYPE_BRANCH 635 gc.Patch(p4, p) 636 p5 := s.Prog(arm64.ACSET) 637 p5.From.Type = obj.TYPE_REG // assembler encodes conditional bits in Reg 638 p5.From.Reg = arm64.COND_EQ 639 p5.To.Type = obj.TYPE_REG 640 p5.To.Reg = out 641 gc.Patch(p2, p5) 642 case ssa.OpARM64LoweredAtomicAnd8, 643 ssa.OpARM64LoweredAtomicOr8: 644 // LDAXRB (Rarg0), Rout 645 // AND/OR Rarg1, Rout 646 // STLXRB Rout, (Rarg0), Rtmp 647 // CBNZ Rtmp, -3(PC) 648 r0 := v.Args[0].Reg() 649 r1 := v.Args[1].Reg() 650 out := v.Reg0() 651 p := s.Prog(arm64.ALDAXRB) 652 p.From.Type = obj.TYPE_MEM 653 p.From.Reg = r0 654 p.To.Type = obj.TYPE_REG 655 p.To.Reg = out 656 p1 := s.Prog(v.Op.Asm()) 657 p1.From.Type = obj.TYPE_REG 658 p1.From.Reg = r1 659 p1.To.Type = obj.TYPE_REG 660 p1.To.Reg = out 661 p2 := s.Prog(arm64.ASTLXRB) 662 p2.From.Type = obj.TYPE_REG 663 p2.From.Reg = out 664 p2.To.Type = obj.TYPE_MEM 665 p2.To.Reg = r0 666 p2.RegTo2 = arm64.REGTMP 667 p3 := s.Prog(arm64.ACBNZ) 668 p3.From.Type = obj.TYPE_REG 669 p3.From.Reg = arm64.REGTMP 670 p3.To.Type = obj.TYPE_BRANCH 671 gc.Patch(p3, p) 672 case ssa.OpARM64MOVBreg, 673 ssa.OpARM64MOVBUreg, 674 ssa.OpARM64MOVHreg, 675 ssa.OpARM64MOVHUreg, 676 ssa.OpARM64MOVWreg, 677 ssa.OpARM64MOVWUreg: 678 a := v.Args[0] 679 for a.Op == ssa.OpCopy || a.Op == ssa.OpARM64MOVDreg { 680 a = a.Args[0] 681 } 682 if a.Op == ssa.OpLoadReg { 683 t := a.Type 684 switch { 685 case v.Op == ssa.OpARM64MOVBreg && t.Size() == 1 && t.IsSigned(), 686 v.Op == ssa.OpARM64MOVBUreg && t.Size() == 1 && !t.IsSigned(), 687 v.Op == ssa.OpARM64MOVHreg && t.Size() == 2 && t.IsSigned(), 688 v.Op == ssa.OpARM64MOVHUreg && t.Size() == 2 && !t.IsSigned(), 689 v.Op == ssa.OpARM64MOVWreg && t.Size() == 4 && t.IsSigned(), 690 v.Op == ssa.OpARM64MOVWUreg && t.Size() == 4 && !t.IsSigned(): 691 // arg is a proper-typed load, already zero/sign-extended, don't extend again 692 if v.Reg() == v.Args[0].Reg() { 693 return 694 } 695 p := s.Prog(arm64.AMOVD) 696 p.From.Type = obj.TYPE_REG 697 p.From.Reg = v.Args[0].Reg() 698 p.To.Type = obj.TYPE_REG 699 p.To.Reg = v.Reg() 700 return 701 default: 702 } 703 } 704 fallthrough 705 case ssa.OpARM64MVN, 706 ssa.OpARM64NEG, 707 ssa.OpARM64FABSD, 708 ssa.OpARM64FMOVDfpgp, 709 ssa.OpARM64FMOVDgpfp, 710 ssa.OpARM64FMOVSfpgp, 711 ssa.OpARM64FMOVSgpfp, 712 ssa.OpARM64FNEGS, 713 ssa.OpARM64FNEGD, 714 ssa.OpARM64FSQRTD, 715 ssa.OpARM64FCVTZSSW, 716 ssa.OpARM64FCVTZSDW, 717 ssa.OpARM64FCVTZUSW, 718 ssa.OpARM64FCVTZUDW, 719 ssa.OpARM64FCVTZSS, 720 ssa.OpARM64FCVTZSD, 721 ssa.OpARM64FCVTZUS, 722 ssa.OpARM64FCVTZUD, 723 ssa.OpARM64SCVTFWS, 724 ssa.OpARM64SCVTFWD, 725 ssa.OpARM64SCVTFS, 726 ssa.OpARM64SCVTFD, 727 ssa.OpARM64UCVTFWS, 728 ssa.OpARM64UCVTFWD, 729 ssa.OpARM64UCVTFS, 730 ssa.OpARM64UCVTFD, 731 ssa.OpARM64FCVTSD, 732 ssa.OpARM64FCVTDS, 733 ssa.OpARM64REV, 734 ssa.OpARM64REVW, 735 ssa.OpARM64REV16W, 736 ssa.OpARM64RBIT, 737 ssa.OpARM64RBITW, 738 ssa.OpARM64CLZ, 739 ssa.OpARM64CLZW, 740 ssa.OpARM64FRINTAD, 741 ssa.OpARM64FRINTMD, 742 ssa.OpARM64FRINTND, 743 ssa.OpARM64FRINTPD, 744 ssa.OpARM64FRINTZD: 745 p := s.Prog(v.Op.Asm()) 746 p.From.Type = obj.TYPE_REG 747 p.From.Reg = v.Args[0].Reg() 748 p.To.Type = obj.TYPE_REG 749 p.To.Reg = v.Reg() 750 case ssa.OpARM64LoweredRound32F, ssa.OpARM64LoweredRound64F: 751 // input is already rounded 752 case ssa.OpARM64VCNT: 753 p := s.Prog(v.Op.Asm()) 754 p.From.Type = obj.TYPE_REG 755 p.From.Reg = (v.Args[0].Reg()-arm64.REG_F0)&31 + arm64.REG_ARNG + ((arm64.ARNG_8B & 15) << 5) 756 p.To.Type = obj.TYPE_REG 757 p.To.Reg = (v.Reg()-arm64.REG_F0)&31 + arm64.REG_ARNG + ((arm64.ARNG_8B & 15) << 5) 758 case ssa.OpARM64VUADDLV: 759 p := s.Prog(v.Op.Asm()) 760 p.From.Type = obj.TYPE_REG 761 p.From.Reg = (v.Args[0].Reg()-arm64.REG_F0)&31 + arm64.REG_ARNG + ((arm64.ARNG_8B & 15) << 5) 762 p.To.Type = obj.TYPE_REG 763 p.To.Reg = v.Reg() - arm64.REG_F0 + arm64.REG_V0 764 case ssa.OpARM64CSEL, ssa.OpARM64CSEL0: 765 r1 := int16(arm64.REGZERO) 766 if v.Op != ssa.OpARM64CSEL0 { 767 r1 = v.Args[1].Reg() 768 } 769 p := s.Prog(v.Op.Asm()) 770 p.From.Type = obj.TYPE_REG // assembler encodes conditional bits in Reg 771 p.From.Reg = condBits[v.Aux.(ssa.Op)] 772 p.Reg = v.Args[0].Reg() 773 p.SetFrom3(obj.Addr{Type: obj.TYPE_REG, Reg: r1}) 774 p.To.Type = obj.TYPE_REG 775 p.To.Reg = v.Reg() 776 case ssa.OpARM64DUFFZERO: 777 // runtime.duffzero expects start address in R16 778 p := s.Prog(obj.ADUFFZERO) 779 p.To.Type = obj.TYPE_MEM 780 p.To.Name = obj.NAME_EXTERN 781 p.To.Sym = gc.Duffzero 782 p.To.Offset = v.AuxInt 783 case ssa.OpARM64LoweredZero: 784 // STP.P (ZR,ZR), 16(R16) 785 // CMP Rarg1, R16 786 // BLE -2(PC) 787 // arg1 is the address of the last 16-byte unit to zero 788 p := s.Prog(arm64.ASTP) 789 p.Scond = arm64.C_XPOST 790 p.From.Type = obj.TYPE_REGREG 791 p.From.Reg = arm64.REGZERO 792 p.From.Offset = int64(arm64.REGZERO) 793 p.To.Type = obj.TYPE_MEM 794 p.To.Reg = arm64.REG_R16 795 p.To.Offset = 16 796 p2 := s.Prog(arm64.ACMP) 797 p2.From.Type = obj.TYPE_REG 798 p2.From.Reg = v.Args[1].Reg() 799 p2.Reg = arm64.REG_R16 800 p3 := s.Prog(arm64.ABLE) 801 p3.To.Type = obj.TYPE_BRANCH 802 gc.Patch(p3, p) 803 case ssa.OpARM64DUFFCOPY: 804 p := s.Prog(obj.ADUFFCOPY) 805 p.To.Type = obj.TYPE_MEM 806 p.To.Name = obj.NAME_EXTERN 807 p.To.Sym = gc.Duffcopy 808 p.To.Offset = v.AuxInt 809 case ssa.OpARM64LoweredMove: 810 // MOVD.P 8(R16), Rtmp 811 // MOVD.P Rtmp, 8(R17) 812 // CMP Rarg2, R16 813 // BLE -3(PC) 814 // arg2 is the address of the last element of src 815 p := s.Prog(arm64.AMOVD) 816 p.Scond = arm64.C_XPOST 817 p.From.Type = obj.TYPE_MEM 818 p.From.Reg = arm64.REG_R16 819 p.From.Offset = 8 820 p.To.Type = obj.TYPE_REG 821 p.To.Reg = arm64.REGTMP 822 p2 := s.Prog(arm64.AMOVD) 823 p2.Scond = arm64.C_XPOST 824 p2.From.Type = obj.TYPE_REG 825 p2.From.Reg = arm64.REGTMP 826 p2.To.Type = obj.TYPE_MEM 827 p2.To.Reg = arm64.REG_R17 828 p2.To.Offset = 8 829 p3 := s.Prog(arm64.ACMP) 830 p3.From.Type = obj.TYPE_REG 831 p3.From.Reg = v.Args[2].Reg() 832 p3.Reg = arm64.REG_R16 833 p4 := s.Prog(arm64.ABLE) 834 p4.To.Type = obj.TYPE_BRANCH 835 gc.Patch(p4, p) 836 case ssa.OpARM64CALLstatic, ssa.OpARM64CALLclosure, ssa.OpARM64CALLinter: 837 s.Call(v) 838 case ssa.OpARM64LoweredWB: 839 p := s.Prog(obj.ACALL) 840 p.To.Type = obj.TYPE_MEM 841 p.To.Name = obj.NAME_EXTERN 842 p.To.Sym = v.Aux.(*obj.LSym) 843 case ssa.OpARM64LoweredNilCheck: 844 // Issue a load which will fault if arg is nil. 845 p := s.Prog(arm64.AMOVB) 846 p.From.Type = obj.TYPE_MEM 847 p.From.Reg = v.Args[0].Reg() 848 gc.AddAux(&p.From, v) 849 p.To.Type = obj.TYPE_REG 850 p.To.Reg = arm64.REGTMP 851 if gc.Debug_checknil != 0 && v.Pos.Line() > 1 { // v.Line==1 in generated wrappers 852 gc.Warnl(v.Pos, "generated nil check") 853 } 854 case ssa.OpARM64Equal, 855 ssa.OpARM64NotEqual, 856 ssa.OpARM64LessThan, 857 ssa.OpARM64LessEqual, 858 ssa.OpARM64GreaterThan, 859 ssa.OpARM64GreaterEqual, 860 ssa.OpARM64LessThanU, 861 ssa.OpARM64LessEqualU, 862 ssa.OpARM64GreaterThanU, 863 ssa.OpARM64GreaterEqualU: 864 // generate boolean values using CSET 865 p := s.Prog(arm64.ACSET) 866 p.From.Type = obj.TYPE_REG // assembler encodes conditional bits in Reg 867 p.From.Reg = condBits[v.Op] 868 p.To.Type = obj.TYPE_REG 869 p.To.Reg = v.Reg() 870 case ssa.OpARM64LoweredGetClosurePtr: 871 // Closure pointer is R26 (arm64.REGCTXT). 872 gc.CheckLoweredGetClosurePtr(v) 873 case ssa.OpARM64LoweredGetCallerSP: 874 // caller's SP is FixedFrameSize below the address of the first arg 875 p := s.Prog(arm64.AMOVD) 876 p.From.Type = obj.TYPE_ADDR 877 p.From.Offset = -gc.Ctxt.FixedFrameSize() 878 p.From.Name = obj.NAME_PARAM 879 p.To.Type = obj.TYPE_REG 880 p.To.Reg = v.Reg() 881 case ssa.OpARM64LoweredGetCallerPC: 882 p := s.Prog(obj.AGETCALLERPC) 883 p.To.Type = obj.TYPE_REG 884 p.To.Reg = v.Reg() 885 case ssa.OpARM64FlagEQ, 886 ssa.OpARM64FlagLT_ULT, 887 ssa.OpARM64FlagLT_UGT, 888 ssa.OpARM64FlagGT_ULT, 889 ssa.OpARM64FlagGT_UGT: 890 v.Fatalf("Flag* ops should never make it to codegen %v", v.LongString()) 891 case ssa.OpARM64InvertFlags: 892 v.Fatalf("InvertFlags should never make it to codegen %v", v.LongString()) 893 case ssa.OpClobber: 894 // TODO: implement for clobberdead experiment. Nop is ok for now. 895 default: 896 v.Fatalf("genValue not implemented: %s", v.LongString()) 897 } 898 } 899 900 var condBits = map[ssa.Op]int16{ 901 ssa.OpARM64Equal: arm64.COND_EQ, 902 ssa.OpARM64NotEqual: arm64.COND_NE, 903 ssa.OpARM64LessThan: arm64.COND_LT, 904 ssa.OpARM64LessThanU: arm64.COND_LO, 905 ssa.OpARM64LessEqual: arm64.COND_LE, 906 ssa.OpARM64LessEqualU: arm64.COND_LS, 907 ssa.OpARM64GreaterThan: arm64.COND_GT, 908 ssa.OpARM64GreaterThanU: arm64.COND_HI, 909 ssa.OpARM64GreaterEqual: arm64.COND_GE, 910 ssa.OpARM64GreaterEqualU: arm64.COND_HS, 911 } 912 913 var blockJump = map[ssa.BlockKind]struct { 914 asm, invasm obj.As 915 }{ 916 ssa.BlockARM64EQ: {arm64.ABEQ, arm64.ABNE}, 917 ssa.BlockARM64NE: {arm64.ABNE, arm64.ABEQ}, 918 ssa.BlockARM64LT: {arm64.ABLT, arm64.ABGE}, 919 ssa.BlockARM64GE: {arm64.ABGE, arm64.ABLT}, 920 ssa.BlockARM64LE: {arm64.ABLE, arm64.ABGT}, 921 ssa.BlockARM64GT: {arm64.ABGT, arm64.ABLE}, 922 ssa.BlockARM64ULT: {arm64.ABLO, arm64.ABHS}, 923 ssa.BlockARM64UGE: {arm64.ABHS, arm64.ABLO}, 924 ssa.BlockARM64UGT: {arm64.ABHI, arm64.ABLS}, 925 ssa.BlockARM64ULE: {arm64.ABLS, arm64.ABHI}, 926 ssa.BlockARM64Z: {arm64.ACBZ, arm64.ACBNZ}, 927 ssa.BlockARM64NZ: {arm64.ACBNZ, arm64.ACBZ}, 928 ssa.BlockARM64ZW: {arm64.ACBZW, arm64.ACBNZW}, 929 ssa.BlockARM64NZW: {arm64.ACBNZW, arm64.ACBZW}, 930 ssa.BlockARM64TBZ: {arm64.ATBZ, arm64.ATBNZ}, 931 ssa.BlockARM64TBNZ: {arm64.ATBNZ, arm64.ATBZ}, 932 } 933 934 func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { 935 switch b.Kind { 936 case ssa.BlockPlain: 937 if b.Succs[0].Block() != next { 938 p := s.Prog(obj.AJMP) 939 p.To.Type = obj.TYPE_BRANCH 940 s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) 941 } 942 943 case ssa.BlockDefer: 944 // defer returns in R0: 945 // 0 if we should continue executing 946 // 1 if we should jump to deferreturn call 947 p := s.Prog(arm64.ACMP) 948 p.From.Type = obj.TYPE_CONST 949 p.From.Offset = 0 950 p.Reg = arm64.REG_R0 951 p = s.Prog(arm64.ABNE) 952 p.To.Type = obj.TYPE_BRANCH 953 s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()}) 954 if b.Succs[0].Block() != next { 955 p := s.Prog(obj.AJMP) 956 p.To.Type = obj.TYPE_BRANCH 957 s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) 958 } 959 960 case ssa.BlockExit: 961 s.Prog(obj.AUNDEF) // tell plive.go that we never reach here 962 963 case ssa.BlockRet: 964 s.Prog(obj.ARET) 965 966 case ssa.BlockRetJmp: 967 p := s.Prog(obj.ARET) 968 p.To.Type = obj.TYPE_MEM 969 p.To.Name = obj.NAME_EXTERN 970 p.To.Sym = b.Aux.(*obj.LSym) 971 972 case ssa.BlockARM64EQ, ssa.BlockARM64NE, 973 ssa.BlockARM64LT, ssa.BlockARM64GE, 974 ssa.BlockARM64LE, ssa.BlockARM64GT, 975 ssa.BlockARM64ULT, ssa.BlockARM64UGT, 976 ssa.BlockARM64ULE, ssa.BlockARM64UGE, 977 ssa.BlockARM64Z, ssa.BlockARM64NZ, 978 ssa.BlockARM64ZW, ssa.BlockARM64NZW: 979 jmp := blockJump[b.Kind] 980 var p *obj.Prog 981 switch next { 982 case b.Succs[0].Block(): 983 p = s.Br(jmp.invasm, b.Succs[1].Block()) 984 case b.Succs[1].Block(): 985 p = s.Br(jmp.asm, b.Succs[0].Block()) 986 default: 987 if b.Likely != ssa.BranchUnlikely { 988 p = s.Br(jmp.asm, b.Succs[0].Block()) 989 s.Br(obj.AJMP, b.Succs[1].Block()) 990 } else { 991 p = s.Br(jmp.invasm, b.Succs[1].Block()) 992 s.Br(obj.AJMP, b.Succs[0].Block()) 993 } 994 } 995 if !b.Control.Type.IsFlags() { 996 p.From.Type = obj.TYPE_REG 997 p.From.Reg = b.Control.Reg() 998 } 999 case ssa.BlockARM64TBZ, ssa.BlockARM64TBNZ: 1000 jmp := blockJump[b.Kind] 1001 var p *obj.Prog 1002 switch next { 1003 case b.Succs[0].Block(): 1004 p = s.Br(jmp.invasm, b.Succs[1].Block()) 1005 case b.Succs[1].Block(): 1006 p = s.Br(jmp.asm, b.Succs[0].Block()) 1007 default: 1008 if b.Likely != ssa.BranchUnlikely { 1009 p = s.Br(jmp.asm, b.Succs[0].Block()) 1010 s.Br(obj.AJMP, b.Succs[1].Block()) 1011 } else { 1012 p = s.Br(jmp.invasm, b.Succs[1].Block()) 1013 s.Br(obj.AJMP, b.Succs[0].Block()) 1014 } 1015 } 1016 p.From.Offset = b.Aux.(int64) 1017 p.From.Type = obj.TYPE_CONST 1018 p.Reg = b.Control.Reg() 1019 1020 default: 1021 b.Fatalf("branch not implemented: %s. Control: %s", b.LongString(), b.Control.LongString()) 1022 } 1023 }