RegAllocFast: Rewrite and improve

This rewrites big parts of the fast register allocator. The basic
strategy of doing block-local allocation hasn't changed but I tweaked
several details:

Track register state on register units instead of physical
registers. This simplifies and speeds up handling of register aliases.
Process basic blocks in reverse order: Definitions are known to end
register livetimes when walking backwards (contrary when walking
forward then uses may or may not be a kill so we need heuristics).

Check register mask operands (calls) instead of conservatively
assuming everything is clobbered.  Enhance heuristics to detect
killing uses: In case of a small number of defs/uses check if they are
all in the same basic block and if so the last one is a killing use.
Enhance heuristic for copy-coalescing through hinting: We check the
first k defs of a register for COPYs rather than relying on there just
being a single definition.  When testing this on the full llvm
test-suite including SPEC externals I measured:

average 5.1% reduction in code size for X86, 4.9% reduction in code on
aarch64. (ranging between 0% and 20% depending on the test) 0.5%
faster compiletime (some analysis suggests the pass is slightly slower
than before, but we more than make up for it because later passes are
faster with the reduced instruction count)

Also adds a few testcases that were broken without this patch, in
particular bug 47278.

Patch mostly by Matthias Braun

Author: arsenm

llvm/lib/CodeGen/RegAllocFast.cpp

@@ -31,9 +31,8 @@ target triple = "arm64-apple-ios13.0.0"
 ; This test checks that we don't re-use the register for the variable descriptor
 ; for the second ldr.
 ; CHECK:        adrp	x[[PTR1:[0-9]+]], _t_val@TLVPPAGE
-; CHECK:	ldr	x[[PTR1]], [x[[PTR1]], _t_val@TLVPPAGEOFF]
-; CHECK:	ldr	x[[FPTR:[0-9]+]], [x[[PTR1]]]
-; CHECK:        mov	x0, x[[PTR1]]
+; CHECK:	ldr	x0, [x[[PTR1]], _t_val@TLVPPAGEOFF]
+; CHECK:	ldr	x[[FPTR:[0-9]+]], [x0]
 ; CHECK:        blr     x[[FPTR]]
 
 define void @_Z4funcPKc(i8* %id) {

llvm/test/CodeGen/AArch64/GlobalISel/darwin-tls-call-clobber.ll

@@ -94,7 +94,7 @@ entry:
   store i32 %c, i32* %c.addr, align 4
   store i64 %d, i64* %d.addr, align 8
   %0 = load i16, i16* %b.addr, align 2
-; CHECK: tbz w8, #0, LBB4_2
+; CHECK: tbz {{w[0-9]+}}, #0, LBB4_2
   %conv = trunc i16 %0 to i1
   br i1 %conv, label %if.then, label %if.end
 

llvm/test/CodeGen/AArch64/arm64-fast-isel-br.ll

@@ -79,8 +79,7 @@ declare i32 @bar(i8 zeroext, i8 zeroext, i8 zeroext, i8 zeroext, i8 zeroext, i8
 define i32 @t2() {
 entry:
 ; CHECK-LABEL: t2
-; CHECK:       mov [[REG1:x[0-9]+]], xzr
-; CHECK:       mov x0, [[REG1]]
+; CHECK:       mov x0, xzr
 ; CHECK:       mov w1, #-8
 ; CHECK:       mov [[REG2:w[0-9]+]], #1023
 ; CHECK:       uxth w2, [[REG2]]

llvm/test/CodeGen/AArch64/arm64-fast-isel-call.ll

@@ -4,9 +4,8 @@
 define i32 @fptosi_wh(half %a) nounwind ssp {
 entry:
 ; CHECK-LABEL: fptosi_wh
-; CHECK: fcvt s0, h0
-; CHECK: fcvtzs [[REG:w[0-9]+]], s0
-; CHECK: mov w0, [[REG]]
+; CHECK: fcvt [[REG:s[0-9]+]], h0
+; CHECK: fcvtzs w0, [[REG]]
   %conv = fptosi half %a to i32
   ret i32 %conv
 }
@@ -15,9 +14,8 @@ entry:
 define i32 @fptoui_swh(half %a) nounwind ssp {
 entry:
 ; CHECK-LABEL: fptoui_swh
-; CHECK: fcvt s0, h0
-; CHECK: fcvtzu [[REG:w[0-9]+]], s0
-; CHECK: mov w0, [[REG]]
+; CHECK: fcvt [[REG:s[0-9]+]], h0
+; CHECK: fcvtzu w0, [[REG]]
   %conv = fptoui half %a to i32
   ret i32 %conv
 }
@@ -26,8 +24,8 @@ entry:
 define half @sitofp_hw_i1(i1 %a) nounwind ssp {
 entry:
 ; CHECK-LABEL: sitofp_hw_i1
-; CHECK: sbfx w8, w0, #0, #1
-; CHECK: scvtf s0, w8
+; CHECK: sbfx [[REG:w[0-9]+]], w0, #0, #1
+; CHECK: scvtf s0, [[REG]]
 ; CHECK: fcvt  h0, s0
   %conv = sitofp i1 %a to half
   ret half %conv
@@ -37,8 +35,8 @@ entry:
 define half @sitofp_hw_i8(i8 %a) nounwind ssp {
 entry:
 ; CHECK-LABEL: sitofp_hw_i8
-; CHECK: sxtb w8, w0
-; CHECK: scvtf s0, w8
+; CHECK: sxtb [[REG:w[0-9]+]], w0
+; CHECK: scvtf s0, [[REG]]
 ; CHECK: fcvt  h0, s0
   %conv = sitofp i8 %a to half
   ret half %conv
@@ -48,8 +46,8 @@ entry:
 define half @sitofp_hw_i16(i16 %a) nounwind ssp {
 entry:
 ; CHECK-LABEL: sitofp_hw_i16
-; CHECK: sxth w8, w0
-; CHECK: scvtf s0, w8
+; CHECK: sxth [[REG:w[0-9]+]], w0
+; CHECK: scvtf s0, [[REG]]
 ; CHECK: fcvt  h0, s0
   %conv = sitofp i16 %a to half
   ret half %conv
@@ -79,8 +77,8 @@ entry:
 define half @uitofp_hw_i1(i1 %a) nounwind ssp {
 entry:
 ; CHECK-LABEL: uitofp_hw_i1
-; CHECK: and w8, w0, #0x1
-; CHECK: ucvtf s0, w8
+; CHECK: and [[REG:w[0-9]+]], w0, #0x1
+; CHECK: ucvtf s0, [[REG]]
 ; CHECK: fcvt  h0, s0
   %conv = uitofp i1 %a to half
   ret half %conv
@@ -90,8 +88,8 @@ entry:
 define half @uitofp_hw_i8(i8 %a) nounwind ssp {
 entry:
 ; CHECK-LABEL: uitofp_hw_i8
-; CHECK: and w8, w0, #0xff
-; CHECK: ucvtf s0, w8
+; CHECK: and [[REG:w[0-9]+]], w0, #0xff
+; CHECK: ucvtf s0, [[REG]]
 ; CHECK: fcvt  h0, s0
   %conv = uitofp i8 %a to half
   ret half %conv
@@ -101,8 +99,8 @@ entry:
 define half @uitofp_hw_i16(i16 %a) nounwind ssp {
 entry:
 ; CHECK-LABEL: uitofp_hw_i16
-; CHECK: and w8, w0, #0xffff
-; CHECK: ucvtf s0, w8
+; CHECK: and [[REG:w[0-9]+]], w0, #0xffff
+; CHECK: ucvtf s0, [[REG]]
 ; CHECK: fcvt  h0, s0
   %conv = uitofp i16 %a to half
   ret half %conv

llvm/test/CodeGen/AArch64/arm64-fast-isel-conversion-fallback.ll

@@ -1,4 +1,4 @@
-; RUN: llc -O0 -fast-isel -fast-isel-abort=1 -verify-machineinstrs -mtriple=arm64-apple-darwin -mcpu=cyclone < %s | FileCheck %s
+; RUN: llc -O0 -fast-isel -fast-isel-abort=1 -verify-machineinstrs -mtriple=arm64-apple-darwin -mcpu=cyclone < %s | FileCheck -enable-var-scope %s
 
 ;; Test various conversions.
 define zeroext i32 @trunc_(i8 zeroext %a, i16 zeroext %b, i32 %c, i64 %d) nounwind ssp {
@@ -49,13 +49,12 @@ entry:
 ; CHECK: strh w1, [sp, #12]
 ; CHECK: str w2, [sp, #8]
 ; CHECK: str x3, [sp]
-; CHECK: ldrb w8, [sp, #15]
-; CHECK: strh w8, [sp, #12]
-; CHECK: ldrh w8, [sp, #12]
-; CHECK: str w8, [sp, #8]
-; CHECK: ldr w8, [sp, #8]
-; CHECK: ; kill: def $x8 killed $w8
-; CHECK: str x8, [sp]
+; CHECK: ldrb [[REG0:w[0-9]+]], [sp, #15]
+; CHECK: strh [[REG0]], [sp, #12]
+; CHECK: ldrh [[REG1:w[0-9]+]], [sp, #12]
+; CHECK: str [[REG1]], [sp, #8]
+; CHECK: ldr w[[REG2:[0-9]+]], [sp, #8]
+; CHECK: str x[[REG2]], [sp]
 ; CHECK: ldr x0, [sp]
 ; CHECK: ret
   %a.addr = alloca i8, align 1
@@ -105,12 +104,12 @@ entry:
 ; CHECK: strh w1, [sp, #12]
 ; CHECK: str w2, [sp, #8]
 ; CHECK: str x3, [sp]
-; CHECK: ldrsb w8, [sp, #15]
-; CHECK: strh w8, [sp, #12]
-; CHECK: ldrsh w8, [sp, #12]
-; CHECK: str w8, [sp, #8]
-; CHECK: ldrsw x8, [sp, #8]
-; CHECK: str x8, [sp]
+; CHECK: ldrsb [[REG0:w[0-9]+]], [sp, #15]
+; CHECK: strh [[REG0]], [sp, #12]
+; CHECK: ldrsh [[REG1:w[0-9]+]], [sp, #12]
+; CHECK: str [[REG1]], [sp, #8]
+; CHECK: ldrsw [[REG2:x[0-9]+]], [sp, #8]
+; CHECK: str [[REG2]], [sp]
 ; CHECK: ldr x0, [sp]
 ; CHECK: ret
   %a.addr = alloca i8, align 1
@@ -166,8 +165,8 @@ entry:
 define signext i16 @sext_i1_i16(i1 %a) nounwind ssp {
 entry:
 ; CHECK-LABEL: sext_i1_i16
-; CHECK: sbfx w8, w0, #0, #1
-; CHECK-NEXT: sxth	w0, w8
+; CHECK: sbfx [[REG:w[0-9]+]], w0, #0, #1
+; CHECK: sxth w0, [[REG]]
   %conv = sext i1 %a to i16
   ret i16 %conv
 }
@@ -176,8 +175,8 @@ entry:
 define signext i8 @sext_i1_i8(i1 %a) nounwind ssp {
 entry:
 ; CHECK-LABEL: sext_i1_i8
-; CHECK: sbfx w8, w0, #0, #1
-; CHECK-NEXT: sxtb w0, w8
+; CHECK: sbfx [[REG:w[0-9]+]], w0, #0, #1
+; CHECK: sxtb w0, [[REG]]
   %conv = sext i1 %a to i8
   ret i8 %conv
 }
@@ -240,8 +239,8 @@ entry:
 define float @sitofp_sw_i1(i1 %a) nounwind ssp {
 entry:
 ; CHECK-LABEL: sitofp_sw_i1
-; CHECK: sbfx w8, w0, #0, #1
-; CHECK: scvtf s0, w8
+; CHECK: sbfx [[REG:w[0-9]+]], w0, #0, #1
+; CHECK: scvtf s0, [[REG]]
   %conv = sitofp i1 %a to float
   ret float %conv
 }
@@ -250,8 +249,8 @@ entry:
 define float @sitofp_sw_i8(i8 %a) nounwind ssp {
 entry:
 ; CHECK-LABEL: sitofp_sw_i8
-; CHECK: sxtb w8, w0
-; CHECK: scvtf s0, w8
+; CHECK: sxtb [[REG:w[0-9]+]], w0
+; CHECK: scvtf s0, [[REG]]
   %conv = sitofp i8 %a to float
   ret float %conv
 }
@@ -304,8 +303,8 @@ entry:
 define float @uitofp_sw_i1(i1 %a) nounwind ssp {
 entry:
 ; CHECK-LABEL: uitofp_sw_i1
-; CHECK: and w8, w0, #0x1
-; CHECK: ucvtf s0, w8
+; CHECK: and [[REG:w[0-9]+]], w0, #0x1
+; CHECK: ucvtf s0, [[REG]]
   %conv = uitofp i1 %a to float
   ret float %conv
 }
@@ -374,7 +373,8 @@ entry:
 define zeroext i16 @i64_trunc_i16(i64 %a) nounwind ssp {
 entry:
 ; CHECK-LABEL: i64_trunc_i16
-; CHECK: and [[REG2:w[0-9]+]], w0, #0xffff
+; CHECK: mov x[[TMP:[0-9]+]], x0
+; CHECK: and [[REG2:w[0-9]+]], w[[TMP]], #0xffff{{$}}
 ; CHECK: uxth w0, [[REG2]]
   %conv = trunc i64 %a to i16
   ret i16 %conv
@@ -383,7 +383,8 @@ entry:
 define zeroext i8 @i64_trunc_i8(i64 %a) nounwind ssp {
 entry:
 ; CHECK-LABEL: i64_trunc_i8
-; CHECK: and [[REG2:w[0-9]+]], w0, #0xff
+; CHECK: mov x[[TMP:[0-9]+]], x0
+; CHECK: and [[REG2:w[0-9]+]], w[[TMP]], #0xff{{$}}
 ; CHECK: uxtb w0, [[REG2]]
   %conv = trunc i64 %a to i8
   ret i8 %conv
@@ -392,7 +393,8 @@ entry:
 define zeroext i1 @i64_trunc_i1(i64 %a) nounwind ssp {
 entry:
 ; CHECK-LABEL: i64_trunc_i1
-; CHECK: and [[REG2:w[0-9]+]], w0, #0x1
+; CHECK: mov x[[TMP:[0-9]+]], x0
+; CHECK: and [[REG2:w[0-9]+]], w[[TMP]], #0x1{{$}}
 ; CHECK: and w0, [[REG2]], #0x1
   %conv = trunc i64 %a to i1
   ret i1 %conv

llvm/test/CodeGen/AArch64/arm64-fast-isel-conversion.ll

@@ -210,10 +210,10 @@ define <4 x float> @test_vcvt_high_f32_f64(<2 x float> %x, <2 x double> %v) noun
 ;
 ; FAST-LABEL: test_vcvt_high_f32_f64:
 ; FAST:       // %bb.0:
-; FAST-NEXT:    // implicit-def: $q2
 ; FAST-NEXT:    mov.16b v2, v0
-; FAST-NEXT:    fcvtn2 v2.4s, v1.2d
+; FAST-NEXT:    // implicit-def: $q0
 ; FAST-NEXT:    mov.16b v0, v2
+; FAST-NEXT:    fcvtn2 v0.4s, v1.2d
 ; FAST-NEXT:    ret
 ;
 ; GISEL-LABEL: test_vcvt_high_f32_f64:
@@ -249,10 +249,10 @@ define <4 x float> @test_vcvtx_high_f32_f64(<2 x float> %x, <2 x double> %v) nou
 ;
 ; FAST-LABEL: test_vcvtx_high_f32_f64:
 ; FAST:       // %bb.0:
-; FAST-NEXT:    // implicit-def: $q2
 ; FAST-NEXT:    mov.16b v2, v0
-; FAST-NEXT:    fcvtxn2 v2.4s, v1.2d
+; FAST-NEXT:    // implicit-def: $q0
 ; FAST-NEXT:    mov.16b v0, v2
+; FAST-NEXT:    fcvtxn2 v0.4s, v1.2d
 ; FAST-NEXT:    ret
 ;
 ; GISEL-LABEL: test_vcvtx_high_f32_f64:
@@ -283,17 +283,12 @@ define i16 @to_half(float %in) {
 ;
 ; FAST-LABEL: to_half:
 ; FAST:       // %bb.0:
-; FAST-NEXT:    sub sp, sp, #16 // =16
-; FAST-NEXT:    .cfi_def_cfa_offset 16
-; FAST-NEXT:    fcvt h0, s0
+; FAST-NEXT:    fcvt h1, s0
 ; FAST-NEXT:    // implicit-def: $w0
-; FAST-NEXT:    fmov s1, w0
-; FAST-NEXT:    mov.16b v1, v0
-; FAST-NEXT:    fmov w8, s1
-; FAST-NEXT:    mov w0, w8
-; FAST-NEXT:    str w0, [sp, #12] // 4-byte Folded Spill
-; FAST-NEXT:    mov w0, w8
-; FAST-NEXT:    add sp, sp, #16 // =16
+; FAST-NEXT:    fmov s0, w0
+; FAST-NEXT:    mov.16b v0, v1
+; FAST-NEXT:    fmov w0, s0
+; FAST-NEXT:    // kill: def $w1 killed $w0
 ; FAST-NEXT:    ret
 ;
 ; GISEL-LABEL: to_half:

llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll

@@ -17,8 +17,9 @@ declare [2 x i32] @callee()
 define void @test_struct_return(i32* %addr) {
 ; CHECK-LABEL: test_struct_return:
 ; CHECK: bl _callee
-; CHECK-DAG: lsr [[HI:x[0-9]+]], x0, #32
-; CHECK-DAG: str w0
+; CHECK: x[[COPYX0:[0-9]+]], x0
+; CHECK-DAG: lsr [[HI:x[0-9]+]], x[[COPYX0]], #32
+; CHECK-DAG: str w[[COPYX0]]
   %res = call [2 x i32] @callee()
   %res.0 = extractvalue [2 x i32] %res, 0
   store i32 %res.0, i32* %addr

llvm/test/CodeGen/AArch64/arm64_32-fastisel.ll

@@ -13,11 +13,12 @@ define void @test_store(i8** %p) {
 define void @test_phi(i8** %p) {
 ; CHECK-LABEL: test_phi:
 ; CHECK: mov [[R1:x[0-9]+]], xzr
-; CHECK: str [[R1]], [sp]
+; CHECK: str [[R1]], [sp, #8]
 ; CHECK: b [[BB:LBB[0-9_]+]]
 ; CHECK: [[BB]]:
-; CHECK: ldr x0, [sp]
-; CHECK: str w0, [x{{.*}}]
+; CHECK: ldr x0, [sp, #8]
+; CHECK: mov w8, w0
+; CHECK: str w8, [x{{.*}}]
 
 bb0:
   br label %bb1

llvm/test/CodeGen/AArch64/arm64_32-null.ll

@@ -64,9 +64,9 @@ bb3:
 ; OPT: b.gt [[L:\.LBB[0-9_]+]]
 ; OPT: tbz w1, #0, [[L]]
 ;
+; NOOPT: str w1, [sp, #[[SLOT2:[0-9]+]]]
 ; NOOPT: subs w{{[0-9]+}}, w{{[0-9]+}}, #0
 ; NOOPT: cset [[R1:w[0-9]+]], gt
-; NOOPT: str w1, [sp, #[[SLOT2:[0-9]+]]]
 ; NOOPT: str [[R1]], [sp, #[[SLOT1:[0-9]+]]]
 ; NOOPT: b .LBB
 ; NOOPT: ldr [[R2:w[0-9]+]], [sp, #[[SLOT1]]]