[MachineLICM][AArch64] Hoist COPY instructions with other uses in the loop (#71403)

When there is a COPY instruction in the loop with other uses, we want to
hoist the COPY, which in turn leads to the users being hoisted as well.

Co-authored-by David Green : [email protected]

Author: Rin

llvm/lib/CodeGen/MachineLICM.cpp

@@ -1262,6 +1262,18 @@ bool MachineLICMBase::IsProfitableToHoist(MachineInstr &MI,
     return false;
   }
 
+  // If we have a COPY with other uses in the loop, hoist to allow the users to
+  // also be hoisted.
+  if (MI.isCopy() && MI.getOperand(0).isReg() &&
+      MI.getOperand(0).getReg().isVirtual() && MI.getOperand(1).isReg() &&
+      MI.getOperand(1).getReg().isVirtual() &&
+      IsLoopInvariantInst(MI, CurLoop) &&
+      any_of(MRI->use_nodbg_instructions(MI.getOperand(0).getReg()),
+             [&CurLoop](MachineInstr &UseMI) {
+               return CurLoop->contains(&UseMI);
+             }))
+    return true;
+
   // High register pressure situation, only hoist if the instruction is going
   // to be remat'ed.
   if (!isTriviallyReMaterializable(MI) &&

llvm/test/CodeGen/AArch64/tbl-loops.ll

@@ -52,19 +52,19 @@ define void @loop1(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n
 ; CHECK-NEXT:    b.eq .LBB0_8
 ; CHECK-NEXT:  .LBB0_6: // %for.body.preheader1
 ; CHECK-NEXT:    movi d0, #0000000000000000
-; CHECK-NEXT:    sub w10, w2, w10
 ; CHECK-NEXT:    mov w11, #1132396544 // =0x437f0000
+; CHECK-NEXT:    sub w10, w2, w10
+; CHECK-NEXT:    fmov s1, w11
 ; CHECK-NEXT:  .LBB0_7: // %for.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    fmov s2, w11
-; CHECK-NEXT:    ldr s1, [x8], #4
-; CHECK-NEXT:    fcmp s1, s2
-; CHECK-NEXT:    fcsel s2, s2, s1, gt
-; CHECK-NEXT:    fcmp s1, #0.0
-; CHECK-NEXT:    fcsel s1, s0, s2, mi
+; CHECK-NEXT:    ldr s2, [x8], #4
+; CHECK-NEXT:    fcmp s2, s1
+; CHECK-NEXT:    fcsel s3, s1, s2, gt
+; CHECK-NEXT:    fcmp s2, #0.0
+; CHECK-NEXT:    fcsel s2, s0, s3, mi
 ; CHECK-NEXT:    subs w10, w10, #1
-; CHECK-NEXT:    fcvtzs w12, s1
-; CHECK-NEXT:    strb w12, [x9], #1
+; CHECK-NEXT:    fcvtzs w11, s2
+; CHECK-NEXT:    strb w11, [x9], #1
 ; CHECK-NEXT:    b.ne .LBB0_7
 ; CHECK-NEXT:  .LBB0_8: // %for.cond.cleanup
 ; CHECK-NEXT:    ret
@@ -165,25 +165,25 @@ define void @loop2(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n
 ; CHECK-NEXT:    mov x9, x0
 ; CHECK-NEXT:  .LBB1_5: // %for.body.preheader1
 ; CHECK-NEXT:    movi d0, #0000000000000000
-; CHECK-NEXT:    sub w10, w2, w10
 ; CHECK-NEXT:    mov w11, #1132396544 // =0x437f0000
+; CHECK-NEXT:    sub w10, w2, w10
+; CHECK-NEXT:    fmov s1, w11
 ; CHECK-NEXT:  .LBB1_6: // %for.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldp s1, s3, [x8], #8
-; CHECK-NEXT:    fmov s2, w11
-; CHECK-NEXT:    fcmp s1, s2
-; CHECK-NEXT:    fcsel s4, s2, s1, gt
-; CHECK-NEXT:    fcmp s1, #0.0
-; CHECK-NEXT:    fcsel s1, s0, s4, mi
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    fcsel s2, s2, s3, gt
+; CHECK-NEXT:    ldp s2, s3, [x8], #8
+; CHECK-NEXT:    fcmp s2, s1
+; CHECK-NEXT:    fcsel s4, s1, s2, gt
+; CHECK-NEXT:    fcmp s2, #0.0
+; CHECK-NEXT:    fcsel s2, s0, s4, mi
+; CHECK-NEXT:    fcmp s3, s1
+; CHECK-NEXT:    fcsel s4, s1, s3, gt
 ; CHECK-NEXT:    fcmp s3, #0.0
-; CHECK-NEXT:    fcvtzs w12, s1
-; CHECK-NEXT:    fcsel s2, s0, s2, mi
+; CHECK-NEXT:    fcvtzs w11, s2
+; CHECK-NEXT:    fcsel s3, s0, s4, mi
 ; CHECK-NEXT:    subs w10, w10, #1
-; CHECK-NEXT:    strb w12, [x9]
-; CHECK-NEXT:    fcvtzs w13, s2
-; CHECK-NEXT:    strb w13, [x9, #1]
+; CHECK-NEXT:    strb w11, [x9]
+; CHECK-NEXT:    fcvtzs w12, s3
+; CHECK-NEXT:    strb w12, [x9, #1]
 ; CHECK-NEXT:    add x9, x9, #2
 ; CHECK-NEXT:    b.ne .LBB1_6
 ; CHECK-NEXT:  .LBB1_7: // %for.cond.cleanup
@@ -380,33 +380,33 @@ define void @loop3(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n
 ; CHECK-NEXT:    mov x9, x0
 ; CHECK-NEXT:  .LBB2_7: // %for.body.preheader1
 ; CHECK-NEXT:    movi d0, #0000000000000000
-; CHECK-NEXT:    sub w10, w2, w10
 ; CHECK-NEXT:    mov w11, #1132396544 // =0x437f0000
+; CHECK-NEXT:    sub w10, w2, w10
+; CHECK-NEXT:    fmov s1, w11
 ; CHECK-NEXT:  .LBB2_8: // %for.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldp s1, s3, [x8]
-; CHECK-NEXT:    fmov s2, w11
-; CHECK-NEXT:    fcmp s1, s2
-; CHECK-NEXT:    fcsel s4, s2, s1, gt
-; CHECK-NEXT:    fcmp s1, #0.0
-; CHECK-NEXT:    fcsel s1, s0, s4, mi
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    fcsel s4, s2, s3, gt
+; CHECK-NEXT:    ldp s2, s3, [x8]
+; CHECK-NEXT:    fcmp s2, s1
+; CHECK-NEXT:    fcsel s4, s1, s2, gt
+; CHECK-NEXT:    fcmp s2, #0.0
+; CHECK-NEXT:    fcsel s2, s0, s4, mi
+; CHECK-NEXT:    fcmp s3, s1
+; CHECK-NEXT:    fcsel s4, s1, s3, gt
 ; CHECK-NEXT:    fcmp s3, #0.0
 ; CHECK-NEXT:    ldr s3, [x8, #8]
-; CHECK-NEXT:    fcvtzs w12, s1
+; CHECK-NEXT:    fcvtzs w11, s2
 ; CHECK-NEXT:    add x8, x8, #12
 ; CHECK-NEXT:    fcsel s4, s0, s4, mi
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    strb w12, [x9]
-; CHECK-NEXT:    fcsel s2, s2, s3, gt
+; CHECK-NEXT:    fcmp s3, s1
+; CHECK-NEXT:    strb w11, [x9]
+; CHECK-NEXT:    fcsel s5, s1, s3, gt
 ; CHECK-NEXT:    fcmp s3, #0.0
-; CHECK-NEXT:    fcvtzs w13, s4
-; CHECK-NEXT:    fcsel s2, s0, s2, mi
+; CHECK-NEXT:    fcvtzs w12, s4
+; CHECK-NEXT:    fcsel s3, s0, s5, mi
 ; CHECK-NEXT:    subs w10, w10, #1
-; CHECK-NEXT:    strb w13, [x9, #1]
-; CHECK-NEXT:    fcvtzs w14, s2
-; CHECK-NEXT:    strb w14, [x9, #2]
+; CHECK-NEXT:    strb w12, [x9, #1]
+; CHECK-NEXT:    fcvtzs w13, s3
+; CHECK-NEXT:    strb w13, [x9, #2]
 ; CHECK-NEXT:    add x9, x9, #3
 ; CHECK-NEXT:    b.ne .LBB2_8
 ; CHECK-NEXT:  .LBB2_9: // %for.cond.cleanup
@@ -549,39 +549,39 @@ define void @loop4(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n
 ; CHECK-NEXT:    mov x9, x0
 ; CHECK-NEXT:  .LBB3_5: // %for.body.preheader1
 ; CHECK-NEXT:    movi d0, #0000000000000000
-; CHECK-NEXT:    sub w10, w2, w10
 ; CHECK-NEXT:    mov w11, #1132396544 // =0x437f0000
+; CHECK-NEXT:    sub w10, w2, w10
+; CHECK-NEXT:    fmov s1, w11
 ; CHECK-NEXT:  .LBB3_6: // %for.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldp s1, s3, [x8]
-; CHECK-NEXT:    fmov s2, w11
-; CHECK-NEXT:    fcmp s1, s2
-; CHECK-NEXT:    fcsel s4, s2, s1, gt
-; CHECK-NEXT:    fcmp s1, #0.0
-; CHECK-NEXT:    fcsel s1, s0, s4, mi
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    fcsel s4, s2, s3, gt
+; CHECK-NEXT:    ldp s2, s3, [x8]
+; CHECK-NEXT:    fcmp s2, s1
+; CHECK-NEXT:    fcsel s4, s1, s2, gt
+; CHECK-NEXT:    fcmp s2, #0.0
+; CHECK-NEXT:    fcsel s2, s0, s4, mi
+; CHECK-NEXT:    fcmp s3, s1
+; CHECK-NEXT:    fcsel s4, s1, s3, gt
 ; CHECK-NEXT:    fcmp s3, #0.0
 ; CHECK-NEXT:    ldp s3, s5, [x8, #8]
-; CHECK-NEXT:    fcvtzs w12, s1
+; CHECK-NEXT:    fcvtzs w11, s2
 ; CHECK-NEXT:    add x8, x8, #16
 ; CHECK-NEXT:    fcsel s4, s0, s4, mi
-; CHECK-NEXT:    fcmp s3, s2
-; CHECK-NEXT:    strb w12, [x9]
-; CHECK-NEXT:    fcsel s6, s2, s3, gt
+; CHECK-NEXT:    fcmp s3, s1
+; CHECK-NEXT:    strb w11, [x9]
+; CHECK-NEXT:    fcsel s6, s1, s3, gt
 ; CHECK-NEXT:    fcmp s3, #0.0
-; CHECK-NEXT:    fcvtzs w13, s4
+; CHECK-NEXT:    fcvtzs w12, s4
 ; CHECK-NEXT:    fcsel s3, s0, s6, mi
-; CHECK-NEXT:    fcmp s5, s2
-; CHECK-NEXT:    strb w13, [x9, #1]
-; CHECK-NEXT:    fcsel s2, s2, s5, gt
+; CHECK-NEXT:    fcmp s5, s1
+; CHECK-NEXT:    strb w12, [x9, #1]
+; CHECK-NEXT:    fcsel s6, s1, s5, gt
 ; CHECK-NEXT:    fcmp s5, #0.0
-; CHECK-NEXT:    fcvtzs w14, s3
-; CHECK-NEXT:    fcsel s2, s0, s2, mi
+; CHECK-NEXT:    fcvtzs w13, s3
+; CHECK-NEXT:    fcsel s5, s0, s6, mi
 ; CHECK-NEXT:    subs w10, w10, #1
-; CHECK-NEXT:    strb w14, [x9, #2]
-; CHECK-NEXT:    fcvtzs w15, s2
-; CHECK-NEXT:    strb w15, [x9, #3]
+; CHECK-NEXT:    strb w13, [x9, #2]
+; CHECK-NEXT:    fcvtzs w14, s5
+; CHECK-NEXT:    strb w14, [x9, #3]
 ; CHECK-NEXT:    add x9, x9, #4
 ; CHECK-NEXT:    b.ne .LBB3_6
 ; CHECK-NEXT:  .LBB3_7: // %for.cond.cleanup

llvm/test/CodeGen/AArch64/zext-to-tbl.ll

@@ -2756,37 +2756,39 @@ exit:
 define i32 @test_pr62620_widening_instr(ptr %p1, ptr %p2, i64 %lx, i32 %h) {
 ; CHECK-LABEL: test_pr62620_widening_instr:
 ; CHECK:       ; %bb.0: ; %entry
-; CHECK-NEXT:    lsl x8, x2, #4
-; CHECK-NEXT:    ldr q0, [x0, x8]
-; CHECK-NEXT:    ldr q1, [x1, x8]
+; CHECK-NEXT:    lsl x9, x2, #4
+; CHECK-NEXT:    mov x8, x0
 ; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    ldr q0, [x8, x9]
+; CHECK-NEXT:    ldr q1, [x1, x9]
+; CHECK-NEXT:    uabdl.8h v2, v0, v1
+; CHECK-NEXT:    uabal2.8h v2, v0, v1
+; CHECK-NEXT:    uaddlv.8h s0, v2
+; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:  LBB23_1: ; %loop
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    uabdl.8h v2, v0, v1
 ; CHECK-NEXT:    subs w3, w3, #1
-; CHECK-NEXT:    uabal2.8h v2, v0, v1
-; CHECK-NEXT:    uaddlv.8h s2, v2
-; CHECK-NEXT:    fmov w8, s2
 ; CHECK-NEXT:    add w0, w8, w0
 ; CHECK-NEXT:    b.ne LBB23_1
 ; CHECK-NEXT:  ; %bb.2: ; %exit
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-BE-LABEL: test_pr62620_widening_instr:
 ; CHECK-BE:       // %bb.0: // %entry
-; CHECK-BE-NEXT:    lsl x8, x2, #4
-; CHECK-BE-NEXT:    add x9, x0, x8
-; CHECK-BE-NEXT:    add x8, x1, x8
+; CHECK-BE-NEXT:    lsl x9, x2, #4
+; CHECK-BE-NEXT:    mov x8, x0
 ; CHECK-BE-NEXT:    mov w0, wzr
-; CHECK-BE-NEXT:    ld1 { v0.16b }, [x9]
-; CHECK-BE-NEXT:    ld1 { v1.16b }, [x8]
+; CHECK-BE-NEXT:    add x8, x8, x9
+; CHECK-BE-NEXT:    add x9, x1, x9
+; CHECK-BE-NEXT:    ld1 { v0.16b }, [x8]
+; CHECK-BE-NEXT:    ld1 { v1.16b }, [x9]
+; CHECK-BE-NEXT:    uabdl v2.8h, v0.8b, v1.8b
+; CHECK-BE-NEXT:    uabal2 v2.8h, v0.16b, v1.16b
+; CHECK-BE-NEXT:    uaddlv s0, v2.8h
+; CHECK-BE-NEXT:    fmov w8, s0
 ; CHECK-BE-NEXT:  .LBB23_1: // %loop
 ; CHECK-BE-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-BE-NEXT:    uabdl v2.8h, v0.8b, v1.8b
 ; CHECK-BE-NEXT:    subs w3, w3, #1
-; CHECK-BE-NEXT:    uabal2 v2.8h, v0.16b, v1.16b
-; CHECK-BE-NEXT:    uaddlv s2, v2.8h
-; CHECK-BE-NEXT:    fmov w8, s2
 ; CHECK-BE-NEXT:    add w0, w8, w0
 ; CHECK-BE-NEXT:    b.ne .LBB23_1
 ; CHECK-BE-NEXT:  // %bb.2: // %exit