From 03892a27d6b89df92def3239338cafbbfa541dbd Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Wed, 24 Feb 2021 08:46:15 +0000
Subject: [PATCH] [ARM] Expand the range of allowed post-incs in load/store
 optimizer

Currently the load/store optimizer will only fold in increments of the
same size as the load/store. This patch expands that to any legal
immediate for the post-inc instruction.

This is a recommit of 3b34b06fc5908b with correctness fixes and extra
tests.

Differential Revision: https://reviews.llvm.org/D95885
---
 llvm/lib/Target/ARM/ARMBaseInstrInfo.h        |  4 +++
 llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp | 25 +++++++++++--------
 llvm/test/CodeGen/ARM/store-prepostinc.mir    | 21 ++++++----------
 .../CodeGen/Thumb2/mve-float32regloops.ll     |  3 +--
 llvm/test/CodeGen/Thumb2/store-prepostinc.mir | 21 ++++++----------
 5 files changed, 34 insertions(+), 40 deletions(-)

diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
index 1b843c428130..c0d94b675c24 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
@@ -888,8 +888,12 @@ inline bool isLegalAddressImm(unsigned Opcode, int Imm,
     return std::abs(Imm) < (((1 << 7) * 4) - 1) && Imm % 4 == 0;
   case ARMII::AddrModeT2_i8:
     return std::abs(Imm) < (((1 << 8) * 1) - 1);
+  case ARMII::AddrMode2:
+    return std::abs(Imm) < (((1 << 12) * 1) - 1);
   case ARMII::AddrModeT2_i12:
     return Imm >= 0 && Imm < (((1 << 12) * 1) - 1);
+  case ARMII::AddrModeT2_i8s4:
+    return std::abs(Imm) < (((1 << 8) * 4) - 1) && Imm % 4 == 0;
   default:
     llvm_unreachable("Unhandled Addressing mode");
   }
diff --git a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
index 5fe61809f31b..28fe01c4a579 100644
--- a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
@@ -1502,12 +1502,16 @@ bool ARMLoadStoreOpt::MergeBaseUpdateLoadStore(MachineInstr *MI) {
     NewOpc = getPreIndexedLoadStoreOpcode(Opcode, ARM_AM::sub);
   } else {
     MergeInstr = findIncDecAfter(MBBI, Base, Pred, PredReg, Offset, TRI);
-    if (Offset == Bytes) {
-      NewOpc = getPostIndexedLoadStoreOpcode(Opcode, ARM_AM::add);
-    } else if (!isAM5 && Offset == -Bytes) {
-      NewOpc = getPostIndexedLoadStoreOpcode(Opcode, ARM_AM::sub);
-    } else
+    if (MergeInstr == MBB.end())
       return false;
+
+    NewOpc = getPostIndexedLoadStoreOpcode(Opcode, ARM_AM::add);
+    if ((isAM5 && Offset != Bytes) ||
+        (!isAM5 && !isLegalAddressImm(NewOpc, Offset, TII))) {
+      NewOpc = getPostIndexedLoadStoreOpcode(Opcode, ARM_AM::sub);
+      if (isAM5 || !isLegalAddressImm(NewOpc, Offset, TII))
+        return false;
+    }
   }
   LLVM_DEBUG(dbgs() << "  Erasing old increment: " << *MergeInstr);
   MBB.erase(MergeInstr);
@@ -1546,7 +1550,7 @@ bool ARMLoadStoreOpt::MergeBaseUpdateLoadStore(MachineInstr *MI) {
         (void)MIB;
         LLVM_DEBUG(dbgs() << "  Added new instruction: " << *MIB);
       } else {
-        int Imm = ARM_AM::getAM2Opc(AddSub, Bytes, ARM_AM::no_shift);
+        int Imm = ARM_AM::getAM2Opc(AddSub, abs(Offset), ARM_AM::no_shift);
         auto MIB =
             BuildMI(MBB, MBBI, DL, TII->get(NewOpc), MI->getOperand(0).getReg())
                 .addReg(Base, RegState::Define)
@@ -1576,7 +1580,7 @@ bool ARMLoadStoreOpt::MergeBaseUpdateLoadStore(MachineInstr *MI) {
     // the vestigal zero-reg offset register. When that's fixed, this clause
     // can be removed entirely.
     if (isAM2 && NewOpc == ARM::STR_POST_IMM) {
-      int Imm = ARM_AM::getAM2Opc(AddSub, Bytes, ARM_AM::no_shift);
+      int Imm = ARM_AM::getAM2Opc(AddSub, abs(Offset), ARM_AM::no_shift);
       // STR_PRE, STR_POST
       auto MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc), Base)
                      .addReg(MO.getReg(), getKillRegState(MO.isKill()))
@@ -1633,9 +1637,10 @@ bool ARMLoadStoreOpt::MergeBaseUpdateLSDouble(MachineInstr &MI) const {
     NewOpc = Opcode == ARM::t2LDRDi8 ? ARM::t2LDRD_PRE : ARM::t2STRD_PRE;
   } else {
     MergeInstr = findIncDecAfter(MBBI, Base, Pred, PredReg, Offset, TRI);
-    if (Offset == 8 || Offset == -8) {
-      NewOpc = Opcode == ARM::t2LDRDi8 ? ARM::t2LDRD_POST : ARM::t2STRD_POST;
-    } else
+    if (MergeInstr == MBB.end())
+      return false;
+    NewOpc = Opcode == ARM::t2LDRDi8 ? ARM::t2LDRD_POST : ARM::t2STRD_POST;
+    if (!isLegalAddressImm(NewOpc, Offset, TII))
       return false;
   }
   LLVM_DEBUG(dbgs() << "  Erasing old increment: " << *MergeInstr);
diff --git a/llvm/test/CodeGen/ARM/store-prepostinc.mir b/llvm/test/CodeGen/ARM/store-prepostinc.mir
index d4e3fe160061..bea3d4ff68a4 100644
--- a/llvm/test/CodeGen/ARM/store-prepostinc.mir
+++ b/llvm/test/CodeGen/ARM/store-prepostinc.mir
@@ -271,8 +271,7 @@ body:             |
 
     ; CHECK-LABEL: name: STR_post8
     ; CHECK: liveins: $r0, $r1
-    ; CHECK: STRi12 killed renamable $r1, renamable $r0, 0, 14 /* CC::al */, $noreg :: (store 4)
-    ; CHECK: renamable $r0 = nuw ADDri killed renamable $r0, 8, 14 /* CC::al */, $noreg, $noreg
+    ; CHECK: early-clobber $r0 = STR_POST_IMM killed $r1, $r0, $noreg, 8, 14 /* CC::al */, $noreg :: (store 4)
     ; CHECK: BX_RET 14 /* CC::al */, $noreg, implicit $r0
     STRi12 killed renamable $r1, renamable $r0, 0, 14 /* CC::al */, $noreg :: (store 4)
     renamable $r0 = nuw ADDri killed renamable $r0, 8, 14 /* CC::al */, $noreg, $noreg
@@ -292,8 +291,7 @@ body:             |
 
     ; CHECK-LABEL: name: STR_post255
     ; CHECK: liveins: $r0, $r1
-    ; CHECK: STRi12 killed renamable $r1, renamable $r0, 0, 14 /* CC::al */, $noreg :: (store 4)
-    ; CHECK: renamable $r0 = nuw ADDri killed renamable $r0, 255, 14 /* CC::al */, $noreg, $noreg
+    ; CHECK: early-clobber $r0 = STR_POST_IMM killed $r1, $r0, $noreg, 255, 14 /* CC::al */, $noreg :: (store 4)
     ; CHECK: BX_RET 14 /* CC::al */, $noreg, implicit $r0
     STRi12 killed renamable $r1, renamable $r0, 0, 14 /* CC::al */, $noreg :: (store 4)
     renamable $r0 = nuw ADDri killed renamable $r0, 255, 14 /* CC::al */, $noreg, $noreg
@@ -313,8 +311,7 @@ body:             |
 
     ; CHECK-LABEL: name: STR_post256
     ; CHECK: liveins: $r0, $r1
-    ; CHECK: STRi12 killed renamable $r1, renamable $r0, 0, 14 /* CC::al */, $noreg :: (store 4)
-    ; CHECK: renamable $r0 = nuw ADDri killed renamable $r0, 256, 14 /* CC::al */, $noreg, $noreg
+    ; CHECK: early-clobber $r0 = STR_POST_IMM killed $r1, $r0, $noreg, 256, 14 /* CC::al */, $noreg :: (store 4)
     ; CHECK: BX_RET 14 /* CC::al */, $noreg, implicit $r0
     STRi12 killed renamable $r1, renamable $r0, 0, 14 /* CC::al */, $noreg :: (store 4)
     renamable $r0 = nuw ADDri killed renamable $r0, 256, 14 /* CC::al */, $noreg, $noreg
@@ -334,8 +331,7 @@ body:             |
 
     ; CHECK-LABEL: name: STR_post1024
     ; CHECK: liveins: $r0, $r1
-    ; CHECK: STRi12 killed renamable $r1, renamable $r0, 0, 14 /* CC::al */, $noreg :: (store 4)
-    ; CHECK: renamable $r0 = nuw ADDri killed renamable $r0, 1024, 14 /* CC::al */, $noreg, $noreg
+    ; CHECK: early-clobber $r0 = STR_POST_IMM killed $r1, $r0, $noreg, 1024, 14 /* CC::al */, $noreg :: (store 4)
     ; CHECK: BX_RET 14 /* CC::al */, $noreg, implicit $r0
     STRi12 killed renamable $r1, renamable $r0, 0, 14 /* CC::al */, $noreg :: (store 4)
     renamable $r0 = nuw ADDri killed renamable $r0, 1024, 14 /* CC::al */, $noreg, $noreg
@@ -355,8 +351,7 @@ body:             |
 
     ; CHECK-LABEL: name: STR_post4095
     ; CHECK: liveins: $r0, $r1
-    ; CHECK: STRi12 killed renamable $r1, renamable $r0, 0, 14 /* CC::al */, $noreg :: (store 4)
-    ; CHECK: renamable $r0 = nuw ADDri killed renamable $r0, 2095, 14 /* CC::al */, $noreg, $noreg
+    ; CHECK: early-clobber $r0 = STR_POST_IMM killed $r1, $r0, $noreg, 2095, 14 /* CC::al */, $noreg :: (store 4)
     ; CHECK: BX_RET 14 /* CC::al */, $noreg, implicit $r0
     STRi12 killed renamable $r1, renamable $r0, 0, 14 /* CC::al */, $noreg :: (store 4)
     renamable $r0 = nuw ADDri killed renamable $r0, 2095, 14 /* CC::al */, $noreg, $noreg
@@ -397,8 +392,7 @@ body:             |
 
     ; CHECK-LABEL: name: STR_postm1024
     ; CHECK: liveins: $r0, $r1
-    ; CHECK: STRi12 killed renamable $r1, renamable $r0, 0, 14 /* CC::al */, $noreg :: (store 4)
-    ; CHECK: renamable $r0 = nuw SUBri killed renamable $r0, 1024, 14 /* CC::al */, $noreg, $noreg
+    ; CHECK: early-clobber $r0 = STR_POST_IMM killed $r1, $r0, $noreg, 5120, 14 /* CC::al */, $noreg :: (store 4)
     ; CHECK: BX_RET 14 /* CC::al */, $noreg, implicit $r0
     STRi12 killed renamable $r1, renamable $r0, 0, 14 /* CC::al */, $noreg :: (store 4)
     renamable $r0 = nuw SUBri killed renamable $r0, 1024, 14 /* CC::al */, $noreg, $noreg
@@ -418,8 +412,7 @@ body:             |
 
     ; CHECK-LABEL: name: STR_postm4095
     ; CHECK: liveins: $r0, $r1
-    ; CHECK: STRi12 killed renamable $r1, renamable $r0, 0, 14 /* CC::al */, $noreg :: (store 4)
-    ; CHECK: renamable $r0 = nuw SUBri killed renamable $r0, 2095, 14 /* CC::al */, $noreg, $noreg
+    ; CHECK: early-clobber $r0 = STR_POST_IMM killed $r1, $r0, $noreg, 6191, 14 /* CC::al */, $noreg :: (store 4)
     ; CHECK: BX_RET 14 /* CC::al */, $noreg, implicit $r0
     STRi12 killed renamable $r1, renamable $r0, 0, 14 /* CC::al */, $noreg :: (store 4)
     renamable $r0 = nuw SUBri killed renamable $r0, 2095, 14 /* CC::al */, $noreg, $noreg
diff --git a/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll b/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll
index 7e4603e4b4c6..44a152b32c0b 100644
--- a/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll
@@ -1715,7 +1715,7 @@ define arm_aapcs_vfpcc void @arm_biquad_cascade_df1_f32(%struct.arm_biquad_casd_
 ; CHECK-NEXT:    vmov r3, s10
 ; CHECK-NEXT:    vldrw.u32 q3, [r11, #48]
 ; CHECK-NEXT:    vfma.f32 q1, q0, r3
-; CHECK-NEXT:    ldr r3, [r1]
+; CHECK-NEXT:    ldr r3, [r1], #16
 ; CHECK-NEXT:    vfma.f32 q1, q7, r6
 ; CHECK-NEXT:    vldrw.u32 q6, [r11, #64]
 ; CHECK-NEXT:    vfma.f32 q1, q3, r3
@@ -1725,7 +1725,6 @@ define arm_aapcs_vfpcc void @arm_biquad_cascade_df1_f32(%struct.arm_biquad_casd_
 ; CHECK-NEXT:    vfma.f32 q1, q5, r0
 ; CHECK-NEXT:    vldrw.u32 q0, [sp, #64] @ 16-byte Reload
 ; CHECK-NEXT:    vfma.f32 q1, q4, r7
-; CHECK-NEXT:    adds r1, #16
 ; CHECK-NEXT:    vfma.f32 q1, q0, r9
 ; CHECK-NEXT:    vmov.f32 s2, s8
 ; CHECK-NEXT:    vstrb.8 q1, [r5], #16
diff --git a/llvm/test/CodeGen/Thumb2/store-prepostinc.mir b/llvm/test/CodeGen/Thumb2/store-prepostinc.mir
index 63493d6cc275..b01983eec466 100644
--- a/llvm/test/CodeGen/Thumb2/store-prepostinc.mir
+++ b/llvm/test/CodeGen/Thumb2/store-prepostinc.mir
@@ -421,8 +421,7 @@ body:             |
 
     ; CHECK-LABEL: name: STR_post8
     ; CHECK: liveins: $r0, $r1
-    ; CHECK: t2STRi12 killed renamable $r1, renamable $r0, 0, 14 /* CC::al */, $noreg :: (store 4)
-    ; CHECK: renamable $r0 = nuw t2ADDri killed renamable $r0, 8, 14 /* CC::al */, $noreg, $noreg
+    ; CHECK: early-clobber $r0 = t2STR_POST killed $r1, $r0, 8, 14 /* CC::al */, $noreg :: (store 4)
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     t2STRi12 killed renamable $r1, renamable $r0, 0, 14 /* CC::al */, $noreg :: (store 4)
     renamable $r0 = nuw t2ADDri killed renamable $r0, 8, 14 /* CC::al */, $noreg, $noreg
@@ -485,8 +484,7 @@ body:             |
 
     ; CHECK-LABEL: name: STRD_post4
     ; CHECK: liveins: $r0, $r1, $r2
-    ; CHECK: t2STRDi8 killed $r1, killed $r2, $r0, 0, 14 /* CC::al */, $noreg :: (store 8)
-    ; CHECK: renamable $r0 = nuw t2ADDri killed renamable $r0, 4, 14 /* CC::al */, $noreg, $noreg
+    ; CHECK: $r0 = t2STRD_POST killed renamable $r1, killed renamable $r2, killed $r0, 4, 14 /* CC::al */, $noreg :: (store 8)
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     t2STRDi8 killed renamable $r1, killed renamable $r2, renamable $r0, 0, 14 /* CC::al */, $noreg :: (store 8)
     renamable $r0 = nuw t2ADDri killed renamable $r0, 4, 14 /* CC::al */, $noreg, $noreg
@@ -550,8 +548,7 @@ body:             |
 
     ; CHECK-LABEL: name: STRD_post256
     ; CHECK: liveins: $r0, $r1, $r2
-    ; CHECK: t2STRDi8 killed $r1, killed $r2, $r0, 0, 14 /* CC::al */, $noreg :: (store 8)
-    ; CHECK: renamable $r0 = nuw t2ADDri killed renamable $r0, 256, 14 /* CC::al */, $noreg, $noreg
+    ; CHECK: $r0 = t2STRD_POST killed renamable $r1, killed renamable $r2, killed $r0, 256, 14 /* CC::al */, $noreg :: (store 8)
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     t2STRDi8 killed renamable $r1, killed renamable $r2, renamable $r0, 0, 14 /* CC::al */, $noreg :: (store 8)
     renamable $r0 = nuw t2ADDri killed renamable $r0, 256, 14 /* CC::al */, $noreg, $noreg
@@ -572,8 +569,7 @@ body:             |
 
     ; CHECK-LABEL: name: STRD_post1020
     ; CHECK: liveins: $r0, $r1, $r2
-    ; CHECK: t2STRDi8 killed $r1, killed $r2, $r0, 0, 14 /* CC::al */, $noreg :: (store 8)
-    ; CHECK: renamable $r0 = nuw t2ADDri killed renamable $r0, 1020, 14 /* CC::al */, $noreg, $noreg
+    ; CHECK: $r0 = t2STRD_POST killed renamable $r1, killed renamable $r2, killed $r0, 1020, 14 /* CC::al */, $noreg :: (store 8)
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     t2STRDi8 killed renamable $r1, killed renamable $r2, renamable $r0, 0, 14 /* CC::al */, $noreg :: (store 8)
     renamable $r0 = nuw t2ADDri killed renamable $r0, 1020, 14 /* CC::al */, $noreg, $noreg
@@ -616,8 +612,7 @@ body:             |
 
     ; CHECK-LABEL: name: STRD_postm4
     ; CHECK: liveins: $r0, $r1, $r2
-    ; CHECK: t2STRDi8 killed $r1, killed $r2, $r0, 0, 14 /* CC::al */, $noreg :: (store 8)
-    ; CHECK: renamable $r0 = nuw t2SUBri killed renamable $r0, 4, 14 /* CC::al */, $noreg, $noreg
+    ; CHECK: $r0 = t2STRD_POST killed renamable $r1, killed renamable $r2, killed $r0, -4, 14 /* CC::al */, $noreg :: (store 8)
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     t2STRDi8 killed renamable $r1, killed renamable $r2, renamable $r0, 0, 14 /* CC::al */, $noreg :: (store 8)
     renamable $r0 = nuw t2SUBri killed renamable $r0, 4, 14 /* CC::al */, $noreg, $noreg
@@ -681,8 +676,7 @@ body:             |
 
     ; CHECK-LABEL: name: STRD_postm256
     ; CHECK: liveins: $r0, $r1, $r2
-    ; CHECK: t2STRDi8 killed $r1, killed $r2, $r0, 0, 14 /* CC::al */, $noreg :: (store 8)
-    ; CHECK: renamable $r0 = nuw t2ADDri killed renamable $r0, 256, 14 /* CC::al */, $noreg, $noreg
+    ; CHECK: $r0 = t2STRD_POST killed renamable $r1, killed renamable $r2, killed $r0, 256, 14 /* CC::al */, $noreg :: (store 8)
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     t2STRDi8 killed renamable $r1, killed renamable $r2, renamable $r0, 0, 14 /* CC::al */, $noreg :: (store 8)
     renamable $r0 = nuw t2ADDri killed renamable $r0, 256, 14 /* CC::al */, $noreg, $noreg
@@ -703,8 +697,7 @@ body:             |
 
     ; CHECK-LABEL: name: STRD_postm1020
     ; CHECK: liveins: $r0, $r1, $r2
-    ; CHECK: t2STRDi8 killed $r1, killed $r2, $r0, 0, 14 /* CC::al */, $noreg :: (store 8)
-    ; CHECK: renamable $r0 = nuw t2SUBri killed renamable $r0, 1020, 14 /* CC::al */, $noreg, $noreg
+    ; CHECK: $r0 = t2STRD_POST killed renamable $r1, killed renamable $r2, killed $r0, -1020, 14 /* CC::al */, $noreg :: (store 8)
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $r0
     t2STRDi8 killed renamable $r1, killed renamable $r2, renamable $r0, 0, 14 /* CC::al */, $noreg :: (store 8)
     renamable $r0 = nuw t2SUBri killed renamable $r0, 1020, 14 /* CC::al */, $noreg, $noreg