From 1eb04d289a6fb4c7cb75f69312a7b949987e7c97 Mon Sep 17 00:00:00 2001 From: Serguei Katkov Date: Thu, 31 Oct 2019 17:18:29 +0700 Subject: [PATCH] [LICM] Invalidate SCEV upon instruction hoisting Since SCEV can cache information about location of an instruction, it should be invalidated when the instruction is moved. There should be similar bug in code sinking part of LICM, it will be fixed in a follow-up change. Patch Author: Daniil Suchkov Reviewers: asbirlea, mkazantsev, reames Reviewed By: asbirlea Subscribers: hiraditya, javed.absar, llvm-commits Differential Revision: https://reviews.llvm.org/D69370 --- .../include/llvm/Transforms/Utils/LoopUtils.h | 2 +- llvm/lib/Transforms/Scalar/LICM.cpp | 34 +++++++++++-------- llvm/unittests/Transforms/Scalar/LICMTest.cpp | 6 ++-- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h index 63cc4484dd5c..9ed96809ed99 100644 --- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h +++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h @@ -133,7 +133,7 @@ bool sinkRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *, /// ORE. It returns changed status. bool hoistRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *, TargetLibraryInfo *, Loop *, AliasSetTracker *, - MemorySSAUpdater *, ICFLoopSafetyInfo *, + MemorySSAUpdater *, ScalarEvolution *, ICFLoopSafetyInfo *, SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *); /// This function deletes dead loops. The caller of this function needs to diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp index 6ce4831a7359..70868ed8c187 100644 --- a/llvm/lib/Transforms/Scalar/LICM.cpp +++ b/llvm/lib/Transforms/Scalar/LICM.cpp @@ -137,7 +137,8 @@ static bool isNotUsedOrFreeInLoop(const Instruction &I, const Loop *CurLoop, TargetTransformInfo *TTI, bool &FreeInLoop); static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop, BasicBlock *Dest, ICFLoopSafetyInfo *SafetyInfo, - MemorySSAUpdater *MSSAU, OptimizationRemarkEmitter *ORE); + MemorySSAUpdater *MSSAU, ScalarEvolution *SE, + OptimizationRemarkEmitter *ORE); static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT, const Loop *CurLoop, ICFLoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU, OptimizationRemarkEmitter *ORE); @@ -162,7 +163,7 @@ static void eraseInstruction(Instruction &I, ICFLoopSafetyInfo &SafetyInfo, static void moveInstructionBefore(Instruction &I, Instruction &Dest, ICFLoopSafetyInfo &SafetyInfo, - MemorySSAUpdater *MSSAU); + MemorySSAUpdater *MSSAU, ScalarEvolution *SE); namespace { struct LoopInvariantCodeMotion { @@ -390,8 +391,9 @@ bool LoopInvariantCodeMotion::runOnLoop( CurAST.get(), MSSAU.get(), &SafetyInfo, Flags, ORE); Flags.IsSink = false; if (Preheader) - Changed |= hoistRegion(DT->getNode(L->getHeader()), AA, LI, DT, TLI, L, - CurAST.get(), MSSAU.get(), &SafetyInfo, Flags, ORE); + Changed |= + hoistRegion(DT->getNode(L->getHeader()), AA, LI, DT, TLI, L, + CurAST.get(), MSSAU.get(), SE, &SafetyInfo, Flags, ORE); // Now that all loop invariants have been removed from the loop, promote any // memory references to scalars that we can. @@ -795,7 +797,7 @@ public: bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI, DominatorTree *DT, TargetLibraryInfo *TLI, Loop *CurLoop, AliasSetTracker *CurAST, MemorySSAUpdater *MSSAU, - ICFLoopSafetyInfo *SafetyInfo, + ScalarEvolution *SE, ICFLoopSafetyInfo *SafetyInfo, SinkAndHoistLICMFlags &Flags, OptimizationRemarkEmitter *ORE) { // Verify inputs. @@ -855,7 +857,7 @@ bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI, I, DT, CurLoop, SafetyInfo, ORE, CurLoop->getLoopPreheader()->getTerminator())) { hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo, - MSSAU, ORE); + MSSAU, SE, ORE); HoistedInstructions.push_back(&I); Changed = true; continue; @@ -882,7 +884,7 @@ bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI, eraseInstruction(I, *SafetyInfo, CurAST, MSSAU); hoist(*ReciprocalDivisor, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), - SafetyInfo, MSSAU, ORE); + SafetyInfo, MSSAU, SE, ORE); HoistedInstructions.push_back(ReciprocalDivisor); Changed = true; continue; @@ -901,7 +903,7 @@ bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI, CurLoop->hasLoopInvariantOperands(&I) && MustExecuteWithoutWritesBefore(I)) { hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo, - MSSAU, ORE); + MSSAU, SE, ORE); HoistedInstructions.push_back(&I); Changed = true; continue; @@ -915,7 +917,7 @@ bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI, PN->setIncomingBlock( i, CFH.getOrCreateHoistedBlock(PN->getIncomingBlock(i))); hoist(*PN, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo, - MSSAU, ORE); + MSSAU, SE, ORE); assert(DT->dominates(PN, BB) && "Conditional PHIs not expected"); Changed = true; continue; @@ -952,7 +954,7 @@ bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI, LLVM_DEBUG(dbgs() << "LICM rehoisting to " << HoistPoint->getParent()->getName() << ": " << *I << "\n"); - moveInstructionBefore(*I, *HoistPoint, *SafetyInfo, MSSAU); + moveInstructionBefore(*I, *HoistPoint, *SafetyInfo, MSSAU, SE); HoistPoint = I; Changed = true; } @@ -1441,7 +1443,8 @@ static void eraseInstruction(Instruction &I, ICFLoopSafetyInfo &SafetyInfo, static void moveInstructionBefore(Instruction &I, Instruction &Dest, ICFLoopSafetyInfo &SafetyInfo, - MemorySSAUpdater *MSSAU) { + MemorySSAUpdater *MSSAU, + ScalarEvolution *SE) { SafetyInfo.removeInstruction(&I); SafetyInfo.insertInstructionTo(&I, Dest.getParent()); I.moveBefore(&Dest); @@ -1449,6 +1452,8 @@ static void moveInstructionBefore(Instruction &I, Instruction &Dest, if (MemoryUseOrDef *OldMemAcc = cast_or_null( MSSAU->getMemorySSA()->getMemoryAccess(&I))) MSSAU->moveToPlace(OldMemAcc, Dest.getParent(), MemorySSA::End); + if (SE) + SE->forgetValue(&I); } static Instruction *sinkThroughTriviallyReplaceablePHI( @@ -1662,7 +1667,8 @@ static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT, /// static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop, BasicBlock *Dest, ICFLoopSafetyInfo *SafetyInfo, - MemorySSAUpdater *MSSAU, OptimizationRemarkEmitter *ORE) { + MemorySSAUpdater *MSSAU, ScalarEvolution *SE, + OptimizationRemarkEmitter *ORE) { LLVM_DEBUG(dbgs() << "LICM hoisting to " << Dest->getName() << ": " << I << "\n"); ORE->emit([&]() { @@ -1683,10 +1689,10 @@ static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop, if (isa(I)) // Move the new node to the end of the phi list in the destination block. - moveInstructionBefore(I, *Dest->getFirstNonPHI(), *SafetyInfo, MSSAU); + moveInstructionBefore(I, *Dest->getFirstNonPHI(), *SafetyInfo, MSSAU, SE); else // Move the new node to the destination block, before its terminator. - moveInstructionBefore(I, *Dest->getTerminator(), *SafetyInfo, MSSAU); + moveInstructionBefore(I, *Dest->getTerminator(), *SafetyInfo, MSSAU, SE); // Apply line 0 debug locations when we are moving instructions to different // basic blocks because we want to avoid jumpy line tables. diff --git a/llvm/unittests/Transforms/Scalar/LICMTest.cpp b/llvm/unittests/Transforms/Scalar/LICMTest.cpp index 91a9b10815a3..3063785332bb 100644 --- a/llvm/unittests/Transforms/Scalar/LICMTest.cpp +++ b/llvm/unittests/Transforms/Scalar/LICMTest.cpp @@ -86,10 +86,8 @@ TEST(LICMTest, TestSCEVInvalidationOnHoisting) { // If LICM have properly invalidated SCEV, // 1. SCEV of should properly dominate the "loop" BB, // 2. extra invalidation shouldn't change result of the query. - // FIXME: these values should be equal! - EXPECT_NE(DispositionBeforeInvalidation, + EXPECT_EQ(DispositionBeforeInvalidation, ScalarEvolution::BlockDisposition::ProperlyDominatesBlock); - // FIXME: these values should be equal! - EXPECT_NE(DispositionBeforeInvalidation, DispositionAfterInvalidation); + EXPECT_EQ(DispositionBeforeInvalidation, DispositionAfterInvalidation); } }