diff --git a/polly/lib/Transform/ForwardOpTree.cpp b/polly/lib/Transform/ForwardOpTree.cpp index 067d3f634298..7fc0d5a3774e 100644 --- a/polly/lib/Transform/ForwardOpTree.cpp +++ b/polly/lib/Transform/ForwardOpTree.cpp @@ -64,6 +64,7 @@ STATISTIC(KnownOutOfQuota, STATISTIC(TotalInstructionsCopied, "Number of copied instructions"); STATISTIC(TotalKnownLoadsForwarded, "Number of forwarded loads because their value was known"); +STATISTIC(TotalReloads, "Number of reloaded values"); STATISTIC(TotalReadOnlyCopied, "Number of copied read-only accesses"); STATISTIC(TotalForwardedTrees, "Number of forwarded operand trees"); STATISTIC(TotalModifiedStmts, @@ -106,12 +107,21 @@ enum ForwardingDecision { /// and can be used anywhere) into the same statement as %add would. FD_CanForwardLeaf, - /// The root instruction can be forwarded in a non-trivial way. This requires - /// the operand tree root to be an instruction in some statement. - FD_CanForwardTree, + /// The root instruction can be forwarded and doing so avoids a scalar + /// dependency. + /// + /// This can be either because the operand tree can be moved to the target + /// statement, or a memory access is redirected to read from a different + /// location. + FD_CanForwardProfitably, - /// Used to indicate that a forwarding has be carried out successfully. - FD_DidForward, + /// Used to indicate that a forwarding has be carried out successfully, and + /// the forwarded memory access can be deleted. + FD_DidForwardTree, + + /// Used to indicate that a forwarding has be carried out successfully, and + /// the forwarded memory access is being reused. + FD_DidForwardLeaf, /// A forwarding method cannot be applied to the operand tree. /// The difference to FD_CannotForward is that there might be other methods @@ -140,6 +150,9 @@ private: /// Number of loads forwarded because their value was known. int NumKnownLoadsForwarded = 0; + /// Number of values reloaded from known array elements. + int NumReloads = 0; + /// How many read-only accesses have been copied. int NumReadOnlyCopied = 0; @@ -293,6 +306,7 @@ public: << '\n'; OS.indent(Indent + 4) << "Known loads forwarded: " << NumKnownLoadsForwarded << '\n'; + OS.indent(Indent + 4) << "Reloads: " << NumReloads<< '\n'; OS.indent(Indent + 4) << "Read-only accesses copied: " << NumReadOnlyCopied << '\n'; OS.indent(Indent + 4) << "Operand trees forwarded: " << NumForwardedTrees @@ -391,12 +405,11 @@ public: /// use DoIt==true if an operand tree is not known to be /// forwardable. /// - /// @return FD_NotApplicable if @p Inst is not a LoadInst. - /// FD_CannotForward if no array element to load from was found. - /// FD_CanForwardLeaf if the load is already in the target statement - /// instance. - /// FD_CanForwardTree if @p Inst is forwardable. - /// FD_DidForward if @p DoIt was true. + /// @return FD_NotApplicable if @p Inst cannot be forwarded by creating a new + /// load. + /// FD_CannotForward if the pointer operand cannot be forwarded. + /// FD_CanForwardProfitably if @p Inst is forwardable. + /// FD_DidForwardTree if @p DoIt was true. ForwardingDecision forwardKnownLoad(ScopStmt *TargetStmt, Instruction *Inst, ScopStmt *UseStmt, Loop *UseLoop, isl::map UseToTarget, ScopStmt *DefStmt, @@ -421,10 +434,7 @@ public: // do not add another MemoryAccess. MemoryAccess *Access = TargetStmt->getArrayAccessOrNULLFor(LI); if (Access && !DoIt) - return FD_CanForwardTree; - - if (DoIt) - TargetStmt->prependInstruction(LI); + return FD_CanForwardProfitably; ForwardingDecision OpDecision = forwardTree(TargetStmt, LI->getPointerOperand(), DefStmt, DefLoop, @@ -435,11 +445,12 @@ public: return OpDecision; case FD_CanForwardLeaf: - case FD_CanForwardTree: + case FD_CanForwardProfitably: assert(!DoIt); break; - case FD_DidForward: + case FD_DidForwardLeaf: + case FD_DidForwardTree: assert(DoIt); break; @@ -462,10 +473,13 @@ public: isl::map SameVal = singleLocation(Candidates, getDomainFor(TargetStmt)); if (!SameVal) - return FD_CannotForward; + return FD_NotApplicable; + + if (DoIt) + TargetStmt->prependInstruction(LI); if (!DoIt) - return FD_CanForwardTree; + return FD_CanForwardProfitably; if (Access) { DEBUG(dbgs() << " forwarded known load with preexisting MemoryAccess" @@ -519,7 +533,77 @@ public: NumKnownLoadsForwarded++; TotalKnownLoadsForwarded++; - return FD_DidForward; + return FD_DidForwardTree; + } + + /// Forward a scalar by redirecting the access to an array element that stores + /// the same value. + /// + /// @param TargetStmt The statement the operand tree will be copied to. + /// @param Inst The scalar to forward. + /// @param UseStmt The statement that uses @p Inst. + /// @param UseLoop The loop @p Inst is used in. + /// @param UseToTarget { DomainUse[] -> DomainTarget[] } + /// A mapping from the statement instance @p Inst is used + /// in, to the statement instance it is forwarded to. + /// @param DefStmt The statement @p Inst is defined in. + /// @param DefLoop The loop which contains @p Inst. + /// @param DefToTarget { DomainDef[] -> DomainTarget[] } + /// A mapping from the statement instance @p Inst is + /// defined in, to the statement instance it is forwarded + /// to. + /// @param DoIt If false, only determine whether an operand tree can be + /// forwarded. If true, carry out the forwarding. Do not + /// use DoIt==true if an operand tree is not known to be + /// forwardable. + /// + /// @return FD_NotApplicable if @p Inst cannot be reloaded. + /// FD_CanForwardLeaf if @p Inst can be reloaded. + /// FD_CanForwardProfitably if @p Inst has been reloaded. + /// FD_DidForwardLeaf if @p DoIt was true. + ForwardingDecision reloadKnownContent(ScopStmt *TargetStmt, Instruction *Inst, + ScopStmt *UseStmt, Loop *UseLoop, + isl::map UseToTarget, ScopStmt *DefStmt, + Loop *DefLoop, isl::map DefToTarget, + bool DoIt) { + // Cannot do anything without successful known analysis. + if (Known.is_null()) + return FD_NotApplicable; + + MemoryAccess *Access = TargetStmt->lookupInputAccessOf(Inst); + if (Access && Access->isLatestArrayKind()) { + if (DoIt) + return FD_DidForwardLeaf; + return FD_CanForwardLeaf; + } + + // { DomainDef[] -> ValInst[] } + isl::union_map ExpectedVal = makeValInst(Inst, UseStmt, UseLoop); + + // { DomainTarget[] -> ValInst[] } + isl::union_map TargetExpectedVal = ExpectedVal.apply_domain(UseToTarget); + isl::union_map TranslatedExpectedVal = + TargetExpectedVal.apply_range(Translator); + + // { DomainTarget[] -> Element[] } + isl::union_map Candidates = findSameContentElements(TranslatedExpectedVal); + + isl::map SameVal = singleLocation(Candidates, getDomainFor(TargetStmt)); + if (!SameVal) + return FD_NotApplicable; + + if (!DoIt) + return FD_CanForwardProfitably; + + if (!Access) + Access = TargetStmt->ensureValueRead(Inst); + + simplify(SameVal); + Access->setNewAccessRelation(SameVal); + + TotalReloads++; + NumReloads++; + return FD_DidForwardLeaf; } /// Forwards a speculatively executable instruction. @@ -584,11 +668,12 @@ public: return FD_CannotForward; case FD_CanForwardLeaf: - case FD_CanForwardTree: + case FD_CanForwardProfitably: assert(!DoIt); break; - case FD_DidForward: + case FD_DidForwardLeaf: + case FD_DidForwardTree: assert(DoIt); break; @@ -598,8 +683,8 @@ public: } if (DoIt) - return FD_DidForward; - return FD_CanForwardTree; + return FD_DidForwardTree; + return FD_CanForwardProfitably; } /// Determines whether an operand tree can be forwarded or carries out a @@ -636,14 +721,14 @@ public: case VirtualUse::Hoisted: // These can be used anywhere without special considerations. if (DoIt) - return FD_DidForward; + return FD_DidForwardTree; return FD_CanForwardLeaf; case VirtualUse::Synthesizable: { // ScopExpander will take care for of generating the code at the new // location. if (DoIt) - return FD_DidForward; + return FD_DidForwardTree; // Check if the value is synthesizable at the new location as well. This // might be possible when leaving a loop for which ScalarEvolution is @@ -682,7 +767,7 @@ public: NumReadOnlyCopied++; TotalReadOnlyCopied++; - return FD_DidForward; + return FD_DidForwardLeaf; case VirtualUse::Intra: // Knowing that UseStmt and DefStmt are the same statement instance, just @@ -725,6 +810,12 @@ public: if (KnownResult != FD_NotApplicable) return KnownResult; + ForwardingDecision ReloadResult = + reloadKnownContent(TargetStmt, Inst, UseStmt, UseLoop, UseToTarget, + DefStmt, DefLoop, DefToTarget, DoIt); + if (ReloadResult != FD_NotApplicable) + return ReloadResult; + // When no method is found to forward the operand tree, we effectively // cannot handle it. DEBUG(dbgs() << " Cannot forward instruction: " << *Inst << "\n"); @@ -751,17 +842,18 @@ public: ForwardingDecision Assessment = forwardTree( Stmt, RA->getAccessValue(), Stmt, InLoop, TargetToUse, false); - assert(Assessment != FD_DidForward); - if (Assessment != FD_CanForwardTree) + assert(Assessment != FD_DidForwardTree && Assessment != FD_DidForwardLeaf); + if (Assessment != FD_CanForwardProfitably) return false; ForwardingDecision Execution = forwardTree(Stmt, RA->getAccessValue(), Stmt, InLoop, TargetToUse, true); - assert(Execution == FD_DidForward && + assert(((Execution == FD_DidForwardTree) || + (Execution == FD_DidForwardLeaf)) && "A previous positive assessment must also be executable"); - (void)Execution; - Stmt->removeSingleMemoryAccess(RA); + if (Execution == FD_DidForwardTree) + Stmt->removeSingleMemoryAccess(RA); return true; } diff --git a/polly/test/ForwardOpTree/forward_store.ll b/polly/test/ForwardOpTree/forward_store.ll new file mode 100644 index 000000000000..96a91cf29fdb --- /dev/null +++ b/polly/test/ForwardOpTree/forward_store.ll @@ -0,0 +1,73 @@ +; RUN: opt %loadPolly -polly-optree -analyze < %s | FileCheck %s -match-full-lines +; +; Rematerialize a load. +; +; for (int j = 0; j < n; j += 1) { +; bodyA: +; double val = B[j]; +; +; bodyB: +; A[j] = val; +; } +; + +declare double @f() #0 + +define void @func(i32 %n, double* noalias nonnull %A, double* noalias nonnull %B) { +entry: + br label %for + +for: + %j = phi i32 [0, %entry], [%j.inc, %inc] + %j.cmp = icmp slt i32 %j, %n + br i1 %j.cmp, label %bodyA, label %exit + + bodyA: + %val = call double @f() + %A_idx = getelementptr inbounds double, double* %A, i32 %j + store double %val, double* %A_idx + br label %bodyB + + bodyB: + %B_idx = getelementptr inbounds double, double* %B, i32 %j + store double %val, double* %B_idx + br label %inc + +inc: + %j.inc = add nuw nsw i32 %j, 1 + br label %for + +exit: + br label %return + +return: + ret void +} + +attributes #0 = { nounwind readnone } + + +; CHECK: Statistics { +; CHECK: Reloads: 1 +; CHECK: } + +; CHECK: After statements { +; CHECK-NEXT: Stmt_bodyA +; CHECK-NEXT: MustWriteAccess := [Reduction Type: NONE] [Scalar: 0] +; CHECK-NEXT: [n] -> { Stmt_bodyA[i0] -> MemRef_A[i0] }; +; CHECK-NEXT: MustWriteAccess := [Reduction Type: NONE] [Scalar: 1] +; CHECK-NEXT: [n] -> { Stmt_bodyA[i0] -> MemRef_val[] }; +; CHECK-NEXT: Instructions { +; CHECK-NEXT: %val = call double @f() +; CHECK-NEXT: store double %val, double* %A_idx +; CHECK-NEXT: } +; CHECK-NEXT: Stmt_bodyB +; CHECK-NEXT: MustWriteAccess := [Reduction Type: NONE] [Scalar: 0] +; CHECK-NEXT: [n] -> { Stmt_bodyB[i0] -> MemRef_B[i0] }; +; CHECK-NEXT: ReadAccess := [Reduction Type: NONE] [Scalar: 1] +; CHECK-NEXT: [n] -> { Stmt_bodyB[i0] -> MemRef_val[] }; +; CHECK-NEXT: new: [n] -> { Stmt_bodyB[i0] -> MemRef_A[i0] }; +; CHECK-NEXT: Instructions { +; CHECK-NEXT: store double %val, double* %B_idx +; CHECK-NEXT: } +; CHECK-NEXT: }