forked from OSchip/llvm-project
1246 lines
50 KiB
C++
1246 lines
50 KiB
C++
//===- CGSCCPassManager.cpp - Managing & running CGSCC passes -------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "llvm/Analysis/CGSCCPassManager.h"
|
|
#include "llvm/ADT/ArrayRef.h"
|
|
#include "llvm/ADT/Optional.h"
|
|
#include "llvm/ADT/STLExtras.h"
|
|
#include "llvm/ADT/SetVector.h"
|
|
#include "llvm/ADT/SmallPtrSet.h"
|
|
#include "llvm/ADT/SmallVector.h"
|
|
#include "llvm/ADT/iterator_range.h"
|
|
#include "llvm/Analysis/LazyCallGraph.h"
|
|
#include "llvm/IR/Constant.h"
|
|
#include "llvm/IR/InstIterator.h"
|
|
#include "llvm/IR/Instruction.h"
|
|
#include "llvm/IR/PassManager.h"
|
|
#include "llvm/IR/PassManagerImpl.h"
|
|
#include "llvm/IR/ValueHandle.h"
|
|
#include "llvm/Support/Casting.h"
|
|
#include "llvm/Support/CommandLine.h"
|
|
#include "llvm/Support/Debug.h"
|
|
#include "llvm/Support/ErrorHandling.h"
|
|
#include "llvm/Support/TimeProfiler.h"
|
|
#include "llvm/Support/raw_ostream.h"
|
|
#include <algorithm>
|
|
#include <cassert>
|
|
#include <iterator>
|
|
|
|
#define DEBUG_TYPE "cgscc"
|
|
|
|
using namespace llvm;
|
|
|
|
// Explicit template instantiations and specialization definitions for core
|
|
// template typedefs.
|
|
namespace llvm {
|
|
|
|
static cl::opt<bool> AbortOnMaxDevirtIterationsReached(
|
|
"abort-on-max-devirt-iterations-reached",
|
|
cl::desc("Abort when the max iterations for devirtualization CGSCC repeat "
|
|
"pass is reached"));
|
|
|
|
// Explicit instantiations for the core proxy templates.
|
|
template class AllAnalysesOn<LazyCallGraph::SCC>;
|
|
template class AnalysisManager<LazyCallGraph::SCC, LazyCallGraph &>;
|
|
template class PassManager<LazyCallGraph::SCC, CGSCCAnalysisManager,
|
|
LazyCallGraph &, CGSCCUpdateResult &>;
|
|
template class InnerAnalysisManagerProxy<CGSCCAnalysisManager, Module>;
|
|
template class OuterAnalysisManagerProxy<ModuleAnalysisManager,
|
|
LazyCallGraph::SCC, LazyCallGraph &>;
|
|
template class OuterAnalysisManagerProxy<CGSCCAnalysisManager, Function>;
|
|
|
|
/// Explicitly specialize the pass manager run method to handle call graph
|
|
/// updates.
|
|
template <>
|
|
PreservedAnalyses
|
|
PassManager<LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &,
|
|
CGSCCUpdateResult &>::run(LazyCallGraph::SCC &InitialC,
|
|
CGSCCAnalysisManager &AM,
|
|
LazyCallGraph &G, CGSCCUpdateResult &UR) {
|
|
// Request PassInstrumentation from analysis manager, will use it to run
|
|
// instrumenting callbacks for the passes later.
|
|
PassInstrumentation PI =
|
|
AM.getResult<PassInstrumentationAnalysis>(InitialC, G);
|
|
|
|
PreservedAnalyses PA = PreservedAnalyses::all();
|
|
|
|
if (DebugLogging)
|
|
dbgs() << "Starting CGSCC pass manager run.\n";
|
|
|
|
// The SCC may be refined while we are running passes over it, so set up
|
|
// a pointer that we can update.
|
|
LazyCallGraph::SCC *C = &InitialC;
|
|
|
|
// Get Function analysis manager from its proxy.
|
|
FunctionAnalysisManager &FAM =
|
|
AM.getCachedResult<FunctionAnalysisManagerCGSCCProxy>(*C)->getManager();
|
|
|
|
for (auto &Pass : Passes) {
|
|
// Check the PassInstrumentation's BeforePass callbacks before running the
|
|
// pass, skip its execution completely if asked to (callback returns false).
|
|
if (!PI.runBeforePass(*Pass, *C))
|
|
continue;
|
|
|
|
PreservedAnalyses PassPA;
|
|
{
|
|
TimeTraceScope TimeScope(Pass->name());
|
|
PassPA = Pass->run(*C, AM, G, UR);
|
|
}
|
|
|
|
if (UR.InvalidatedSCCs.count(C))
|
|
PI.runAfterPassInvalidated<LazyCallGraph::SCC>(*Pass, PassPA);
|
|
else
|
|
PI.runAfterPass<LazyCallGraph::SCC>(*Pass, *C, PassPA);
|
|
|
|
// Update the SCC if necessary.
|
|
C = UR.UpdatedC ? UR.UpdatedC : C;
|
|
if (UR.UpdatedC) {
|
|
// If C is updated, also create a proxy and update FAM inside the result.
|
|
auto *ResultFAMCP =
|
|
&AM.getResult<FunctionAnalysisManagerCGSCCProxy>(*C, G);
|
|
ResultFAMCP->updateFAM(FAM);
|
|
}
|
|
|
|
// If the CGSCC pass wasn't able to provide a valid updated SCC, the
|
|
// current SCC may simply need to be skipped if invalid.
|
|
if (UR.InvalidatedSCCs.count(C)) {
|
|
LLVM_DEBUG(dbgs() << "Skipping invalidated root or island SCC!\n");
|
|
break;
|
|
}
|
|
// Check that we didn't miss any update scenario.
|
|
assert(C->begin() != C->end() && "Cannot have an empty SCC!");
|
|
|
|
// Update the analysis manager as each pass runs and potentially
|
|
// invalidates analyses.
|
|
AM.invalidate(*C, PassPA);
|
|
|
|
// Finally, we intersect the final preserved analyses to compute the
|
|
// aggregate preserved set for this pass manager.
|
|
PA.intersect(std::move(PassPA));
|
|
|
|
// FIXME: Historically, the pass managers all called the LLVM context's
|
|
// yield function here. We don't have a generic way to acquire the
|
|
// context and it isn't yet clear what the right pattern is for yielding
|
|
// in the new pass manager so it is currently omitted.
|
|
// ...getContext().yield();
|
|
}
|
|
|
|
// Before we mark all of *this* SCC's analyses as preserved below, intersect
|
|
// this with the cross-SCC preserved analysis set. This is used to allow
|
|
// CGSCC passes to mutate ancestor SCCs and still trigger proper invalidation
|
|
// for them.
|
|
UR.CrossSCCPA.intersect(PA);
|
|
|
|
// Invalidation was handled after each pass in the above loop for the current
|
|
// SCC. Therefore, the remaining analysis results in the AnalysisManager are
|
|
// preserved. We mark this with a set so that we don't need to inspect each
|
|
// one individually.
|
|
PA.preserveSet<AllAnalysesOn<LazyCallGraph::SCC>>();
|
|
|
|
if (DebugLogging)
|
|
dbgs() << "Finished CGSCC pass manager run.\n";
|
|
|
|
return PA;
|
|
}
|
|
|
|
PreservedAnalyses
|
|
ModuleToPostOrderCGSCCPassAdaptor::run(Module &M, ModuleAnalysisManager &AM) {
|
|
// Setup the CGSCC analysis manager from its proxy.
|
|
CGSCCAnalysisManager &CGAM =
|
|
AM.getResult<CGSCCAnalysisManagerModuleProxy>(M).getManager();
|
|
|
|
// Get the call graph for this module.
|
|
LazyCallGraph &CG = AM.getResult<LazyCallGraphAnalysis>(M);
|
|
|
|
// Get Function analysis manager from its proxy.
|
|
FunctionAnalysisManager &FAM =
|
|
AM.getCachedResult<FunctionAnalysisManagerModuleProxy>(M)->getManager();
|
|
|
|
// We keep worklists to allow us to push more work onto the pass manager as
|
|
// the passes are run.
|
|
SmallPriorityWorklist<LazyCallGraph::RefSCC *, 1> RCWorklist;
|
|
SmallPriorityWorklist<LazyCallGraph::SCC *, 1> CWorklist;
|
|
|
|
// Keep sets for invalidated SCCs and RefSCCs that should be skipped when
|
|
// iterating off the worklists.
|
|
SmallPtrSet<LazyCallGraph::RefSCC *, 4> InvalidRefSCCSet;
|
|
SmallPtrSet<LazyCallGraph::SCC *, 4> InvalidSCCSet;
|
|
|
|
SmallDenseSet<std::pair<LazyCallGraph::Node *, LazyCallGraph::SCC *>, 4>
|
|
InlinedInternalEdges;
|
|
|
|
CGSCCUpdateResult UR = {
|
|
RCWorklist, CWorklist, InvalidRefSCCSet, InvalidSCCSet,
|
|
nullptr, nullptr, PreservedAnalyses::all(), InlinedInternalEdges,
|
|
{}};
|
|
|
|
// Request PassInstrumentation from analysis manager, will use it to run
|
|
// instrumenting callbacks for the passes later.
|
|
PassInstrumentation PI = AM.getResult<PassInstrumentationAnalysis>(M);
|
|
|
|
PreservedAnalyses PA = PreservedAnalyses::all();
|
|
CG.buildRefSCCs();
|
|
for (auto RCI = CG.postorder_ref_scc_begin(),
|
|
RCE = CG.postorder_ref_scc_end();
|
|
RCI != RCE;) {
|
|
assert(RCWorklist.empty() &&
|
|
"Should always start with an empty RefSCC worklist");
|
|
// The postorder_ref_sccs range we are walking is lazily constructed, so
|
|
// we only push the first one onto the worklist. The worklist allows us
|
|
// to capture *new* RefSCCs created during transformations.
|
|
//
|
|
// We really want to form RefSCCs lazily because that makes them cheaper
|
|
// to update as the program is simplified and allows us to have greater
|
|
// cache locality as forming a RefSCC touches all the parts of all the
|
|
// functions within that RefSCC.
|
|
//
|
|
// We also eagerly increment the iterator to the next position because
|
|
// the CGSCC passes below may delete the current RefSCC.
|
|
RCWorklist.insert(&*RCI++);
|
|
|
|
do {
|
|
LazyCallGraph::RefSCC *RC = RCWorklist.pop_back_val();
|
|
if (InvalidRefSCCSet.count(RC)) {
|
|
LLVM_DEBUG(dbgs() << "Skipping an invalid RefSCC...\n");
|
|
continue;
|
|
}
|
|
|
|
assert(CWorklist.empty() &&
|
|
"Should always start with an empty SCC worklist");
|
|
|
|
LLVM_DEBUG(dbgs() << "Running an SCC pass across the RefSCC: " << *RC
|
|
<< "\n");
|
|
|
|
// The top of the worklist may *also* be the same SCC we just ran over
|
|
// (and invalidated for). Keep track of that last SCC we processed due
|
|
// to SCC update to avoid redundant processing when an SCC is both just
|
|
// updated itself and at the top of the worklist.
|
|
LazyCallGraph::SCC *LastUpdatedC = nullptr;
|
|
|
|
// Push the initial SCCs in reverse post-order as we'll pop off the
|
|
// back and so see this in post-order.
|
|
for (LazyCallGraph::SCC &C : llvm::reverse(*RC))
|
|
CWorklist.insert(&C);
|
|
|
|
do {
|
|
LazyCallGraph::SCC *C = CWorklist.pop_back_val();
|
|
// Due to call graph mutations, we may have invalid SCCs or SCCs from
|
|
// other RefSCCs in the worklist. The invalid ones are dead and the
|
|
// other RefSCCs should be queued above, so we just need to skip both
|
|
// scenarios here.
|
|
if (InvalidSCCSet.count(C)) {
|
|
LLVM_DEBUG(dbgs() << "Skipping an invalid SCC...\n");
|
|
continue;
|
|
}
|
|
if (LastUpdatedC == C) {
|
|
LLVM_DEBUG(dbgs() << "Skipping redundant run on SCC: " << *C << "\n");
|
|
continue;
|
|
}
|
|
if (&C->getOuterRefSCC() != RC) {
|
|
LLVM_DEBUG(dbgs() << "Skipping an SCC that is now part of some other "
|
|
"RefSCC...\n");
|
|
continue;
|
|
}
|
|
|
|
// Ensure we can proxy analysis updates from the CGSCC analysis manager
|
|
// into the the Function analysis manager by getting a proxy here.
|
|
// This also needs to update the FunctionAnalysisManager, as this may be
|
|
// the first time we see this SCC.
|
|
CGAM.getResult<FunctionAnalysisManagerCGSCCProxy>(*C, CG).updateFAM(
|
|
FAM);
|
|
|
|
// Each time we visit a new SCC pulled off the worklist,
|
|
// a transformation of a child SCC may have also modified this parent
|
|
// and invalidated analyses. So we invalidate using the update record's
|
|
// cross-SCC preserved set. This preserved set is intersected by any
|
|
// CGSCC pass that handles invalidation (primarily pass managers) prior
|
|
// to marking its SCC as preserved. That lets us track everything that
|
|
// might need invalidation across SCCs without excessive invalidations
|
|
// on a single SCC.
|
|
//
|
|
// This essentially allows SCC passes to freely invalidate analyses
|
|
// of any ancestor SCC. If this becomes detrimental to successfully
|
|
// caching analyses, we could force each SCC pass to manually
|
|
// invalidate the analyses for any SCCs other than themselves which
|
|
// are mutated. However, that seems to lose the robustness of the
|
|
// pass-manager driven invalidation scheme.
|
|
CGAM.invalidate(*C, UR.CrossSCCPA);
|
|
|
|
do {
|
|
// Check that we didn't miss any update scenario.
|
|
assert(!InvalidSCCSet.count(C) && "Processing an invalid SCC!");
|
|
assert(C->begin() != C->end() && "Cannot have an empty SCC!");
|
|
assert(&C->getOuterRefSCC() == RC &&
|
|
"Processing an SCC in a different RefSCC!");
|
|
|
|
LastUpdatedC = UR.UpdatedC;
|
|
UR.UpdatedRC = nullptr;
|
|
UR.UpdatedC = nullptr;
|
|
|
|
// Check the PassInstrumentation's BeforePass callbacks before
|
|
// running the pass, skip its execution completely if asked to
|
|
// (callback returns false).
|
|
if (!PI.runBeforePass<LazyCallGraph::SCC>(*Pass, *C))
|
|
continue;
|
|
|
|
PreservedAnalyses PassPA;
|
|
{
|
|
TimeTraceScope TimeScope(Pass->name());
|
|
PassPA = Pass->run(*C, CGAM, CG, UR);
|
|
}
|
|
|
|
if (UR.InvalidatedSCCs.count(C))
|
|
PI.runAfterPassInvalidated<LazyCallGraph::SCC>(*Pass, PassPA);
|
|
else
|
|
PI.runAfterPass<LazyCallGraph::SCC>(*Pass, *C, PassPA);
|
|
|
|
// Update the SCC and RefSCC if necessary.
|
|
C = UR.UpdatedC ? UR.UpdatedC : C;
|
|
RC = UR.UpdatedRC ? UR.UpdatedRC : RC;
|
|
|
|
if (UR.UpdatedC) {
|
|
// If we're updating the SCC, also update the FAM inside the proxy's
|
|
// result.
|
|
CGAM.getResult<FunctionAnalysisManagerCGSCCProxy>(*C, CG).updateFAM(
|
|
FAM);
|
|
}
|
|
|
|
// If the CGSCC pass wasn't able to provide a valid updated SCC,
|
|
// the current SCC may simply need to be skipped if invalid.
|
|
if (UR.InvalidatedSCCs.count(C)) {
|
|
LLVM_DEBUG(dbgs() << "Skipping invalidated root or island SCC!\n");
|
|
break;
|
|
}
|
|
// Check that we didn't miss any update scenario.
|
|
assert(C->begin() != C->end() && "Cannot have an empty SCC!");
|
|
|
|
// We handle invalidating the CGSCC analysis manager's information
|
|
// for the (potentially updated) SCC here. Note that any other SCCs
|
|
// whose structure has changed should have been invalidated by
|
|
// whatever was updating the call graph. This SCC gets invalidated
|
|
// late as it contains the nodes that were actively being
|
|
// processed.
|
|
CGAM.invalidate(*C, PassPA);
|
|
|
|
// Then intersect the preserved set so that invalidation of module
|
|
// analyses will eventually occur when the module pass completes.
|
|
// Also intersect with the cross-SCC preserved set to capture any
|
|
// cross-SCC invalidation.
|
|
UR.CrossSCCPA.intersect(PassPA);
|
|
PA.intersect(std::move(PassPA));
|
|
|
|
// The pass may have restructured the call graph and refined the
|
|
// current SCC and/or RefSCC. We need to update our current SCC and
|
|
// RefSCC pointers to follow these. Also, when the current SCC is
|
|
// refined, re-run the SCC pass over the newly refined SCC in order
|
|
// to observe the most precise SCC model available. This inherently
|
|
// cannot cycle excessively as it only happens when we split SCCs
|
|
// apart, at most converging on a DAG of single nodes.
|
|
// FIXME: If we ever start having RefSCC passes, we'll want to
|
|
// iterate there too.
|
|
if (UR.UpdatedC)
|
|
LLVM_DEBUG(dbgs()
|
|
<< "Re-running SCC passes after a refinement of the "
|
|
"current SCC: "
|
|
<< *UR.UpdatedC << "\n");
|
|
|
|
// Note that both `C` and `RC` may at this point refer to deleted,
|
|
// invalid SCC and RefSCCs respectively. But we will short circuit
|
|
// the processing when we check them in the loop above.
|
|
} while (UR.UpdatedC);
|
|
} while (!CWorklist.empty());
|
|
|
|
// We only need to keep internal inlined edge information within
|
|
// a RefSCC, clear it to save on space and let the next time we visit
|
|
// any of these functions have a fresh start.
|
|
InlinedInternalEdges.clear();
|
|
} while (!RCWorklist.empty());
|
|
}
|
|
|
|
// By definition we preserve the call garph, all SCC analyses, and the
|
|
// analysis proxies by handling them above and in any nested pass managers.
|
|
PA.preserveSet<AllAnalysesOn<LazyCallGraph::SCC>>();
|
|
PA.preserve<LazyCallGraphAnalysis>();
|
|
PA.preserve<CGSCCAnalysisManagerModuleProxy>();
|
|
PA.preserve<FunctionAnalysisManagerModuleProxy>();
|
|
return PA;
|
|
}
|
|
|
|
PreservedAnalyses DevirtSCCRepeatedPass::run(LazyCallGraph::SCC &InitialC,
|
|
CGSCCAnalysisManager &AM,
|
|
LazyCallGraph &CG,
|
|
CGSCCUpdateResult &UR) {
|
|
PreservedAnalyses PA = PreservedAnalyses::all();
|
|
PassInstrumentation PI =
|
|
AM.getResult<PassInstrumentationAnalysis>(InitialC, CG);
|
|
|
|
// The SCC may be refined while we are running passes over it, so set up
|
|
// a pointer that we can update.
|
|
LazyCallGraph::SCC *C = &InitialC;
|
|
|
|
// Struct to track the counts of direct and indirect calls in each function
|
|
// of the SCC.
|
|
struct CallCount {
|
|
int Direct;
|
|
int Indirect;
|
|
};
|
|
|
|
// Put value handles on all of the indirect calls and return the number of
|
|
// direct calls for each function in the SCC.
|
|
auto ScanSCC = [](LazyCallGraph::SCC &C,
|
|
SmallMapVector<Value *, WeakTrackingVH, 16> &CallHandles) {
|
|
assert(CallHandles.empty() && "Must start with a clear set of handles.");
|
|
|
|
SmallDenseMap<Function *, CallCount> CallCounts;
|
|
CallCount CountLocal = {0, 0};
|
|
for (LazyCallGraph::Node &N : C) {
|
|
CallCount &Count =
|
|
CallCounts.insert(std::make_pair(&N.getFunction(), CountLocal))
|
|
.first->second;
|
|
for (Instruction &I : instructions(N.getFunction()))
|
|
if (auto *CB = dyn_cast<CallBase>(&I)) {
|
|
if (CB->getCalledFunction()) {
|
|
++Count.Direct;
|
|
} else {
|
|
++Count.Indirect;
|
|
CallHandles.insert({CB, WeakTrackingVH(CB)});
|
|
}
|
|
}
|
|
}
|
|
|
|
return CallCounts;
|
|
};
|
|
|
|
UR.IndirectVHs.clear();
|
|
// Populate the initial call handles and get the initial call counts.
|
|
auto CallCounts = ScanSCC(*C, UR.IndirectVHs);
|
|
|
|
for (int Iteration = 0;; ++Iteration) {
|
|
if (!PI.runBeforePass<LazyCallGraph::SCC>(*Pass, *C))
|
|
continue;
|
|
|
|
PreservedAnalyses PassPA = Pass->run(*C, AM, CG, UR);
|
|
|
|
if (UR.InvalidatedSCCs.count(C))
|
|
PI.runAfterPassInvalidated<LazyCallGraph::SCC>(*Pass, PassPA);
|
|
else
|
|
PI.runAfterPass<LazyCallGraph::SCC>(*Pass, *C, PassPA);
|
|
|
|
// If the SCC structure has changed, bail immediately and let the outer
|
|
// CGSCC layer handle any iteration to reflect the refined structure.
|
|
if (UR.UpdatedC && UR.UpdatedC != C) {
|
|
PA.intersect(std::move(PassPA));
|
|
break;
|
|
}
|
|
|
|
// Check that we didn't miss any update scenario.
|
|
assert(!UR.InvalidatedSCCs.count(C) && "Processing an invalid SCC!");
|
|
assert(C->begin() != C->end() && "Cannot have an empty SCC!");
|
|
|
|
// Check whether any of the handles were devirtualized.
|
|
bool Devirt = llvm::any_of(UR.IndirectVHs, [](auto &P) -> bool {
|
|
if (P.second) {
|
|
if (CallBase *CB = dyn_cast<CallBase>(P.second)) {
|
|
if (CB->getCalledFunction()) {
|
|
LLVM_DEBUG(dbgs() << "Found devirtualized call: " << *CB << "\n");
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
return false;
|
|
});
|
|
|
|
// Rescan to build up a new set of handles and count how many direct
|
|
// calls remain. If we decide to iterate, this also sets up the input to
|
|
// the next iteration.
|
|
UR.IndirectVHs.clear();
|
|
auto NewCallCounts = ScanSCC(*C, UR.IndirectVHs);
|
|
|
|
// If we haven't found an explicit devirtualization already see if we
|
|
// have decreased the number of indirect calls and increased the number
|
|
// of direct calls for any function in the SCC. This can be fooled by all
|
|
// manner of transformations such as DCE and other things, but seems to
|
|
// work well in practice.
|
|
if (!Devirt)
|
|
// Iterate over the keys in NewCallCounts, if Function also exists in
|
|
// CallCounts, make the check below.
|
|
for (auto &Pair : NewCallCounts) {
|
|
auto &CallCountNew = Pair.second;
|
|
auto CountIt = CallCounts.find(Pair.first);
|
|
if (CountIt != CallCounts.end()) {
|
|
const auto &CallCountOld = CountIt->second;
|
|
if (CallCountOld.Indirect > CallCountNew.Indirect &&
|
|
CallCountOld.Direct < CallCountNew.Direct) {
|
|
Devirt = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!Devirt) {
|
|
PA.intersect(std::move(PassPA));
|
|
break;
|
|
}
|
|
|
|
// Otherwise, if we've already hit our max, we're done.
|
|
if (Iteration >= MaxIterations) {
|
|
if (AbortOnMaxDevirtIterationsReached)
|
|
report_fatal_error("Max devirtualization iterations reached");
|
|
LLVM_DEBUG(
|
|
dbgs() << "Found another devirtualization after hitting the max "
|
|
"number of repetitions ("
|
|
<< MaxIterations << ") on SCC: " << *C << "\n");
|
|
PA.intersect(std::move(PassPA));
|
|
break;
|
|
}
|
|
|
|
LLVM_DEBUG(
|
|
dbgs() << "Repeating an SCC pass after finding a devirtualization in: "
|
|
<< *C << "\n");
|
|
|
|
// Move over the new call counts in preparation for iterating.
|
|
CallCounts = std::move(NewCallCounts);
|
|
|
|
// Update the analysis manager with each run and intersect the total set
|
|
// of preserved analyses so we're ready to iterate.
|
|
AM.invalidate(*C, PassPA);
|
|
|
|
PA.intersect(std::move(PassPA));
|
|
}
|
|
|
|
// Note that we don't add any preserved entries here unlike a more normal
|
|
// "pass manager" because we only handle invalidation *between* iterations,
|
|
// not after the last iteration.
|
|
return PA;
|
|
}
|
|
|
|
PreservedAnalyses CGSCCToFunctionPassAdaptor::run(LazyCallGraph::SCC &C,
|
|
CGSCCAnalysisManager &AM,
|
|
LazyCallGraph &CG,
|
|
CGSCCUpdateResult &UR) {
|
|
// Setup the function analysis manager from its proxy.
|
|
FunctionAnalysisManager &FAM =
|
|
AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, CG).getManager();
|
|
|
|
SmallVector<LazyCallGraph::Node *, 4> Nodes;
|
|
for (LazyCallGraph::Node &N : C)
|
|
Nodes.push_back(&N);
|
|
|
|
// The SCC may get split while we are optimizing functions due to deleting
|
|
// edges. If this happens, the current SCC can shift, so keep track of
|
|
// a pointer we can overwrite.
|
|
LazyCallGraph::SCC *CurrentC = &C;
|
|
|
|
LLVM_DEBUG(dbgs() << "Running function passes across an SCC: " << C << "\n");
|
|
|
|
PreservedAnalyses PA = PreservedAnalyses::all();
|
|
for (LazyCallGraph::Node *N : Nodes) {
|
|
// Skip nodes from other SCCs. These may have been split out during
|
|
// processing. We'll eventually visit those SCCs and pick up the nodes
|
|
// there.
|
|
if (CG.lookupSCC(*N) != CurrentC)
|
|
continue;
|
|
|
|
Function &F = N->getFunction();
|
|
|
|
PassInstrumentation PI = FAM.getResult<PassInstrumentationAnalysis>(F);
|
|
if (!PI.runBeforePass<Function>(*Pass, F))
|
|
continue;
|
|
|
|
PreservedAnalyses PassPA;
|
|
{
|
|
TimeTraceScope TimeScope(Pass->name());
|
|
PassPA = Pass->run(F, FAM);
|
|
}
|
|
|
|
PI.runAfterPass<Function>(*Pass, F, PassPA);
|
|
|
|
// We know that the function pass couldn't have invalidated any other
|
|
// function's analyses (that's the contract of a function pass), so
|
|
// directly handle the function analysis manager's invalidation here.
|
|
FAM.invalidate(F, PassPA);
|
|
|
|
// Then intersect the preserved set so that invalidation of module
|
|
// analyses will eventually occur when the module pass completes.
|
|
PA.intersect(std::move(PassPA));
|
|
|
|
// If the call graph hasn't been preserved, update it based on this
|
|
// function pass. This may also update the current SCC to point to
|
|
// a smaller, more refined SCC.
|
|
auto PAC = PA.getChecker<LazyCallGraphAnalysis>();
|
|
if (!PAC.preserved() && !PAC.preservedSet<AllAnalysesOn<Module>>()) {
|
|
CurrentC = &updateCGAndAnalysisManagerForFunctionPass(CG, *CurrentC, *N,
|
|
AM, UR, FAM);
|
|
assert(CG.lookupSCC(*N) == CurrentC &&
|
|
"Current SCC not updated to the SCC containing the current node!");
|
|
}
|
|
}
|
|
|
|
// By definition we preserve the proxy. And we preserve all analyses on
|
|
// Functions. This precludes *any* invalidation of function analyses by the
|
|
// proxy, but that's OK because we've taken care to invalidate analyses in
|
|
// the function analysis manager incrementally above.
|
|
PA.preserveSet<AllAnalysesOn<Function>>();
|
|
PA.preserve<FunctionAnalysisManagerCGSCCProxy>();
|
|
|
|
// We've also ensured that we updated the call graph along the way.
|
|
PA.preserve<LazyCallGraphAnalysis>();
|
|
|
|
return PA;
|
|
}
|
|
|
|
bool CGSCCAnalysisManagerModuleProxy::Result::invalidate(
|
|
Module &M, const PreservedAnalyses &PA,
|
|
ModuleAnalysisManager::Invalidator &Inv) {
|
|
// If literally everything is preserved, we're done.
|
|
if (PA.areAllPreserved())
|
|
return false; // This is still a valid proxy.
|
|
|
|
// If this proxy or the call graph is going to be invalidated, we also need
|
|
// to clear all the keys coming from that analysis.
|
|
//
|
|
// We also directly invalidate the FAM's module proxy if necessary, and if
|
|
// that proxy isn't preserved we can't preserve this proxy either. We rely on
|
|
// it to handle module -> function analysis invalidation in the face of
|
|
// structural changes and so if it's unavailable we conservatively clear the
|
|
// entire SCC layer as well rather than trying to do invalidation ourselves.
|
|
auto PAC = PA.getChecker<CGSCCAnalysisManagerModuleProxy>();
|
|
if (!(PAC.preserved() || PAC.preservedSet<AllAnalysesOn<Module>>()) ||
|
|
Inv.invalidate<LazyCallGraphAnalysis>(M, PA) ||
|
|
Inv.invalidate<FunctionAnalysisManagerModuleProxy>(M, PA)) {
|
|
InnerAM->clear();
|
|
|
|
// And the proxy itself should be marked as invalid so that we can observe
|
|
// the new call graph. This isn't strictly necessary because we cheat
|
|
// above, but is still useful.
|
|
return true;
|
|
}
|
|
|
|
// Directly check if the relevant set is preserved so we can short circuit
|
|
// invalidating SCCs below.
|
|
bool AreSCCAnalysesPreserved =
|
|
PA.allAnalysesInSetPreserved<AllAnalysesOn<LazyCallGraph::SCC>>();
|
|
|
|
// Ok, we have a graph, so we can propagate the invalidation down into it.
|
|
G->buildRefSCCs();
|
|
for (auto &RC : G->postorder_ref_sccs())
|
|
for (auto &C : RC) {
|
|
Optional<PreservedAnalyses> InnerPA;
|
|
|
|
// Check to see whether the preserved set needs to be adjusted based on
|
|
// module-level analysis invalidation triggering deferred invalidation
|
|
// for this SCC.
|
|
if (auto *OuterProxy =
|
|
InnerAM->getCachedResult<ModuleAnalysisManagerCGSCCProxy>(C))
|
|
for (const auto &OuterInvalidationPair :
|
|
OuterProxy->getOuterInvalidations()) {
|
|
AnalysisKey *OuterAnalysisID = OuterInvalidationPair.first;
|
|
const auto &InnerAnalysisIDs = OuterInvalidationPair.second;
|
|
if (Inv.invalidate(OuterAnalysisID, M, PA)) {
|
|
if (!InnerPA)
|
|
InnerPA = PA;
|
|
for (AnalysisKey *InnerAnalysisID : InnerAnalysisIDs)
|
|
InnerPA->abandon(InnerAnalysisID);
|
|
}
|
|
}
|
|
|
|
// Check if we needed a custom PA set. If so we'll need to run the inner
|
|
// invalidation.
|
|
if (InnerPA) {
|
|
InnerAM->invalidate(C, *InnerPA);
|
|
continue;
|
|
}
|
|
|
|
// Otherwise we only need to do invalidation if the original PA set didn't
|
|
// preserve all SCC analyses.
|
|
if (!AreSCCAnalysesPreserved)
|
|
InnerAM->invalidate(C, PA);
|
|
}
|
|
|
|
// Return false to indicate that this result is still a valid proxy.
|
|
return false;
|
|
}
|
|
|
|
template <>
|
|
CGSCCAnalysisManagerModuleProxy::Result
|
|
CGSCCAnalysisManagerModuleProxy::run(Module &M, ModuleAnalysisManager &AM) {
|
|
// Force the Function analysis manager to also be available so that it can
|
|
// be accessed in an SCC analysis and proxied onward to function passes.
|
|
// FIXME: It is pretty awkward to just drop the result here and assert that
|
|
// we can find it again later.
|
|
(void)AM.getResult<FunctionAnalysisManagerModuleProxy>(M);
|
|
|
|
return Result(*InnerAM, AM.getResult<LazyCallGraphAnalysis>(M));
|
|
}
|
|
|
|
AnalysisKey FunctionAnalysisManagerCGSCCProxy::Key;
|
|
|
|
FunctionAnalysisManagerCGSCCProxy::Result
|
|
FunctionAnalysisManagerCGSCCProxy::run(LazyCallGraph::SCC &C,
|
|
CGSCCAnalysisManager &AM,
|
|
LazyCallGraph &CG) {
|
|
// Note: unconditionally getting checking that the proxy exists may get it at
|
|
// this point. There are cases when this is being run unnecessarily, but
|
|
// it is cheap and having the assertion in place is more valuable.
|
|
auto &MAMProxy = AM.getResult<ModuleAnalysisManagerCGSCCProxy>(C, CG);
|
|
Module &M = *C.begin()->getFunction().getParent();
|
|
bool ProxyExists =
|
|
MAMProxy.cachedResultExists<FunctionAnalysisManagerModuleProxy>(M);
|
|
assert(ProxyExists &&
|
|
"The CGSCC pass manager requires that the FAM module proxy is run "
|
|
"on the module prior to entering the CGSCC walk");
|
|
(void)ProxyExists;
|
|
|
|
// We just return an empty result. The caller will use the updateFAM interface
|
|
// to correctly register the relevant FunctionAnalysisManager based on the
|
|
// context in which this proxy is run.
|
|
return Result();
|
|
}
|
|
|
|
bool FunctionAnalysisManagerCGSCCProxy::Result::invalidate(
|
|
LazyCallGraph::SCC &C, const PreservedAnalyses &PA,
|
|
CGSCCAnalysisManager::Invalidator &Inv) {
|
|
// If literally everything is preserved, we're done.
|
|
if (PA.areAllPreserved())
|
|
return false; // This is still a valid proxy.
|
|
|
|
// All updates to preserve valid results are done below, so we don't need to
|
|
// invalidate this proxy.
|
|
//
|
|
// Note that in order to preserve this proxy, a module pass must ensure that
|
|
// the FAM has been completely updated to handle the deletion of functions.
|
|
// Specifically, any FAM-cached results for those functions need to have been
|
|
// forcibly cleared. When preserved, this proxy will only invalidate results
|
|
// cached on functions *still in the module* at the end of the module pass.
|
|
auto PAC = PA.getChecker<FunctionAnalysisManagerCGSCCProxy>();
|
|
if (!PAC.preserved() && !PAC.preservedSet<AllAnalysesOn<LazyCallGraph::SCC>>()) {
|
|
for (LazyCallGraph::Node &N : C)
|
|
FAM->clear(N.getFunction(), N.getFunction().getName());
|
|
|
|
return false;
|
|
}
|
|
|
|
// Directly check if the relevant set is preserved.
|
|
bool AreFunctionAnalysesPreserved =
|
|
PA.allAnalysesInSetPreserved<AllAnalysesOn<Function>>();
|
|
|
|
// Now walk all the functions to see if any inner analysis invalidation is
|
|
// necessary.
|
|
for (LazyCallGraph::Node &N : C) {
|
|
Function &F = N.getFunction();
|
|
Optional<PreservedAnalyses> FunctionPA;
|
|
|
|
// Check to see whether the preserved set needs to be pruned based on
|
|
// SCC-level analysis invalidation that triggers deferred invalidation
|
|
// registered with the outer analysis manager proxy for this function.
|
|
if (auto *OuterProxy =
|
|
FAM->getCachedResult<CGSCCAnalysisManagerFunctionProxy>(F))
|
|
for (const auto &OuterInvalidationPair :
|
|
OuterProxy->getOuterInvalidations()) {
|
|
AnalysisKey *OuterAnalysisID = OuterInvalidationPair.first;
|
|
const auto &InnerAnalysisIDs = OuterInvalidationPair.second;
|
|
if (Inv.invalidate(OuterAnalysisID, C, PA)) {
|
|
if (!FunctionPA)
|
|
FunctionPA = PA;
|
|
for (AnalysisKey *InnerAnalysisID : InnerAnalysisIDs)
|
|
FunctionPA->abandon(InnerAnalysisID);
|
|
}
|
|
}
|
|
|
|
// Check if we needed a custom PA set, and if so we'll need to run the
|
|
// inner invalidation.
|
|
if (FunctionPA) {
|
|
FAM->invalidate(F, *FunctionPA);
|
|
continue;
|
|
}
|
|
|
|
// Otherwise we only need to do invalidation if the original PA set didn't
|
|
// preserve all function analyses.
|
|
if (!AreFunctionAnalysesPreserved)
|
|
FAM->invalidate(F, PA);
|
|
}
|
|
|
|
// Return false to indicate that this result is still a valid proxy.
|
|
return false;
|
|
}
|
|
|
|
} // end namespace llvm
|
|
|
|
/// When a new SCC is created for the graph we first update the
|
|
/// FunctionAnalysisManager in the Proxy's result.
|
|
/// As there might be function analysis results cached for the functions now in
|
|
/// that SCC, two forms of updates are required.
|
|
///
|
|
/// First, a proxy from the SCC to the FunctionAnalysisManager needs to be
|
|
/// created so that any subsequent invalidation events to the SCC are
|
|
/// propagated to the function analysis results cached for functions within it.
|
|
///
|
|
/// Second, if any of the functions within the SCC have analysis results with
|
|
/// outer analysis dependencies, then those dependencies would point to the
|
|
/// *wrong* SCC's analysis result. We forcibly invalidate the necessary
|
|
/// function analyses so that they don't retain stale handles.
|
|
static void updateNewSCCFunctionAnalyses(LazyCallGraph::SCC &C,
|
|
LazyCallGraph &G,
|
|
CGSCCAnalysisManager &AM,
|
|
FunctionAnalysisManager &FAM) {
|
|
AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, G).updateFAM(FAM);
|
|
|
|
// Now walk the functions in this SCC and invalidate any function analysis
|
|
// results that might have outer dependencies on an SCC analysis.
|
|
for (LazyCallGraph::Node &N : C) {
|
|
Function &F = N.getFunction();
|
|
|
|
auto *OuterProxy =
|
|
FAM.getCachedResult<CGSCCAnalysisManagerFunctionProxy>(F);
|
|
if (!OuterProxy)
|
|
// No outer analyses were queried, nothing to do.
|
|
continue;
|
|
|
|
// Forcibly abandon all the inner analyses with dependencies, but
|
|
// invalidate nothing else.
|
|
auto PA = PreservedAnalyses::all();
|
|
for (const auto &OuterInvalidationPair :
|
|
OuterProxy->getOuterInvalidations()) {
|
|
const auto &InnerAnalysisIDs = OuterInvalidationPair.second;
|
|
for (AnalysisKey *InnerAnalysisID : InnerAnalysisIDs)
|
|
PA.abandon(InnerAnalysisID);
|
|
}
|
|
|
|
// Now invalidate anything we found.
|
|
FAM.invalidate(F, PA);
|
|
}
|
|
}
|
|
|
|
/// Helper function to update both the \c CGSCCAnalysisManager \p AM and the \c
|
|
/// CGSCCPassManager's \c CGSCCUpdateResult \p UR based on a range of newly
|
|
/// added SCCs.
|
|
///
|
|
/// The range of new SCCs must be in postorder already. The SCC they were split
|
|
/// out of must be provided as \p C. The current node being mutated and
|
|
/// triggering updates must be passed as \p N.
|
|
///
|
|
/// This function returns the SCC containing \p N. This will be either \p C if
|
|
/// no new SCCs have been split out, or it will be the new SCC containing \p N.
|
|
template <typename SCCRangeT>
|
|
static LazyCallGraph::SCC *
|
|
incorporateNewSCCRange(const SCCRangeT &NewSCCRange, LazyCallGraph &G,
|
|
LazyCallGraph::Node &N, LazyCallGraph::SCC *C,
|
|
CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR) {
|
|
using SCC = LazyCallGraph::SCC;
|
|
|
|
if (NewSCCRange.begin() == NewSCCRange.end())
|
|
return C;
|
|
|
|
// Add the current SCC to the worklist as its shape has changed.
|
|
UR.CWorklist.insert(C);
|
|
LLVM_DEBUG(dbgs() << "Enqueuing the existing SCC in the worklist:" << *C
|
|
<< "\n");
|
|
|
|
SCC *OldC = C;
|
|
|
|
// Update the current SCC. Note that if we have new SCCs, this must actually
|
|
// change the SCC.
|
|
assert(C != &*NewSCCRange.begin() &&
|
|
"Cannot insert new SCCs without changing current SCC!");
|
|
C = &*NewSCCRange.begin();
|
|
assert(G.lookupSCC(N) == C && "Failed to update current SCC!");
|
|
|
|
// If we had a cached FAM proxy originally, we will want to create more of
|
|
// them for each SCC that was split off.
|
|
FunctionAnalysisManager *FAM = nullptr;
|
|
if (auto *FAMProxy =
|
|
AM.getCachedResult<FunctionAnalysisManagerCGSCCProxy>(*OldC))
|
|
FAM = &FAMProxy->getManager();
|
|
|
|
// We need to propagate an invalidation call to all but the newly current SCC
|
|
// because the outer pass manager won't do that for us after splitting them.
|
|
// FIXME: We should accept a PreservedAnalysis from the CG updater so that if
|
|
// there are preserved analysis we can avoid invalidating them here for
|
|
// split-off SCCs.
|
|
// We know however that this will preserve any FAM proxy so go ahead and mark
|
|
// that.
|
|
PreservedAnalyses PA;
|
|
PA.preserve<FunctionAnalysisManagerCGSCCProxy>();
|
|
AM.invalidate(*OldC, PA);
|
|
|
|
// Ensure the now-current SCC's function analyses are updated.
|
|
if (FAM)
|
|
updateNewSCCFunctionAnalyses(*C, G, AM, *FAM);
|
|
|
|
for (SCC &NewC : llvm::reverse(make_range(std::next(NewSCCRange.begin()),
|
|
NewSCCRange.end()))) {
|
|
assert(C != &NewC && "No need to re-visit the current SCC!");
|
|
assert(OldC != &NewC && "Already handled the original SCC!");
|
|
UR.CWorklist.insert(&NewC);
|
|
LLVM_DEBUG(dbgs() << "Enqueuing a newly formed SCC:" << NewC << "\n");
|
|
|
|
// Ensure new SCCs' function analyses are updated.
|
|
if (FAM)
|
|
updateNewSCCFunctionAnalyses(NewC, G, AM, *FAM);
|
|
|
|
// Also propagate a normal invalidation to the new SCC as only the current
|
|
// will get one from the pass manager infrastructure.
|
|
AM.invalidate(NewC, PA);
|
|
}
|
|
return C;
|
|
}
|
|
|
|
static LazyCallGraph::SCC &updateCGAndAnalysisManagerForPass(
|
|
LazyCallGraph &G, LazyCallGraph::SCC &InitialC, LazyCallGraph::Node &N,
|
|
CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR,
|
|
FunctionAnalysisManager &FAM, bool FunctionPass) {
|
|
using Node = LazyCallGraph::Node;
|
|
using Edge = LazyCallGraph::Edge;
|
|
using SCC = LazyCallGraph::SCC;
|
|
using RefSCC = LazyCallGraph::RefSCC;
|
|
|
|
RefSCC &InitialRC = InitialC.getOuterRefSCC();
|
|
SCC *C = &InitialC;
|
|
RefSCC *RC = &InitialRC;
|
|
Function &F = N.getFunction();
|
|
|
|
// Walk the function body and build up the set of retained, promoted, and
|
|
// demoted edges.
|
|
SmallVector<Constant *, 16> Worklist;
|
|
SmallPtrSet<Constant *, 16> Visited;
|
|
SmallPtrSet<Node *, 16> RetainedEdges;
|
|
SmallSetVector<Node *, 4> PromotedRefTargets;
|
|
SmallSetVector<Node *, 4> DemotedCallTargets;
|
|
SmallSetVector<Node *, 4> NewCallEdges;
|
|
SmallSetVector<Node *, 4> NewRefEdges;
|
|
SmallSetVector<Node *, 4> NewNodes;
|
|
|
|
// First walk the function and handle all called functions. We do this first
|
|
// because if there is a single call edge, whether there are ref edges is
|
|
// irrelevant.
|
|
for (Instruction &I : instructions(F)) {
|
|
if (auto *CB = dyn_cast<CallBase>(&I)) {
|
|
if (Function *Callee = CB->getCalledFunction()) {
|
|
if (Visited.insert(Callee).second && !Callee->isDeclaration()) {
|
|
Node *CalleeN = G.lookup(*Callee);
|
|
if (!CalleeN) {
|
|
CalleeN = &G.get(*Callee);
|
|
NewNodes.insert(CalleeN);
|
|
}
|
|
Edge *E = N->lookup(*CalleeN);
|
|
assert((E || !FunctionPass) &&
|
|
"No function transformations should introduce *new* "
|
|
"call edges! Any new calls should be modeled as "
|
|
"promoted existing ref edges!");
|
|
bool Inserted = RetainedEdges.insert(CalleeN).second;
|
|
(void)Inserted;
|
|
assert(Inserted && "We should never visit a function twice.");
|
|
if (!E)
|
|
NewCallEdges.insert(CalleeN);
|
|
else if (!E->isCall())
|
|
PromotedRefTargets.insert(CalleeN);
|
|
}
|
|
} else {
|
|
// We can miss devirtualization if an indirect call is created then
|
|
// promoted before updateCGAndAnalysisManagerForPass runs.
|
|
auto *Entry = UR.IndirectVHs.find(CB);
|
|
if (Entry == UR.IndirectVHs.end())
|
|
UR.IndirectVHs.insert({CB, WeakTrackingVH(CB)});
|
|
else if (!Entry->second)
|
|
Entry->second = WeakTrackingVH(CB);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Now walk all references.
|
|
for (Instruction &I : instructions(F))
|
|
for (Value *Op : I.operand_values())
|
|
if (auto *OpC = dyn_cast<Constant>(Op))
|
|
if (Visited.insert(OpC).second)
|
|
Worklist.push_back(OpC);
|
|
|
|
auto VisitRef = [&](Function &Referee) {
|
|
Node *RefereeN = G.lookup(Referee);
|
|
if (!RefereeN) {
|
|
RefereeN = &G.get(Referee);
|
|
NewNodes.insert(RefereeN);
|
|
}
|
|
Edge *E = N->lookup(*RefereeN);
|
|
assert((E || !FunctionPass) &&
|
|
"No function transformations should introduce *new* ref "
|
|
"edges! Any new ref edges would require IPO which "
|
|
"function passes aren't allowed to do!");
|
|
bool Inserted = RetainedEdges.insert(RefereeN).second;
|
|
(void)Inserted;
|
|
assert(Inserted && "We should never visit a function twice.");
|
|
if (!E)
|
|
NewRefEdges.insert(RefereeN);
|
|
else if (E->isCall())
|
|
DemotedCallTargets.insert(RefereeN);
|
|
};
|
|
LazyCallGraph::visitReferences(Worklist, Visited, VisitRef);
|
|
|
|
for (Node *NewNode : NewNodes)
|
|
G.initNode(*NewNode, *C);
|
|
|
|
// Handle new ref edges.
|
|
for (Node *RefTarget : NewRefEdges) {
|
|
SCC &TargetC = *G.lookupSCC(*RefTarget);
|
|
RefSCC &TargetRC = TargetC.getOuterRefSCC();
|
|
(void)TargetRC;
|
|
// TODO: This only allows trivial edges to be added for now.
|
|
assert((RC == &TargetRC ||
|
|
RC->isAncestorOf(TargetRC)) && "New ref edge is not trivial!");
|
|
RC->insertTrivialRefEdge(N, *RefTarget);
|
|
}
|
|
|
|
// Handle new call edges.
|
|
for (Node *CallTarget : NewCallEdges) {
|
|
SCC &TargetC = *G.lookupSCC(*CallTarget);
|
|
RefSCC &TargetRC = TargetC.getOuterRefSCC();
|
|
(void)TargetRC;
|
|
// TODO: This only allows trivial edges to be added for now.
|
|
assert((RC == &TargetRC ||
|
|
RC->isAncestorOf(TargetRC)) && "New call edge is not trivial!");
|
|
// Add a trivial ref edge to be promoted later on alongside
|
|
// PromotedRefTargets.
|
|
RC->insertTrivialRefEdge(N, *CallTarget);
|
|
}
|
|
|
|
// Include synthetic reference edges to known, defined lib functions.
|
|
for (auto *LibFn : G.getLibFunctions())
|
|
// While the list of lib functions doesn't have repeats, don't re-visit
|
|
// anything handled above.
|
|
if (!Visited.count(LibFn))
|
|
VisitRef(*LibFn);
|
|
|
|
// First remove all of the edges that are no longer present in this function.
|
|
// The first step makes these edges uniformly ref edges and accumulates them
|
|
// into a separate data structure so removal doesn't invalidate anything.
|
|
SmallVector<Node *, 4> DeadTargets;
|
|
for (Edge &E : *N) {
|
|
if (RetainedEdges.count(&E.getNode()))
|
|
continue;
|
|
|
|
SCC &TargetC = *G.lookupSCC(E.getNode());
|
|
RefSCC &TargetRC = TargetC.getOuterRefSCC();
|
|
if (&TargetRC == RC && E.isCall()) {
|
|
if (C != &TargetC) {
|
|
// For separate SCCs this is trivial.
|
|
RC->switchTrivialInternalEdgeToRef(N, E.getNode());
|
|
} else {
|
|
// Now update the call graph.
|
|
C = incorporateNewSCCRange(RC->switchInternalEdgeToRef(N, E.getNode()),
|
|
G, N, C, AM, UR);
|
|
}
|
|
}
|
|
|
|
// Now that this is ready for actual removal, put it into our list.
|
|
DeadTargets.push_back(&E.getNode());
|
|
}
|
|
// Remove the easy cases quickly and actually pull them out of our list.
|
|
DeadTargets.erase(
|
|
llvm::remove_if(DeadTargets,
|
|
[&](Node *TargetN) {
|
|
SCC &TargetC = *G.lookupSCC(*TargetN);
|
|
RefSCC &TargetRC = TargetC.getOuterRefSCC();
|
|
|
|
// We can't trivially remove internal targets, so skip
|
|
// those.
|
|
if (&TargetRC == RC)
|
|
return false;
|
|
|
|
RC->removeOutgoingEdge(N, *TargetN);
|
|
LLVM_DEBUG(dbgs() << "Deleting outgoing edge from '"
|
|
<< N << "' to '" << TargetN << "'\n");
|
|
return true;
|
|
}),
|
|
DeadTargets.end());
|
|
|
|
// Now do a batch removal of the internal ref edges left.
|
|
auto NewRefSCCs = RC->removeInternalRefEdge(N, DeadTargets);
|
|
if (!NewRefSCCs.empty()) {
|
|
// The old RefSCC is dead, mark it as such.
|
|
UR.InvalidatedRefSCCs.insert(RC);
|
|
|
|
// Note that we don't bother to invalidate analyses as ref-edge
|
|
// connectivity is not really observable in any way and is intended
|
|
// exclusively to be used for ordering of transforms rather than for
|
|
// analysis conclusions.
|
|
|
|
// Update RC to the "bottom".
|
|
assert(G.lookupSCC(N) == C && "Changed the SCC when splitting RefSCCs!");
|
|
RC = &C->getOuterRefSCC();
|
|
assert(G.lookupRefSCC(N) == RC && "Failed to update current RefSCC!");
|
|
|
|
// The RC worklist is in reverse postorder, so we enqueue the new ones in
|
|
// RPO except for the one which contains the source node as that is the
|
|
// "bottom" we will continue processing in the bottom-up walk.
|
|
assert(NewRefSCCs.front() == RC &&
|
|
"New current RefSCC not first in the returned list!");
|
|
for (RefSCC *NewRC : llvm::reverse(make_range(std::next(NewRefSCCs.begin()),
|
|
NewRefSCCs.end()))) {
|
|
assert(NewRC != RC && "Should not encounter the current RefSCC further "
|
|
"in the postorder list of new RefSCCs.");
|
|
UR.RCWorklist.insert(NewRC);
|
|
LLVM_DEBUG(dbgs() << "Enqueuing a new RefSCC in the update worklist: "
|
|
<< *NewRC << "\n");
|
|
}
|
|
}
|
|
|
|
// Next demote all the call edges that are now ref edges. This helps make
|
|
// the SCCs small which should minimize the work below as we don't want to
|
|
// form cycles that this would break.
|
|
for (Node *RefTarget : DemotedCallTargets) {
|
|
SCC &TargetC = *G.lookupSCC(*RefTarget);
|
|
RefSCC &TargetRC = TargetC.getOuterRefSCC();
|
|
|
|
// The easy case is when the target RefSCC is not this RefSCC. This is
|
|
// only supported when the target RefSCC is a child of this RefSCC.
|
|
if (&TargetRC != RC) {
|
|
assert(RC->isAncestorOf(TargetRC) &&
|
|
"Cannot potentially form RefSCC cycles here!");
|
|
RC->switchOutgoingEdgeToRef(N, *RefTarget);
|
|
LLVM_DEBUG(dbgs() << "Switch outgoing call edge to a ref edge from '" << N
|
|
<< "' to '" << *RefTarget << "'\n");
|
|
continue;
|
|
}
|
|
|
|
// We are switching an internal call edge to a ref edge. This may split up
|
|
// some SCCs.
|
|
if (C != &TargetC) {
|
|
// For separate SCCs this is trivial.
|
|
RC->switchTrivialInternalEdgeToRef(N, *RefTarget);
|
|
continue;
|
|
}
|
|
|
|
// Now update the call graph.
|
|
C = incorporateNewSCCRange(RC->switchInternalEdgeToRef(N, *RefTarget), G, N,
|
|
C, AM, UR);
|
|
}
|
|
|
|
// We added a ref edge earlier for new call edges, promote those to call edges
|
|
// alongside PromotedRefTargets.
|
|
for (Node *E : NewCallEdges)
|
|
PromotedRefTargets.insert(E);
|
|
|
|
// Now promote ref edges into call edges.
|
|
for (Node *CallTarget : PromotedRefTargets) {
|
|
SCC &TargetC = *G.lookupSCC(*CallTarget);
|
|
RefSCC &TargetRC = TargetC.getOuterRefSCC();
|
|
|
|
// The easy case is when the target RefSCC is not this RefSCC. This is
|
|
// only supported when the target RefSCC is a child of this RefSCC.
|
|
if (&TargetRC != RC) {
|
|
assert(RC->isAncestorOf(TargetRC) &&
|
|
"Cannot potentially form RefSCC cycles here!");
|
|
RC->switchOutgoingEdgeToCall(N, *CallTarget);
|
|
LLVM_DEBUG(dbgs() << "Switch outgoing ref edge to a call edge from '" << N
|
|
<< "' to '" << *CallTarget << "'\n");
|
|
continue;
|
|
}
|
|
LLVM_DEBUG(dbgs() << "Switch an internal ref edge to a call edge from '"
|
|
<< N << "' to '" << *CallTarget << "'\n");
|
|
|
|
// Otherwise we are switching an internal ref edge to a call edge. This
|
|
// may merge away some SCCs, and we add those to the UpdateResult. We also
|
|
// need to make sure to update the worklist in the event SCCs have moved
|
|
// before the current one in the post-order sequence
|
|
bool HasFunctionAnalysisProxy = false;
|
|
auto InitialSCCIndex = RC->find(*C) - RC->begin();
|
|
bool FormedCycle = RC->switchInternalEdgeToCall(
|
|
N, *CallTarget, [&](ArrayRef<SCC *> MergedSCCs) {
|
|
for (SCC *MergedC : MergedSCCs) {
|
|
assert(MergedC != &TargetC && "Cannot merge away the target SCC!");
|
|
|
|
HasFunctionAnalysisProxy |=
|
|
AM.getCachedResult<FunctionAnalysisManagerCGSCCProxy>(
|
|
*MergedC) != nullptr;
|
|
|
|
// Mark that this SCC will no longer be valid.
|
|
UR.InvalidatedSCCs.insert(MergedC);
|
|
|
|
// FIXME: We should really do a 'clear' here to forcibly release
|
|
// memory, but we don't have a good way of doing that and
|
|
// preserving the function analyses.
|
|
auto PA = PreservedAnalyses::allInSet<AllAnalysesOn<Function>>();
|
|
PA.preserve<FunctionAnalysisManagerCGSCCProxy>();
|
|
AM.invalidate(*MergedC, PA);
|
|
}
|
|
});
|
|
|
|
// If we formed a cycle by creating this call, we need to update more data
|
|
// structures.
|
|
if (FormedCycle) {
|
|
C = &TargetC;
|
|
assert(G.lookupSCC(N) == C && "Failed to update current SCC!");
|
|
|
|
// If one of the invalidated SCCs had a cached proxy to a function
|
|
// analysis manager, we need to create a proxy in the new current SCC as
|
|
// the invalidated SCCs had their functions moved.
|
|
if (HasFunctionAnalysisProxy)
|
|
AM.getResult<FunctionAnalysisManagerCGSCCProxy>(*C, G).updateFAM(FAM);
|
|
|
|
// Any analyses cached for this SCC are no longer precise as the shape
|
|
// has changed by introducing this cycle. However, we have taken care to
|
|
// update the proxies so it remains valide.
|
|
auto PA = PreservedAnalyses::allInSet<AllAnalysesOn<Function>>();
|
|
PA.preserve<FunctionAnalysisManagerCGSCCProxy>();
|
|
AM.invalidate(*C, PA);
|
|
}
|
|
auto NewSCCIndex = RC->find(*C) - RC->begin();
|
|
// If we have actually moved an SCC to be topologically "below" the current
|
|
// one due to merging, we will need to revisit the current SCC after
|
|
// visiting those moved SCCs.
|
|
//
|
|
// It is critical that we *do not* revisit the current SCC unless we
|
|
// actually move SCCs in the process of merging because otherwise we may
|
|
// form a cycle where an SCC is split apart, merged, split, merged and so
|
|
// on infinitely.
|
|
if (InitialSCCIndex < NewSCCIndex) {
|
|
// Put our current SCC back onto the worklist as we'll visit other SCCs
|
|
// that are now definitively ordered prior to the current one in the
|
|
// post-order sequence, and may end up observing more precise context to
|
|
// optimize the current SCC.
|
|
UR.CWorklist.insert(C);
|
|
LLVM_DEBUG(dbgs() << "Enqueuing the existing SCC in the worklist: " << *C
|
|
<< "\n");
|
|
// Enqueue in reverse order as we pop off the back of the worklist.
|
|
for (SCC &MovedC : llvm::reverse(make_range(RC->begin() + InitialSCCIndex,
|
|
RC->begin() + NewSCCIndex))) {
|
|
UR.CWorklist.insert(&MovedC);
|
|
LLVM_DEBUG(dbgs() << "Enqueuing a newly earlier in post-order SCC: "
|
|
<< MovedC << "\n");
|
|
}
|
|
}
|
|
}
|
|
|
|
assert(!UR.InvalidatedSCCs.count(C) && "Invalidated the current SCC!");
|
|
assert(!UR.InvalidatedRefSCCs.count(RC) && "Invalidated the current RefSCC!");
|
|
assert(&C->getOuterRefSCC() == RC && "Current SCC not in current RefSCC!");
|
|
|
|
// Record the current RefSCC and SCC for higher layers of the CGSCC pass
|
|
// manager now that all the updates have been applied.
|
|
if (RC != &InitialRC)
|
|
UR.UpdatedRC = RC;
|
|
if (C != &InitialC)
|
|
UR.UpdatedC = C;
|
|
|
|
return *C;
|
|
}
|
|
|
|
LazyCallGraph::SCC &llvm::updateCGAndAnalysisManagerForFunctionPass(
|
|
LazyCallGraph &G, LazyCallGraph::SCC &InitialC, LazyCallGraph::Node &N,
|
|
CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR,
|
|
FunctionAnalysisManager &FAM) {
|
|
return updateCGAndAnalysisManagerForPass(G, InitialC, N, AM, UR, FAM,
|
|
/* FunctionPass */ true);
|
|
}
|
|
LazyCallGraph::SCC &llvm::updateCGAndAnalysisManagerForCGSCCPass(
|
|
LazyCallGraph &G, LazyCallGraph::SCC &InitialC, LazyCallGraph::Node &N,
|
|
CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR,
|
|
FunctionAnalysisManager &FAM) {
|
|
return updateCGAndAnalysisManagerForPass(G, InitialC, N, AM, UR, FAM,
|
|
/* FunctionPass */ false);
|
|
}
|