2017-09-22 07:20:16 +08:00
|
|
|
//===- SpillPlacement.cpp - Optimal Spill Code Placement ------------------===//
|
2011-01-06 09:21:53 +08:00
|
|
|
//
|
2019-01-19 16:50:56 +08:00
|
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
2011-01-06 09:21:53 +08:00
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
//
|
|
|
|
// This file implements the spill code placement analysis.
|
|
|
|
//
|
|
|
|
// Each edge bundle corresponds to a node in a Hopfield network. Constraints on
|
|
|
|
// basic blocks are weighted by the block frequency and added to become the node
|
|
|
|
// bias.
|
|
|
|
//
|
|
|
|
// Transparent basic blocks have the variable live through, but don't care if it
|
|
|
|
// is spilled or in a register. These blocks become connections in the Hopfield
|
|
|
|
// network, again weighted by block frequency.
|
|
|
|
//
|
|
|
|
// The Hopfield network minimizes (possibly locally) its energy function:
|
|
|
|
//
|
|
|
|
// E = -sum_n V_n * ( B_n + sum_{n, m linked by b} V_m * F_b )
|
|
|
|
//
|
|
|
|
// The energy function represents the expected spill code execution frequency,
|
|
|
|
// or the cost of spilling. This is a Lyapunov function which never increases
|
|
|
|
// when a node is updated. It is guaranteed to converge to a local minimum.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
#include "SpillPlacement.h"
|
2017-09-22 07:20:16 +08:00
|
|
|
#include "llvm/ADT/ArrayRef.h"
|
2013-03-19 07:45:45 +08:00
|
|
|
#include "llvm/ADT/BitVector.h"
|
2017-09-22 07:20:16 +08:00
|
|
|
#include "llvm/ADT/SmallVector.h"
|
|
|
|
#include "llvm/ADT/SparseSet.h"
|
2011-01-06 09:21:53 +08:00
|
|
|
#include "llvm/CodeGen/EdgeBundles.h"
|
|
|
|
#include "llvm/CodeGen/MachineBasicBlock.h"
|
2013-06-18 03:00:36 +08:00
|
|
|
#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
|
2011-01-06 09:21:53 +08:00
|
|
|
#include "llvm/CodeGen/MachineFunction.h"
|
|
|
|
#include "llvm/CodeGen/MachineLoopInfo.h"
|
|
|
|
#include "llvm/CodeGen/Passes.h"
|
Sink all InitializePasses.h includes
This file lists every pass in LLVM, and is included by Pass.h, which is
very popular. Every time we add, remove, or rename a pass in LLVM, it
caused lots of recompilation.
I found this fact by looking at this table, which is sorted by the
number of times a file was changed over the last 100,000 git commits
multiplied by the number of object files that depend on it in the
current checkout:
recompiles touches affected_files header
342380 95 3604 llvm/include/llvm/ADT/STLExtras.h
314730 234 1345 llvm/include/llvm/InitializePasses.h
307036 118 2602 llvm/include/llvm/ADT/APInt.h
213049 59 3611 llvm/include/llvm/Support/MathExtras.h
170422 47 3626 llvm/include/llvm/Support/Compiler.h
162225 45 3605 llvm/include/llvm/ADT/Optional.h
158319 63 2513 llvm/include/llvm/ADT/Triple.h
140322 39 3598 llvm/include/llvm/ADT/StringRef.h
137647 59 2333 llvm/include/llvm/Support/Error.h
131619 73 1803 llvm/include/llvm/Support/FileSystem.h
Before this change, touching InitializePasses.h would cause 1345 files
to recompile. After this change, touching it only causes 550 compiles in
an incremental rebuild.
Reviewers: bkramer, asbirlea, bollu, jdoerfert
Differential Revision: https://reviews.llvm.org/D70211
2019-11-14 05:15:01 +08:00
|
|
|
#include "llvm/InitializePasses.h"
|
2017-09-22 07:20:16 +08:00
|
|
|
#include "llvm/Pass.h"
|
|
|
|
#include "llvm/Support/BlockFrequency.h"
|
|
|
|
#include <algorithm>
|
|
|
|
#include <cassert>
|
|
|
|
#include <cstdint>
|
|
|
|
#include <utility>
|
2011-01-06 09:21:53 +08:00
|
|
|
|
|
|
|
using namespace llvm;
|
|
|
|
|
2017-05-26 05:26:32 +08:00
|
|
|
#define DEBUG_TYPE "spill-code-placement"
|
2014-04-22 10:02:50 +08:00
|
|
|
|
2011-01-06 09:21:53 +08:00
|
|
|
char SpillPlacement::ID = 0;
|
2017-09-22 07:20:16 +08:00
|
|
|
|
|
|
|
char &llvm::SpillPlacementID = SpillPlacement::ID;
|
|
|
|
|
2017-05-26 05:26:32 +08:00
|
|
|
INITIALIZE_PASS_BEGIN(SpillPlacement, DEBUG_TYPE,
|
2011-01-06 09:21:53 +08:00
|
|
|
"Spill Code Placement Analysis", true, true)
|
|
|
|
INITIALIZE_PASS_DEPENDENCY(EdgeBundles)
|
|
|
|
INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)
|
2017-05-26 05:26:32 +08:00
|
|
|
INITIALIZE_PASS_END(SpillPlacement, DEBUG_TYPE,
|
2011-01-06 09:21:53 +08:00
|
|
|
"Spill Code Placement Analysis", true, true)
|
|
|
|
|
|
|
|
void SpillPlacement::getAnalysisUsage(AnalysisUsage &AU) const {
|
|
|
|
AU.setPreservesAll();
|
2013-06-18 03:00:36 +08:00
|
|
|
AU.addRequired<MachineBlockFrequencyInfo>();
|
2011-01-06 09:21:53 +08:00
|
|
|
AU.addRequiredTransitive<EdgeBundles>();
|
|
|
|
AU.addRequiredTransitive<MachineLoopInfo>();
|
|
|
|
MachineFunctionPass::getAnalysisUsage(AU);
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Node - Each edge bundle corresponds to a Hopfield node.
|
|
|
|
///
|
|
|
|
/// The node contains precomputed frequency data that only depends on the CFG,
|
|
|
|
/// but Bias and Links are computed each time placeSpills is called.
|
|
|
|
///
|
|
|
|
/// The node Value is positive when the variable should be in a register. The
|
|
|
|
/// value can change when linked nodes change, but convergence is very fast
|
|
|
|
/// because all weights are positive.
|
|
|
|
struct SpillPlacement::Node {
|
2013-07-17 02:26:15 +08:00
|
|
|
/// BiasN - Sum of blocks that prefer a spill.
|
|
|
|
BlockFrequency BiasN;
|
2017-09-22 07:20:16 +08:00
|
|
|
|
2013-07-17 02:26:15 +08:00
|
|
|
/// BiasP - Sum of blocks that prefer a register.
|
|
|
|
BlockFrequency BiasP;
|
2011-01-06 09:21:53 +08:00
|
|
|
|
|
|
|
/// Value - Output value of this node computed from the Bias and links.
|
2013-07-17 02:26:15 +08:00
|
|
|
/// This is always on of the values {-1, 0, 1}. A positive number means the
|
|
|
|
/// variable should go in a register through this bundle.
|
|
|
|
int Value;
|
2011-01-06 09:21:53 +08:00
|
|
|
|
2017-09-22 07:20:16 +08:00
|
|
|
using LinkVector = SmallVector<std::pair<BlockFrequency, unsigned>, 4>;
|
2011-01-06 09:21:53 +08:00
|
|
|
|
|
|
|
/// Links - (Weight, BundleNo) for all transparent blocks connecting to other
|
2013-07-17 02:26:15 +08:00
|
|
|
/// bundles. The weights are all positive block frequencies.
|
2011-01-06 09:21:53 +08:00
|
|
|
LinkVector Links;
|
|
|
|
|
2013-07-17 02:26:15 +08:00
|
|
|
/// SumLinkWeights - Cached sum of the weights of all links + ThresHold.
|
|
|
|
BlockFrequency SumLinkWeights;
|
|
|
|
|
2011-01-06 09:21:53 +08:00
|
|
|
/// preferReg - Return true when this node prefers to be in a register.
|
|
|
|
bool preferReg() const {
|
|
|
|
// Undecided nodes (Value==0) go on the stack.
|
|
|
|
return Value > 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/// mustSpill - Return True if this node is so biased that it must spill.
|
|
|
|
bool mustSpill() const {
|
2013-07-17 02:26:15 +08:00
|
|
|
// We must spill if Bias < -sum(weights) or the MustSpill flag was set.
|
|
|
|
// BiasN is saturated when MustSpill is set, make sure this still returns
|
|
|
|
// true when the RHS saturates. Note that SumLinkWeights includes Threshold.
|
|
|
|
return BiasN >= BiasP + SumLinkWeights;
|
2011-01-06 09:21:53 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/// clear - Reset per-query data, but preserve frequencies that only depend on
|
2017-09-22 07:20:16 +08:00
|
|
|
/// the CFG.
|
2014-10-03 06:23:14 +08:00
|
|
|
void clear(const BlockFrequency &Threshold) {
|
2013-07-17 02:26:15 +08:00
|
|
|
BiasN = BiasP = Value = 0;
|
2014-10-03 06:23:14 +08:00
|
|
|
SumLinkWeights = Threshold;
|
2011-01-06 09:21:53 +08:00
|
|
|
Links.clear();
|
|
|
|
}
|
|
|
|
|
|
|
|
/// addLink - Add a link to bundle b with weight w.
|
2013-07-17 02:26:15 +08:00
|
|
|
void addLink(unsigned b, BlockFrequency w) {
|
|
|
|
// Update cached sum.
|
|
|
|
SumLinkWeights += w;
|
2011-01-06 09:21:53 +08:00
|
|
|
|
|
|
|
// There can be multiple links to the same bundle, add them up.
|
|
|
|
for (LinkVector::iterator I = Links.begin(), E = Links.end(); I != E; ++I)
|
|
|
|
if (I->second == b) {
|
|
|
|
I->first += w;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
// This must be the first link to b.
|
|
|
|
Links.push_back(std::make_pair(w, b));
|
|
|
|
}
|
|
|
|
|
2013-07-17 02:26:15 +08:00
|
|
|
/// addBias - Bias this node.
|
|
|
|
void addBias(BlockFrequency freq, BorderConstraint direction) {
|
|
|
|
switch (direction) {
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
case PrefReg:
|
|
|
|
BiasP += freq;
|
|
|
|
break;
|
|
|
|
case PrefSpill:
|
|
|
|
BiasN += freq;
|
|
|
|
break;
|
|
|
|
case MustSpill:
|
|
|
|
BiasN = BlockFrequency::getMaxFrequency();
|
|
|
|
break;
|
|
|
|
}
|
2011-01-06 09:21:53 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/// update - Recompute Value from Bias and Links. Return true when node
|
|
|
|
/// preference changes.
|
2014-10-03 06:23:14 +08:00
|
|
|
bool update(const Node nodes[], const BlockFrequency &Threshold) {
|
2011-01-06 09:21:53 +08:00
|
|
|
// Compute the weighted sum of inputs.
|
2013-07-17 02:26:15 +08:00
|
|
|
BlockFrequency SumN = BiasN;
|
|
|
|
BlockFrequency SumP = BiasP;
|
|
|
|
for (LinkVector::iterator I = Links.begin(), E = Links.end(); I != E; ++I) {
|
|
|
|
if (nodes[I->second].Value == -1)
|
|
|
|
SumN += I->first;
|
|
|
|
else if (nodes[I->second].Value == 1)
|
|
|
|
SumP += I->first;
|
|
|
|
}
|
2011-01-06 09:21:53 +08:00
|
|
|
|
2013-07-17 02:26:15 +08:00
|
|
|
// Each weighted sum is going to be less than the total frequency of the
|
|
|
|
// bundle. Ideally, we should simply set Value = sign(SumP - SumN), but we
|
|
|
|
// will add a dead zone around 0 for two reasons:
|
|
|
|
//
|
2011-01-06 09:21:53 +08:00
|
|
|
// 1. It avoids arbitrary bias when all links are 0 as is possible during
|
|
|
|
// initial iterations.
|
|
|
|
// 2. It helps tame rounding errors when the links nominally sum to 0.
|
2013-07-17 02:26:15 +08:00
|
|
|
//
|
2011-01-06 09:21:53 +08:00
|
|
|
bool Before = preferReg();
|
2014-10-03 06:23:14 +08:00
|
|
|
if (SumN >= SumP + Threshold)
|
2011-01-06 09:21:53 +08:00
|
|
|
Value = -1;
|
2014-10-03 06:23:14 +08:00
|
|
|
else if (SumP >= SumN + Threshold)
|
2011-01-06 09:21:53 +08:00
|
|
|
Value = 1;
|
|
|
|
else
|
|
|
|
Value = 0;
|
|
|
|
return Before != preferReg();
|
|
|
|
}
|
Reapply r263460: [SpillPlacement] Fix a quadratic behavior in spill placement.
Using Chandler's words from r265331:
This commit was greatly exacerbating PR17409 and effectively regressed
build time for lot of (very large) code when compiled with ASan or MSan.
PR17409 is fixed by r269249, so this is fine to reapply r263460.
Original commit message:
The bad behavior happens when we have a function with a long linear
chain of basic blocks, and have a live range spanning most of this
chain, but with very few uses.
Let say we have only 2 uses.
The Hopfield network is only seeded with two active blocks where the
uses are, and each iteration of the outer loop in
`RAGreedy::growRegion()` only adds two new nodes to the network due to
the completely linear shape of the CFG. Meanwhile,
`SpillPlacer->iterate()` visits the whole set of discovered nodes, which
adds up to a quadratic algorithm.
This is an historical accident effect from r129188.
When the Hopfield network is expanding, most of the action is happening
on the frontier where new nodes are being added. The internal nodes in
the network are not likely to be flip-flopping much, or they will at
least settle down very quickly. This means that while
`SpillPlacer->iterate()` is recomputing all the nodes in the network, it
is probably only the two frontier nodes that are changing their output.
Instead of recomputing the whole network on each iteration, we can
maintain a SparseSet of nodes that need to be updated:
- `SpillPlacement::activate()` adds the node to the todo list.
- When a node changes value (i.e., `update()` returns true), its
neighbors are added to the todo list.
- `SpillPlacement::iterate()` only updates the nodes in the list.
The result of Hopfield iterations is not necessarily exact. It should
converge to a local minimum, but there is no guarantee that it will find
a global minimum. It is possible that updating nodes in a different
order will cause us to switch to a different local minimum. In other
words, this is not NFC, but although I saw a few runtime improvements
and regressions when I benchmarked this change, those were side effects
and actually the performance change is in the noise as expected.
Huge thanks to Jakob Stoklund Olesen <stoklund@2pi.dk> for his
feedbacks, guidance and time for the review.
llvm-svn: 270149
2016-05-20 06:40:37 +08:00
|
|
|
|
|
|
|
void getDissentingNeighbors(SparseSet<unsigned> &List,
|
|
|
|
const Node nodes[]) const {
|
|
|
|
for (const auto &Elt : Links) {
|
|
|
|
unsigned n = Elt.second;
|
|
|
|
// Neighbors that already have the same value are not going to
|
|
|
|
// change because of this node changing.
|
|
|
|
if (Value != nodes[n].Value)
|
|
|
|
List.insert(n);
|
|
|
|
}
|
|
|
|
}
|
2011-01-06 09:21:53 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
bool SpillPlacement::runOnMachineFunction(MachineFunction &mf) {
|
|
|
|
MF = &mf;
|
|
|
|
bundles = &getAnalysis<EdgeBundles>();
|
|
|
|
loops = &getAnalysis<MachineLoopInfo>();
|
|
|
|
|
|
|
|
assert(!nodes && "Leaking node array");
|
|
|
|
nodes = new Node[bundles->getNumBundles()];
|
Reapply r263460: [SpillPlacement] Fix a quadratic behavior in spill placement.
Using Chandler's words from r265331:
This commit was greatly exacerbating PR17409 and effectively regressed
build time for lot of (very large) code when compiled with ASan or MSan.
PR17409 is fixed by r269249, so this is fine to reapply r263460.
Original commit message:
The bad behavior happens when we have a function with a long linear
chain of basic blocks, and have a live range spanning most of this
chain, but with very few uses.
Let say we have only 2 uses.
The Hopfield network is only seeded with two active blocks where the
uses are, and each iteration of the outer loop in
`RAGreedy::growRegion()` only adds two new nodes to the network due to
the completely linear shape of the CFG. Meanwhile,
`SpillPlacer->iterate()` visits the whole set of discovered nodes, which
adds up to a quadratic algorithm.
This is an historical accident effect from r129188.
When the Hopfield network is expanding, most of the action is happening
on the frontier where new nodes are being added. The internal nodes in
the network are not likely to be flip-flopping much, or they will at
least settle down very quickly. This means that while
`SpillPlacer->iterate()` is recomputing all the nodes in the network, it
is probably only the two frontier nodes that are changing their output.
Instead of recomputing the whole network on each iteration, we can
maintain a SparseSet of nodes that need to be updated:
- `SpillPlacement::activate()` adds the node to the todo list.
- When a node changes value (i.e., `update()` returns true), its
neighbors are added to the todo list.
- `SpillPlacement::iterate()` only updates the nodes in the list.
The result of Hopfield iterations is not necessarily exact. It should
converge to a local minimum, but there is no guarantee that it will find
a global minimum. It is possible that updating nodes in a different
order will cause us to switch to a different local minimum. In other
words, this is not NFC, but although I saw a few runtime improvements
and regressions when I benchmarked this change, those were side effects
and actually the performance change is in the noise as expected.
Huge thanks to Jakob Stoklund Olesen <stoklund@2pi.dk> for his
feedbacks, guidance and time for the review.
llvm-svn: 270149
2016-05-20 06:40:37 +08:00
|
|
|
TodoList.clear();
|
|
|
|
TodoList.setUniverse(bundles->getNumBundles());
|
2011-01-06 09:21:53 +08:00
|
|
|
|
|
|
|
// Compute total ingoing and outgoing block frequencies for all bundles.
|
2013-07-17 02:26:15 +08:00
|
|
|
BlockFrequencies.resize(mf.getNumBlockIDs());
|
2013-12-14 08:25:47 +08:00
|
|
|
MBFI = &getAnalysis<MachineBlockFrequencyInfo>();
|
2014-04-09 03:18:56 +08:00
|
|
|
setThreshold(MBFI->getEntryFreq());
|
2015-10-10 06:56:24 +08:00
|
|
|
for (auto &I : mf) {
|
|
|
|
unsigned Num = I.getNumber();
|
|
|
|
BlockFrequencies[Num] = MBFI->getBlockFreq(&I);
|
2011-01-06 09:21:53 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// We never change the function.
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
void SpillPlacement::releaseMemory() {
|
|
|
|
delete[] nodes;
|
2014-04-14 08:51:57 +08:00
|
|
|
nodes = nullptr;
|
Reapply r263460: [SpillPlacement] Fix a quadratic behavior in spill placement.
Using Chandler's words from r265331:
This commit was greatly exacerbating PR17409 and effectively regressed
build time for lot of (very large) code when compiled with ASan or MSan.
PR17409 is fixed by r269249, so this is fine to reapply r263460.
Original commit message:
The bad behavior happens when we have a function with a long linear
chain of basic blocks, and have a live range spanning most of this
chain, but with very few uses.
Let say we have only 2 uses.
The Hopfield network is only seeded with two active blocks where the
uses are, and each iteration of the outer loop in
`RAGreedy::growRegion()` only adds two new nodes to the network due to
the completely linear shape of the CFG. Meanwhile,
`SpillPlacer->iterate()` visits the whole set of discovered nodes, which
adds up to a quadratic algorithm.
This is an historical accident effect from r129188.
When the Hopfield network is expanding, most of the action is happening
on the frontier where new nodes are being added. The internal nodes in
the network are not likely to be flip-flopping much, or they will at
least settle down very quickly. This means that while
`SpillPlacer->iterate()` is recomputing all the nodes in the network, it
is probably only the two frontier nodes that are changing their output.
Instead of recomputing the whole network on each iteration, we can
maintain a SparseSet of nodes that need to be updated:
- `SpillPlacement::activate()` adds the node to the todo list.
- When a node changes value (i.e., `update()` returns true), its
neighbors are added to the todo list.
- `SpillPlacement::iterate()` only updates the nodes in the list.
The result of Hopfield iterations is not necessarily exact. It should
converge to a local minimum, but there is no guarantee that it will find
a global minimum. It is possible that updating nodes in a different
order will cause us to switch to a different local minimum. In other
words, this is not NFC, but although I saw a few runtime improvements
and regressions when I benchmarked this change, those were side effects
and actually the performance change is in the noise as expected.
Huge thanks to Jakob Stoklund Olesen <stoklund@2pi.dk> for his
feedbacks, guidance and time for the review.
llvm-svn: 270149
2016-05-20 06:40:37 +08:00
|
|
|
TodoList.clear();
|
2011-01-06 09:21:53 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/// activate - mark node n as active if it wasn't already.
|
|
|
|
void SpillPlacement::activate(unsigned n) {
|
Reapply r263460: [SpillPlacement] Fix a quadratic behavior in spill placement.
Using Chandler's words from r265331:
This commit was greatly exacerbating PR17409 and effectively regressed
build time for lot of (very large) code when compiled with ASan or MSan.
PR17409 is fixed by r269249, so this is fine to reapply r263460.
Original commit message:
The bad behavior happens when we have a function with a long linear
chain of basic blocks, and have a live range spanning most of this
chain, but with very few uses.
Let say we have only 2 uses.
The Hopfield network is only seeded with two active blocks where the
uses are, and each iteration of the outer loop in
`RAGreedy::growRegion()` only adds two new nodes to the network due to
the completely linear shape of the CFG. Meanwhile,
`SpillPlacer->iterate()` visits the whole set of discovered nodes, which
adds up to a quadratic algorithm.
This is an historical accident effect from r129188.
When the Hopfield network is expanding, most of the action is happening
on the frontier where new nodes are being added. The internal nodes in
the network are not likely to be flip-flopping much, or they will at
least settle down very quickly. This means that while
`SpillPlacer->iterate()` is recomputing all the nodes in the network, it
is probably only the two frontier nodes that are changing their output.
Instead of recomputing the whole network on each iteration, we can
maintain a SparseSet of nodes that need to be updated:
- `SpillPlacement::activate()` adds the node to the todo list.
- When a node changes value (i.e., `update()` returns true), its
neighbors are added to the todo list.
- `SpillPlacement::iterate()` only updates the nodes in the list.
The result of Hopfield iterations is not necessarily exact. It should
converge to a local minimum, but there is no guarantee that it will find
a global minimum. It is possible that updating nodes in a different
order will cause us to switch to a different local minimum. In other
words, this is not NFC, but although I saw a few runtime improvements
and regressions when I benchmarked this change, those were side effects
and actually the performance change is in the noise as expected.
Huge thanks to Jakob Stoklund Olesen <stoklund@2pi.dk> for his
feedbacks, guidance and time for the review.
llvm-svn: 270149
2016-05-20 06:40:37 +08:00
|
|
|
TodoList.insert(n);
|
2011-01-06 09:21:53 +08:00
|
|
|
if (ActiveNodes->test(n))
|
|
|
|
return;
|
|
|
|
ActiveNodes->set(n);
|
2014-10-03 06:23:14 +08:00
|
|
|
nodes[n].clear(Threshold);
|
2012-05-21 11:11:23 +08:00
|
|
|
|
|
|
|
// Very large bundles usually come from big switches, indirect branches,
|
|
|
|
// landing pads, or loops with many 'continue' statements. It is difficult to
|
|
|
|
// allocate registers when so many different blocks are involved.
|
|
|
|
//
|
2013-07-17 02:26:15 +08:00
|
|
|
// Give a small negative bias to large bundles such that a substantial
|
|
|
|
// fraction of the connected blocks need to be interested before we consider
|
|
|
|
// expanding the region through the bundle. This helps compile time by
|
|
|
|
// limiting the number of blocks visited and the number of links in the
|
|
|
|
// Hopfield network.
|
|
|
|
if (bundles->getBlocks(n).size() > 100) {
|
|
|
|
nodes[n].BiasP = 0;
|
2013-12-14 10:37:38 +08:00
|
|
|
nodes[n].BiasN = (MBFI->getEntryFreq() / 16);
|
2013-07-17 02:26:15 +08:00
|
|
|
}
|
2011-01-06 09:21:53 +08:00
|
|
|
}
|
|
|
|
|
2018-05-01 23:54:18 +08:00
|
|
|
/// Set the threshold for a given entry frequency.
|
2014-10-03 06:23:14 +08:00
|
|
|
///
|
|
|
|
/// Set the threshold relative to \c Entry. Since the threshold is used as a
|
|
|
|
/// bound on the open interval (-Threshold;Threshold), 1 is the minimum
|
|
|
|
/// threshold.
|
|
|
|
void SpillPlacement::setThreshold(const BlockFrequency &Entry) {
|
|
|
|
// Apparently 2 is a good threshold when Entry==2^14, but we need to scale
|
|
|
|
// it. Divide by 2^13, rounding as appropriate.
|
|
|
|
uint64_t Freq = Entry.getFrequency();
|
|
|
|
uint64_t Scaled = (Freq >> 13) + bool(Freq & (1 << 12));
|
|
|
|
Threshold = std::max(UINT64_C(1), Scaled);
|
|
|
|
}
|
2011-01-06 09:21:53 +08:00
|
|
|
|
2011-04-07 03:13:57 +08:00
|
|
|
/// addConstraints - Compute node biases and weights from a set of constraints.
|
2011-01-06 09:21:53 +08:00
|
|
|
/// Set a bit in NodeMask for each active node.
|
2011-04-07 03:13:57 +08:00
|
|
|
void SpillPlacement::addConstraints(ArrayRef<BlockConstraint> LiveBlocks) {
|
|
|
|
for (ArrayRef<BlockConstraint>::iterator I = LiveBlocks.begin(),
|
2011-01-06 09:21:53 +08:00
|
|
|
E = LiveBlocks.end(); I != E; ++I) {
|
2013-07-17 02:26:15 +08:00
|
|
|
BlockFrequency Freq = BlockFrequencies[I->Number];
|
2011-01-06 09:21:53 +08:00
|
|
|
|
|
|
|
// Live-in to block?
|
|
|
|
if (I->Entry != DontCare) {
|
2017-09-22 07:20:16 +08:00
|
|
|
unsigned ib = bundles->getBundle(I->Number, false);
|
2011-01-06 09:21:53 +08:00
|
|
|
activate(ib);
|
2013-07-17 02:26:15 +08:00
|
|
|
nodes[ib].addBias(Freq, I->Entry);
|
2011-01-06 09:21:53 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// Live-out from block?
|
|
|
|
if (I->Exit != DontCare) {
|
2017-09-22 07:20:16 +08:00
|
|
|
unsigned ob = bundles->getBundle(I->Number, true);
|
2011-01-06 09:21:53 +08:00
|
|
|
activate(ob);
|
2013-07-17 02:26:15 +08:00
|
|
|
nodes[ob].addBias(Freq, I->Exit);
|
2011-01-06 09:21:53 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-07-23 11:10:19 +08:00
|
|
|
/// addPrefSpill - Same as addConstraints(PrefSpill)
|
2011-08-04 07:09:38 +08:00
|
|
|
void SpillPlacement::addPrefSpill(ArrayRef<unsigned> Blocks, bool Strong) {
|
2011-07-23 11:10:19 +08:00
|
|
|
for (ArrayRef<unsigned>::iterator I = Blocks.begin(), E = Blocks.end();
|
|
|
|
I != E; ++I) {
|
2013-07-17 02:26:15 +08:00
|
|
|
BlockFrequency Freq = BlockFrequencies[*I];
|
2011-08-04 07:09:38 +08:00
|
|
|
if (Strong)
|
|
|
|
Freq += Freq;
|
2017-09-22 07:20:16 +08:00
|
|
|
unsigned ib = bundles->getBundle(*I, false);
|
|
|
|
unsigned ob = bundles->getBundle(*I, true);
|
2011-07-23 11:10:19 +08:00
|
|
|
activate(ib);
|
|
|
|
activate(ob);
|
2013-07-17 02:26:15 +08:00
|
|
|
nodes[ib].addBias(Freq, PrefSpill);
|
|
|
|
nodes[ob].addBias(Freq, PrefSpill);
|
2011-07-23 11:10:19 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-04-08 01:27:46 +08:00
|
|
|
void SpillPlacement::addLinks(ArrayRef<unsigned> Links) {
|
|
|
|
for (ArrayRef<unsigned>::iterator I = Links.begin(), E = Links.end(); I != E;
|
|
|
|
++I) {
|
|
|
|
unsigned Number = *I;
|
2017-09-22 07:20:16 +08:00
|
|
|
unsigned ib = bundles->getBundle(Number, false);
|
|
|
|
unsigned ob = bundles->getBundle(Number, true);
|
2011-04-08 01:27:46 +08:00
|
|
|
|
|
|
|
// Ignore self-loops.
|
|
|
|
if (ib == ob)
|
|
|
|
continue;
|
|
|
|
activate(ib);
|
|
|
|
activate(ob);
|
2013-07-17 02:26:15 +08:00
|
|
|
BlockFrequency Freq = BlockFrequencies[Number];
|
|
|
|
nodes[ib].addLink(ob, Freq);
|
|
|
|
nodes[ob].addLink(ib, Freq);
|
2011-04-08 01:27:46 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-04-09 10:59:09 +08:00
|
|
|
bool SpillPlacement::scanActiveBundles() {
|
|
|
|
RecentPositive.clear();
|
2017-05-17 09:07:53 +08:00
|
|
|
for (unsigned n : ActiveNodes->set_bits()) {
|
Reapply r263460: [SpillPlacement] Fix a quadratic behavior in spill placement.
Using Chandler's words from r265331:
This commit was greatly exacerbating PR17409 and effectively regressed
build time for lot of (very large) code when compiled with ASan or MSan.
PR17409 is fixed by r269249, so this is fine to reapply r263460.
Original commit message:
The bad behavior happens when we have a function with a long linear
chain of basic blocks, and have a live range spanning most of this
chain, but with very few uses.
Let say we have only 2 uses.
The Hopfield network is only seeded with two active blocks where the
uses are, and each iteration of the outer loop in
`RAGreedy::growRegion()` only adds two new nodes to the network due to
the completely linear shape of the CFG. Meanwhile,
`SpillPlacer->iterate()` visits the whole set of discovered nodes, which
adds up to a quadratic algorithm.
This is an historical accident effect from r129188.
When the Hopfield network is expanding, most of the action is happening
on the frontier where new nodes are being added. The internal nodes in
the network are not likely to be flip-flopping much, or they will at
least settle down very quickly. This means that while
`SpillPlacer->iterate()` is recomputing all the nodes in the network, it
is probably only the two frontier nodes that are changing their output.
Instead of recomputing the whole network on each iteration, we can
maintain a SparseSet of nodes that need to be updated:
- `SpillPlacement::activate()` adds the node to the todo list.
- When a node changes value (i.e., `update()` returns true), its
neighbors are added to the todo list.
- `SpillPlacement::iterate()` only updates the nodes in the list.
The result of Hopfield iterations is not necessarily exact. It should
converge to a local minimum, but there is no guarantee that it will find
a global minimum. It is possible that updating nodes in a different
order will cause us to switch to a different local minimum. In other
words, this is not NFC, but although I saw a few runtime improvements
and regressions when I benchmarked this change, those were side effects
and actually the performance change is in the noise as expected.
Huge thanks to Jakob Stoklund Olesen <stoklund@2pi.dk> for his
feedbacks, guidance and time for the review.
llvm-svn: 270149
2016-05-20 06:40:37 +08:00
|
|
|
update(n);
|
2011-04-09 10:59:09 +08:00
|
|
|
// A node that must spill, or a node without any links is not going to
|
|
|
|
// change its value ever again, so exclude it from iterations.
|
|
|
|
if (nodes[n].mustSpill())
|
|
|
|
continue;
|
|
|
|
if (nodes[n].preferReg())
|
|
|
|
RecentPositive.push_back(n);
|
|
|
|
}
|
|
|
|
return !RecentPositive.empty();
|
|
|
|
}
|
|
|
|
|
Reapply r263460: [SpillPlacement] Fix a quadratic behavior in spill placement.
Using Chandler's words from r265331:
This commit was greatly exacerbating PR17409 and effectively regressed
build time for lot of (very large) code when compiled with ASan or MSan.
PR17409 is fixed by r269249, so this is fine to reapply r263460.
Original commit message:
The bad behavior happens when we have a function with a long linear
chain of basic blocks, and have a live range spanning most of this
chain, but with very few uses.
Let say we have only 2 uses.
The Hopfield network is only seeded with two active blocks where the
uses are, and each iteration of the outer loop in
`RAGreedy::growRegion()` only adds two new nodes to the network due to
the completely linear shape of the CFG. Meanwhile,
`SpillPlacer->iterate()` visits the whole set of discovered nodes, which
adds up to a quadratic algorithm.
This is an historical accident effect from r129188.
When the Hopfield network is expanding, most of the action is happening
on the frontier where new nodes are being added. The internal nodes in
the network are not likely to be flip-flopping much, or they will at
least settle down very quickly. This means that while
`SpillPlacer->iterate()` is recomputing all the nodes in the network, it
is probably only the two frontier nodes that are changing their output.
Instead of recomputing the whole network on each iteration, we can
maintain a SparseSet of nodes that need to be updated:
- `SpillPlacement::activate()` adds the node to the todo list.
- When a node changes value (i.e., `update()` returns true), its
neighbors are added to the todo list.
- `SpillPlacement::iterate()` only updates the nodes in the list.
The result of Hopfield iterations is not necessarily exact. It should
converge to a local minimum, but there is no guarantee that it will find
a global minimum. It is possible that updating nodes in a different
order will cause us to switch to a different local minimum. In other
words, this is not NFC, but although I saw a few runtime improvements
and regressions when I benchmarked this change, those were side effects
and actually the performance change is in the noise as expected.
Huge thanks to Jakob Stoklund Olesen <stoklund@2pi.dk> for his
feedbacks, guidance and time for the review.
llvm-svn: 270149
2016-05-20 06:40:37 +08:00
|
|
|
bool SpillPlacement::update(unsigned n) {
|
|
|
|
if (!nodes[n].update(nodes, Threshold))
|
|
|
|
return false;
|
|
|
|
nodes[n].getDissentingNeighbors(TodoList, nodes);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2011-01-06 09:21:53 +08:00
|
|
|
/// iterate - Repeatedly update the Hopfield nodes until stability or the
|
|
|
|
/// maximum number of iterations is reached.
|
2011-04-09 10:59:09 +08:00
|
|
|
void SpillPlacement::iterate() {
|
Reapply r263460: [SpillPlacement] Fix a quadratic behavior in spill placement.
Using Chandler's words from r265331:
This commit was greatly exacerbating PR17409 and effectively regressed
build time for lot of (very large) code when compiled with ASan or MSan.
PR17409 is fixed by r269249, so this is fine to reapply r263460.
Original commit message:
The bad behavior happens when we have a function with a long linear
chain of basic blocks, and have a live range spanning most of this
chain, but with very few uses.
Let say we have only 2 uses.
The Hopfield network is only seeded with two active blocks where the
uses are, and each iteration of the outer loop in
`RAGreedy::growRegion()` only adds two new nodes to the network due to
the completely linear shape of the CFG. Meanwhile,
`SpillPlacer->iterate()` visits the whole set of discovered nodes, which
adds up to a quadratic algorithm.
This is an historical accident effect from r129188.
When the Hopfield network is expanding, most of the action is happening
on the frontier where new nodes are being added. The internal nodes in
the network are not likely to be flip-flopping much, or they will at
least settle down very quickly. This means that while
`SpillPlacer->iterate()` is recomputing all the nodes in the network, it
is probably only the two frontier nodes that are changing their output.
Instead of recomputing the whole network on each iteration, we can
maintain a SparseSet of nodes that need to be updated:
- `SpillPlacement::activate()` adds the node to the todo list.
- When a node changes value (i.e., `update()` returns true), its
neighbors are added to the todo list.
- `SpillPlacement::iterate()` only updates the nodes in the list.
The result of Hopfield iterations is not necessarily exact. It should
converge to a local minimum, but there is no guarantee that it will find
a global minimum. It is possible that updating nodes in a different
order will cause us to switch to a different local minimum. In other
words, this is not NFC, but although I saw a few runtime improvements
and regressions when I benchmarked this change, those were side effects
and actually the performance change is in the noise as expected.
Huge thanks to Jakob Stoklund Olesen <stoklund@2pi.dk> for his
feedbacks, guidance and time for the review.
llvm-svn: 270149
2016-05-20 06:40:37 +08:00
|
|
|
// We do not need to push those node in the todolist.
|
|
|
|
// They are already been proceeded as part of the previous iteration.
|
|
|
|
RecentPositive.clear();
|
2016-04-05 02:57:50 +08:00
|
|
|
|
Reapply r263460: [SpillPlacement] Fix a quadratic behavior in spill placement.
Using Chandler's words from r265331:
This commit was greatly exacerbating PR17409 and effectively regressed
build time for lot of (very large) code when compiled with ASan or MSan.
PR17409 is fixed by r269249, so this is fine to reapply r263460.
Original commit message:
The bad behavior happens when we have a function with a long linear
chain of basic blocks, and have a live range spanning most of this
chain, but with very few uses.
Let say we have only 2 uses.
The Hopfield network is only seeded with two active blocks where the
uses are, and each iteration of the outer loop in
`RAGreedy::growRegion()` only adds two new nodes to the network due to
the completely linear shape of the CFG. Meanwhile,
`SpillPlacer->iterate()` visits the whole set of discovered nodes, which
adds up to a quadratic algorithm.
This is an historical accident effect from r129188.
When the Hopfield network is expanding, most of the action is happening
on the frontier where new nodes are being added. The internal nodes in
the network are not likely to be flip-flopping much, or they will at
least settle down very quickly. This means that while
`SpillPlacer->iterate()` is recomputing all the nodes in the network, it
is probably only the two frontier nodes that are changing their output.
Instead of recomputing the whole network on each iteration, we can
maintain a SparseSet of nodes that need to be updated:
- `SpillPlacement::activate()` adds the node to the todo list.
- When a node changes value (i.e., `update()` returns true), its
neighbors are added to the todo list.
- `SpillPlacement::iterate()` only updates the nodes in the list.
The result of Hopfield iterations is not necessarily exact. It should
converge to a local minimum, but there is no guarantee that it will find
a global minimum. It is possible that updating nodes in a different
order will cause us to switch to a different local minimum. In other
words, this is not NFC, but although I saw a few runtime improvements
and regressions when I benchmarked this change, those were side effects
and actually the performance change is in the noise as expected.
Huge thanks to Jakob Stoklund Olesen <stoklund@2pi.dk> for his
feedbacks, guidance and time for the review.
llvm-svn: 270149
2016-05-20 06:40:37 +08:00
|
|
|
// Since the last iteration, the todolist have been augmented by calls
|
|
|
|
// to addConstraints, addLinks, and co.
|
|
|
|
// Update the network energy starting at this new frontier.
|
|
|
|
// The call to ::update will add the nodes that changed into the todolist.
|
|
|
|
unsigned Limit = bundles->getNumBundles() * 10;
|
|
|
|
while(Limit-- > 0 && !TodoList.empty()) {
|
|
|
|
unsigned n = TodoList.pop_back_val();
|
|
|
|
if (!update(n))
|
|
|
|
continue;
|
|
|
|
if (nodes[n].preferReg())
|
|
|
|
RecentPositive.push_back(n);
|
2011-01-06 09:21:53 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-04-07 03:13:57 +08:00
|
|
|
void SpillPlacement::prepare(BitVector &RegBundles) {
|
2011-04-09 10:59:09 +08:00
|
|
|
RecentPositive.clear();
|
Reapply r263460: [SpillPlacement] Fix a quadratic behavior in spill placement.
Using Chandler's words from r265331:
This commit was greatly exacerbating PR17409 and effectively regressed
build time for lot of (very large) code when compiled with ASan or MSan.
PR17409 is fixed by r269249, so this is fine to reapply r263460.
Original commit message:
The bad behavior happens when we have a function with a long linear
chain of basic blocks, and have a live range spanning most of this
chain, but with very few uses.
Let say we have only 2 uses.
The Hopfield network is only seeded with two active blocks where the
uses are, and each iteration of the outer loop in
`RAGreedy::growRegion()` only adds two new nodes to the network due to
the completely linear shape of the CFG. Meanwhile,
`SpillPlacer->iterate()` visits the whole set of discovered nodes, which
adds up to a quadratic algorithm.
This is an historical accident effect from r129188.
When the Hopfield network is expanding, most of the action is happening
on the frontier where new nodes are being added. The internal nodes in
the network are not likely to be flip-flopping much, or they will at
least settle down very quickly. This means that while
`SpillPlacer->iterate()` is recomputing all the nodes in the network, it
is probably only the two frontier nodes that are changing their output.
Instead of recomputing the whole network on each iteration, we can
maintain a SparseSet of nodes that need to be updated:
- `SpillPlacement::activate()` adds the node to the todo list.
- When a node changes value (i.e., `update()` returns true), its
neighbors are added to the todo list.
- `SpillPlacement::iterate()` only updates the nodes in the list.
The result of Hopfield iterations is not necessarily exact. It should
converge to a local minimum, but there is no guarantee that it will find
a global minimum. It is possible that updating nodes in a different
order will cause us to switch to a different local minimum. In other
words, this is not NFC, but although I saw a few runtime improvements
and regressions when I benchmarked this change, those were side effects
and actually the performance change is in the noise as expected.
Huge thanks to Jakob Stoklund Olesen <stoklund@2pi.dk> for his
feedbacks, guidance and time for the review.
llvm-svn: 270149
2016-05-20 06:40:37 +08:00
|
|
|
TodoList.clear();
|
2011-01-06 09:21:53 +08:00
|
|
|
// Reuse RegBundles as our ActiveNodes vector.
|
|
|
|
ActiveNodes = &RegBundles;
|
|
|
|
ActiveNodes->clear();
|
|
|
|
ActiveNodes->resize(bundles->getNumBundles());
|
2011-04-07 03:13:57 +08:00
|
|
|
}
|
2011-01-06 09:21:53 +08:00
|
|
|
|
2011-04-07 03:13:57 +08:00
|
|
|
bool
|
|
|
|
SpillPlacement::finish() {
|
|
|
|
assert(ActiveNodes && "Call prepare() first");
|
2011-01-06 09:21:53 +08:00
|
|
|
|
2011-04-07 03:13:57 +08:00
|
|
|
// Write preferences back to ActiveNodes.
|
2011-01-06 09:21:53 +08:00
|
|
|
bool Perfect = true;
|
2017-05-17 09:07:53 +08:00
|
|
|
for (unsigned n : ActiveNodes->set_bits())
|
2011-01-06 09:21:53 +08:00
|
|
|
if (!nodes[n].preferReg()) {
|
2011-04-07 03:13:57 +08:00
|
|
|
ActiveNodes->reset(n);
|
2011-01-06 09:21:53 +08:00
|
|
|
Perfect = false;
|
|
|
|
}
|
2014-04-14 08:51:57 +08:00
|
|
|
ActiveNodes = nullptr;
|
2011-01-06 09:21:53 +08:00
|
|
|
return Perfect;
|
|
|
|
}
|