2017-09-14 05:15:20 +08:00
|
|
|
//===- BranchRelaxation.cpp -----------------------------------------------===//
|
2014-03-29 18:18:08 +08:00
|
|
|
//
|
2019-01-19 16:50:56 +08:00
|
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
2014-03-29 18:18:08 +08:00
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
#include "llvm/ADT/SmallVector.h"
|
2014-07-25 19:42:14 +08:00
|
|
|
#include "llvm/ADT/Statistic.h"
|
2016-12-17 07:55:37 +08:00
|
|
|
#include "llvm/CodeGen/LivePhysRegs.h"
|
2017-09-14 05:15:20 +08:00
|
|
|
#include "llvm/CodeGen/MachineBasicBlock.h"
|
|
|
|
#include "llvm/CodeGen/MachineFunction.h"
|
2014-03-29 18:18:08 +08:00
|
|
|
#include "llvm/CodeGen/MachineFunctionPass.h"
|
2017-09-14 05:15:20 +08:00
|
|
|
#include "llvm/CodeGen/MachineInstr.h"
|
2016-10-07 00:20:41 +08:00
|
|
|
#include "llvm/CodeGen/RegisterScavenging.h"
|
2017-11-08 09:01:31 +08:00
|
|
|
#include "llvm/CodeGen/TargetInstrInfo.h"
|
2017-11-17 09:07:10 +08:00
|
|
|
#include "llvm/CodeGen/TargetRegisterInfo.h"
|
|
|
|
#include "llvm/CodeGen/TargetSubtargetInfo.h"
|
2018-04-30 22:59:11 +08:00
|
|
|
#include "llvm/Config/llvm-config.h"
|
2017-09-14 05:15:20 +08:00
|
|
|
#include "llvm/IR/DebugLoc.h"
|
Sink all InitializePasses.h includes
This file lists every pass in LLVM, and is included by Pass.h, which is
very popular. Every time we add, remove, or rename a pass in LLVM, it
caused lots of recompilation.
I found this fact by looking at this table, which is sorted by the
number of times a file was changed over the last 100,000 git commits
multiplied by the number of object files that depend on it in the
current checkout:
recompiles touches affected_files header
342380 95 3604 llvm/include/llvm/ADT/STLExtras.h
314730 234 1345 llvm/include/llvm/InitializePasses.h
307036 118 2602 llvm/include/llvm/ADT/APInt.h
213049 59 3611 llvm/include/llvm/Support/MathExtras.h
170422 47 3626 llvm/include/llvm/Support/Compiler.h
162225 45 3605 llvm/include/llvm/ADT/Optional.h
158319 63 2513 llvm/include/llvm/ADT/Triple.h
140322 39 3598 llvm/include/llvm/ADT/StringRef.h
137647 59 2333 llvm/include/llvm/Support/Error.h
131619 73 1803 llvm/include/llvm/Support/FileSystem.h
Before this change, touching InitializePasses.h would cause 1345 files
to recompile. After this change, touching it only causes 550 compiles in
an incremental rebuild.
Reviewers: bkramer, asbirlea, bollu, jdoerfert
Differential Revision: https://reviews.llvm.org/D70211
2019-11-14 05:15:01 +08:00
|
|
|
#include "llvm/InitializePasses.h"
|
2017-09-14 05:15:20 +08:00
|
|
|
#include "llvm/Pass.h"
|
|
|
|
#include "llvm/Support/Compiler.h"
|
2014-03-29 18:18:08 +08:00
|
|
|
#include "llvm/Support/Debug.h"
|
|
|
|
#include "llvm/Support/Format.h"
|
2017-09-14 05:15:20 +08:00
|
|
|
#include "llvm/Support/MathExtras.h"
|
2014-03-29 18:18:08 +08:00
|
|
|
#include "llvm/Support/raw_ostream.h"
|
2017-09-14 05:15:20 +08:00
|
|
|
#include <cassert>
|
|
|
|
#include <cstdint>
|
|
|
|
#include <iterator>
|
|
|
|
#include <memory>
|
2016-10-06 23:38:53 +08:00
|
|
|
|
2014-03-29 18:18:08 +08:00
|
|
|
using namespace llvm;
|
|
|
|
|
2016-10-06 23:38:53 +08:00
|
|
|
#define DEBUG_TYPE "branch-relaxation"
|
2014-04-22 10:41:26 +08:00
|
|
|
|
2014-03-29 18:18:08 +08:00
|
|
|
STATISTIC(NumSplit, "Number of basic blocks split");
|
2016-08-23 09:30:30 +08:00
|
|
|
STATISTIC(NumConditionalRelaxed, "Number of conditional branches relaxed");
|
2016-10-07 00:20:41 +08:00
|
|
|
STATISTIC(NumUnconditionalRelaxed, "Number of unconditional branches relaxed");
|
2014-03-29 18:18:08 +08:00
|
|
|
|
2016-10-06 23:38:53 +08:00
|
|
|
#define BRANCH_RELAX_NAME "Branch relaxation pass"
|
2015-08-06 00:12:10 +08:00
|
|
|
|
2014-03-29 18:18:08 +08:00
|
|
|
namespace {
|
2017-09-14 05:15:20 +08:00
|
|
|
|
2016-10-06 23:38:53 +08:00
|
|
|
class BranchRelaxation : public MachineFunctionPass {
|
2014-03-29 18:18:08 +08:00
|
|
|
/// BasicBlockInfo - Information about the offset and size of a single
|
|
|
|
/// basic block.
|
|
|
|
struct BasicBlockInfo {
|
|
|
|
/// Offset - Distance from the beginning of the function to the beginning
|
|
|
|
/// of this basic block.
|
|
|
|
///
|
|
|
|
/// The offset is always aligned as required by the basic block.
|
2017-09-14 05:15:20 +08:00
|
|
|
unsigned Offset = 0;
|
2014-03-29 18:18:08 +08:00
|
|
|
|
|
|
|
/// Size - Size of the basic block in bytes. If the block contains
|
|
|
|
/// inline assembly, this is a worst case estimate.
|
|
|
|
///
|
|
|
|
/// The size does not include any alignment padding whether from the
|
|
|
|
/// beginning of the block, or from an aligned jump table at the end.
|
2017-09-14 05:15:20 +08:00
|
|
|
unsigned Size = 0;
|
2014-03-29 18:18:08 +08:00
|
|
|
|
2017-09-14 05:15:20 +08:00
|
|
|
BasicBlockInfo() = default;
|
2014-03-29 18:18:08 +08:00
|
|
|
|
2016-10-07 00:00:58 +08:00
|
|
|
/// Compute the offset immediately following this block. \p MBB is the next
|
|
|
|
/// block.
|
|
|
|
unsigned postOffset(const MachineBasicBlock &MBB) const {
|
[Alignment] Use llvm::Align in MachineFunction and TargetLowering - fixes mir parsing
Summary:
This catches malformed mir files which specify alignment as log2 instead of pow2.
See https://reviews.llvm.org/D65945 for reference,
This is patch is part of a series to introduce an Alignment type.
See this thread for context: http://lists.llvm.org/pipermail/llvm-dev/2019-July/133851.html
See this patch for the introduction of the type: https://reviews.llvm.org/D64790
Reviewers: courbet
Subscribers: MatzeB, qcolombet, dschuff, arsenm, sdardis, nemanjai, jvesely, nhaehnle, hiraditya, kbarton, asb, rbar, johnrusso, simoncook, apazos, sabuasal, niosHD, jrtc27, MaskRay, zzheng, edward-jones, atanasyan, rogfer01, MartinMosbeck, brucehoult, the_o, PkmX, jocewei, jsji, Petar.Avramovic, asbirlea, s.egerton, pzheng, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D67433
llvm-svn: 371608
2019-09-11 19:16:48 +08:00
|
|
|
const unsigned PO = Offset + Size;
|
2019-09-27 20:54:21 +08:00
|
|
|
const Align Alignment = MBB.getAlignment();
|
|
|
|
const Align ParentAlign = MBB.getParent()->getAlignment();
|
|
|
|
if (Alignment <= ParentAlign)
|
2020-01-20 06:53:45 +08:00
|
|
|
return alignTo(PO, Alignment);
|
2016-10-07 00:00:58 +08:00
|
|
|
|
|
|
|
// The alignment of this MBB is larger than the function's alignment, so we
|
|
|
|
// can't tell whether or not it will insert nops. Assume that it will.
|
2020-01-20 06:53:45 +08:00
|
|
|
return alignTo(PO, Alignment) + Alignment.value() - ParentAlign.value();
|
2014-03-29 18:18:08 +08:00
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
SmallVector<BasicBlockInfo, 16> BlockInfo;
|
2016-10-07 00:20:41 +08:00
|
|
|
std::unique_ptr<RegScavenger> RS;
|
2016-12-17 07:55:37 +08:00
|
|
|
LivePhysRegs LiveRegs;
|
2014-03-29 18:18:08 +08:00
|
|
|
|
|
|
|
MachineFunction *MF;
|
2016-12-17 07:55:37 +08:00
|
|
|
const TargetRegisterInfo *TRI;
|
2016-10-06 23:38:53 +08:00
|
|
|
const TargetInstrInfo *TII;
|
2014-03-29 18:18:08 +08:00
|
|
|
|
|
|
|
bool relaxBranchInstructions();
|
|
|
|
void scanFunction();
|
2016-10-07 00:20:41 +08:00
|
|
|
|
|
|
|
MachineBasicBlock *createNewBlockAfter(MachineBasicBlock &BB);
|
|
|
|
|
2016-11-03 00:18:29 +08:00
|
|
|
MachineBasicBlock *splitBlockBeforeInstr(MachineInstr &MI,
|
|
|
|
MachineBasicBlock *DestBB);
|
2018-07-17 02:51:40 +08:00
|
|
|
void adjustBlockOffsets(MachineBasicBlock &Start);
|
2016-08-02 16:06:17 +08:00
|
|
|
bool isBlockInRange(const MachineInstr &MI, const MachineBasicBlock &BB) const;
|
2016-08-02 16:30:06 +08:00
|
|
|
|
2016-08-02 15:20:09 +08:00
|
|
|
bool fixupConditionalBranch(MachineInstr &MI);
|
2016-10-07 00:20:41 +08:00
|
|
|
bool fixupUnconditionalBranch(MachineInstr &MI);
|
2016-10-06 23:38:53 +08:00
|
|
|
uint64_t computeBlockSize(const MachineBasicBlock &MBB) const;
|
2016-08-02 16:06:17 +08:00
|
|
|
unsigned getInstrOffset(const MachineInstr &MI) const;
|
2014-03-29 18:18:08 +08:00
|
|
|
void dumpBBs();
|
|
|
|
void verify();
|
|
|
|
|
|
|
|
public:
|
|
|
|
static char ID;
|
2017-09-14 05:15:20 +08:00
|
|
|
|
|
|
|
BranchRelaxation() : MachineFunctionPass(ID) {}
|
2014-03-29 18:18:08 +08:00
|
|
|
|
2014-04-29 15:58:25 +08:00
|
|
|
bool runOnMachineFunction(MachineFunction &MF) override;
|
2014-03-29 18:18:08 +08:00
|
|
|
|
2017-09-14 05:15:20 +08:00
|
|
|
StringRef getPassName() const override { return BRANCH_RELAX_NAME; }
|
2014-03-29 18:18:08 +08:00
|
|
|
};
|
2016-10-06 23:38:53 +08:00
|
|
|
|
2017-09-14 05:15:20 +08:00
|
|
|
} // end anonymous namespace
|
2014-03-29 18:18:08 +08:00
|
|
|
|
2016-10-06 23:38:53 +08:00
|
|
|
char BranchRelaxation::ID = 0;
|
2017-09-14 05:15:20 +08:00
|
|
|
|
2016-10-06 23:38:53 +08:00
|
|
|
char &llvm::BranchRelaxationPassID = BranchRelaxation::ID;
|
|
|
|
|
|
|
|
INITIALIZE_PASS(BranchRelaxation, DEBUG_TYPE, BRANCH_RELAX_NAME, false, false)
|
2015-08-06 00:12:10 +08:00
|
|
|
|
2014-03-29 18:18:08 +08:00
|
|
|
/// verify - check BBOffsets, BBSizes, alignment of islands
|
2016-10-06 23:38:53 +08:00
|
|
|
void BranchRelaxation::verify() {
|
2014-03-29 18:18:08 +08:00
|
|
|
#ifndef NDEBUG
|
|
|
|
unsigned PrevNum = MF->begin()->getNumber();
|
2014-04-16 08:42:46 +08:00
|
|
|
for (MachineBasicBlock &MBB : *MF) {
|
[Alignment][NFC] Remove LogAlignment functions
Summary:
This is patch is part of a series to introduce an Alignment type.
See this thread for context: http://lists.llvm.org/pipermail/llvm-dev/2019-July/133851.html
See this patch for the introduction of the type: https://reviews.llvm.org/D64790
Reviewers: courbet
Subscribers: arsenm, sdardis, nemanjai, jvesely, nhaehnle, hiraditya, kbarton, jrtc27, MaskRay, atanasyan, jsji, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D67620
llvm-svn: 372231
2019-09-18 23:49:49 +08:00
|
|
|
const unsigned Num = MBB.getNumber();
|
2016-10-07 00:00:58 +08:00
|
|
|
assert(!Num || BlockInfo[PrevNum].postOffset(MBB) <= BlockInfo[Num].Offset);
|
2016-11-03 00:18:29 +08:00
|
|
|
assert(BlockInfo[Num].Size == computeBlockSize(MBB));
|
2014-03-29 18:18:08 +08:00
|
|
|
PrevNum = Num;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2017-10-15 22:32:27 +08:00
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
2014-03-29 18:18:08 +08:00
|
|
|
/// print block size and offset information - debugging
|
2017-01-28 10:02:38 +08:00
|
|
|
LLVM_DUMP_METHOD void BranchRelaxation::dumpBBs() {
|
2014-04-04 07:43:26 +08:00
|
|
|
for (auto &MBB : *MF) {
|
2014-04-03 02:00:39 +08:00
|
|
|
const BasicBlockInfo &BBI = BlockInfo[MBB.getNumber()];
|
2019-08-17 02:23:54 +08:00
|
|
|
dbgs() << format("%%bb.%u\toffset=%08x\t", MBB.getNumber(), BBI.Offset)
|
2014-03-29 18:18:08 +08:00
|
|
|
<< format("size=%#x\n", BBI.Size);
|
|
|
|
}
|
|
|
|
}
|
2017-01-28 10:02:38 +08:00
|
|
|
#endif
|
2014-03-29 18:18:08 +08:00
|
|
|
|
|
|
|
/// scanFunction - Do the initial scan of the function, building up
|
|
|
|
/// information about each block.
|
2016-10-06 23:38:53 +08:00
|
|
|
void BranchRelaxation::scanFunction() {
|
2014-03-29 18:18:08 +08:00
|
|
|
BlockInfo.clear();
|
|
|
|
BlockInfo.resize(MF->getNumBlockIDs());
|
|
|
|
|
|
|
|
// First thing, compute the size of all basic blocks, and see if the function
|
|
|
|
// has any inline assembly in it. If so, we have to be conservative about
|
|
|
|
// alignment assumptions, as we don't know for sure the size of any
|
|
|
|
// instructions in the inline assembly.
|
2014-04-16 08:42:46 +08:00
|
|
|
for (MachineBasicBlock &MBB : *MF)
|
2016-10-06 23:38:53 +08:00
|
|
|
BlockInfo[MBB.getNumber()].Size = computeBlockSize(MBB);
|
2014-03-29 18:18:08 +08:00
|
|
|
|
|
|
|
// Compute block offsets and known bits.
|
2014-04-16 08:42:46 +08:00
|
|
|
adjustBlockOffsets(*MF->begin());
|
2014-03-29 18:18:08 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/// computeBlockSize - Compute the size for MBB.
|
2016-10-06 23:38:53 +08:00
|
|
|
uint64_t BranchRelaxation::computeBlockSize(const MachineBasicBlock &MBB) const {
|
|
|
|
uint64_t Size = 0;
|
2014-04-16 08:42:46 +08:00
|
|
|
for (const MachineInstr &MI : MBB)
|
2016-07-29 00:32:22 +08:00
|
|
|
Size += TII->getInstSizeInBytes(MI);
|
2016-10-06 23:38:53 +08:00
|
|
|
return Size;
|
2014-03-29 18:18:08 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/// getInstrOffset - Return the current offset of the specified machine
|
|
|
|
/// instruction from the start of the function. This offset changes as stuff is
|
|
|
|
/// moved around inside the function.
|
2016-10-06 23:38:53 +08:00
|
|
|
unsigned BranchRelaxation::getInstrOffset(const MachineInstr &MI) const {
|
2016-08-02 16:06:17 +08:00
|
|
|
const MachineBasicBlock *MBB = MI.getParent();
|
2014-03-29 18:18:08 +08:00
|
|
|
|
|
|
|
// The offset is composed of two things: the sum of the sizes of all MBB's
|
|
|
|
// before this instruction's block, and the offset from the start of the block
|
|
|
|
// it is in.
|
|
|
|
unsigned Offset = BlockInfo[MBB->getNumber()].Offset;
|
|
|
|
|
|
|
|
// Sum instructions before MI in MBB.
|
2016-08-02 16:06:17 +08:00
|
|
|
for (MachineBasicBlock::const_iterator I = MBB->begin(); &*I != &MI; ++I) {
|
2014-03-29 18:18:08 +08:00
|
|
|
assert(I != MBB->end() && "Didn't find MI in its own basic block?");
|
2016-07-29 00:32:22 +08:00
|
|
|
Offset += TII->getInstSizeInBytes(*I);
|
2014-03-29 18:18:08 +08:00
|
|
|
}
|
2016-08-02 15:20:09 +08:00
|
|
|
|
2014-03-29 18:18:08 +08:00
|
|
|
return Offset;
|
|
|
|
}
|
|
|
|
|
2016-10-06 23:38:53 +08:00
|
|
|
void BranchRelaxation::adjustBlockOffsets(MachineBasicBlock &Start) {
|
2014-04-16 08:42:46 +08:00
|
|
|
unsigned PrevNum = Start.getNumber();
|
2020-01-20 06:53:45 +08:00
|
|
|
for (auto &MBB :
|
|
|
|
make_range(std::next(MachineFunction::iterator(Start)), MF->end())) {
|
2014-04-16 08:42:46 +08:00
|
|
|
unsigned Num = MBB.getNumber();
|
2014-03-29 18:18:08 +08:00
|
|
|
// Get the offset and known bits at the end of the layout predecessor.
|
|
|
|
// Include the alignment of the current block.
|
2016-10-07 00:00:58 +08:00
|
|
|
BlockInfo[Num].Offset = BlockInfo[PrevNum].postOffset(MBB);
|
|
|
|
|
2014-03-29 18:18:08 +08:00
|
|
|
PrevNum = Num;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-09-14 05:15:20 +08:00
|
|
|
/// Insert a new empty basic block and insert it after \BB
|
2016-10-07 00:20:41 +08:00
|
|
|
MachineBasicBlock *BranchRelaxation::createNewBlockAfter(MachineBasicBlock &BB) {
|
|
|
|
// Create a new MBB for the code after the OrigBB.
|
|
|
|
MachineBasicBlock *NewBB =
|
|
|
|
MF->CreateMachineBasicBlock(BB.getBasicBlock());
|
|
|
|
MF->insert(++BB.getIterator(), NewBB);
|
|
|
|
|
|
|
|
// Insert an entry into BlockInfo to align it properly with the block numbers.
|
|
|
|
BlockInfo.insert(BlockInfo.begin() + NewBB->getNumber(), BasicBlockInfo());
|
|
|
|
|
|
|
|
return NewBB;
|
|
|
|
}
|
|
|
|
|
2014-03-29 18:18:08 +08:00
|
|
|
/// Split the basic block containing MI into two blocks, which are joined by
|
|
|
|
/// an unconditional branch. Update data structures and renumber blocks to
|
|
|
|
/// account for this change and returns the newly created block.
|
2016-11-03 00:18:29 +08:00
|
|
|
MachineBasicBlock *BranchRelaxation::splitBlockBeforeInstr(MachineInstr &MI,
|
|
|
|
MachineBasicBlock *DestBB) {
|
2016-08-02 15:20:09 +08:00
|
|
|
MachineBasicBlock *OrigBB = MI.getParent();
|
2014-03-29 18:18:08 +08:00
|
|
|
|
|
|
|
// Create a new MBB for the code after the OrigBB.
|
|
|
|
MachineBasicBlock *NewBB =
|
|
|
|
MF->CreateMachineBasicBlock(OrigBB->getBasicBlock());
|
2015-10-14 04:02:15 +08:00
|
|
|
MF->insert(++OrigBB->getIterator(), NewBB);
|
2014-03-29 18:18:08 +08:00
|
|
|
|
|
|
|
// Splice the instructions starting with MI over to NewBB.
|
2016-08-02 15:20:09 +08:00
|
|
|
NewBB->splice(NewBB->end(), OrigBB, MI.getIterator(), OrigBB->end());
|
2014-03-29 18:18:08 +08:00
|
|
|
|
|
|
|
// Add an unconditional branch from OrigBB to NewBB.
|
|
|
|
// Note the new unconditional branch is not being recorded.
|
|
|
|
// There doesn't seem to be meaningful DebugInfo available; this doesn't
|
|
|
|
// correspond to anything in the source.
|
2016-09-15 01:23:48 +08:00
|
|
|
TII->insertUnconditionalBranch(*OrigBB, NewBB, DebugLoc());
|
2014-03-29 18:18:08 +08:00
|
|
|
|
|
|
|
// Insert an entry into BlockInfo to align it properly with the block numbers.
|
|
|
|
BlockInfo.insert(BlockInfo.begin() + NewBB->getNumber(), BasicBlockInfo());
|
|
|
|
|
2016-11-03 00:18:29 +08:00
|
|
|
NewBB->transferSuccessors(OrigBB);
|
|
|
|
OrigBB->addSuccessor(NewBB);
|
|
|
|
OrigBB->addSuccessor(DestBB);
|
|
|
|
|
|
|
|
// Cleanup potential unconditional branch to successor block.
|
|
|
|
// Note that updateTerminator may change the size of the blocks.
|
|
|
|
NewBB->updateTerminator();
|
|
|
|
OrigBB->updateTerminator();
|
|
|
|
|
2014-03-29 18:18:08 +08:00
|
|
|
// Figure out how large the OrigBB is. As the first half of the original
|
|
|
|
// block, it cannot contain a tablejump. The size includes
|
|
|
|
// the new jump we added. (It should be possible to do this without
|
|
|
|
// recounting everything, but it's very confusing, and this is rarely
|
|
|
|
// executed.)
|
2016-10-06 23:38:53 +08:00
|
|
|
BlockInfo[OrigBB->getNumber()].Size = computeBlockSize(*OrigBB);
|
2014-03-29 18:18:08 +08:00
|
|
|
|
2016-10-06 23:38:53 +08:00
|
|
|
// Figure out how large the NewMBB is. As the second half of the original
|
2014-03-29 18:18:08 +08:00
|
|
|
// block, it may contain a tablejump.
|
2016-10-06 23:38:53 +08:00
|
|
|
BlockInfo[NewBB->getNumber()].Size = computeBlockSize(*NewBB);
|
2014-03-29 18:18:08 +08:00
|
|
|
|
|
|
|
// All BBOffsets following these blocks must be modified.
|
2014-04-16 08:42:46 +08:00
|
|
|
adjustBlockOffsets(*OrigBB);
|
2014-03-29 18:18:08 +08:00
|
|
|
|
2016-12-17 07:55:37 +08:00
|
|
|
// Need to fix live-in lists if we track liveness.
|
|
|
|
if (TRI->trackLivenessAfterRegAlloc(*MF))
|
2017-09-07 04:45:24 +08:00
|
|
|
computeAndAddLiveIns(LiveRegs, *NewBB);
|
2016-12-17 07:55:37 +08:00
|
|
|
|
2014-03-29 18:18:08 +08:00
|
|
|
++NumSplit;
|
|
|
|
|
|
|
|
return NewBB;
|
|
|
|
}
|
|
|
|
|
|
|
|
/// isBlockInRange - Returns true if the distance between specific MI and
|
|
|
|
/// specific BB can fit in MI's displacement field.
|
2016-10-06 23:38:53 +08:00
|
|
|
bool BranchRelaxation::isBlockInRange(
|
2016-08-02 16:06:17 +08:00
|
|
|
const MachineInstr &MI, const MachineBasicBlock &DestBB) const {
|
2016-10-06 23:38:09 +08:00
|
|
|
int64_t BrOffset = getInstrOffset(MI);
|
|
|
|
int64_t DestOffset = BlockInfo[DestBB.getNumber()].Offset;
|
2014-03-29 18:18:08 +08:00
|
|
|
|
2016-10-06 23:38:09 +08:00
|
|
|
if (TII->isBranchOffsetInRange(MI.getOpcode(), DestOffset - BrOffset))
|
2016-08-02 16:06:17 +08:00
|
|
|
return true;
|
2014-03-29 18:18:08 +08:00
|
|
|
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << "Out of range branch to destination "
|
|
|
|
<< printMBBReference(DestBB) << " from "
|
|
|
|
<< printMBBReference(*MI.getParent()) << " to "
|
|
|
|
<< DestOffset << " offset " << DestOffset - BrOffset << '\t'
|
|
|
|
<< MI);
|
2014-03-29 18:18:08 +08:00
|
|
|
|
2016-08-02 16:06:17 +08:00
|
|
|
return false;
|
2014-03-29 18:18:08 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/// fixupConditionalBranch - Fix up a conditional branch whose destination is
|
|
|
|
/// too far away to fit in its displacement field. It is converted to an inverse
|
|
|
|
/// conditional branch + an unconditional branch to the destination.
|
2016-10-06 23:38:53 +08:00
|
|
|
bool BranchRelaxation::fixupConditionalBranch(MachineInstr &MI) {
|
2019-04-23 09:42:07 +08:00
|
|
|
DebugLoc DL = MI.getDebugLoc();
|
2016-09-15 01:23:48 +08:00
|
|
|
MachineBasicBlock *MBB = MI.getParent();
|
|
|
|
MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
|
2018-01-04 15:08:45 +08:00
|
|
|
MachineBasicBlock *NewBB = nullptr;
|
2016-09-15 01:23:48 +08:00
|
|
|
SmallVector<MachineOperand, 4> Cond;
|
|
|
|
|
2018-01-04 15:08:45 +08:00
|
|
|
auto insertUncondBranch = [&](MachineBasicBlock *MBB,
|
|
|
|
MachineBasicBlock *DestBB) {
|
|
|
|
unsigned &BBSize = BlockInfo[MBB->getNumber()].Size;
|
|
|
|
int NewBrSize = 0;
|
|
|
|
TII->insertUnconditionalBranch(*MBB, DestBB, DL, &NewBrSize);
|
|
|
|
BBSize += NewBrSize;
|
|
|
|
};
|
|
|
|
auto insertBranch = [&](MachineBasicBlock *MBB, MachineBasicBlock *TBB,
|
|
|
|
MachineBasicBlock *FBB,
|
|
|
|
SmallVectorImpl<MachineOperand>& Cond) {
|
|
|
|
unsigned &BBSize = BlockInfo[MBB->getNumber()].Size;
|
|
|
|
int NewBrSize = 0;
|
|
|
|
TII->insertBranch(*MBB, TBB, FBB, Cond, DL, &NewBrSize);
|
|
|
|
BBSize += NewBrSize;
|
|
|
|
};
|
|
|
|
auto removeBranch = [&](MachineBasicBlock *MBB) {
|
|
|
|
unsigned &BBSize = BlockInfo[MBB->getNumber()].Size;
|
|
|
|
int RemovedSize = 0;
|
|
|
|
TII->removeBranch(*MBB, &RemovedSize);
|
|
|
|
BBSize -= RemovedSize;
|
|
|
|
};
|
|
|
|
|
|
|
|
auto finalizeBlockChanges = [&](MachineBasicBlock *MBB,
|
|
|
|
MachineBasicBlock *NewBB) {
|
|
|
|
// Keep the block offsets up to date.
|
|
|
|
adjustBlockOffsets(*MBB);
|
|
|
|
|
|
|
|
// Need to fix live-in lists if we track liveness.
|
|
|
|
if (NewBB && TRI->trackLivenessAfterRegAlloc(*MF))
|
|
|
|
computeAndAddLiveIns(LiveRegs, *NewBB);
|
|
|
|
};
|
|
|
|
|
2016-09-15 01:23:48 +08:00
|
|
|
bool Fail = TII->analyzeBranch(*MBB, TBB, FBB, Cond);
|
|
|
|
assert(!Fail && "branches to be relaxed must be analyzable");
|
|
|
|
(void)Fail;
|
2014-03-29 18:18:08 +08:00
|
|
|
|
|
|
|
// Add an unconditional branch to the destination and invert the branch
|
|
|
|
// condition to jump over it:
|
|
|
|
// tbz L1
|
|
|
|
// =>
|
|
|
|
// tbnz L2
|
|
|
|
// b L1
|
|
|
|
// L2:
|
|
|
|
|
2018-01-04 15:08:45 +08:00
|
|
|
bool ReversedCond = !TII->reverseBranchCondition(Cond);
|
|
|
|
if (ReversedCond) {
|
|
|
|
if (FBB && isBlockInRange(MI, *FBB)) {
|
|
|
|
// Last MI in the BB is an unconditional branch. We can simply invert the
|
|
|
|
// condition and swap destinations:
|
|
|
|
// beq L1
|
|
|
|
// b L2
|
|
|
|
// =>
|
|
|
|
// bne L2
|
|
|
|
// b L1
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << " Invert condition and swap "
|
|
|
|
"its destination with "
|
|
|
|
<< MBB->back());
|
2018-01-04 15:08:45 +08:00
|
|
|
|
|
|
|
removeBranch(MBB);
|
|
|
|
insertBranch(MBB, FBB, TBB, Cond);
|
|
|
|
finalizeBlockChanges(MBB, nullptr);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
if (FBB) {
|
|
|
|
// We need to split the basic block here to obtain two long-range
|
|
|
|
// unconditional branches.
|
|
|
|
NewBB = createNewBlockAfter(*MBB);
|
|
|
|
|
|
|
|
insertUncondBranch(NewBB, FBB);
|
|
|
|
// Update the succesor lists according to the transformation to follow.
|
|
|
|
// Do it here since if there's no split, no update is needed.
|
|
|
|
MBB->replaceSuccessor(FBB, NewBB);
|
|
|
|
NewBB->addSuccessor(FBB);
|
|
|
|
}
|
2017-05-27 08:53:48 +08:00
|
|
|
|
2018-01-04 15:08:45 +08:00
|
|
|
// We now have an appropriate fall-through block in place (either naturally or
|
|
|
|
// just created), so we can use the inverted the condition.
|
|
|
|
MachineBasicBlock &NextBB = *std::next(MachineFunction::iterator(MBB));
|
|
|
|
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << " Insert B to " << printMBBReference(*TBB)
|
|
|
|
<< ", invert condition and change dest. to "
|
|
|
|
<< printMBBReference(NextBB) << '\n');
|
2018-01-04 15:08:45 +08:00
|
|
|
|
|
|
|
removeBranch(MBB);
|
|
|
|
// Insert a new conditional branch and a new unconditional branch.
|
|
|
|
insertBranch(MBB, &NextBB, TBB, Cond);
|
|
|
|
|
|
|
|
finalizeBlockChanges(MBB, NewBB);
|
|
|
|
return true;
|
2014-03-29 18:18:08 +08:00
|
|
|
}
|
2018-01-04 15:08:45 +08:00
|
|
|
// Branch cond can't be inverted.
|
|
|
|
// In this case we always add a block after the MBB.
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << " The branch condition can't be inverted. "
|
|
|
|
<< " Insert a new BB after " << MBB->back());
|
2018-01-04 15:08:45 +08:00
|
|
|
|
|
|
|
if (!FBB)
|
|
|
|
FBB = &(*std::next(MachineFunction::iterator(MBB)));
|
2016-08-02 16:06:17 +08:00
|
|
|
|
2018-01-04 15:08:45 +08:00
|
|
|
// This is the block with cond. branch and the distance to TBB is too long.
|
|
|
|
// beq L1
|
|
|
|
// L2:
|
|
|
|
|
|
|
|
// We do the following transformation:
|
|
|
|
// beq NewBB
|
|
|
|
// b L2
|
|
|
|
// NewBB:
|
|
|
|
// b L1
|
|
|
|
// L2:
|
2014-03-29 18:18:08 +08:00
|
|
|
|
2018-01-04 15:08:45 +08:00
|
|
|
NewBB = createNewBlockAfter(*MBB);
|
|
|
|
insertUncondBranch(NewBB, TBB);
|
2014-03-29 18:18:08 +08:00
|
|
|
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << " Insert cond B to the new BB "
|
|
|
|
<< printMBBReference(*NewBB)
|
|
|
|
<< " Keep the exiting condition.\n"
|
|
|
|
<< " Insert B to " << printMBBReference(*FBB) << ".\n"
|
|
|
|
<< " In the new BB: Insert B to "
|
|
|
|
<< printMBBReference(*TBB) << ".\n");
|
2016-08-02 16:30:06 +08:00
|
|
|
|
2018-01-04 15:08:45 +08:00
|
|
|
// Update the successor lists according to the transformation to follow.
|
|
|
|
MBB->replaceSuccessor(TBB, NewBB);
|
|
|
|
NewBB->addSuccessor(TBB);
|
2016-09-15 01:23:48 +08:00
|
|
|
|
2018-01-04 15:08:45 +08:00
|
|
|
// Replace branch in the current (MBB) block.
|
|
|
|
removeBranch(MBB);
|
|
|
|
insertBranch(MBB, NewBB, FBB, Cond);
|
2014-03-29 18:18:08 +08:00
|
|
|
|
2018-01-04 15:08:45 +08:00
|
|
|
finalizeBlockChanges(MBB, NewBB);
|
2014-03-29 18:18:08 +08:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2016-10-07 00:20:41 +08:00
|
|
|
bool BranchRelaxation::fixupUnconditionalBranch(MachineInstr &MI) {
|
|
|
|
MachineBasicBlock *MBB = MI.getParent();
|
|
|
|
|
|
|
|
unsigned OldBrSize = TII->getInstSizeInBytes(MI);
|
|
|
|
MachineBasicBlock *DestBB = TII->getBranchDestBlock(MI);
|
|
|
|
|
|
|
|
int64_t DestOffset = BlockInfo[DestBB->getNumber()].Offset;
|
|
|
|
int64_t SrcOffset = getInstrOffset(MI);
|
|
|
|
|
|
|
|
assert(!TII->isBranchOffsetInRange(MI.getOpcode(), DestOffset - SrcOffset));
|
|
|
|
|
|
|
|
BlockInfo[MBB->getNumber()].Size -= OldBrSize;
|
|
|
|
|
|
|
|
MachineBasicBlock *BranchBB = MBB;
|
|
|
|
|
|
|
|
// If this was an expanded conditional branch, there is already a single
|
|
|
|
// unconditional branch in a block.
|
|
|
|
if (!MBB->empty()) {
|
|
|
|
BranchBB = createNewBlockAfter(*MBB);
|
|
|
|
|
|
|
|
// Add live outs.
|
|
|
|
for (const MachineBasicBlock *Succ : MBB->successors()) {
|
|
|
|
for (const MachineBasicBlock::RegisterMaskPair &LiveIn : Succ->liveins())
|
|
|
|
BranchBB->addLiveIn(LiveIn);
|
|
|
|
}
|
|
|
|
|
2016-10-12 23:32:04 +08:00
|
|
|
BranchBB->sortUniqueLiveIns();
|
2016-10-07 00:20:41 +08:00
|
|
|
BranchBB->addSuccessor(DestBB);
|
|
|
|
MBB->replaceSuccessor(DestBB, BranchBB);
|
|
|
|
}
|
|
|
|
|
2019-04-23 09:42:07 +08:00
|
|
|
DebugLoc DL = MI.getDebugLoc();
|
2016-10-07 00:20:41 +08:00
|
|
|
MI.eraseFromParent();
|
2016-11-03 00:18:29 +08:00
|
|
|
BlockInfo[BranchBB->getNumber()].Size += TII->insertIndirectBranch(
|
2016-10-07 00:20:41 +08:00
|
|
|
*BranchBB, *DestBB, DL, DestOffset - SrcOffset, RS.get());
|
|
|
|
|
|
|
|
adjustBlockOffsets(*MBB);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2016-10-06 23:38:53 +08:00
|
|
|
bool BranchRelaxation::relaxBranchInstructions() {
|
2014-03-29 18:18:08 +08:00
|
|
|
bool Changed = false;
|
2016-10-07 00:20:41 +08:00
|
|
|
|
2014-03-29 18:18:08 +08:00
|
|
|
// Relaxing branches involves creating new basic blocks, so re-eval
|
|
|
|
// end() for termination.
|
2016-06-17 05:21:49 +08:00
|
|
|
for (MachineFunction::iterator I = MF->begin(); I != MF->end(); ++I) {
|
|
|
|
MachineBasicBlock &MBB = *I;
|
2016-10-06 23:38:53 +08:00
|
|
|
|
2016-12-17 07:55:37 +08:00
|
|
|
// Empty block?
|
|
|
|
MachineBasicBlock::iterator Last = MBB.getLastNonDebugInstr();
|
|
|
|
if (Last == MBB.end())
|
2016-11-02 02:34:00 +08:00
|
|
|
continue;
|
|
|
|
|
|
|
|
// Expand the unconditional branch first if necessary. If there is a
|
|
|
|
// conditional branch, this will end up changing the branch destination of
|
|
|
|
// it to be over the newly inserted indirect branch block, which may avoid
|
|
|
|
// the need to try expanding the conditional branch first, saving an extra
|
|
|
|
// jump.
|
|
|
|
if (Last->isUnconditionalBranch()) {
|
|
|
|
// Unconditional branch destination might be unanalyzable, assume these
|
|
|
|
// are OK.
|
|
|
|
if (MachineBasicBlock *DestBB = TII->getBranchDestBlock(*Last)) {
|
|
|
|
if (!isBlockInRange(*Last, *DestBB)) {
|
|
|
|
fixupUnconditionalBranch(*Last);
|
|
|
|
++NumUnconditionalRelaxed;
|
|
|
|
Changed = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Loop over the conditional branches.
|
2016-08-23 09:30:30 +08:00
|
|
|
MachineBasicBlock::iterator Next;
|
|
|
|
for (MachineBasicBlock::iterator J = MBB.getFirstTerminator();
|
|
|
|
J != MBB.end(); J = Next) {
|
|
|
|
Next = std::next(J);
|
|
|
|
MachineInstr &MI = *J;
|
|
|
|
|
|
|
|
if (MI.isConditionalBranch()) {
|
2016-10-06 23:38:09 +08:00
|
|
|
MachineBasicBlock *DestBB = TII->getBranchDestBlock(MI);
|
2016-08-23 09:30:30 +08:00
|
|
|
if (!isBlockInRange(MI, *DestBB)) {
|
|
|
|
if (Next != MBB.end() && Next->isConditionalBranch()) {
|
|
|
|
// If there are multiple conditional branches, this isn't an
|
|
|
|
// analyzable block. Split later terminators into a new block so
|
|
|
|
// each one will be analyzable.
|
|
|
|
|
2016-11-03 00:18:29 +08:00
|
|
|
splitBlockBeforeInstr(*Next, DestBB);
|
2016-08-23 09:30:30 +08:00
|
|
|
} else {
|
|
|
|
fixupConditionalBranch(MI);
|
|
|
|
++NumConditionalRelaxed;
|
|
|
|
}
|
|
|
|
|
|
|
|
Changed = true;
|
|
|
|
|
|
|
|
// This may have modified all of the terminators, so start over.
|
|
|
|
Next = MBB.getFirstTerminator();
|
|
|
|
}
|
|
|
|
}
|
2014-03-29 18:18:08 +08:00
|
|
|
}
|
|
|
|
}
|
2016-08-23 09:30:30 +08:00
|
|
|
|
2014-03-29 18:18:08 +08:00
|
|
|
return Changed;
|
|
|
|
}
|
|
|
|
|
2016-10-06 23:38:53 +08:00
|
|
|
bool BranchRelaxation::runOnMachineFunction(MachineFunction &mf) {
|
2014-03-29 18:18:08 +08:00
|
|
|
MF = &mf;
|
|
|
|
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << "***** BranchRelaxation *****\n");
|
2014-03-29 18:18:08 +08:00
|
|
|
|
2016-10-07 00:20:41 +08:00
|
|
|
const TargetSubtargetInfo &ST = MF->getSubtarget();
|
|
|
|
TII = ST.getInstrInfo();
|
|
|
|
|
2016-12-17 07:55:37 +08:00
|
|
|
TRI = ST.getRegisterInfo();
|
2016-10-07 00:20:41 +08:00
|
|
|
if (TRI->trackLivenessAfterRegAlloc(*MF))
|
|
|
|
RS.reset(new RegScavenger());
|
2014-03-29 18:18:08 +08:00
|
|
|
|
|
|
|
// Renumber all of the machine basic blocks in the function, guaranteeing that
|
|
|
|
// the numbers agree with the position of the block in the function.
|
|
|
|
MF->RenumberBlocks();
|
|
|
|
|
|
|
|
// Do the initial scan of the function, building up information about the
|
|
|
|
// sizes of each block.
|
|
|
|
scanFunction();
|
|
|
|
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << " Basic blocks before relaxation\n"; dumpBBs(););
|
2014-03-29 18:18:08 +08:00
|
|
|
|
|
|
|
bool MadeChange = false;
|
|
|
|
while (relaxBranchInstructions())
|
|
|
|
MadeChange = true;
|
|
|
|
|
|
|
|
// After a while, this might be made debug-only, but it is not expensive.
|
|
|
|
verify();
|
|
|
|
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << " Basic blocks after relaxation\n\n"; dumpBBs());
|
2014-03-29 18:18:08 +08:00
|
|
|
|
|
|
|
BlockInfo.clear();
|
|
|
|
|
|
|
|
return MadeChange;
|
|
|
|
}
|