maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[llvm-reduce] Add support for LTO bitcode files 

Adds support for reading and writing LTO bitcode files.

  - Emit a summary if the original bitcode file had a summary
  - Use split LTO units if the original bitcode file used them.

Reviewed By: arsenm

Differential Revision: https://reviews.llvm.org/D127168
This commit is contained in:
Matthew Voss 2022-06-30 08:53:00 -07:00
parent 13f9089ac9
commit 6b3956e123
11 changed files with 154 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
llvm/test/tools/llvm-reduce/Inputs/llvm-dis-and-filecheck.py
View File

@ -13,6 +13,7 @@ Usage:
import sys
import os
import subprocess
llvm_dis = sys.argv[1]
@ -20,10 +21,23 @@ filecheck = sys.argv[2]
filecheck_args = [filecheck, ]
filecheck_args.extend(sys.argv[3:-1])
bitcode_file = sys.argv[-1]
ir_file = bitcode_file + ".ll"
disassemble = subprocess.Popen([llvm_dis, "-o", "-", bitcode_file],
stdout=subprocess.PIPE)
check = subprocess.Popen(filecheck_args, stdin=disassemble.stdout)
disassemble.stdout.close()
disassemble = subprocess.Popen([llvm_dis, "-o", ir_file, bitcode_file])
if os.path.exists(ir_file + ".0"):
ir_file = ir_file + ".0"
disassemble.communicate()
if disassemble.returncode != 0:
print("stderr:")
print(disassemble.stderr)
print("stdout:")
print(disassemble.stdout)
sys.exit(1)
check=None
with open(ir_file, "r") as ir:
check = subprocess.Popen(filecheck_args, stdin=ir, stdout=sys.stdout)
check.communicate()
sys.exit(check.returncode)

2
llvm/test/tools/llvm-reduce/fail-file-open.test
View File

@ -2,4 +2,4 @@
This file will not be read. An invalid file path is fed to llvm-reduce.
# CHECK: llvm-reduce{{.*}}: {{.*}}.NotAFileInTestingDir: error:
# CHECK: llvm-reduce{{.*}}: error: {{.*}}.NotAFileInTestingDir:

3
llvm/test/tools/llvm-reduce/invalid-bitcode-error.ll Normal file
View File

@ -0,0 +1,3 @@
; RUN: not %python %p/Inputs/llvm-dis-and-filecheck.py llvm-dis FileCheck %s %s 2>&1 | FileCheck %s
; CHECK: stderr
; CHECK: stdout

27
llvm/test/tools/llvm-reduce/temporary-files-as-bitcode-split.ll Normal file
View File

@ -0,0 +1,27 @@
; RUN: opt --thinlto-bc --thinlto-split-lto-unit %s -o %t0
; RUN: llvm-reduce -write-tmp-files-as-bitcode --delta-passes=basic-blocks %t0 -o %t1 \
; RUN: --test %python --test-arg %p/Inputs/llvm-dis-and-filecheck.py --test-arg llvm-dis --test-arg FileCheck --test-arg --check-prefixes=CHECK-ALL,CHECK-INTERESTINGNESS --test-arg %s
; RUN: cat %t1* | FileCheck --check-prefixes=CHECK-ALL,CHECK-FINAL %s
@g = internal global i8 42, !type !0
; CHECK-INTERESTINGNESS: @callee(
; CHECK-FINAL: declare void @callee()
define void @callee() {
ret void
}
; CHECK-ALL: define void @caller()
define void @caller() {
entry:
; CHECK-ALL: call void @callee()
; CHECK-ALL: ret void
call void @callee()
ret void
}
define i8* @f() {
; CHECK-ALL: ret i8* @g.{{([0-9a-f]{32})}}
ret i8* @g
}
!0 = !{i32 0, !"typeid"}

1
llvm/tools/llvm-reduce/CMakeLists.txt
View File

@ -14,6 +14,7 @@ set(LLVM_LINK_COMPONENTS
Support
Target
TransformUtils
IPO
)
add_llvm_tool(llvm-reduce

41
llvm/tools/llvm-reduce/ReducerWorkItem.cpp
View File

@ -7,6 +7,7 @@
//===----------------------------------------------------------------------===//
#include "ReducerWorkItem.h"
#include "llvm/Bitcode/BitcodeReader.h"
#include "llvm/CodeGen/CommandFlags.h"
#include "llvm/CodeGen/MIRParser/MIRParser.h"
#include "llvm/CodeGen/MIRPrinter.h"
@ -17,11 +18,14 @@
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/IR/ModuleSummaryIndex.h"
#include "llvm/IR/Verifier.h"
#include "llvm/IRReader/IRReader.h"
#include "llvm/MC/TargetRegistry.h"
#include "llvm/Support/Host.h"
#include "llvm/Support/MemoryBufferRef.h"
#include "llvm/Support/SourceMgr.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/WithColor.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/Utils/Cloning.h"
@ -31,6 +35,8 @@ static cl::opt<std::string> TargetTriple("mtriple",
cl::desc("Set the target triple"),
cl::cat(LLVMReduceOptions));
void readBitcode(ReducerWorkItem &M, MemoryBufferRef Data, LLVMContext &Ctx, const char *ToolName);
static void cloneFrameInfo(
MachineFrameInfo &DstMFI, const MachineFrameInfo &SrcMFI,
const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB) {
@ -352,6 +358,13 @@ static std::unique_ptr<MachineFunction> cloneMF(MachineFunction *SrcMF,
return DstMF;
}
static void initializeTargetInfo() {
InitializeAllTargets();
InitializeAllTargetMCs();
InitializeAllAsmPrinters();
InitializeAllAsmParsers();
}
std::unique_ptr<ReducerWorkItem>
parseReducerWorkItem(const char *ToolName, StringRef Filename,
LLVMContext &Ctxt, std::unique_ptr<TargetMachine> &TM,
@ -361,6 +374,8 @@ parseReducerWorkItem(const char *ToolName, StringRef Filename,
auto MMM = std::make_unique<ReducerWorkItem>();
if (IsMIR) {
initializeTargetInfo();
auto FileOrErr = MemoryBuffer::getFileOrSTDIN(Filename, /*IsText=*/true);
if (std::error_code EC = FileOrErr.getError()) {
WithColor::error(errs(), ToolName) << EC.message() << '\n';
@ -409,17 +424,31 @@ parseReducerWorkItem(const char *ToolName, StringRef Filename,
MMM->M = std::move(M);
} else {
SMDiagnostic Err;
std::unique_ptr<Module> Result = parseIRFile(Filename, Err, Ctxt);
if (!Result) {
Err.print(ToolName, errs());
return std::unique_ptr<ReducerWorkItem>();
ErrorOr<std::unique_ptr<MemoryBuffer>> MB = MemoryBuffer::getFileOrSTDIN(Filename);
if (std::error_code EC = MB.getError()) {
WithColor::error(errs(), ToolName) << Filename << ": " << EC.message() << "\n";
return nullptr;
}
if (!isBitcode((const unsigned char *)(*MB)->getBufferStart(),
(const unsigned char *)(*MB)->getBufferEnd())) {
std::unique_ptr<Module> Result = parseIRFile(Filename, Err, Ctxt);
if (!Result) {
Err.print(ToolName, errs());
return nullptr;
}
MMM->M = std::move(Result);
} else {
readBitcode(*MMM, MemoryBufferRef(**MB), Ctxt, ToolName);
if (MMM->LTOInfo->IsThinLTO && MMM->LTOInfo->EnableSplitLTOUnit)
initializeTargetInfo();
}
MMM->M = std::move(Result);
}
if (verifyReducerWorkItem(*MMM, &errs())) {
WithColor::error(errs(), ToolName)
<< Filename << " - input module is broken!\n";
return std::unique_ptr<ReducerWorkItem>();
return nullptr;
}
return MMM;
}

3
llvm/tools/llvm-reduce/ReducerWorkItem.h
View File

@ -9,9 +9,11 @@
#ifndef LLVM_TOOLS_LLVM_REDUCE_REDUCERWORKITEM_H
#define LLVM_TOOLS_LLVM_REDUCE_REDUCERWORKITEM_H
#include "llvm/Bitcode/BitcodeReader.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/ModuleSummaryIndex.h"
#include "llvm/Target/TargetMachine.h"
using namespace llvm;
@ -19,6 +21,7 @@ using namespace llvm;
class ReducerWorkItem {
public:
std::shared_ptr<Module> M;
std::unique_ptr<BitcodeLTOInfo> LTOInfo;
std::unique_ptr<MachineModuleInfo> MMI;
bool isMIR() const { return MMI != nullptr; }

4
llvm/tools/llvm-reduce/TestRunner.cpp
View File

@ -15,9 +15,9 @@ using namespace llvm;
TestRunner::TestRunner(StringRef TestName,
const std::vector<std::string> &TestArgs,
std::unique_ptr<ReducerWorkItem> Program,
std::unique_ptr<TargetMachine> TM)
std::unique_ptr<TargetMachine> TM, const char *ToolName)
: TestName(TestName), TestArgs(TestArgs), Program(std::move(Program)),
TM(std::move(TM)) {
TM(std::move(TM)), ToolName(ToolName) {
assert(this->Program && "Initialized with null program?");
}

5
llvm/tools/llvm-reduce/TestRunner.h
View File

@ -28,7 +28,7 @@ class TestRunner {
public:
TestRunner(StringRef TestName, const std::vector<std::string> &TestArgs,
std::unique_ptr<ReducerWorkItem> Program,
std::unique_ptr<TargetMachine> TM);
std::unique_ptr<TargetMachine> TM, const char *ToolName);
/// Runs the interesting-ness test for the specified file
/// @returns 0 if test was successful, 1 if otherwise
@ -41,8 +41,11 @@ public:
const TargetMachine *getTargetMachine() const { return TM.get(); }
const char *getToolName() const { return ToolName; }
private:
StringRef TestName;
const char *ToolName;
const std::vector<std::string> &TestArgs;
std::unique_ptr<ReducerWorkItem> Program;
std::unique_ptr<TargetMachine> TM;

37
llvm/tools/llvm-reduce/deltas/Delta.cpp
View File

@ -15,11 +15,16 @@
#include "Delta.h"
#include "ReducerWorkItem.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/ModuleSummaryAnalysis.h"
#include "llvm/Bitcode/BitcodeReader.h"
#include "llvm/Bitcode/BitcodeWriter.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Verifier.h"
#include "llvm/MC/TargetRegistry.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/MemoryBufferRef.h"
#include "llvm/Support/ThreadPool.h"
#include "llvm/Support/ToolOutputFile.h"
#include <fstream>
@ -56,6 +61,11 @@ unsigned NumJobs = 1;
void writeOutput(ReducerWorkItem &M, llvm::StringRef Message);
void writeBitcode(ReducerWorkItem &M, raw_ostream &OutStream);
void readBitcode(ReducerWorkItem &M, MemoryBufferRef Data, LLVMContext &Ctx,
const char *ToolName);
bool isReduced(ReducerWorkItem &M, TestRunner &Test,
SmallString<128> &CurrentFilepath) {
// Write ReducerWorkItem to tmp file
@ -70,7 +80,7 @@ bool isReduced(ReducerWorkItem &M, TestRunner &Test,
if (TmpFilesAsBitcode) {
llvm::raw_fd_ostream OutStream(FD, true);
WriteBitcodeToFile(M, OutStream);
writeBitcode(M, OutStream);
OutStream.close();
if (OutStream.has_error()) {
errs() << "Error emitting bitcode to file '" << CurrentFilepath << "'!\n";
@ -192,14 +202,10 @@ SmallString<0> ProcessChunkFromSerializedBitcode(
std::vector<Chunk> &ChunksStillConsideredInteresting,
SmallString<0> &OriginalBC, std::atomic<bool> &AnyReduced) {
LLVMContext Ctx;
Expected<std::unique_ptr<Module>> MOrErr = parseBitcodeFile(
MemoryBufferRef(StringRef(OriginalBC.data(), OriginalBC.size()),
"<llvm-reduce tmp module>"),
Ctx);
if (!MOrErr)
report_fatal_error("Failed to read bitcode");
auto CloneMMM = std::make_unique<ReducerWorkItem>();
CloneMMM->M = std::move(MOrErr.get());
auto Data = MemoryBufferRef(StringRef(OriginalBC.data(), OriginalBC.size()),
"<bc file>");
readBitcode(*CloneMMM, Data, Ctx, Test.getToolName());
SmallString<0> Result;
if (std::unique_ptr<ReducerWorkItem> ChunkResult =
@ -207,7 +213,7 @@ SmallString<0> ProcessChunkFromSerializedBitcode(
Test, ExtractChunksFromModule, UninterestingChunks,
ChunksStillConsideredInteresting)) {
raw_svector_ostream BCOS(Result);
WriteBitcodeToFile(*ChunkResult->M, BCOS);
writeBitcode(*ChunkResult, BCOS);
// Communicate that the task reduced a chunk.
AnyReduced = true;
}
@ -284,7 +290,7 @@ void llvm::runDeltaPass(TestRunner &Test,
SmallString<0> OriginalBC;
if (NumJobs > 1) {
raw_svector_ostream BCOS(OriginalBC);
WriteBitcodeToFile(*Test.getProgram().M, BCOS);
writeBitcode(Test.getProgram(), BCOS);
}
std::deque<std::shared_future<SmallString<0>>> TaskQueue;
@ -351,14 +357,11 @@ void llvm::runDeltaPass(TestRunner &Test,
continue;
}
Expected<std::unique_ptr<Module>> MOrErr = parseBitcodeFile(
MemoryBufferRef(StringRef(Res.data(), Res.size()),
"<llvm-reduce tmp module>"),
Test.getProgram().M->getContext());
if (!MOrErr)
report_fatal_error("Failed to read bitcode");
Result = std::make_unique<ReducerWorkItem>();
Result->M = std::move(MOrErr.get());
auto Data = MemoryBufferRef(StringRef(Res.data(), Res.size()),
"<bc file>");
readBitcode(*Result, Data, Test.getProgram().M->getContext(),
Test.getToolName());
break;
}
// Forward I to the last chunk processed in parallel.

51
llvm/tools/llvm-reduce/llvm-reduce.cpp
View File

@ -17,10 +17,15 @@
#include "DeltaManager.h"
#include "ReducerWorkItem.h"
#include "TestRunner.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/ModuleSummaryAnalysis.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/Bitcode/BitcodeReader.h"
#include "llvm/Bitcode/BitcodeWriter.h"
#include "llvm/CodeGen/CommandFlags.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/IR/LegacyPassManager.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Verifier.h"
#include "llvm/IRReader/IRReader.h"
@ -32,6 +37,7 @@
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/WithColor.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/IPO.h"
#include <system_error>
#include <vector>
@ -94,13 +100,6 @@ static cl::opt<int>
static codegen::RegisterCodeGenFlags CGF;
static void initializeTargetInfo() {
InitializeAllTargets();
InitializeAllTargetMCs();
InitializeAllAsmPrinters();
InitializeAllAsmParsers();
}
void writeOutput(ReducerWorkItem &M, StringRef Message) {
if (ReplaceInput) // In-place
OutputFilename = InputFilename.c_str();
@ -116,6 +115,39 @@ void writeOutput(ReducerWorkItem &M, StringRef Message) {
errs() << Message << OutputFilename << "\n";
}
void writeBitcode(ReducerWorkItem &M, llvm::raw_ostream &OutStream) {
if (M.LTOInfo && M.LTOInfo->IsThinLTO && M.LTOInfo->EnableSplitLTOUnit) {
legacy::PassManager PM;
PM.add(llvm::createWriteThinLTOBitcodePass(OutStream));
PM.run(*(M.M));
} else {
std::unique_ptr<ModuleSummaryIndex> Index;
if (M.LTOInfo && M.LTOInfo->HasSummary) {
ProfileSummaryInfo PSI(M);
Index = std::make_unique<ModuleSummaryIndex>(
buildModuleSummaryIndex(M, nullptr, &PSI));
}
WriteBitcodeToFile(M, OutStream, Index.get());
}
}
void readBitcode(ReducerWorkItem &M, MemoryBufferRef Data, LLVMContext &Ctx, const char *ToolName) {
Expected<BitcodeFileContents> IF = llvm::getBitcodeFileContents(Data);
if (!IF) {
WithColor::error(errs(), ToolName) << IF.takeError();
exit(1);
}
BitcodeModule BM = IF->Mods[0];
Expected<BitcodeLTOInfo> LI = BM.getLTOInfo();
Expected<std::unique_ptr<Module>> MOrErr = BM.parseModule(Ctx);
if (!LI || !MOrErr) {
WithColor::error(errs(), ToolName) << IF.takeError();
exit(1);
}
M.LTOInfo = std::make_unique<BitcodeLTOInfo>(*LI);
M.M = std::move(MOrErr.get());
}
int main(int Argc, char **Argv) {
InitLLVM X(Argc, Argv);
@ -135,9 +167,6 @@ int main(int Argc, char **Argv) {
return 0;
}
if (ReduceModeMIR)
initializeTargetInfo();
LLVMContext Context;
std::unique_ptr<TargetMachine> TM;
@ -149,7 +178,7 @@ int main(int Argc, char **Argv) {
// Initialize test environment
TestRunner Tester(TestFilename, TestArguments, std::move(OriginalProgram),
std::move(TM));
std::move(TM), Argv[0]);
// Try to reduce code
runDeltaPasses(Tester, MaxPassIterations);