Added support for ThinLTO plugin options : thinlto-index-only and thinlto-prefix-replace

Differential Revision: https://reviews.llvm.org/D46034

llvm-svn: 331405
This commit is contained in:
Rumeet Dhindsa 2018-05-02 21:40:07 +00:00
parent e3b437935f
commit d366e36bbf
10 changed files with 221 additions and 31 deletions

View File

@ -94,6 +94,8 @@ struct Configuration {
llvm::StringRef SoName;
llvm::StringRef Sysroot;
llvm::StringRef ThinLTOCacheDir;
llvm::StringRef ThinLTOIndexOnlyObjectsFile;
llvm::StringRef ThinLTOPrefixReplace;
std::string Rpath;
std::vector<VersionDefinition> VersionDefinitions;
std::vector<llvm::StringRef> AuxiliaryList;
@ -156,6 +158,7 @@ struct Configuration {
bool SysvHash = false;
bool Target1Rel;
bool Trace;
bool ThinLTOIndexOnly;
bool UndefinedVersion;
bool WarnBackrefs;
bool WarnCommon;

View File

@ -777,27 +777,37 @@ void LinkerDriver::readConfigs(opt::InputArgList &Args) {
// Parse LTO plugin-related options for compatibility with gold.
for (auto *Arg : Args.filtered(OPT_plugin_opt)) {
StringRef S = Arg->getValue();
if (S == "disable-verify")
if (S == "disable-verify") {
Config->DisableVerify = true;
else if (S == "save-temps")
} else if (S == "save-temps") {
Config->SaveTemps = true;
else if (S.startswith("O"))
} else if (S.startswith("O")) {
Config->LTOO = parseInt(S.substr(1), Arg);
else if (S.startswith("lto-partitions="))
} else if (S.startswith("lto-partitions=")) {
Config->LTOPartitions = parseInt(S.substr(15), Arg);
else if (S.startswith("jobs="))
} else if (S.startswith("jobs=")) {
Config->ThinLTOJobs = parseInt(S.substr(5), Arg);
else if (S.startswith("mcpu="))
} else if (S.startswith("mcpu=")) {
parseClangOption(Saver.save("-" + S), Arg->getSpelling());
else if (S == "new-pass-manager")
} else if (S == "new-pass-manager") {
Config->LTONewPassManager = true;
else if (S == "debug-pass-manager")
} else if (S == "debug-pass-manager") {
Config->LTODebugPassManager = true;
else if (S.startswith("sample-profile="))
Config->LTOSampleProfile = S.substr(strlen("sample-profile="));
else if (!S.startswith("/") && !S.startswith("-fresolution=") &&
!S.startswith("-pass-through=") && !S.startswith("thinlto"))
} else if (S.startswith("sample-profile=")) {
Config->LTOSampleProfile = S.substr(15);
} else if (S == "thinlto-index-only") {
Config->ThinLTOIndexOnly = true;
} else if (S.startswith("thinlto-index-only=")) {
Config->ThinLTOIndexOnly = true;
Config->ThinLTOIndexOnlyObjectsFile = S.substr(19);
} else if (S.startswith("thinlto-prefix-replace=")) {
Config->ThinLTOPrefixReplace = S.substr(23);
if (!Config->ThinLTOPrefixReplace.contains(';'))
error("thinlto-prefix-replace expects 'old;new' format");
} else if (!S.startswith("/") && !S.startswith("-fresolution=") &&
!S.startswith("-pass-through=") && !S.startswith("thinlto")) {
parseClangOption(S, Arg->getSpelling());
}
}
// Parse -mllvm options.

View File

@ -42,6 +42,7 @@ bool InputFile::IsInGroup;
uint32_t InputFile::NextGroupId;
std::vector<BinaryFile *> elf::BinaryFiles;
std::vector<BitcodeFile *> elf::BitcodeFiles;
std::vector<LazyObjFile *> elf::LazyObjFiles;
std::vector<InputFile *> elf::ObjectFiles;
std::vector<InputFile *> elf::SharedFiles;
@ -1024,9 +1025,10 @@ BitcodeFile::BitcodeFile(MemoryBufferRef MB, StringRef ArchiveName,
// this causes a collision which result in only one of the objects being
// taken into consideration at LTO time (which very likely causes undefined
// symbols later in the link stage).
MemoryBufferRef MBRef(MB.getBuffer(),
Saver.save(ArchiveName + MB.getBufferIdentifier() +
utostr(OffsetInArchive)));
MemoryBufferRef MBRef(
MB.getBuffer(),
Saver.save(ArchiveName + MB.getBufferIdentifier() +
(ArchiveName.empty() ? "" : utostr(OffsetInArchive))));
Obj = CHECK(lto::InputFile::create(MBRef), this);
Triple T(Obj->getTargetTriple());
@ -1128,11 +1130,6 @@ void BinaryFile::parse() {
Data.size(), 0, STB_GLOBAL, nullptr, nullptr);
}
static bool isBitcode(MemoryBufferRef MB) {
using namespace sys::fs;
return identify_magic(MB.getBuffer()) == file_magic::bitcode;
}
InputFile *elf::createObjectFile(MemoryBufferRef MB, StringRef ArchiveName,
uint64_t OffsetInArchive) {
if (isBitcode(MB))
@ -1168,9 +1165,9 @@ InputFile *elf::createSharedFile(MemoryBufferRef MB, StringRef DefaultSoName) {
}
MemoryBufferRef LazyObjFile::getBuffer() {
if (Seen)
if (AddedToLink)
return MemoryBufferRef();
Seen = true;
AddedToLink = true;
return MB;
}

View File

@ -259,11 +259,11 @@ public:
template <class ELFT> void parse();
MemoryBufferRef getBuffer();
InputFile *fetch();
bool AddedToLink = false;
private:
template <class ELFT> void addElfSymbols();
bool Seen = false;
uint64_t OffsetInArchive;
};
@ -351,8 +351,13 @@ InputFile *createObjectFile(MemoryBufferRef MB, StringRef ArchiveName = "",
uint64_t OffsetInArchive = 0);
InputFile *createSharedFile(MemoryBufferRef MB, StringRef DefaultSoName);
inline bool isBitcode(MemoryBufferRef MB) {
return identify_magic(MB.getBuffer()) == llvm::file_magic::bitcode;
}
extern std::vector<BinaryFile *> BinaryFiles;
extern std::vector<BitcodeFile *> BitcodeFiles;
extern std::vector<LazyObjFile *> LazyObjFiles;
extern std::vector<InputFile *> ObjectFiles;
extern std::vector<InputFile *> SharedFiles;

View File

@ -20,6 +20,8 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
#include "llvm/BinaryFormat/ELF.h"
#include "llvm/Bitcode/BitcodeReader.h"
#include "llvm/Bitcode/BitcodeWriter.h"
#include "llvm/IR/DiagnosticPrinter.h"
#include "llvm/LTO/Caching.h"
#include "llvm/LTO/Config.h"
@ -29,7 +31,6 @@
#include "llvm/Support/Error.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/raw_ostream.h"
#include <algorithm>
#include <cstddef>
#include <memory>
@ -66,7 +67,57 @@ static void checkError(Error E) {
[&](ErrorInfoBase &EIB) { error(EIB.message()); });
}
static std::unique_ptr<lto::LTO> createLTO() {
// With the ThinLTOIndexOnly option, only the thin link is performed, and will
// generate index files for the ThinLTO backends in a distributed build system.
// The distributed build system may expect that index files are created for all
// input bitcode objects provided to the linker for the thin link. However,
// index files will not normally be created for input bitcode objects that
// either aren't selected by the linker (i.e. in a static library and not
// needed), or because they don't have a summary. Therefore we need to create
// empty dummy index file outputs in those cases.
// If SkipModule is true then .thinlto.bc should contain just
// SkipModuleByDistributedBackend flag which requests distributed backend
// to skip the compilation of the corresponding module and produce an empty
// object file.
static void writeEmptyDistributedBuildOutputs(const std::string &ModulePath,
const std::string &OldPrefix,
const std::string &NewPrefix,
bool SkipModule) {
std::string NewModulePath =
lto::getThinLTOOutputFile(ModulePath, OldPrefix, NewPrefix);
std::error_code EC;
raw_fd_ostream OS(NewModulePath + ".thinlto.bc", EC,
sys::fs::OpenFlags::F_None);
if (EC)
error("failed to write " + NewModulePath + ".thinlto.bc" + ": " +
EC.message());
if (SkipModule) {
ModuleSummaryIndex Index(false);
Index.setSkipModuleByDistributedBackend();
WriteIndexToFile(Index, OS);
}
}
// Creates and returns output stream with a list of object files for final
// linking of distributed ThinLTO.
static std::unique_ptr<raw_fd_ostream> createLinkedObjectsFile() {
if (Config->ThinLTOIndexOnlyObjectsFile.empty())
return nullptr;
std::error_code EC;
auto LinkedObjectsFile = llvm::make_unique<raw_fd_ostream>(
Config->ThinLTOIndexOnlyObjectsFile, EC, sys::fs::OpenFlags::F_None);
if (EC)
error("cannot create " + Config->ThinLTOIndexOnlyObjectsFile + ": " +
EC.message());
return LinkedObjectsFile;
}
// Creates instance of LTO.
// LinkedObjectsFile is an output stream to write the list of object files for
// the final ThinLTO linking. Can be nullptr.
static std::unique_ptr<lto::LTO> createLTO(raw_fd_ostream *LinkedObjectsFile) {
lto::Config Conf;
// LLD supports the new relocations.
@ -105,6 +156,13 @@ static std::unique_ptr<lto::LTO> createLTO() {
if (Config->ThinLTOJobs != -1u)
Backend = lto::createInProcessThinBackend(Config->ThinLTOJobs);
if (Config->ThinLTOIndexOnly) {
std::string OldPrefix, NewPrefix;
std::tie(OldPrefix, NewPrefix) = Config->ThinLTOPrefixReplace.split(';');
Backend = lto::createWriteIndexesThinBackend(OldPrefix, NewPrefix, true,
LinkedObjectsFile, nullptr);
}
Conf.SampleProfile = Config->LTOSampleProfile;
Conf.UseNewPM = Config->LTONewPassManager;
Conf.DebugPassManager = Config->LTODebugPassManager;
@ -113,7 +171,9 @@ static std::unique_ptr<lto::LTO> createLTO() {
Config->LTOPartitions);
}
BitcodeCompiler::BitcodeCompiler() : LTOObj(createLTO()) {
BitcodeCompiler::BitcodeCompiler() {
LinkedObjects = createLinkedObjectsFile();
LTOObj = createLTO(LinkedObjects.get());
for (Symbol *Sym : Symtab->getSymbols()) {
StringRef Name = Sym->getName();
for (StringRef Prefix : {"__start_", "__stop_"})
@ -131,6 +191,13 @@ static void undefine(Symbol *S) {
void BitcodeCompiler::add(BitcodeFile &F) {
lto::InputFile &Obj = *F.Obj;
std::string OldPrefix, NewPrefix;
std::tie(OldPrefix, NewPrefix) = Config->ThinLTOPrefixReplace.split(';');
// Create the empty files which, if indexed, will be overwritten later.
writeEmptyDistributedBuildOutputs(Obj.getName(), OldPrefix, NewPrefix, false);
unsigned SymNum = 0;
std::vector<Symbol *> Syms = F.getSymbols();
std::vector<lto::SymbolResolution> Resols(Syms.size());
@ -188,6 +255,12 @@ std::vector<InputFile *> BitcodeCompiler::compile() {
Buff.resize(MaxTasks);
Files.resize(MaxTasks);
// If LazyObjFile has not been added to link, emit empty index files
if (Config->ThinLTOIndexOnly)
for (LazyObjFile *F : LazyObjFiles)
if (!F->AddedToLink && isBitcode(F->MB))
addLazyObjFile(F);
// The --thinlto-cache-dir option specifies the path to a directory in which
// to cache native object files for ThinLTO incremental builds. If a path was
// specified, configure LTO to use it as the cache directory.
@ -222,9 +295,24 @@ std::vector<InputFile *> BitcodeCompiler::compile() {
Ret.push_back(Obj);
}
// ThinLTO with index only option is required to generate only the index
// files. After that, we exit from linker and ThinLTO backend runs in a
// distributed environment.
if (Config->ThinLTOIndexOnly)
exit(0);
for (std::unique_ptr<MemoryBuffer> &File : Files)
if (File)
Ret.push_back(createObjectFile(*File));
return Ret;
}
// For lazy object files not added to link, adds empty index files
void BitcodeCompiler::addLazyObjFile(LazyObjFile *File) {
StringRef Identifier = File->getBuffer().getBufferIdentifier();
std::string OldPrefix, NewPrefix;
std::tie(OldPrefix, NewPrefix) = Config->ThinLTOPrefixReplace.split(';');
writeEmptyDistributedBuildOutputs(Identifier, OldPrefix, NewPrefix,
/* SkipModule */ true);
}

View File

@ -24,6 +24,7 @@
#include "lld/Common/LLVM.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/Support/raw_ostream.h"
#include <memory>
#include <vector>
@ -38,6 +39,7 @@ namespace elf {
class BitcodeFile;
class InputFile;
class LazyObjFile;
class BitcodeCompiler {
public:
@ -46,12 +48,14 @@ public:
void add(BitcodeFile &F);
std::vector<InputFile *> compile();
void addLazyObjFile(LazyObjFile *File);
private:
std::unique_ptr<llvm::lto::LTO> LTOObj;
std::vector<SmallString<0>> Buff;
std::vector<std::unique_ptr<MemoryBuffer>> Files;
llvm::DenseSet<StringRef> UsedStartStop;
std::unique_ptr<llvm::raw_fd_ostream> LinkedObjects;
};
} // namespace elf
} // namespace lld

View File

@ -82,6 +82,7 @@ template <class ELFT> void SymbolTable::addFile(InputFile *File) {
// Lazy object file
if (auto *F = dyn_cast<LazyObjFile>(File)) {
LazyObjFiles.push_back(F);
F->parse<ELFT>();
return;
}

View File

@ -0,0 +1,2 @@
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

View File

@ -1,7 +1,37 @@
; REQUIRES: x86
; First ensure that the ThinLTO handling in lld handles
; bitcode without summary sections gracefully and generates index file.
; RUN: llvm-as %s -o %t.o
; RUN: llvm-as %p/Inputs/thinlto.ll -o %t2.o
; RUN: ld.lld -m elf_x86_64 --plugin-opt=thinlto-index-only -shared %t.o %t2.o -o %t3
; RUN: ls %t2.o.thinlto.bc
; RUN: not test -e %t3
; RUN: ld.lld -m elf_x86_64 -shared %t.o %t2.o -o %t4
; RUN: llvm-nm %t4 | FileCheck %s --check-prefix=NM
; Basic ThinLTO tests.
; RUN: opt -module-summary %s -o %t.o
; RUN: opt -module-summary %p/Inputs/thinlto.ll -o %t2.o
; RUN: opt -module-summary %p/Inputs/thinlto_empty.ll -o %t4.o
; Ensure lld generates an index and not a binary if requested.
; RUN: ld.lld -m elf_x86_64 --plugin-opt=thinlto-index-only -shared %t.o %t2.o -o %t3
; RUN: llvm-bcanalyzer -dump %t.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND1
; RUN: llvm-bcanalyzer -dump %t2.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND2
; RUN: not test -e %t3
; Ensure lld generates an index even if the file is wrapped in --start-lib/--end-lib
; RUN: rm -f %t2.o.thinlto.bc
; RUN: ld.lld -m elf_x86_64 --plugin-opt=thinlto-index-only -shared %t.o %t4.o --start-lib %t2.o --end-lib -o %t5
; RUN: ls %t2.o.thinlto.bc
; Ensure lld generates error if unable to write to index file
; RUN: rm -f %t4.o.thinlto.bc
; RUN: touch %t4.o.thinlto.bc
; RUN: chmod 400 %t4.o.thinlto.bc
; RUN: ld.lld -m elf_x86_64 --plugin-opt=thinlto-index-only -shared %t.o %t4.o -o %t5 2>&1 | FileCheck %s --check-prefix=ERR
; ERR: failed to write {{.*}}4.o.thinlto.bc: Permission denied
; First force single-threaded mode
; RUN: rm -f %t.lto.o %t1.lto.o
@ -15,16 +45,43 @@
; RUN: llvm-nm %t21.lto.o | FileCheck %s --check-prefix=NM1
; RUN: llvm-nm %t22.lto.o | FileCheck %s --check-prefix=NM2
; NM1: T f
; NM1-NOT: U g
; NM2: T g
; Then check without --thinlto-jobs (which currently default to hardware_concurrency)
; We just check that we don't crash or fail (as it's not sure which tests are
; stable on the final output file itself.
; RUN: ld.lld -shared %t.o %t2.o -o %t2
; NM: T f
; NM1: T f
; NM1-NOT: U g
; NM2: T g
; The backend index for this module contains summaries from itself and
; Inputs/thinlto.ll, as it imports from the latter.
; BACKEND1: <MODULE_STRTAB_BLOCK
; BACKEND1-NEXT: <ENTRY {{.*}} record string = '{{.*}}/thinlto.ll.tmp{{.*}}.o'
; BACKEND1-NEXT: <ENTRY {{.*}} record string = '{{.*}}/thinlto.ll.tmp{{.*}}.o'
; BACKEND1-NEXT: </MODULE_STRTAB_BLOCK
; BACKEND1: <GLOBALVAL_SUMMARY_BLOCK
; BACKEND1: <VERSION
; BACKEND1: <FLAGS
; BACKEND1: <VALUE_GUID op0={{1|2}} op1={{-3706093650706652785|-5300342847281564238}}
; BACKEND1: <VALUE_GUID op0={{1|2}} op1={{-3706093650706652785|-5300342847281564238}}
; BACKEND1: <COMBINED
; BACKEND1: <COMBINED
; BACKEND1: </GLOBALVAL_SUMMARY_BLOCK
; The backend index for Input/thinlto.ll contains summaries from itself only,
; as it does not import anything.
; BACKEND2: <MODULE_STRTAB_BLOCK
; BACKEND2-NEXT: <ENTRY {{.*}} record string = '{{.*}}/thinlto.ll.tmp2.o'
; BACKEND2-NEXT: </MODULE_STRTAB_BLOCK
; BACKEND2-NEXT: <GLOBALVAL_SUMMARY_BLOCK
; BACKEND2-NEXT: <VERSION
; BACKEND2-NEXT: <FLAGS
; BACKEND2-NEXT: <VALUE_GUID op0=1 op1=-5300342847281564238
; BACKEND2-NEXT: <COMBINED
; BACKEND2-NEXT: </GLOBALVAL_SUMMARY_BLOCK
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

View File

@ -0,0 +1,23 @@
; REQUIRES: x86
; Check that changing the output path via thinlto-prefix-replace works
; RUN: mkdir -p %t/oldpath
; RUN: opt -module-summary %s -o %t/oldpath/thinlto_prefix_replace.o
; Ensure that there is no existing file at the new path, so we properly
; test the creation of the new file there.
; RUN: rm -f %t/newpath/thinlto_prefix_replace.o.thinlto.bc
; RUN: ld.lld --plugin-opt=thinlto-index-only --plugin-opt=thinlto-prefix-replace="%t/oldpath/;%t/newpath/" -shared %t/oldpath/thinlto_prefix_replace.o -o %t/thinlto_prefix_replace
; RUN: ls %t/newpath/thinlto_prefix_replace.o.thinlto.bc
; Ensure that lld generates error if prefix replace option does not have 'old;new' format
; RUN: rm -f %t/newpath/thinlto_prefix_replace.o.thinlto.bc
; RUN: not ld.lld --plugin-opt=thinlto-index-only --plugin-opt=thinlto-prefix-replace="%t/oldpath/:%t/newpath/" -shared %t/oldpath/thinlto_prefix_replace.o -o %t/thinlto_prefix_replace 2>&1 | FileCheck %s --check-prefix=ERR
; ERR: thinlto-prefix-replace expects 'old;new' format
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"
define void @f() {
entry:
ret void
}