2019-06-26 15:45:27 +08:00
|
|
|
//===--- QueryDriverDatabase.cpp ---------------------------------*- C++-*-===//
|
|
|
|
//
|
|
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
// Some compiler drivers have implicit search mechanism for system headers.
|
|
|
|
// This compilation database implementation tries to extract that information by
|
|
|
|
// executing the driver in verbose mode. gcc-compatible drivers print something
|
|
|
|
// like:
|
|
|
|
// ....
|
|
|
|
// ....
|
|
|
|
// #include <...> search starts here:
|
|
|
|
// /usr/lib/gcc/x86_64-linux-gnu/7/include
|
|
|
|
// /usr/local/include
|
|
|
|
// /usr/lib/gcc/x86_64-linux-gnu/7/include-fixed
|
|
|
|
// /usr/include/x86_64-linux-gnu
|
|
|
|
// /usr/include
|
|
|
|
// End of search list.
|
|
|
|
// ....
|
|
|
|
// ....
|
|
|
|
// This component parses that output and adds each path to command line args
|
|
|
|
// provided by Base, after prepending them with -isystem. Therefore current
|
|
|
|
// implementation would not work with a driver that is not gcc-compatible.
|
|
|
|
//
|
|
|
|
// First argument of the command line received from underlying compilation
|
|
|
|
// database is used as compiler driver path. Due to this arbitrary binary
|
|
|
|
// execution, this mechanism is not used by default and only executes binaries
|
2020-06-20 06:41:06 +08:00
|
|
|
// in the paths that are explicitly included by the user.
|
2019-06-26 15:45:27 +08:00
|
|
|
|
|
|
|
#include "GlobalCompilationDatabase.h"
|
[clangd] Move non-clang base pieces into separate support/ lib. NFCI
Summary:
This enforces layering, reduces a sprawling clangd/ directory, and makes life
easier for embedders.
Reviewers: kbobyrev
Subscribers: mgorny, ilya-biryukov, javed.absar, MaskRay, jkorous, arphaman, jfb, kadircet, usaxena95, cfe-commits
Tags: #clang
Differential Revision: https://reviews.llvm.org/D79014
2020-04-28 23:49:17 +08:00
|
|
|
#include "support/Logger.h"
|
|
|
|
#include "support/Path.h"
|
|
|
|
#include "support/Trace.h"
|
2020-11-26 19:58:22 +08:00
|
|
|
#include "clang/Basic/Diagnostic.h"
|
|
|
|
#include "clang/Basic/TargetInfo.h"
|
|
|
|
#include "clang/Basic/TargetOptions.h"
|
2019-06-26 15:45:27 +08:00
|
|
|
#include "clang/Driver/Types.h"
|
|
|
|
#include "clang/Tooling/CompilationDatabase.h"
|
|
|
|
#include "llvm/ADT/DenseMap.h"
|
|
|
|
#include "llvm/ADT/ScopeExit.h"
|
|
|
|
#include "llvm/ADT/SmallString.h"
|
|
|
|
#include "llvm/ADT/StringExtras.h"
|
|
|
|
#include "llvm/ADT/StringRef.h"
|
|
|
|
#include "llvm/ADT/iterator_range.h"
|
|
|
|
#include "llvm/Support/FileSystem.h"
|
|
|
|
#include "llvm/Support/MemoryBuffer.h"
|
|
|
|
#include "llvm/Support/Path.h"
|
|
|
|
#include "llvm/Support/Program.h"
|
|
|
|
#include "llvm/Support/Regex.h"
|
|
|
|
#include "llvm/Support/ScopedPrinter.h"
|
|
|
|
#include <algorithm>
|
2019-07-04 20:24:17 +08:00
|
|
|
#include <map>
|
2019-06-26 15:45:27 +08:00
|
|
|
#include <string>
|
|
|
|
#include <vector>
|
|
|
|
|
|
|
|
namespace clang {
|
|
|
|
namespace clangd {
|
|
|
|
namespace {
|
|
|
|
|
2020-11-26 19:58:22 +08:00
|
|
|
struct DriverInfo {
|
2019-06-26 15:45:27 +08:00
|
|
|
std::vector<std::string> SystemIncludes;
|
2020-11-26 19:58:22 +08:00
|
|
|
std::string Target;
|
|
|
|
};
|
|
|
|
|
|
|
|
bool isValidTarget(llvm::StringRef Triple) {
|
|
|
|
std::shared_ptr<TargetOptions> TargetOpts(new TargetOptions);
|
|
|
|
TargetOpts->Triple = Triple.str();
|
|
|
|
DiagnosticsEngine Diags(new DiagnosticIDs, new DiagnosticOptions,
|
|
|
|
new IgnoringDiagConsumer);
|
|
|
|
IntrusiveRefCntPtr<TargetInfo> Target =
|
|
|
|
TargetInfo::CreateTargetInfo(Diags, TargetOpts);
|
|
|
|
return bool(Target);
|
|
|
|
}
|
|
|
|
|
|
|
|
llvm::Optional<DriverInfo> parseDriverOutput(llvm::StringRef Output) {
|
|
|
|
DriverInfo Info;
|
2019-07-18 12:23:54 +08:00
|
|
|
const char SIS[] = "#include <...> search starts here:";
|
2019-07-19 17:08:22 +08:00
|
|
|
const char SIE[] = "End of search list.";
|
2020-11-26 19:58:22 +08:00
|
|
|
const char TS[] = "Target: ";
|
2020-12-10 20:36:35 +08:00
|
|
|
llvm::SmallVector<llvm::StringRef> Lines;
|
2019-06-26 15:45:27 +08:00
|
|
|
Output.split(Lines, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
|
|
|
|
|
2020-11-26 19:58:22 +08:00
|
|
|
enum {
|
|
|
|
Initial, // Initial state: searching for target or includes list.
|
|
|
|
IncludesExtracting, // Includes extracting.
|
|
|
|
Done // Includes and target extraction done.
|
|
|
|
} State = Initial;
|
|
|
|
bool SeenIncludes = false;
|
|
|
|
bool SeenTarget = false;
|
|
|
|
for (auto *It = Lines.begin(); State != Done && It != Lines.end(); ++It) {
|
|
|
|
auto Line = *It;
|
|
|
|
switch (State) {
|
|
|
|
case Initial:
|
|
|
|
if (!SeenIncludes && Line.trim() == SIS) {
|
|
|
|
SeenIncludes = true;
|
|
|
|
State = IncludesExtracting;
|
|
|
|
} else if (!SeenTarget && Line.trim().startswith(TS)) {
|
|
|
|
SeenTarget = true;
|
|
|
|
llvm::StringRef TargetLine = Line.trim();
|
|
|
|
TargetLine.consume_front(TS);
|
|
|
|
// Only detect targets that clang understands
|
|
|
|
if (!isValidTarget(TargetLine)) {
|
|
|
|
elog("System include extraction: invalid target \"{0}\", ignoring",
|
|
|
|
TargetLine);
|
|
|
|
} else {
|
|
|
|
Info.Target = TargetLine.str();
|
|
|
|
vlog("System include extraction: target extracted: \"{0}\"",
|
|
|
|
TargetLine);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case IncludesExtracting:
|
|
|
|
if (Line.trim() == SIE) {
|
|
|
|
State = SeenTarget ? Done : Initial;
|
|
|
|
} else {
|
|
|
|
Info.SystemIncludes.push_back(Line.trim().str());
|
|
|
|
vlog("System include extraction: adding {0}", Line);
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
llvm_unreachable("Impossible state of the driver output parser");
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (!SeenIncludes) {
|
2019-06-26 15:45:27 +08:00
|
|
|
elog("System include extraction: start marker not found: {0}", Output);
|
2020-11-26 19:58:22 +08:00
|
|
|
return llvm::None;
|
2019-06-26 15:45:27 +08:00
|
|
|
}
|
2020-11-26 19:58:22 +08:00
|
|
|
if (State == IncludesExtracting) {
|
2019-06-26 15:45:27 +08:00
|
|
|
elog("System include extraction: end marker missing: {0}", Output);
|
2020-11-26 19:58:22 +08:00
|
|
|
return llvm::None;
|
2019-06-26 15:45:27 +08:00
|
|
|
}
|
2020-11-26 19:58:22 +08:00
|
|
|
return std::move(Info);
|
2019-06-26 15:45:27 +08:00
|
|
|
}
|
|
|
|
|
2020-11-26 19:58:22 +08:00
|
|
|
llvm::Optional<DriverInfo>
|
|
|
|
extractSystemIncludesAndTarget(PathRef Driver, llvm::StringRef Lang,
|
|
|
|
llvm::ArrayRef<std::string> CommandLine,
|
|
|
|
const llvm::Regex &QueryDriverRegex) {
|
|
|
|
trace::Span Tracer("Extract system includes and target");
|
2019-06-26 15:45:27 +08:00
|
|
|
SPAN_ATTACH(Tracer, "driver", Driver);
|
2019-07-11 21:47:46 +08:00
|
|
|
SPAN_ATTACH(Tracer, "lang", Lang);
|
2019-06-26 15:45:27 +08:00
|
|
|
|
|
|
|
if (!QueryDriverRegex.match(Driver)) {
|
2020-06-20 06:41:06 +08:00
|
|
|
vlog("System include extraction: not allowed driver {0}", Driver);
|
2020-11-26 19:58:22 +08:00
|
|
|
return llvm::None;
|
2019-06-26 15:45:27 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
if (!llvm::sys::fs::exists(Driver)) {
|
|
|
|
elog("System include extraction: {0} does not exist.", Driver);
|
2020-11-26 19:58:22 +08:00
|
|
|
return llvm::None;
|
2019-06-26 15:45:27 +08:00
|
|
|
}
|
|
|
|
if (!llvm::sys::fs::can_execute(Driver)) {
|
|
|
|
elog("System include extraction: {0} is not executable.", Driver);
|
2020-11-26 19:58:22 +08:00
|
|
|
return llvm::None;
|
2019-06-26 15:45:27 +08:00
|
|
|
}
|
|
|
|
|
2019-07-04 19:34:23 +08:00
|
|
|
llvm::SmallString<128> StdErrPath;
|
|
|
|
if (auto EC = llvm::sys::fs::createTemporaryFile("system-includes", "clangd",
|
|
|
|
StdErrPath)) {
|
2019-06-26 15:45:27 +08:00
|
|
|
elog("System include extraction: failed to create temporary file with "
|
|
|
|
"error {0}",
|
|
|
|
EC.message());
|
2020-11-26 19:58:22 +08:00
|
|
|
return llvm::None;
|
2019-06-26 15:45:27 +08:00
|
|
|
}
|
|
|
|
auto CleanUp = llvm::make_scope_exit(
|
2019-07-04 19:34:23 +08:00
|
|
|
[&StdErrPath]() { llvm::sys::fs::remove(StdErrPath); });
|
2019-06-26 15:45:27 +08:00
|
|
|
|
|
|
|
llvm::Optional<llvm::StringRef> Redirects[] = {
|
2019-07-04 19:34:23 +08:00
|
|
|
{""}, {""}, llvm::StringRef(StdErrPath)};
|
2019-06-26 15:45:27 +08:00
|
|
|
|
2020-12-10 20:36:35 +08:00
|
|
|
llvm::SmallVector<llvm::StringRef> Args = {Driver, "-E", "-x",
|
|
|
|
Lang, "-", "-v"};
|
2020-02-05 16:55:56 +08:00
|
|
|
|
|
|
|
// These flags will be preserved
|
|
|
|
const llvm::StringRef FlagsToPreserve[] = {
|
|
|
|
"-nostdinc", "--no-standard-includes", "-nostdinc++", "-nobuiltininc"};
|
|
|
|
// Preserves these flags and their values, either as separate args or with an
|
|
|
|
// equalsbetween them
|
|
|
|
const llvm::StringRef ArgsToPreserve[] = {"--sysroot", "-isysroot"};
|
|
|
|
|
|
|
|
for (size_t I = 0, E = CommandLine.size(); I < E; ++I) {
|
|
|
|
llvm::StringRef Arg = CommandLine[I];
|
|
|
|
if (llvm::any_of(FlagsToPreserve,
|
|
|
|
[&Arg](llvm::StringRef S) { return S == Arg; })) {
|
|
|
|
Args.push_back(Arg);
|
|
|
|
} else {
|
|
|
|
const auto *Found =
|
|
|
|
llvm::find_if(ArgsToPreserve, [&Arg](llvm::StringRef S) {
|
|
|
|
return Arg.startswith(S);
|
|
|
|
});
|
|
|
|
if (Found == std::end(ArgsToPreserve))
|
|
|
|
continue;
|
2020-06-19 20:13:01 +08:00
|
|
|
Arg = Arg.drop_front(Found->size());
|
2020-02-05 16:55:56 +08:00
|
|
|
if (Arg.empty() && I + 1 < E) {
|
|
|
|
Args.push_back(CommandLine[I]);
|
|
|
|
Args.push_back(CommandLine[++I]);
|
|
|
|
} else if (Arg.startswith("=")) {
|
|
|
|
Args.push_back(CommandLine[I]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2019-06-26 15:45:27 +08:00
|
|
|
|
2019-07-04 19:34:23 +08:00
|
|
|
if (int RC = llvm::sys::ExecuteAndWait(Driver, Args, /*Env=*/llvm::None,
|
|
|
|
Redirects)) {
|
2019-06-26 15:45:27 +08:00
|
|
|
elog("System include extraction: driver execution failed with return code: "
|
2020-02-05 16:55:56 +08:00
|
|
|
"{0}. Args: ['{1}']",
|
|
|
|
llvm::to_string(RC), llvm::join(Args, "', '"));
|
2020-11-26 19:58:22 +08:00
|
|
|
return llvm::None;
|
2019-06-26 15:45:27 +08:00
|
|
|
}
|
|
|
|
|
2019-07-04 19:34:23 +08:00
|
|
|
auto BufOrError = llvm::MemoryBuffer::getFile(StdErrPath);
|
2019-06-26 15:45:27 +08:00
|
|
|
if (!BufOrError) {
|
|
|
|
elog("System include extraction: failed to read {0} with error {1}",
|
2019-07-04 19:34:23 +08:00
|
|
|
StdErrPath, BufOrError.getError().message());
|
2020-11-26 19:58:22 +08:00
|
|
|
return llvm::None;
|
2019-06-26 15:45:27 +08:00
|
|
|
}
|
|
|
|
|
2020-11-26 19:58:22 +08:00
|
|
|
llvm::Optional<DriverInfo> Info =
|
|
|
|
parseDriverOutput(BufOrError->get()->getBuffer());
|
|
|
|
if (!Info)
|
|
|
|
return llvm::None;
|
|
|
|
log("System includes extractor: successfully executed {0}\n\tgot includes: "
|
|
|
|
"\"{1}\"\n\tgot target: \"{2}\"",
|
|
|
|
Driver, llvm::join(Info->SystemIncludes, ", "), Info->Target);
|
|
|
|
return Info;
|
2019-06-26 15:45:27 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
tooling::CompileCommand &
|
|
|
|
addSystemIncludes(tooling::CompileCommand &Cmd,
|
|
|
|
llvm::ArrayRef<std::string> SystemIncludes) {
|
|
|
|
for (llvm::StringRef Include : SystemIncludes) {
|
|
|
|
// FIXME(kadircet): This doesn't work when we have "--driver-mode=cl"
|
|
|
|
Cmd.CommandLine.push_back("-isystem");
|
|
|
|
Cmd.CommandLine.push_back(Include.str());
|
|
|
|
}
|
|
|
|
return Cmd;
|
|
|
|
}
|
|
|
|
|
2020-11-26 19:58:22 +08:00
|
|
|
tooling::CompileCommand &setTarget(tooling::CompileCommand &Cmd,
|
|
|
|
const std::string &Target) {
|
|
|
|
if (!Target.empty()) {
|
|
|
|
// We do not want to override existing target with extracted one.
|
|
|
|
for (llvm::StringRef Arg : Cmd.CommandLine) {
|
|
|
|
if (Arg == "-target" || Arg.startswith("--target="))
|
|
|
|
return Cmd;
|
|
|
|
}
|
|
|
|
Cmd.CommandLine.push_back("--target=" + Target);
|
|
|
|
}
|
|
|
|
return Cmd;
|
|
|
|
}
|
|
|
|
|
2019-06-26 15:45:27 +08:00
|
|
|
/// Converts a glob containing only ** or * into a regex.
|
|
|
|
std::string convertGlobToRegex(llvm::StringRef Glob) {
|
|
|
|
std::string RegText;
|
|
|
|
llvm::raw_string_ostream RegStream(RegText);
|
|
|
|
RegStream << '^';
|
|
|
|
for (size_t I = 0, E = Glob.size(); I < E; ++I) {
|
|
|
|
if (Glob[I] == '*') {
|
|
|
|
if (I + 1 < E && Glob[I + 1] == '*') {
|
|
|
|
// Double star, accept any sequence.
|
|
|
|
RegStream << ".*";
|
|
|
|
// Also skip the second star.
|
|
|
|
++I;
|
|
|
|
} else {
|
|
|
|
// Single star, accept any sequence without a slash.
|
|
|
|
RegStream << "[^/]*";
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
RegStream << llvm::Regex::escape(Glob.substr(I, 1));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
RegStream << '$';
|
|
|
|
RegStream.flush();
|
|
|
|
return RegText;
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Converts a glob containing only ** or * into a regex.
|
|
|
|
llvm::Regex convertGlobsToRegex(llvm::ArrayRef<std::string> Globs) {
|
|
|
|
assert(!Globs.empty() && "Globs cannot be empty!");
|
|
|
|
std::vector<std::string> RegTexts;
|
|
|
|
RegTexts.reserve(Globs.size());
|
|
|
|
for (llvm::StringRef Glob : Globs)
|
|
|
|
RegTexts.push_back(convertGlobToRegex(Glob));
|
|
|
|
|
|
|
|
llvm::Regex Reg(llvm::join(RegTexts, "|"));
|
|
|
|
assert(Reg.isValid(RegTexts.front()) &&
|
|
|
|
"Created an invalid regex from globs");
|
|
|
|
return Reg;
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Extracts system includes from a trusted driver by parsing the output of
|
|
|
|
/// include search path and appends them to the commands coming from underlying
|
|
|
|
/// compilation database.
|
|
|
|
class QueryDriverDatabase : public GlobalCompilationDatabase {
|
|
|
|
public:
|
|
|
|
QueryDriverDatabase(llvm::ArrayRef<std::string> QueryDriverGlobs,
|
|
|
|
std::unique_ptr<GlobalCompilationDatabase> Base)
|
|
|
|
: QueryDriverRegex(convertGlobsToRegex(QueryDriverGlobs)),
|
|
|
|
Base(std::move(Base)) {
|
|
|
|
assert(this->Base);
|
|
|
|
BaseChanged =
|
|
|
|
this->Base->watch([this](const std::vector<std::string> &Changes) {
|
|
|
|
OnCommandChanged.broadcast(Changes);
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
llvm::Optional<tooling::CompileCommand>
|
2019-07-11 17:54:31 +08:00
|
|
|
getCompileCommand(PathRef File) const override {
|
|
|
|
auto Cmd = Base->getCompileCommand(File);
|
2019-06-26 15:45:27 +08:00
|
|
|
if (!Cmd || Cmd->CommandLine.empty())
|
|
|
|
return Cmd;
|
|
|
|
|
2019-07-11 21:47:46 +08:00
|
|
|
llvm::StringRef Lang;
|
|
|
|
for (size_t I = 0, E = Cmd->CommandLine.size(); I < E; ++I) {
|
|
|
|
llvm::StringRef Arg = Cmd->CommandLine[I];
|
|
|
|
if (Arg == "-x" && I + 1 < E)
|
|
|
|
Lang = Cmd->CommandLine[I + 1];
|
|
|
|
else if (Arg.startswith("-x"))
|
|
|
|
Lang = Arg.drop_front(2).trim();
|
|
|
|
}
|
|
|
|
if (Lang.empty()) {
|
|
|
|
llvm::StringRef Ext = llvm::sys::path::extension(File).trim('.');
|
|
|
|
auto Type = driver::types::lookupTypeForExtension(Ext);
|
|
|
|
if (Type == driver::types::TY_INVALID) {
|
|
|
|
elog("System include extraction: invalid file type for {0}", Ext);
|
|
|
|
return {};
|
|
|
|
}
|
|
|
|
Lang = driver::types::getTypeName(Type);
|
|
|
|
}
|
|
|
|
|
2019-06-26 15:45:27 +08:00
|
|
|
llvm::SmallString<128> Driver(Cmd->CommandLine.front());
|
|
|
|
llvm::sys::fs::make_absolute(Cmd->Directory, Driver);
|
|
|
|
|
2020-11-26 19:58:22 +08:00
|
|
|
if (auto Info =
|
|
|
|
QueriedDrivers.get(/*Key=*/(Driver + ":" + Lang).str(), [&] {
|
|
|
|
return extractSystemIncludesAndTarget(
|
|
|
|
Driver, Lang, Cmd->CommandLine, QueryDriverRegex);
|
|
|
|
})) {
|
|
|
|
setTarget(addSystemIncludes(*Cmd, Info->SystemIncludes), Info->Target);
|
|
|
|
}
|
|
|
|
return Cmd;
|
2019-06-26 15:45:27 +08:00
|
|
|
}
|
|
|
|
|
2019-07-11 17:54:31 +08:00
|
|
|
llvm::Optional<ProjectInfo> getProjectInfo(PathRef File) const override {
|
|
|
|
return Base->getProjectInfo(File);
|
|
|
|
}
|
|
|
|
|
2019-06-26 15:45:27 +08:00
|
|
|
private:
|
2020-06-10 04:54:42 +08:00
|
|
|
// Caches includes extracted from a driver. Key is driver:lang.
|
2020-11-26 19:58:22 +08:00
|
|
|
Memoize<llvm::StringMap<llvm::Optional<DriverInfo>>> QueriedDrivers;
|
2020-06-10 04:54:42 +08:00
|
|
|
llvm::Regex QueryDriverRegex;
|
2019-06-26 15:45:27 +08:00
|
|
|
|
|
|
|
std::unique_ptr<GlobalCompilationDatabase> Base;
|
|
|
|
CommandChanged::Subscription BaseChanged;
|
|
|
|
};
|
|
|
|
} // namespace
|
|
|
|
|
|
|
|
std::unique_ptr<GlobalCompilationDatabase>
|
|
|
|
getQueryDriverDatabase(llvm::ArrayRef<std::string> QueryDriverGlobs,
|
|
|
|
std::unique_ptr<GlobalCompilationDatabase> Base) {
|
|
|
|
assert(Base && "Null base to SystemIncludeExtractor");
|
|
|
|
if (QueryDriverGlobs.empty())
|
|
|
|
return Base;
|
2019-08-15 07:52:23 +08:00
|
|
|
return std::make_unique<QueryDriverDatabase>(QueryDriverGlobs,
|
2020-02-05 16:55:56 +08:00
|
|
|
std::move(Base));
|
2019-06-26 15:45:27 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
} // namespace clangd
|
|
|
|
} // namespace clang
|