maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

Revert "[clang][clang-scan-deps] Aggregate the full dependency information." 

This reverts commit f978ea4983.

It broke clang-ppc64be-linux, but not sure why yet.
This commit is contained in:
Michael Spencer 2019-12-11 16:34:31 -08:00
parent 56232f950d
commit 5bcd34a03f
8 changed files with 155 additions and 648 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h
View File

@ -11,69 +11,13 @@
#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
#include "clang/Tooling/JSONCompilationDatabase.h"
#include "llvm/ADT/StringSet.h"
#include <string>
namespace clang{
namespace tooling{
namespace dependencies{
/// The full dependencies and module graph for a specific input.
struct FullDependencies {
/// The name of the C++20 module this translation unit exports. This may
/// include `:` for C++20 module partitons.
///
/// If the translation unit is not a module then this will be empty.
std::string ExportedModuleName;
/// The context hash represents the set of compiler options that may make one
/// version of a module incompatible with another. This includes things like
/// language mode, predefined macros, header search paths, etc...
///
/// Modules with the same name but a different \c ContextHash should be
/// treated as separate modules for the purpose of a build.
std::string ContextHash;
/// A collection of absolute paths to files that this translation unit
/// directly depends on, not including transitive dependencies.
std::vector<std::string> FileDeps;
/// A list of modules this translation unit directly depends on, not including
/// transitive dependencies.
///
/// This may include modules with a different context hash when it can be
/// determined that the differences are benign for this compilation.
std::vector<ClangModuleDep> ClangModuleDeps;
/// A partial addtional set of command line arguments that can be used to
/// build this translation unit.
///
/// Call \c getFullAdditionalCommandLine() to get a command line suitable for
/// appending to the original command line to pass to clang.
std::vector<std::string> AdditionalNonPathCommandLine;
/// Gets the full addtional command line suitable for appending to the
/// original command line to pass to clang.
///
/// \param LookupPCMPath this function is called to fill in `-fmodule-file=`
/// flags and for the `-o` flag. It needs to return a
/// path for where the PCM for the given module is to
/// be located.
/// \param LookupModuleDeps this fucntion is called to collect the full
/// transitive set of dependencies for this
/// compilation.
std::vector<std::string> getAdditionalCommandLine(
std::function<StringRef(ClangModuleDep)> LookupPCMPath,
std::function<const ModuleDeps &(ClangModuleDep)> LookupModuleDeps) const;
};
struct FullDependenciesResult {
FullDependencies FullDeps;
std::vector<ModuleDeps> DiscoveredModules;
};
/// The high-level implementation of the dependency discovery tool that runs on
/// an individual worker thread.
class DependencyScanningTool {
@ -91,23 +35,8 @@ public:
getDependencyFile(const tooling::CompilationDatabase &Compilations,
StringRef CWD);
/// Collect the full module depenedency graph for the input, ignoring any
/// modules which have already been seen.
///
/// \param AlreadySeen this is used to not report modules that have previously
/// been reported. Use the same `llvm::StringSet<>` for all
/// calls to `getFullDependencies` for a single
/// `DependencyScanningTool` for a single build. Use a
/// different one for different tools, and clear it between
/// builds.
///
/// \returns a \c StringError with the diagnostic output if clang errors
/// occurred, \c FullDependencies otherwise.
llvm::Expected<FullDependenciesResult>
getFullDependencies(const tooling::CompilationDatabase &Compilations,
StringRef CWD, const llvm::StringSet<> &AlreadySeen);
private:
const ScanningOutputFormat Format;
DependencyScanningWorker Worker;
};

87
clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h
View File

@ -28,82 +28,16 @@ namespace dependencies {
class DependencyConsumer;
/// This is used to refer to a specific module.
///
/// See \c ModuleDeps for details about what these members mean.
struct ClangModuleDep {
std::string ModuleName;
std::string ContextHash;
};
struct ModuleDeps {
/// The name of the module. This may include `:` for C++20 module partitons,
/// or a header-name for C++20 header units.
std::string ModuleName;
/// The context hash of a module represents the set of compiler options that
/// may make one version of a module incompatible with another. This includes
/// things like language mode, predefined macros, header search paths, etc...
///
/// Modules with the same name but a different \c ContextHash should be
/// treated as separate modules for the purpose of a build.
std::string ContextHash;
/// The path to the modulemap file which defines this module.
///
/// This can be used to explicitly build this module. This file will
/// additionally appear in \c FileDeps as a dependency.
std::string ClangModuleMapFile;
/// The path to where an implicit build would put the PCM for this module.
std::string ImplicitModulePCMPath;
/// A collection of absolute paths to files that this module directly depends
/// on, not including transitive dependencies.
std::string ModulePCMPath;
std::string ContextHash;
llvm::StringSet<> FileDeps;
/// A list of modules this module directly depends on, not including
/// transitive dependencies.
///
/// This may include modules with a different context hash when it can be
/// determined that the differences are benign for this compilation.
std::vector<ClangModuleDep> ClangModuleDeps;
/// A partial command line that can be used to build this module.
///
/// Call \c getFullCommandLine() to get a command line suitable for passing to
/// clang.
std::vector<std::string> NonPathCommandLine;
// Used to track which modules that were discovered were directly imported by
// the primary TU.
llvm::StringSet<> ClangModuleDeps;
bool ImportedByMainFile = false;
/// Gets the full command line suitable for passing to clang.
///
/// \param LookupPCMPath this function is called to fill in `-fmodule-file=`
/// flags and for the `-o` flag. It needs to return a
/// path for where the PCM for the given module is to
/// be located.
/// \param LookupModuleDeps this fucntion is called to collect the full
/// transitive set of dependencies for this
/// compilation.
std::vector<std::string> getFullCommandLine(
std::function<StringRef(ClangModuleDep)> LookupPCMPath,
std::function<const ModuleDeps &(ClangModuleDep)> LookupModuleDeps) const;
};
namespace detail {
/// Append the `-fmodule-file=` and `-fmodule-map-file=` arguments for the
/// modules in \c Modules transitively, along with other needed arguments to
/// use explicitly built modules.
void appendCommonModuleArguments(
llvm::ArrayRef<ClangModuleDep> Modules,
std::function<StringRef(ClangModuleDep)> LookupPCMPath,
std::function<const ModuleDeps &(ClangModuleDep)> LookupModuleDeps,
std::vector<std::string> &Result);
} // namespace detail
class ModuleDepCollector;
class ModuleDepCollectorPP final : public PPCallbacks {
@ -120,8 +54,6 @@ public:
StringRef SearchPath, StringRef RelativePath,
const Module *Imported,
SrcMgr::CharacteristicKind FileType) override;
void moduleImport(SourceLocation ImportLoc, ModuleIdPath Path,
const Module *Imported) override;
void EndOfMainFile() override;
@ -130,18 +62,16 @@ private:
ModuleDepCollector &MDC;
llvm::DenseSet<const Module *> DirectDeps;
void handleImport(const Module *Imported);
void handleTopLevelModule(const Module *M);
void addAllSubmoduleDeps(const Module *M, ModuleDeps &MD,
llvm::DenseSet<const Module *> &AddedModules);
void addModuleDep(const Module *M, ModuleDeps &MD,
llvm::DenseSet<const Module *> &AddedModules);
void addAllSubmoduleDeps(const Module *M, ModuleDeps &MD);
void addModuleDep(const Module *M, ModuleDeps &MD);
void addDirectDependencies(const Module *Mod);
};
class ModuleDepCollector final : public DependencyCollector {
public:
ModuleDepCollector(std::unique_ptr<DependencyOutputOptions> Opts,
CompilerInstance &I, DependencyConsumer &C);
ModuleDepCollector(CompilerInstance &I, DependencyConsumer &C);
void attachToPreprocessor(Preprocessor &PP) override;
void attachToASTReader(ASTReader &R) override;
@ -155,7 +85,6 @@ private:
std::string ContextHash;
std::vector<std::string> MainDeps;
std::unordered_map<std::string, ModuleDeps> Deps;
std::unique_ptr<DependencyOutputOptions> Opts;
};
} // end namespace dependencies

127
clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp
View File

@ -8,25 +8,24 @@
#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
#include "clang/Frontend/Utils.h"
#include "llvm/Support/JSON.h"
static llvm::json::Array toJSONSorted(const llvm::StringSet<> &Set) {
std::vector<llvm::StringRef> Strings;
for (auto &&I : Set)
Strings.push_back(I.getKey());
std::sort(Strings.begin(), Strings.end());
return llvm::json::Array(Strings);
}
namespace clang{
namespace tooling{
namespace dependencies{
std::vector<std::string> FullDependencies::getAdditionalCommandLine(
std::function<StringRef(ClangModuleDep)> LookupPCMPath,
std::function<const ModuleDeps &(ClangModuleDep)> LookupModuleDeps) const {
std::vector<std::string> Ret = AdditionalNonPathCommandLine;
dependencies::detail::appendCommonModuleArguments(
ClangModuleDeps, LookupPCMPath, LookupModuleDeps, Ret);
return Ret;
}
DependencyScanningTool::DependencyScanningTool(
DependencyScanningService &Service)
: Worker(Service) {}
: Format(Service.getFormat()), Worker(Service) {
}
llvm::Expected<std::string> DependencyScanningTool::getDependencyFile(
const tooling::CompilationDatabase &Compilations, StringRef CWD) {
@ -76,33 +75,8 @@ llvm::Expected<std::string> DependencyScanningTool::getDependencyFile(
std::vector<std::string> Dependencies;
};
// We expect a single command here because if a source file occurs multiple
// times in the original CDB, then `computeDependencies` would run the
// `DependencyScanningAction` once for every time the input occured in the
// CDB. Instead we split up the CDB into single command chunks to avoid this
// behavior.
assert(Compilations.getAllCompileCommands().size() == 1 &&
"Expected a compilation database with a single command!");
std::string Input = Compilations.getAllCompileCommands().front().Filename;
MakeDependencyPrinterConsumer Consumer;
auto Result = Worker.computeDependencies(Input, CWD, Compilations, Consumer);
if (Result)
return std::move(Result);
std::string Output;
Consumer.printDependencies(Output);
return Output;
}
llvm::Expected<FullDependenciesResult>
DependencyScanningTool::getFullDependencies(
const tooling::CompilationDatabase &Compilations, StringRef CWD,
const llvm::StringSet<> &AlreadySeen) {
class FullDependencyPrinterConsumer : public DependencyConsumer {
public:
FullDependencyPrinterConsumer(const llvm::StringSet<> &AlreadySeen)
: AlreadySeen(AlreadySeen) {}
void handleFileDependency(const DependencyOutputOptions &Opts,
StringRef File) override {
Dependencies.push_back(File);
@ -116,41 +90,55 @@ DependencyScanningTool::getFullDependencies(
ContextHash = std::move(Hash);
}
FullDependenciesResult getFullDependencies() const {
FullDependencies FD;
void printDependencies(std::string &S, StringRef MainFile) {
// Sort the modules by name to get a deterministic order.
std::vector<StringRef> Modules;
for (auto &&Dep : ClangModuleDeps)
Modules.push_back(Dep.first);
std::sort(Modules.begin(), Modules.end());
FD.ContextHash = std::move(ContextHash);
llvm::raw_string_ostream OS(S);
FD.FileDeps.assign(Dependencies.begin(), Dependencies.end());
using namespace llvm::json;
for (auto &&M : ClangModuleDeps) {
auto &MD = M.second;
Array Imports;
for (auto &&ModName : Modules) {
auto &MD = ClangModuleDeps[ModName];
if (MD.ImportedByMainFile)
FD.ClangModuleDeps.push_back({MD.ModuleName, ContextHash});
Imports.push_back(MD.ModuleName);
}
FullDependenciesResult FDR;
for (auto &&M : ClangModuleDeps) {
// TODO: Avoid handleModuleDependency even being called for modules
// we've already seen.
if (AlreadySeen.count(M.first))
continue;
FDR.DiscoveredModules.push_back(std::move(M.second));
Array Mods;
for (auto &&ModName : Modules) {
auto &MD = ClangModuleDeps[ModName];
Object Mod{
{"name", MD.ModuleName},
{"file-deps", toJSONSorted(MD.FileDeps)},
{"clang-module-deps", toJSONSorted(MD.ClangModuleDeps)},
{"clang-modulemap-file", MD.ClangModuleMapFile},
};
Mods.push_back(std::move(Mod));
}
FDR.FullDeps = std::move(FD);
return FDR;
Object O{
{"input-file", MainFile},
{"clang-context-hash", ContextHash},
{"file-deps", Dependencies},
{"clang-module-deps", std::move(Imports)},
{"clang-modules", std::move(Mods)},
};
S = llvm::formatv("{0:2},\n", Value(std::move(O))).str();
return;
}
private:
std::vector<std::string> Dependencies;
std::unordered_map<std::string, ModuleDeps> ClangModuleDeps;
std::string ContextHash;
std::vector<std::string> OutputPaths;
const llvm::StringSet<> &AlreadySeen;
};
// We expect a single command here because if a source file occurs multiple
// times in the original CDB, then `computeDependencies` would run the
// `DependencyScanningAction` once for every time the input occured in the
@ -159,13 +147,26 @@ DependencyScanningTool::getFullDependencies(
assert(Compilations.getAllCompileCommands().size() == 1 &&
"Expected a compilation database with a single command!");
std::string Input = Compilations.getAllCompileCommands().front().Filename;
FullDependencyPrinterConsumer Consumer(AlreadySeen);
llvm::Error Result =
Worker.computeDependencies(Input, CWD, Compilations, Consumer);
if (Result)
return std::move(Result);
return Consumer.getFullDependencies();
if (Format == ScanningOutputFormat::Make) {
MakeDependencyPrinterConsumer Consumer;
auto Result =
Worker.computeDependencies(Input, CWD, Compilations, Consumer);
if (Result)
return std::move(Result);
std::string Output;
Consumer.printDependencies(Output);
return Output;
} else {
FullDependencyPrinterConsumer Consumer;
auto Result =
Worker.computeDependencies(Input, CWD, Compilations, Consumer);
if (Result)
return std::move(Result);
std::string Output;
Consumer.printDependencies(Output, Input);
return Output;
}
}
} // end namespace dependencies

11
clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
View File

@ -142,18 +142,11 @@ public:
Consumer));
break;
case ScanningOutputFormat::Full:
Compiler.addDependencyCollector(std::make_shared<ModuleDepCollector>(
std::move(Opts), Compiler, Consumer));
Compiler.addDependencyCollector(
std::make_shared<ModuleDepCollector>(Compiler, Consumer));
break;
}
// Consider different header search and diagnostic options to create
// different modules. This avoids the unsound aliasing of module PCMs.
//
// TODO: Implement diagnostic bucketing and header search pruning to reduce
// the impact of strict context hashing.
Compiler.getHeaderSearchOpts().ModulesStrictContextHash = false;
Consumer.handleContextHash(Compiler.getInvocation().getModuleHash());
auto Action = std::make_unique<PreprocessOnlyAction>();

85
clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
View File

@ -17,47 +17,6 @@ using namespace clang;
using namespace tooling;
using namespace dependencies;
std::vector<std::string> ModuleDeps::getFullCommandLine(
std::function<StringRef(ClangModuleDep)> LookupPCMPath,
std::function<const ModuleDeps &(ClangModuleDep)> LookupModuleDeps) const {
std::vector<std::string> Ret = NonPathCommandLine;
// TODO: Build full command line. That also means capturing the original
// command line into NonPathCommandLine.
dependencies::detail::appendCommonModuleArguments(
ClangModuleDeps, LookupPCMPath, LookupModuleDeps, Ret);
return Ret;
}
void dependencies::detail::appendCommonModuleArguments(
llvm::ArrayRef<ClangModuleDep> Modules,
std::function<StringRef(ClangModuleDep)> LookupPCMPath,
std::function<const ModuleDeps &(ClangModuleDep)> LookupModuleDeps,
std::vector<std::string> &Result) {
llvm::StringSet<> AlreadyAdded;
std::function<void(llvm::ArrayRef<ClangModuleDep>)> AddArgs =
[&](llvm::ArrayRef<ClangModuleDep> Modules) {
for (const ClangModuleDep &CMD : Modules) {
if (!AlreadyAdded.insert(CMD.ModuleName + CMD.ContextHash).second)
continue;
const ModuleDeps &M = LookupModuleDeps(CMD);
// Depth first traversal.
AddArgs(M.ClangModuleDeps);
Result.push_back(("-fmodule-file=" + LookupPCMPath(CMD)).str());
if (!M.ClangModuleMapFile.empty()) {
Result.push_back("-fmodule-map-file=" + M.ClangModuleMapFile);
}
}
};
Result.push_back("-fno-implicit-modules");
Result.push_back("-fno-implicit-module-maps");
AddArgs(Modules);
}
void ModuleDepCollectorPP::FileChanged(SourceLocation Loc,
FileChangeReason Reason,
SrcMgr::CharacteristicKind FileType,
@ -91,16 +50,7 @@ void ModuleDepCollectorPP::InclusionDirective(
// here as `FileChanged` will never see it.
MDC.MainDeps.push_back(FileName);
}
handleImport(Imported);
}
void ModuleDepCollectorPP::moduleImport(SourceLocation ImportLoc,
ModuleIdPath Path,
const Module *Imported) {
handleImport(Imported);
}
void ModuleDepCollectorPP::handleImport(const Module *Imported) {
if (!Imported)
return;
@ -121,8 +71,9 @@ void ModuleDepCollectorPP::EndOfMainFile() {
for (auto &&I : MDC.Deps)
MDC.Consumer.handleModuleDependency(I.second);
DependencyOutputOptions Opts;
for (auto &&I : MDC.MainDeps)
MDC.Consumer.handleFileDependency(*MDC.Opts, I);
MDC.Consumer.handleFileDependency(Opts, I);
}
void ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
@ -143,7 +94,7 @@ void ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
MD.ClangModuleMapFile = ModuleMap ? ModuleMap->getName() : "";
MD.ModuleName = M->getFullModuleName();
MD.ImplicitModulePCMPath = M->getASTFile()->getName();
MD.ModulePCMPath = M->getASTFile()->getName();
MD.ContextHash = MDC.ContextHash;
serialization::ModuleFile *MF =
MDC.Instance.getASTReader()->getModuleManager().lookup(M->getASTFile());
@ -152,38 +103,30 @@ void ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
MD.FileDeps.insert(IF.getFile()->getName());
});
llvm::DenseSet<const Module *> AddedModules;
addAllSubmoduleDeps(M, MD, AddedModules);
addAllSubmoduleDeps(M, MD);
}
void ModuleDepCollectorPP::addAllSubmoduleDeps(
const Module *M, ModuleDeps &MD,
llvm::DenseSet<const Module *> &AddedModules) {
addModuleDep(M, MD, AddedModules);
void ModuleDepCollectorPP::addAllSubmoduleDeps(const Module *M,
ModuleDeps &MD) {
addModuleDep(M, MD);
for (const Module *SubM : M->submodules())
addAllSubmoduleDeps(SubM, MD, AddedModules);
addAllSubmoduleDeps(SubM, MD);
}
void ModuleDepCollectorPP::addModuleDep(
const Module *M, ModuleDeps &MD,
llvm::DenseSet<const Module *> &AddedModules) {
void ModuleDepCollectorPP::addModuleDep(const Module *M, ModuleDeps &MD) {
for (const Module *Import : M->Imports) {
if (Import->getTopLevelModule() != M->getTopLevelModule()) {
if (AddedModules.insert(Import->getTopLevelModule()).second)
MD.ClangModuleDeps.push_back(
{Import->getTopLevelModuleName(),
Instance.getInvocation().getModuleHash()});
MD.ClangModuleDeps.insert(Import->getTopLevelModuleName());
handleTopLevelModule(Import->getTopLevelModule());
}
}
}
ModuleDepCollector::ModuleDepCollector(
std::unique_ptr<DependencyOutputOptions> Opts, CompilerInstance &I,
DependencyConsumer &C)
: Instance(I), Consumer(C), ContextHash(I.getInvocation().getModuleHash()),
Opts(std::move(Opts)) {}
ModuleDepCollector::ModuleDepCollector(CompilerInstance &I,
DependencyConsumer &C)
: Instance(I), Consumer(C), ContextHash(I.getInvocation().getModuleHash()) {
}
void ModuleDepCollector::attachToPreprocessor(Preprocessor &PP) {
PP.addPPCallbacks(std::make_unique<ModuleDepCollectorPP>(Instance, *this));

13
clang/test/ClangScanDeps/Inputs/modules_cdb.json
View File

@ -1,22 +1,13 @@
[
{
"directory": "DIR",
"command": "clang -E DIR/modules_cdb_input2.cpp -IInputs -D INCLUDE_HEADER2 -MD -MF DIR/modules_cdb2.d -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps",
"command": "clang -E -fsyntax-only DIR/modules_cdb_input2.cpp -IInputs -D INCLUDE_HEADER2 -MD -MF DIR/modules_cdb2.d -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps",
"file": "DIR/modules_cdb_input2.cpp"
},
{
"directory": "DIR",
"command": "clang -E DIR/modules_cdb_input.cpp -IInputs -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps",
"file": "DIR/modules_cdb_input.cpp"
},
{
"directory": "DIR",
"command": "clang -E DIR/modules_cdb_input.cpp -IInputs -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps -o a.o",
"file": "DIR/modules_cdb_input.cpp"
},
{
"directory": "DIR",
"command": "clang -E DIR/modules_cdb_input.cpp -IInputs -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps -o b.o",
"file": "DIR/modules_cdb_input.cpp"
}
]

192
clang/test/ClangScanDeps/modules-full.cpp
View File

@ -1,5 +1,6 @@
// RUN: rm -rf %t.dir
// RUN: rm -rf %t.cdb
// RUN: rm -rf %t.module-cache
// RUN: mkdir -p %t.dir
// RUN: cp %s %t.dir/modules_cdb_input.cpp
// RUN: cp %s %t.dir/modules_cdb_input2.cpp
@ -10,146 +11,67 @@
// RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/modules_cdb.json > %t.cdb
//
// RUN: echo %t.dir > %t.result
// RUN: clang-scan-deps -compilation-database %t.cdb -j 4 -full-command-line \
// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 \
// RUN: -mode preprocess-minimized-sources -format experimental-full >> %t.result
// RUN: cat %t.result | sed 's/\\/\//g' | FileCheck --check-prefixes=CHECK %s
// RUN: cat %t.result | FileCheck --check-prefixes=CHECK %s
// FIXME: Backslash issues.
// XFAIL: system-windows
#include "header.h"
// CHECK: [[PREFIX:.*]]
// CHECK-NEXT: {
// CHECK-NEXT: "modules": [
// CHECK: [[PREFIX:(.*[/\\])+[a-zA-Z0-9.-]+]]
// CHECK-NEXT: {
// CHECK-NEXT: "clang-module-deps": [
// CHECK-NEXT: {
// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H1:[A-Z0-9]+]]",
// CHECK-NEXT: "module-name": "header2"
// CHECK-NEXT: }
// CHECK-NEXT: ],
// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/Inputs/module.modulemap",
// CHECK-NEXT: "command-line": [
// CHECK-NEXT: "-fno-implicit-modules",
// CHECK-NEXT: "-fno-implicit-module-maps",
// CHECK-NEXT: "-fmodule-file=[[PREFIX]]/module-cache/[[CONTEXT_HASH_H1]]/header2-{{[A-Z0-9]+}}.pcm",
// CHECK-NEXT: "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
// CHECK-NEXT: ],
// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H1]]",
// CHECK-NEXT: "file-deps": [
// CHECK-NEXT: "[[PREFIX]]/Inputs/header.h",
// CHECK-NEXT: "[[PREFIX]]/Inputs/module.modulemap"
// CHECK-NEXT: ],
// CHECK-NEXT: "name": "header1"
// CHECK-NEXT: },
// CHECK-NEXT: {
// CHECK-NEXT: "clang-module-deps": [],
// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/Inputs/module.modulemap",
// CHECK-NEXT: "command-line": [
// CHECK-NEXT: "-fno-implicit-modules",
// CHECK-NEXT: "-fno-implicit-module-maps"
// CHECK-NEXT: ],
// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H2:[A-Z0-9]+]]",
// CHECK-NEXT: "file-deps": [
// CHECK-NEXT: "[[PREFIX]]/Inputs/header.h",
// CHECK-NEXT: "[[PREFIX]]/Inputs/module.modulemap"
// CHECK-NEXT: ],
// CHECK-NEXT: "name": "header1"
// CHECK-NEXT: },
// CHECK-NEXT: {
// CHECK-NEXT: "clang-module-deps": [],
// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/Inputs/module.modulemap",
// CHECK-NEXT: "command-line": [
// CHECK-NEXT: "-fno-implicit-modules",
// CHECK-NEXT: "-fno-implicit-module-maps"
// CHECK-NEXT: ],
// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H1]]",
// CHECK-NEXT: "file-deps": [
// CHECK-NEXT: "[[PREFIX]]/Inputs/header2.h",
// CHECK-NEXT: "[[PREFIX]]/Inputs/module.modulemap"
// CHECK-NEXT: ],
// CHECK-NEXT: "name": "header2"
// CHECK-NEXT: }
// CHECK-NEXT: ],
// CHECK-NEXT: "translation-units": [
// CHECK-NEXT: {
// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH_H2]]",
// CHECK-NEXT: "clang-module-deps": [
// CHECK-NEXT: {
// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H2]]",
// CHECK-NEXT: "module-name": "header1"
// CHECK-NEXT: }
// CHECK-NEXT: ],
// CHECK-NEXT: "command-line": [
// CHECK-NEXT: "-fno-implicit-modules",
// CHECK-NEXT: "-fno-implicit-module-maps",
// CHECK-NEXT: "-fmodule-file=[[PREFIX]]/module-cache/[[CONTEXT_HASH_H2]]/header1-{{[A-Z0-9]+}}.pcm",
// CHECK-NEXT: "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
// CHECK-NEXT: ],
// CHECK-NEXT: "file-deps": [
// CHECK-NEXT: "[[PREFIX]]/modules_cdb_input.cpp"
// CHECK-NEXT: ],
// CHECK-NEXT: "input-file": "[[PREFIX]]/modules_cdb_input.cpp"
// CHECK-NEXT: },
// CHECK-NEXT: {
// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH_H2]]",
// CHECK-NEXT: "clang-module-deps": [
// CHECK-NEXT: {
// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H2]]",
// CHECK-NEXT: "module-name": "header1"
// CHECK-NEXT: }
// CHECK-NEXT: ],
// CHECK-NEXT: "command-line": [
// CHECK-NEXT: "-fno-implicit-modules",
// CHECK-NEXT: "-fno-implicit-module-maps",
// CHECK-NEXT: "-fmodule-file=[[PREFIX]]/module-cache/[[CONTEXT_HASH_H2]]/header1-{{[A-Z0-9]+}}.pcm",
// CHECK-NEXT: "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
// CHECK-NEXT: ],
// CHECK-NEXT: "file-deps": [
// CHECK-NEXT: "[[PREFIX]]/modules_cdb_input.cpp"
// CHECK-NEXT: ],
// CHECK-NEXT: "input-file": "[[PREFIX]]/modules_cdb_input.cpp"
// CHECK-NEXT: },
// CHECK-NEXT: {
// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH_H2]]",
// CHECK-NEXT: "clang-module-deps": [
// CHECK-NEXT: {
// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H2]]",
// CHECK-NEXT: "module-name": "header1"
// CHECK-NEXT: }
// CHECK-NEXT: ],
// CHECK-NEXT: "command-line": [
// CHECK-NEXT: "-fno-implicit-modules",
// CHECK-NEXT: "-fno-implicit-module-maps",
// CHECK-NEXT: "-fmodule-file=[[PREFIX]]/module-cache/[[CONTEXT_HASH_H2]]/header1-{{[A-Z0-9]+}}.pcm",
// CHECK-NEXT: "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
// CHECK-NEXT: ],
// CHECK-NEXT: "file-deps": [
// CHECK-NEXT: "[[PREFIX]]/modules_cdb_input.cpp"
// CHECK-NEXT: ],
// CHECK-NEXT: "input-file": "[[PREFIX]]/modules_cdb_input.cpp"
// CHECK-NEXT: },
// CHECK-NEXT: {
// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH_H1]]",
// CHECK-NEXT: "clang-module-deps": [
// CHECK-NEXT: {
// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H1]]",
// CHECK-NEXT: "module-name": "header1"
// CHECK-NEXT: }
// CHECK-NEXT: ],
// CHECK-NEXT: "command-line": [
// CHECK-NEXT: "-fno-implicit-modules",
// CHECK-NEXT: "-fno-implicit-module-maps",
// CHECK-NEXT: "-fmodule-file=[[PREFIX]]/module-cache/[[CONTEXT_HASH_H1]]/header2-{{[A-Z0-9]+}}.pcm",
// CHECK-NEXT: "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap",
// CHECK-NEXT: "-fmodule-file=[[PREFIX]]/module-cache/[[CONTEXT_HASH_H1]]/header1-{{[A-Z0-9]+}}.pcm",
// CHECK-NEXT: "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap"
// CHECK-NEXT: ],
// CHECK-NEXT: "file-deps": [
// CHECK-NEXT: "[[PREFIX]]/modules_cdb_input2.cpp"
// CHECK-NEXT: ],
// CHECK-NEXT: "input-file": "[[PREFIX]]/modules_cdb_input2.cpp"
// CHECK-NEXT: }
// CHECK-NEXT: ]
// CHECK-NEXT: }
// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH:[A-Z0-9]+]]",
// CHECK-NEXT: "clang-module-deps": [
// CHECK-NEXT: "header1"
// CHECK-NEXT: ],
// CHECK-NEXT: "clang-modules": [
// CHECK-NEXT: {
// CHECK-NEXT: "clang-module-deps": [
// CHECK-NEXT: "header2"
// CHECK-NEXT: ],
// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap",
// CHECK-NEXT: "file-deps": [
// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}header.h",
// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap"
// CHECK-NEXT: ],
// CHECK-NEXT: "name": "header1"
// CHECK-NEXT: },
// CHECK-NEXT: {
// CHECK-NEXT: "clang-module-deps": [],
// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap",
// CHECK-NEXT: "file-deps": [
// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}header2.h",
// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap"
// CHECK-NEXT: ],
// CHECK-NEXT: "name": "header2"
// CHECK-NEXT: }
// CHECK-NEXT: ],
// CHECK-NEXT: "file-deps": [
// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}modules_cdb_input2.cpp"
// CHECK-NEXT: ],
// CHECK-NEXT: "input-file": "[[PREFIX]]{{[/\\]}}modules_cdb_input2.cpp"
// CHECK-NEXT:},
// CHECK-NEXT:{
// CHECK-NOT: "clang-context-hash": "[[CONTEXT_HASH]]",
// CHECK-NEXT: "clang-context-hash": "{{[A-Z0-9]+}}",
// CHECK-NEXT: "clang-module-deps": [
// CHECK-NEXT: "header1"
// CHECK-NEXT: ],
// CHECK-NEXT: "clang-modules": [
// CHECK-NEXT: {
// CHECK-NEXT: "clang-module-deps": [],
// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap",
// CHECK-NEXT: "file-deps": [
// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}header.h",
// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap"
// CHECK-NEXT: ],
// CHECK-NEXT: "name": "header1"
// CHECK-NEXT: }
// CHECK-NEXT: ],
// CHECK-NEXT: "file-deps": [
// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}modules_cdb_input.cpp"
// CHECK-NEXT: ],
// CHECK-NEXT: "input-file": "[[PREFIX]]{{[/\\]}}modules_cdb_input.cpp"
// CHECK-NEXT:},

215
clang/tools/clang-scan-deps/ClangScanDeps.cpp
View File

@ -15,7 +15,6 @@
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/FileUtilities.h"
#include "llvm/Support/InitLLVM.h"
#include "llvm/Support/JSON.h"
#include "llvm/Support/Program.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/Threading.h"
@ -130,11 +129,6 @@ static llvm::cl::opt<ScanningOutputFormat> Format(
llvm::cl::init(ScanningOutputFormat::Make),
llvm::cl::cat(DependencyScannerCategory));
static llvm::cl::opt<bool> FullCommandLine(
"full-command-line",
llvm::cl::desc("Include the full command lines to use to build modules"),
llvm::cl::init(false), llvm::cl::cat(DependencyScannerCategory));
llvm::cl::opt<unsigned>
NumThreads("j", llvm::cl::Optional,
llvm::cl::desc("Number of worker threads to use (default: use "
@ -195,10 +189,9 @@ private:
/// based on the result.
///
/// \returns True on error.
static bool
handleMakeDependencyToolResult(const std::string &Input,
llvm::Expected<std::string> &MaybeFile,
SharedStream &OS, SharedStream &Errs) {
static bool handleDependencyToolResult(const std::string &Input,
llvm::Expected<std::string> &MaybeFile,
SharedStream &OS, SharedStream &Errs) {
if (!MaybeFile) {
llvm::handleAllErrors(
MaybeFile.takeError(), [&Input, &Errs](llvm::StringError &Err) {
@ -213,184 +206,6 @@ handleMakeDependencyToolResult(const std::string &Input,
return false;
}
static llvm::json::Array toJSONSorted(const llvm::StringSet<> &Set) {
std::vector<llvm::StringRef> Strings;
for (auto &&I : Set)
Strings.push_back(I.getKey());
std::sort(Strings.begin(), Strings.end());
return llvm::json::Array(Strings);
}
static llvm::json::Array toJSONSorted(std::vector<ClangModuleDep> V) {
std::sort(V.begin(), V.end(),
[](const ClangModuleDep &A, const ClangModuleDep &B) {
return std::tie(A.ModuleName, A.ContextHash) <
std::tie(B.ModuleName, B.ContextHash);
});
llvm::json::Array Ret;
for (const ClangModuleDep &CMD : V)
Ret.push_back(llvm::json::Object(
{{"module-name", CMD.ModuleName}, {"context-hash", CMD.ContextHash}}));
return Ret;
}
// Thread safe.
class FullDeps {
public:
void mergeDeps(StringRef Input, FullDependenciesResult FDR,
size_t InputIndex) {
const FullDependencies &FD = FDR.FullDeps;
InputDeps ID;
ID.FileName = Input;
ID.ContextHash = std::move(FD.ContextHash);
ID.FileDeps = std::move(FD.FileDeps);
ID.ModuleDeps = std::move(FD.ClangModuleDeps);
std::unique_lock<std::mutex> ul(Lock);
for (const ModuleDeps &MD : FDR.DiscoveredModules) {
auto I = Modules.find({MD.ContextHash, MD.ModuleName, 0});
if (I != Modules.end()) {
I->first.InputIndex = std::min(I->first.InputIndex, InputIndex);
continue;
}
Modules.insert(
I, {{MD.ContextHash, MD.ModuleName, InputIndex}, std::move(MD)});
}
if (FullCommandLine)
ID.AdditonalCommandLine = FD.getAdditionalCommandLine(
[&](ClangModuleDep CMD) { return lookupPCMPath(CMD); },
[&](ClangModuleDep CMD) -> const ModuleDeps & {
return lookupModuleDeps(CMD);
});
Inputs.push_back(std::move(ID));
}
void printFullOutput(raw_ostream &OS) {
// Sort the modules by name to get a deterministic order.
std::vector<ContextModulePair> ModuleNames;
for (auto &&M : Modules)
ModuleNames.push_back(M.first);
std::sort(ModuleNames.begin(), ModuleNames.end(),
[](const ContextModulePair &A, const ContextModulePair &B) {
return std::tie(A.ModuleName, A.InputIndex) <
std::tie(B.ModuleName, B.InputIndex);
});
std::sort(Inputs.begin(), Inputs.end(),
[](const InputDeps &A, const InputDeps &B) {
return A.FileName < B.FileName;
});
using namespace llvm::json;
Array OutModules;
for (auto &&ModName : ModuleNames) {
auto &MD = Modules[ModName];
Object O{
{"name", MD.ModuleName},
{"context-hash", MD.ContextHash},
{"file-deps", toJSONSorted(MD.FileDeps)},
{"clang-module-deps", toJSONSorted(MD.ClangModuleDeps)},
{"clang-modulemap-file", MD.ClangModuleMapFile},
{"command-line",
FullCommandLine
? MD.getFullCommandLine(
[&](ClangModuleDep CMD) { return lookupPCMPath(CMD); },
[&](ClangModuleDep CMD) -> const ModuleDeps & {
return lookupModuleDeps(CMD);
})
: MD.NonPathCommandLine},
};
OutModules.push_back(std::move(O));
}
Array TUs;
for (auto &&I : Inputs) {
Object O{
{"input-file", I.FileName},
{"clang-context-hash", I.ContextHash},
{"file-deps", I.FileDeps},
{"clang-module-deps", toJSONSorted(I.ModuleDeps)},
{"command-line", I.AdditonalCommandLine},
};
TUs.push_back(std::move(O));
}
Object Output{
{"modules", std::move(OutModules)},
{"translation-units", std::move(TUs)},
};
OS << llvm::formatv("{0:2}\n", Value(std::move(Output)));
}
private:
StringRef lookupPCMPath(ClangModuleDep CMD) {
return Modules[ContextModulePair{CMD.ContextHash, CMD.ModuleName, 0}]
.ImplicitModulePCMPath;
}
const ModuleDeps &lookupModuleDeps(ClangModuleDep CMD) {
auto I =
Modules.find(ContextModulePair{CMD.ContextHash, CMD.ModuleName, 0});
assert(I != Modules.end());
return I->second;
};
struct ContextModulePair {
std::string ContextHash;
std::string ModuleName;
mutable size_t InputIndex;
bool operator==(const ContextModulePair &Other) const {
return ContextHash == Other.ContextHash && ModuleName == Other.ModuleName;
}
};
struct ContextModulePairHasher {
std::size_t operator()(const ContextModulePair &CMP) const {
using llvm::hash_combine;
return hash_combine(CMP.ContextHash, CMP.ModuleName);
}
};
struct InputDeps {
std::string FileName;
std::string ContextHash;
std::vector<std::string> FileDeps;
std::vector<ClangModuleDep> ModuleDeps;
std::vector<std::string> AdditonalCommandLine;
};
std::mutex Lock;
std::unordered_map<ContextModulePair, ModuleDeps, ContextModulePairHasher>
Modules;
std::vector<InputDeps> Inputs;
};
static bool handleFullDependencyToolResult(
const std::string &Input,
llvm::Expected<FullDependenciesResult> &MaybeFullDeps, FullDeps &FD,
size_t InputIndex, SharedStream &OS, SharedStream &Errs) {
if (!MaybeFullDeps) {
llvm::handleAllErrors(
MaybeFullDeps.takeError(), [&Input, &Errs](llvm::StringError &Err) {
Errs.applyLocked([&](raw_ostream &OS) {
OS << "Error while scanning dependencies for " << Input << ":\n";
OS << Err.getMessage();
});
});
return true;
}
FD.mergeDeps(Input, std::move(*MaybeFullDeps), InputIndex);
return false;
}
int main(int argc, const char **argv) {
llvm::InitLLVM X(argc, argv);
llvm::cl::HideUnrelatedOptions(DependencyScannerCategory);
@ -501,7 +316,6 @@ int main(int argc, const char **argv) {
std::vector<std::thread> WorkerThreads;
std::atomic<bool> HadErrors(false);
FullDeps FD;
std::mutex Lock;
size_t Index = 0;
@ -510,38 +324,26 @@ int main(int argc, const char **argv) {
<< " files using " << NumWorkers << " workers\n";
}
for (unsigned I = 0; I < NumWorkers; ++I) {
auto Worker = [I, &Lock, &Index, &Inputs, &HadErrors, &FD, &WorkerTools,
auto Worker = [I, &Lock, &Index, &Inputs, &HadErrors, &WorkerTools,
&DependencyOS, &Errs]() {
llvm::StringSet<> AlreadySeenModules;
while (true) {
const SingleCommandCompilationDatabase *Input;
std::string Filename;
std::string CWD;
size_t LocalIndex;
// Take the next input.
{
std::unique_lock<std::mutex> LockGuard(Lock);
if (Index >= Inputs.size())
return;
LocalIndex = Index;
Input = &Inputs[Index++];
tooling::CompileCommand Cmd = Input->getAllCompileCommands()[0];
Filename = std::move(Cmd.Filename);
CWD = std::move(Cmd.Directory);
}
// Run the tool on it.
if (Format == ScanningOutputFormat::Make) {
auto MaybeFile = WorkerTools[I]->getDependencyFile(*Input, CWD);
if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS,
Errs))
HadErrors = true;
} else {
auto MaybeFullDeps = WorkerTools[I]->getFullDependencies(
*Input, CWD, AlreadySeenModules);
if (handleFullDependencyToolResult(Filename, MaybeFullDeps, FD,
LocalIndex, DependencyOS, Errs))
HadErrors = true;
}
auto MaybeFile = WorkerTools[I]->getDependencyFile(*Input, CWD);
if (handleDependencyToolResult(Filename, MaybeFile, DependencyOS, Errs))
HadErrors = true;
}
};
#if LLVM_ENABLE_THREADS
@ -554,8 +356,5 @@ int main(int argc, const char **argv) {
for (auto &W : WorkerThreads)
W.join();
if (Format == ScanningOutputFormat::Full)
FD.printFullOutput(llvm::outs());
return HadErrors;
}