maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[clangd] Use in-memory preambles in clangd. 

Reviewers: klimek, bkramer, sammccall

Reviewed By: sammccall

Subscribers: cfe-commits

Differential Revision: https://reviews.llvm.org/D39843

llvm-svn: 318412
This commit is contained in:
Ilya Biryukov 2017-11-16 16:25:18 +00:00
parent 417085ac37
commit e9eb7f0cb8
12 changed files with 92 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
clang-tools-extra/clangd/ClangdLSPServer.cpp
View File

@ -235,11 +235,13 @@ void ClangdLSPServer::onSwitchSourceHeader(Ctx C,
}
ClangdLSPServer::ClangdLSPServer(JSONOutput &Out, unsigned AsyncThreadsCount,
bool StorePreamblesInMemory,
bool SnippetCompletions,
llvm::Optional<StringRef> ResourceDir,
llvm::Optional<Path> CompileCommandsDir)
: Out(Out), CDB(/*Logger=*/Out, std::move(CompileCommandsDir)),
Server(CDB, /*DiagConsumer=*/*this, FSProvider, AsyncThreadsCount,
StorePreamblesInMemory,
clangd::CodeCompleteOptions(
/*EnableSnippetsAndCodePatterns=*/SnippetCompletions),
/*Logger=*/Out, ResourceDir) {}

2
clang-tools-extra/clangd/ClangdLSPServer.h
View File

@ -31,7 +31,7 @@ public:
/// loaded only from \p CompileCommandsDir. Otherwise, clangd will look
/// for compile_commands.json in all parent directories of each file.
ClangdLSPServer(JSONOutput &Out, unsigned AsyncThreadsCount,
bool SnippetCompletions,
bool StorePreamblesInMemory, bool SnippetCompletions,
llvm::Optional<StringRef> ResourceDir,
llvm::Optional<Path> CompileCommandsDir);

21
clang-tools-extra/clangd/ClangdServer.cpp
View File

@ -169,17 +169,16 @@ ClangdScheduler::~ClangdScheduler() {
Worker.join();
}
ClangdServer::ClangdServer(GlobalCompilationDatabase &CDB,
DiagnosticsConsumer &DiagConsumer,
FileSystemProvider &FSProvider,
unsigned AsyncThreadsCount,
clangd::CodeCompleteOptions CodeCompleteOpts,
clangd::Logger &Logger,
llvm::Optional<StringRef> ResourceDir)
ClangdServer::ClangdServer(
GlobalCompilationDatabase &CDB, DiagnosticsConsumer &DiagConsumer,
FileSystemProvider &FSProvider, unsigned AsyncThreadsCount,
bool StorePreamblesInMemory, clangd::CodeCompleteOptions CodeCompleteOpts,
clangd::Logger &Logger, llvm::Optional<StringRef> ResourceDir)
: Logger(Logger), CDB(CDB), DiagConsumer(DiagConsumer),
FSProvider(FSProvider),
ResourceDir(ResourceDir ? ResourceDir->str() : getStandardResourceDir()),
PCHs(std::make_shared<PCHContainerOperations>()),
StorePreamblesInMemory(StorePreamblesInMemory),
CodeCompleteOpts(CodeCompleteOpts), WorkScheduler(AsyncThreadsCount) {}
void ClangdServer::setRootPath(PathRef RootPath) {
@ -193,8 +192,8 @@ std::future<void> ClangdServer::addDocument(PathRef File, StringRef Contents) {
DocVersion Version = DraftMgr.updateDraft(File, Contents);
auto TaggedFS = FSProvider.getTaggedFileSystem(File);
std::shared_ptr<CppFile> Resources =
Units.getOrCreateFile(File, ResourceDir, CDB, PCHs, Logger);
std::shared_ptr<CppFile> Resources = Units.getOrCreateFile(
File, ResourceDir, CDB, StorePreamblesInMemory, PCHs, Logger);
return scheduleReparseAndDiags(File, VersionedDraft{Version, Contents.str()},
std::move(Resources), std::move(TaggedFS));
}
@ -211,8 +210,8 @@ std::future<void> ClangdServer::forceReparse(PathRef File) {
"forceReparse() was called for non-added document");
auto TaggedFS = FSProvider.getTaggedFileSystem(File);
auto Recreated = Units.recreateFileIfCompileCommandChanged(File, ResourceDir,
CDB, PCHs, Logger);
auto Recreated = Units.recreateFileIfCompileCommandChanged(
File, ResourceDir, CDB, StorePreamblesInMemory, PCHs, Logger);
// Note that std::future from this cleanup action is ignored.
scheduleCancelRebuild(std::move(Recreated.RemovedFile));

5
clang-tools-extra/clangd/ClangdServer.h
View File

@ -205,10 +205,14 @@ public:
/// worker thread. Therefore, instances of \p DiagConsumer must properly
/// synchronize access to shared state.
///
/// \p StorePreamblesInMemory defines whether the Preambles generated by
/// clangd are stored in-memory or on disk.
///
/// Various messages are logged using \p Logger.
ClangdServer(GlobalCompilationDatabase &CDB,
DiagnosticsConsumer &DiagConsumer,
FileSystemProvider &FSProvider, unsigned AsyncThreadsCount,
bool StorePreamblesInMemory,
clangd::CodeCompleteOptions CodeCompleteOpts,
clangd::Logger &Logger,
llvm::Optional<StringRef> ResourceDir = llvm::None);
@ -325,6 +329,7 @@ private:
// If set, this represents the workspace path.
llvm::Optional<std::string> RootPath;
std::shared_ptr<PCHContainerOperations> PCHs;
bool StorePreamblesInMemory;
clangd::CodeCompleteOptions CodeCompleteOpts;
/// Used to serialize diagnostic callbacks.
/// FIXME(ibiryukov): get rid of an extra map and put all version counters

13
clang-tools-extra/clangd/ClangdUnit.cpp
View File

@ -235,7 +235,7 @@ prepareCompilerInstance(std::unique_ptr<clang::CompilerInvocation> CI,
// NOTE: we use Buffer.get() when adding remapped files, so we have to make
// sure it will be released if no error is emitted.
if (Preamble) {
Preamble->AddImplicitPreamble(*CI, Buffer.get());
Preamble->AddImplicitPreamble(*CI, VFS, Buffer.get());
} else {
CI->getPreprocessorOpts().addRemappedFile(
CI->getFrontendOpts().Inputs[0].getFile(), Buffer.get());
@ -1137,16 +1137,20 @@ PreambleData::PreambleData(PrecompiledPreamble Preamble,
std::shared_ptr<CppFile>
CppFile::Create(PathRef FileName, tooling::CompileCommand Command,
bool StorePreamblesInMemory,
std::shared_ptr<PCHContainerOperations> PCHs,
clangd::Logger &Logger) {
return std::shared_ptr<CppFile>(
new CppFile(FileName, std::move(Command), std::move(PCHs), Logger));
return std::shared_ptr<CppFile>(new CppFile(FileName, std::move(Command),
StorePreamblesInMemory,
std::move(PCHs), Logger));
}
CppFile::CppFile(PathRef FileName, tooling::CompileCommand Command,
bool StorePreamblesInMemory,
std::shared_ptr<PCHContainerOperations> PCHs,
clangd::Logger &Logger)
: FileName(FileName), Command(std::move(Command)), RebuildCounter(0),
: FileName(FileName), Command(std::move(Command)),
StorePreamblesInMemory(StorePreamblesInMemory), RebuildCounter(0),
RebuildInProgress(false), PCHs(std::move(PCHs)), Logger(Logger) {
std::lock_guard<std::mutex> Lock(Mutex);
@ -1290,6 +1294,7 @@ CppFile::deferRebuild(StringRef NewContents,
CppFilePreambleCallbacks SerializedDeclsCollector;
auto BuiltPreamble = PrecompiledPreamble::Build(
*CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine, VFS, PCHs,
/*StoreInMemory=*/That->StorePreamblesInMemory,
SerializedDeclsCollector);
if (BuiltPreamble) {

3
clang-tools-extra/clangd/ClangdUnit.h
View File

@ -143,10 +143,12 @@ public:
// deferRebuild will hold references to it.
static std::shared_ptr<CppFile>
Create(PathRef FileName, tooling::CompileCommand Command,
bool StorePreamblesInMemory,
std::shared_ptr<PCHContainerOperations> PCHs, clangd::Logger &Logger);
private:
CppFile(PathRef FileName, tooling::CompileCommand Command,
bool StorePreamblesInMemory,
std::shared_ptr<PCHContainerOperations> PCHs, clangd::Logger &Logger);
public:
@ -222,6 +224,7 @@ private:
Path FileName;
tooling::CompileCommand Command;
bool StorePreamblesInMemory;
/// Mutex protects all fields, declared below it, FileName and Command are not
/// mutated.

7
clang-tools-extra/clangd/ClangdUnitStore.cpp
View File

@ -29,7 +29,8 @@ std::shared_ptr<CppFile> CppFileCollection::removeIfPresent(PathRef File) {
CppFileCollection::RecreateResult
CppFileCollection::recreateFileIfCompileCommandChanged(
PathRef File, PathRef ResourceDir, GlobalCompilationDatabase &CDB,
std::shared_ptr<PCHContainerOperations> PCHs, clangd::Logger &Logger) {
bool StorePreamblesInMemory, std::shared_ptr<PCHContainerOperations> PCHs,
clangd::Logger &Logger) {
auto NewCommand = getCompileCommand(CDB, File, ResourceDir);
std::lock_guard<std::mutex> Lock(Mutex);
@ -40,13 +41,15 @@ CppFileCollection::recreateFileIfCompileCommandChanged(
if (It == OpenedFiles.end()) {
It = OpenedFiles
.try_emplace(File, CppFile::Create(File, std::move(NewCommand),
StorePreamblesInMemory,
std::move(PCHs), Logger))
.first;
} else if (!compileCommandsAreEqual(It->second->getCompileCommand(),
NewCommand)) {
Result.RemovedFile = std::move(It->second);
It->second =
CppFile::Create(File, std::move(NewCommand), std::move(PCHs), Logger);
CppFile::Create(File, std::move(NewCommand), StorePreamblesInMemory,
std::move(PCHs), Logger);
}
Result.FileInCollection = It->second;
return Result;

12
clang-tools-extra/clangd/ClangdUnitStore.h
View File

@ -25,9 +25,11 @@ class Logger;
/// Thread-safe mapping from FileNames to CppFile.
class CppFileCollection {
public:
std::shared_ptr<CppFile> getOrCreateFile(
PathRef File, PathRef ResourceDir, GlobalCompilationDatabase &CDB,
std::shared_ptr<PCHContainerOperations> PCHs, clangd::Logger &Logger) {
std::shared_ptr<CppFile>
getOrCreateFile(PathRef File, PathRef ResourceDir,
GlobalCompilationDatabase &CDB, bool StorePreamblesInMemory,
std::shared_ptr<PCHContainerOperations> PCHs,
clangd::Logger &Logger) {
std::lock_guard<std::mutex> Lock(Mutex);
auto It = OpenedFiles.find(File);
@ -36,6 +38,7 @@ public:
It = OpenedFiles
.try_emplace(File, CppFile::Create(File, std::move(Command),
StorePreamblesInMemory,
std::move(PCHs), Logger))
.first;
}
@ -58,7 +61,8 @@ public:
/// will be returned in RecreateResult.RemovedFile.
RecreateResult recreateFileIfCompileCommandChanged(
PathRef File, PathRef ResourceDir, GlobalCompilationDatabase &CDB,
std::shared_ptr<PCHContainerOperations> PCHs, clangd::Logger &Logger);
bool StorePreamblesInMemory, std::shared_ptr<PCHContainerOperations> PCHs,
clangd::Logger &Logger);
std::shared_ptr<CppFile> getFile(PathRef File) {
std::lock_guard<std::mutex> Lock(Mutex);

28
clang-tools-extra/clangd/tool/ClangdMain.cpp
View File

@ -24,6 +24,10 @@
using namespace clang;
using namespace clang::clangd;
namespace {
enum class PCHStorageFlag { Disk, Memory };
}
static llvm::cl::opt<Path> CompileCommandsDir(
"compile-commands-dir",
llvm::cl::desc("Specify a path to look for compile_commands.json. If path "
@ -45,6 +49,15 @@ static llvm::cl::opt<bool>
PrettyPrint("pretty", llvm::cl::desc("Pretty-print JSON output"),
llvm::cl::init(false));
static llvm::cl::opt<PCHStorageFlag> PCHStorage(
"pch-storage",
llvm::cl::desc("Storing PCHs in memory increases memory usages, but may "
"improve performance"),
llvm::cl::values(
clEnumValN(PCHStorageFlag::Disk, "disk", "store PCHs on disk"),
clEnumValN(PCHStorageFlag::Memory, "memory", "store PCHs in memory")),
llvm::cl::init(PCHStorageFlag::Disk));
static llvm::cl::opt<bool> RunSynchronously(
"run-synchronously",
llvm::cl::desc("Parse on main thread. If set, -j is ignored"),
@ -127,6 +140,16 @@ int main(int argc, char *argv[]) {
CompileCommandsDirPath = CompileCommandsDir;
}
bool StorePreamblesInMemory;
switch (PCHStorage) {
case PCHStorageFlag::Memory:
StorePreamblesInMemory = true;
break;
case PCHStorageFlag::Disk:
StorePreamblesInMemory = false;
break;
}
llvm::Optional<StringRef> ResourceDirRef = None;
if (!ResourceDir.empty())
ResourceDirRef = ResourceDir;
@ -135,8 +158,9 @@ int main(int argc, char *argv[]) {
llvm::sys::ChangeStdinToBinary();
// Initialize and run ClangdLSPServer.
ClangdLSPServer LSPServer(Out, WorkerThreadsCount, EnableSnippets,
ResourceDirRef, CompileCommandsDirPath);
ClangdLSPServer LSPServer(Out, WorkerThreadsCount, StorePreamblesInMemory,
EnableSnippets, ResourceDirRef,
CompileCommandsDirPath);
constexpr int NoShutdownRequestErrorCode = 1;
llvm::set_thread_name("clangd.main");
return LSPServer.run(std::cin) ? 0 : NoShutdownRequestErrorCode;

1
clang-tools-extra/test/clangd/completion.test
View File

@ -1,4 +1,5 @@
# RUN: clangd -pretty -run-synchronously < %s | FileCheck -strict-whitespace %s
# RUN: clangd -pretty -run-synchronously -pch-storage=memory < %s | FileCheck -strict-whitespace %s
# It is absolutely vital that this file has CRLF line endings.
#
Content-Length: 125

1
clang-tools-extra/test/clangd/diagnostics-preamble.test
View File

@ -1,4 +1,5 @@
# RUN: clangd -pretty -run-synchronously < %s | FileCheck -strict-whitespace %s
# RUN: clangd -pretty -run-synchronously -pch-storage=memory < %s | FileCheck -strict-whitespace %s
# It is absolutely vital that this file has CRLF line endings.
#
Content-Length: 125

27
clang-tools-extra/unittests/clangd/ClangdTests.cpp
View File

@ -332,6 +332,7 @@ protected:
ErrorCheckingDiagConsumer DiagConsumer;
MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true);
ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(),
/*StorePreamblesInMemory=*/true,
clangd::CodeCompleteOptions(),
EmptyLogger::getInstance());
for (const auto &FileWithContents : ExtraFiles)
@ -396,6 +397,7 @@ TEST_F(ClangdVFSTest, Reparse) {
ErrorCheckingDiagConsumer DiagConsumer;
MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true);
ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(),
/*StorePreamblesInMemory=*/true,
clangd::CodeCompleteOptions(),
EmptyLogger::getInstance());
@ -442,6 +444,7 @@ TEST_F(ClangdVFSTest, ReparseOnHeaderChange) {
MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true);
ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(),
/*StorePreamblesInMemory=*/true,
clangd::CodeCompleteOptions(),
EmptyLogger::getInstance());
@ -490,7 +493,9 @@ TEST_F(ClangdVFSTest, CheckVersions) {
MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true);
// Run ClangdServer synchronously.
ClangdServer Server(CDB, DiagConsumer, FS,
/*AsyncThreadsCount=*/0, clangd::CodeCompleteOptions(),
/*AsyncThreadsCount=*/0,
/*StorePreamblesInMemory=*/true,
clangd::CodeCompleteOptions(),
EmptyLogger::getInstance());
auto FooCpp = getVirtualTestFilePath("foo.cpp");
@ -524,7 +529,9 @@ TEST_F(ClangdVFSTest, SearchLibDir) {
"-stdlib=libstdc++"});
// Run ClangdServer synchronously.
ClangdServer Server(CDB, DiagConsumer, FS,
/*AsyncThreadsCount=*/0, clangd::CodeCompleteOptions(),
/*AsyncThreadsCount=*/0,
/*StorePreamblesInMemory=*/true,
clangd::CodeCompleteOptions(),
EmptyLogger::getInstance());
// Just a random gcc version string
@ -573,7 +580,9 @@ TEST_F(ClangdVFSTest, ForceReparseCompileCommand) {
ErrorCheckingDiagConsumer DiagConsumer;
MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true);
ClangdServer Server(CDB, DiagConsumer, FS,
/*AsyncThreadsCount=*/0, clangd::CodeCompleteOptions(),
/*AsyncThreadsCount=*/0,
/*StorePreamblesInMemory=*/true,
clangd::CodeCompleteOptions(),
EmptyLogger::getInstance());
// No need to sync reparses, because reparses are performed on the calling
// thread to true.
@ -641,6 +650,7 @@ TEST_F(ClangdCompletionTest, CheckContentsOverride) {
MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true);
ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(),
/*StorePreamblesInMemory=*/true,
clangd::CodeCompleteOptions(),
EmptyLogger::getInstance());
@ -701,7 +711,8 @@ TEST_F(ClangdCompletionTest, Limit) {
clangd::CodeCompleteOptions Opts;
Opts.Limit = 2;
ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(),
Opts, EmptyLogger::getInstance());
/*StorePreamblesInMemory=*/true, Opts,
EmptyLogger::getInstance());
auto FooCpp = getVirtualTestFilePath("foo.cpp");
FS.Files[FooCpp] = "";
@ -796,7 +807,8 @@ int test() {
auto TestWithOpts = [&](clangd::CodeCompleteOptions Opts) {
ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(),
Opts, EmptyLogger::getInstance());
/*StorePreamblesInMemory=*/true, Opts,
EmptyLogger::getInstance());
// No need to sync reparses here as there are no asserts on diagnostics (or
// other async operations).
Server.addDocument(FooCpp, GlobalCompletion.Text);
@ -989,6 +1001,7 @@ int d;
{
MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true);
ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(),
/*StorePreamblesInMemory=*/true,
clangd::CodeCompleteOptions(),
EmptyLogger::getInstance());
@ -1149,6 +1162,7 @@ TEST_F(ClangdVFSTest, CheckSourceHeaderSwitch) {
MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true);
ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(),
/*StorePreamblesInMemory=*/true,
clangd::CodeCompleteOptions(),
EmptyLogger::getInstance());
@ -1275,7 +1289,8 @@ int d;
std::move(StartSecondReparsePromise));
MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true);
ClangdServer Server(CDB, DiagConsumer, FS, 4, clangd::CodeCompleteOptions(),
ClangdServer Server(CDB, DiagConsumer, FS, 4, /*StorePreamblesInMemory=*/true,
clangd::CodeCompleteOptions(),
EmptyLogger::getInstance());
Server.addDocument(FooCpp, SourceContentsWithErrors);
StartSecondReparse.wait();