[clangd] Rewrote AST and Preamble management.

Summary: The new implementation allows code completion that never waits for AST.

Reviewers: bkramer, krasimir, klimek

Reviewed By: bkramer

Subscribers: cfe-commits

Differential Revision: https://reviews.llvm.org/D36133

llvm-svn: 309696
This commit is contained in:
Ilya Biryukov 2017-08-01 15:51:38 +00:00
parent cb870c57b3
commit 02d5870de3
8 changed files with 764 additions and 441 deletions

View File

@ -23,6 +23,16 @@ using namespace clang::clangd;
namespace {
class FulfillPromiseGuard {
public:
FulfillPromiseGuard(std::promise<void> &Promise) : Promise(Promise) {}
~FulfillPromiseGuard() { Promise.set_value(); }
private:
std::promise<void> &Promise;
};
std::vector<tooling::Replacement> formatCode(StringRef Code, StringRef Filename,
ArrayRef<tooling::Range> Ranges) {
// Call clang-format.
@ -79,7 +89,7 @@ ClangdScheduler::ClangdScheduler(bool RunSynchronously)
// using not-yet-initialized members
Worker = std::thread([this]() {
while (true) {
std::function<void()> Request;
std::future<void> Request;
// Pick request from the queue
{
@ -99,7 +109,7 @@ ClangdScheduler::ClangdScheduler(bool RunSynchronously)
RequestQueue.pop_front();
} // unlock Mutex
Request();
Request.get();
}
});
}
@ -117,32 +127,6 @@ ClangdScheduler::~ClangdScheduler() {
Worker.join();
}
void ClangdScheduler::addToFront(std::function<void()> Request) {
if (RunSynchronously) {
Request();
return;
}
{
std::lock_guard<std::mutex> Lock(Mutex);
RequestQueue.push_front(Request);
}
RequestCV.notify_one();
}
void ClangdScheduler::addToEnd(std::function<void()> Request) {
if (RunSynchronously) {
Request();
return;
}
{
std::lock_guard<std::mutex> Lock(Mutex);
RequestQueue.push_back(Request);
}
RequestCV.notify_one();
}
ClangdServer::ClangdServer(GlobalCompilationDatabase &CDB,
DiagnosticsConsumer &DiagConsumer,
FileSystemProvider &FSProvider,
@ -153,41 +137,73 @@ ClangdServer::ClangdServer(GlobalCompilationDatabase &CDB,
PCHs(std::make_shared<PCHContainerOperations>()),
WorkScheduler(RunSynchronously) {}
void ClangdServer::addDocument(PathRef File, StringRef Contents) {
std::future<void> ClangdServer::addDocument(PathRef File, StringRef Contents) {
DocVersion Version = DraftMgr.updateDraft(File, Contents);
Path FileStr = File;
WorkScheduler.addToFront([this, FileStr, Version]() {
auto FileContents = DraftMgr.getDraft(FileStr);
if (FileContents.Version != Version)
return; // This request is outdated, do nothing
assert(FileContents.Draft &&
"No contents inside a file that was scheduled for reparse");
auto TaggedFS = FSProvider.getTaggedFileSystem(FileStr);
Units.runOnUnit(
FileStr, *FileContents.Draft, ResourceDir, CDB, PCHs, TaggedFS.Value,
[&](ClangdUnit const &Unit) {
DiagConsumer.onDiagnosticsReady(
FileStr, make_tagged(Unit.getLocalDiagnostics(), TaggedFS.Tag));
});
});
auto TaggedFS = FSProvider.getTaggedFileSystem(File);
std::shared_ptr<CppFile> Resources =
Units.getOrCreateFile(File, ResourceDir, CDB, PCHs, TaggedFS.Value);
std::future<llvm::Optional<std::vector<DiagWithFixIts>>> DeferredRebuild =
Resources->deferRebuild(Contents, TaggedFS.Value);
std::promise<void> DonePromise;
std::future<void> DoneFuture = DonePromise.get_future();
Path FileStr = File;
VFSTag Tag = TaggedFS.Tag;
auto ReparseAndPublishDiags =
[this, FileStr, Version,
Tag](std::future<llvm::Optional<std::vector<DiagWithFixIts>>>
DeferredRebuild,
std::promise<void> DonePromise) -> void {
FulfillPromiseGuard Guard(DonePromise);
auto CurrentVersion = DraftMgr.getVersion(FileStr);
if (CurrentVersion != Version)
return; // This request is outdated
auto Diags = DeferredRebuild.get();
if (!Diags)
return; // A new reparse was requested before this one completed.
DiagConsumer.onDiagnosticsReady(FileStr,
make_tagged(std::move(*Diags), Tag));
};
WorkScheduler.addToFront(std::move(ReparseAndPublishDiags),
std::move(DeferredRebuild), std::move(DonePromise));
return DoneFuture;
}
void ClangdServer::removeDocument(PathRef File) {
std::future<void> ClangdServer::removeDocument(PathRef File) {
auto Version = DraftMgr.removeDraft(File);
Path FileStr = File;
WorkScheduler.addToFront([this, FileStr, Version]() {
std::promise<void> DonePromise;
std::future<void> DoneFuture = DonePromise.get_future();
auto RemoveDocFromCollection = [this, FileStr,
Version](std::promise<void> DonePromise) {
FulfillPromiseGuard Guard(DonePromise);
if (Version != DraftMgr.getVersion(FileStr))
return; // This request is outdated, do nothing
Units.removeUnitIfPresent(FileStr);
});
std::shared_ptr<CppFile> File = Units.removeIfPresent(FileStr);
if (!File)
return;
// Cancel all ongoing rebuilds, so that we don't do extra work before
// deleting this file.
File->cancelRebuilds();
};
WorkScheduler.addToFront(std::move(RemoveDocFromCollection),
std::move(DonePromise));
return DoneFuture;
}
void ClangdServer::forceReparse(PathRef File) {
std::future<void> ClangdServer::forceReparse(PathRef File) {
// The addDocument schedules the reparse even if the contents of the file
// never changed, so we just call it here.
addDocument(File, getDocument(File));
return addDocument(File, getDocument(File));
}
Tagged<std::vector<CompletionItem>>
@ -208,12 +224,14 @@ ClangdServer::codeComplete(PathRef File, Position Pos,
if (UsedFS)
*UsedFS = TaggedFS.Value;
std::vector<CompletionItem> Result;
Units.runOnUnitWithoutReparse(File, *OverridenContents, ResourceDir, CDB,
PCHs, TaggedFS.Value, [&](ClangdUnit &Unit) {
Result = Unit.codeComplete(
*OverridenContents, Pos, TaggedFS.Value);
});
std::shared_ptr<CppFile> Resources = Units.getFile(File);
assert(Resources && "Calling completion on non-added file");
auto Preamble = Resources->getPossiblyStalePreamble();
std::vector<CompletionItem> Result =
clangd::codeComplete(File, Resources->getCompileCommand(),
Preamble ? &Preamble->Preamble : nullptr,
*OverridenContents, Pos, TaggedFS.Value, PCHs);
return make_tagged(std::move(Result), TaggedFS.Tag);
}
@ -253,37 +271,38 @@ std::string ClangdServer::getDocument(PathRef File) {
}
std::string ClangdServer::dumpAST(PathRef File) {
std::promise<std::string> DumpPromise;
auto DumpFuture = DumpPromise.get_future();
auto Version = DraftMgr.getVersion(File);
std::shared_ptr<CppFile> Resources = Units.getFile(File);
assert(Resources && "dumpAST is called for non-added document");
WorkScheduler.addToEnd([this, &DumpPromise, File, Version]() {
assert(DraftMgr.getVersion(File) == Version && "Version has changed");
(void)Version;
Units.runOnExistingUnit(File, [&DumpPromise](ClangdUnit &Unit) {
std::string Result;
Resources->getAST().get().runUnderLock([&Result](ParsedAST *AST) {
llvm::raw_string_ostream ResultOS(Result);
Unit.dumpAST(ResultOS);
if (AST) {
clangd::dumpAST(*AST, ResultOS);
} else {
ResultOS << "<no-ast>";
}
ResultOS.flush();
DumpPromise.set_value(std::move(Result));
});
});
return DumpFuture.get();
return Result;
}
Tagged<std::vector<Location>>
ClangdServer::findDefinitions(PathRef File, Position Pos) {
Tagged<std::vector<Location>> ClangdServer::findDefinitions(PathRef File,
Position Pos) {
auto FileContents = DraftMgr.getDraft(File);
assert(FileContents.Draft && "findDefinitions is called for non-added document");
assert(FileContents.Draft &&
"findDefinitions is called for non-added document");
auto TaggedFS = FSProvider.getTaggedFileSystem(File);
std::shared_ptr<CppFile> Resources = Units.getFile(File);
assert(Resources && "Calling findDefinitions on non-added file");
std::vector<Location> Result;
auto TaggedFS = FSProvider.getTaggedFileSystem(File);
Units.runOnUnit(File, *FileContents.Draft, ResourceDir, CDB, PCHs,
TaggedFS.Value, [&](ClangdUnit &Unit) {
Result = Unit.findDefinitions(Pos);
Resources->getAST().get().runUnderLock([Pos, &Result](ParsedAST *AST) {
if (!AST)
return;
Result = clangd::findDefinitions(*AST, Pos);
});
return make_tagged(std::move(Result), TaggedFS.Tag);
}

View File

@ -108,21 +108,45 @@ public:
ClangdScheduler(bool RunSynchronously);
~ClangdScheduler();
/// Add \p Request to the start of the queue. \p Request will be run on a
/// separate worker thread.
/// \p Request is scheduled to be executed before all currently added
/// requests.
void addToFront(std::function<void()> Request);
/// Add \p Request to the end of the queue. \p Request will be run on a
/// separate worker thread.
/// \p Request is scheduled to be executed after all currently added
/// requests.
void addToEnd(std::function<void()> Request);
/// Add a new request to run function \p F with args \p As to the start of the
/// queue. The request will be run on a separate thread.
template <class Func, class... Args>
void addToFront(Func &&F, Args &&... As) {
if (RunSynchronously) {
std::forward<Func>(F)(std::forward<Args>(As)...);
return;
}
{
std::lock_guard<std::mutex> Lock(Mutex);
RequestQueue.push_front(std::async(std::launch::deferred,
std::forward<Func>(F),
std::forward<Args>(As)...));
}
RequestCV.notify_one();
}
/// Add a new request to run function \p F with args \p As to the end of the
/// queue. The request will be run on a separate thread.
template <class Func, class... Args> void addToEnd(Func &&F, Args &&... As) {
if (RunSynchronously) {
std::forward<Func>(F)(std::forward<Args>(As)...);
return;
}
{
std::lock_guard<std::mutex> Lock(Mutex);
RequestQueue.push_back(std::async(std::launch::deferred,
std::forward<Func>(F),
std::forward<Args>(As)...));
}
RequestCV.notify_one();
}
private:
bool RunSynchronously;
std::mutex Mutex;
/// We run some tasks on a separate thread(parsing, ClangdUnit cleanup).
/// We run some tasks on a separate threads(parsing, CppFile cleanup).
/// This thread looks into RequestQueue to find requests to handle and
/// terminates when Done is set to true.
std::thread Worker;
@ -131,7 +155,7 @@ private:
/// A queue of requests.
/// FIXME(krasimir): code completion should always have priority over parsing
/// for diagnostics.
std::deque<std::function<void()>> RequestQueue;
std::deque<std::future<void>> RequestQueue;
/// Condition variable to wake up the worker thread.
std::condition_variable RequestCV;
};
@ -163,12 +187,16 @@ public:
/// \p File is already tracked. Also schedules parsing of the AST for it on a
/// separate thread. When the parsing is complete, DiagConsumer passed in
/// constructor will receive onDiagnosticsReady callback.
void addDocument(PathRef File, StringRef Contents);
/// \return A future that will become ready when the rebuild (including
/// diagnostics) is finished.
std::future<void> addDocument(PathRef File, StringRef Contents);
/// Remove \p File from list of tracked files, schedule a request to free
/// resources associated with it.
void removeDocument(PathRef File);
/// \return A future that will become ready the file is removed and all
/// associated reosources are freed.
std::future<void> removeDocument(PathRef File);
/// Force \p File to be reparsed using the latest contents.
void forceReparse(PathRef File);
std::future<void> forceReparse(PathRef File);
/// Run code completion for \p File at \p Pos. If \p OverridenContents is not
/// None, they will used only for code completion, i.e. no diagnostics update
@ -209,7 +237,7 @@ private:
DiagnosticsConsumer &DiagConsumer;
FileSystemProvider &FSProvider;
DraftStore DraftMgr;
ClangdUnitStore Units;
CppFileCollection Units;
std::string ResourceDir;
std::shared_ptr<PCHContainerOperations> PCHs;
// WorkScheduler has to be the last member, because its destructor has to be

View File

@ -28,6 +28,7 @@
#include "llvm/Support/Format.h"
#include <algorithm>
#include <chrono>
using namespace clang::clangd;
using namespace clang;
@ -70,7 +71,7 @@ private:
std::vector<const Decl *> TopLevelDecls;
};
class ClangdUnitPreambleCallbacks : public PreambleCallbacks {
class CppFilePreambleCallbacks : public PreambleCallbacks {
public:
std::vector<serialization::DeclID> takeTopLevelDeclIDs() {
return std::move(TopLevelDeclIDs);
@ -220,73 +221,12 @@ prepareCompilerInstance(std::unique_ptr<clang::CompilerInvocation> CI,
return Clang;
}
template <class T> bool futureIsReady(std::shared_future<T> const &Future) {
return Future.wait_for(std::chrono::seconds(0)) == std::future_status::ready;
}
} // namespace
ClangdUnit::ClangdUnit(PathRef FileName, StringRef Contents,
StringRef ResourceDir,
std::shared_ptr<PCHContainerOperations> PCHs,
std::vector<tooling::CompileCommand> Commands,
IntrusiveRefCntPtr<vfs::FileSystem> VFS)
: FileName(FileName), PCHs(PCHs) {
assert(!Commands.empty() && "No compile commands provided");
// Inject the resource dir.
// FIXME: Don't overwrite it if it's already there.
Commands.front().CommandLine.push_back("-resource-dir=" +
std::string(ResourceDir));
Command = std::move(Commands.front());
reparse(Contents, VFS);
}
void ClangdUnit::reparse(StringRef Contents,
IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
std::vector<const char *> ArgStrs;
for (const auto &S : Command.CommandLine)
ArgStrs.push_back(S.c_str());
VFS->setCurrentWorkingDirectory(Command.Directory);
std::unique_ptr<CompilerInvocation> CI;
{
// FIXME(ibiryukov): store diagnostics from CommandLine when we start
// reporting them.
EmptyDiagsConsumer CommandLineDiagsConsumer;
IntrusiveRefCntPtr<DiagnosticsEngine> CommandLineDiagsEngine =
CompilerInstance::createDiagnostics(new DiagnosticOptions,
&CommandLineDiagsConsumer, false);
CI = createCompilerInvocation(ArgStrs, CommandLineDiagsEngine, VFS);
}
assert(CI && "Couldn't create CompilerInvocation");
std::unique_ptr<llvm::MemoryBuffer> ContentsBuffer =
llvm::MemoryBuffer::getMemBufferCopy(Contents, FileName);
// Rebuild the preamble if it is missing or can not be reused.
auto Bounds =
ComputePreambleBounds(*CI->getLangOpts(), ContentsBuffer.get(), 0);
if (!Preamble || !Preamble->Preamble.CanReuse(*CI, ContentsBuffer.get(),
Bounds, VFS.get())) {
std::vector<DiagWithFixIts> PreambleDiags;
StoreDiagsConsumer PreambleDiagnosticsConsumer(/*ref*/ PreambleDiags);
IntrusiveRefCntPtr<DiagnosticsEngine> PreambleDiagsEngine =
CompilerInstance::createDiagnostics(
&CI->getDiagnosticOpts(), &PreambleDiagnosticsConsumer, false);
ClangdUnitPreambleCallbacks SerializedDeclsCollector;
auto BuiltPreamble = PrecompiledPreamble::Build(
*CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine, VFS, PCHs,
SerializedDeclsCollector);
if (BuiltPreamble)
Preamble = PreambleData(std::move(*BuiltPreamble),
SerializedDeclsCollector.takeTopLevelDeclIDs(),
std::move(PreambleDiags));
}
Unit = ParsedAST::Build(
std::move(CI), Preamble ? &Preamble->Preamble : nullptr,
Preamble ? llvm::makeArrayRef(Preamble->TopLevelDeclIDs) : llvm::None,
std::move(ContentsBuffer), PCHs, VFS);
}
namespace {
CompletionItemKind getKind(CXCursorKind K) {
@ -390,8 +330,10 @@ public:
} // namespace
std::vector<CompletionItem>
ClangdUnit::codeComplete(StringRef Contents, Position Pos,
IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
clangd::codeComplete(PathRef FileName, tooling::CompileCommand Command,
PrecompiledPreamble const *Preamble, StringRef Contents,
Position Pos, IntrusiveRefCntPtr<vfs::FileSystem> VFS,
std::shared_ptr<PCHContainerOperations> PCHs) {
std::vector<const char *> ArgStrs;
for (const auto &S : Command.CommandLine)
ArgStrs.push_back(S.c_str());
@ -412,16 +354,14 @@ ClangdUnit::codeComplete(StringRef Contents, Position Pos,
llvm::MemoryBuffer::getMemBufferCopy(Contents, FileName);
// Attempt to reuse the PCH from precompiled preamble, if it was built.
const PrecompiledPreamble *PreambleForCompletion = nullptr;
if (Preamble) {
auto Bounds =
ComputePreambleBounds(*CI->getLangOpts(), ContentsBuffer.get(), 0);
if (Preamble->Preamble.CanReuse(*CI, ContentsBuffer.get(), Bounds,
VFS.get()))
PreambleForCompletion = &Preamble->Preamble;
if (!Preamble->CanReuse(*CI, ContentsBuffer.get(), Bounds, VFS.get()))
Preamble = nullptr;
}
auto Clang = prepareCompilerInstance(std::move(CI), PreambleForCompletion,
auto Clang = prepareCompilerInstance(std::move(CI), Preamble,
std::move(ContentsBuffer), PCHs, VFS,
DummyDiagsConsumer);
auto &DiagOpts = Clang->getDiagnosticOpts();
@ -457,31 +397,12 @@ ClangdUnit::codeComplete(StringRef Contents, Position Pos,
return Items;
}
std::vector<DiagWithFixIts> ClangdUnit::getLocalDiagnostics() const {
if (!Unit)
return {}; // Parsing failed.
std::vector<DiagWithFixIts> Result;
auto PreambleDiagsSize = Preamble ? Preamble->Diags.size() : 0;
const auto &Diags = Unit->getDiagnostics();
Result.reserve(PreambleDiagsSize + Diags.size());
if (Preamble)
Result.insert(Result.end(), Preamble->Diags.begin(), Preamble->Diags.end());
Result.insert(Result.end(), Diags.begin(), Diags.end());
return Result;
void clangd::dumpAST(ParsedAST &AST, llvm::raw_ostream &OS) {
AST.getASTContext().getTranslationUnitDecl()->dump(OS, true);
}
void ClangdUnit::dumpAST(llvm::raw_ostream &OS) const {
if (!Unit) {
OS << "<no-ast-in-clang>";
return; // Parsing failed.
}
Unit->getASTContext().getTranslationUnitDecl()->dump(OS, true);
}
llvm::Optional<ClangdUnit::ParsedAST>
ClangdUnit::ParsedAST::Build(std::unique_ptr<clang::CompilerInvocation> CI,
llvm::Optional<ParsedAST>
ParsedAST::Build(std::unique_ptr<clang::CompilerInvocation> CI,
const PrecompiledPreamble *Preamble,
ArrayRef<serialization::DeclID> PreambleDeclIDs,
std::unique_ptr<llvm::MemoryBuffer> Buffer,
@ -563,10 +484,11 @@ public:
return std::move(DeclarationLocations);
}
bool handleDeclOccurence(const Decl* D, index::SymbolRoleSet Roles,
ArrayRef<index::SymbolRelation> Relations, FileID FID, unsigned Offset,
index::IndexDataConsumer::ASTNodeInfo ASTNode) override
{
bool
handleDeclOccurence(const Decl *D, index::SymbolRoleSet Roles,
ArrayRef<index::SymbolRelation> Relations, FileID FID,
unsigned Offset,
index::IndexDataConsumer::ASTNodeInfo ASTNode) override {
if (isSearchedLocation(FID, Offset)) {
addDeclarationLocation(D->getSourceRange());
}
@ -622,48 +544,20 @@ private:
PP.getMacroDefinitionAtLoc(IdentifierInfo, BeforeSearchedLocation);
MacroInfo *MacroInf = MacroDef.getMacroInfo();
if (MacroInf) {
addDeclarationLocation(
SourceRange(MacroInf->getDefinitionLoc(),
addDeclarationLocation(SourceRange(MacroInf->getDefinitionLoc(),
MacroInf->getDefinitionEndLoc()));
}
}
}
}
};
} // namespace
std::vector<Location> ClangdUnit::findDefinitions(Position Pos) {
if (!Unit)
return {}; // Parsing failed.
const SourceManager &SourceMgr = Unit->getASTContext().getSourceManager();
const FileEntry *FE = SourceMgr.getFileEntryForID(SourceMgr.getMainFileID());
if (!FE)
return {};
SourceLocation SourceLocationBeg = getBeginningOfIdentifier(Pos, FE);
auto DeclLocationsFinder = std::make_shared<DeclarationLocationsFinder>(
llvm::errs(), SourceLocationBeg, Unit->getASTContext(),
Unit->getPreprocessor());
index::IndexingOptions IndexOpts;
IndexOpts.SystemSymbolFilter =
index::IndexingOptions::SystemSymbolFilterKind::All;
IndexOpts.IndexFunctionLocals = true;
indexTopLevelDecls(Unit->getASTContext(), Unit->getTopLevelDecls(),
DeclLocationsFinder, IndexOpts);
return DeclLocationsFinder->takeLocations();
}
SourceLocation ClangdUnit::getBeginningOfIdentifier(const Position &Pos,
const FileEntry *FE) const {
SourceLocation getBeginningOfIdentifier(ParsedAST &Unit, const Position &Pos,
const FileEntry *FE) {
// The language server protocol uses zero-based line and column numbers.
// Clang uses one-based numbers.
const ASTContext &AST = Unit->getASTContext();
const ASTContext &AST = Unit.getASTContext();
const SourceManager &SourceMgr = AST.getSourceManager();
SourceLocation InputLocation =
@ -691,13 +585,36 @@ SourceLocation ClangdUnit::getBeginningOfIdentifier(const Position &Pos,
if (Result.is(tok::raw_identifier)) {
return Lexer::GetBeginningOfToken(PeekBeforeLocation, SourceMgr,
Unit->getASTContext().getLangOpts());
AST.getLangOpts());
}
return InputLocation;
}
} // namespace
void ClangdUnit::ParsedAST::ensurePreambleDeclsDeserialized() {
std::vector<Location> clangd::findDefinitions(ParsedAST &AST, Position Pos) {
const SourceManager &SourceMgr = AST.getASTContext().getSourceManager();
const FileEntry *FE = SourceMgr.getFileEntryForID(SourceMgr.getMainFileID());
if (!FE)
return {};
SourceLocation SourceLocationBeg = getBeginningOfIdentifier(AST, Pos, FE);
auto DeclLocationsFinder = std::make_shared<DeclarationLocationsFinder>(
llvm::errs(), SourceLocationBeg, AST.getASTContext(),
AST.getPreprocessor());
index::IndexingOptions IndexOpts;
IndexOpts.SystemSymbolFilter =
index::IndexingOptions::SystemSymbolFilterKind::All;
IndexOpts.IndexFunctionLocals = true;
indexTopLevelDecls(AST.getASTContext(), AST.getTopLevelDecls(),
DeclLocationsFinder, IndexOpts);
return DeclLocationsFinder->takeLocations();
}
void ParsedAST::ensurePreambleDeclsDeserialized() {
if (PendingTopLevelDecls.empty())
return;
@ -718,45 +635,38 @@ void ClangdUnit::ParsedAST::ensurePreambleDeclsDeserialized() {
PendingTopLevelDecls.clear();
}
ClangdUnit::ParsedAST::ParsedAST(ParsedAST &&Other) = default;
ParsedAST::ParsedAST(ParsedAST &&Other) = default;
ClangdUnit::ParsedAST &ClangdUnit::ParsedAST::
operator=(ParsedAST &&Other) = default;
ParsedAST &ParsedAST::operator=(ParsedAST &&Other) = default;
ClangdUnit::ParsedAST::~ParsedAST() {
ParsedAST::~ParsedAST() {
if (Action) {
Action->EndSourceFile();
}
}
ASTContext &ClangdUnit::ParsedAST::getASTContext() {
ASTContext &ParsedAST::getASTContext() { return Clang->getASTContext(); }
const ASTContext &ParsedAST::getASTContext() const {
return Clang->getASTContext();
}
const ASTContext &ClangdUnit::ParsedAST::getASTContext() const {
return Clang->getASTContext();
}
Preprocessor &ParsedAST::getPreprocessor() { return Clang->getPreprocessor(); }
Preprocessor &ClangdUnit::ParsedAST::getPreprocessor() {
const Preprocessor &ParsedAST::getPreprocessor() const {
return Clang->getPreprocessor();
}
const Preprocessor &ClangdUnit::ParsedAST::getPreprocessor() const {
return Clang->getPreprocessor();
}
ArrayRef<const Decl *> ClangdUnit::ParsedAST::getTopLevelDecls() {
ArrayRef<const Decl *> ParsedAST::getTopLevelDecls() {
ensurePreambleDeclsDeserialized();
return TopLevelDecls;
}
const std::vector<DiagWithFixIts> &
ClangdUnit::ParsedAST::getDiagnostics() const {
const std::vector<DiagWithFixIts> &ParsedAST::getDiagnostics() const {
return Diags;
}
ClangdUnit::ParsedAST::ParsedAST(
std::unique_ptr<CompilerInstance> Clang,
ParsedAST::ParsedAST(std::unique_ptr<CompilerInstance> Clang,
std::unique_ptr<FrontendAction> Action,
std::vector<const Decl *> TopLevelDecls,
std::vector<serialization::DeclID> PendingTopLevelDecls,
@ -768,9 +678,274 @@ ClangdUnit::ParsedAST::ParsedAST(
assert(this->Action);
}
ClangdUnit::PreambleData::PreambleData(
PrecompiledPreamble Preamble,
ParsedASTWrapper::ParsedASTWrapper(ParsedASTWrapper &&Wrapper)
: AST(std::move(Wrapper.AST)) {}
ParsedASTWrapper::ParsedASTWrapper(llvm::Optional<ParsedAST> AST)
: AST(std::move(AST)) {}
PreambleData::PreambleData(PrecompiledPreamble Preamble,
std::vector<serialization::DeclID> TopLevelDeclIDs,
std::vector<DiagWithFixIts> Diags)
: Preamble(std::move(Preamble)),
TopLevelDeclIDs(std::move(TopLevelDeclIDs)), Diags(std::move(Diags)) {}
std::shared_ptr<CppFile>
CppFile::Create(PathRef FileName, tooling::CompileCommand Command,
std::shared_ptr<PCHContainerOperations> PCHs) {
return std::shared_ptr<CppFile>(
new CppFile(FileName, std::move(Command), std::move(PCHs)));
}
CppFile::CppFile(PathRef FileName, tooling::CompileCommand Command,
std::shared_ptr<PCHContainerOperations> PCHs)
: FileName(FileName), Command(std::move(Command)), RebuildCounter(0),
RebuildInProgress(false), PCHs(std::move(PCHs)) {
std::lock_guard<std::mutex> Lock(Mutex);
LatestAvailablePreamble = nullptr;
PreamblePromise.set_value(nullptr);
PreambleFuture = PreamblePromise.get_future();
ASTPromise.set_value(ParsedASTWrapper(llvm::None));
ASTFuture = ASTPromise.get_future();
}
void CppFile::cancelRebuilds() {
std::unique_lock<std::mutex> Lock(Mutex);
// Cancel an ongoing rebuild, if any, and wait for it to finish.
++this->RebuildCounter;
// Rebuild asserts that futures aren't ready if rebuild is cancelled.
// We want to keep this invariant.
if (futureIsReady(PreambleFuture)) {
PreamblePromise = std::promise<std::shared_ptr<const PreambleData>>();
PreambleFuture = PreamblePromise.get_future();
}
if (futureIsReady(ASTFuture)) {
ASTPromise = std::promise<ParsedASTWrapper>();
ASTFuture = ASTPromise.get_future();
}
// Now wait for rebuild to finish.
RebuildCond.wait(Lock, [this]() { return !this->RebuildInProgress; });
// Return empty results for futures.
PreamblePromise.set_value(nullptr);
ASTPromise.set_value(ParsedASTWrapper(llvm::None));
}
llvm::Optional<std::vector<DiagWithFixIts>>
CppFile::rebuild(StringRef NewContents,
IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
return deferRebuild(NewContents, std::move(VFS)).get();
}
std::future<llvm::Optional<std::vector<DiagWithFixIts>>>
CppFile::deferRebuild(StringRef NewContents,
IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
std::shared_ptr<const PreambleData> OldPreamble;
std::shared_ptr<PCHContainerOperations> PCHs;
unsigned RequestRebuildCounter;
{
std::unique_lock<std::mutex> Lock(Mutex);
// Increase RebuildCounter to cancel all ongoing FinishRebuild operations.
// They will try to exit as early as possible and won't call set_value on
// our promises.
RequestRebuildCounter = ++this->RebuildCounter;
PCHs = this->PCHs;
// Remember the preamble to be used during rebuild.
OldPreamble = this->LatestAvailablePreamble;
// Setup std::promises and std::futures for Preamble and AST. Corresponding
// futures will wait until the rebuild process is finished.
if (futureIsReady(this->PreambleFuture)) {
this->PreamblePromise =
std::promise<std::shared_ptr<const PreambleData>>();
this->PreambleFuture = this->PreamblePromise.get_future();
}
if (futureIsReady(this->ASTFuture)) {
this->ASTPromise = std::promise<ParsedASTWrapper>();
this->ASTFuture = this->ASTPromise.get_future();
}
} // unlock Mutex.
// A helper to function to finish the rebuild. May be run on a different
// thread.
// Don't let this CppFile die before rebuild is finished.
std::shared_ptr<CppFile> That = shared_from_this();
auto FinishRebuild = [OldPreamble, VFS, RequestRebuildCounter, PCHs,
That](std::string NewContents)
-> llvm::Optional<std::vector<DiagWithFixIts>> {
// Only one execution of this method is possible at a time.
// RebuildGuard will wait for any ongoing rebuilds to finish and will put us
// into a state for doing a rebuild.
RebuildGuard Rebuild(*That, RequestRebuildCounter);
if (Rebuild.wasCancelledBeforeConstruction())
return llvm::None;
std::vector<const char *> ArgStrs;
for (const auto &S : That->Command.CommandLine)
ArgStrs.push_back(S.c_str());
VFS->setCurrentWorkingDirectory(That->Command.Directory);
std::unique_ptr<CompilerInvocation> CI;
{
// FIXME(ibiryukov): store diagnostics from CommandLine when we start
// reporting them.
EmptyDiagsConsumer CommandLineDiagsConsumer;
IntrusiveRefCntPtr<DiagnosticsEngine> CommandLineDiagsEngine =
CompilerInstance::createDiagnostics(new DiagnosticOptions,
&CommandLineDiagsConsumer, false);
CI = createCompilerInvocation(ArgStrs, CommandLineDiagsEngine, VFS);
}
assert(CI && "Couldn't create CompilerInvocation");
std::unique_ptr<llvm::MemoryBuffer> ContentsBuffer =
llvm::MemoryBuffer::getMemBufferCopy(NewContents, That->FileName);
// A helper function to rebuild the preamble or reuse the existing one. Does
// not mutate any fields, only does the actual computation.
auto DoRebuildPreamble = [&]() -> std::shared_ptr<const PreambleData> {
auto Bounds =
ComputePreambleBounds(*CI->getLangOpts(), ContentsBuffer.get(), 0);
if (OldPreamble && OldPreamble->Preamble.CanReuse(
*CI, ContentsBuffer.get(), Bounds, VFS.get())) {
return OldPreamble;
}
std::vector<DiagWithFixIts> PreambleDiags;
StoreDiagsConsumer PreambleDiagnosticsConsumer(/*ref*/ PreambleDiags);
IntrusiveRefCntPtr<DiagnosticsEngine> PreambleDiagsEngine =
CompilerInstance::createDiagnostics(
&CI->getDiagnosticOpts(), &PreambleDiagnosticsConsumer, false);
CppFilePreambleCallbacks SerializedDeclsCollector;
auto BuiltPreamble = PrecompiledPreamble::Build(
*CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine, VFS, PCHs,
SerializedDeclsCollector);
if (BuiltPreamble) {
return std::make_shared<PreambleData>(
std::move(*BuiltPreamble),
SerializedDeclsCollector.takeTopLevelDeclIDs(),
std::move(PreambleDiags));
} else {
return nullptr;
}
};
// Compute updated Preamble.
std::shared_ptr<const PreambleData> NewPreamble = DoRebuildPreamble();
// Publish the new Preamble.
{
std::lock_guard<std::mutex> Lock(That->Mutex);
// We always set LatestAvailablePreamble to the new value, hoping that it
// will still be usable in the further requests.
That->LatestAvailablePreamble = NewPreamble;
if (RequestRebuildCounter != That->RebuildCounter)
return llvm::None; // Our rebuild request was cancelled, do nothing.
That->PreamblePromise.set_value(NewPreamble);
} // unlock Mutex
// Prepare the Preamble and supplementary data for rebuilding AST.
const PrecompiledPreamble *PreambleForAST = nullptr;
ArrayRef<serialization::DeclID> SerializedPreambleDecls = llvm::None;
std::vector<DiagWithFixIts> Diagnostics;
if (NewPreamble) {
PreambleForAST = &NewPreamble->Preamble;
SerializedPreambleDecls = NewPreamble->TopLevelDeclIDs;
Diagnostics.insert(Diagnostics.begin(), NewPreamble->Diags.begin(),
NewPreamble->Diags.end());
}
// Compute updated AST.
llvm::Optional<ParsedAST> NewAST =
ParsedAST::Build(std::move(CI), PreambleForAST, SerializedPreambleDecls,
std::move(ContentsBuffer), PCHs, VFS);
if (NewAST) {
Diagnostics.insert(Diagnostics.end(), NewAST->getDiagnostics().begin(),
NewAST->getDiagnostics().end());
} else {
// Don't report even Preamble diagnostics if we coulnd't build AST.
Diagnostics.clear();
}
// Publish the new AST.
{
std::lock_guard<std::mutex> Lock(That->Mutex);
if (RequestRebuildCounter != That->RebuildCounter)
return Diagnostics; // Our rebuild request was cancelled, don't set
// ASTPromise.
That->ASTPromise.set_value(ParsedASTWrapper(std::move(NewAST)));
} // unlock Mutex
return Diagnostics;
};
return std::async(std::launch::deferred, FinishRebuild, NewContents.str());
}
std::shared_future<std::shared_ptr<const PreambleData>>
CppFile::getPreamble() const {
std::lock_guard<std::mutex> Lock(Mutex);
return PreambleFuture;
}
std::shared_ptr<const PreambleData> CppFile::getPossiblyStalePreamble() const {
std::lock_guard<std::mutex> Lock(Mutex);
return LatestAvailablePreamble;
}
std::shared_future<ParsedASTWrapper> CppFile::getAST() const {
std::lock_guard<std::mutex> Lock(Mutex);
return ASTFuture;
}
tooling::CompileCommand const &CppFile::getCompileCommand() const {
return Command;
}
CppFile::RebuildGuard::RebuildGuard(CppFile &File,
unsigned RequestRebuildCounter)
: File(File), RequestRebuildCounter(RequestRebuildCounter) {
std::unique_lock<std::mutex> Lock(File.Mutex);
WasCancelledBeforeConstruction = File.RebuildCounter != RequestRebuildCounter;
if (WasCancelledBeforeConstruction)
return;
File.RebuildCond.wait(Lock, [&File]() { return !File.RebuildInProgress; });
WasCancelledBeforeConstruction = File.RebuildCounter != RequestRebuildCounter;
if (WasCancelledBeforeConstruction)
return;
File.RebuildInProgress = true;
}
bool CppFile::RebuildGuard::wasCancelledBeforeConstruction() const {
return WasCancelledBeforeConstruction;
}
CppFile::RebuildGuard::~RebuildGuard() {
if (WasCancelledBeforeConstruction)
return;
std::unique_lock<std::mutex> Lock(File.Mutex);
assert(File.RebuildInProgress);
File.RebuildInProgress = false;
if (File.RebuildCounter == RequestRebuildCounter) {
// Our rebuild request was successful.
assert(futureIsReady(File.ASTFuture));
assert(futureIsReady(File.PreambleFuture));
} else {
// Our rebuild request was cancelled, because further reparse was requested.
assert(!futureIsReady(File.ASTFuture));
assert(!futureIsReady(File.PreambleFuture));
}
Lock.unlock();
File.RebuildCond.notify_all();
}

View File

@ -17,7 +17,10 @@
#include "clang/Serialization/ASTBitCodes.h"
#include "clang/Tooling/CompilationDatabase.h"
#include "clang/Tooling/Core/Replacement.h"
#include <atomic>
#include <future>
#include <memory>
#include <mutex>
namespace llvm {
class raw_ostream;
@ -43,36 +46,6 @@ struct DiagWithFixIts {
llvm::SmallVector<tooling::Replacement, 1> FixIts;
};
/// Stores parsed C++ AST and provides implementations of all operations clangd
/// would want to perform on parsed C++ files.
class ClangdUnit {
public:
ClangdUnit(PathRef FileName, StringRef Contents, StringRef ResourceDir,
std::shared_ptr<PCHContainerOperations> PCHs,
std::vector<tooling::CompileCommand> Commands,
IntrusiveRefCntPtr<vfs::FileSystem> VFS);
/// Reparse with new contents.
void reparse(StringRef Contents, IntrusiveRefCntPtr<vfs::FileSystem> VFS);
/// Get code completions at a specified \p Line and \p Column in \p File.
///
/// This function is thread-safe and returns completion items that own the
/// data they contain.
std::vector<CompletionItem>
codeComplete(StringRef Contents, Position Pos,
IntrusiveRefCntPtr<vfs::FileSystem> VFS);
/// Get definition of symbol at a specified \p Line and \p Column in \p File.
std::vector<Location> findDefinitions(Position Pos);
/// Returns diagnostics and corresponding FixIts for each diagnostic that are
/// located in the current file.
std::vector<DiagWithFixIts> getLocalDiagnostics() const;
/// For testing/debugging purposes. Note that this method deserializes all
/// unserialized Decls, so use with care.
void dumpAST(llvm::raw_ostream &OS) const;
private:
/// Stores and provides access to parsed AST.
class ParsedAST {
public:
@ -128,7 +101,28 @@ private:
std::vector<serialization::DeclID> PendingTopLevelDecls;
};
// Store Preamble and all associated data
// Provides thread-safe access to ParsedAST.
class ParsedASTWrapper {
public:
ParsedASTWrapper(ParsedASTWrapper &&Wrapper);
ParsedASTWrapper(llvm::Optional<ParsedAST> AST);
/// Runs \p F on wrapped ParsedAST under lock. Ensures it is not accessed
/// concurrently.
template <class Func> void runUnderLock(Func F) const {
std::lock_guard<std::mutex> Lock(Mutex);
F(AST ? AST.getPointer() : nullptr);
}
private:
// This wrapper is used as an argument to std::shared_future (and it returns a
// const ref in get()), but we need to have non-const ref in order to
// implement some features.
mutable std::mutex Mutex;
mutable llvm::Optional<ParsedAST> AST;
};
// Stores Preamble and associated data.
struct PreambleData {
PreambleData(PrecompiledPreamble Preamble,
std::vector<serialization::DeclID> TopLevelDeclIDs,
@ -139,18 +133,122 @@ private:
std::vector<DiagWithFixIts> Diags;
};
SourceLocation getBeginningOfIdentifier(const Position &Pos,
const FileEntry *FE) const;
/// Manages resources, required by clangd. Allows to rebuild file with new
/// contents, and provides AST and Preamble for it.
class CppFile : public std::enable_shared_from_this<CppFile> {
public:
// We only allow to create CppFile as shared_ptr, because a future returned by
// deferRebuild will hold references to it.
static std::shared_ptr<CppFile>
Create(PathRef FileName, tooling::CompileCommand Command,
std::shared_ptr<PCHContainerOperations> PCHs);
private:
CppFile(PathRef FileName, tooling::CompileCommand Command,
std::shared_ptr<PCHContainerOperations> PCHs);
public:
CppFile(CppFile const &) = delete;
CppFile(CppFile &&) = delete;
/// Cancels all scheduled rebuilds and sets AST and Preamble to nulls.
/// If a rebuild is in progress, will wait for it to finish.
void cancelRebuilds();
/// Rebuild AST and Preamble synchronously on the calling thread.
/// Returns a list of diagnostics or a llvm::None, if another rebuild was
/// requested in parallel (effectively cancelling this rebuild) before
/// diagnostics were produced.
llvm::Optional<std::vector<DiagWithFixIts>>
rebuild(StringRef NewContents, IntrusiveRefCntPtr<vfs::FileSystem> VFS);
/// Schedule a rebuild and return a deferred computation that will finish the
/// rebuild, that can be called on a different thread.
/// After calling this method, resources, available via futures returned by
/// getPreamble() and getAST(), will be waiting for rebuild to finish. A
/// future fininshing rebuild, returned by this function, must be
/// computed(i.e. get() should be called on it) in order to make those
/// resources ready. If deferRebuild is called again before the rebuild is
/// finished (either because returned future had not been called or because it
/// had not returned yet), the previous rebuild request is cancelled and the
/// resource futures (returned by getPreamble() or getAST()) that were not
/// ready will be waiting for the last rebuild to finish instead.
/// The future to finish rebuild returns a list of diagnostics built during
/// reparse, or None, if another deferRebuild was called before this
/// rebuild was finished.
std::future<llvm::Optional<std::vector<DiagWithFixIts>>>
deferRebuild(StringRef NewContents, IntrusiveRefCntPtr<vfs::FileSystem> VFS);
/// Returns a future to get the most fresh PreambleData for a file. The
/// future will wait until the Preamble is rebuilt.
std::shared_future<std::shared_ptr<const PreambleData>> getPreamble() const;
/// Return some preamble for a file. It might be stale, but won't wait for
/// rebuild to finish.
std::shared_ptr<const PreambleData> getPossiblyStalePreamble() const;
/// Returns a future to get the most fresh AST for a file. Returned AST is
/// wrapped to prevent concurrent accesses.
std::shared_future<ParsedASTWrapper> getAST() const;
/// Get CompileCommand used to build this CppFile.
tooling::CompileCommand const &getCompileCommand() const;
private:
/// A helper guard that manages the state of CppFile during rebuild.
class RebuildGuard {
public:
RebuildGuard(CppFile &File, unsigned RequestRebuildCounter);
~RebuildGuard();
bool wasCancelledBeforeConstruction() const;
private:
CppFile &File;
unsigned RequestRebuildCounter;
bool WasCancelledBeforeConstruction;
};
Path FileName;
tooling::CompileCommand Command;
llvm::Optional<PreambleData> Preamble;
llvm::Optional<ParsedAST> Unit;
/// Mutex protects all fields, declared below it, FileName and Command are not
/// mutated.
mutable std::mutex Mutex;
/// A counter to cancel old rebuilds.
unsigned RebuildCounter;
/// Used to wait when rebuild is finished before starting another one.
bool RebuildInProgress;
/// Condition variable to indicate changes to RebuildInProgress.
std::condition_variable RebuildCond;
/// Promise and future for the latests AST. Fulfilled during rebuild.
std::promise<ParsedASTWrapper> ASTPromise;
std::shared_future<ParsedASTWrapper> ASTFuture;
/// Promise and future for the latests Preamble. Fulfilled during rebuild.
std::promise<std::shared_ptr<const PreambleData>> PreamblePromise;
std::shared_future<std::shared_ptr<const PreambleData>> PreambleFuture;
/// Latest preamble that was built. May be stale, but always available without
/// waiting for rebuild to finish.
std::shared_ptr<const PreambleData> LatestAvailablePreamble;
/// Utility class, required by clang.
std::shared_ptr<PCHContainerOperations> PCHs;
};
/// Get code completions at a specified \p Pos in \p FileName.
std::vector<CompletionItem>
codeComplete(PathRef FileName, tooling::CompileCommand Command,
PrecompiledPreamble const *Preamble, StringRef Contents,
Position Pos, IntrusiveRefCntPtr<vfs::FileSystem> VFS,
std::shared_ptr<PCHContainerOperations> PCHs);
/// Get definition of symbol at a specified \p Pos.
std::vector<Location> findDefinitions(ParsedAST &AST, Position Pos);
/// For testing/debugging purposes. Note that this method deserializes all
/// unserialized Decls, so use with care.
void dumpAST(ParsedAST &AST, llvm::raw_ostream &OS);
} // namespace clangd
} // namespace clang
#endif

View File

@ -13,21 +13,29 @@
using namespace clang::clangd;
using namespace clang;
void ClangdUnitStore::removeUnitIfPresent(PathRef File) {
std::shared_ptr<CppFile> CppFileCollection::removeIfPresent(PathRef File) {
std::lock_guard<std::mutex> Lock(Mutex);
auto It = OpenedFiles.find(File);
if (It == OpenedFiles.end())
return;
return nullptr;
std::shared_ptr<CppFile> Result = It->second;
OpenedFiles.erase(It);
return Result;
}
std::vector<tooling::CompileCommand>
ClangdUnitStore::getCompileCommands(GlobalCompilationDatabase &CDB,
PathRef File) {
tooling::CompileCommand
CppFileCollection::getCompileCommand(GlobalCompilationDatabase &CDB,
PathRef File, PathRef ResourceDir) {
std::vector<tooling::CompileCommand> Commands = CDB.getCompileCommands(File);
if (Commands.empty())
// Add a fake command line if we know nothing.
Commands.push_back(getDefaultCompileCommand(File));
return Commands;
// Inject the resource dir.
// FIXME: Don't overwrite it if it's already there.
Commands.front().CommandLine.push_back("-resource-dir=" +
std::string(ResourceDir));
return std::move(Commands.front());
}

View File

@ -1,4 +1,4 @@
//===--- ClangdUnitStore.h - A ClangdUnits container -------------*-C++-*-===//
//===--- ClangdUnitStore.h - A container of CppFiles -------------*-C++-*-===//
//
// The LLVM Compiler Infrastructure
//
@ -20,87 +20,47 @@
namespace clang {
namespace clangd {
/// Thread-safe collection of ASTs built for specific files. Provides
/// synchronized access to ASTs.
class ClangdUnitStore {
/// Thread-safe mapping from FileNames to CppFile.
class CppFileCollection {
public:
/// Run specified \p Action on the ClangdUnit for \p File.
/// If the file is not present in ClangdUnitStore, a new ClangdUnit will be
/// created from the \p FileContents. If the file is already present in the
/// store, ClangdUnit::reparse will be called with the new contents before
/// running \p Action.
template <class Func>
void runOnUnit(PathRef File, StringRef FileContents, StringRef ResourceDir,
std::shared_ptr<CppFile>
getOrCreateFile(PathRef File, PathRef ResourceDir,
GlobalCompilationDatabase &CDB,
std::shared_ptr<PCHContainerOperations> PCHs,
IntrusiveRefCntPtr<vfs::FileSystem> VFS, Func Action) {
runOnUnitImpl(File, FileContents, ResourceDir, CDB, PCHs,
/*ReparseBeforeAction=*/true, VFS,
std::forward<Func>(Action));
}
/// Run specified \p Action on the ClangdUnit for \p File.
/// If the file is not present in ClangdUnitStore, a new ClangdUnit will be
/// created from the \p FileContents. If the file is already present in the
/// store, the \p Action will be run directly on it.
template <class Func>
void runOnUnitWithoutReparse(PathRef File, StringRef FileContents,
StringRef ResourceDir,
GlobalCompilationDatabase &CDB,
std::shared_ptr<PCHContainerOperations> PCHs,
IntrusiveRefCntPtr<vfs::FileSystem> VFS,
Func Action) {
runOnUnitImpl(File, FileContents, ResourceDir, CDB, PCHs,
/*ReparseBeforeAction=*/false, VFS,
std::forward<Func>(Action));
}
/// Run the specified \p Action on the ClangdUnit for \p File.
/// Unit for \p File should exist in the store.
template <class Func> void runOnExistingUnit(PathRef File, Func Action) {
std::lock_guard<std::mutex> Lock(Mutex);
auto It = OpenedFiles.find(File);
assert(It != OpenedFiles.end() && "File is not in OpenedFiles");
Action(It->second);
}
/// Remove ClangdUnit for \p File, if any
void removeUnitIfPresent(PathRef File);
private:
/// Run specified \p Action on the ClangdUnit for \p File.
template <class Func>
void runOnUnitImpl(PathRef File, StringRef FileContents,
StringRef ResourceDir, GlobalCompilationDatabase &CDB,
std::shared_ptr<PCHContainerOperations> PCHs,
bool ReparseBeforeAction,
IntrusiveRefCntPtr<vfs::FileSystem> VFS, Func Action) {
IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
std::lock_guard<std::mutex> Lock(Mutex);
auto It = OpenedFiles.find(File);
if (It == OpenedFiles.end()) {
auto Commands = getCompileCommands(CDB, File);
assert(!Commands.empty() &&
"getCompileCommands should add default command");
auto Command = getCompileCommand(CDB, File, ResourceDir);
It = OpenedFiles
.insert(std::make_pair(File, ClangdUnit(File, FileContents,
ResourceDir, PCHs,
Commands, VFS)))
.try_emplace(File, CppFile::Create(File, std::move(Command),
std::move(PCHs)))
.first;
} else if (ReparseBeforeAction) {
It->second.reparse(FileContents, VFS);
}
return Action(It->second);
return It->second;
}
std::vector<tooling::CompileCommand>
getCompileCommands(GlobalCompilationDatabase &CDB, PathRef File);
std::shared_ptr<CppFile> getFile(PathRef File) {
std::lock_guard<std::mutex> Lock(Mutex);
auto It = OpenedFiles.find(File);
if (It == OpenedFiles.end())
return nullptr;
return It->second;
}
/// Removes a CppFile, stored for \p File, if it's inside collection and
/// returns it.
std::shared_ptr<CppFile> removeIfPresent(PathRef File);
private:
tooling::CompileCommand getCompileCommand(GlobalCompilationDatabase &CDB,
PathRef File, PathRef ResourceDir);
std::mutex Mutex;
llvm::StringMap<ClangdUnit> OpenedFiles;
llvm::StringMap<std::shared_ptr<CppFile>> OpenedFiles;
};
} // namespace clangd
} // namespace clang

View File

@ -28,7 +28,7 @@ namespace clangd {
/// Returns a default compile command to use for \p File.
tooling::CompileCommand getDefaultCompileCommand(PathRef File);
/// Provides compilation arguments used for building ClangdUnit.
/// Provides compilation arguments used for parsing C and C++ files.
class GlobalCompilationDatabase {
public:
virtual ~GlobalCompilationDatabase() = default;

View File

@ -17,6 +17,7 @@
#include "llvm/Support/Regex.h"
#include "gtest/gtest.h"
#include <algorithm>
#include <chrono>
#include <iostream>
#include <string>
#include <vector>
@ -130,9 +131,15 @@ IntrusiveRefCntPtr<vfs::FileSystem> getTempOnlyFS() {
namespace clangd {
namespace {
// Don't wait for async ops in clangd test more than that to avoid blocking
// indefinitely in case of bugs.
static const std::chrono::seconds DefaultFutureTimeout =
std::chrono::seconds(10);
class ErrorCheckingDiagConsumer : public DiagnosticsConsumer {
public:
void onDiagnosticsReady(PathRef File,
void
onDiagnosticsReady(PathRef File,
Tagged<std::vector<DiagWithFixIts>> Diagnostics) override {
bool HadError = false;
for (const auto &DiagAndFixIts : Diagnostics.Value) {
@ -152,9 +159,7 @@ public:
return HadErrorInLastDiags;
}
VFSTag lastVFSTag() {
return LastVFSTag;
}
VFSTag lastVFSTag() { return LastVFSTag; }
private:
std::mutex Mutex;
@ -276,9 +281,15 @@ protected:
auto SourceFilename = getVirtualTestFilePath(SourceFileRelPath);
FS.ExpectedFile = SourceFilename;
Server.addDocument(SourceFilename, SourceContents);
// Have to sync reparses because RunSynchronously is false.
auto AddDocFuture = Server.addDocument(SourceFilename, SourceContents);
auto Result = dumpASTWithoutMemoryLocs(Server, SourceFilename);
// Wait for reparse to finish before checking for errors.
EXPECT_EQ(AddDocFuture.wait_for(DefaultFutureTimeout),
std::future_status::ready);
EXPECT_EQ(ExpectErrors, DiagConsumer.hadErrorInLastDiags());
return Result;
}
@ -338,16 +349,25 @@ int b = a;
FS.Files[FooCpp] = SourceContents;
FS.ExpectedFile = FooCpp;
Server.addDocument(FooCpp, SourceContents);
// To sync reparses before checking for errors.
std::future<void> ParseFuture;
ParseFuture = Server.addDocument(FooCpp, SourceContents);
auto DumpParse1 = dumpASTWithoutMemoryLocs(Server, FooCpp);
ASSERT_EQ(ParseFuture.wait_for(DefaultFutureTimeout),
std::future_status::ready);
EXPECT_FALSE(DiagConsumer.hadErrorInLastDiags());
Server.addDocument(FooCpp, "");
ParseFuture = Server.addDocument(FooCpp, "");
auto DumpParseEmpty = dumpASTWithoutMemoryLocs(Server, FooCpp);
ASSERT_EQ(ParseFuture.wait_for(DefaultFutureTimeout),
std::future_status::ready);
EXPECT_FALSE(DiagConsumer.hadErrorInLastDiags());
Server.addDocument(FooCpp, SourceContents);
ParseFuture = Server.addDocument(FooCpp, SourceContents);
auto DumpParse2 = dumpASTWithoutMemoryLocs(Server, FooCpp);
ASSERT_EQ(ParseFuture.wait_for(DefaultFutureTimeout),
std::future_status::ready);
EXPECT_FALSE(DiagConsumer.hadErrorInLastDiags());
EXPECT_EQ(DumpParse1, DumpParse2);
@ -374,18 +394,27 @@ int b = a;
FS.Files[FooCpp] = SourceContents;
FS.ExpectedFile = FooCpp;
Server.addDocument(FooCpp, SourceContents);
// To sync reparses before checking for errors.
std::future<void> ParseFuture;
ParseFuture = Server.addDocument(FooCpp, SourceContents);
auto DumpParse1 = dumpASTWithoutMemoryLocs(Server, FooCpp);
ASSERT_EQ(ParseFuture.wait_for(DefaultFutureTimeout),
std::future_status::ready);
EXPECT_FALSE(DiagConsumer.hadErrorInLastDiags());
FS.Files[FooH] = "";
Server.forceReparse(FooCpp);
ParseFuture = Server.forceReparse(FooCpp);
auto DumpParseDifferent = dumpASTWithoutMemoryLocs(Server, FooCpp);
ASSERT_EQ(ParseFuture.wait_for(DefaultFutureTimeout),
std::future_status::ready);
EXPECT_TRUE(DiagConsumer.hadErrorInLastDiags());
FS.Files[FooH] = "int a;";
Server.forceReparse(FooCpp);
ParseFuture = Server.forceReparse(FooCpp);
auto DumpParse2 = dumpASTWithoutMemoryLocs(Server, FooCpp);
EXPECT_EQ(ParseFuture.wait_for(DefaultFutureTimeout),
std::future_status::ready);
EXPECT_FALSE(DiagConsumer.hadErrorInLastDiags());
EXPECT_EQ(DumpParse1, DumpParse2);
@ -404,10 +433,12 @@ TEST_F(ClangdVFSTest, CheckVersions) {
FS.Files[FooCpp] = SourceContents;
FS.ExpectedFile = FooCpp;
// No need to sync reparses, because RunSynchronously is set
// to true.
FS.Tag = "123";
Server.addDocument(FooCpp, SourceContents);
EXPECT_EQ(DiagConsumer.lastVFSTag(), FS.Tag);
EXPECT_EQ(Server.codeComplete(FooCpp, Position{0, 0}).Tag, FS.Tag);
EXPECT_EQ(DiagConsumer.lastVFSTag(), FS.Tag);
FS.Tag = "321";
Server.addDocument(FooCpp, SourceContents);
@ -455,6 +486,8 @@ mock_string x;
)cpp";
FS.Files[FooCpp] = SourceContents;
// No need to sync reparses, because RunSynchronously is set
// to true.
Server.addDocument(FooCpp, SourceContents);
EXPECT_FALSE(DiagConsumer.hadErrorInLastDiags());
@ -505,6 +538,8 @@ int b = ;
FS.Files[FooCpp] = SourceContents;
FS.ExpectedFile = FooCpp;
// No need to sync reparses here as there are no asserts on diagnostics (or
// other async operations).
Server.addDocument(FooCpp, SourceContents);
{