Revert "Reland(2) "[clangd] Indexing of standard library""

This reverts commit ca875539f7.
This commit is contained in:
Sam McCall 2022-05-18 17:35:50 +02:00
parent e44fe27251
commit 77533ea443
18 changed files with 21 additions and 713 deletions

View File

@ -119,7 +119,6 @@ add_clang_library(clangDaemon
index/Ref.cpp
index/Relation.cpp
index/Serialization.cpp
index/StdLib.cpp
index/Symbol.cpp
index/SymbolCollector.cpp
index/SymbolID.cpp

View File

@ -26,7 +26,6 @@
#include "index/CanonicalIncludes.h"
#include "index/FileIndex.h"
#include "index/Merge.h"
#include "index/StdLib.h"
#include "refactor/Rename.h"
#include "refactor/Tweak.h"
#include "support/Cancellation.h"
@ -60,39 +59,16 @@ namespace {
// Update the FileIndex with new ASTs and plumb the diagnostics responses.
struct UpdateIndexCallbacks : public ParsingCallbacks {
UpdateIndexCallbacks(FileIndex *FIndex,
ClangdServer::Callbacks *ServerCallbacks,
const ThreadsafeFS &TFS, AsyncTaskRunner *Tasks)
: FIndex(FIndex), ServerCallbacks(ServerCallbacks), TFS(TFS),
Tasks(Tasks) {}
ClangdServer::Callbacks *ServerCallbacks)
: FIndex(FIndex), ServerCallbacks(ServerCallbacks) {}
void onPreambleAST(PathRef Path, llvm::StringRef Version,
const CompilerInvocation &CI, ASTContext &Ctx,
void onPreambleAST(PathRef Path, llvm::StringRef Version, ASTContext &Ctx,
Preprocessor &PP,
const CanonicalIncludes &CanonIncludes) override {
// If this preamble uses a standard library we haven't seen yet, index it.
if (FIndex)
if (auto Loc = Stdlib.add(*CI.getLangOpts(), PP.getHeaderSearchInfo()))
indexStdlib(CI, std::move(*Loc));
if (FIndex)
FIndex->updatePreamble(Path, Version, Ctx, PP, CanonIncludes);
}
void indexStdlib(const CompilerInvocation &CI, StdLibLocation Loc) {
auto Task = [this, LO(*CI.getLangOpts()), Loc(std::move(Loc)),
CI(std::make_unique<CompilerInvocation>(CI))]() mutable {
IndexFileIn IF;
IF.Symbols = indexStandardLibrary(std::move(CI), Loc, TFS);
if (Stdlib.isBest(LO))
FIndex->updatePreamble(std::move(IF));
};
if (Tasks)
// This doesn't have a semaphore to enforce -j, but it's rare.
Tasks->runAsync("IndexStdlib", std::move(Task));
else
Task();
}
void onMainAST(PathRef Path, ParsedAST &AST, PublishFn Publish) override {
if (FIndex)
FIndex->updateMain(Path, AST);
@ -127,9 +103,6 @@ struct UpdateIndexCallbacks : public ParsingCallbacks {
private:
FileIndex *FIndex;
ClangdServer::Callbacks *ServerCallbacks;
const ThreadsafeFS &TFS;
StdLibSet Stdlib;
AsyncTaskRunner *Tasks;
};
class DraftStoreFS : public ThreadsafeFS {
@ -181,15 +154,12 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB,
Transient(Opts.ImplicitCancellation ? TUScheduler::InvalidateOnUpdate
: TUScheduler::NoInvalidation),
DirtyFS(std::make_unique<DraftStoreFS>(TFS, DraftMgr)) {
if (Opts.AsyncThreadsCount != 0)
IndexTasks.emplace();
// Pass a callback into `WorkScheduler` to extract symbols from a newly
// parsed file and rebuild the file index synchronously each time an AST
// is parsed.
WorkScheduler.emplace(CDB, TUScheduler::Options(Opts),
std::make_unique<UpdateIndexCallbacks>(
DynamicIdx.get(), Callbacks, TFS,
IndexTasks ? IndexTasks.getPointer() : nullptr));
WorkScheduler.emplace(
CDB, TUScheduler::Options(Opts),
std::make_unique<UpdateIndexCallbacks>(DynamicIdx.get(), Callbacks));
// Adds an index to the stack, at higher priority than existing indexes.
auto AddIndex = [&](SymbolIndex *Idx) {
if (this->Index != nullptr) {
@ -1005,9 +975,6 @@ ClangdServer::blockUntilIdleForTest(llvm::Optional<double> TimeoutSeconds) {
// and we're blocking the main thread.
if (!WorkScheduler->blockUntilIdle(timeoutSeconds(TimeoutSeconds)))
return false;
// TUScheduler is the only thing that starts background indexing work.
if (IndexTasks && !IndexTasks->wait(timeoutSeconds(TimeoutSeconds)))
return false;
// Unfortunately we don't have strict topological order between the rest of
// the components. E.g. CDB broadcast triggers backrgound indexing.

View File

@ -428,7 +428,6 @@ private:
mutable std::mutex CachedCompletionFuzzyFindRequestMutex;
llvm::Optional<std::string> WorkspaceRoot;
llvm::Optional<AsyncTaskRunner> IndexTasks; // for stdlib indexing.
llvm::Optional<TUScheduler> WorkScheduler;
// Invalidation policy used for actions that we assume are "transient".
TUScheduler::ASTActionInvalidation Transient;

View File

@ -81,12 +81,11 @@ struct Config {
/// forward-slashes.
std::string MountPoint;
};
/// Controls index behavior.
/// Controls background-index behavior.
struct {
/// Whether this TU should be background-indexed.
/// Whether this TU should be indexed.
BackgroundPolicy Background = BackgroundPolicy::Build;
ExternalIndexSpec External;
bool StandardLibrary = false;
} Index;
enum UnusedIncludesPolicy { Strict, None };

View File

@ -332,11 +332,6 @@ struct FragmentCompiler {
}
if (F.External)
compile(std::move(**F.External), F.External->Range);
if (F.StandardLibrary)
Out.Apply.push_back(
[Val(**F.StandardLibrary)](const Params &, Config &C) {
C.Index.StandardLibrary = Val;
});
}
void compile(Fragment::IndexBlock::ExternalBlock &&External,

View File

@ -199,9 +199,6 @@ struct Fragment {
llvm::Optional<Located<std::string>> MountPoint;
};
llvm::Optional<Located<ExternalBlock>> External;
// Whether the standard library visible from this file should be indexed.
// This makes all standard library symbols available, included or not.
llvm::Optional<Located<bool>> StandardLibrary;
};
IndexBlock Index;

View File

@ -184,10 +184,6 @@ private:
F.External.emplace(std::move(External));
F.External->Range = N.getSourceRange();
});
Dict.handle("StandardLibrary", [&](Node &N) {
if (auto StandardLibrary = boolValue(N, "StandardLibrary"))
F.StandardLibrary = *StandardLibrary;
});
Dict.parse(N);
}

View File

@ -1013,10 +1013,9 @@ void PreambleThread::build(Request Req) {
bool IsFirstPreamble = !LatestBuild;
LatestBuild = clang::clangd::buildPreamble(
FileName, *Req.CI, Inputs, StoreInMemory,
[&](ASTContext &Ctx, Preprocessor &PP,
const CanonicalIncludes &CanonIncludes) {
Callbacks.onPreambleAST(FileName, Inputs.Version, *Req.CI, Ctx, PP,
CanonIncludes);
[this, Version(Inputs.Version)](ASTContext &Ctx, Preprocessor &PP,
const CanonicalIncludes &CanonIncludes) {
Callbacks.onPreambleAST(FileName, Version, Ctx, PP, CanonIncludes);
},
&Stats);
if (!LatestBuild)

View File

@ -133,8 +133,8 @@ public:
/// contains only AST nodes from the #include directives at the start of the
/// file. AST node in the current file should be observed on onMainAST call.
virtual void onPreambleAST(PathRef Path, llvm::StringRef Version,
const CompilerInvocation &CI, ASTContext &Ctx,
Preprocessor &PP, const CanonicalIncludes &) {}
ASTContext &Ctx, Preprocessor &PP,
const CanonicalIncludes &) {}
/// The argument function is run under the critical section guarding against
/// races when closing the files.

View File

@ -425,7 +425,12 @@ FileIndex::FileIndex()
MainFileSymbols(IndexContents::All),
MainFileIndex(std::make_unique<MemIndex>()) {}
void FileIndex::updatePreamble(IndexFileIn IF) {
void FileIndex::updatePreamble(PathRef Path, llvm::StringRef Version,
ASTContext &AST, Preprocessor &PP,
const CanonicalIncludes &Includes) {
IndexFileIn IF;
std::tie(IF.Symbols, std::ignore, IF.Relations) =
indexHeaderSymbols(Version, AST, PP, Includes);
FileShardedIndex ShardedIndex(std::move(IF));
for (auto Uri : ShardedIndex.getAllSources()) {
auto IF = ShardedIndex.getShard(Uri);
@ -456,15 +461,6 @@ void FileIndex::updatePreamble(IndexFileIn IF) {
}
}
void FileIndex::updatePreamble(PathRef Path, llvm::StringRef Version,
ASTContext &AST, Preprocessor &PP,
const CanonicalIncludes &Includes) {
IndexFileIn IF;
std::tie(IF.Symbols, std::ignore, IF.Relations) =
indexHeaderSymbols(Version, AST, PP, Includes);
updatePreamble(std::move(IF));
}
void FileIndex::updateMain(PathRef Path, ParsedAST &AST) {
auto Contents = indexMainDecls(AST);
MainFileSymbols.update(

View File

@ -114,7 +114,6 @@ public:
/// and macros in \p PP.
void updatePreamble(PathRef Path, llvm::StringRef Version, ASTContext &AST,
Preprocessor &PP, const CanonicalIncludes &Includes);
void updatePreamble(IndexFileIn);
/// Update symbols and references from main file \p Path with
/// `indexMainDecls`.

View File

@ -1,363 +0,0 @@
//===-- StdLib.cpp ----------------------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "StdLib.h"
#include <fstream>
#include <memory>
#include <string>
#include <vector>
#include "Compiler.h"
#include "Config.h"
#include "SymbolCollector.h"
#include "index/IndexAction.h"
#include "support/Logger.h"
#include "support/ThreadsafeFS.h"
#include "support/Trace.h"
#include "clang/Basic/LangOptions.h"
#include "clang/Frontend/CompilerInvocation.h"
#include "clang/Lex/PreprocessorOptions.h"
#include "llvm/ADT/IntrusiveRefCntPtr.h"
#include "llvm/ADT/None.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
namespace clang {
namespace clangd {
namespace {
enum Lang { C, CXX };
Lang langFromOpts(const LangOptions &LO) { return LO.CPlusPlus ? CXX : C; }
llvm::StringLiteral mandatoryHeader(Lang L) {
switch (L) {
case C:
return "stdio.h";
case CXX:
return "vector";
}
llvm_unreachable("unhandled Lang");
}
LangStandard::Kind standardFromOpts(const LangOptions &LO) {
if (LO.CPlusPlus) {
if (LO.CPlusPlus2b)
return LangStandard::lang_cxx2b;
if (LO.CPlusPlus20)
return LangStandard::lang_cxx20;
if (LO.CPlusPlus17)
return LangStandard::lang_cxx17;
if (LO.CPlusPlus14)
return LangStandard::lang_cxx14;
if (LO.CPlusPlus11)
return LangStandard::lang_cxx11;
return LangStandard::lang_cxx98;
}
if (LO.C2x)
return LangStandard::lang_c2x;
// C17 has no new features, so treat {C11,C17} as C17.
if (LO.C11)
return LangStandard::lang_c17;
return LangStandard::lang_c99;
}
std::string buildUmbrella(llvm::StringLiteral Mandatory,
std::vector<llvm::StringLiteral> Headers) {
std::string Result;
llvm::raw_string_ostream OS(Result);
// We __has_include guard all our #includes to avoid errors when using older
// stdlib version that don't have headers for the newest language standards.
// But make sure we get *some* error if things are totally broken.
OS << llvm::formatv(
"#if !__has_include(<{0}>)\n"
"#error Mandatory header <{0}> not found in standard library!\n"
"#endif\n",
Mandatory);
llvm::sort(Headers.begin(), Headers.end());
auto Last = std::unique(Headers.begin(), Headers.end());
for (auto Header = Headers.begin(); Header != Last; ++Header) {
OS << llvm::formatv("#if __has_include({0})\n"
"#include {0}\n"
"#endif\n",
*Header);
}
OS.flush();
return Result;
}
} // namespace
llvm::StringRef getStdlibUmbrellaHeader(const LangOptions &LO) {
// The umbrella header is the same for all versions of each language.
// Headers that are unsupported in old lang versions are usually guarded by
// #if. Some headers may be not present in old stdlib versions, the umbrella
// header guards with __has_include for this purpose.
Lang L = langFromOpts(LO);
switch (L) {
case CXX:
static std::string *UmbrellaCXX =
new std::string(buildUmbrella(mandatoryHeader(L), {
#define SYMBOL(Name, NameSpace, Header) #Header,
#include "clang/Tooling/Inclusions/StdSymbolMap.inc"
#undef SYMBOL
}));
return *UmbrellaCXX;
case C:
static std::string *UmbrellaC =
new std::string(buildUmbrella(mandatoryHeader(L), {
#define SYMBOL(Name, NameSpace, Header) #Header,
#include "clang/Tooling/Inclusions/CSymbolMap.inc"
#undef SYMBOL
}));
return *UmbrellaC;
}
}
namespace {
// Including the standard library leaks unwanted transitively included symbols.
//
// We want to drop these, they're a bit tricky to identify:
// - we don't want to limit to symbols on our list, as our list has only
// top-level symbols (and there may be legitimate stdlib extensions).
// - we can't limit to only symbols defined in known stdlib headers, as stdlib
// internal structure is murky
// - we can't strictly require symbols to come from a particular path, e.g.
// libstdc++ is mostly under /usr/include/c++/10/...
// but std::ctype_base is under /usr/include/<platform>/c++/10/...
// We require the symbol to come from a header that is *either* from
// the standard library path (as identified by the location of <vector>), or
// another header that defines a symbol from our stdlib list.
SymbolSlab filter(SymbolSlab Slab, const StdLibLocation &Loc) {
SymbolSlab::Builder Result;
static auto &StandardHeaders = *[] {
auto *Set = new llvm::DenseSet<llvm::StringRef>();
for (llvm::StringRef Header : {
#define SYMBOL(Name, NameSpace, Header) #Header,
#include "clang/Tooling/Inclusions/CSymbolMap.inc"
#include "clang/Tooling/Inclusions/StdSymbolMap.inc"
#undef SYMBOL
})
Set->insert(Header);
return Set;
}();
// Form prefixes like file:///usr/include/c++/10/
// These can be trivially prefix-compared with URIs in the indexed symbols.
llvm::SmallVector<std::string> StdLibURIPrefixes;
for (const auto &Path : Loc.Paths) {
StdLibURIPrefixes.push_back(URI::create(Path).toString());
if (StdLibURIPrefixes.back().back() != '/')
StdLibURIPrefixes.back().push_back('/');
}
// For each header URI, is it *either* prefixed by StdLibURIPrefixes *or*
// owner of a symbol whose insertable header is in StandardHeaders?
// Pointer key because strings in a SymbolSlab are interned.
llvm::DenseMap<const char *, bool> GoodHeader;
for (const Symbol &S : Slab) {
if (!S.IncludeHeaders.empty() &&
StandardHeaders.contains(S.IncludeHeaders.front().IncludeHeader)) {
GoodHeader[S.CanonicalDeclaration.FileURI] = true;
GoodHeader[S.Definition.FileURI] = true;
continue;
}
for (const char *URI :
{S.CanonicalDeclaration.FileURI, S.Definition.FileURI}) {
auto R = GoodHeader.try_emplace(URI, false);
if (R.second) {
R.first->second = llvm::any_of(
StdLibURIPrefixes,
[&, URIStr(llvm::StringRef(URI))](const std::string &Prefix) {
return URIStr.startswith(Prefix);
});
}
}
}
#ifndef NDEBUG
for (const auto &Good : GoodHeader)
if (Good.second && *Good.first)
dlog("Stdlib header: {0}", Good.first);
#endif
// Empty URIs aren't considered good. (Definition can be blank).
auto IsGoodHeader = [&](const char *C) { return *C && GoodHeader.lookup(C); };
for (const Symbol &S : Slab) {
if (!(IsGoodHeader(S.CanonicalDeclaration.FileURI) ||
IsGoodHeader(S.Definition.FileURI))) {
dlog("Ignoring wrong-header symbol {0}{1} in {2}", S.Scope, S.Name,
S.CanonicalDeclaration.FileURI);
continue;
}
Result.insert(S);
}
return std::move(Result).build();
}
} // namespace
SymbolSlab indexStandardLibrary(llvm::StringRef HeaderSources,
std::unique_ptr<CompilerInvocation> CI,
const StdLibLocation &Loc,
const ThreadsafeFS &TFS) {
if (CI->getFrontendOpts().Inputs.size() != 1 ||
!CI->getPreprocessorOpts().ImplicitPCHInclude.empty()) {
elog("Indexing standard library failed: bad CompilerInvocation");
assert(false && "indexing stdlib with a dubious CompilerInvocation!");
return SymbolSlab();
}
const FrontendInputFile &Input = CI->getFrontendOpts().Inputs.front();
trace::Span Tracer("StandardLibraryIndex");
LangStandard::Kind LangStd = standardFromOpts(*CI->getLangOpts());
log("Indexing {0} standard library in the context of {1}",
LangStandard::getLangStandardForKind(LangStd).getName(), Input.getFile());
SymbolSlab Symbols;
IgnoreDiagnostics IgnoreDiags;
// CompilerInvocation is taken from elsewhere, and may map a dirty buffer.
CI->getPreprocessorOpts().clearRemappedFiles();
auto Clang = prepareCompilerInstance(
std::move(CI), /*Preamble=*/nullptr,
llvm::MemoryBuffer::getMemBuffer(HeaderSources, Input.getFile()),
TFS.view(/*CWD=*/llvm::None), IgnoreDiags);
if (!Clang) {
elog("Standard Library Index: Couldn't build compiler instance");
return Symbols;
}
SymbolCollector::Options IndexOpts;
IndexOpts.Origin = SymbolOrigin::StdLib;
IndexOpts.CollectMainFileSymbols = false;
IndexOpts.CollectMainFileRefs = false;
IndexOpts.CollectMacro = true;
IndexOpts.StoreAllDocumentation = true;
// Sadly we can't use IndexOpts.FileFilter to restrict indexing scope.
// Files from outside the StdLibLocation may define true std symbols anyway.
// We end up "blessing" such headers, and can only do that by indexing
// everything first.
// Refs, relations, include graph in the stdlib mostly aren't useful.
auto Action = createStaticIndexingAction(
IndexOpts, [&](SymbolSlab S) { Symbols = std::move(S); }, nullptr,
nullptr, nullptr);
if (!Action->BeginSourceFile(*Clang, Input)) {
elog("Standard Library Index: BeginSourceFile() failed");
return Symbols;
}
if (llvm::Error Err = Action->Execute()) {
elog("Standard Library Index: Execute failed: {0}", std::move(Err));
return Symbols;
}
Action->EndSourceFile();
unsigned SymbolsBeforeFilter = Symbols.size();
Symbols = filter(std::move(Symbols), Loc);
bool Errors = Clang->hasDiagnostics() &&
Clang->getDiagnostics().hasUncompilableErrorOccurred();
log("Indexed {0} standard library{3}: {1} symbols, {2} filtered",
LangStandard::getLangStandardForKind(LangStd).getName(), Symbols.size(),
SymbolsBeforeFilter - Symbols.size(),
Errors ? " (incomplete due to errors)" : "");
SPAN_ATTACH(Tracer, "symbols", int(Symbols.size()));
return Symbols;
}
SymbolSlab indexStandardLibrary(std::unique_ptr<CompilerInvocation> Invocation,
const StdLibLocation &Loc,
const ThreadsafeFS &TFS) {
return indexStandardLibrary(
getStdlibUmbrellaHeader(*Invocation->getLangOpts()),
std::move(Invocation), Loc, TFS);
}
bool StdLibSet::isBest(const LangOptions &LO) const {
return standardFromOpts(LO) >=
Best[langFromOpts(LO)].load(std::memory_order_acquire);
}
llvm::Optional<StdLibLocation> StdLibSet::add(const LangOptions &LO,
const HeaderSearch &HS) {
Lang L = langFromOpts(LO);
int OldVersion = Best[L].load(std::memory_order_acquire);
int NewVersion = standardFromOpts(LO);
dlog("Index stdlib? {0}",
LangStandard::getLangStandardForKind(standardFromOpts(LO)).getName());
if (!Config::current().Index.StandardLibrary) {
dlog("No: disabled in config");
return llvm::None;
}
if (NewVersion <= OldVersion) {
dlog("No: have {0}, {1}>={2}",
LangStandard::getLangStandardForKind(
static_cast<LangStandard::Kind>(NewVersion))
.getName(),
OldVersion, NewVersion);
return llvm::None;
}
// We'd like to index a standard library here if there is one.
// Check for the existence of <vector> on the search path.
// We could cache this, but we only get here repeatedly when there's no
// stdlib, and even then only once per preamble build.
llvm::StringLiteral ProbeHeader = mandatoryHeader(L);
llvm::SmallString<256> Path; // Scratch space.
llvm::SmallVector<std::string> SearchPaths;
auto RecordHeaderPath = [&](llvm::StringRef HeaderPath) {
llvm::StringRef DirPath = llvm::sys::path::parent_path(HeaderPath);
if (!HS.getFileMgr().getVirtualFileSystem().getRealPath(DirPath, Path))
SearchPaths.emplace_back(Path);
};
for (const auto &DL :
llvm::make_range(HS.search_dir_begin(), HS.search_dir_end())) {
switch (DL.getLookupType()) {
case DirectoryLookup::LT_NormalDir: {
Path = DL.getDir()->getName();
llvm::sys::path::append(Path, ProbeHeader);
llvm::vfs::Status Stat;
if (!HS.getFileMgr().getNoncachedStatValue(Path, Stat) &&
Stat.isRegularFile())
RecordHeaderPath(Path);
break;
}
case DirectoryLookup::LT_Framework:
// stdlib can't be a framework (framework includes must have a slash)
continue;
case DirectoryLookup::LT_HeaderMap:
llvm::StringRef Target =
DL.getHeaderMap()->lookupFilename(ProbeHeader, Path);
if (!Target.empty())
RecordHeaderPath(Target);
break;
}
}
if (SearchPaths.empty())
return llvm::None;
dlog("Found standard library in {0}", llvm::join(SearchPaths, ", "));
while (!Best[L].compare_exchange_weak(OldVersion, NewVersion,
std::memory_order_acq_rel))
if (OldVersion >= NewVersion) {
dlog("No: lost the race");
return llvm::None; // Another thread won the race while we were checking.
}
dlog("Yes, index stdlib!");
return StdLibLocation{std::move(SearchPaths)};
}
} // namespace clangd
} // namespace clang

View File

@ -1,110 +0,0 @@
//===--- StdLib.h - Index the C and C++ standard library ---------*- C++-*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
// Eagerly indexing the standard library gives a much friendlier "warm start"
// with working code completion in a standalone file or small project.
//
// We act as if we saw a file which included the whole standard library:
// #include <array>
// #include <bitset>
// #include <chrono>
// ...
// We index this TU and feed the result into the dynamic index.
//
// This happens within the context of some particular open file, and we reuse
// its CompilerInvocation. Matching its include path, LangOpts etc ensures that
// we see the standard library and configuration that matches the project.
//===----------------------------------------------------------------------===//
#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_STDLIB_H
#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_STDLIB_H
#include "index/Symbol.h"
#include "support/ThreadsafeFS.h"
#include "llvm/ADT/StringRef.h"
#include <string>
namespace clang {
class CompilerInvocation;
class LangOptions;
class HeaderSearch;
namespace clangd {
// The filesystem location where a standard library was found.
//
// This is the directory containing <vector> or <stdio.h>.
// It's used to ensure we only index files that are in the standard library.
//
// The paths are canonicalized (FS "real path" with symlinks resolved).
// This allows them to be easily compared against paths the indexer returns.
struct StdLibLocation {
llvm::SmallVector<std::string> Paths;
};
// Tracks the state of standard library indexing within a particular index.
//
// In general, we don't want to index the standard library multiple times.
// In most cases, this class just acts as a flag to ensure we only do it once.
//
// However, if we first open a C++11 file, and then a C++20 file, we *do*
// want the index to be upgraded to include the extra symbols.
// Similarly, the C and C++ standard library can coexist.
class StdLibSet {
std::atomic<int> Best[2] = {{-1}, {-1}};
public:
// Determines if we should index the standard library in a configuration.
//
// This is true if:
// - standard library indexing is enabled for the file
// - the language version is higher than any previous add() for the language
// - the standard library headers exist on the search path
// Returns the location where the standard library was found.
//
// This function is threadsafe.
llvm::Optional<StdLibLocation> add(const LangOptions &, const HeaderSearch &);
// Indicates whether a built index should be used.
// It should not be used if a newer version has subsequently been added.
//
// Intended pattern is:
// if (add()) {
// symbols = indexStandardLibrary();
// if (isBest())
// index.update(symbols);
// }
//
// This is still technically racy: we could return true here, then another
// thread could add->index->update a better library before we can update.
// We'd then overwrite it with the older version.
// However, it's very unlikely: indexing takes a long time.
bool isBest(const LangOptions &) const;
};
// Index a standard library and return the discovered symbols.
//
// The compiler invocation should describe the file whose config we're reusing.
// We overwrite its virtual buffer with a lot of #include statements.
SymbolSlab indexStandardLibrary(std::unique_ptr<CompilerInvocation> Invocation,
const StdLibLocation &Loc,
const ThreadsafeFS &TFS);
// Variant that allows the umbrella header source to be specified.
// Exposed for testing.
SymbolSlab indexStandardLibrary(llvm::StringRef HeaderSources,
std::unique_ptr<CompilerInvocation> CI,
const StdLibLocation &Loc,
const ThreadsafeFS &TFS);
// Generate header containing #includes for all standard library headers.
// Exposed for testing.
llvm::StringRef getStdlibUmbrellaHeader(const LangOptions &);
} // namespace clangd
} // namespace clang
#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_STDLIB_H

View File

@ -14,7 +14,7 @@ namespace clangd {
llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, SymbolOrigin O) {
if (O == SymbolOrigin::Unknown)
return OS << "unknown";
constexpr static char Sigils[] = "AOSMIRP7BL012345";
constexpr static char Sigils[] = "AOSMIRP7B9012345";
for (unsigned I = 0; I < sizeof(Sigils); ++I)
if (static_cast<uint16_t>(O) & 1u << I)
OS << Sigils[I];

View File

@ -29,7 +29,6 @@ enum class SymbolOrigin : uint16_t {
Preamble = 1 << 6, // From the dynamic index of preambles.
// 7 reserved
Background = 1 << 8, // From the automatic project index.
StdLib = 1 << 9, // Standard library index.
};
inline SymbolOrigin operator|(SymbolOrigin A, SymbolOrigin B) {

View File

@ -81,7 +81,6 @@ add_unittest(ClangdUnitTests ClangdTests
SemanticSelectionTests.cpp
SerializationTests.cpp
SourceCodeTests.cpp
StdLibTests.cpp
SymbolCollectorTests.cpp
SymbolInfoTests.cpp
SyncAPI.cpp

View File

@ -1,162 +0,0 @@
//===-- StdLibTests.cpp -----------------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "Annotations.h"
#include "ClangdServer.h"
#include "CodeComplete.h"
#include "Compiler.h"
#include "Config.h"
#include "SyncAPI.h"
#include "TestFS.h"
#include "index/StdLib.h"
#include "clang/Basic/LangOptions.h"
#include "clang/Basic/SourceManager.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include <memory>
using namespace testing;
namespace clang {
namespace clangd {
namespace {
// Check the generated header sources contains usual standard library headers.
TEST(StdLibTests, getStdlibUmbrellaHeader) {
LangOptions LO;
LO.CPlusPlus = true;
auto CXX = getStdlibUmbrellaHeader(LO).str();
EXPECT_THAT(CXX, HasSubstr("#include <string>"));
EXPECT_THAT(CXX, HasSubstr("#include <cstdio>"));
EXPECT_THAT(CXX, Not(HasSubstr("#include <stdio.h>")));
LO.CPlusPlus = false;
auto C = getStdlibUmbrellaHeader(LO).str();
EXPECT_THAT(C, Not(HasSubstr("#include <string>")));
EXPECT_THAT(C, Not(HasSubstr("#include <cstdio>")));
EXPECT_THAT(C, HasSubstr("#include <stdio.h>"));
}
MATCHER_P(Named, Name, "") { return arg.Name == Name; }
// Build an index, and check if it contains the right symbols.
TEST(StdLibTests, indexStandardLibrary) {
MockFS FS;
FS.Files["std/foo.h"] = R"cpp(
#include <platform_stuff.h>
#if __cplusplus >= 201703L
int foo17();
#elif __cplusplus >= 201402L
int foo14();
#else
bool foo98();
#endif
)cpp";
FS.Files["nonstd/platform_stuff.h"] = "int magic = 42;";
ParseInputs OriginalInputs;
OriginalInputs.TFS = &FS;
OriginalInputs.CompileCommand.Filename = testPath("main.cc");
OriginalInputs.CompileCommand.CommandLine = {"clang++", testPath("main.cc"),
"-isystemstd/",
"-isystemnonstd/", "-std=c++14"};
OriginalInputs.CompileCommand.Directory = testRoot();
IgnoreDiagnostics Diags;
auto CI = buildCompilerInvocation(OriginalInputs, Diags);
ASSERT_TRUE(CI);
StdLibLocation Loc;
Loc.Paths.push_back(testPath("std/"));
auto Symbols =
indexStandardLibrary("#include <foo.h>", std::move(CI), Loc, FS);
EXPECT_THAT(Symbols, ElementsAre(Named("foo14")));
}
TEST(StdLibTests, StdLibSet) {
StdLibSet Set;
MockFS FS;
FS.Files["std/_"] = "";
FS.Files["libc/_"] = "";
auto Add = [&](const LangOptions &LO,
std::vector<llvm::StringRef> SearchPath) {
SourceManagerForFile SM("scratch", "");
SM.get().getFileManager().setVirtualFileSystem(FS.view(llvm::None));
HeaderSearch HS(/*HSOpts=*/nullptr, SM.get(), SM.get().getDiagnostics(), LO,
/*Target=*/nullptr);
for (auto P : SearchPath)
HS.AddSearchPath(
DirectoryLookup(
cantFail(SM.get().getFileManager().getDirectoryRef(testPath(P))),
SrcMgr::C_System, /*isFramework=*/false),
true);
return Set.add(LO, HS);
};
Config Cfg;
Cfg.Index.StandardLibrary = false;
WithContextValue Disabled(Config::Key, std::move(Cfg));
LangOptions LO;
LO.CPlusPlus = true;
EXPECT_FALSE(Add(LO, {"std"})) << "Disabled in config";
Cfg = Config();
Cfg.Index.StandardLibrary = true;
WithContextValue Enabled(Config::Key, std::move(Cfg));
EXPECT_FALSE(Add(LO, {"std"})) << "No <vector> found";
FS.Files["std/vector"] = "class vector;";
EXPECT_TRUE(Add(LO, {"std"})) << "Indexing as C++98";
EXPECT_FALSE(Add(LO, {"std"})) << "Don't reindex";
LO.CPlusPlus11 = true;
EXPECT_TRUE(Add(LO, {"std"})) << "Indexing as C++11";
LO.CPlusPlus = false;
EXPECT_FALSE(Add(LO, {"libc"})) << "No <stdio.h>";
FS.Files["libc/stdio.h"] = true;
EXPECT_TRUE(Add(LO, {"libc"})) << "Indexing as C";
}
MATCHER_P(StdlibSymbol, Name, "") {
return arg.Name == Name && arg.Includes.size() == 1 &&
llvm::StringRef(arg.Includes.front().Header).startswith("<");
}
TEST(StdLibTests, EndToEnd) {
Config Cfg;
Cfg.Index.StandardLibrary = true;
WithContextValue Enabled(Config::Key, std::move(Cfg));
MockFS FS;
FS.Files["stdlib/vector"] =
"namespace std { template <class> class vector; }";
FS.Files["stdlib/list"] =
" namespace std { template <typename T> class list; }";
MockCompilationDatabase CDB;
CDB.ExtraClangFlags.push_back("-isystem" + testPath("stdlib"));
ClangdServer::Options Opts = ClangdServer::optsForTest();
Opts.BuildDynamicSymbolIndex = true; // also used for stdlib index
ClangdServer Server(CDB, FS, Opts);
Annotations A("std::^");
Server.addDocument(testPath("foo.cc"), A.code());
ASSERT_TRUE(Server.blockUntilIdleForTest());
clangd::CodeCompleteOptions CCOpts;
auto Completions =
cantFail(runCodeComplete(Server, testPath("foo.cc"), A.point(), CCOpts));
EXPECT_THAT(
Completions.Completions,
UnorderedElementsAre(StdlibSymbol("list"), StdlibSymbol("vector")));
}
} // namespace
} // namespace clangd
} // namespace clang

View File

@ -1123,8 +1123,7 @@ TEST_F(TUSchedulerTests, AsyncPreambleThread) {
public:
BlockPreambleThread(llvm::StringRef BlockVersion, Notification &N)
: BlockVersion(BlockVersion), N(N) {}
void onPreambleAST(PathRef Path, llvm::StringRef Version,
const CompilerInvocation &, ASTContext &Ctx,
void onPreambleAST(PathRef Path, llvm::StringRef Version, ASTContext &Ctx,
Preprocessor &, const CanonicalIncludes &) override {
if (Version == BlockVersion)
N.wait();