forked from OSchip/llvm-project
[clangd] Shard preamble symbols in dynamic index
Summary: This reduces memory usage by dynamic index from more than 400MB to 32MB when all files in clang-tools-extra/clangd/*.cpp are active in clangd. Reviewers: sammccall Subscribers: ilya-biryukov, MaskRay, jkorous, arphaman, usaxena95, cfe-commits Tags: #clang Differential Revision: https://reviews.llvm.org/D77732
This commit is contained in:
Kadir Cetinkaya
parent
9bd6b77ac6
commit
dffa9dfbda
|
|
@ -61,51 +61,6 @@ namespace clang {
|
|||
namespace clangd {
|
||||
namespace {
|
||||
|
||||
// Resolves URI to file paths with cache.
|
||||
class URIToFileCache {
|
||||
public:
|
||||
URIToFileCache(llvm::StringRef HintPath) : HintPath(HintPath) {}
|
||||
|
||||
llvm::StringRef resolve(llvm::StringRef FileURI) {
|
||||
auto I = URIToPathCache.try_emplace(FileURI);
|
||||
if (I.second) {
|
||||
auto Path = URI::resolve(FileURI, HintPath);
|
||||
if (!Path) {
|
||||
elog("Failed to resolve URI {0}: {1}", FileURI, Path.takeError());
|
||||
assert(false && "Failed to resolve URI");
|
||||
return "";
|
||||
}
|
||||
I.first->second = *Path;
|
||||
}
|
||||
return I.first->second;
|
||||
}
|
||||
|
||||
private:
|
||||
std::string HintPath;
|
||||
llvm::StringMap<std::string> URIToPathCache;
|
||||
};
|
||||
|
||||
// We keep only the node "U" and its edges. Any node other than "U" will be
|
||||
// empty in the resultant graph.
|
||||
IncludeGraph getSubGraph(const URI &U, const IncludeGraph &FullGraph) {
|
||||
IncludeGraph IG;
|
||||
|
||||
std::string FileURI = U.toString();
|
||||
auto Entry = IG.try_emplace(FileURI).first;
|
||||
auto &Node = Entry->getValue();
|
||||
Node = FullGraph.lookup(Entry->getKey());
|
||||
Node.URI = Entry->getKey();
|
||||
|
||||
// URIs inside nodes must point into the keys of the same IncludeGraph.
|
||||
for (auto &Include : Node.DirectIncludes) {
|
||||
auto I = IG.try_emplace(Include).first;
|
||||
I->getValue().URI = I->getKey();
|
||||
Include = I->getKey();
|
||||
}
|
||||
|
||||
return IG;
|
||||
}
|
||||
|
||||
// We cannot use vfs->makeAbsolute because Cmd.FileName is either absolute or
|
||||
// relative to Cmd.Directory, which might not be the same as current working
|
||||
// directory.
|
||||
|
|
@ -219,108 +174,44 @@ void BackgroundIndex::update(
|
|||
llvm::StringRef MainFile, IndexFileIn Index,
|
||||
const llvm::StringMap<ShardVersion> &ShardVersionsSnapshot,
|
||||
bool HadErrors) {
|
||||
// Partition symbols/references into files.
|
||||
struct File {
|
||||
llvm::DenseSet<const Symbol *> Symbols;
|
||||
llvm::DenseSet<const Ref *> Refs;
|
||||
llvm::DenseSet<const Relation *> Relations;
|
||||
FileDigest Digest;
|
||||
};
|
||||
llvm::StringMap<File> Files;
|
||||
URIToFileCache URICache(MainFile);
|
||||
llvm::StringMap<FileDigest> FilesToUpdate;
|
||||
for (const auto &IndexIt : *Index.Sources) {
|
||||
const auto &IGN = IndexIt.getValue();
|
||||
// Note that sources do not contain any information regarding missing
|
||||
// headers, since we don't even know what absolute path they should fall in.
|
||||
const auto AbsPath = URICache.resolve(IGN.URI);
|
||||
auto AbsPath = llvm::cantFail(URI::resolve(IGN.URI, MainFile),
|
||||
"Failed to resovle URI");
|
||||
const auto DigestIt = ShardVersionsSnapshot.find(AbsPath);
|
||||
// File has different contents, or indexing was successful this time.
|
||||
if (DigestIt == ShardVersionsSnapshot.end() ||
|
||||
DigestIt->getValue().Digest != IGN.Digest ||
|
||||
(DigestIt->getValue().HadErrors && !HadErrors))
|
||||
Files.try_emplace(AbsPath).first->getValue().Digest = IGN.Digest;
|
||||
}
|
||||
// This map is used to figure out where to store relations.
|
||||
llvm::DenseMap<SymbolID, File *> SymbolIDToFile;
|
||||
for (const auto &Sym : *Index.Symbols) {
|
||||
if (Sym.CanonicalDeclaration) {
|
||||
auto DeclPath = URICache.resolve(Sym.CanonicalDeclaration.FileURI);
|
||||
const auto FileIt = Files.find(DeclPath);
|
||||
if (FileIt != Files.end()) {
|
||||
FileIt->second.Symbols.insert(&Sym);
|
||||
SymbolIDToFile[Sym.ID] = &FileIt->second;
|
||||
}
|
||||
}
|
||||
// For symbols with different declaration and definition locations, we store
|
||||
// the full symbol in both the header file and the implementation file, so
|
||||
// that merging can tell the preferred symbols (from canonical headers) from
|
||||
// other symbols (e.g. forward declarations).
|
||||
if (Sym.Definition &&
|
||||
Sym.Definition.FileURI != Sym.CanonicalDeclaration.FileURI) {
|
||||
auto DefPath = URICache.resolve(Sym.Definition.FileURI);
|
||||
const auto FileIt = Files.find(DefPath);
|
||||
if (FileIt != Files.end())
|
||||
FileIt->second.Symbols.insert(&Sym);
|
||||
}
|
||||
}
|
||||
llvm::DenseMap<const Ref *, SymbolID> RefToIDs;
|
||||
for (const auto &SymRefs : *Index.Refs) {
|
||||
for (const auto &R : SymRefs.second) {
|
||||
auto Path = URICache.resolve(R.Location.FileURI);
|
||||
const auto FileIt = Files.find(Path);
|
||||
if (FileIt != Files.end()) {
|
||||
auto &F = FileIt->getValue();
|
||||
RefToIDs[&R] = SymRefs.first;
|
||||
F.Refs.insert(&R);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const auto &Rel : *Index.Relations) {
|
||||
const auto FileIt = SymbolIDToFile.find(Rel.Subject);
|
||||
if (FileIt != SymbolIDToFile.end())
|
||||
FileIt->second->Relations.insert(&Rel);
|
||||
FilesToUpdate[AbsPath] = IGN.Digest;
|
||||
}
|
||||
|
||||
// Shard slabs into files.
|
||||
FileShardedIndex ShardedIndex(std::move(Index), MainFile);
|
||||
|
||||
// Build and store new slabs for each updated file.
|
||||
for (const auto &FileIt : Files) {
|
||||
llvm::StringRef Path = FileIt.getKey();
|
||||
SymbolSlab::Builder Syms;
|
||||
RefSlab::Builder Refs;
|
||||
RelationSlab::Builder Relations;
|
||||
for (const auto *S : FileIt.second.Symbols)
|
||||
Syms.insert(*S);
|
||||
for (const auto *R : FileIt.second.Refs)
|
||||
Refs.insert(RefToIDs[R], *R);
|
||||
for (const auto *Rel : FileIt.second.Relations)
|
||||
Relations.insert(*Rel);
|
||||
auto SS = std::make_unique<SymbolSlab>(std::move(Syms).build());
|
||||
auto RS = std::make_unique<RefSlab>(std::move(Refs).build());
|
||||
auto RelS = std::make_unique<RelationSlab>(std::move(Relations).build());
|
||||
auto IG = std::make_unique<IncludeGraph>(
|
||||
getSubGraph(URI::create(Path), Index.Sources.getValue()));
|
||||
for (const auto &FileIt : FilesToUpdate) {
|
||||
PathRef Path = FileIt.first();
|
||||
auto IF = ShardedIndex.getShard(Path);
|
||||
|
||||
// Only store command line hash for main files of the TU, since our
|
||||
// current model keeps only one version of a header file.
|
||||
if (Path != MainFile)
|
||||
IF.Cmd.reset();
|
||||
|
||||
// We need to store shards before updating the index, since the latter
|
||||
// consumes slabs.
|
||||
// FIXME: Also skip serializing the shard if it is already up-to-date.
|
||||
BackgroundIndexStorage *IndexStorage = IndexStorageFactory(Path);
|
||||
IndexFileOut Shard;
|
||||
Shard.Symbols = SS.get();
|
||||
Shard.Refs = RS.get();
|
||||
Shard.Relations = RelS.get();
|
||||
Shard.Sources = IG.get();
|
||||
|
||||
// Only store command line hash for main files of the TU, since our
|
||||
// current model keeps only one version of a header file.
|
||||
if (Path == MainFile)
|
||||
Shard.Cmd = Index.Cmd.getPointer();
|
||||
|
||||
if (auto Error = IndexStorage->storeShard(Path, Shard))
|
||||
if (auto Error = IndexStorageFactory(Path)->storeShard(Path, IF))
|
||||
elog("Failed to write background-index shard for file {0}: {1}", Path,
|
||||
std::move(Error));
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> Lock(ShardVersionsMu);
|
||||
auto Hash = FileIt.second.Digest;
|
||||
const auto &Hash = FileIt.getValue();
|
||||
auto DigestIt = ShardVersions.try_emplace(Path);
|
||||
ShardVersion &SV = DigestIt.first->second;
|
||||
// Skip if file is already up to date, unless previous index was broken
|
||||
|
|
@ -333,8 +224,11 @@ void BackgroundIndex::update(
|
|||
// This can override a newer version that is added in another thread, if
|
||||
// this thread sees the older version but finishes later. This should be
|
||||
// rare in practice.
|
||||
IndexedSymbols.update(Path, std::move(SS), std::move(RS), std::move(RelS),
|
||||
Path == MainFile);
|
||||
IndexedSymbols.update(
|
||||
Path, std::make_unique<SymbolSlab>(std::move(*IF.Symbols)),
|
||||
std::make_unique<RefSlab>(std::move(*IF.Refs)),
|
||||
std::make_unique<RelationSlab>(std::move(*IF.Relations)),
|
||||
Path == MainFile);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,11 +10,17 @@
|
|||
#include "CollectMacros.h"
|
||||
#include "Logger.h"
|
||||
#include "ParsedAST.h"
|
||||
#include "Path.h"
|
||||
#include "SymbolCollector.h"
|
||||
#include "index/CanonicalIncludes.h"
|
||||
#include "index/Index.h"
|
||||
#include "index/MemIndex.h"
|
||||
#include "index/Merge.h"
|
||||
#include "index/Ref.h"
|
||||
#include "index/Relation.h"
|
||||
#include "index/Serialization.h"
|
||||
#include "index/Symbol.h"
|
||||
#include "index/SymbolID.h"
|
||||
#include "index/SymbolOrigin.h"
|
||||
#include "index/dex/Dex.h"
|
||||
#include "clang/AST/ASTContext.h"
|
||||
|
|
@ -23,19 +29,24 @@
|
|||
#include "clang/Lex/MacroInfo.h"
|
||||
#include "clang/Lex/Preprocessor.h"
|
||||
#include "llvm/ADT/DenseMap.h"
|
||||
#include "llvm/ADT/DenseSet.h"
|
||||
#include "llvm/ADT/STLExtras.h"
|
||||
#include "llvm/ADT/StringMap.h"
|
||||
#include "llvm/ADT/StringRef.h"
|
||||
#include "llvm/Support/Error.h"
|
||||
#include <memory>
|
||||
#include <tuple>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace clang {
|
||||
namespace clangd {
|
||||
namespace {
|
||||
|
||||
static SlabTuple indexSymbols(ASTContext &AST, std::shared_ptr<Preprocessor> PP,
|
||||
llvm::ArrayRef<Decl *> DeclsToIndex,
|
||||
const MainFileMacros *MacroRefsToIndex,
|
||||
const CanonicalIncludes &Includes,
|
||||
bool IsIndexMainAST, llvm::StringRef Version) {
|
||||
SlabTuple indexSymbols(ASTContext &AST, std::shared_ptr<Preprocessor> PP,
|
||||
llvm::ArrayRef<Decl *> DeclsToIndex,
|
||||
const MainFileMacros *MacroRefsToIndex,
|
||||
const CanonicalIncludes &Includes, bool IsIndexMainAST,
|
||||
llvm::StringRef Version) {
|
||||
SymbolCollector::Options CollectorOpts;
|
||||
CollectorOpts.CollectIncludePath = true;
|
||||
CollectorOpts.Includes = &Includes;
|
||||
|
|
@ -85,6 +96,131 @@ static SlabTuple indexSymbols(ASTContext &AST, std::shared_ptr<Preprocessor> PP,
|
|||
std::move(Relations));
|
||||
}
|
||||
|
||||
// Resolves URI to file paths with cache.
|
||||
class URIToFileCache {
|
||||
public:
|
||||
URIToFileCache(PathRef HintPath) : HintPath(HintPath) {}
|
||||
|
||||
llvm::StringRef operator[](llvm::StringRef FileURI) {
|
||||
if (FileURI.empty())
|
||||
return "";
|
||||
auto I = URIToPathCache.try_emplace(FileURI);
|
||||
if (I.second) {
|
||||
I.first->second = llvm::cantFail(URI::resolve(FileURI, HintPath),
|
||||
"Failed to resolve URI");
|
||||
}
|
||||
return I.first->second;
|
||||
}
|
||||
|
||||
private:
|
||||
PathRef HintPath;
|
||||
llvm::StringMap<std::string> URIToPathCache;
|
||||
};
|
||||
|
||||
// We keep only the node "U" and its edges. Any node other than "U" will be
|
||||
// empty in the resultant graph.
|
||||
IncludeGraph getSubGraph(llvm::StringRef URI, const IncludeGraph &FullGraph) {
|
||||
IncludeGraph IG;
|
||||
|
||||
auto Entry = IG.try_emplace(URI).first;
|
||||
auto &Node = Entry->getValue();
|
||||
Node = FullGraph.lookup(Entry->getKey());
|
||||
Node.URI = Entry->getKey();
|
||||
|
||||
// URIs inside nodes must point into the keys of the same IncludeGraph.
|
||||
for (auto &Include : Node.DirectIncludes) {
|
||||
auto I = IG.try_emplace(Include).first;
|
||||
I->getValue().URI = I->getKey();
|
||||
Include = I->getKey();
|
||||
}
|
||||
return IG;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
FileShardedIndex::FileShardedIndex(IndexFileIn Input, PathRef HintPath)
|
||||
: Index(std::move(Input)) {
|
||||
URIToFileCache UriToFile(HintPath);
|
||||
// Used to build RelationSlabs.
|
||||
llvm::DenseMap<SymbolID, FileShard *> SymbolIDToFile;
|
||||
|
||||
// Attribute each Symbol to both their declaration and definition locations.
|
||||
if (Index.Symbols) {
|
||||
for (const auto &S : *Index.Symbols) {
|
||||
auto File = UriToFile[S.CanonicalDeclaration.FileURI];
|
||||
auto It = Shards.try_emplace(File);
|
||||
It.first->getValue().Symbols.insert(&S);
|
||||
SymbolIDToFile[S.ID] = &It.first->getValue();
|
||||
// Only bother if definition file is different than declaration file.
|
||||
if (S.Definition &&
|
||||
S.Definition.FileURI != S.CanonicalDeclaration.FileURI) {
|
||||
auto File = UriToFile[S.Definition.FileURI];
|
||||
auto It = Shards.try_emplace(File);
|
||||
It.first->getValue().Symbols.insert(&S);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Attribute references into each file they occured in.
|
||||
if (Index.Refs) {
|
||||
for (const auto &SymRefs : *Index.Refs) {
|
||||
for (const auto &R : SymRefs.second) {
|
||||
auto File = UriToFile[R.Location.FileURI];
|
||||
const auto It = Shards.try_emplace(File);
|
||||
It.first->getValue().Refs.insert(&R);
|
||||
RefToSymID[&R] = SymRefs.first;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Attribute relations to the file declaraing their Subject as Object might
|
||||
// not have been indexed, see SymbolCollector::processRelations for details.
|
||||
if (Index.Relations) {
|
||||
for (const auto &R : *Index.Relations) {
|
||||
auto *File = SymbolIDToFile.lookup(R.Subject);
|
||||
assert(File && "unknown subject in relation");
|
||||
File->Relations.insert(&R);
|
||||
}
|
||||
}
|
||||
// Store only the direct includes of a file in a shard.
|
||||
if (Index.Sources) {
|
||||
const auto &FullGraph = *Index.Sources;
|
||||
for (const auto &It : FullGraph) {
|
||||
auto File = UriToFile[It.first()];
|
||||
auto ShardIt = Shards.try_emplace(File);
|
||||
ShardIt.first->getValue().IG = getSubGraph(It.first(), FullGraph);
|
||||
}
|
||||
}
|
||||
}
|
||||
std::vector<PathRef> FileShardedIndex::getAllFiles() const {
|
||||
return {Shards.keys().begin(), Shards.keys().end()};
|
||||
}
|
||||
|
||||
IndexFileIn FileShardedIndex::getShard(PathRef File) const {
|
||||
auto It = Shards.find(File);
|
||||
assert(It != Shards.end() && "received unknown file");
|
||||
|
||||
IndexFileIn IF;
|
||||
IF.Sources = It->getValue().IG;
|
||||
IF.Cmd = Index.Cmd;
|
||||
|
||||
SymbolSlab::Builder SymB;
|
||||
for (const auto *S : It->getValue().Symbols)
|
||||
SymB.insert(*S);
|
||||
IF.Symbols = std::move(SymB).build();
|
||||
|
||||
RefSlab::Builder RefB;
|
||||
for (const auto *Ref : It->getValue().Refs) {
|
||||
auto SID = RefToSymID.lookup(Ref);
|
||||
RefB.insert(SID, *Ref);
|
||||
}
|
||||
IF.Refs = std::move(RefB).build();
|
||||
|
||||
RelationSlab::Builder RelB;
|
||||
for (const auto *Rel : It->getValue().Relations) {
|
||||
RelB.insert(*Rel);
|
||||
}
|
||||
IF.Relations = std::move(RelB).build();
|
||||
return IF;
|
||||
}
|
||||
|
||||
SlabTuple indexMainDecls(ParsedAST &AST) {
|
||||
return indexSymbols(AST.getASTContext(), AST.getPreprocessorPtr(),
|
||||
AST.getLocalTopLevelDecls(), &AST.getMacros(),
|
||||
|
|
@ -254,15 +390,24 @@ void FileIndex::updatePreamble(PathRef Path, llvm::StringRef Version,
|
|||
ASTContext &AST,
|
||||
std::shared_ptr<Preprocessor> PP,
|
||||
const CanonicalIncludes &Includes) {
|
||||
auto Slabs = indexHeaderSymbols(Version, AST, std::move(PP), Includes);
|
||||
PreambleSymbols.update(
|
||||
Path, std::make_unique<SymbolSlab>(std::move(std::get<0>(Slabs))),
|
||||
std::make_unique<RefSlab>(),
|
||||
std::make_unique<RelationSlab>(std::move(std::get<2>(Slabs))),
|
||||
/*CountReferences=*/false);
|
||||
IndexFileIn IF;
|
||||
std::tie(IF.Symbols, std::ignore, IF.Relations) =
|
||||
indexHeaderSymbols(Version, AST, std::move(PP), Includes);
|
||||
FileShardedIndex ShardedIndex(std::move(IF), Path);
|
||||
for (PathRef File : ShardedIndex.getAllFiles()) {
|
||||
auto IF = ShardedIndex.getShard(File);
|
||||
PreambleSymbols.update(
|
||||
File, std::make_unique<SymbolSlab>(std::move(*IF.Symbols)),
|
||||
std::make_unique<RefSlab>(),
|
||||
std::make_unique<RelationSlab>(std::move(*IF.Relations)),
|
||||
/*CountReferences=*/false);
|
||||
}
|
||||
PreambleIndex.reset(
|
||||
PreambleSymbols.buildIndex(UseDex ? IndexType::Heavy : IndexType::Light,
|
||||
DuplicateHandling::PickOne));
|
||||
vlog("Build dynamic index for header symbols with estimated memory usage of "
|
||||
"{0} bytes",
|
||||
PreambleIndex.estimateMemoryUsage());
|
||||
}
|
||||
|
||||
void FileIndex::updateMain(PathRef Path, ParsedAST &AST) {
|
||||
|
|
@ -274,6 +419,9 @@ void FileIndex::updateMain(PathRef Path, ParsedAST &AST) {
|
|||
/*CountReferences=*/true);
|
||||
MainFileIndex.reset(
|
||||
MainFileSymbols.buildIndex(IndexType::Light, DuplicateHandling::Merge));
|
||||
vlog("Build dynamic index for main-file symbols with estimated memory usage "
|
||||
"of {0} bytes",
|
||||
MainFileIndex.estimateMemoryUsage());
|
||||
}
|
||||
|
||||
} // namespace clangd
|
||||
|
|
|
|||
|
|
@ -15,14 +15,24 @@
|
|||
#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_FILEINDEX_H
|
||||
#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_FILEINDEX_H
|
||||
|
||||
#include "Headers.h"
|
||||
#include "Index.h"
|
||||
#include "MemIndex.h"
|
||||
#include "Merge.h"
|
||||
#include "Path.h"
|
||||
#include "index/CanonicalIncludes.h"
|
||||
#include "index/Ref.h"
|
||||
#include "index/Relation.h"
|
||||
#include "index/Serialization.h"
|
||||
#include "index/Symbol.h"
|
||||
#include "clang/Lex/Preprocessor.h"
|
||||
#include "clang/Tooling/CompilationDatabase.h"
|
||||
#include "llvm/ADT/DenseSet.h"
|
||||
#include "llvm/ADT/Optional.h"
|
||||
#include "llvm/ADT/StringMap.h"
|
||||
#include "llvm/ADT/StringRef.h"
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
namespace clang {
|
||||
class ASTContext;
|
||||
|
|
@ -109,17 +119,15 @@ public:
|
|||
private:
|
||||
bool UseDex; // FIXME: this should be always on.
|
||||
|
||||
// Contains information from each file's preamble only.
|
||||
// These are large, but update fairly infrequently (preambles are stable).
|
||||
// Contains information from each file's preamble only. Symbols and relations
|
||||
// are sharded per declaration file to deduplicate multiple symbols and reduce
|
||||
// memory usage.
|
||||
// Missing information:
|
||||
// - symbol refs (these are always "from the main file")
|
||||
// - definition locations in the main file
|
||||
//
|
||||
// FIXME: Because the preambles for different TUs have large overlap and
|
||||
// FileIndex doesn't deduplicate, this uses lots of extra RAM.
|
||||
// The biggest obstacle in fixing this: the obvious approach of partitioning
|
||||
// by declaring file (rather than main file) fails if headers provide
|
||||
// different symbols based on preprocessor state.
|
||||
// Note that we store only one version of a header, hence symbols appearing in
|
||||
// different PP states will be missing.
|
||||
FileSymbols PreambleSymbols;
|
||||
SwapIndex PreambleIndex;
|
||||
|
||||
|
|
@ -146,6 +154,43 @@ SlabTuple indexHeaderSymbols(llvm::StringRef Version, ASTContext &AST,
|
|||
std::shared_ptr<Preprocessor> PP,
|
||||
const CanonicalIncludes &Includes);
|
||||
|
||||
/// Takes slabs coming from a TU (multiple files) and shards them per
|
||||
/// declaration location.
|
||||
struct FileShardedIndex {
|
||||
/// \p HintPath is used to convert file URIs stored in symbols into absolute
|
||||
/// paths.
|
||||
explicit FileShardedIndex(IndexFileIn Input, PathRef HintPath);
|
||||
|
||||
/// Returns absolute paths for all files that has a shard.
|
||||
std::vector<PathRef> getAllFiles() const;
|
||||
|
||||
/// Generates index shard for the \p File. Note that this function results in
|
||||
/// a copy of all the relevant data.
|
||||
/// Returned index will always have Symbol/Refs/Relation Slabs set, even if
|
||||
/// they are empty.
|
||||
IndexFileIn getShard(PathRef File) const;
|
||||
|
||||
private:
|
||||
// Contains all the information that belongs to a single file.
|
||||
struct FileShard {
|
||||
// Either declared or defined in the file.
|
||||
llvm::DenseSet<const Symbol *> Symbols;
|
||||
// Reference occurs in the file.
|
||||
llvm::DenseSet<const Ref *> Refs;
|
||||
// Subject is declared in the file.
|
||||
llvm::DenseSet<const Relation *> Relations;
|
||||
// Contains edges for only the direct includes.
|
||||
IncludeGraph IG;
|
||||
};
|
||||
|
||||
// Keeps all the information alive.
|
||||
const IndexFileIn Index;
|
||||
// Mapping from absolute paths to slab information.
|
||||
llvm::StringMap<FileShard> Shards;
|
||||
// Used to build RefSlabs.
|
||||
llvm::DenseMap<const Ref *, SymbolID> RefToSymID;
|
||||
};
|
||||
|
||||
} // namespace clangd
|
||||
} // namespace clang
|
||||
|
||||
|
|
|
|||
|
|
@ -283,6 +283,18 @@ bool SymbolCollector::handleDeclOccurrence(
|
|||
if (!ID)
|
||||
return true;
|
||||
|
||||
// ND is the canonical (i.e. first) declaration. If it's in the main file
|
||||
// (which is not a header), then no public declaration was visible, so assume
|
||||
// it's main-file only.
|
||||
bool IsMainFileOnly =
|
||||
SM.isWrittenInMainFile(SM.getExpansionLoc(ND->getBeginLoc())) &&
|
||||
!isHeaderFile(SM.getFileEntryForID(SM.getMainFileID())->getName(),
|
||||
ASTCtx->getLangOpts());
|
||||
// In C, printf is a redecl of an implicit builtin! So check OrigD instead.
|
||||
if (ASTNode.OrigD->isImplicit() ||
|
||||
!shouldCollectSymbol(*ND, *ASTCtx, Opts, IsMainFileOnly))
|
||||
return true;
|
||||
|
||||
// Note: we need to process relations for all decl occurrences, including
|
||||
// refs, because the indexing code only populates relations for specific
|
||||
// occurrences. For example, RelationBaseOf is only populated for the
|
||||
|
|
@ -297,17 +309,6 @@ bool SymbolCollector::handleDeclOccurrence(
|
|||
if (IsOnlyRef && !CollectRef)
|
||||
return true;
|
||||
|
||||
// ND is the canonical (i.e. first) declaration. If it's in the main file
|
||||
// (which is not a header), then no public declaration was visible, so assume
|
||||
// it's main-file only.
|
||||
bool IsMainFileOnly =
|
||||
SM.isWrittenInMainFile(SM.getExpansionLoc(ND->getBeginLoc())) &&
|
||||
!isHeaderFile(SM.getFileEntryForID(SM.getMainFileID())->getName(),
|
||||
ASTCtx->getLangOpts());
|
||||
// In C, printf is a redecl of an implicit builtin! So check OrigD instead.
|
||||
if (ASTNode.OrigD->isImplicit() ||
|
||||
!shouldCollectSymbol(*ND, *ASTCtx, Opts, IsMainFileOnly))
|
||||
return true;
|
||||
// Do not store references to main-file symbols.
|
||||
// Unlike other fields, e.g. Symbols (which use spelling locations), we use
|
||||
// file locations for references (as it aligns the behavior of clangd's
|
||||
|
|
|
|||
|
|
@ -9,13 +9,19 @@
|
|||
#include "AST.h"
|
||||
#include "Annotations.h"
|
||||
#include "Compiler.h"
|
||||
#include "Headers.h"
|
||||
#include "ParsedAST.h"
|
||||
#include "SyncAPI.h"
|
||||
#include "TestFS.h"
|
||||
#include "TestTU.h"
|
||||
#include "URI.h"
|
||||
#include "index/CanonicalIncludes.h"
|
||||
#include "index/FileIndex.h"
|
||||
#include "index/Index.h"
|
||||
#include "index/Ref.h"
|
||||
#include "index/Relation.h"
|
||||
#include "index/Serialization.h"
|
||||
#include "index/Symbol.h"
|
||||
#include "clang/Frontend/CompilerInvocation.h"
|
||||
#include "clang/Frontend/Utils.h"
|
||||
#include "clang/Index/IndexSymbol.h"
|
||||
|
|
@ -23,6 +29,7 @@
|
|||
#include "clang/Tooling/CompilationDatabase.h"
|
||||
#include "gmock/gmock.h"
|
||||
#include "gtest/gtest.h"
|
||||
#include <utility>
|
||||
|
||||
using ::testing::_;
|
||||
using ::testing::AllOf;
|
||||
|
|
@ -151,8 +158,9 @@ void update(FileIndex &M, llvm::StringRef Basename, llvm::StringRef Code) {
|
|||
File.HeaderFilename = (Basename + ".h").str();
|
||||
File.HeaderCode = std::string(Code);
|
||||
auto AST = File.build();
|
||||
M.updatePreamble(File.Filename, /*Version=*/"null", AST.getASTContext(),
|
||||
AST.getPreprocessorPtr(), AST.getCanonicalIncludes());
|
||||
M.updatePreamble(testPath(File.Filename), /*Version=*/"null",
|
||||
AST.getASTContext(), AST.getPreprocessorPtr(),
|
||||
AST.getCanonicalIncludes());
|
||||
}
|
||||
|
||||
TEST(FileIndexTest, CustomizedURIScheme) {
|
||||
|
|
@ -393,8 +401,9 @@ TEST(FileIndexTest, Relations) {
|
|||
TU.HeaderCode = "class A {}; class B : public A {};";
|
||||
auto AST = TU.build();
|
||||
FileIndex Index;
|
||||
Index.updatePreamble(TU.Filename, /*Version=*/"null", AST.getASTContext(),
|
||||
AST.getPreprocessorPtr(), AST.getCanonicalIncludes());
|
||||
Index.updatePreamble(testPath(TU.Filename), /*Version=*/"null",
|
||||
AST.getASTContext(), AST.getPreprocessorPtr(),
|
||||
AST.getCanonicalIncludes());
|
||||
SymbolID A = findSymbol(TU.headerSymbols(), "A").ID;
|
||||
uint32_t Results = 0;
|
||||
RelationsRequest Req;
|
||||
|
|
@ -477,6 +486,128 @@ TEST(FileSymbolsTest, CountReferencesWithRefSlabs) {
|
|||
AllOf(QName("2"), NumReferences(1u)),
|
||||
AllOf(QName("3"), NumReferences(1u))));
|
||||
}
|
||||
|
||||
TEST(FileIndexTest, StalePreambleSymbolsDeleted) {
|
||||
FileIndex M;
|
||||
TestTU File;
|
||||
File.HeaderFilename = "a.h";
|
||||
|
||||
File.Filename = "f1.cpp";
|
||||
File.HeaderCode = "int a;";
|
||||
auto AST = File.build();
|
||||
M.updatePreamble(testPath(File.Filename), /*Version=*/"null",
|
||||
AST.getASTContext(), AST.getPreprocessorPtr(),
|
||||
AST.getCanonicalIncludes());
|
||||
EXPECT_THAT(runFuzzyFind(M, ""), UnorderedElementsAre(QName("a")));
|
||||
|
||||
File.Filename = "f2.cpp";
|
||||
File.HeaderCode = "int b;";
|
||||
AST = File.build();
|
||||
M.updatePreamble(testPath(File.Filename), /*Version=*/"null",
|
||||
AST.getASTContext(), AST.getPreprocessorPtr(),
|
||||
AST.getCanonicalIncludes());
|
||||
EXPECT_THAT(runFuzzyFind(M, ""), UnorderedElementsAre(QName("b")));
|
||||
}
|
||||
|
||||
TEST(FileShardedIndexTest, Sharding) {
|
||||
auto AHeaderUri = URI::create(testPath("a.h")).toString();
|
||||
auto BHeaderUri = URI::create(testPath("b.h")).toString();
|
||||
auto BSourceUri = URI::create(testPath("b.cc")).toString();
|
||||
|
||||
auto Sym1 = symbol("1");
|
||||
Sym1.CanonicalDeclaration.FileURI = AHeaderUri.c_str();
|
||||
|
||||
auto Sym2 = symbol("2");
|
||||
Sym2.CanonicalDeclaration.FileURI = BHeaderUri.c_str();
|
||||
Sym2.Definition.FileURI = BSourceUri.c_str();
|
||||
|
||||
IndexFileIn IF;
|
||||
{
|
||||
SymbolSlab::Builder B;
|
||||
// Should be stored in only a.h
|
||||
B.insert(Sym1);
|
||||
// Should be stored in both b.h and b.cc
|
||||
B.insert(Sym2);
|
||||
IF.Symbols = std::move(B).build();
|
||||
}
|
||||
{
|
||||
// Should be stored in b.cc
|
||||
IF.Refs = std::move(*refSlab(Sym1.ID, BSourceUri.c_str()).release());
|
||||
}
|
||||
{
|
||||
RelationSlab::Builder B;
|
||||
// Should be stored in a.h
|
||||
B.insert(Relation{Sym1.ID, RelationKind::BaseOf, Sym2.ID});
|
||||
// Should be stored in b.h
|
||||
B.insert(Relation{Sym2.ID, RelationKind::BaseOf, Sym1.ID});
|
||||
IF.Relations = std::move(B).build();
|
||||
}
|
||||
|
||||
IF.Sources.emplace();
|
||||
IncludeGraph &IG = *IF.Sources;
|
||||
{
|
||||
// b.cc includes b.h
|
||||
auto &Node = IG[BSourceUri];
|
||||
Node.DirectIncludes = {BHeaderUri};
|
||||
Node.URI = BSourceUri;
|
||||
}
|
||||
{
|
||||
// b.h includes a.h
|
||||
auto &Node = IG[BHeaderUri];
|
||||
Node.DirectIncludes = {AHeaderUri};
|
||||
Node.URI = BHeaderUri;
|
||||
}
|
||||
{
|
||||
// a.h includes nothing.
|
||||
auto &Node = IG[AHeaderUri];
|
||||
Node.DirectIncludes = {};
|
||||
Node.URI = AHeaderUri;
|
||||
}
|
||||
|
||||
IF.Cmd = tooling::CompileCommand(testRoot(), "b.cc", {"clang"}, "out");
|
||||
|
||||
FileShardedIndex ShardedIndex(std::move(IF), testPath("b.cc"));
|
||||
ASSERT_THAT(
|
||||
ShardedIndex.getAllFiles(),
|
||||
UnorderedElementsAre(testPath("a.h"), testPath("b.h"), testPath("b.cc")));
|
||||
|
||||
{
|
||||
auto Shard = ShardedIndex.getShard(testPath("a.h"));
|
||||
EXPECT_THAT(Shard.Symbols.getValue(), UnorderedElementsAre(QName("1")));
|
||||
EXPECT_THAT(Shard.Refs.getValue(), IsEmpty());
|
||||
EXPECT_THAT(
|
||||
Shard.Relations.getValue(),
|
||||
UnorderedElementsAre(Relation{Sym1.ID, RelationKind::BaseOf, Sym2.ID}));
|
||||
ASSERT_THAT(Shard.Sources.getValue().keys(),
|
||||
UnorderedElementsAre(AHeaderUri));
|
||||
EXPECT_THAT(Shard.Sources->lookup(AHeaderUri).DirectIncludes, IsEmpty());
|
||||
EXPECT_TRUE(Shard.Cmd.hasValue());
|
||||
}
|
||||
{
|
||||
auto Shard = ShardedIndex.getShard(testPath("b.h"));
|
||||
EXPECT_THAT(Shard.Symbols.getValue(), UnorderedElementsAre(QName("2")));
|
||||
EXPECT_THAT(Shard.Refs.getValue(), IsEmpty());
|
||||
EXPECT_THAT(
|
||||
Shard.Relations.getValue(),
|
||||
UnorderedElementsAre(Relation{Sym2.ID, RelationKind::BaseOf, Sym1.ID}));
|
||||
ASSERT_THAT(Shard.Sources.getValue().keys(),
|
||||
UnorderedElementsAre(BHeaderUri, AHeaderUri));
|
||||
EXPECT_THAT(Shard.Sources->lookup(BHeaderUri).DirectIncludes,
|
||||
UnorderedElementsAre(AHeaderUri));
|
||||
EXPECT_TRUE(Shard.Cmd.hasValue());
|
||||
}
|
||||
{
|
||||
auto Shard = ShardedIndex.getShard(testPath("b.cc"));
|
||||
EXPECT_THAT(Shard.Symbols.getValue(), UnorderedElementsAre(QName("2")));
|
||||
EXPECT_THAT(Shard.Refs.getValue(), UnorderedElementsAre(Pair(Sym1.ID, _)));
|
||||
EXPECT_THAT(Shard.Relations.getValue(), IsEmpty());
|
||||
ASSERT_THAT(Shard.Sources.getValue().keys(),
|
||||
UnorderedElementsAre(BSourceUri, BHeaderUri));
|
||||
EXPECT_THAT(Shard.Sources->lookup(BSourceUri).DirectIncludes,
|
||||
UnorderedElementsAre(BHeaderUri));
|
||||
EXPECT_TRUE(Shard.Cmd.hasValue());
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
} // namespace clangd
|
||||
} // namespace clang
|
||||
|
|
|
|||
|
|
@ -116,9 +116,10 @@ SymbolSlab TestTU::headerSymbols() const {
|
|||
std::unique_ptr<SymbolIndex> TestTU::index() const {
|
||||
auto AST = build();
|
||||
auto Idx = std::make_unique<FileIndex>(/*UseDex=*/true);
|
||||
Idx->updatePreamble(Filename, /*Version=*/"null", AST.getASTContext(),
|
||||
AST.getPreprocessorPtr(), AST.getCanonicalIncludes());
|
||||
Idx->updateMain(Filename, AST);
|
||||
Idx->updatePreamble(testPath(Filename), /*Version=*/"null",
|
||||
AST.getASTContext(), AST.getPreprocessorPtr(),
|
||||
AST.getCanonicalIncludes());
|
||||
Idx->updateMain(testPath(Filename), AST);
|
||||
return std::move(Idx);
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue