2017-12-15 20:25:02 +08:00
|
|
|
//===--- FileIndex.h - Index for files. ---------------------------- C++-*-===//
|
2017-12-14 22:50:58 +08:00
|
|
|
//
|
2019-01-19 16:50:56 +08:00
|
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
2017-12-14 22:50:58 +08:00
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
2017-12-15 20:25:02 +08:00
|
|
|
//
|
|
|
|
// FileIndex implements SymbolIndex for symbols from a set of files. Symbols are
|
|
|
|
// maintained at source-file granuality (e.g. with ASTs), and files can be
|
|
|
|
// updated dynamically.
|
|
|
|
//
|
2018-08-15 00:03:32 +08:00
|
|
|
//===----------------------------------------------------------------------===//
|
2017-12-14 22:50:58 +08:00
|
|
|
|
2017-12-15 20:25:02 +08:00
|
|
|
#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_FILEINDEX_H
|
|
|
|
#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_FILEINDEX_H
|
2017-12-14 22:50:58 +08:00
|
|
|
|
2018-09-07 17:40:36 +08:00
|
|
|
#include "ClangdUnit.h"
|
2017-12-14 22:50:58 +08:00
|
|
|
#include "Index.h"
|
2017-12-15 20:25:02 +08:00
|
|
|
#include "MemIndex.h"
|
2018-10-04 22:20:22 +08:00
|
|
|
#include "Merge.h"
|
2019-02-05 00:19:57 +08:00
|
|
|
#include "index/CanonicalIncludes.h"
|
2019-02-28 21:23:03 +08:00
|
|
|
#include "index/Symbol.h"
|
2018-05-24 23:50:15 +08:00
|
|
|
#include "clang/Lex/Preprocessor.h"
|
2018-09-18 18:30:44 +08:00
|
|
|
#include <memory>
|
2017-12-14 22:50:58 +08:00
|
|
|
|
|
|
|
namespace clang {
|
|
|
|
namespace clangd {
|
|
|
|
|
2018-10-16 16:53:52 +08:00
|
|
|
/// Select between in-memory index implementations, which have tradeoffs.
|
|
|
|
enum class IndexType {
|
|
|
|
// MemIndex is trivially cheap to build, but has poor query performance.
|
|
|
|
Light,
|
|
|
|
// Dex is relatively expensive to build and uses more memory, but is fast.
|
|
|
|
Heavy,
|
|
|
|
};
|
|
|
|
|
2018-11-06 18:55:21 +08:00
|
|
|
/// How to handle duplicated symbols across multiple files.
|
|
|
|
enum class DuplicateHandling {
|
|
|
|
// Pick a random symbol. Less accurate but faster.
|
|
|
|
PickOne,
|
|
|
|
// Merge symbols. More accurate but slower.
|
|
|
|
Merge,
|
|
|
|
};
|
|
|
|
|
[clangd] Factor out the data-swapping functionality from MemIndex/DexIndex.
Summary:
This is now handled by a wrapper class SwapIndex, so MemIndex/DexIndex can be
immutable and focus on their job.
Old and busted:
I have a MemIndex, which holds a shared_ptr<vector<Symbol*>>, which keeps the
symbol slab alive. I update by calling build(shared_ptr<vector<Symbol*>>).
New hotness: I have a SwapIndex, which holds a unique_ptr<SymbolIndex>, which
holds a MemIndex, which holds a shared_ptr<void>, which keeps backing
data alive.
I update by building a new MemIndex and calling SwapIndex::reset().
Reviewers: kbobyrev, ioeric
Subscribers: ilya-biryukov, ioeric, MaskRay, jkorous, mgrang, arphaman, kadircet, cfe-commits
Differential Revision: https://reviews.llvm.org/D51422
llvm-svn: 341318
2018-09-03 22:37:43 +08:00
|
|
|
/// A container of Symbols from several source files. It can be updated
|
2017-12-14 22:50:58 +08:00
|
|
|
/// at source-file granularity, replacing all symbols from one file with a new
|
|
|
|
/// set.
|
|
|
|
///
|
|
|
|
/// This implements a snapshot semantics for symbols in a file. Each update to a
|
|
|
|
/// file will create a new snapshot for all symbols in the file. Snapshots are
|
|
|
|
/// managed with shared pointers that are shared between this class and the
|
|
|
|
/// users. For each file, this class only stores a pointer pointing to the
|
|
|
|
/// newest snapshot, and an outdated snapshot is deleted by the last owner of
|
|
|
|
/// the snapshot, either this class or the symbol index.
|
|
|
|
///
|
|
|
|
/// The snapshot semantics keeps critical sections minimal since we only need
|
2018-09-01 15:47:03 +08:00
|
|
|
/// locking when we swap or obtain references to snapshots.
|
2017-12-14 22:50:58 +08:00
|
|
|
class FileSymbols {
|
|
|
|
public:
|
[clangd] SymbolOccurrences -> Refs and cleanup
Summary:
A few things that I noticed while merging the SwapIndex patch:
- SymbolOccurrences and particularly SymbolOccurrenceSlab are unwieldy names,
and these names appear *a lot*. Ref, RefSlab, etc seem clear enough
and read/format much better.
- The asymmetry between SymbolSlab and RefSlab (build() vs freeze()) is
confusing and irritating, and doesn't even save much code.
Avoiding RefSlab::Builder was my idea, but it was a bad one; add it.
- DenseMap<SymbolID, ArrayRef<Ref>> seems like a reasonable compromise for
constructing MemIndex - and means many less wasted allocations than the
current DenseMap<SymbolID, vector<Ref*>> for FileIndex, and none for
slabs.
- RefSlab::find() is not actually used for anything, so we can throw
away the DenseMap and keep the representation much more compact.
- A few naming/consistency fixes: e.g. Slabs,Refs -> Symbols,Refs.
Reviewers: ioeric
Subscribers: ilya-biryukov, MaskRay, jkorous, mgrang, arphaman, kadircet, cfe-commits
Differential Revision: https://reviews.llvm.org/D51605
llvm-svn: 341368
2018-09-04 22:39:56 +08:00
|
|
|
/// Updates all symbols and refs in a file.
|
[clangd] Factor out the data-swapping functionality from MemIndex/DexIndex.
Summary:
This is now handled by a wrapper class SwapIndex, so MemIndex/DexIndex can be
immutable and focus on their job.
Old and busted:
I have a MemIndex, which holds a shared_ptr<vector<Symbol*>>, which keeps the
symbol slab alive. I update by calling build(shared_ptr<vector<Symbol*>>).
New hotness: I have a SwapIndex, which holds a unique_ptr<SymbolIndex>, which
holds a MemIndex, which holds a shared_ptr<void>, which keeps backing
data alive.
I update by building a new MemIndex and calling SwapIndex::reset().
Reviewers: kbobyrev, ioeric
Subscribers: ilya-biryukov, ioeric, MaskRay, jkorous, mgrang, arphaman, kadircet, cfe-commits
Differential Revision: https://reviews.llvm.org/D51422
llvm-svn: 341318
2018-09-03 22:37:43 +08:00
|
|
|
/// If either is nullptr, corresponding data for \p Path will be removed.
|
2018-09-01 03:53:37 +08:00
|
|
|
void update(PathRef Path, std::unique_ptr<SymbolSlab> Slab,
|
[clangd] SymbolOccurrences -> Refs and cleanup
Summary:
A few things that I noticed while merging the SwapIndex patch:
- SymbolOccurrences and particularly SymbolOccurrenceSlab are unwieldy names,
and these names appear *a lot*. Ref, RefSlab, etc seem clear enough
and read/format much better.
- The asymmetry between SymbolSlab and RefSlab (build() vs freeze()) is
confusing and irritating, and doesn't even save much code.
Avoiding RefSlab::Builder was my idea, but it was a bad one; add it.
- DenseMap<SymbolID, ArrayRef<Ref>> seems like a reasonable compromise for
constructing MemIndex - and means many less wasted allocations than the
current DenseMap<SymbolID, vector<Ref*>> for FileIndex, and none for
slabs.
- RefSlab::find() is not actually used for anything, so we can throw
away the DenseMap and keep the representation much more compact.
- A few naming/consistency fixes: e.g. Slabs,Refs -> Symbols,Refs.
Reviewers: ioeric
Subscribers: ilya-biryukov, MaskRay, jkorous, mgrang, arphaman, kadircet, cfe-commits
Differential Revision: https://reviews.llvm.org/D51605
llvm-svn: 341368
2018-09-04 22:39:56 +08:00
|
|
|
std::unique_ptr<RefSlab> Refs);
|
2017-12-14 22:50:58 +08:00
|
|
|
|
[clangd] Factor out the data-swapping functionality from MemIndex/DexIndex.
Summary:
This is now handled by a wrapper class SwapIndex, so MemIndex/DexIndex can be
immutable and focus on their job.
Old and busted:
I have a MemIndex, which holds a shared_ptr<vector<Symbol*>>, which keeps the
symbol slab alive. I update by calling build(shared_ptr<vector<Symbol*>>).
New hotness: I have a SwapIndex, which holds a unique_ptr<SymbolIndex>, which
holds a MemIndex, which holds a shared_ptr<void>, which keeps backing
data alive.
I update by building a new MemIndex and calling SwapIndex::reset().
Reviewers: kbobyrev, ioeric
Subscribers: ilya-biryukov, ioeric, MaskRay, jkorous, mgrang, arphaman, kadircet, cfe-commits
Differential Revision: https://reviews.llvm.org/D51422
llvm-svn: 341318
2018-09-03 22:37:43 +08:00
|
|
|
// The index keeps the symbols alive.
|
2018-10-16 16:53:52 +08:00
|
|
|
std::unique_ptr<SymbolIndex>
|
2018-11-06 18:55:21 +08:00
|
|
|
buildIndex(IndexType,
|
[clangd] Cleanup: stop passing around list of supported URI schemes.
Summary:
Instead of passing around a list of supported URI schemes in clangd, we
expose an interface to convert a path to URI using any compatible scheme
that has been registered. It favors customized schemes and falls
back to "file" when no other scheme works.
Changes in this patch are:
- URI::create(AbsPath, URISchemes) -> URI::create(AbsPath). The new API finds a
compatible scheme from the registry.
- Remove URISchemes option everywhere (ClangdServer, SymbolCollecter, FileIndex etc).
- Unit tests will use "unittest" by default.
- Move "test" scheme from ClangdLSPServer to ClangdMain.cpp, and only
register the test scheme when lit-test or enable-lit-scheme is set.
(The new flag is added to make lit protocol.test work; I wonder if there
is alternative here.)
Reviewers: sammccall
Reviewed By: sammccall
Subscribers: ilya-biryukov, MaskRay, jkorous, arphaman, kadircet, cfe-commits
Differential Revision: https://reviews.llvm.org/D54800
llvm-svn: 347467
2018-11-22 23:02:05 +08:00
|
|
|
DuplicateHandling DuplicateHandle = DuplicateHandling::PickOne);
|
2018-09-01 03:53:37 +08:00
|
|
|
|
2017-12-14 22:50:58 +08:00
|
|
|
private:
|
|
|
|
mutable std::mutex Mutex;
|
|
|
|
|
[clangd] SymbolOccurrences -> Refs and cleanup
Summary:
A few things that I noticed while merging the SwapIndex patch:
- SymbolOccurrences and particularly SymbolOccurrenceSlab are unwieldy names,
and these names appear *a lot*. Ref, RefSlab, etc seem clear enough
and read/format much better.
- The asymmetry between SymbolSlab and RefSlab (build() vs freeze()) is
confusing and irritating, and doesn't even save much code.
Avoiding RefSlab::Builder was my idea, but it was a bad one; add it.
- DenseMap<SymbolID, ArrayRef<Ref>> seems like a reasonable compromise for
constructing MemIndex - and means many less wasted allocations than the
current DenseMap<SymbolID, vector<Ref*>> for FileIndex, and none for
slabs.
- RefSlab::find() is not actually used for anything, so we can throw
away the DenseMap and keep the representation much more compact.
- A few naming/consistency fixes: e.g. Slabs,Refs -> Symbols,Refs.
Reviewers: ioeric
Subscribers: ilya-biryukov, MaskRay, jkorous, mgrang, arphaman, kadircet, cfe-commits
Differential Revision: https://reviews.llvm.org/D51605
llvm-svn: 341368
2018-09-04 22:39:56 +08:00
|
|
|
/// Stores the latest symbol snapshots for all active files.
|
|
|
|
llvm::StringMap<std::shared_ptr<SymbolSlab>> FileToSymbols;
|
|
|
|
/// Stores the latest ref snapshots for all active files.
|
|
|
|
llvm::StringMap<std::shared_ptr<RefSlab>> FileToRefs;
|
2017-12-14 22:50:58 +08:00
|
|
|
};
|
|
|
|
|
[clangd] Factor out the data-swapping functionality from MemIndex/DexIndex.
Summary:
This is now handled by a wrapper class SwapIndex, so MemIndex/DexIndex can be
immutable and focus on their job.
Old and busted:
I have a MemIndex, which holds a shared_ptr<vector<Symbol*>>, which keeps the
symbol slab alive. I update by calling build(shared_ptr<vector<Symbol*>>).
New hotness: I have a SwapIndex, which holds a unique_ptr<SymbolIndex>, which
holds a MemIndex, which holds a shared_ptr<void>, which keeps backing
data alive.
I update by building a new MemIndex and calling SwapIndex::reset().
Reviewers: kbobyrev, ioeric
Subscribers: ilya-biryukov, ioeric, MaskRay, jkorous, mgrang, arphaman, kadircet, cfe-commits
Differential Revision: https://reviews.llvm.org/D51422
llvm-svn: 341318
2018-09-03 22:37:43 +08:00
|
|
|
/// This manages symbols from files and an in-memory index on all symbols.
|
2018-09-18 18:30:44 +08:00
|
|
|
/// FIXME: Expose an interface to remove files that are closed.
|
2018-10-04 22:20:22 +08:00
|
|
|
class FileIndex : public MergedIndex {
|
2017-12-15 20:25:02 +08:00
|
|
|
public:
|
[clangd] Cleanup: stop passing around list of supported URI schemes.
Summary:
Instead of passing around a list of supported URI schemes in clangd, we
expose an interface to convert a path to URI using any compatible scheme
that has been registered. It favors customized schemes and falls
back to "file" when no other scheme works.
Changes in this patch are:
- URI::create(AbsPath, URISchemes) -> URI::create(AbsPath). The new API finds a
compatible scheme from the registry.
- Remove URISchemes option everywhere (ClangdServer, SymbolCollecter, FileIndex etc).
- Unit tests will use "unittest" by default.
- Move "test" scheme from ClangdLSPServer to ClangdMain.cpp, and only
register the test scheme when lit-test or enable-lit-scheme is set.
(The new flag is added to make lit protocol.test work; I wonder if there
is alternative here.)
Reviewers: sammccall
Reviewed By: sammccall
Subscribers: ilya-biryukov, MaskRay, jkorous, arphaman, kadircet, cfe-commits
Differential Revision: https://reviews.llvm.org/D54800
llvm-svn: 347467
2018-11-22 23:02:05 +08:00
|
|
|
FileIndex(bool UseDex = true);
|
2018-06-15 16:55:00 +08:00
|
|
|
|
2018-09-18 18:30:44 +08:00
|
|
|
/// Update preamble symbols of file \p Path with all declarations in \p AST
|
|
|
|
/// and macros in \p PP.
|
|
|
|
void updatePreamble(PathRef Path, ASTContext &AST,
|
2019-02-05 00:19:57 +08:00
|
|
|
std::shared_ptr<Preprocessor> PP,
|
|
|
|
const CanonicalIncludes &Includes);
|
2018-09-18 18:30:44 +08:00
|
|
|
|
2018-09-18 21:35:16 +08:00
|
|
|
/// Update symbols and references from main file \p Path with
|
|
|
|
/// `indexMainDecls`.
|
|
|
|
void updateMain(PathRef Path, ParsedAST &AST);
|
[clangd] Factor out the data-swapping functionality from MemIndex/DexIndex.
Summary:
This is now handled by a wrapper class SwapIndex, so MemIndex/DexIndex can be
immutable and focus on their job.
Old and busted:
I have a MemIndex, which holds a shared_ptr<vector<Symbol*>>, which keeps the
symbol slab alive. I update by calling build(shared_ptr<vector<Symbol*>>).
New hotness: I have a SwapIndex, which holds a unique_ptr<SymbolIndex>, which
holds a MemIndex, which holds a shared_ptr<void>, which keeps backing
data alive.
I update by building a new MemIndex and calling SwapIndex::reset().
Reviewers: kbobyrev, ioeric
Subscribers: ilya-biryukov, ioeric, MaskRay, jkorous, mgrang, arphaman, kadircet, cfe-commits
Differential Revision: https://reviews.llvm.org/D51422
llvm-svn: 341318
2018-09-03 22:37:43 +08:00
|
|
|
|
2018-09-18 18:30:44 +08:00
|
|
|
private:
|
2018-10-16 16:53:52 +08:00
|
|
|
bool UseDex; // FIXME: this should be always on.
|
2018-09-18 18:30:44 +08:00
|
|
|
|
|
|
|
// Contains information from each file's preamble only.
|
|
|
|
// These are large, but update fairly infrequently (preambles are stable).
|
|
|
|
// Missing information:
|
|
|
|
// - symbol refs (these are always "from the main file")
|
|
|
|
// - definition locations in the main file
|
|
|
|
//
|
|
|
|
// FIXME: Because the preambles for different TUs have large overlap and
|
|
|
|
// FileIndex doesn't deduplicate, this uses lots of extra RAM.
|
|
|
|
// The biggest obstacle in fixing this: the obvious approach of partitioning
|
|
|
|
// by declaring file (rather than main file) fails if headers provide
|
|
|
|
// different symbols based on preprocessor state.
|
|
|
|
FileSymbols PreambleSymbols;
|
|
|
|
SwapIndex PreambleIndex;
|
|
|
|
|
|
|
|
// Contains information from each file's main AST.
|
|
|
|
// These are updated frequently (on file change), but are relatively small.
|
|
|
|
// Mostly contains:
|
|
|
|
// - refs to symbols declared in the preamble and referenced from main
|
|
|
|
// - symbols declared both in the main file and the preamble
|
|
|
|
// (Note that symbols *only* in the main file are not indexed).
|
|
|
|
FileSymbols MainFileSymbols;
|
|
|
|
SwapIndex MainFileIndex;
|
2017-12-15 20:25:02 +08:00
|
|
|
};
|
|
|
|
|
2018-09-18 21:35:16 +08:00
|
|
|
/// Retrieves symbols and refs of local top level decls in \p AST (i.e.
|
|
|
|
/// `AST.getLocalTopLevelDecls()`).
|
2018-04-30 23:24:17 +08:00
|
|
|
/// Exposed to assist in unit tests.
|
[clangd] Cleanup: stop passing around list of supported URI schemes.
Summary:
Instead of passing around a list of supported URI schemes in clangd, we
expose an interface to convert a path to URI using any compatible scheme
that has been registered. It favors customized schemes and falls
back to "file" when no other scheme works.
Changes in this patch are:
- URI::create(AbsPath, URISchemes) -> URI::create(AbsPath). The new API finds a
compatible scheme from the registry.
- Remove URISchemes option everywhere (ClangdServer, SymbolCollecter, FileIndex etc).
- Unit tests will use "unittest" by default.
- Move "test" scheme from ClangdLSPServer to ClangdMain.cpp, and only
register the test scheme when lit-test or enable-lit-scheme is set.
(The new flag is added to make lit protocol.test work; I wonder if there
is alternative here.)
Reviewers: sammccall
Reviewed By: sammccall
Subscribers: ilya-biryukov, MaskRay, jkorous, arphaman, kadircet, cfe-commits
Differential Revision: https://reviews.llvm.org/D54800
llvm-svn: 347467
2018-11-22 23:02:05 +08:00
|
|
|
std::pair<SymbolSlab, RefSlab> indexMainDecls(ParsedAST &AST);
|
2018-09-18 18:30:44 +08:00
|
|
|
|
|
|
|
/// Idex declarations from \p AST and macros from \p PP that are declared in
|
|
|
|
/// included headers.
|
2019-02-05 00:19:57 +08:00
|
|
|
SymbolSlab indexHeaderSymbols(ASTContext &AST, std::shared_ptr<Preprocessor> PP,
|
|
|
|
const CanonicalIncludes &Includes);
|
2018-04-30 23:24:17 +08:00
|
|
|
|
2017-12-14 22:50:58 +08:00
|
|
|
} // namespace clangd
|
|
|
|
} // namespace clang
|
|
|
|
|
2017-12-15 20:25:02 +08:00
|
|
|
#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_FILEINDEX_H
|