llvm-project/clang-tools-extra/clangd/TUScheduler.cpp

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

1409 lines
53 KiB
C++
Raw Normal View History

//===--- TUScheduler.cpp -----------------------------------------*-C++-*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
// TUScheduler manages a worker per active file. This ASTWorker processes
// updates (modifications to file contents) and reads (actions performed on
// preamble/AST) to the file.
//
// Each ASTWorker owns a dedicated thread to process updates and reads to the
// relevant file. Any request gets queued in FIFO order to be processed by that
// thread.
//
// An update request replaces current praser inputs to ensure any subsequent
// read sees the version of the file they were requested. It will also issue a
// build for new inputs.
//
// ASTWorker processes the file in two parts, a preamble and a main-file
// section. A preamble can be reused between multiple versions of the file until
// invalidated by a modification to a header, compile commands or modification
// to relevant part of the current file. Such a preamble is called compatible.
// An update is considered dead if no read was issued for that version and
// diagnostics weren't requested by client or could be generated for a later
// version of the file. ASTWorker eliminates such requests as they are
// redundant.
//
// In the presence of stale (non-compatible) preambles, ASTWorker won't publish
// diagnostics for update requests. Read requests will be served with ASTs build
// with stale preambles, unless the read is picky and requires a compatible
// preamble. In such cases it will block until new preamble is built.
//
// ASTWorker owns a PreambleThread for building preambles. If the preamble gets
// invalidated by an update request, a new build will be requested on
// PreambleThread. Since PreambleThread only receives requests for newer
// versions of the file, in case of multiple requests it will only build the
// last one and skip requests in between. Unless client force requested
// diagnostics(WantDiagnostics::Yes).
//
// When a new preamble is built, a "golden" AST is immediately built from that
// version of the file. This ensures diagnostics get updated even if the queue
// is full.
//
// Some read requests might just need preamble. Since preambles can be read
// concurrently, ASTWorker runs these requests on their own thread. These
// requests will receive latest build preamble, which might possibly be stale.
#include "TUScheduler.h"
#include "Cancellation.h"
#include "Compiler.h"
#include "Context.h"
[clangd] Surface errors from command-line parsing Summary: Those errors are exposed at the first character of a file, for a lack of a better place. Previously, all errors were stored inside the AST and report accordingly. However, errors in command-line argument parsing could result in failure to produce the AST, so we need an alternative ways to report those errors. We take the following approach in this patch: - buildCompilerInvocation() now requires an explicit DiagnosticConsumer. - TUScheduler and TestTU now collect the diagnostics produced when parsing command line arguments. If pasing of the AST failed, diagnostics are reported via a new ParsingCallbacks::onFailedAST method. If parsing of the AST succeeded, any errors produced during command-line parsing are stored alongside the AST inside the ParsedAST instance and reported as previously by calling the ParsingCallbacks::onMainAST method; - The client code that uses ClangdServer's DiagnosticConsumer does not need to change, it will receive new diagnostics in the onDiagnosticsReady() callback Errors produced when parsing command-line arguments are collected using the same StoreDiags class that is used to collect all other errors. They are recognized by their location being invalid. IIUC, the location is invalid as there is no source manager at this point, it is created at a later stage. Although technically we might also get diagnostics that mention the command-line arguments FileID with after the source manager was created (and they have valid source locations), we choose to not handle those and they are dropped as not coming from the main file. AFAICT, those diagnostics should always be notes, therefore it's safe to drop them without loosing too much information. Reviewers: kadircet Reviewed By: kadircet Subscribers: nridge, javed.absar, MaskRay, jkorous, arphaman, cfe-commits, gribozavr Tags: #clang Differential Revision: https://reviews.llvm.org/D66759 llvm-svn: 370177
2019-08-28 17:24:55 +08:00
#include "Diagnostics.h"
#include "GlobalCompilationDatabase.h"
#include "Logger.h"
#include "ParsedAST.h"
#include "Path.h"
#include "Preamble.h"
#include "Threading.h"
#include "Trace.h"
#include "index/CanonicalIncludes.h"
#include "clang/Frontend/CompilerInvocation.h"
#include "clang/Tooling/CompilationDatabase.h"
#include "llvm/ADT/FunctionExtras.h"
#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/Errc.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Threading.h"
#include <algorithm>
#include <chrono>
#include <functional>
#include <memory>
#include <mutex>
#include <queue>
#include <string>
#include <thread>
#include <type_traits>
#include <utility>
#include <vector>
namespace clang {
namespace clangd {
using std::chrono::steady_clock;
namespace {
class ASTWorker;
} // namespace
static clang::clangd::Key<std::string> kFileBeingProcessed;
llvm::Optional<llvm::StringRef> TUScheduler::getFileBeingProcessedInContext() {
if (auto *File = Context::current().get(kFileBeingProcessed))
return llvm::StringRef(*File);
return None;
}
/// An LRU cache of idle ASTs.
/// Because we want to limit the overall number of these we retain, the cache
/// owns ASTs (and may evict them) while their workers are idle.
/// Workers borrow ASTs when active, and return them when done.
class TUScheduler::ASTCache {
public:
using Key = const ASTWorker *;
ASTCache(unsigned MaxRetainedASTs) : MaxRetainedASTs(MaxRetainedASTs) {}
/// Returns result of getUsedBytes() for the AST cached by \p K.
/// If no AST is cached, 0 is returned.
std::size_t getUsedBytes(Key K) {
std::lock_guard<std::mutex> Lock(Mut);
auto It = findByKey(K);
if (It == LRU.end() || !It->second)
return 0;
return It->second->getUsedBytes();
}
/// Store the value in the pool, possibly removing the last used AST.
/// The value should not be in the pool when this function is called.
void put(Key K, std::unique_ptr<ParsedAST> V) {
std::unique_lock<std::mutex> Lock(Mut);
assert(findByKey(K) == LRU.end());
LRU.insert(LRU.begin(), {K, std::move(V)});
if (LRU.size() <= MaxRetainedASTs)
return;
// We're past the limit, remove the last element.
std::unique_ptr<ParsedAST> ForCleanup = std::move(LRU.back().second);
LRU.pop_back();
// Run the expensive destructor outside the lock.
Lock.unlock();
ForCleanup.reset();
}
/// Returns the cached value for \p K, or llvm::None if the value is not in
/// the cache anymore. If nullptr was cached for \p K, this function will
/// return a null unique_ptr wrapped into an optional.
llvm::Optional<std::unique_ptr<ParsedAST>> take(Key K) {
std::unique_lock<std::mutex> Lock(Mut);
auto Existing = findByKey(K);
if (Existing == LRU.end())
return None;
std::unique_ptr<ParsedAST> V = std::move(Existing->second);
LRU.erase(Existing);
// GCC 4.8 fails to compile `return V;`, as it tries to call the copy
// constructor of unique_ptr, so we call the move ctor explicitly to avoid
// this miscompile.
return llvm::Optional<std::unique_ptr<ParsedAST>>(std::move(V));
}
private:
using KVPair = std::pair<Key, std::unique_ptr<ParsedAST>>;
std::vector<KVPair>::iterator findByKey(Key K) {
return llvm::find_if(LRU, [K](const KVPair &P) { return P.first == K; });
}
std::mutex Mut;
unsigned MaxRetainedASTs;
/// Items sorted in LRU order, i.e. first item is the most recently accessed
/// one.
std::vector<KVPair> LRU; /* GUARDED_BY(Mut) */
};
namespace {
/// Threadsafe manager for updating a TUStatus and emitting it after each
/// update.
class SynchronizedTUStatus {
public:
SynchronizedTUStatus(PathRef FileName, ParsingCallbacks &Callbacks)
: FileName(FileName), Callbacks(Callbacks) {}
void update(llvm::function_ref<void(TUStatus &)> Mutator) {
std::lock_guard<std::mutex> Lock(StatusMu);
Mutator(Status);
emitStatusLocked();
}
/// Prevents emitting of further updates.
void stop() {
std::lock_guard<std::mutex> Lock(StatusMu);
CanPublish = false;
}
private:
void emitStatusLocked() {
if (CanPublish)
Callbacks.onFileUpdated(FileName, Status);
}
const Path FileName;
std::mutex StatusMu;
TUStatus Status;
bool CanPublish = true;
ParsingCallbacks &Callbacks;
};
/// Responsible for building preambles. Whenever the thread is idle and the
/// preamble is outdated, it starts to build a fresh preamble from the latest
/// inputs. If RunSync is true, preambles are built synchronously in update()
/// instead.
class PreambleThread {
public:
PreambleThread(llvm::StringRef FileName, ParsingCallbacks &Callbacks,
bool StorePreambleInMemory, bool RunSync,
SynchronizedTUStatus &Status, ASTWorker &AW)
: FileName(FileName), Callbacks(Callbacks),
StoreInMemory(StorePreambleInMemory), RunSync(RunSync), Status(Status),
ASTPeer(AW) {}
/// It isn't guaranteed that each requested version will be built. If there
/// are multiple update requests while building a preamble, only the last one
/// will be built.
void update(std::unique_ptr<CompilerInvocation> CI, ParseInputs PI,
std::vector<Diag> CIDiags, WantDiagnostics WantDiags) {
Request Req = {std::move(CI), std::move(PI), std::move(CIDiags), WantDiags,
Context::current().clone()};
if (RunSync) {
build(std::move(Req));
Status.update([](TUStatus &Status) {
Status.PreambleActivity = PreambleAction::Idle;
});
return;
}
{
std::lock_guard<std::mutex> Lock(Mutex);
// If shutdown is issued, don't bother building.
if (Done)
return;
NextReq = std::move(Req);
}
// Let the worker thread know there's a request, notify_one is safe as there
// should be a single worker thread waiting on it.
ReqCV.notify_all();
}
void run() {
while (true) {
{
std::unique_lock<std::mutex> Lock(Mutex);
assert(!CurrentReq && "Already processing a request?");
// Wait until stop is called or there is a request.
ReqCV.wait(Lock, [this] { return NextReq || Done; });
if (Done)
break;
CurrentReq = std::move(*NextReq);
NextReq.reset();
}
WithContext Guard(std::move(CurrentReq->Ctx));
// Build the preamble and let the waiters know about it.
build(std::move(*CurrentReq));
bool IsEmpty = false;
{
std::lock_guard<std::mutex> Lock(Mutex);
CurrentReq.reset();
IsEmpty = !NextReq.hasValue();
}
if (IsEmpty) {
// We don't perform this above, before waiting for a request to make
// tests more deterministic. As there can be a race between this thread
// and client thread(clangdserver).
Status.update([](TUStatus &Status) {
Status.PreambleActivity = PreambleAction::Idle;
});
}
ReqCV.notify_all();
}
dlog("Preamble worker for {0} stopped", FileName);
}
/// Signals the run loop to exit.
void stop() {
dlog("Preamble worker for {0} received stop", FileName);
{
std::lock_guard<std::mutex> Lock(Mutex);
Done = true;
NextReq.reset();
}
// Let the worker thread know that it should stop.
ReqCV.notify_all();
}
bool blockUntilIdle(Deadline Timeout) const {
std::unique_lock<std::mutex> Lock(Mutex);
return wait(Lock, ReqCV, Timeout, [&] { return !NextReq && !CurrentReq; });
}
private:
/// Holds inputs required for building a preamble. CI is guaranteed to be
/// non-null.
struct Request {
std::unique_ptr<CompilerInvocation> CI;
ParseInputs Inputs;
std::vector<Diag> CIDiags;
WantDiagnostics WantDiags;
Context Ctx;
};
bool isDone() {
std::lock_guard<std::mutex> Lock(Mutex);
return Done;
}
/// Builds a preamble for \p Req, might reuse LatestBuild if possible.
/// Notifies ASTWorker after build finishes.
void build(Request Req);
mutable std::mutex Mutex;
bool Done = false; /* GUARDED_BY(Mutex) */
llvm::Optional<Request> NextReq; /* GUARDED_BY(Mutex) */
llvm::Optional<Request> CurrentReq; /* GUARDED_BY(Mutex) */
// Signaled whenever a thread populates NextReq or worker thread builds a
// Preamble.
mutable std::condition_variable ReqCV; /* GUARDED_BY(Mutex) */
// Accessed only by preamble thread.
std::shared_ptr<const PreambleData> LatestBuild;
const Path FileName;
ParsingCallbacks &Callbacks;
const bool StoreInMemory;
const bool RunSync;
SynchronizedTUStatus &Status;
ASTWorker &ASTPeer;
};
class ASTWorkerHandle;
/// Owns one instance of the AST, schedules updates and reads of it.
/// Also responsible for building and providing access to the preamble.
/// Each ASTWorker processes the async requests sent to it on a separate
/// dedicated thread.
/// The ASTWorker that manages the AST is shared by both the processing thread
/// and the TUScheduler. The TUScheduler should discard an ASTWorker when
/// remove() is called, but its thread may be busy and we don't want to block.
/// So the workers are accessed via an ASTWorkerHandle. Destroying the handle
/// signals the worker to exit its run loop and gives up shared ownership of the
/// worker.
class ASTWorker {
friend class ASTWorkerHandle;
ASTWorker(PathRef FileName, const GlobalCompilationDatabase &CDB,
TUScheduler::ASTCache &LRUCache, Semaphore &Barrier, bool RunSync,
DebouncePolicy UpdateDebounce, bool StorePreamblesInMemory,
ParsingCallbacks &Callbacks);
public:
/// Create a new ASTWorker and return a handle to it.
/// The processing thread is spawned using \p Tasks. However, when \p Tasks
/// is null, all requests will be processed on the calling thread
/// synchronously instead. \p Barrier is acquired when processing each
/// request, it is used to limit the number of actively running threads.
static ASTWorkerHandle
create(PathRef FileName, const GlobalCompilationDatabase &CDB,
TUScheduler::ASTCache &IdleASTs, AsyncTaskRunner *Tasks,
Semaphore &Barrier, DebouncePolicy UpdateDebounce,
bool StorePreamblesInMemory, ParsingCallbacks &Callbacks);
~ASTWorker();
void update(ParseInputs Inputs, WantDiagnostics);
void
runWithAST(llvm::StringRef Name,
llvm::unique_function<void(llvm::Expected<InputsAndAST>)> Action,
TUScheduler::ASTActionInvalidation);
bool blockUntilIdle(Deadline Timeout) const;
std::shared_ptr<const PreambleData> getPossiblyStalePreamble() const;
/// Used to inform ASTWorker about a new preamble build by PreambleThread.
/// Diagnostics are only published through this callback. This ensures they
/// are always for newer versions of the file, as the callback gets called in
/// the same order as update requests.
void updatePreamble(std::unique_ptr<CompilerInvocation> CI, ParseInputs PI,
std::shared_ptr<const PreambleData> Preamble,
std::vector<Diag> CIDiags, WantDiagnostics WantDiags);
/// Obtain a preamble reflecting all updates so far. Threadsafe.
/// It may be delivered immediately, or later on the worker thread.
void getCurrentPreamble(
llvm::unique_function<void(std::shared_ptr<const PreambleData>)>);
/// Returns compile command from the current file inputs.
tooling::CompileCommand getCurrentCompileCommand() const;
/// Wait for the first build of preamble to finish. Preamble itself can be
/// accessed via getPossiblyStalePreamble(). Note that this function will
/// return after an unsuccessful build of the preamble too, i.e. result of
/// getPossiblyStalePreamble() can be null even after this function returns.
void waitForFirstPreamble() const;
std::size_t getUsedBytes() const;
bool isASTCached() const;
private:
/// Publishes diagnostics for \p Inputs. It will build an AST or reuse the
/// cached one if applicable. Assumes LatestPreamble is compatible for \p
/// Inputs.
void generateDiagnostics(std::unique_ptr<CompilerInvocation> Invocation,
ParseInputs Inputs, std::vector<Diag> CIDiags);
// Must be called exactly once on processing thread. Will return after
// stop() is called on a separate thread and all pending requests are
// processed.
void run();
/// Signal that run() should finish processing pending requests and exit.
void stop();
/// Adds a new task to the end of the request queue.
void startTask(llvm::StringRef Name, llvm::unique_function<void()> Task,
llvm::Optional<WantDiagnostics> UpdateType,
TUScheduler::ASTActionInvalidation);
/// Determines the next action to perform.
/// All actions that should never run are discarded.
/// Returns a deadline for the next action. If it's expired, run now.
/// scheduleLocked() is called again at the deadline, or if requests arrive.
Deadline scheduleLocked();
/// Should the first task in the queue be skipped instead of run?
bool shouldSkipHeadLocked() const;
/// This is private because `FileInputs.FS` is not thread-safe and thus not
/// safe to share. Callers should make sure not to expose `FS` via a public
/// interface.
std::shared_ptr<const ParseInputs> getCurrentFileInputs() const;
struct Request {
llvm::unique_function<void()> Action;
std::string Name;
steady_clock::time_point AddTime;
Context Ctx;
llvm::Optional<WantDiagnostics> UpdateType;
TUScheduler::ASTActionInvalidation InvalidationPolicy;
Canceler Invalidate;
};
/// Handles retention of ASTs.
TUScheduler::ASTCache &IdleASTs;
const bool RunSync;
/// Time to wait after an update to see whether another update obsoletes it.
const DebouncePolicy UpdateDebounce;
/// File that ASTWorker is responsible for.
const Path FileName;
const GlobalCompilationDatabase &CDB;
/// Callback invoked when preamble or main file AST is built.
ParsingCallbacks &Callbacks;
/// Latest build preamble for current TU.
std::shared_ptr<const PreambleData> LatestPreamble;
Notification BuiltFirstPreamble;
Semaphore &Barrier;
/// Whether the 'onMainAST' callback ran for the current FileInputs.
bool RanASTCallback = false;
/// Guards members used by both TUScheduler and the worker thread.
mutable std::mutex Mutex;
/// File inputs, currently being used by the worker.
/// Inputs are written and read by the worker thread, compile command can also
/// be consumed by clients of ASTWorker.
std::shared_ptr<const ParseInputs> FileInputs; /* GUARDED_BY(Mutex) */
/// Times of recent AST rebuilds, used for UpdateDebounce computation.
llvm::SmallVector<DebouncePolicy::clock::duration, 8>
RebuildTimes; /* GUARDED_BY(Mutex) */
/// Set to true to signal run() to finish processing.
bool Done; /* GUARDED_BY(Mutex) */
std::deque<Request> Requests; /* GUARDED_BY(Mutex) */
std::queue<Request> PreambleRequests; /* GUARDED_BY(Mutex) */
llvm::Optional<Request> CurrentRequest; /* GUARDED_BY(Mutex) */
mutable std::condition_variable RequestsCV;
/// Guards the callback that publishes results of AST-related computations
/// (diagnostics, highlightings) and file statuses.
std::mutex PublishMu;
// Used to prevent remove document + add document races that lead to
// out-of-order callbacks for publishing results of onMainAST callback.
//
// The lifetime of the old/new ASTWorkers will overlap, but their handles
// don't. When the old handle is destroyed, the old worker will stop reporting
// any results to the user.
bool CanPublishResults = true; /* GUARDED_BY(PublishMu) */
SynchronizedTUStatus Status;
PreambleThread PreamblePeer;
};
/// A smart-pointer-like class that points to an active ASTWorker.
/// In destructor, signals to the underlying ASTWorker that no new requests will
/// be sent and the processing loop may exit (after running all pending
/// requests).
class ASTWorkerHandle {
friend class ASTWorker;
ASTWorkerHandle(std::shared_ptr<ASTWorker> Worker)
: Worker(std::move(Worker)) {
assert(this->Worker);
}
public:
ASTWorkerHandle(const ASTWorkerHandle &) = delete;
ASTWorkerHandle &operator=(const ASTWorkerHandle &) = delete;
ASTWorkerHandle(ASTWorkerHandle &&) = default;
ASTWorkerHandle &operator=(ASTWorkerHandle &&) = default;
~ASTWorkerHandle() {
if (Worker)
Worker->stop();
}
ASTWorker &operator*() {
assert(Worker && "Handle was moved from");
return *Worker;
}
ASTWorker *operator->() {
assert(Worker && "Handle was moved from");
return Worker.get();
}
/// Returns an owning reference to the underlying ASTWorker that can outlive
/// the ASTWorkerHandle. However, no new requests to an active ASTWorker can
/// be schedule via the returned reference, i.e. only reads of the preamble
/// are possible.
std::shared_ptr<const ASTWorker> lock() { return Worker; }
private:
std::shared_ptr<ASTWorker> Worker;
};
ASTWorkerHandle
ASTWorker::create(PathRef FileName, const GlobalCompilationDatabase &CDB,
TUScheduler::ASTCache &IdleASTs, AsyncTaskRunner *Tasks,
Semaphore &Barrier, DebouncePolicy UpdateDebounce,
bool StorePreamblesInMemory, ParsingCallbacks &Callbacks) {
std::shared_ptr<ASTWorker> Worker(
new ASTWorker(FileName, CDB, IdleASTs, Barrier, /*RunSync=*/!Tasks,
UpdateDebounce, StorePreamblesInMemory, Callbacks));
if (Tasks) {
Tasks->runAsync("ASTWorker:" + llvm::sys::path::filename(FileName),
[Worker]() { Worker->run(); });
Tasks->runAsync("PreambleWorker:" + llvm::sys::path::filename(FileName),
[Worker]() { Worker->PreamblePeer.run(); });
}
return ASTWorkerHandle(std::move(Worker));
}
ASTWorker::ASTWorker(PathRef FileName, const GlobalCompilationDatabase &CDB,
TUScheduler::ASTCache &LRUCache, Semaphore &Barrier,
bool RunSync, DebouncePolicy UpdateDebounce,
bool StorePreamblesInMemory, ParsingCallbacks &Callbacks)
: IdleASTs(LRUCache), RunSync(RunSync), UpdateDebounce(UpdateDebounce),
FileName(FileName), CDB(CDB), Callbacks(Callbacks), Barrier(Barrier),
Done(false), Status(FileName, Callbacks),
PreamblePeer(FileName, Callbacks, StorePreamblesInMemory,
// FIXME: Run PreamblePeer asynchronously once ast patching
// is available.
/*RunSync=*/true, Status, *this) {
auto Inputs = std::make_shared<ParseInputs>();
// Set a fallback command because compile command can be accessed before
// `Inputs` is initialized. Other fields are only used after initialization
// from client inputs.
Inputs->CompileCommand = CDB.getFallbackCommand(FileName);
FileInputs = std::move(Inputs);
}
ASTWorker::~ASTWorker() {
// Make sure we remove the cached AST, if any.
IdleASTs.take(this);
#ifndef NDEBUG
std::lock_guard<std::mutex> Lock(Mutex);
assert(Done && "handle was not destroyed");
assert(Requests.empty() && !CurrentRequest &&
"unprocessed requests when destroying ASTWorker");
#endif
}
void ASTWorker::update(ParseInputs Inputs, WantDiagnostics WantDiags) {
std::string TaskName = llvm::formatv("Update ({0})", Inputs.Version);
auto Task = [=]() mutable {
// Get the actual command as `Inputs` does not have a command.
// FIXME: some build systems like Bazel will take time to preparing
// environment to build the file, it would be nice if we could emit a
// "PreparingBuild" status to inform users, it is non-trivial given the
// current implementation.
if (auto Cmd = CDB.getCompileCommand(FileName))
Inputs.CompileCommand = *Cmd;
else
// FIXME: consider using old command if it's not a fallback one.
Inputs.CompileCommand = CDB.getFallbackCommand(FileName);
bool InputsAreTheSame =
std::tie(FileInputs->CompileCommand, FileInputs->Contents) ==
std::tie(Inputs.CompileCommand, Inputs.Contents);
// Cached AST is invalidated.
if (!InputsAreTheSame) {
IdleASTs.take(this);
RanASTCallback = false;
}
// Update current inputs so that subsequent reads can see them.
{
std::lock_guard<std::mutex> Lock(Mutex);
FileInputs = std::make_shared<ParseInputs>(Inputs);
}
log("ASTWorker building file {0} version {1} with command {2}\n[{3}]\n{4}",
FileName, Inputs.Version, Inputs.CompileCommand.Heuristic,
Inputs.CompileCommand.Directory,
llvm::join(Inputs.CompileCommand.CommandLine, " "));
[clangd] Surface errors from command-line parsing Summary: Those errors are exposed at the first character of a file, for a lack of a better place. Previously, all errors were stored inside the AST and report accordingly. However, errors in command-line argument parsing could result in failure to produce the AST, so we need an alternative ways to report those errors. We take the following approach in this patch: - buildCompilerInvocation() now requires an explicit DiagnosticConsumer. - TUScheduler and TestTU now collect the diagnostics produced when parsing command line arguments. If pasing of the AST failed, diagnostics are reported via a new ParsingCallbacks::onFailedAST method. If parsing of the AST succeeded, any errors produced during command-line parsing are stored alongside the AST inside the ParsedAST instance and reported as previously by calling the ParsingCallbacks::onMainAST method; - The client code that uses ClangdServer's DiagnosticConsumer does not need to change, it will receive new diagnostics in the onDiagnosticsReady() callback Errors produced when parsing command-line arguments are collected using the same StoreDiags class that is used to collect all other errors. They are recognized by their location being invalid. IIUC, the location is invalid as there is no source manager at this point, it is created at a later stage. Although technically we might also get diagnostics that mention the command-line arguments FileID with after the source manager was created (and they have valid source locations), we choose to not handle those and they are dropped as not coming from the main file. AFAICT, those diagnostics should always be notes, therefore it's safe to drop them without loosing too much information. Reviewers: kadircet Reviewed By: kadircet Subscribers: nridge, javed.absar, MaskRay, jkorous, arphaman, cfe-commits, gribozavr Tags: #clang Differential Revision: https://reviews.llvm.org/D66759 llvm-svn: 370177
2019-08-28 17:24:55 +08:00
StoreDiags CompilerInvocationDiagConsumer;
std::vector<std::string> CC1Args;
std::unique_ptr<CompilerInvocation> Invocation = buildCompilerInvocation(
Inputs, CompilerInvocationDiagConsumer, &CC1Args);
// Log cc1 args even (especially!) if creating invocation failed.
if (!CC1Args.empty())
vlog("Driver produced command: cc1 {0}", llvm::join(CC1Args, " "));
[clangd] Surface errors from command-line parsing Summary: Those errors are exposed at the first character of a file, for a lack of a better place. Previously, all errors were stored inside the AST and report accordingly. However, errors in command-line argument parsing could result in failure to produce the AST, so we need an alternative ways to report those errors. We take the following approach in this patch: - buildCompilerInvocation() now requires an explicit DiagnosticConsumer. - TUScheduler and TestTU now collect the diagnostics produced when parsing command line arguments. If pasing of the AST failed, diagnostics are reported via a new ParsingCallbacks::onFailedAST method. If parsing of the AST succeeded, any errors produced during command-line parsing are stored alongside the AST inside the ParsedAST instance and reported as previously by calling the ParsingCallbacks::onMainAST method; - The client code that uses ClangdServer's DiagnosticConsumer does not need to change, it will receive new diagnostics in the onDiagnosticsReady() callback Errors produced when parsing command-line arguments are collected using the same StoreDiags class that is used to collect all other errors. They are recognized by their location being invalid. IIUC, the location is invalid as there is no source manager at this point, it is created at a later stage. Although technically we might also get diagnostics that mention the command-line arguments FileID with after the source manager was created (and they have valid source locations), we choose to not handle those and they are dropped as not coming from the main file. AFAICT, those diagnostics should always be notes, therefore it's safe to drop them without loosing too much information. Reviewers: kadircet Reviewed By: kadircet Subscribers: nridge, javed.absar, MaskRay, jkorous, arphaman, cfe-commits, gribozavr Tags: #clang Differential Revision: https://reviews.llvm.org/D66759 llvm-svn: 370177
2019-08-28 17:24:55 +08:00
std::vector<Diag> CompilerInvocationDiags =
CompilerInvocationDiagConsumer.take();
if (!Invocation) {
elog("Could not build CompilerInvocation for file {0}", FileName);
// Remove the old AST if it's still in cache.
IdleASTs.take(this);
RanASTCallback = false;
[clangd] Surface errors from command-line parsing Summary: Those errors are exposed at the first character of a file, for a lack of a better place. Previously, all errors were stored inside the AST and report accordingly. However, errors in command-line argument parsing could result in failure to produce the AST, so we need an alternative ways to report those errors. We take the following approach in this patch: - buildCompilerInvocation() now requires an explicit DiagnosticConsumer. - TUScheduler and TestTU now collect the diagnostics produced when parsing command line arguments. If pasing of the AST failed, diagnostics are reported via a new ParsingCallbacks::onFailedAST method. If parsing of the AST succeeded, any errors produced during command-line parsing are stored alongside the AST inside the ParsedAST instance and reported as previously by calling the ParsingCallbacks::onMainAST method; - The client code that uses ClangdServer's DiagnosticConsumer does not need to change, it will receive new diagnostics in the onDiagnosticsReady() callback Errors produced when parsing command-line arguments are collected using the same StoreDiags class that is used to collect all other errors. They are recognized by their location being invalid. IIUC, the location is invalid as there is no source manager at this point, it is created at a later stage. Although technically we might also get diagnostics that mention the command-line arguments FileID with after the source manager was created (and they have valid source locations), we choose to not handle those and they are dropped as not coming from the main file. AFAICT, those diagnostics should always be notes, therefore it's safe to drop them without loosing too much information. Reviewers: kadircet Reviewed By: kadircet Subscribers: nridge, javed.absar, MaskRay, jkorous, arphaman, cfe-commits, gribozavr Tags: #clang Differential Revision: https://reviews.llvm.org/D66759 llvm-svn: 370177
2019-08-28 17:24:55 +08:00
// Report the diagnostics we collected when parsing the command line.
Callbacks.onFailedAST(FileName, Inputs.Version,
std::move(CompilerInvocationDiags),
[&](llvm::function_ref<void()> Publish) {
// Ensure we only publish results from the worker
// if the file was not removed, making sure there
// are not race conditions.
std::lock_guard<std::mutex> Lock(PublishMu);
if (CanPublishResults)
Publish();
});
// Make sure anyone waiting for the preamble gets notified it could not be
// built.
BuiltFirstPreamble.notify();
return;
}
PreamblePeer.update(std::move(Invocation), std::move(Inputs),
std::move(CompilerInvocationDiags), WantDiags);
return;
};
startTask(TaskName, std::move(Task), WantDiags, TUScheduler::NoInvalidation);
}
void ASTWorker::runWithAST(
llvm::StringRef Name,
llvm::unique_function<void(llvm::Expected<InputsAndAST>)> Action,
TUScheduler::ASTActionInvalidation Invalidation) {
auto Task = [=, Action = std::move(Action)]() mutable {
if (isCancelled())
return Action(llvm::make_error<CancelledError>());
llvm::Optional<std::unique_ptr<ParsedAST>> AST = IdleASTs.take(this);
auto CurrentInputs = getCurrentFileInputs();
if (!AST) {
[clangd] Surface errors from command-line parsing Summary: Those errors are exposed at the first character of a file, for a lack of a better place. Previously, all errors were stored inside the AST and report accordingly. However, errors in command-line argument parsing could result in failure to produce the AST, so we need an alternative ways to report those errors. We take the following approach in this patch: - buildCompilerInvocation() now requires an explicit DiagnosticConsumer. - TUScheduler and TestTU now collect the diagnostics produced when parsing command line arguments. If pasing of the AST failed, diagnostics are reported via a new ParsingCallbacks::onFailedAST method. If parsing of the AST succeeded, any errors produced during command-line parsing are stored alongside the AST inside the ParsedAST instance and reported as previously by calling the ParsingCallbacks::onMainAST method; - The client code that uses ClangdServer's DiagnosticConsumer does not need to change, it will receive new diagnostics in the onDiagnosticsReady() callback Errors produced when parsing command-line arguments are collected using the same StoreDiags class that is used to collect all other errors. They are recognized by their location being invalid. IIUC, the location is invalid as there is no source manager at this point, it is created at a later stage. Although technically we might also get diagnostics that mention the command-line arguments FileID with after the source manager was created (and they have valid source locations), we choose to not handle those and they are dropped as not coming from the main file. AFAICT, those diagnostics should always be notes, therefore it's safe to drop them without loosing too much information. Reviewers: kadircet Reviewed By: kadircet Subscribers: nridge, javed.absar, MaskRay, jkorous, arphaman, cfe-commits, gribozavr Tags: #clang Differential Revision: https://reviews.llvm.org/D66759 llvm-svn: 370177
2019-08-28 17:24:55 +08:00
StoreDiags CompilerInvocationDiagConsumer;
std::unique_ptr<CompilerInvocation> Invocation = buildCompilerInvocation(
*CurrentInputs, CompilerInvocationDiagConsumer);
// Try rebuilding the AST.
vlog("ASTWorker rebuilding evicted AST to run {0}: {1} version {2}", Name,
FileName, CurrentInputs->Version);
// FIXME: We might need to build a patched ast once preamble thread starts
// running async. Currently getPossiblyStalePreamble below will always
// return a compatible preamble as ASTWorker::update blocks.
llvm::Optional<ParsedAST> NewAST =
Invocation ? buildAST(FileName, std::move(Invocation),
CompilerInvocationDiagConsumer.take(),
*CurrentInputs, getPossiblyStalePreamble())
: None;
AST = NewAST ? std::make_unique<ParsedAST>(std::move(*NewAST)) : nullptr;
}
// Make sure we put the AST back into the LRU cache.
auto _ = llvm::make_scope_exit(
[&AST, this]() { IdleASTs.put(this, std::move(*AST)); });
// Run the user-provided action.
if (!*AST)
return Action(llvm::make_error<llvm::StringError>(
"invalid AST", llvm::errc::invalid_argument));
vlog("ASTWorker running {0} on version {2} of {1}", Name, FileName,
CurrentInputs->Version);
Action(InputsAndAST{*CurrentInputs, **AST});
};
startTask(Name, std::move(Task), /*UpdateType=*/None, Invalidation);
}
void PreambleThread::build(Request Req) {
assert(Req.CI && "Got preamble request with null compiler invocation");
const ParseInputs &Inputs = Req.Inputs;
Status.update([&](TUStatus &Status) {
Status.PreambleActivity = PreambleAction::Building;
});
auto _ = llvm::make_scope_exit([this, &Req] {
ASTPeer.updatePreamble(std::move(Req.CI), std::move(Req.Inputs),
LatestBuild, std::move(Req.CIDiags),
std::move(Req.WantDiags));
});
if (!LatestBuild || Inputs.ForceRebuild) {
vlog("Building first preamble for {0} version {1}", FileName,
Inputs.Version);
} else if (isPreambleCompatible(*LatestBuild, Inputs, FileName, *Req.CI)) {
vlog("Reusing preamble version {0} for version {1} of {2}",
LatestBuild->Version, Inputs.Version, FileName);
return;
} else {
vlog("Rebuilding invalidated preamble for {0} version {1} (previous was "
"version {2})",
FileName, Inputs.Version, LatestBuild->Version);
}
LatestBuild = clang::clangd::buildPreamble(
FileName, *Req.CI, Inputs, StoreInMemory,
[this, Version(Inputs.Version)](ASTContext &Ctx,
std::shared_ptr<clang::Preprocessor> PP,
const CanonicalIncludes &CanonIncludes) {
Callbacks.onPreambleAST(FileName, Version, Ctx, std::move(PP),
CanonIncludes);
});
}
void ASTWorker::updatePreamble(std::unique_ptr<CompilerInvocation> CI,
ParseInputs PI,
std::shared_ptr<const PreambleData> Preamble,
std::vector<Diag> CIDiags,
WantDiagnostics WantDiags) {
std::string TaskName = llvm::formatv("Build AST for ({0})", PI.Version);
// Store preamble and build diagnostics with new preamble if requested.
auto Task = [this, Preamble = std::move(Preamble), CI = std::move(CI),
PI = std::move(PI), CIDiags = std::move(CIDiags),
WantDiags = std::move(WantDiags)]() mutable {
// Update the preamble inside ASTWorker queue to ensure atomicity. As a task
// running inside ASTWorker assumes internals won't change until it
// finishes.
if (Preamble != LatestPreamble) {
// Cached AST is no longer valid.
IdleASTs.take(this);
RanASTCallback = false;
std::lock_guard<std::mutex> Lock(Mutex);
// LatestPreamble might be the last reference to old preamble, do not
// trigger destructor while holding the lock.
std::swap(LatestPreamble, Preamble);
}
// Give up our ownership to old preamble before starting expensive AST
// build.
Preamble.reset();
BuiltFirstPreamble.notify();
// We only need to build the AST if diagnostics were requested.
if (WantDiags == WantDiagnostics::No)
return;
// Report diagnostics with the new preamble to ensure progress. Otherwise
// diagnostics might get stale indefinitely if user keeps invalidating the
// preamble.
generateDiagnostics(std::move(CI), std::move(PI), std::move(CIDiags));
};
if (RunSync) {
Task();
return;
}
{
std::lock_guard<std::mutex> Lock(Mutex);
PreambleRequests.push({std::move(Task), std::move(TaskName),
steady_clock::now(), Context::current().clone(),
llvm::None, TUScheduler::NoInvalidation, nullptr});
}
RequestsCV.notify_all();
}
void ASTWorker::generateDiagnostics(
std::unique_ptr<CompilerInvocation> Invocation, ParseInputs Inputs,
std::vector<Diag> CIDiags) {
assert(Invocation);
// No need to rebuild the AST if we won't send the diagnostics.
{
std::lock_guard<std::mutex> Lock(PublishMu);
if (!CanPublishResults)
return;
}
// Used to check whether we can update AST cache.
bool InputsAreLatest =
std::tie(FileInputs->CompileCommand, FileInputs->Contents) ==
std::tie(Inputs.CompileCommand, Inputs.Contents);
// Take a shortcut and don't report the diagnostics, since they should be the
// same. All the clients should handle the lack of OnUpdated() call anyway to
// handle empty result from buildAST.
// FIXME: the AST could actually change if non-preamble includes changed,
// but we choose to ignore it.
if (InputsAreLatest && RanASTCallback)
return;
// Get the AST for diagnostics, either build it or use the cached one.
std::string TaskName = llvm::formatv("Build AST ({0})", Inputs.Version);
Status.update([&](TUStatus &Status) {
Status.ASTActivity.K = ASTAction::Building;
Status.ASTActivity.Name = std::move(TaskName);
});
// We might be able to reuse the last we've built for a read request.
// FIXME: It might be better to not reuse this AST. That way queued AST builds
// won't be required for diags.
llvm::Optional<std::unique_ptr<ParsedAST>> AST = IdleASTs.take(this);
if (!AST) {
auto RebuildStartTime = DebouncePolicy::clock::now();
llvm::Optional<ParsedAST> NewAST = buildAST(
FileName, std::move(Invocation), CIDiags, Inputs, LatestPreamble);
auto RebuildDuration = DebouncePolicy::clock::now() - RebuildStartTime;
// Try to record the AST-build time, to inform future update debouncing.
// This is best-effort only: if the lock is held, don't bother.
std::unique_lock<std::mutex> Lock(Mutex, std::try_to_lock);
if (Lock.owns_lock()) {
// Do not let RebuildTimes grow beyond its small-size (i.e.
// capacity).
if (RebuildTimes.size() == RebuildTimes.capacity())
RebuildTimes.erase(RebuildTimes.begin());
RebuildTimes.push_back(RebuildDuration);
Lock.unlock();
}
Status.update([&](TUStatus &Status) {
Status.Details.ReuseAST = false;
Status.Details.BuildFailed = !NewAST.hasValue();
});
AST = NewAST ? std::make_unique<ParsedAST>(std::move(*NewAST)) : nullptr;
} else {
assert(InputsAreLatest && !RanASTCallback &&
"forgot to invalidate cached ast?");
log("Skipping rebuild of the AST for {0}, inputs are the same.", FileName);
Status.update([](TUStatus &Status) {
Status.Details.ReuseAST = true;
Status.Details.BuildFailed = false;
});
}
// Publish diagnostics.
auto RunPublish = [&](llvm::function_ref<void()> Publish) {
// Ensure we only publish results from the worker if the file was not
// removed, making sure there are not race conditions.
std::lock_guard<std::mutex> Lock(PublishMu);
if (CanPublishResults)
Publish();
};
if (*AST) {
trace::Span Span("Running main AST callback");
Callbacks.onMainAST(FileName, **AST, RunPublish);
} else {
// Failed to build the AST, at least report diagnostics from the
// command line if there were any.
// FIXME: we might have got more errors while trying to build the
// AST, surface them too.
Callbacks.onFailedAST(FileName, Inputs.Version, CIDiags, RunPublish);
}
// AST might've been built for an older version of the source, as ASTWorker
// queue raced ahead while we were waiting on the preamble. In that case the
// queue can't reuse the AST.
if (InputsAreLatest) {
RanASTCallback = *AST != nullptr;
IdleASTs.put(this, std::move(*AST));
}
}
std::shared_ptr<const PreambleData>
ASTWorker::getPossiblyStalePreamble() const {
std::lock_guard<std::mutex> Lock(Mutex);
return LatestPreamble;
}
void ASTWorker::getCurrentPreamble(
llvm::unique_function<void(std::shared_ptr<const PreambleData>)> Callback) {
// We could just call startTask() to throw the read on the queue, knowing
// it will run after any updates. But we know this task is cheap, so to
// improve latency we cheat: insert it on the queue after the last update.
std::unique_lock<std::mutex> Lock(Mutex);
auto LastUpdate =
std::find_if(Requests.rbegin(), Requests.rend(),
[](const Request &R) { return R.UpdateType.hasValue(); });
// If there were no writes in the queue, and CurrentRequest is not a write,
// the preamble is ready now.
if (LastUpdate == Requests.rend() &&
(!CurrentRequest || CurrentRequest->UpdateType.hasValue())) {
Lock.unlock();
return Callback(getPossiblyStalePreamble());
}
assert(!RunSync && "Running synchronously, but queue is non-empty!");
Requests.insert(LastUpdate.base(),
Request{[Callback = std::move(Callback), this]() mutable {
Callback(getPossiblyStalePreamble());
},
"GetPreamble", steady_clock::now(),
Context::current().clone(),
/*UpdateType=*/None,
/*InvalidationPolicy=*/TUScheduler::NoInvalidation,
/*Invalidate=*/nullptr});
Lock.unlock();
RequestsCV.notify_all();
}
void ASTWorker::waitForFirstPreamble() const { BuiltFirstPreamble.wait(); }
std::shared_ptr<const ParseInputs> ASTWorker::getCurrentFileInputs() const {
std::unique_lock<std::mutex> Lock(Mutex);
return FileInputs;
}
tooling::CompileCommand ASTWorker::getCurrentCompileCommand() const {
std::unique_lock<std::mutex> Lock(Mutex);
return FileInputs->CompileCommand;
}
std::size_t ASTWorker::getUsedBytes() const {
// Note that we don't report the size of ASTs currently used for processing
// the in-flight requests. We used this information for debugging purposes
// only, so this should be fine.
std::size_t Result = IdleASTs.getUsedBytes(this);
if (auto Preamble = getPossiblyStalePreamble())
Result += Preamble->Preamble.getSize();
return Result;
}
bool ASTWorker::isASTCached() const { return IdleASTs.getUsedBytes(this) != 0; }
void ASTWorker::stop() {
{
std::lock_guard<std::mutex> Lock(PublishMu);
CanPublishResults = false;
}
{
std::lock_guard<std::mutex> Lock(Mutex);
assert(!Done && "stop() called twice");
Done = true;
}
// We are no longer going to build any preambles, let the waiters know that.
BuiltFirstPreamble.notify();
PreamblePeer.stop();
Status.stop();
RequestsCV.notify_all();
}
void ASTWorker::startTask(llvm::StringRef Name,
llvm::unique_function<void()> Task,
llvm::Optional<WantDiagnostics> UpdateType,
TUScheduler::ASTActionInvalidation Invalidation) {
if (RunSync) {
assert(!Done && "running a task after stop()");
trace::Span Tracer(Name + ":" + llvm::sys::path::filename(FileName));
Task();
return;
}
{
std::lock_guard<std::mutex> Lock(Mutex);
assert(!Done && "running a task after stop()");
// Cancel any requests invalidated by this request.
if (UpdateType) {
for (auto &R : llvm::reverse(Requests)) {
if (R.InvalidationPolicy == TUScheduler::InvalidateOnUpdate)
R.Invalidate();
if (R.UpdateType)
break; // Older requests were already invalidated by the older update.
}
}
// Allow this request to be cancelled if invalidated.
Context Ctx = Context::current().derive(kFileBeingProcessed, FileName);
Canceler Invalidate = nullptr;
if (Invalidation) {
WithContext WC(std::move(Ctx));
std::tie(Ctx, Invalidate) = cancelableTask();
}
Requests.push_back({std::move(Task), std::string(Name), steady_clock::now(),
std::move(Ctx), UpdateType, Invalidation,
std::move(Invalidate)});
}
RequestsCV.notify_all();
}
void ASTWorker::run() {
while (true) {
{
std::unique_lock<std::mutex> Lock(Mutex);
assert(!CurrentRequest && "A task is already running, multiple workers?");
for (auto Wait = scheduleLocked(); !Wait.expired();
Wait = scheduleLocked()) {
assert(PreambleRequests.empty() &&
"Preamble updates should be scheduled immediately");
if (Done) {
if (Requests.empty())
return;
else // Even though Done is set, finish pending requests.
break; // However, skip delays to shutdown fast.
}
// Tracing: we have a next request, attribute this sleep to it.
llvm::Optional<WithContext> Ctx;
llvm::Optional<trace::Span> Tracer;
if (!Requests.empty()) {
Ctx.emplace(Requests.front().Ctx.clone());
Tracer.emplace("Debounce");
SPAN_ATTACH(*Tracer, "next_request", Requests.front().Name);
if (!(Wait == Deadline::infinity())) {
Status.update([&](TUStatus &Status) {
Status.ASTActivity.K = ASTAction::Queued;
Status.ASTActivity.Name = Requests.front().Name;
});
SPAN_ATTACH(*Tracer, "sleep_ms",
std::chrono::duration_cast<std::chrono::milliseconds>(
Wait.time() - steady_clock::now())
.count());
}
}
wait(Lock, RequestsCV, Wait);
}
// Any request in ReceivedPreambles is at least as old as the
// Requests.front(), so prefer them first to preserve LSP order.
if (!PreambleRequests.empty()) {
CurrentRequest = std::move(PreambleRequests.front());
PreambleRequests.pop();
} else {
CurrentRequest = std::move(Requests.front());
Requests.pop_front();
}
} // unlock Mutex
// It is safe to perform reads to CurrentRequest without holding the lock as
// only writer is also this thread.
{
std::unique_lock<Semaphore> Lock(Barrier, std::try_to_lock);
if (!Lock.owns_lock()) {
Status.update([&](TUStatus &Status) {
Status.ASTActivity.K = ASTAction::Queued;
Status.ASTActivity.Name = CurrentRequest->Name;
});
Lock.lock();
}
WithContext Guard(std::move(CurrentRequest->Ctx));
trace::Span Tracer(CurrentRequest->Name);
Status.update([&](TUStatus &Status) {
Status.ASTActivity.K = ASTAction::RunningAction;
Status.ASTActivity.Name = CurrentRequest->Name;
});
CurrentRequest->Action();
}
bool IsEmpty = false;
{
std::lock_guard<std::mutex> Lock(Mutex);
CurrentRequest.reset();
IsEmpty = Requests.empty() && PreambleRequests.empty();
}
if (IsEmpty) {
Status.update([&](TUStatus &Status) {
Status.ASTActivity.K = ASTAction::Idle;
Status.ASTActivity.Name = "";
});
}
RequestsCV.notify_all();
}
}
Deadline ASTWorker::scheduleLocked() {
// Process new preambles immediately.
if (!PreambleRequests.empty())
return Deadline::zero();
if (Requests.empty())
return Deadline::infinity(); // Wait for new requests.
// Handle cancelled requests first so the rest of the scheduler doesn't.
for (auto I = Requests.begin(), E = Requests.end(); I != E; ++I) {
if (!isCancelled(I->Ctx)) {
// Cancellations after the first read don't affect current scheduling.
if (I->UpdateType == None)
break;
continue;
}
// Cancelled reads are moved to the front of the queue and run immediately.
if (I->UpdateType == None) {
Request R = std::move(*I);
Requests.erase(I);
Requests.push_front(std::move(R));
return Deadline::zero();
}
// Cancelled updates are downgraded to auto-diagnostics, and may be elided.
if (I->UpdateType == WantDiagnostics::Yes)
I->UpdateType = WantDiagnostics::Auto;
}
while (shouldSkipHeadLocked()) {
vlog("ASTWorker skipping {0} for {1}", Requests.front().Name, FileName);
Requests.pop_front();
}
assert(!Requests.empty() && "skipped the whole queue");
// Some updates aren't dead yet, but never end up being used.
// e.g. the first keystroke is live until obsoleted by the second.
// We debounce "maybe-unused" writes, sleeping in case they become dead.
// But don't delay reads (including updates where diagnostics are needed).
for (const auto &R : Requests)
if (R.UpdateType == None || R.UpdateType == WantDiagnostics::Yes)
return Deadline::zero();
// Front request needs to be debounced, so determine when we're ready.
Deadline D(Requests.front().AddTime + UpdateDebounce.compute(RebuildTimes));
return D;
}
// Returns true if Requests.front() is a dead update that can be skipped.
bool ASTWorker::shouldSkipHeadLocked() const {
assert(!Requests.empty());
auto Next = Requests.begin();
auto UpdateType = Next->UpdateType;
if (!UpdateType) // Only skip updates.
return false;
++Next;
// An update is live if its AST might still be read.
// That is, if it's not immediately followed by another update.
if (Next == Requests.end() || !Next->UpdateType)
return false;
// The other way an update can be live is if its diagnostics might be used.
switch (*UpdateType) {
case WantDiagnostics::Yes:
return false; // Always used.
case WantDiagnostics::No:
return true; // Always dead.
case WantDiagnostics::Auto:
// Used unless followed by an update that generates diagnostics.
for (; Next != Requests.end(); ++Next)
if (Next->UpdateType == WantDiagnostics::Yes ||
Next->UpdateType == WantDiagnostics::Auto)
return true; // Prefer later diagnostics.
return false;
}
llvm_unreachable("Unknown WantDiagnostics");
}
bool ASTWorker::blockUntilIdle(Deadline Timeout) const {
std::unique_lock<std::mutex> Lock(Mutex);
return wait(Lock, RequestsCV, Timeout, [&] {
return PreambleRequests.empty() && Requests.empty() && !CurrentRequest;
});
}
// Render a TUAction to a user-facing string representation.
// TUAction represents clangd-internal states, we don't intend to expose them
// to users (say C++ programmers) directly to avoid confusion, we use terms that
// are familiar by C++ programmers.
std::string renderTUAction(const PreambleAction PA, const ASTAction &AA) {
llvm::SmallVector<std::string, 2> Result;
switch (PA) {
case PreambleAction::Building:
Result.push_back("parsing includes");
break;
case PreambleAction::Idle:
// We handle idle specially below.
break;
}
switch (AA.K) {
case ASTAction::Queued:
Result.push_back("file is queued");
break;
case ASTAction::RunningAction:
Result.push_back("running " + AA.Name);
break;
case ASTAction::Building:
Result.push_back("parsing main file");
break;
case ASTAction::Idle:
// We handle idle specially below.
break;
}
if (Result.empty())
return "idle";
return llvm::join(Result, ",");
}
} // namespace
unsigned getDefaultAsyncThreadsCount() {
[Support] On Windows, ensure hardware_concurrency() extends to all CPU sockets and all NUMA groups The goal of this patch is to maximize CPU utilization on multi-socket or high core count systems, so that parallel computations such as LLD/ThinLTO can use all hardware threads in the system. Before this patch, on Windows, a maximum of 64 hardware threads could be used at most, in some cases dispatched only on one CPU socket. == Background == Windows doesn't have a flat cpu_set_t like Linux. Instead, it projects hardware CPUs (or NUMA nodes) to applications through a concept of "processor groups". A "processor" is the smallest unit of execution on a CPU, that is, an hyper-thread if SMT is active; a core otherwise. There's a limit of 32-bit processors on older 32-bit versions of Windows, which later was raised to 64-processors with 64-bit versions of Windows. This limit comes from the affinity mask, which historically is represented by the sizeof(void*). Consequently, the concept of "processor groups" was introduced for dealing with systems with more than 64 hyper-threads. By default, the Windows OS assigns only one "processor group" to each starting application, in a round-robin manner. If the application wants to use more processors, it needs to programmatically enable it, by assigning threads to other "processor groups". This also means that affinity cannot cross "processor group" boundaries; one can only specify a "preferred" group on start-up, but the application is free to allocate more groups if it wants to. This creates a peculiar situation, where newer CPUs like the AMD EPYC 7702P (64-cores, 128-hyperthreads) are projected by the OS as two (2) "processor groups". This means that by default, an application can only use half of the cores. This situation could only get worse in the years to come, as dies with more cores will appear on the market. == The problem == The heavyweight_hardware_concurrency() API was introduced so that only *one hardware thread per core* was used. Once that API returns, that original intention is lost, only the number of threads is retained. Consider a situation, on Windows, where the system has 2 CPU sockets, 18 cores each, each core having 2 hyper-threads, for a total of 72 hyper-threads. Both heavyweight_hardware_concurrency() and hardware_concurrency() currently return 36, because on Windows they are simply wrappers over std::thread::hardware_concurrency() -- which can only return processors from the current "processor group". == The changes in this patch == To solve this situation, we capture (and retain) the initial intention until the point of usage, through a new ThreadPoolStrategy class. The number of threads to use is deferred as late as possible, until the moment where the std::threads are created (ThreadPool in the case of ThinLTO). When using hardware_concurrency(), setting ThreadCount to 0 now means to use all the possible hardware CPU (SMT) threads. Providing a ThreadCount above to the maximum number of threads will have no effect, the maximum will be used instead. The heavyweight_hardware_concurrency() is similar to hardware_concurrency(), except that only one thread per hardware *core* will be used. When LLVM_ENABLE_THREADS is OFF, the threading APIs will always return 1, to ensure any caller loops will be exercised at least once. Differential Revision: https://reviews.llvm.org/D71775
2020-02-14 11:49:57 +08:00
return llvm::heavyweight_hardware_concurrency().compute_thread_count();
}
FileStatus TUStatus::render(PathRef File) const {
FileStatus FStatus;
FStatus.uri = URIForFile::canonicalize(File, /*TUPath=*/File);
FStatus.state = renderTUAction(PreambleActivity, ASTActivity);
return FStatus;
}
struct TUScheduler::FileData {
/// Latest inputs, passed to TUScheduler::update().
std::string Contents;
ASTWorkerHandle Worker;
};
TUScheduler::TUScheduler(const GlobalCompilationDatabase &CDB,
const Options &Opts,
std::unique_ptr<ParsingCallbacks> Callbacks)
: CDB(CDB), StorePreamblesInMemory(Opts.StorePreamblesInMemory),
Callbacks(Callbacks ? move(Callbacks)
: std::make_unique<ParsingCallbacks>()),
Barrier(Opts.AsyncThreadsCount),
IdleASTs(
std::make_unique<ASTCache>(Opts.RetentionPolicy.MaxRetainedASTs)),
UpdateDebounce(Opts.UpdateDebounce) {
if (0 < Opts.AsyncThreadsCount) {
PreambleTasks.emplace();
WorkerThreads.emplace();
}
}
TUScheduler::~TUScheduler() {
// Notify all workers that they need to stop.
Files.clear();
// Wait for all in-flight tasks to finish.
if (PreambleTasks)
PreambleTasks->wait();
if (WorkerThreads)
WorkerThreads->wait();
}
bool TUScheduler::blockUntilIdle(Deadline D) const {
for (auto &File : Files)
if (!File.getValue()->Worker->blockUntilIdle(D))
return false;
if (PreambleTasks)
if (!PreambleTasks->wait(D))
return false;
return true;
}
bool TUScheduler::update(PathRef File, ParseInputs Inputs,
WantDiagnostics WantDiags) {
std::unique_ptr<FileData> &FD = Files[File];
bool NewFile = FD == nullptr;
if (!FD) {
// Create a new worker to process the AST-related tasks.
ASTWorkerHandle Worker = ASTWorker::create(
File, CDB, *IdleASTs,
WorkerThreads ? WorkerThreads.getPointer() : nullptr, Barrier,
UpdateDebounce, StorePreamblesInMemory, *Callbacks);
FD = std::unique_ptr<FileData>(
new FileData{Inputs.Contents, std::move(Worker)});
} else {
FD->Contents = Inputs.Contents;
}
FD->Worker->update(std::move(Inputs), WantDiags);
return NewFile;
}
void TUScheduler::remove(PathRef File) {
bool Removed = Files.erase(File);
if (!Removed)
elog("Trying to remove file from TUScheduler that is not tracked: {0}",
File);
}
llvm::StringMap<std::string> TUScheduler::getAllFileContents() const {
llvm::StringMap<std::string> Results;
for (auto &It : Files)
Results.try_emplace(It.getKey(), It.getValue()->Contents);
return Results;
}
void TUScheduler::run(llvm::StringRef Name,
llvm::unique_function<void()> Action) {
if (!PreambleTasks)
return Action();
PreambleTasks->runAsync(Name, [this, Ctx = Context::current().clone(),
Action = std::move(Action)]() mutable {
std::lock_guard<Semaphore> BarrierLock(Barrier);
WithContext WC(std::move(Ctx));
Action();
});
}
void TUScheduler::runWithAST(
llvm::StringRef Name, PathRef File,
llvm::unique_function<void(llvm::Expected<InputsAndAST>)> Action,
TUScheduler::ASTActionInvalidation Invalidation) {
auto It = Files.find(File);
if (It == Files.end()) {
Action(llvm::make_error<LSPError>(
"trying to get AST for non-added document", ErrorCode::InvalidParams));
return;
}
It->second->Worker->runWithAST(Name, std::move(Action), Invalidation);
}
void TUScheduler::runWithPreamble(llvm::StringRef Name, PathRef File,
PreambleConsistency Consistency,
Callback<InputsAndPreamble> Action) {
auto It = Files.find(File);
if (It == Files.end()) {
Action(llvm::make_error<LSPError>(
"trying to get preamble for non-added document",
ErrorCode::InvalidParams));
return;
}
if (!PreambleTasks) {
trace::Span Tracer(Name);
SPAN_ATTACH(Tracer, "file", File);
std::shared_ptr<const PreambleData> Preamble =
It->second->Worker->getPossiblyStalePreamble();
Action(InputsAndPreamble{It->second->Contents,
It->second->Worker->getCurrentCompileCommand(),
Preamble.get()});
return;
}
// Future is populated if the task needs a specific preamble.
std::future<std::shared_ptr<const PreambleData>> ConsistentPreamble;
// FIXME: Currently this only holds because ASTWorker blocks after issuing a
// preamble build. Get rid of consistent reads or make them build on the
// calling thread instead.
if (Consistency == Consistent) {
std::promise<std::shared_ptr<const PreambleData>> Promise;
ConsistentPreamble = Promise.get_future();
It->second->Worker->getCurrentPreamble(
[Promise = std::move(Promise)](
std::shared_ptr<const PreambleData> Preamble) mutable {
Promise.set_value(std::move(Preamble));
});
}
std::shared_ptr<const ASTWorker> Worker = It->second->Worker.lock();
auto Task =
[Worker, Consistency, Name = Name.str(), File = File.str(),
Contents = It->second->Contents,
Command = Worker->getCurrentCompileCommand(),
Ctx = Context::current().derive(kFileBeingProcessed, std::string(File)),
ConsistentPreamble = std::move(ConsistentPreamble),
Action = std::move(Action), this]() mutable {
std::shared_ptr<const PreambleData> Preamble;
if (ConsistentPreamble.valid()) {
Preamble = ConsistentPreamble.get();
} else {
if (Consistency != PreambleConsistency::StaleOrAbsent) {
// Wait until the preamble is built for the first time, if preamble
// is required. This avoids extra work of processing the preamble
// headers in parallel multiple times.
Worker->waitForFirstPreamble();
}
Preamble = Worker->getPossiblyStalePreamble();
}
std::lock_guard<Semaphore> BarrierLock(Barrier);
WithContext Guard(std::move(Ctx));
trace::Span Tracer(Name);
SPAN_ATTACH(Tracer, "file", File);
Action(InputsAndPreamble{Contents, Command, Preamble.get()});
};
PreambleTasks->runAsync("task:" + llvm::sys::path::filename(File),
std::move(Task));
}
std::vector<std::pair<Path, std::size_t>>
TUScheduler::getUsedBytesPerFile() const {
std::vector<std::pair<Path, std::size_t>> Result;
Result.reserve(Files.size());
for (auto &&PathAndFile : Files)
Result.push_back({std::string(PathAndFile.first()),
PathAndFile.second->Worker->getUsedBytes()});
return Result;
}
std::vector<Path> TUScheduler::getFilesWithCachedAST() const {
std::vector<Path> Result;
for (auto &&PathAndFile : Files) {
if (!PathAndFile.second->Worker->isASTCached())
continue;
Result.push_back(std::string(PathAndFile.first()));
}
return Result;
}
DebouncePolicy::clock::duration
DebouncePolicy::compute(llvm::ArrayRef<clock::duration> History) const {
assert(Min <= Max && "Invalid policy");
if (History.empty())
return Max; // Arbitrary.
// Base the result on the median rebuild.
// nth_element needs a mutable array, take the chance to bound the data size.
History = History.take_back(15);
llvm::SmallVector<clock::duration, 15> Recent(History.begin(), History.end());
auto Median = Recent.begin() + Recent.size() / 2;
std::nth_element(Recent.begin(), Median, Recent.end());
clock::duration Target =
std::chrono::duration_cast<clock::duration>(RebuildRatio * *Median);
if (Target > Max)
return Max;
if (Target < Min)
return Min;
return Target;
}
DebouncePolicy DebouncePolicy::fixed(clock::duration T) {
DebouncePolicy P;
P.Min = P.Max = T;
return P;
}
} // namespace clangd
} // namespace clang