[clangd] Config: compile Fragment -> CompiledFragment -> Config

Subscribers: mgorny, ilya-biryukov, MaskRay, jkorous, arphaman, kadircet, usaxena95, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D82612
This commit is contained in:
Sam McCall 2020-06-26 01:49:53 +02:00
parent a8e582c830
commit f12cd99c44
9 changed files with 411 additions and 67 deletions

View File

@ -37,6 +37,7 @@ add_clang_library(clangDaemon
CompileCommands.cpp
Compiler.cpp
Config.cpp
ConfigCompile.cpp
ConfigYAML.cpp
Diagnostics.cpp
DraftStore.cpp

View File

@ -0,0 +1,156 @@
//===--- ConfigCompile.cpp - Translating Fragments into Config ------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Fragments are applied to Configs in two steps:
//
// 1. (When the fragment is first loaded)
// FragmentCompiler::compile() traverses the Fragment and creates
// function objects that know how to apply the configuration.
// 2. (Every time a config is required)
// CompiledFragment() executes these functions to populate the Config.
//
// Work could be split between these steps in different ways. We try to
// do as much work as possible in the first step. For example, regexes are
// compiled in stage 1 and captured by the apply function. This is because:
//
// - it's more efficient, as the work done in stage 1 must only be done once
// - problems can be reported in stage 1, in stage 2 we must silently recover
//
//===----------------------------------------------------------------------===//
#include "Config.h"
#include "ConfigFragment.h"
#include "support/Logger.h"
#include "support/Trace.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/Regex.h"
#include "llvm/Support/SMLoc.h"
#include "llvm/Support/SourceMgr.h"
namespace clang {
namespace clangd {
namespace config {
namespace {
struct CompiledFragmentImpl {
// The independent conditions to check before using settings from this config.
// The following fragment has *two* conditions:
// If: { Platform: [mac, linux], PathMatch: foo/.* }
// All of them must be satisfied: the platform and path conditions are ANDed.
// The OR logic for the platform condition is implemented inside the function.
std::vector<llvm::unique_function<bool(const Params &) const>> Conditions;
// Mutations that this fragment will apply to the configuration.
// These are invoked only if the conditions are satisfied.
std::vector<llvm::unique_function<void(Config &) const>> Apply;
bool operator()(const Params &P, Config &C) const {
for (const auto &C : Conditions) {
if (!C(P)) {
dlog("Config fragment {0}: condition not met", this);
return false;
}
}
dlog("Config fragment {0}: applying {1} rules", this, Apply.size());
for (const auto &A : Apply)
A(C);
return true;
}
};
// Wrapper around condition compile() functions to reduce arg-passing.
struct FragmentCompiler {
CompiledFragmentImpl &Out;
DiagnosticCallback Diagnostic;
llvm::SourceMgr *SourceMgr;
llvm::Optional<llvm::Regex> compileRegex(const Located<std::string> &Text) {
std::string Anchored = "^(" + *Text + ")$";
llvm::Regex Result(Anchored);
std::string RegexError;
if (!Result.isValid(RegexError)) {
diag(Error, "Invalid regex " + Anchored + ": " + RegexError, Text.Range);
return llvm::None;
}
return Result;
}
void compile(Fragment &&F) {
compile(std::move(F.If));
compile(std::move(F.CompileFlags));
}
void compile(Fragment::IfBlock &&F) {
if (F.HasUnrecognizedCondition)
Out.Conditions.push_back([&](const Params &) { return false; });
auto PathMatch = std::make_unique<std::vector<llvm::Regex>>();
for (auto &Entry : F.PathMatch) {
if (auto RE = compileRegex(Entry))
PathMatch->push_back(std::move(*RE));
}
if (!PathMatch->empty()) {
Out.Conditions.push_back(
[PathMatch(std::move(PathMatch))](const Params &P) {
if (P.Path.empty())
return false;
return llvm::any_of(*PathMatch, [&](const llvm::Regex &RE) {
return RE.match(P.Path);
});
});
}
}
void compile(Fragment::CompileFlagsBlock &&F) {
if (!F.Add.empty()) {
std::vector<std::string> Add;
for (auto &A : F.Add)
Add.push_back(std::move(*A));
Out.Apply.push_back([Add(std::move(Add))](Config &C) {
C.CompileFlags.Edits.push_back([Add](std::vector<std::string> &Args) {
Args.insert(Args.end(), Add.begin(), Add.end());
});
});
}
}
constexpr static llvm::SourceMgr::DiagKind Error = llvm::SourceMgr::DK_Error;
void diag(llvm::SourceMgr::DiagKind Kind, llvm::StringRef Message,
llvm::SMRange Range) {
if (Range.isValid() && SourceMgr != nullptr)
Diagnostic(SourceMgr->GetMessage(Range.Start, Kind, Message, Range));
else
Diagnostic(llvm::SMDiagnostic("", Kind, Message));
}
};
} // namespace
CompiledFragment Fragment::compile(DiagnosticCallback D) && {
llvm::StringRef ConfigFile = "<unknown>";
std::pair<unsigned, unsigned> LineCol = {0, 0};
if (auto *SM = Source.Manager.get()) {
unsigned BufID = SM->getMainFileID();
LineCol = SM->getLineAndColumn(Source.Location, BufID);
ConfigFile = SM->getBufferInfo(BufID).Buffer->getBufferIdentifier();
}
trace::Span Tracer("ConfigCompile");
SPAN_ATTACH(Tracer, "ConfigFile", ConfigFile);
auto Result = std::make_shared<CompiledFragmentImpl>();
vlog("Config fragment: compiling {0}:{1} -> {2}", ConfigFile, LineCol.first,
Result.get());
FragmentCompiler{*Result, D, Source.Manager.get()}.compile(std::move(*this));
// Return as cheaply-copyable wrapper.
return [Result(std::move(Result))](const Params &P, Config &C) {
return (*Result)(P, C);
};
}
} // namespace config
} // namespace clangd
} // namespace clang

View File

@ -32,6 +32,7 @@
#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIGFRAGMENT_H
#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIGFRAGMENT_H
#include "ConfigProvider.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/Support/Error.h"
@ -59,11 +60,6 @@ private:
T Value;
};
/// Used to report problems in parsing or interpreting a config.
/// Errors reflect structurally invalid config that should be user-visible.
/// Warnings reflect e.g. unknown properties that are recoverable.
using DiagnosticCallback = llvm::function_ref<void(const llvm::SMDiagnostic &)>;
/// A chunk of configuration obtained from a config file, LSP, or elsewhere.
struct Fragment {
/// Parses fragments from a YAML file (one from each --- delimited document).
@ -73,6 +69,17 @@ struct Fragment {
llvm::StringRef BufferName,
DiagnosticCallback);
/// Analyzes and consumes this fragment, possibly yielding more diagnostics.
/// This always produces a usable result (errors are recovered).
///
/// Typically, providers will compile a Fragment once when it's first loaded,
/// caching the result for reuse.
/// Like a compiled program, this is good for performance and also encourages
/// errors to be reported early and only once.
///
/// The returned function is a cheap-copyable wrapper of refcounted internals.
CompiledFragment compile(DiagnosticCallback) &&;
/// These fields are not part of the user-specified configuration, but
/// instead are populated by the parser to describe the configuration source.
struct SourceInfo {
@ -87,24 +94,25 @@ struct Fragment {
};
SourceInfo Source;
/// Conditions restrict when a Fragment applies.
/// Conditions in the If block restrict when a Fragment applies.
///
/// Each separate condition must match (combined with AND).
/// When one condition has multiple values, any may match (combined with OR).
/// e.g. `PathMatch: [foo/.*, bar/.*]` matches files in either directory.
///
/// Conditions based on a file's path use the following form:
/// - if the fragment came from a project directory, the path is relative
/// - if the fragment is global (e.g. user config), the path is absolute
/// - paths always use forward-slashes (UNIX-style)
/// If no file is being processed, these conditions will not match.
struct ConditionBlock {
struct IfBlock {
/// The file being processed must fully match a regular expression.
std::vector<Located<std::string>> PathMatch;
/// An unrecognized key was found while parsing the condition.
/// The condition will evaluate to false.
bool HasUnrecognizedCondition = false;
};
ConditionBlock Condition;
IfBlock If;
struct CompileFlagsBlock {
std::vector<Located<std::string>> Add;

View File

@ -0,0 +1,54 @@
//===--- ConfigProvider.h - Loading of user configuration --------*- C++-*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Various clangd features have configurable behaviour (or can be disabled).
// The configuration system allows users to control this:
// - in a user config file, a project config file, via LSP, or via flags
// - specifying different settings for different files
// This file defines the structures used for this, that produce a Config.
//
//===----------------------------------------------------------------------===//
#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIGPROVIDER_H
#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIGPROVIDER_H
#include "llvm/ADT/FunctionExtras.h"
#include "llvm/Support/SMLoc.h"
#include "llvm/Support/SourceMgr.h"
#include <string>
#include <vector>
namespace clang {
namespace clangd {
struct Config;
namespace config {
/// Describes the context used to evaluate configuration fragments.
struct Params {
/// Absolute path to a source file we're applying the config to. Unix slashes.
/// Empty if not configuring a particular file.
llvm::StringRef Path;
};
/// Used to report problems in parsing or interpreting a config.
/// Errors reflect structurally invalid config that should be user-visible.
/// Warnings reflect e.g. unknown properties that are recoverable.
using DiagnosticCallback = llvm::function_ref<void(const llvm::SMDiagnostic &)>;
/// A chunk of configuration that has been fully analyzed and is ready to apply.
/// Typically this is obtained from a Fragment by calling Fragment::compile().
///
/// Calling it updates the configuration to reflect settings from the fragment.
/// Returns true if the condition was met and the settings were used.
using CompiledFragment = std::function<bool(const Params &, Config &)>;
} // namespace config
} // namespace clangd
} // namespace clang
#endif

View File

@ -35,15 +35,15 @@ public:
// The private parse() helpers follow the same pattern.
bool parse(Fragment &F, Node &N) {
DictParser Dict("Config", this);
Dict.handle("If", [&](Node &N) { return parse(F.Condition, N); });
Dict.handle("If", [&](Node &N) { return parse(F.If, N); });
Dict.handle("CompileFlags",
[&](Node &N) { return parse(F.CompileFlags, N); });
return Dict.parse(N);
}
private:
bool parse(Fragment::ConditionBlock &F, Node &N) {
DictParser Dict("Condition", this);
bool parse(Fragment::IfBlock &F, Node &N) {
DictParser Dict("If", this);
Dict.unrecognized(
[&](llvm::StringRef) { F.HasUnrecognizedCondition = true; });
Dict.handle("PathMatch", [&](Node &N) {

View File

@ -41,6 +41,7 @@ add_unittest(ClangdUnitTests ClangdTests
CollectMacrosTests.cpp
CompileCommandsTests.cpp
CompilerTests.cpp
ConfigCompileTests.cpp
ConfigYAMLTests.cpp
DexTests.cpp
DiagnosticsTests.cpp

View File

@ -0,0 +1,97 @@
//===-- ConfigCompileTests.cpp --------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "Config.h"
#include "ConfigFragment.h"
#include "ConfigTesting.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
namespace clang {
namespace clangd {
namespace config {
namespace {
using ::testing::ElementsAre;
using ::testing::IsEmpty;
using ::testing::SizeIs;
using ::testing::StartsWith;
class ConfigCompileTests : public ::testing::Test {
protected:
CapturedDiags Diags;
Config Conf;
Fragment Frag;
Params Parm;
bool compileAndApply() {
Conf = Config();
Diags.Diagnostics.clear();
auto Compiled = std::move(Frag).compile(Diags.callback());
return Compiled(Parm, Conf);
}
};
TEST_F(ConfigCompileTests, Condition) {
// No condition.
Frag = {};
Frag.CompileFlags.Add.emplace_back("X");
EXPECT_TRUE(compileAndApply()) << "Empty config";
EXPECT_THAT(Diags.Diagnostics, IsEmpty());
EXPECT_THAT(Conf.CompileFlags.Edits, SizeIs(1));
// Regex with no file.
Frag = {};
Frag.If.PathMatch.emplace_back("fo*");
EXPECT_FALSE(compileAndApply());
EXPECT_THAT(Diags.Diagnostics, IsEmpty());
EXPECT_THAT(Conf.CompileFlags.Edits, SizeIs(0));
// Following tests have a file path set.
Parm.Path = "bar";
// Non-matching regex.
Frag = {};
Frag.If.PathMatch.emplace_back("fo*");
EXPECT_FALSE(compileAndApply());
EXPECT_THAT(Diags.Diagnostics, IsEmpty());
// Matching regex.
Frag = {};
Frag.If.PathMatch.emplace_back("fo*");
Frag.If.PathMatch.emplace_back("ba*r");
EXPECT_TRUE(compileAndApply());
EXPECT_THAT(Diags.Diagnostics, IsEmpty());
// Invalid regex.
Frag = {};
Frag.If.PathMatch.emplace_back("**]@theu");
EXPECT_TRUE(compileAndApply());
EXPECT_THAT(Diags.Diagnostics, SizeIs(1));
EXPECT_THAT(Diags.Diagnostics.front().Message, StartsWith("Invalid regex"));
// Valid regex and unknown key.
Frag = {};
Frag.If.HasUnrecognizedCondition = true;
Frag.If.PathMatch.emplace_back("ba*r");
EXPECT_FALSE(compileAndApply());
EXPECT_THAT(Diags.Diagnostics, IsEmpty());
}
TEST_F(ConfigCompileTests, CompileCommands) {
Frag.CompileFlags.Add.emplace_back("-foo");
std::vector<std::string> Argv = {"clang", "a.cc"};
EXPECT_TRUE(compileAndApply());
EXPECT_THAT(Conf.CompileFlags.Edits, SizeIs(1));
Conf.CompileFlags.Edits.front()(Argv);
EXPECT_THAT(Argv, ElementsAre("clang", "a.cc", "-foo"));
}
} // namespace
} // namespace config
} // namespace clangd
} // namespace clang

View File

@ -0,0 +1,77 @@
//===-- ConfigTesting.h - Helpers for configuration tests -------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_UNITTESTS_CONFIGTESTING_H
#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_UNITTESTS_CONFIGTESTING_H
#include "Protocol.h"
#include "llvm/Support/ScopedPrinter.h"
#include "llvm/Support/SourceMgr.h"
#include "gmock/gmock.h"
#include <functional>
namespace clang {
namespace clangd {
namespace config {
// Provides a DiagnosticsCallback that records diganostics.
// Unlike just pushing them into a vector, underlying storage need not survive.
struct CapturedDiags {
std::function<void(const llvm::SMDiagnostic &)> callback() {
return [this](const llvm::SMDiagnostic &D) {
Diagnostics.emplace_back();
Diag &Out = Diagnostics.back();
Out.Message = D.getMessage().str();
Out.Kind = D.getKind();
Out.Pos.line = D.getLineNo() - 1;
Out.Pos.character = D.getColumnNo(); // Zero-based - bug in SourceMgr?
if (!D.getRanges().empty()) {
const auto &R = D.getRanges().front();
Out.Range.emplace();
Out.Range->start.line = Out.Range->end.line = Out.Pos.line;
Out.Range->start.character = R.first;
Out.Range->end.character = R.second;
}
};
}
struct Diag {
std::string Message;
llvm::SourceMgr::DiagKind Kind;
Position Pos;
llvm::Optional<Range> Range;
friend void PrintTo(const Diag &D, std::ostream *OS) {
*OS << (D.Kind == llvm::SourceMgr::DK_Error ? "error: " : "warning: ")
<< D.Message << "@" << llvm::to_string(D.Pos);
}
};
std::vector<Diag> Diagnostics;
};
MATCHER_P(DiagMessage, M, "") { return arg.Message == M; }
MATCHER_P(DiagKind, K, "") { return arg.Kind == K; }
MATCHER_P(DiagPos, P, "") { return arg.Pos == P; }
MATCHER_P(DiagRange, R, "") { return arg.Range == R; }
inline Position toPosition(llvm::SMLoc L, const llvm::SourceMgr &SM) {
auto LineCol = SM.getLineAndColumn(L);
Position P;
P.line = LineCol.first - 1;
P.character = LineCol.second - 1;
return P;
}
inline Range toRange(llvm::SMRange R, const llvm::SourceMgr &SM) {
return Range{toPosition(R.Start, SM), toPosition(R.End, SM)};
}
} // namespace config
} // namespace clangd
} // namespace clang
#endif

View File

@ -8,14 +8,13 @@
#include "Annotations.h"
#include "ConfigFragment.h"
#include "Matchers.h"
#include "ConfigTesting.h"
#include "Protocol.h"
#include "llvm/Support/SMLoc.h"
#include "llvm/Support/ScopedPrinter.h"
#include "llvm/Support/SourceMgr.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include "gtest/internal/gtest-internal.h"
namespace clang {
namespace clangd {
@ -36,55 +35,6 @@ MATCHER_P(Val, Value, "") {
return false;
}
Position toPosition(llvm::SMLoc L, const llvm::SourceMgr &SM) {
auto LineCol = SM.getLineAndColumn(L);
Position P;
P.line = LineCol.first - 1;
P.character = LineCol.second - 1;
return P;
}
Range toRange(llvm::SMRange R, const llvm::SourceMgr &SM) {
return Range{toPosition(R.Start, SM), toPosition(R.End, SM)};
}
struct CapturedDiags {
std::function<void(const llvm::SMDiagnostic &)> callback() {
return [this](const llvm::SMDiagnostic &D) {
Diagnostics.emplace_back();
Diag &Out = Diagnostics.back();
Out.Message = D.getMessage().str();
Out.Kind = D.getKind();
Out.Pos.line = D.getLineNo() - 1;
Out.Pos.character = D.getColumnNo(); // Zero-based - bug in SourceMgr?
if (!D.getRanges().empty()) {
const auto &R = D.getRanges().front();
Out.Rng.emplace();
Out.Rng->start.line = Out.Rng->end.line = Out.Pos.line;
Out.Rng->start.character = R.first;
Out.Rng->end.character = R.second;
}
};
}
struct Diag {
std::string Message;
llvm::SourceMgr::DiagKind Kind;
Position Pos;
llvm::Optional<Range> Rng;
friend void PrintTo(const Diag &D, std::ostream *OS) {
*OS << (D.Kind == llvm::SourceMgr::DK_Error ? "error: " : "warning: ")
<< D.Message << "@" << llvm::to_string(D.Pos);
}
};
std::vector<Diag> Diagnostics;
};
MATCHER_P(DiagMessage, M, "") { return arg.Message == M; }
MATCHER_P(DiagKind, K, "") { return arg.Kind == K; }
MATCHER_P(DiagPos, P, "") { return arg.Pos == P; }
MATCHER_P(DiagRange, R, "") { return arg.Rng == R; }
TEST(ParseYAML, SyntacticForms) {
CapturedDiags Diags;
const char *YAML = R"yaml(
@ -101,8 +51,8 @@ CompileFlags:
auto Results = Fragment::parseYAML(YAML, "config.yaml", Diags.callback());
EXPECT_THAT(Diags.Diagnostics, IsEmpty());
ASSERT_EQ(Results.size(), 2u);
EXPECT_FALSE(Results.front().Condition.HasUnrecognizedCondition);
EXPECT_THAT(Results.front().Condition.PathMatch, ElementsAre(Val("abc")));
EXPECT_FALSE(Results.front().If.HasUnrecognizedCondition);
EXPECT_THAT(Results.front().If.PathMatch, ElementsAre(Val("abc")));
EXPECT_THAT(Results.front().CompileFlags.Add,
ElementsAre(Val("foo"), Val("bar")));
@ -120,7 +70,7 @@ If:
EXPECT_THAT(Diags.Diagnostics, IsEmpty());
ASSERT_EQ(Results.size(), 1u);
ASSERT_NE(Results.front().Source.Manager, nullptr);
EXPECT_EQ(toRange(Results.front().Condition.PathMatch.front().Range,
EXPECT_EQ(toRange(Results.front().If.PathMatch.front().Range,
*Results.front().Source.Manager),
YAML.range());
}
@ -140,7 +90,7 @@ CompileFlags: {^
ASSERT_THAT(
Diags.Diagnostics,
ElementsAre(AllOf(DiagMessage("Unknown Condition key UnknownCondition"),
ElementsAre(AllOf(DiagMessage("Unknown If key UnknownCondition"),
DiagKind(llvm::SourceMgr::DK_Warning),
DiagPos(YAML.range().start), DiagRange(YAML.range())),
AllOf(DiagMessage("Unexpected token. Expected Key, Flow "
@ -150,7 +100,7 @@ CompileFlags: {^
ASSERT_EQ(Results.size(), 2u);
EXPECT_THAT(Results.front().CompileFlags.Add, ElementsAre(Val("first")));
EXPECT_TRUE(Results.front().Condition.HasUnrecognizedCondition);
EXPECT_TRUE(Results.front().If.HasUnrecognizedCondition);
EXPECT_THAT(Results.back().CompileFlags.Add, IsEmpty());
}