[clangd] allow clients to control the compilation database by passing in

compilationDatabaseChanges in the 'workspace/didChangeConfiguration' request

This commit allows clangd to use an in-memory compilation database that's
controlled from the LSP client (-compile_args_from=lsp). It extends the
'workspace/didChangeConfiguration' request to allow the client to pass in a
compilation database subset that needs to be updated in the workspace.

Differential Revision: https://reviews.llvm.org/D49758

llvm-svn: 338597

clang-tools-extra/clangd/ClangdLSPServer.cpp

@@ -135,11 +135,8 @@ void ClangdLSPServer::onExit(ExitParams &Params) { IsDone = true; }
 
 void ClangdLSPServer::onDocumentDidOpen(DidOpenTextDocumentParams &Params) {
   PathRef File = Params.textDocument.uri.file();
-  if (Params.metadata && !Params.metadata->extraFlags.empty()) {
-    NonCachedCDB.setExtraFlagsForFile(File,
-                                      std::move(Params.metadata->extraFlags));
-    CDB.invalidate(File);
-  }
+  if (Params.metadata && !Params.metadata->extraFlags.empty())
+    CDB.setExtraFlagsForFile(File, std::move(Params.metadata->extraFlags));
 
   std::string &Contents = Params.textDocument.text;
 
@@ -250,6 +247,7 @@ void ClangdLSPServer::onDocumentDidClose(DidCloseTextDocumentParams &Params) {
   PathRef File = Params.textDocument.uri.file();
   DraftMgr.removeDraft(File);
   Server.removeDocument(File);
+  CDB.invalidate(File);
 }
 
 void ClangdLSPServer::onDocumentOnTypeFormatting(
@@ -405,12 +403,29 @@ void ClangdLSPServer::applyConfiguration(
     const ClangdConfigurationParamsChange &Settings) {
   // Compilation database change.
   if (Settings.compilationDatabasePath.hasValue()) {
-    NonCachedCDB.setCompileCommandsDir(
-        Settings.compilationDatabasePath.getValue());
-    CDB.clear();
+    CDB.setCompileCommandsDir(Settings.compilationDatabasePath.getValue());
 
     reparseOpenedFiles();
   }
+
+  // Update to the compilation database.
+  if (Settings.compilationDatabaseChanges) {
+    const auto &CompileCommandUpdates = *Settings.compilationDatabaseChanges;
+    bool ShouldReparseOpenFiles = false;
+    for (auto &Entry : CompileCommandUpdates) {
+      /// The opened files need to be reparsed only when some existing
+      /// entries are changed.
+      PathRef File = Entry.first;
+      if (!CDB.setCompilationCommandForFile(
+              File, tooling::CompileCommand(
+                        std::move(Entry.second.workingDirectory), File,
+                        std::move(Entry.second.compilationCommand),
+                        /*Output=*/"")))
+        ShouldReparseOpenFiles = true;
+    }
+    if (ShouldReparseOpenFiles)
+      reparseOpenedFiles();
+  }
 }
 
 // FIXME: This function needs to be properly tested.
@@ -422,10 +437,13 @@ void ClangdLSPServer::onChangeConfiguration(
 ClangdLSPServer::ClangdLSPServer(JSONOutput &Out,
                                  const clangd::CodeCompleteOptions &CCOpts,
                                  llvm::Optional<Path> CompileCommandsDir,
+                                 bool ShouldUseInMemoryCDB,
                                  const ClangdServer::Options &Opts)
-    : Out(Out), NonCachedCDB(std::move(CompileCommandsDir)), CDB(NonCachedCDB),
+    : Out(Out), CDB(ShouldUseInMemoryCDB ? CompilationDB::makeInMemory()
+                                         : CompilationDB::makeDirectoryBased(
+                                               std::move(CompileCommandsDir))),
       CCOpts(CCOpts), SupportedSymbolKinds(defaultSymbolKinds()),
-      Server(CDB, FSProvider, /*DiagConsumer=*/*this, Opts) {}
+      Server(CDB.getCDB(), FSProvider, /*DiagConsumer=*/*this, Opts) {}
 
 bool ClangdLSPServer::run(std::FILE *In, JSONStreamStyle InputStyle) {
   assert(!IsDone && "Run was called before");
@@ -504,3 +522,67 @@ void ClangdLSPServer::reparseOpenedFiles() {
     Server.addDocument(FilePath, *DraftMgr.getDraft(FilePath),
                        WantDiagnostics::Auto);
 }
+
+ClangdLSPServer::CompilationDB ClangdLSPServer::CompilationDB::makeInMemory() {
+  return CompilationDB(llvm::make_unique<InMemoryCompilationDb>(), nullptr,
+                       /*IsDirectoryBased=*/false);
+}
+
+ClangdLSPServer::CompilationDB
+ClangdLSPServer::CompilationDB::makeDirectoryBased(
+    llvm::Optional<Path> CompileCommandsDir) {
+  auto CDB = llvm::make_unique<DirectoryBasedGlobalCompilationDatabase>(
+      std::move(CompileCommandsDir));
+  auto CachingCDB = llvm::make_unique<CachingCompilationDb>(*CDB);
+  return CompilationDB(std::move(CDB), std::move(CachingCDB),
+                       /*IsDirectoryBased=*/true);
+}
+
+void ClangdLSPServer::CompilationDB::invalidate(PathRef File) {
+  if (!IsDirectoryBased)
+    static_cast<InMemoryCompilationDb *>(CDB.get())->invalidate(File);
+  else
+    CachingCDB->invalidate(File);
+}
+
+bool ClangdLSPServer::CompilationDB::setCompilationCommandForFile(
+    PathRef File, tooling::CompileCommand CompilationCommand) {
+  if (IsDirectoryBased) {
+    elog("Trying to set compile command for {0} while using directory-based "
+         "compilation database",
+         File);
+    return false;
+  }
+  return static_cast<InMemoryCompilationDb *>(CDB.get())
+      ->setCompilationCommandForFile(File, std::move(CompilationCommand));
+}
+
+void ClangdLSPServer::CompilationDB::setExtraFlagsForFile(
+    PathRef File, std::vector<std::string> ExtraFlags) {
+  if (!IsDirectoryBased) {
+    elog("Trying to set extra flags for {0} while using in-memory compilation "
+         "database",
+         File);
+    return;
+  }
+  static_cast<DirectoryBasedGlobalCompilationDatabase *>(CDB.get())
+      ->setExtraFlagsForFile(File, std::move(ExtraFlags));
+  CachingCDB->invalidate(File);
+}
+
+void ClangdLSPServer::CompilationDB::setCompileCommandsDir(Path P) {
+  if (!IsDirectoryBased) {
+    elog("Trying to set compile commands dir while using in-memory compilation "
+         "database");
+    return;
+  }
+  static_cast<DirectoryBasedGlobalCompilationDatabase *>(CDB.get())
+      ->setCompileCommandsDir(P);
+  CachingCDB->clear();
+}
+
+GlobalCompilationDatabase &ClangdLSPServer::CompilationDB::getCDB() {
+  if (CachingCDB)
+    return *CachingCDB;
+  return *CDB;
+}

clang-tools-extra/clangd/ClangdLSPServer.h

@@ -35,7 +35,7 @@ public:
   /// for compile_commands.json in all parent directories of each file.
   ClangdLSPServer(JSONOutput &Out, const clangd::CodeCompleteOptions &CCOpts,
                   llvm::Optional<Path> CompileCommandsDir,
-                  const ClangdServer::Options &Opts);
+                  bool ShouldUseInMemoryCDB, const ClangdServer::Options &Opts);
 
   /// Run LSP server loop, receiving input for it from \p In. \p In must be
   /// opened in binary mode. Output will be written using Out variable passed to
@@ -100,10 +100,57 @@ private:
   /// Caches FixIts per file and diagnostics
   llvm::StringMap<DiagnosticToReplacementMap> FixItsMap;
 
+  /// Encapsulates the directory-based or the in-memory compilation database
+  /// that's used by the LSP server.
+  class CompilationDB {
+  public:
+    static CompilationDB makeInMemory();
+    static CompilationDB
+    makeDirectoryBased(llvm::Optional<Path> CompileCommandsDir);
+
+    void invalidate(PathRef File);
+
+    /// Sets the compilation command for a particular file.
+    /// Only valid for in-memory CDB, no-op and error log on DirectoryBasedCDB.
+    ///
+    /// \returns True if the File had no compilation command before.
+    bool
+    setCompilationCommandForFile(PathRef File,
+                                 tooling::CompileCommand CompilationCommand);
+
+    /// Adds extra compilation flags to the compilation command for a particular
+    /// file. Only valid for directory-based CDB, no-op and error log on
+    /// InMemoryCDB;
+    void setExtraFlagsForFile(PathRef File,
+                              std::vector<std::string> ExtraFlags);
+
+    /// Set the compile commands directory to \p P.
+    /// Only valid for directory-based CDB, no-op and error log on InMemoryCDB;
+    void setCompileCommandsDir(Path P);
+
+    /// Returns a CDB that should be used to get compile commands for the
+    /// current instance of ClangdLSPServer.
+    GlobalCompilationDatabase &getCDB();
+
+  private:
+    CompilationDB(std::unique_ptr<GlobalCompilationDatabase> CDB,
+                  std::unique_ptr<CachingCompilationDb> CachingCDB,
+                  bool IsDirectoryBased)
+        : CDB(std::move(CDB)), CachingCDB(std::move(CachingCDB)),
+          IsDirectoryBased(IsDirectoryBased) {}
+
+    // if IsDirectoryBased is true, an instance of InMemoryCDB.
+    // If IsDirectoryBased is false, an instance of DirectoryBasedCDB.
+    // unique_ptr<GlobalCompilationDatabase> CDB;
+    std::unique_ptr<GlobalCompilationDatabase> CDB;
+    // Non-null only for directory-based CDB
+    std::unique_ptr<CachingCompilationDb> CachingCDB;
+    bool IsDirectoryBased;
+  };
+
   // Various ClangdServer parameters go here. It's important they're created
   // before ClangdServer.
-  DirectoryBasedGlobalCompilationDatabase NonCachedCDB;
-  CachingCompilationDb CDB;
+  CompilationDB CDB;
 
   RealFileSystemProvider FSProvider;
   /// Options used for code completion

clang-tools-extra/clangd/GlobalCompilationDatabase.cpp

@@ -152,5 +152,29 @@ void CachingCompilationDb::clear() {
   Cached.clear();
 }
 
+llvm::Optional<tooling::CompileCommand>
+InMemoryCompilationDb::getCompileCommand(PathRef File) const {
+  std::lock_guard<std::mutex> Lock(Mutex);
+  auto It = Commands.find(File);
+  if (It == Commands.end())
+    return None;
+  return It->second;
+}
+
+bool InMemoryCompilationDb::setCompilationCommandForFile(
+    PathRef File, tooling::CompileCommand CompilationCommand) {
+  std::unique_lock<std::mutex> Lock(Mutex);
+  auto ItInserted = Commands.insert(std::make_pair(File, CompilationCommand));
+  if (ItInserted.second)
+    return true;
+  ItInserted.first->setValue(std::move(CompilationCommand));
+  return false;
+}
+
+void InMemoryCompilationDb::invalidate(PathRef File) {
+  std::unique_lock<std::mutex> Lock(Mutex);
+  Commands.erase(File);
+}
+
 } // namespace clangd
 } // namespace clang

clang-tools-extra/clangd/GlobalCompilationDatabase.h

@@ -113,6 +113,29 @@ private:
       Cached; /* GUARDED_BY(Mut) */
 };
 
+/// Gets compile args from an in-memory mapping based on a filepath. Typically
+/// used by clients who provide the compile commands themselves.
+class InMemoryCompilationDb : public GlobalCompilationDatabase {
+public:
+  /// Gets compile command for \p File from the stored mapping.
+  llvm::Optional<tooling::CompileCommand>
+  getCompileCommand(PathRef File) const override;
+
+  /// Sets the compilation command for a particular file.
+  ///
+  /// \returns True if the File had no compilation command before.
+  bool setCompilationCommandForFile(PathRef File,
+                                    tooling::CompileCommand CompilationCommand);
+
+  /// Removes the compilation command for \p File if it's present in the
+  /// mapping.
+  void invalidate(PathRef File);
+
+private:
+  mutable std::mutex Mutex;
+  llvm::StringMap<tooling::CompileCommand> Commands; /* GUARDED_BY(Mut) */
+};
+
 } // namespace clangd
 } // namespace clang
 

clang-tools-extra/clangd/Protocol.cpp

@@ -591,10 +591,18 @@ bool fromJSON(const json::Value &Params, DidChangeConfigurationParams &CCP) {
   return O && O.map("settings", CCP.settings);
 }
 
+bool fromJSON(const llvm::json::Value &Params,
+              ClangdCompileCommand &CDbUpdate) {
+  json::ObjectMapper O(Params);
+  return O && O.map("workingDirectory", CDbUpdate.workingDirectory) &&
+         O.map("compilationCommand", CDbUpdate.compilationCommand);
+}
+
 bool fromJSON(const json::Value &Params,
               ClangdConfigurationParamsChange &CCPC) {
   json::ObjectMapper O(Params);
-  return O && O.map("compilationDatabasePath", CCPC.compilationDatabasePath);
+  return O && O.map("compilationDatabasePath", CCPC.compilationDatabasePath) &&
+         O.map("compilationDatabaseChanges", CCPC.compilationDatabaseChanges);
 }
 
 } // namespace clangd

clang-tools-extra/clangd/Protocol.h

@@ -322,11 +322,25 @@ struct ClientCapabilities {
 
 bool fromJSON(const llvm::json::Value &, ClientCapabilities &);
 
+/// Clangd extension that's used in the 'compilationDatabaseChanges' in
+/// workspace/didChangeConfiguration to record updates to the in-memory
+/// compilation database.
+struct ClangdCompileCommand {
+  std::string workingDirectory;
+  std::vector<std::string> compilationCommand;
+};
+bool fromJSON(const llvm::json::Value &, ClangdCompileCommand &);
+
 /// Clangd extension to set clangd-specific "initializationOptions" in the
 /// "initialize" request and for the "workspace/didChangeConfiguration"
 /// notification since the data received is described as 'any' type in LSP.
 struct ClangdConfigurationParamsChange {
   llvm::Optional<std::string> compilationDatabasePath;
+
+  // The changes that happened to the compilation database.
+  // The key of the map is a file name.
+  llvm::Optional<std::map<std::string, ClangdCompileCommand>>
+      compilationDatabaseChanges;
 };
 bool fromJSON(const llvm::json::Value &, ClangdConfigurationParamsChange &);
 

clang-tools-extra/clangd/tool/ClangdMain.cpp

@@ -173,6 +173,18 @@ static llvm::cl::opt<Path> YamlSymbolFile(
         "eventually. Don't rely on it."),
     llvm::cl::init(""), llvm::cl::Hidden);
 
+enum CompileArgsFrom { LSPCompileArgs, FilesystemCompileArgs };
+
+static llvm::cl::opt<CompileArgsFrom> CompileArgsFrom(
+    "compile_args_from", llvm::cl::desc("The source of compile commands"),
+    llvm::cl::values(clEnumValN(LSPCompileArgs, "lsp",
+                                "All compile commands come from LSP and "
+                                "'compile_commands.json' files are ignored"),
+                     clEnumValN(FilesystemCompileArgs, "filesystem",
+                                "All compile commands come from the "
+                                "'compile_commands.json' files")),
+    llvm::cl::init(FilesystemCompileArgs), llvm::cl::Hidden);
+
 int main(int argc, char *argv[]) {
   llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
   llvm::cl::SetVersionPrinter([](llvm::raw_ostream &OS) {
@@ -289,7 +301,9 @@ int main(int argc, char *argv[]) {
   }
 
   // Initialize and run ClangdLSPServer.
-  ClangdLSPServer LSPServer(Out, CCOpts, CompileCommandsDirPath, Opts);
+  ClangdLSPServer LSPServer(
+      Out, CCOpts, CompileCommandsDirPath,
+      /*ShouldUseInMemoryCDB=*/CompileArgsFrom == LSPCompileArgs, Opts);
   constexpr int NoShutdownRequestErrorCode = 1;
   llvm::set_thread_name("clangd.main");
   // Change stdin to binary to not lose \r\n on windows.

clang-tools-extra/test/clangd/did-change-configuration-params.test

@@ -0,0 +1,52 @@
+# RUN: clangd -compile_args_from=lsp -lit-test < %s 2> %t | FileCheck -strict-whitespace %s
+# RUN: cat %t | FileCheck --check-prefix=ERR %s
+# UNSUPPORTED: mingw32,win32
+{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{},"trace":"off"}}
+---
+{"jsonrpc":"2.0","method":"workspace/didChangeConfiguration","params":{"settings":{"compilationDatabaseChanges":{"/clangd-test/foo.c": {"workingDirectory":"/clangd-test", "compilationCommand": ["clang", "-c", "foo.c"]}}}}}
+---
+{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///foo.c","languageId":"c","version":1,"text":"int main() { int i; return i; }"}}}
+#      CHECK:  "method": "textDocument/publishDiagnostics",
+# CHECK-NEXT:  "params": {
+# CHECK-NEXT:    "diagnostics": [],
+# CHECK-NEXT:    "uri": "file://{{.*}}/foo.c"
+# CHECK-NEXT:  }
+---
+{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///bar.c","languageId":"c","version":1,"text":"int main() { int i; return i; }"}}}
+#      CHECK:  "method": "textDocument/publishDiagnostics",
+# CHECK-NEXT:  "params": {
+# CHECK-NEXT:    "diagnostics": [],
+# CHECK-NEXT:    "uri": "file://{{.*}}/bar.c"
+# CHECK-NEXT:  }
+---
+{"jsonrpc":"2.0","method":"workspace/didChangeConfiguration","params":{"settings":{"compilationDatabaseChanges":{"/clangd-test/foo.c": {"workingDirectory":"/clangd-test2", "compilationCommand": ["clang", "-c", "foo.c", "-Wall", "-Werror"]}}}}}
+#      CHECK:  "method": "textDocument/publishDiagnostics",
+# CHECK-NEXT:  "params": {
+# CHECK-NEXT:    "diagnostics": [
+# CHECK-NEXT:      {
+# CHECK-NEXT:        "message": "variable 'i' is uninitialized when used here",
+# CHECK-NEXT:        "range": {
+# CHECK-NEXT:          "end": {
+# CHECK-NEXT:            "character": 28,
+# CHECK-NEXT:            "line": 0
+# CHECK-NEXT:          },
+# CHECK-NEXT:          "start": {
+# CHECK-NEXT:            "character": 27,
+# CHECK-NEXT:            "line": 0
+# CHECK-NEXT:          }
+# CHECK-NEXT:        },
+# CHECK-NEXT:        "severity": 1
+# CHECK-NEXT:      }
+# CHECK-NEXT:    ],
+# CHECK-NEXT:    "uri": "file://{{.*}}/foo.c"
+# CHECK-NEXT:  }
+#
+# ERR: Updating file {{.*}}foo.c with command [{{.*}}clangd-test2] clang -c foo.c -Wall -Werror
+# Don't reparse the second file:
+# ERR: Skipping rebuild of the AST for {{.*}}bar.c
+---
+{"jsonrpc":"2.0","id":5,"method":"shutdown"}
+---
+{"jsonrpc":"2.0","method":"exit"}
+
+