[clangd] Introduce client state invalidation

Clangd can invalidate client state of features like semantic higlighting
without client explicitly triggering, for example after a preamble build
caused by an onSave notification on a different file.

This patch introduces a mechanism to let client know of such actions,
and also calls the workspace/semanticTokens/refresh request to
demonstrate the situation after each preamble build.

Fixes https://github.com/clangd/clangd/issues/699.

Differential Revision: https://reviews.llvm.org/D97548

clang-tools-extra/clangd/ClangdLSPServer.cpp

@@ -579,7 +579,7 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params,
 
   {
     LSPBinder Binder(Handlers, *this);
-    bindMethods(Binder);
+    bindMethods(Binder, Params.capabilities);
     if (Opts.Modules)
       for (auto &Mod : *Opts.Modules)
         Mod.initializeLSP(Binder, Params.rawCapabilities, ServerCaps);
@@ -1406,8 +1406,7 @@ void ClangdLSPServer::onAST(const ASTParams &Params,
   Server->getAST(Params.textDocument.uri.file(), Params.range, std::move(CB));
 }
 
-ClangdLSPServer::ClangdLSPServer(Transport &Transp,
-                                 const ThreadsafeFS &TFS,
+ClangdLSPServer::ClangdLSPServer(Transport &Transp, const ThreadsafeFS &TFS,
                                  const ClangdLSPServer::Options &Opts)
     : ShouldProfile(/*Period=*/std::chrono::minutes(5),
                     /*Delay=*/std::chrono::minutes(1)),
@@ -1427,7 +1426,8 @@ ClangdLSPServer::ClangdLSPServer(Transport &Transp,
   Bind.method("initialize", this, &ClangdLSPServer::onInitialize);
 }
 
-void ClangdLSPServer::bindMethods(LSPBinder &Bind) {
+void ClangdLSPServer::bindMethods(LSPBinder &Bind,
+                                  const ClientCapabilities &Caps) {
   // clang-format off
   Bind.notification("initialized", this, &ClangdLSPServer::onInitialized);
   Bind.method("shutdown", this, &ClangdLSPServer::onShutdown);
@@ -1481,6 +1481,8 @@ void ClangdLSPServer::bindMethods(LSPBinder &Bind) {
   BeginWorkDoneProgress = Bind.outgoingNotification("$/progress");
   ReportWorkDoneProgress = Bind.outgoingNotification("$/progress");
   EndWorkDoneProgress = Bind.outgoingNotification("$/progress");
+  if(Caps.SemanticTokenRefreshSupport)
+    SemanticTokensRefresh = Bind.outgoingMethod("workspace/semanticTokens/refresh");
   // clang-format on
 }
 
@@ -1656,5 +1658,14 @@ void ClangdLSPServer::onFileUpdated(PathRef File, const TUStatus &Status) {
   NotifyFileStatus(Status.render(File));
 }
 
+void ClangdLSPServer::onSemanticsMaybeChanged(PathRef File) {
+  if (SemanticTokensRefresh) {
+    SemanticTokensRefresh(NoParams{}, [](llvm::Expected<std::nullptr_t> E) {
+      if (E)
+        return;
+      elog("Failed to refresh semantic tokens: {0}", E.takeError());
+    });
+  }
+}
 } // namespace clangd
 } // namespace clang

clang-tools-extra/clangd/ClangdLSPServer.h

@@ -87,6 +87,7 @@ private:
                           std::vector<Diag> Diagnostics) override;
   void onFileUpdated(PathRef File, const TUStatus &Status) override;
   void onBackgroundIndexProgress(const BackgroundQueue::Stats &Stats) override;
+  void onSemanticsMaybeChanged(PathRef File) override;
 
   // LSP methods. Notifications have signature void(const Params&).
   // Calls have signature void(const Params&, Callback<Response>).
@@ -181,11 +182,12 @@ private:
       ReportWorkDoneProgress;
   LSPBinder::OutgoingNotification<ProgressParams<WorkDoneProgressEnd>>
       EndWorkDoneProgress;
+  LSPBinder::OutgoingMethod<NoParams, std::nullptr_t> SemanticTokensRefresh;
 
   void applyEdit(WorkspaceEdit WE, llvm::json::Value Success,
                  Callback<llvm::json::Value> Reply);
 
-  void bindMethods(LSPBinder &);
+  void bindMethods(LSPBinder &, const ClientCapabilities &Caps);
   std::vector<Fix> getFixes(StringRef File, const clangd::Diagnostic &D);
 
   /// Checks if completion request should be ignored. We need this due to the

clang-tools-extra/clangd/ClangdServer.cpp

@@ -73,6 +73,8 @@ struct UpdateIndexCallbacks : public ParsingCallbacks {
                      const CanonicalIncludes &CanonIncludes) override {
     if (FIndex)
       FIndex->updatePreamble(Path, Version, Ctx, std::move(PP), CanonIncludes);
+    if (ServerCallbacks)
+      ServerCallbacks->onSemanticsMaybeChanged(Path);
   }
 
   void onMainAST(PathRef Path, ParsedAST &AST, PublishFn Publish) override {

clang-tools-extra/clangd/ClangdServer.h

@@ -28,6 +28,7 @@
 #include "support/Cancellation.h"
 #include "support/Function.h"
 #include "support/MemoryTree.h"
+#include "support/Path.h"
 #include "support/ThreadsafeFS.h"
 #include "clang/Tooling/CompilationDatabase.h"
 #include "clang/Tooling/Core/Replacement.h"
@@ -74,6 +75,14 @@ public:
     /// Not called concurrently.
     virtual void
     onBackgroundIndexProgress(const BackgroundQueue::Stats &Stats) {}
+
+    /// Called when the meaning of a source code may have changed without an
+    /// edit. Usually clients assume that responses to requests are valid until
+    /// they next edit the file. If they're invalidated at other times, we
+    /// should tell the client. In particular, when an asynchronous preamble
+    /// build finishes, we can provide more accurate semantic tokens, so we
+    /// should tell the client to refresh.
+    virtual void onSemanticsMaybeChanged(PathRef File) {}
   };
   /// Creates a context provider that loads and installs config.
   /// Errors in loading config are reported as diagnostics via Callbacks.

clang-tools-extra/clangd/Protocol.cpp

@@ -403,6 +403,10 @@ bool fromJSON(const llvm::json::Value &Params, ClientCapabilities &R,
         }
       }
     }
+    if (auto *SemanticTokens = Workspace->getObject("semanticTokens")) {
+      if (auto RefreshSupport = SemanticTokens->getBoolean("refreshSupport"))
+        R.SemanticTokenRefreshSupport = *RefreshSupport;
+    }
   }
   if (auto *Window = O->getObject("window")) {
     if (auto WorkDoneProgress = Window->getBoolean("workDoneProgress"))

clang-tools-extra/clangd/Protocol.h

@@ -261,6 +261,7 @@ enum class TraceLevel {
 bool fromJSON(const llvm::json::Value &E, TraceLevel &Out, llvm::json::Path);
 
 struct NoParams {};
+inline llvm::json::Value toJSON(const NoParams &) { return nullptr; }
 inline bool fromJSON(const llvm::json::Value &, NoParams &, llvm::json::Path) {
   return true;
 }
@@ -473,6 +474,10 @@ struct ClientCapabilities {
   /// This is a clangd extension.
   /// window.implicitWorkDoneProgressCreate
   bool ImplicitProgressCreation = false;
+
+  /// Whether the client implementation supports a refresh request sent from the
+  /// server to the client.
+  bool SemanticTokenRefreshSupport = false;
 };
 bool fromJSON(const llvm::json::Value &, ClientCapabilities &,
               llvm::json::Path);

clang-tools-extra/clangd/test/semantic-tokens-refresh.test

@@ -0,0 +1,42 @@
+# RUN: clangd -lit-test < %s | FileCheck -strict-whitespace %s
+{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"capabilities":{
+  "workspace":{"semanticTokens":{"refreshSupport":true}}
+}}}
+---
+{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{
+  "uri": "test:///foo.cpp",
+  "languageId": "cpp",
+  "text": "int x = 2;"
+}}}
+# Expect a request after initial preamble build.
+# CHECK:        "method": "workspace/semanticTokens/refresh",
+# CHECK-NEXT:   "params": null
+# CHECK-NEXT:  }
+---
+# Reply with success.
+{"jsonrpc":"2.0","id":0}
+---
+# Preamble stays the same, no refresh requests.
+{"jsonrpc":"2.0","method":"textDocument/didChange","params":{
+  "textDocument": {"uri":"test:///foo.cpp","version":2},
+  "contentChanges":[{"text":"int x = 2;\nint y = 3;"}]
+}}
+# CHECK-NOT:  "method": "workspace/semanticTokens/refresh"
+---
+# Preamble changes
+{"jsonrpc":"2.0","method":"textDocument/didChange","params":{
+  "textDocument": {"uri":"test:///foo.cpp","version":2},
+  "contentChanges":[{"text":"#define FOO"}]
+}}
+# Expect a request after initial preamble build.
+# CHECK:        "method": "workspace/semanticTokens/refresh",
+# CHECK-NEXT:   "params": null
+# CHECK-NEXT:  }
+---
+# Reply with error, to make sure there are no crashes.
+{"jsonrpc":"2.0","id":1,"error":{"code": 0, "message": "msg"}}
+---
+{"jsonrpc":"2.0","id":3,"method":"shutdown"}
+---
+{"jsonrpc":"2.0","method":"exit"}
+