[clangd] Support new semanticTokens request from LSP 3.16.

Summary:
This is a simpler request/response protocol.

Reference: https://github.com/microsoft/vscode-languageserver-node/blob/master/protocol/src/protocol.semanticTokens.proposed.ts

No attempt to support incremental formatting (yet).

Reviewers: hokein

Subscribers: ilya-biryukov, MaskRay, jkorous, arphaman, kadircet, usaxena95, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D76663

clang-tools-extra/clangd/ClangdLSPServer.cpp

@@ -458,6 +458,14 @@ void ClangdLSPServer::notify(llvm::StringRef Method, llvm::json::Value Params) {
   Transp.notify(Method, std::move(Params));
 }
 
+static std::vector<llvm::StringRef> semanticTokenTypes() {
+  std::vector<llvm::StringRef> Types;
+  for (unsigned I = 0; I <= static_cast<unsigned>(HighlightingKind::LastKind);
+       ++I)
+    Types.push_back(toSemanticTokenType(static_cast<HighlightingKind>(I)));
+  return Types;
+}
+
 void ClangdLSPServer::onInitialize(const InitializeParams &Params,
                                    Callback<llvm::json::Value> Reply) {
   // Determine character encoding first as it affects constructed ClangdServer.
@@ -569,6 +577,14 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params,
                  // trigger on '->' and '::'.
                  {"triggerCharacters", {".", ">", ":"}},
              }},
+            {"semanticTokensProvider",
+             llvm::json::Object{
+                 {"documentProvider", true},
+                 {"rangeProvider", false},
+                 {"legend",
+                  llvm::json::Object{{"tokenTypes", semanticTokenTypes()},
+                                     {"tokenModifiers", llvm::json::Array()}}},
+             }},
             {"signatureHelpProvider",
              llvm::json::Object{
                  {"triggerCharacters", {"(", ","}},
@@ -1220,6 +1236,20 @@ void ClangdLSPServer::onDocumentLink(
       });
 }
 
+void ClangdLSPServer::onSemanticTokens(const SemanticTokensParams &Params,
+                                       Callback<SemanticTokens> CB) {
+  Server->semanticHighlights(
+      Params.textDocument.uri.file(),
+      [CB(std::move(CB))](
+          llvm::Expected<std::vector<HighlightingToken>> Toks) mutable {
+        if (!Toks)
+          return CB(Toks.takeError());
+        SemanticTokens Result;
+        Result.data = toSemanticTokens(*Toks);
+        CB(std::move(Result));
+      });
+}
+
 ClangdLSPServer::ClangdLSPServer(
     class Transport &Transp, const FileSystemProvider &FSProvider,
     const clangd::CodeCompleteOptions &CCOpts,
@@ -1267,6 +1297,7 @@ ClangdLSPServer::ClangdLSPServer(
   MsgHandler->bind("typeHierarchy/resolve", &ClangdLSPServer::onResolveTypeHierarchy);
   MsgHandler->bind("textDocument/selectionRange", &ClangdLSPServer::onSelectionRange);
   MsgHandler->bind("textDocument/documentLink", &ClangdLSPServer::onDocumentLink);
+  MsgHandler->bind("textDocument/semanticTokens", &ClangdLSPServer::onSemanticTokens);
   // clang-format on
 }
 

clang-tools-extra/clangd/ClangdLSPServer.h

@@ -119,6 +119,7 @@ private:
                         Callback<std::vector<SelectionRange>>);
   void onDocumentLink(const DocumentLinkParams &,
                       Callback<std::vector<DocumentLink>>);
+  void onSemanticTokens(const SemanticTokensParams &, Callback<SemanticTokens>);
 
   std::vector<Fix> getFixes(StringRef File, const clangd::Diagnostic &D);
 

clang-tools-extra/clangd/ClangdServer.cpp

@@ -685,6 +685,18 @@ void ClangdServer::documentLinks(PathRef File,
                            TUScheduler::InvalidateOnUpdate);
 }
 
+void ClangdServer::semanticHighlights(
+    PathRef File, Callback<std::vector<HighlightingToken>> CB) {
+  auto Action =
+      [CB = std::move(CB)](llvm::Expected<InputsAndAST> InpAST) mutable {
+        if (!InpAST)
+          return CB(InpAST.takeError());
+        CB(clangd::getSemanticHighlightings(InpAST->AST));
+      };
+  WorkScheduler.runWithAST("SemanticHighlights", File, std::move(Action),
+                           TUScheduler::InvalidateOnUpdate);
+}
+
 std::vector<std::pair<Path, std::size_t>>
 ClangdServer::getUsedBytesPerFile() const {
   return WorkScheduler.getUsedBytesPerFile();

clang-tools-extra/clangd/ClangdServer.h

@@ -297,7 +297,10 @@ public:
 
   /// Get all document links in a file.
   void documentLinks(PathRef File, Callback<std::vector<DocumentLink>> CB);
- 
+
+  void semanticHighlights(PathRef File,
+                          Callback<std::vector<HighlightingToken>>);
+
   /// Returns estimated memory usage for each of the currently open files.
   /// The order of results is unspecified.
   /// Overall memory usage of clangd may be significantly more than reported

clang-tools-extra/clangd/Protocol.cpp

@@ -984,6 +984,29 @@ llvm::json::Value toJSON(const FileStatus &FStatus) {
   };
 }
 
+void SemanticToken::encode(std::vector<unsigned int> &Out) const {
+  Out.push_back(deltaLine);
+  Out.push_back(deltaStart);
+  Out.push_back(length);
+  Out.push_back(tokenType);
+  Out.push_back(tokenModifiers);
+}
+
+llvm::json::Value toJSON(const SemanticTokens &Tokens) {
+  std::vector<unsigned> Data;
+  for (const auto &Tok : Tokens.data)
+    Tok.encode(Data);
+  llvm::json::Object Result{{"data", std::move(Data)}};
+  if (Tokens.resultId)
+    Result["resultId"] = *Tokens.resultId;
+  return Result;
+}
+
+bool fromJSON(const llvm::json::Value &Params, SemanticTokensParams &R) {
+  llvm::json::ObjectMapper O(Params);
+  return O && O.map("textDocument", R.textDocument);
+}
+
 llvm::raw_ostream &operator<<(llvm::raw_ostream &O,
                               const DocumentHighlight &V) {
   O << V.range;

clang-tools-extra/clangd/Protocol.h

@@ -1340,7 +1340,47 @@ struct FileStatus {
   std::string state;
   // FIXME: add detail messages.
 };
-llvm::json::Value toJSON(const FileStatus &FStatus);
+llvm::json::Value toJSON(const FileStatus &);
+
+/// Specifies a single semantic token in the document.
+/// This struct is not part of LSP, which just encodes lists of tokens as
+/// arrays of numbers directly.
+struct SemanticToken {
+  /// token line number, relative to the previous token
+  unsigned deltaLine = 0;
+  /// token start character, relative to the previous token
+  /// (relative to 0 or the previous token's start if they are on the same line)
+  unsigned deltaStart = 0;
+  /// the length of the token. A token cannot be multiline
+  unsigned length = 0;
+  /// will be looked up in `SemanticTokensLegend.tokenTypes`
+  unsigned tokenType = 0;
+  /// each set bit will be looked up in `SemanticTokensLegend.tokenModifiers`
+  unsigned tokenModifiers = 0;
+
+  void encode(std::vector<unsigned> &Out) const;
+};
+
+/// A versioned set of tokens.
+struct SemanticTokens {
+  // An optional result id. If provided and clients support delta updating
+  // the client will include the result id in the next semantic token request.
+  // A server can then instead of computing all semantic tokens again simply
+  // send a delta.
+  llvm::Optional<std::string> resultId;
+
+  /// The actual tokens. For a detailed description about how the data is
+  /// structured pls see
+  /// https://github.com/microsoft/vscode-extension-samples/blob/5ae1f7787122812dcc84e37427ca90af5ee09f14/semantic-tokens-sample/vscode.proposed.d.ts#L71
+  std::vector<SemanticToken> data;
+};
+llvm::json::Value toJSON(const SemanticTokens &);
+
+struct SemanticTokensParams {
+  /// The text document.
+  TextDocumentIdentifier textDocument;
+};
+bool fromJSON(const llvm::json::Value &, SemanticTokensParams &);
 
 /// Represents a semantic highlighting information that has to be applied on a
 /// specific line of the text document.

clang-tools-extra/clangd/SemanticHighlighting.cpp

@@ -445,6 +445,83 @@ bool operator==(const LineHighlightings &L, const LineHighlightings &R) {
   return std::tie(L.Line, L.Tokens) == std::tie(R.Line, R.Tokens);
 }
 
+std::vector<SemanticToken>
+toSemanticTokens(llvm::ArrayRef<HighlightingToken> Tokens) {
+  assert(std::is_sorted(Tokens.begin(), Tokens.end()));
+  std::vector<SemanticToken> Result;
+  const HighlightingToken *Last = nullptr;
+  for (const HighlightingToken &Tok : Tokens) {
+    // FIXME: support inactive code - we need to provide the actual bounds.
+    if (Tok.Kind == HighlightingKind::InactiveCode)
+      continue;
+    Result.emplace_back();
+    SemanticToken &Out = Result.back();
+    // deltaStart/deltaLine are relative if possible.
+    if (Last) {
+      assert(Tok.R.start.line >= Last->R.start.line);
+      Out.deltaLine = Tok.R.start.line - Last->R.start.line;
+      if (Out.deltaLine == 0) {
+        assert(Tok.R.start.character >= Last->R.start.character);
+        Out.deltaStart = Tok.R.start.character - Last->R.start.character;
+      } else {
+        Out.deltaStart = Tok.R.start.character;
+      }
+    } else {
+      Out.deltaLine = Tok.R.start.line;
+      Out.deltaStart = Tok.R.start.character;
+    }
+    assert(Tok.R.end.line == Tok.R.start.line);
+    Out.length = Tok.R.end.character - Tok.R.start.character;
+    Out.tokenType = static_cast<unsigned>(Tok.Kind);
+
+    Last = &Tok;
+  }
+  return Result;
+}
+llvm::StringRef toSemanticTokenType(HighlightingKind Kind) {
+  switch (Kind) {
+  case HighlightingKind::Variable:
+  case HighlightingKind::LocalVariable:
+  case HighlightingKind::StaticField:
+    return "variable";
+  case HighlightingKind::Parameter:
+    return "parameter";
+  case HighlightingKind::Function:
+    return "function";
+  case HighlightingKind::Method:
+    return "member";
+  case HighlightingKind::StaticMethod:
+    // FIXME: better function/member with static modifier?
+    return "function";
+  case HighlightingKind::Field:
+    return "member";
+  case HighlightingKind::Class:
+    return "class";
+  case HighlightingKind::Enum:
+    return "enum";
+  case HighlightingKind::EnumConstant:
+    return "enumConstant"; // nonstandard
+  case HighlightingKind::Typedef:
+    return "type";
+  case HighlightingKind::DependentType:
+    return "dependent"; // nonstandard
+  case HighlightingKind::DependentName:
+    return "dependent"; // nonstandard
+  case HighlightingKind::Namespace:
+    return "namespace";
+  case HighlightingKind::TemplateParameter:
+    return "typeParameter";
+  case HighlightingKind::Concept:
+    return "concept"; // nonstandard
+  case HighlightingKind::Primitive:
+    return "type";
+  case HighlightingKind::Macro:
+    return "macro";
+  case HighlightingKind::InactiveCode:
+    return "comment";
+  }
+}
+
 std::vector<TheiaSemanticHighlightingInformation>
 toTheiaSemanticHighlightingInformation(
     llvm::ArrayRef<LineHighlightings> Tokens) {

clang-tools-extra/clangd/SemanticHighlighting.h

@@ -6,8 +6,21 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// An implementation of semantic highlighting based on this proposal:
-// https://github.com/microsoft/vscode-languageserver-node/pull/367 in clangd.
+// This file supports semantic highlighting: categorizing tokens in the file so
+// that the editor can color/style them differently.
+//
+// This is particularly valuable for C++: its complex and context-dependent
+// grammar is a challenge for simple syntax-highlighting techniques.
+//
+// We support two protocols for providing highlights to the client:
+// - the `textDocument/semanticTokens` request from LSP 3.16
+//   https://github.com/microsoft/vscode-languageserver-node/blob/release/protocol/3.16.0-next.1/protocol/src/protocol.semanticTokens.proposed.ts
+// - the earlier proposed `textDocument/semanticHighlighting` notification
+//   https://github.com/microsoft/vscode-languageserver-node/pull/367
+//   This is referred to as "Theia" semantic highlighting in the code.
+//   It was supported from clangd 9 but should be considered deprecated as of
+//   clangd 11 and eventually removed.
+//
 // Semantic highlightings are calculated for an AST by visiting every AST node
 // and classifying nodes that are interesting to highlight (variables/function
 // calls etc.).
@@ -75,6 +88,9 @@ bool operator==(const LineHighlightings &L, const LineHighlightings &R);
 // main AST.
 std::vector<HighlightingToken> getSemanticHighlightings(ParsedAST &AST);
 
+std::vector<SemanticToken> toSemanticTokens(llvm::ArrayRef<HighlightingToken>);
+llvm::StringRef toSemanticTokenType(HighlightingKind Kind);
+
 /// Converts a HighlightingKind to a corresponding TextMate scope
 /// (https://manual.macromates.com/en/language_grammars).
 llvm::StringRef toTextMateScope(HighlightingKind Kind);

clang-tools-extra/clangd/test/initialize-params.test

@@ -38,6 +38,16 @@
 # CHECK-NEXT:      "referencesProvider": true,
 # CHECK-NEXT:      "renameProvider": true,
 # CHECK-NEXT:      "selectionRangeProvider": true,
+# CHECK-NEXT:      "semanticTokensProvider": {
+# CHECK-NEXT:        "documentProvider": true,
+# CHECK-NEXT:        "legend": {
+# CHECK-NEXT:          "tokenModifiers": [],
+# CHECK-NEXT:          "tokenTypes": [
+# CHECK-NEXT:            "variable",
+# CHECK:               ]
+# CHECK-NEXT:        },
+# CHECK-NEXT:        "rangeProvider": false
+# CHECK-NEXT:      },
 # CHECK-NEXT:      "signatureHelpProvider": {
 # CHECK-NEXT:        "triggerCharacters": [
 # CHECK-NEXT:          "(",

clang-tools-extra/clangd/test/semantic-tokens.test

@@ -0,0 +1,22 @@
+# RUN: clangd -lit-test < %s | FileCheck -strict-whitespace %s
+{"jsonrpc":"2.0","id":0,"method":"initialize","params":{}}
+---
+{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///foo.cpp","languageId":"cpp","text":"int x = 2;"}}}
+---
+{"jsonrpc":"2.0","id":1,"method":"textDocument/semanticTokens","params":{"textDocument":{"uri":"test:///foo.cpp"}}}
+# CHECK:       "id": 1,
+# CHECK-NEXT:  "jsonrpc": "2.0",
+# CHECK-NEXT:  "result": {
+# CHECK-NEXT:    "data": [
+#                  First line, char 5, variable, no modifiers.
+# CHECK-NEXT:      0,
+# CHECK-NEXT:      4,
+# CHECK-NEXT:      1,
+# CHECK-NEXT:      0,
+# CHECK-NEXT:      0
+# CHECK-NEXT:    ]
+# CHECK-NEXT:  }
+---
+{"jsonrpc":"2.0","id":2,"method":"shutdown"}
+---
+{"jsonrpc":"2.0","method":"exit"}

clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

@@ -720,6 +720,41 @@ TEST(SemanticHighlighting, GeneratesHighlightsWhenFileChange) {
   ASSERT_EQ(Counter.Count, 1);
 }
 
+TEST(SemanticHighlighting, toSemanticTokens) {
+  auto CreatePosition = [](int Line, int Character) -> Position {
+    Position Pos;
+    Pos.line = Line;
+    Pos.character = Character;
+    return Pos;
+  };
+
+  std::vector<HighlightingToken> Tokens = {
+      {HighlightingKind::Variable,
+       Range{CreatePosition(1, 1), CreatePosition(1, 5)}},
+      {HighlightingKind::Function,
+       Range{CreatePosition(3, 4), CreatePosition(3, 7)}},
+      {HighlightingKind::Variable,
+       Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
+  };
+
+  std::vector<SemanticToken> Results = toSemanticTokens(Tokens);
+  EXPECT_EQ(Tokens.size(), Results.size());
+  EXPECT_EQ(Results[0].tokenType, unsigned(HighlightingKind::Variable));
+  EXPECT_EQ(Results[0].deltaLine, 1u);
+  EXPECT_EQ(Results[0].deltaStart, 1u);
+  EXPECT_EQ(Results[0].length, 4u);
+
+  EXPECT_EQ(Results[1].tokenType, unsigned(HighlightingKind::Function));
+  EXPECT_EQ(Results[1].deltaLine, 2u);
+  EXPECT_EQ(Results[1].deltaStart, 4u);
+  EXPECT_EQ(Results[1].length, 3u);
+
+  EXPECT_EQ(Results[2].tokenType, unsigned(HighlightingKind::Variable));
+  EXPECT_EQ(Results[2].deltaLine, 0u);
+  EXPECT_EQ(Results[2].deltaStart, 4u);
+  EXPECT_EQ(Results[2].length, 4u);
+}
+
 TEST(SemanticHighlighting, toTheiaSemanticHighlightingInformation) {
   auto CreatePosition = [](int Line, int Character) -> Position {
     Position Pos;