[clangd] Emit publishSemanticHighlighting in LSP if enabled

Summary: Emit publishSemanticHighlighting in LSP if enabled

Reviewers: hokein, kadircet

Subscribers: ilya-biryukov, MaskRay, jkorous, arphaman, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D63919

llvm-svn: 365111

clang-tools-extra/clangd/ClangdLSPServer.cpp

@@ -11,6 +11,7 @@
 #include "FormattedString.h"
 #include "GlobalCompilationDatabase.h"
 #include "Protocol.h"
+#include "SemanticHighlighting.h"
 #include "SourceCode.h"
 #include "Trace.h"
 #include "URI.h"
@@ -328,6 +329,8 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params,
     WithOffsetEncoding.emplace(kCurrentOffsetEncoding,
                                *NegotiatedOffsetEncoding);
 
+  ClangdServerOpts.SemanticHighlighting =
+      Params.capabilities.SemanticHighlighting;
   if (Params.rootUri && *Params.rootUri)
     ClangdServerOpts.WorkspaceRoot = Params.rootUri->file();
   else if (Params.rootPath && !Params.rootPath->empty())
@@ -407,6 +410,11 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params,
         }}}};
   if (NegotiatedOffsetEncoding)
     Result["offsetEncoding"] = *NegotiatedOffsetEncoding;
+  if (Params.capabilities.SemanticHighlighting)
+    Result.getObject("capabilities")
+        ->insert(
+            {"semanticHighlighting",
+             llvm::json::Object{{"scopes", getTextMateScopeLookupTable()}}});
   Reply(std::move(Result));
 }
 
@@ -927,6 +935,11 @@ void ClangdLSPServer::applyConfiguration(
     reparseOpenedFiles();
 }
 
+void ClangdLSPServer::publishSemanticHighlighting(
+    SemanticHighlightingParams Params) {
+  notify("textDocument/semanticHighlighting", Params);
+}
+
 void ClangdLSPServer::publishDiagnostics(
     const URIForFile &File, std::vector<clangd::Diagnostic> Diagnostics) {
   // Publish diagnostics.
@@ -1063,6 +1076,13 @@ bool ClangdLSPServer::shouldRunCompletion(
   return true;
 }
 
+void ClangdLSPServer::onHighlightingsReady(
+    PathRef File, std::vector<HighlightingToken> Highlightings) {
+  publishSemanticHighlighting(
+      {{URIForFile::canonicalize(File, /*TUPath=*/File)},
+       toSemanticHighlightingInformation(Highlightings)});
+}
+
 void ClangdLSPServer::onDiagnosticsReady(PathRef File,
                                          std::vector<Diag> Diagnostics) {
   auto URI = URIForFile::canonicalize(File, /*TUPath=*/File);

clang-tools-extra/clangd/ClangdLSPServer.h

@@ -55,6 +55,8 @@ private:
   // Implement DiagnosticsConsumer.
   void onDiagnosticsReady(PathRef File, std::vector<Diag> Diagnostics) override;
   void onFileUpdated(PathRef File, const TUStatus &Status) override;
+  void onHighlightingsReady(PathRef File,
+                         std::vector<HighlightingToken> Highlightings) override;
 
   // LSP methods. Notifications have signature void(const Params&).
   // Calls have signature void(const Params&, Callback<Response>).
@@ -115,6 +117,9 @@ private:
   void reparseOpenedFiles();
   void applyConfiguration(const ConfigurationSettings &Settings);
 
+  /// Sends a "publishSemanticHighlighting" notification to the LSP client.
+  void publishSemanticHighlighting(SemanticHighlightingParams Params);
+
   /// Sends a "publishDiagnostics" notification to the LSP client.
   void publishDiagnostics(const URIForFile &File,
                           std::vector<clangd::Diagnostic> Diagnostics);

clang-tools-extra/clangd/Protocol.cpp

@@ -273,6 +273,12 @@ bool fromJSON(const llvm::json::Value &Params, ClientCapabilities &R) {
   if (!O)
     return false;
   if (auto *TextDocument = O->getObject("textDocument")) {
+    if (auto *SemanticHighlighting =
+            TextDocument->getObject("semanticHighlightingCapabilities")) {
+      if (auto SemanticHighlightingSupport =
+              SemanticHighlighting->getBoolean("semanticHighlighting"))
+        R.SemanticHighlighting = *SemanticHighlightingSupport;
+    }
     if (auto *Diagnostics = TextDocument->getObject("publishDiagnostics")) {
       if (auto CategorySupport = Diagnostics->getBoolean("categorySupport"))
         R.DiagnosticCategory = *CategorySupport;
@@ -1027,5 +1033,22 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, OffsetEncoding Enc) {
   return OS << toString(Enc);
 }
 
+bool operator==(const SemanticHighlightingInformation &Lhs,
+                const SemanticHighlightingInformation &Rhs) {
+  return Lhs.Line == Rhs.Line && Lhs.Tokens == Rhs.Tokens;
+}
+
+llvm::json::Value toJSON(const SemanticHighlightingInformation &Highlighting) {
+  return llvm::json::Object{{"line", Highlighting.Line},
+                            {"tokens", Highlighting.Tokens}};
+}
+
+llvm::json::Value toJSON(const SemanticHighlightingParams &Highlighting) {
+  return llvm::json::Object{
+      {"textDocument", Highlighting.TextDocument},
+      {"lines", std::move(Highlighting.Lines)},
+  };
+}
+
 } // namespace clangd
 } // namespace clang

clang-tools-extra/clangd/Protocol.h

@@ -406,6 +406,9 @@ struct ClientCapabilities {
   /// textDocument.codeAction.codeActionLiteralSupport.
   bool CodeActionStructure = false;
 
+  /// Client supports semantic highlighting.
+  bool SemanticHighlighting = false;
+
   /// Supported encodings for LSP character offsets. (clangd extension).
   llvm::Optional<std::vector<OffsetEncoding>> offsetEncoding;
 
@@ -1173,6 +1176,27 @@ struct FileStatus {
 };
 llvm::json::Value toJSON(const FileStatus &FStatus);
 
+/// Represents a semantic highlighting information that has to be applied on a
+/// specific line of the text document.
+struct SemanticHighlightingInformation {
+  /// The line these highlightings belong to.
+  int Line;
+  /// The base64 encoded string of highlighting tokens.
+  std::string Tokens;
+};
+bool operator==(const SemanticHighlightingInformation &Lhs,
+                const SemanticHighlightingInformation &Rhs);
+llvm::json::Value toJSON(const SemanticHighlightingInformation &Highlighting);
+
+/// Parameters for the semantic highlighting (server-side) push notification.
+struct SemanticHighlightingParams {
+  /// The textdocument these highlightings belong to.
+  TextDocumentIdentifier TextDocument;
+  /// The lines of highlightings that should be sent.
+  std::vector<SemanticHighlightingInformation> Lines;
+};
+llvm::json::Value toJSON(const SemanticHighlightingParams &Highlighting);
+
 } // namespace clangd
 } // namespace clang
 

clang-tools-extra/clangd/SemanticHighlighting.cpp

@@ -8,6 +8,7 @@
 
 #include "SemanticHighlighting.h"
 #include "Logger.h"
+#include "Protocol.h"
 #include "SourceCode.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/RecursiveASTVisitor.h"
@@ -63,6 +64,48 @@ private:
   }
 };
 
+// Encode binary data into base64.
+// This was copied from compiler-rt/lib/fuzzer/FuzzerUtil.cpp.
+// FIXME: Factor this out into llvm/Support?
+std::string encodeBase64(const llvm::SmallVectorImpl<char> &Bytes) {
+  static const char Table[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+                              "abcdefghijklmnopqrstuvwxyz"
+                              "0123456789+/";
+  std::string Res;
+  size_t I;
+  for (I = 0; I + 2 < Bytes.size(); I += 3) {
+    uint32_t X = (Bytes[I] << 16) + (Bytes[I + 1] << 8) + Bytes[I + 2];
+    Res += Table[(X >> 18) & 63];
+    Res += Table[(X >> 12) & 63];
+    Res += Table[(X >> 6) & 63];
+    Res += Table[X & 63];
+  }
+  if (I + 1 == Bytes.size()) {
+    uint32_t X = (Bytes[I] << 16);
+    Res += Table[(X >> 18) & 63];
+    Res += Table[(X >> 12) & 63];
+    Res += "==";
+  } else if (I + 2 == Bytes.size()) {
+    uint32_t X = (Bytes[I] << 16) + (Bytes[I + 1] << 8);
+    Res += Table[(X >> 18) & 63];
+    Res += Table[(X >> 12) & 63];
+    Res += Table[(X >> 6) & 63];
+    Res += "=";
+  }
+  return Res;
+}
+
+void write32be(uint32_t I, llvm::raw_ostream &OS) {
+  std::array<char, 4> Buf;
+  llvm::support::endian::write32be(Buf.data(), I);
+  OS.write(Buf.data(), Buf.size());
+}
+
+void write16be(uint16_t I, llvm::raw_ostream &OS) {
+  std::array<char, 2> Buf;
+  llvm::support::endian::write16be(Buf.data(), I);
+  OS.write(Buf.data(), Buf.size());
+}
 } // namespace
 
 bool operator==(const HighlightingToken &Lhs, const HighlightingToken &Rhs) {
@@ -73,5 +116,50 @@ std::vector<HighlightingToken> getSemanticHighlightings(ParsedAST &AST) {
   return HighlightingTokenCollector(AST).collectTokens();
 }
 
+std::vector<SemanticHighlightingInformation>
+toSemanticHighlightingInformation(llvm::ArrayRef<HighlightingToken> Tokens) {
+  if (Tokens.size() == 0)
+    return {};
+
+  // FIXME: Tokens might be multiple lines long (block comments) in this case
+  // this needs to add multiple lines for those tokens.
+  std::map<int, std::vector<HighlightingToken>> TokenLines;
+  for (const HighlightingToken &Token : Tokens)
+    TokenLines[Token.R.start.line].push_back(Token);
+
+  std::vector<SemanticHighlightingInformation> Lines;
+  Lines.reserve(TokenLines.size());
+  for (const auto &Line : TokenLines) {
+    llvm::SmallVector<char, 128> LineByteTokens;
+    llvm::raw_svector_ostream OS(LineByteTokens);
+    for (const auto &Token : Line.second) {
+      // Writes the token to LineByteTokens in the byte format specified by the
+      // LSP proposal. Described below.
+      // |<---- 4 bytes ---->|<-- 2 bytes -->|<--- 2 bytes -->|
+      // |    character      |  length       |    index       |
+
+      write32be(Token.R.start.character, OS);
+      write16be(Token.R.end.character - Token.R.start.character, OS);
+      write16be(static_cast<int>(Token.Kind), OS);
+    }
+
+    Lines.push_back({Line.first, encodeBase64(LineByteTokens)});
+  }
+
+  return Lines;
+}
+
+std::vector<std::vector<std::string>> getTextMateScopeLookupTable() {
+  // FIXME: Add scopes for C and Objective C.
+  std::map<HighlightingKind, std::vector<std::string>> Scopes = {
+      {HighlightingKind::Variable, {"variable.cpp"}},
+      {HighlightingKind::Function, {"entity.name.function.cpp"}}};
+  std::vector<std::vector<std::string>> NestedScopes(Scopes.size());
+  for (const auto &Scope : Scopes)
+    NestedScopes[static_cast<int>(Scope.first)] = Scope.second;
+
+  return NestedScopes;
+}
+
 } // namespace clangd
 } // namespace clang

clang-tools-extra/clangd/SemanticHighlighting.h

@@ -5,18 +5,27 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+//
+// An implementation of semantic highlighting based on this proposal:
+// https://github.com/microsoft/vscode-languageserver-node/pull/367 in clangd.
+// Semantic highlightings are calculated for an AST by visiting every AST node
+// and classifying nodes that are interesting to highlight (variables/function
+// calls etc.).
+//
+//===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTICHIGHLIGHTING_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTICHIGHLIGHTING_H
 
 #include "ClangdUnit.h"
+#include "Protocol.h"
 
 namespace clang {
 namespace clangd {
 
 enum class HighlightingKind {
-  Variable,
+  Variable = 0,
-  Function,
+  Function = 1,
 };
 
 // Contains all information needed for the highlighting a token.
@@ -31,6 +40,14 @@ bool operator==(const HighlightingToken &Lhs, const HighlightingToken &Rhs);
 // main AST.
 std::vector<HighlightingToken> getSemanticHighlightings(ParsedAST &AST);
 
+// Gets the TextMate scopes as a double nested array where the
+// SemanticHighlightKind indexes correctly into this vector.
+std::vector<std::vector<std::string>> getTextMateScopeLookupTable();
+
+// Convert to LSP's semantic highlighting information.
+std::vector<SemanticHighlightingInformation>
+toSemanticHighlightingInformation(llvm::ArrayRef<HighlightingToken> Tokens);
+
 } // namespace clangd
 } // namespace clang
 

clang-tools-extra/clangd/test/semantic-highlighting.test

@@ -0,0 +1,75 @@
+# RUN: clangd -lit-test < %s | FileCheck -strict-whitespace %s
+{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{"textDocument":{"semanticHighlightingCapabilities":{"semanticHighlighting":true}}},"trace":"off"}}
+---
+#      CHECK:  "id": 0,
+# CHECK-NEXT:  "jsonrpc": "2.0",
+# CHECK-NEXT:  "result": {
+# CHECK-NEXT:    "capabilities": {
+# CHECK-NEXT:      "codeActionProvider": true,
+# CHECK-NEXT:      "completionProvider": {
+# CHECK-NEXT:        "resolveProvider": false,
+# CHECK-NEXT:        "triggerCharacters": [
+# CHECK-NEXT:          ".",
+# CHECK-NEXT:          ">",
+# CHECK-NEXT:          ":"
+# CHECK-NEXT:        ]
+# CHECK-NEXT:      },
+# CHECK-NEXT:      "declarationProvider": true,
+# CHECK-NEXT:      "definitionProvider": true,
+# CHECK-NEXT:      "documentFormattingProvider": true,
+# CHECK-NEXT:      "documentHighlightProvider": true,
+# CHECK-NEXT:      "documentOnTypeFormattingProvider": {
+# CHECK-NEXT:        "firstTriggerCharacter": "\n",
+# CHECK-NEXT:        "moreTriggerCharacter": []
+# CHECK-NEXT:      },
+# CHECK-NEXT:      "documentRangeFormattingProvider": true,
+# CHECK-NEXT:      "documentSymbolProvider": true,
+# CHECK-NEXT:      "executeCommandProvider": {
+# CHECK-NEXT:        "commands": [
+# CHECK-NEXT:          "clangd.applyFix",
+# CHECK-NEXT:          "clangd.applyTweak"
+# CHECK-NEXT:        ]
+# CHECK-NEXT:      },
+# CHECK-NEXT:      "hoverProvider": true,
+# CHECK-NEXT:      "referencesProvider": true,
+# CHECK-NEXT:      "renameProvider": true,
+# CHECK-NEXT:      "semanticHighlighting": {
+# CHECK-NEXT:        "scopes": [
+# CHECK-NEXT:          [
+# CHECK-NEXT:            "variable.cpp"
+# CHECK-NEXT:          ],
+# CHECK-NEXT:          [
+# CHECK-NEXT:            "entity.name.function.cpp"
+# CHECK-NEXT:          ]
+# CHECK-NEXT:        ]
+# CHECK-NEXT:      },
+# CHECK-NEXT:      "signatureHelpProvider": {
+# CHECK-NEXT:        "triggerCharacters": [
+# CHECK-NEXT:          "(",
+# CHECK-NEXT:          ","
+# CHECK-NEXT:        ]
+# CHECK-NEXT:      },
+# CHECK-NEXT:      "textDocumentSync": 2,
+# CHECK-NEXT:      "typeHierarchyProvider": true
+# CHECK-NEXT:      "workspaceSymbolProvider": true
+# CHECK-NEXT:    }
+# CHECK-NEXT:  }
+---
+{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///foo.cpp","languageId":"cpp","version":1,"text":"int x = 2;"}}}
+#      CHECK:  "method": "textDocument/semanticHighlighting",
+# CHECK-NEXT:  "params": {
+# CHECK-NEXT:    "lines": [
+# CHECK-NEXT:      {
+# CHECK-NEXT:        "line": 0,
+# CHECK-NEXT:        "tokens": "AAAABAABAAA="
+# CHECK-NEXT:      }
+# CHECK-NEXT:    ],
+# CHECK-NEXT:    "textDocument": {
+# CHECK-NEXT:      "uri": "file:///clangd-test/foo.cpp"
+# CHECK-NEXT:    }
+# CHECK-NEXT:  }
+# CHECK-NEXT:}
+---
+{"jsonrpc":"2.0","id":3,"method":"shutdown"}
+---
+{"jsonrpc":"2.0","method":"exit"}

clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

@@ -8,6 +8,7 @@
 
 #include "Annotations.h"
 #include "ClangdServer.h"
+#include "Protocol.h"
 #include "SemanticHighlighting.h"
 #include "TestFS.h"
 #include "TestTU.h"
@@ -66,7 +67,7 @@ TEST(SemanticHighlighting, GetsCorrectTokens) {
   }
 }
 
-TEST(ClangdSemanticHighlightingTest, GeneratesHighlightsWhenFileChange) {
+TEST(SemanticHighlighting, GeneratesHighlightsWhenFileChange) {
   class HighlightingsCounterDiagConsumer : public DiagnosticsConsumer {
   public:
     std::atomic<int> Count = {0};
@@ -90,6 +91,29 @@ TEST(ClangdSemanticHighlightingTest, GeneratesHighlightsWhenFileChange) {
   ASSERT_EQ(DiagConsumer.Count, 1);
 }
 
+TEST(SemanticHighlighting, toSemanticHighlightingInformation) {
+  auto CreatePosition = [](int Line, int Character) -> Position {
+    Position Pos;
+    Pos.line = Line;
+    Pos.character = Character;
+    return Pos;
+  };
+
+  std::vector<HighlightingToken> Tokens{
+      {HighlightingKind::Variable,
+                        Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
+      {HighlightingKind::Function,
+                        Range{CreatePosition(3, 4), CreatePosition(3, 7)}},
+      {HighlightingKind::Variable,
+                        Range{CreatePosition(1, 1), CreatePosition(1, 5)}}};
+  std::vector<SemanticHighlightingInformation> ActualResults =
+      toSemanticHighlightingInformation(Tokens);
+  std::vector<SemanticHighlightingInformation> ExpectedResults = {
+      {1, "AAAAAQAEAAA="},
+      {3, "AAAACAAEAAAAAAAEAAMAAQ=="}};
+  EXPECT_EQ(ActualResults, ExpectedResults);
+}
+
 } // namespace
 } // namespace clangd
 } // namespace clang