From a848dab7a31d818da3d454e7a52c811d4a10fd21 Mon Sep 17 00:00:00 2001 From: Johan Vikstrom Date: Thu, 4 Jul 2019 07:53:12 +0000 Subject: [PATCH] [clangd] Emit publishSemanticHighlighting in LSP if enabled Summary: Emit publishSemanticHighlighting in LSP if enabled Reviewers: hokein, kadircet Subscribers: ilya-biryukov, MaskRay, jkorous, arphaman, cfe-commits Tags: #clang Differential Revision: https://reviews.llvm.org/D63919 llvm-svn: 365111 --- clang-tools-extra/clangd/ClangdLSPServer.cpp | 20 +++++ clang-tools-extra/clangd/ClangdLSPServer.h | 5 ++ clang-tools-extra/clangd/Protocol.cpp | 23 +++++ clang-tools-extra/clangd/Protocol.h | 24 +++++ .../clangd/SemanticHighlighting.cpp | 88 +++++++++++++++++++ .../clangd/SemanticHighlighting.h | 21 ++++- .../clangd/test/semantic-highlighting.test | 75 ++++++++++++++++ .../unittests/SemanticHighlightingTests.cpp | 26 +++++- 8 files changed, 279 insertions(+), 3 deletions(-) create mode 100644 clang-tools-extra/clangd/test/semantic-highlighting.test diff --git a/clang-tools-extra/clangd/ClangdLSPServer.cpp b/clang-tools-extra/clangd/ClangdLSPServer.cpp index a65aab0a0827..df2e3c5da699 100644 --- a/clang-tools-extra/clangd/ClangdLSPServer.cpp +++ b/clang-tools-extra/clangd/ClangdLSPServer.cpp @@ -11,6 +11,7 @@ #include "FormattedString.h" #include "GlobalCompilationDatabase.h" #include "Protocol.h" +#include "SemanticHighlighting.h" #include "SourceCode.h" #include "Trace.h" #include "URI.h" @@ -328,6 +329,8 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params, WithOffsetEncoding.emplace(kCurrentOffsetEncoding, *NegotiatedOffsetEncoding); + ClangdServerOpts.SemanticHighlighting = + Params.capabilities.SemanticHighlighting; if (Params.rootUri && *Params.rootUri) ClangdServerOpts.WorkspaceRoot = Params.rootUri->file(); else if (Params.rootPath && !Params.rootPath->empty()) @@ -407,6 +410,11 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params, }}}}; if (NegotiatedOffsetEncoding) Result["offsetEncoding"] = *NegotiatedOffsetEncoding; + if (Params.capabilities.SemanticHighlighting) + Result.getObject("capabilities") + ->insert( + {"semanticHighlighting", + llvm::json::Object{{"scopes", getTextMateScopeLookupTable()}}}); Reply(std::move(Result)); } @@ -927,6 +935,11 @@ void ClangdLSPServer::applyConfiguration( reparseOpenedFiles(); } +void ClangdLSPServer::publishSemanticHighlighting( + SemanticHighlightingParams Params) { + notify("textDocument/semanticHighlighting", Params); +} + void ClangdLSPServer::publishDiagnostics( const URIForFile &File, std::vector Diagnostics) { // Publish diagnostics. @@ -1063,6 +1076,13 @@ bool ClangdLSPServer::shouldRunCompletion( return true; } +void ClangdLSPServer::onHighlightingsReady( + PathRef File, std::vector Highlightings) { + publishSemanticHighlighting( + {{URIForFile::canonicalize(File, /*TUPath=*/File)}, + toSemanticHighlightingInformation(Highlightings)}); +} + void ClangdLSPServer::onDiagnosticsReady(PathRef File, std::vector Diagnostics) { auto URI = URIForFile::canonicalize(File, /*TUPath=*/File); diff --git a/clang-tools-extra/clangd/ClangdLSPServer.h b/clang-tools-extra/clangd/ClangdLSPServer.h index 2b2d9c0b0080..ae438ddbe182 100644 --- a/clang-tools-extra/clangd/ClangdLSPServer.h +++ b/clang-tools-extra/clangd/ClangdLSPServer.h @@ -55,6 +55,8 @@ private: // Implement DiagnosticsConsumer. void onDiagnosticsReady(PathRef File, std::vector Diagnostics) override; void onFileUpdated(PathRef File, const TUStatus &Status) override; + void onHighlightingsReady(PathRef File, + std::vector Highlightings) override; // LSP methods. Notifications have signature void(const Params&). // Calls have signature void(const Params&, Callback). @@ -115,6 +117,9 @@ private: void reparseOpenedFiles(); void applyConfiguration(const ConfigurationSettings &Settings); + /// Sends a "publishSemanticHighlighting" notification to the LSP client. + void publishSemanticHighlighting(SemanticHighlightingParams Params); + /// Sends a "publishDiagnostics" notification to the LSP client. void publishDiagnostics(const URIForFile &File, std::vector Diagnostics); diff --git a/clang-tools-extra/clangd/Protocol.cpp b/clang-tools-extra/clangd/Protocol.cpp index f8becaaa3615..e91271275ef1 100644 --- a/clang-tools-extra/clangd/Protocol.cpp +++ b/clang-tools-extra/clangd/Protocol.cpp @@ -273,6 +273,12 @@ bool fromJSON(const llvm::json::Value &Params, ClientCapabilities &R) { if (!O) return false; if (auto *TextDocument = O->getObject("textDocument")) { + if (auto *SemanticHighlighting = + TextDocument->getObject("semanticHighlightingCapabilities")) { + if (auto SemanticHighlightingSupport = + SemanticHighlighting->getBoolean("semanticHighlighting")) + R.SemanticHighlighting = *SemanticHighlightingSupport; + } if (auto *Diagnostics = TextDocument->getObject("publishDiagnostics")) { if (auto CategorySupport = Diagnostics->getBoolean("categorySupport")) R.DiagnosticCategory = *CategorySupport; @@ -1027,5 +1033,22 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, OffsetEncoding Enc) { return OS << toString(Enc); } +bool operator==(const SemanticHighlightingInformation &Lhs, + const SemanticHighlightingInformation &Rhs) { + return Lhs.Line == Rhs.Line && Lhs.Tokens == Rhs.Tokens; +} + +llvm::json::Value toJSON(const SemanticHighlightingInformation &Highlighting) { + return llvm::json::Object{{"line", Highlighting.Line}, + {"tokens", Highlighting.Tokens}}; +} + +llvm::json::Value toJSON(const SemanticHighlightingParams &Highlighting) { + return llvm::json::Object{ + {"textDocument", Highlighting.TextDocument}, + {"lines", std::move(Highlighting.Lines)}, + }; +} + } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/Protocol.h b/clang-tools-extra/clangd/Protocol.h index b800e6bf99fe..6f9b7a9470d6 100644 --- a/clang-tools-extra/clangd/Protocol.h +++ b/clang-tools-extra/clangd/Protocol.h @@ -406,6 +406,9 @@ struct ClientCapabilities { /// textDocument.codeAction.codeActionLiteralSupport. bool CodeActionStructure = false; + /// Client supports semantic highlighting. + bool SemanticHighlighting = false; + /// Supported encodings for LSP character offsets. (clangd extension). llvm::Optional> offsetEncoding; @@ -1173,6 +1176,27 @@ struct FileStatus { }; llvm::json::Value toJSON(const FileStatus &FStatus); +/// Represents a semantic highlighting information that has to be applied on a +/// specific line of the text document. +struct SemanticHighlightingInformation { + /// The line these highlightings belong to. + int Line; + /// The base64 encoded string of highlighting tokens. + std::string Tokens; +}; +bool operator==(const SemanticHighlightingInformation &Lhs, + const SemanticHighlightingInformation &Rhs); +llvm::json::Value toJSON(const SemanticHighlightingInformation &Highlighting); + +/// Parameters for the semantic highlighting (server-side) push notification. +struct SemanticHighlightingParams { + /// The textdocument these highlightings belong to. + TextDocumentIdentifier TextDocument; + /// The lines of highlightings that should be sent. + std::vector Lines; +}; +llvm::json::Value toJSON(const SemanticHighlightingParams &Highlighting); + } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp index 420693653df3..a4517af9fb6a 100644 --- a/clang-tools-extra/clangd/SemanticHighlighting.cpp +++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp @@ -8,6 +8,7 @@ #include "SemanticHighlighting.h" #include "Logger.h" +#include "Protocol.h" #include "SourceCode.h" #include "clang/AST/ASTContext.h" #include "clang/AST/RecursiveASTVisitor.h" @@ -63,6 +64,48 @@ private: } }; +// Encode binary data into base64. +// This was copied from compiler-rt/lib/fuzzer/FuzzerUtil.cpp. +// FIXME: Factor this out into llvm/Support? +std::string encodeBase64(const llvm::SmallVectorImpl &Bytes) { + static const char Table[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789+/"; + std::string Res; + size_t I; + for (I = 0; I + 2 < Bytes.size(); I += 3) { + uint32_t X = (Bytes[I] << 16) + (Bytes[I + 1] << 8) + Bytes[I + 2]; + Res += Table[(X >> 18) & 63]; + Res += Table[(X >> 12) & 63]; + Res += Table[(X >> 6) & 63]; + Res += Table[X & 63]; + } + if (I + 1 == Bytes.size()) { + uint32_t X = (Bytes[I] << 16); + Res += Table[(X >> 18) & 63]; + Res += Table[(X >> 12) & 63]; + Res += "=="; + } else if (I + 2 == Bytes.size()) { + uint32_t X = (Bytes[I] << 16) + (Bytes[I + 1] << 8); + Res += Table[(X >> 18) & 63]; + Res += Table[(X >> 12) & 63]; + Res += Table[(X >> 6) & 63]; + Res += "="; + } + return Res; +} + +void write32be(uint32_t I, llvm::raw_ostream &OS) { + std::array Buf; + llvm::support::endian::write32be(Buf.data(), I); + OS.write(Buf.data(), Buf.size()); +} + +void write16be(uint16_t I, llvm::raw_ostream &OS) { + std::array Buf; + llvm::support::endian::write16be(Buf.data(), I); + OS.write(Buf.data(), Buf.size()); +} } // namespace bool operator==(const HighlightingToken &Lhs, const HighlightingToken &Rhs) { @@ -73,5 +116,50 @@ std::vector getSemanticHighlightings(ParsedAST &AST) { return HighlightingTokenCollector(AST).collectTokens(); } +std::vector +toSemanticHighlightingInformation(llvm::ArrayRef Tokens) { + if (Tokens.size() == 0) + return {}; + + // FIXME: Tokens might be multiple lines long (block comments) in this case + // this needs to add multiple lines for those tokens. + std::map> TokenLines; + for (const HighlightingToken &Token : Tokens) + TokenLines[Token.R.start.line].push_back(Token); + + std::vector Lines; + Lines.reserve(TokenLines.size()); + for (const auto &Line : TokenLines) { + llvm::SmallVector LineByteTokens; + llvm::raw_svector_ostream OS(LineByteTokens); + for (const auto &Token : Line.second) { + // Writes the token to LineByteTokens in the byte format specified by the + // LSP proposal. Described below. + // |<---- 4 bytes ---->|<-- 2 bytes -->|<--- 2 bytes -->| + // | character | length | index | + + write32be(Token.R.start.character, OS); + write16be(Token.R.end.character - Token.R.start.character, OS); + write16be(static_cast(Token.Kind), OS); + } + + Lines.push_back({Line.first, encodeBase64(LineByteTokens)}); + } + + return Lines; +} + +std::vector> getTextMateScopeLookupTable() { + // FIXME: Add scopes for C and Objective C. + std::map> Scopes = { + {HighlightingKind::Variable, {"variable.cpp"}}, + {HighlightingKind::Function, {"entity.name.function.cpp"}}}; + std::vector> NestedScopes(Scopes.size()); + for (const auto &Scope : Scopes) + NestedScopes[static_cast(Scope.first)] = Scope.second; + + return NestedScopes; +} + } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/SemanticHighlighting.h b/clang-tools-extra/clangd/SemanticHighlighting.h index 31007f0fc63c..345c29ecd5c5 100644 --- a/clang-tools-extra/clangd/SemanticHighlighting.h +++ b/clang-tools-extra/clangd/SemanticHighlighting.h @@ -5,18 +5,27 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// +// +// An implementation of semantic highlighting based on this proposal: +// https://github.com/microsoft/vscode-languageserver-node/pull/367 in clangd. +// Semantic highlightings are calculated for an AST by visiting every AST node +// and classifying nodes that are interesting to highlight (variables/function +// calls etc.). +// +//===----------------------------------------------------------------------===// #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTICHIGHLIGHTING_H #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTICHIGHLIGHTING_H #include "ClangdUnit.h" +#include "Protocol.h" namespace clang { namespace clangd { enum class HighlightingKind { - Variable, - Function, + Variable = 0, + Function = 1, }; // Contains all information needed for the highlighting a token. @@ -31,6 +40,14 @@ bool operator==(const HighlightingToken &Lhs, const HighlightingToken &Rhs); // main AST. std::vector getSemanticHighlightings(ParsedAST &AST); +// Gets the TextMate scopes as a double nested array where the +// SemanticHighlightKind indexes correctly into this vector. +std::vector> getTextMateScopeLookupTable(); + +// Convert to LSP's semantic highlighting information. +std::vector +toSemanticHighlightingInformation(llvm::ArrayRef Tokens); + } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/test/semantic-highlighting.test b/clang-tools-extra/clangd/test/semantic-highlighting.test new file mode 100644 index 000000000000..9afec3729c58 --- /dev/null +++ b/clang-tools-extra/clangd/test/semantic-highlighting.test @@ -0,0 +1,75 @@ +# RUN: clangd -lit-test < %s | FileCheck -strict-whitespace %s +{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{"textDocument":{"semanticHighlightingCapabilities":{"semanticHighlighting":true}}},"trace":"off"}} +--- +# CHECK: "id": 0, +# CHECK-NEXT: "jsonrpc": "2.0", +# CHECK-NEXT: "result": { +# CHECK-NEXT: "capabilities": { +# CHECK-NEXT: "codeActionProvider": true, +# CHECK-NEXT: "completionProvider": { +# CHECK-NEXT: "resolveProvider": false, +# CHECK-NEXT: "triggerCharacters": [ +# CHECK-NEXT: ".", +# CHECK-NEXT: ">", +# CHECK-NEXT: ":" +# CHECK-NEXT: ] +# CHECK-NEXT: }, +# CHECK-NEXT: "declarationProvider": true, +# CHECK-NEXT: "definitionProvider": true, +# CHECK-NEXT: "documentFormattingProvider": true, +# CHECK-NEXT: "documentHighlightProvider": true, +# CHECK-NEXT: "documentOnTypeFormattingProvider": { +# CHECK-NEXT: "firstTriggerCharacter": "\n", +# CHECK-NEXT: "moreTriggerCharacter": [] +# CHECK-NEXT: }, +# CHECK-NEXT: "documentRangeFormattingProvider": true, +# CHECK-NEXT: "documentSymbolProvider": true, +# CHECK-NEXT: "executeCommandProvider": { +# CHECK-NEXT: "commands": [ +# CHECK-NEXT: "clangd.applyFix", +# CHECK-NEXT: "clangd.applyTweak" +# CHECK-NEXT: ] +# CHECK-NEXT: }, +# CHECK-NEXT: "hoverProvider": true, +# CHECK-NEXT: "referencesProvider": true, +# CHECK-NEXT: "renameProvider": true, +# CHECK-NEXT: "semanticHighlighting": { +# CHECK-NEXT: "scopes": [ +# CHECK-NEXT: [ +# CHECK-NEXT: "variable.cpp" +# CHECK-NEXT: ], +# CHECK-NEXT: [ +# CHECK-NEXT: "entity.name.function.cpp" +# CHECK-NEXT: ] +# CHECK-NEXT: ] +# CHECK-NEXT: }, +# CHECK-NEXT: "signatureHelpProvider": { +# CHECK-NEXT: "triggerCharacters": [ +# CHECK-NEXT: "(", +# CHECK-NEXT: "," +# CHECK-NEXT: ] +# CHECK-NEXT: }, +# CHECK-NEXT: "textDocumentSync": 2, +# CHECK-NEXT: "typeHierarchyProvider": true +# CHECK-NEXT: "workspaceSymbolProvider": true +# CHECK-NEXT: } +# CHECK-NEXT: } +--- +{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///foo.cpp","languageId":"cpp","version":1,"text":"int x = 2;"}}} +# CHECK: "method": "textDocument/semanticHighlighting", +# CHECK-NEXT: "params": { +# CHECK-NEXT: "lines": [ +# CHECK-NEXT: { +# CHECK-NEXT: "line": 0, +# CHECK-NEXT: "tokens": "AAAABAABAAA=" +# CHECK-NEXT: } +# CHECK-NEXT: ], +# CHECK-NEXT: "textDocument": { +# CHECK-NEXT: "uri": "file:///clangd-test/foo.cpp" +# CHECK-NEXT: } +# CHECK-NEXT: } +# CHECK-NEXT:} +--- +{"jsonrpc":"2.0","id":3,"method":"shutdown"} +--- +{"jsonrpc":"2.0","method":"exit"} diff --git a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp index d7c6d4d502ab..a5c793251a4a 100644 --- a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp +++ b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp @@ -8,6 +8,7 @@ #include "Annotations.h" #include "ClangdServer.h" +#include "Protocol.h" #include "SemanticHighlighting.h" #include "TestFS.h" #include "TestTU.h" @@ -66,7 +67,7 @@ TEST(SemanticHighlighting, GetsCorrectTokens) { } } -TEST(ClangdSemanticHighlightingTest, GeneratesHighlightsWhenFileChange) { +TEST(SemanticHighlighting, GeneratesHighlightsWhenFileChange) { class HighlightingsCounterDiagConsumer : public DiagnosticsConsumer { public: std::atomic Count = {0}; @@ -90,6 +91,29 @@ TEST(ClangdSemanticHighlightingTest, GeneratesHighlightsWhenFileChange) { ASSERT_EQ(DiagConsumer.Count, 1); } +TEST(SemanticHighlighting, toSemanticHighlightingInformation) { + auto CreatePosition = [](int Line, int Character) -> Position { + Position Pos; + Pos.line = Line; + Pos.character = Character; + return Pos; + }; + + std::vector Tokens{ + {HighlightingKind::Variable, + Range{CreatePosition(3, 8), CreatePosition(3, 12)}}, + {HighlightingKind::Function, + Range{CreatePosition(3, 4), CreatePosition(3, 7)}}, + {HighlightingKind::Variable, + Range{CreatePosition(1, 1), CreatePosition(1, 5)}}}; + std::vector ActualResults = + toSemanticHighlightingInformation(Tokens); + std::vector ExpectedResults = { + {1, "AAAAAQAEAAA="}, + {3, "AAAACAAEAAAAAAAEAAMAAQ=="}}; + EXPECT_EQ(ActualResults, ExpectedResults); +} + } // namespace } // namespace clangd } // namespace clang