[clangd] Add highlighting for macro expansions.

Summary: https://github.com/clangd/clangd/issues/134

Reviewers: hokein, ilya-biryukov

Subscribers: MaskRay, jkorous, arphaman, kadircet, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D66995

llvm-svn: 370482

clang-tools-extra/clangd/SemanticHighlighting.cpp

@@ -24,16 +24,20 @@ namespace {
 class HighlightingTokenCollector
     : public RecursiveASTVisitor<HighlightingTokenCollector> {
   std::vector<HighlightingToken> Tokens;
-  ASTContext &Ctx;
-  const SourceManager &SM;
+  ParsedAST &AST;
 
 public:
-  HighlightingTokenCollector(ParsedAST &AST)
-      : Ctx(AST.getASTContext()), SM(AST.getSourceManager()) {}
+  HighlightingTokenCollector(ParsedAST &AST) : AST(AST) {}
 
   std::vector<HighlightingToken> collectTokens() {
     Tokens.clear();
-    TraverseAST(Ctx);
+    TraverseAST(AST.getASTContext());
+    // Add highlightings for macro expansions as they are not traversed by the
+    // visitor.
+    // FIXME: Should add highlighting to the macro definitions as well. But this
+    // information is not collected in ParsedAST right now.
+    for (const SourceLocation &L : AST.getMainFileExpansions())
+      addToken(L, HighlightingKind::Macro);
     // Initializer lists can give duplicates of tokens, therefore all tokens
     // must be deduplicated.
     llvm::sort(Tokens);
@@ -264,6 +268,7 @@ private:
   }
 
   void addToken(SourceLocation Loc, HighlightingKind Kind) {
+    const auto &SM = AST.getSourceManager();
     if (Loc.isMacroID()) {
       // Only intereseted in highlighting arguments in macros (DEF_X(arg)).
       if (!SM.isMacroArgExpansion(Loc))
@@ -279,7 +284,7 @@ private:
     if (!isInsideMainFile(Loc, SM))
       return;
 
-    auto R = getTokenRange(SM, Ctx.getLangOpts(), Loc);
+    auto R = getTokenRange(SM, AST.getASTContext().getLangOpts(), Loc);
     if (!R) {
       // R should always have a value, if it doesn't something is very wrong.
       elog("Tried to add semantic token with an invalid range");
@@ -466,6 +471,8 @@ llvm::StringRef toTextMateScope(HighlightingKind Kind) {
     return "entity.name.type.template.cpp";
   case HighlightingKind::Primitive:
     return "storage.type.primitive.cpp";
+  case HighlightingKind::Macro:
+    return "entity.name.function.preprocessor.cpp";
   case HighlightingKind::NumKinds:
     llvm_unreachable("must not pass NumKinds to the function");
   }

clang-tools-extra/clangd/SemanticHighlighting.h

@@ -38,6 +38,7 @@ enum class HighlightingKind {
   Namespace,
   TemplateParameter,
   Primitive,
+  Macro,
 
   NumKinds,
 };

clang-tools-extra/clangd/test/semantic-highlighting.test

@@ -45,6 +45,9 @@
 # CHECK-NEXT:          ],
 # CHECK-NEXT:          [
 # CHECK-NEXT:            "storage.type.primitive.cpp"
+# CHECK-NEXT:          ],
+# CHECK-NEXT:          [
+# CHECK-NEXT:            "entity.name.function.preprocessor.cpp"
 # CHECK-NEXT:          ]
 # CHECK-NEXT:        ]
 # CHECK-NEXT:      },

clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

@@ -47,7 +47,8 @@ std::vector<HighlightingToken> getExpectedTokens(Annotations &Test) {
       {HighlightingKind::Method, "Method"},
       {HighlightingKind::StaticMethod, "StaticMethod"},
       {HighlightingKind::TemplateParameter, "TemplateParameter"},
-      {HighlightingKind::Primitive, "Primitive"}};
+      {HighlightingKind::Primitive, "Primitive"},
+      {HighlightingKind::Macro, "Macro"}};
   std::vector<HighlightingToken> ExpectedTokens;
   for (const auto &KindString : KindToString) {
     std::vector<HighlightingToken> Toks = makeHighlightingTokens(
@@ -391,9 +392,9 @@ TEST(SemanticHighlighting, GetsCorrectTokens) {
       R"cpp(
       #define DEF_MULTIPLE(X) namespace X { class X { int X; }; }
       #define DEF_CLASS(T) class T {};
-      DEF_MULTIPLE(XYZ);
-      DEF_MULTIPLE(XYZW);
-      DEF_CLASS($Class[[A]])
+      $Macro[[DEF_MULTIPLE]](XYZ);
+      $Macro[[DEF_MULTIPLE]](XYZW);
+      $Macro[[DEF_CLASS]]($Class[[A]])
       #define MACRO_CONCAT(X, V, T) T foo##X = V
       #define DEF_VAR(X, V) int X = V
       #define DEF_VAR_T(T, X, V) T X = V
@@ -404,26 +405,27 @@ TEST(SemanticHighlighting, GetsCorrectTokens) {
       #define SOME_NAME_SET variable2 = 123
       #define INC_VAR(X) X += 2
       $Primitive[[void]] $Function[[foo]]() {
-        DEF_VAR($LocalVariable[[X]],  123);
-        DEF_VAR_REV(908, $LocalVariable[[XY]]);
-        $Primitive[[int]] CPY( $LocalVariable[[XX]] );
-        DEF_VAR_TYPE($Class[[A]], $LocalVariable[[AA]]);
-        $Primitive[[double]] SOME_NAME;
-        $Primitive[[int]] SOME_NAME_SET;
+        $Macro[[DEF_VAR]]($LocalVariable[[X]],  123);
+        $Macro[[DEF_VAR_REV]](908, $LocalVariable[[XY]]);
+        $Primitive[[int]] $Macro[[CPY]]( $LocalVariable[[XX]] );
+        $Macro[[DEF_VAR_TYPE]]($Class[[A]], $LocalVariable[[AA]]);
+        $Primitive[[double]] $Macro[[SOME_NAME]];
+        $Primitive[[int]] $Macro[[SOME_NAME_SET]];
         $LocalVariable[[variable]] = 20.1;
-        MACRO_CONCAT(var, 2, $Primitive[[float]]);
-        DEF_VAR_T($Class[[A]], CPY(CPY($LocalVariable[[Nested]])),
-              CPY($Class[[A]]()));
-        INC_VAR($LocalVariable[[variable]]);
+        $Macro[[MACRO_CONCAT]](var, 2, $Primitive[[float]]);
+        $Macro[[DEF_VAR_T]]($Class[[A]], $Macro[[CPY]](
+              $Macro[[CPY]]($LocalVariable[[Nested]])),
+            $Macro[[CPY]]($Class[[A]]()));
+        $Macro[[INC_VAR]]($LocalVariable[[variable]]);
       }
-      $Primitive[[void]] SOME_NAME();
-      DEF_VAR($Variable[[XYZ]], 567);
-      DEF_VAR_REV(756, $Variable[[AB]]);
+      $Primitive[[void]] $Macro[[SOME_NAME]]();
+      $Macro[[DEF_VAR]]($Variable[[XYZ]], 567);
+      $Macro[[DEF_VAR_REV]](756, $Variable[[AB]]);
 
       #define CALL_FN(F) F();
       #define DEF_FN(F) void F ()
-      DEF_FN($Function[[g]]) {
-        CALL_FN($Function[[foo]]);
+      $Macro[[DEF_FN]]($Function[[g]]) {
+        $Macro[[CALL_FN]]($Function[[foo]]);
       }
     )cpp",
       R"cpp(
@@ -433,8 +435,8 @@ TEST(SemanticHighlighting, GetsCorrectTokens) {
       $Primitive[[int]] $Variable[[y]];
       $Primitive[[int]] $Function[[f]]();
       $Primitive[[void]] $Function[[foo]]() {
-        assert($Variable[[x]] != $Variable[[y]]);
-        assert($Variable[[x]] != $Function[[f]]());
+        $Macro[[assert]]($Variable[[x]] != $Variable[[y]]);
+        $Macro[[assert]]($Variable[[x]] != $Function[[f]]());
       }
     )cpp",
     R"cpp(
@@ -470,8 +472,8 @@ TEST(SemanticHighlighting, GetsCorrectTokens) {
   // A separate test for macros in headers.
   checkHighlightings(R"cpp(
     #include "imp.h"
-    DEFINE_Y
-    DXYZ_Y(A);
+    $Macro[[DEFINE_Y]]
+    $Macro[[DXYZ_Y]](A);
   )cpp",
                      {{"imp.h", R"cpp(
     #define DXYZ(X) class X {};