More token-annotation experimentation, preprocessing the annotated

token sequence to detect macro instantiations (that produce at least
token). WIP.

llvm-svn: 98826

clang/include/clang-c/Index.h

@@ -800,9 +800,9 @@ enum CXCursorKind {
      
   /* Preprocessing */
   CXCursor_PreprocessingDirective        = 500,
-  
+  CXCursor_MacroInstantiation            = 501,
   CXCursor_FirstPreprocessing            = CXCursor_PreprocessingDirective,
-  CXCursor_LastPreprocessing             = CXCursor_PreprocessingDirective
+  CXCursor_LastPreprocessing             = CXCursor_MacroInstantiation
 };
 
 /**

clang/test/Index/annotate-tokens-pp.c

@@ -1,6 +1,6 @@
 #define BAR baz
-#define WIBBLE(X, Y)
-WIBBLE(int, float)
+#define WIBBLE(X, Y) X##Y
+float WIBBLE(int, float);
 int BAR;
 #include "foo.h"
 
@@ -17,14 +17,15 @@ int BAR;
 // CHECK: Punctuation: "," [2:17 - 2:18] preprocessing directive=
 // CHECK: Identifier: "Y" [2:19 - 2:20] preprocessing directive=
 // CHECK: Punctuation: ")" [2:20 - 2:21] preprocessing directive=
-// CHECK: Identifier: "WIBBLE" [3:1 - 3:7]
-// CHECK: Punctuation: "(" [3:7 - 3:8]
-// CHECK: Keyword: "int" [3:8 - 3:11]
-// CHECK: Punctuation: "," [3:11 - 3:12]
-// CHECK: Keyword: "float" [3:13 - 3:18]
-// CHECK: Punctuation: ")" [3:18 - 3:19]
+// CHECK: Identifier: "WIBBLE" [3:7 - 3:13] macro instantiation=
+// CHECK: Punctuation: "(" [3:13 - 3:14]
+// CHECK: Keyword: "int" [3:14 - 3:17]
+// CHECK: Punctuation: "," [3:17 - 3:18]
+// CHECK: Keyword: "float" [3:19 - 3:24]
+// CHECK: Punctuation: ")" [3:24 - 3:25]
+// CHECK: Punctuation: ";" [3:25 - 3:26]
 // CHECK: Keyword: "int" [4:1 - 4:4]
-// CHECK: Identifier: "BAR" [4:5 - 4:8]
+// CHECK: Identifier: "BAR" [4:5 - 4:8] macro instantiation=
 // CHECK: Punctuation: ";" [4:8 - 4:9]
 // CHECK: Punctuation: "#" [5:1 - 5:2] preprocessing directive=
 // CHECK: Identifier: "include" [5:2 - 5:9] preprocessing directive=

clang/tools/CIndex/CIndex.cpp

@@ -1522,6 +1522,8 @@ CXString clang_getCursorKindSpelling(enum CXCursorKind Kind) {
      return createCXString("attribute(iboutlet)");
   case CXCursor_PreprocessingDirective:
     return createCXString("preprocessing directive");
+  case CXCursor_MacroInstantiation:
+    return createCXString("macro instantiation");
   }
 
   llvm_unreachable("Unhandled CXCursorKind");
@@ -1652,6 +1654,11 @@ CXSourceLocation clang_getCursorLocation(CXCursor C) {
     SourceLocation L = cxcursor::getCursorPreprocessingDirective(C).getBegin();
     return cxloc::translateSourceLocation(getCursorContext(C), L);
   }
+
+  if (C.kind == CXCursor_MacroInstantiation) {
+    SourceLocation L = cxcursor::getCursorMacroInstantiation(C).getBegin();
+    return cxloc::translateSourceLocation(getCursorContext(C), L);
+  }
   
   if (!getCursorDecl(C))
     return clang_getNullLocation();
@@ -1708,6 +1715,11 @@ CXSourceRange clang_getCursorExtent(CXCursor C) {
     SourceRange R = cxcursor::getCursorPreprocessingDirective(C);
     return cxloc::translateSourceRange(getCursorContext(C), R);
   }
+
+  if (C.kind == CXCursor_MacroInstantiation) {
+    SourceRange R = cxcursor::getCursorMacroInstantiation(C);
+    return cxloc::translateSourceRange(getCursorContext(C), R);
+  }
   
   if (!getCursorDecl(C))
     return clang_getNullRange();
@@ -2017,6 +2029,17 @@ void clang_enableStackTraces(void) {
 // Token-based Operations.
 //===----------------------------------------------------------------------===//
 
+namespace {
+/// IgnoringDiagClient - This is a diagnostic client that just ignores all
+/// diags.
+class IgnoringDiagClient : public DiagnosticClient {
+  void HandleDiagnostic(Diagnostic::Level DiagLevel,
+                        const DiagnosticInfo &Info) {
+    // Just ignore it.
+  }
+};
+}
+
 /* CXToken layout:
  *   int_data[0]: a CXTokenKind
  *   int_data[1]: starting token location
@@ -2281,9 +2304,8 @@ void clang_annotateTokens(CXTranslationUnit TU,
     // Lex tokens in raw mode until we hit the end of the range, to avoid 
     // entering #includes or expanding macros.
     std::vector<Token> TokenStream;
-    const char *EffectiveBufferEnd = Buffer.data() + EndLocInfo.second;
     Preprocessor &PP = CXXUnit->getPreprocessor();
-    while (Lex.getBufferLocation() <= EffectiveBufferEnd) {
+    while (true) {
       Token Tok;
       Lex.LexFromRawLexer(Tok);
       
@@ -2311,23 +2333,21 @@ void clang_annotateTokens(CXTranslationUnit TU,
           Annotated[Locations[I].getRawEncoding()] = Cursor;
         }
         
-        if (Tok.is(tok::eof))
-          break;
-        
         if (Tok.isAtStartOfLine())
           goto reprocess;
         
         continue;
       }
       
-      // If this is a ## token, change its kind to unknown so that repreprocessing
-      // it will not produce an error.
+      // If this is a ## token, change its kind to unknown so that
+      // repreprocessing it will not produce an error.
       if (Tok.is(tok::hashhash))
         Tok.setKind(tok::unknown);
       
-      // If this raw token is an identifier, the raw lexer won't have looked up
-      // the corresponding identifier info for it.  Do this now so that it will be
-      // macro expanded when we re-preprocess it.
+      // If this raw token is an identifier, the raw lexer won't have
+      // looked up the corresponding identifier info for it.  Do this
+      // now so that it will be macro expanded when we re-preprocess
+      // it.
       if (Tok.is(tok::identifier)) {
         // Change the kind of this identifier to the appropriate token kind, e.g.
         // turning "for" into a keyword.
@@ -2336,9 +2356,67 @@ void clang_annotateTokens(CXTranslationUnit TU,
       
       TokenStream.push_back(Tok);
       
-      if (Tok.is(tok::eof)) 
+      if (Tok.is(tok::eof))
         break;
     }
+
+    // Temporarily change the diagnostics object so that we ignore any
+    // generated diagnostics from this pass.
+    IgnoringDiagClient TmpDC;
+    Diagnostic TmpDiags(&TmpDC);
+    Diagnostic *OldDiags = &PP.getDiagnostics();
+    PP.setDiagnostics(TmpDiags);
+
+    // Inform the preprocessor that we don't want comments.
+    PP.SetCommentRetentionState(false, false);
+    
+    // Enter the tokens we just lexed.  This will cause them to be macro expanded
+    // but won't enter sub-files (because we removed #'s).
+    PP.EnterTokenStream(&TokenStream[0], TokenStream.size(), false, false);
+
+    // Lex all the tokens.
+    Token Tok;
+    PP.Lex(Tok);
+    while (Tok.isNot(tok::eof)) {
+      // Ignore non-macro tokens.
+      if (!Tok.getLocation().isMacroID()) {
+        PP.Lex(Tok);
+        continue;
+      }
+
+      // Okay, we have the first token of a macro expansion. Keep
+      // track of the range of the macro expansion.
+      std::pair<SourceLocation, SourceLocation> LLoc =
+        SourceMgr.getInstantiationRange(Tok.getLocation());
+
+      // Ignore tokens whose instantiation location was not the main file.
+      if (SourceMgr.getFileID(LLoc.first) != BeginLocInfo.first) {
+        PP.Lex(Tok);
+        continue;
+      }
+
+      assert(SourceMgr.getFileID(LLoc.second) == BeginLocInfo.first &&
+             "Start and end of expansion must be in the same ultimate file!");
+
+      // Okay, eat this token, getting the next one.
+      PP.Lex(Tok);
+
+      // Skip all the rest of the tokens that are part of this macro
+      // instantiation.  It would be really nice to pop up a window with all the
+      // spelling of the tokens or something.
+      while (!Tok.is(tok::eof) &&
+             SourceMgr.getInstantiationLoc(Tok.getLocation()) == LLoc.first)
+        PP.Lex(Tok);
+
+      CXCursor Cursor
+        = cxcursor::MakeMacroInstantiationCursor(SourceRange(LLoc.first, 
+                                                             LLoc.second), 
+                                                 CXXUnit);
+      Annotated[LLoc.first.getRawEncoding()] = Cursor;
+    }
+
+    // Restore diagnostics object back to its own thing.
+    PP.setDiagnostics(*OldDiags);
   }
   
   for (unsigned I = 0; I != NumTokens; ++I) {

clang/tools/CIndex/CXCursor.cpp

@@ -314,6 +314,24 @@ SourceRange cxcursor::getCursorPreprocessingDirective(CXCursor C) {
                                       reinterpret_cast<uintptr_t> (C.data[1])));
 }
 
+CXCursor cxcursor::MakeMacroInstantiationCursor(SourceRange Range, 
+                                                ASTUnit *TU) {
+  CXCursor C = { CXCursor_MacroInstantiation, 
+                 { reinterpret_cast<void *>(Range.getBegin().getRawEncoding()),
+                   reinterpret_cast<void *>(Range.getEnd().getRawEncoding()),
+                   TU }
+               };
+  return C;
+}
+
+SourceRange cxcursor::getCursorMacroInstantiation(CXCursor C) {
+  assert(C.kind == CXCursor_MacroInstantiation);
+  return SourceRange(SourceLocation::getFromRawEncoding(
+                                      reinterpret_cast<uintptr_t> (C.data[0])),
+                     SourceLocation::getFromRawEncoding(
+                                      reinterpret_cast<uintptr_t> (C.data[1])));
+}
+
 Decl *cxcursor::getCursorDecl(CXCursor Cursor) {
   return (Decl *)Cursor.data[0];
 }

clang/tools/CIndex/CXCursor.h

@@ -78,7 +78,14 @@ CXCursor MakePreprocessingDirectiveCursor(SourceRange Range, ASTUnit *TU);
 
 /// \brief Unpack a given preprocessing directive to retrieve its source range.
 SourceRange getCursorPreprocessingDirective(CXCursor C);
-  
+
+/// \brief Create a macro instantiation cursor.
+CXCursor MakeMacroInstantiationCursor(SourceRange Range, ASTUnit *TU);
+
+/// \brief Unpack a given macro instantiation cursor to retrieve its
+/// source range.
+SourceRange getCursorMacroInstantiation(CXCursor C);
+
 Decl *getCursorDecl(CXCursor Cursor);
 Expr *getCursorExpr(CXCursor Cursor);
 Stmt *getCursorStmt(CXCursor Cursor);