Teach Preprocessor::macro_begin/macro_end to lazily load all macro

definitions from a precompiled header. This ensures that
code-completion with macro names behaves the same with or without
precompiled headers.

llvm-svn: 92497

clang/include/clang/Frontend/PCHReader.h

@@ -20,6 +20,7 @@
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/Type.h"
 #include "clang/AST/TemplateBase.h"
+#include "clang/Lex/ExternalPreprocessorSource.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/SourceManager.h"
@@ -146,7 +147,8 @@ public:
 /// required when traversing the AST. Only those AST nodes that are
 /// actually required will be de-serialized.
 class PCHReader
-  : public ExternalSemaSource,
+  : public ExternalPreprocessorSource,
+    public ExternalSemaSource,
     public IdentifierInfoLookup,
     public ExternalIdentifierLookup,
     public ExternalSLocEntrySource {
@@ -183,6 +185,10 @@ private:
   llvm::BitstreamReader StreamFile;
   llvm::BitstreamCursor Stream;
 
+  /// \brief The cursor to the start of the preprocessor block, which stores
+  /// all of the macro definitions.
+  llvm::BitstreamCursor MacroCursor;
+      
   /// DeclsCursor - This is a cursor to the start of the DECLS_BLOCK block.  It
   /// has read all the abbreviations at the start of the block and is ready to
   /// jump around with these in context.
@@ -634,8 +640,7 @@ public:
   /// This routine builds a new IdentifierInfo for the given identifier. If any
   /// declarations with this name are visible from translation unit scope, their
   /// declarations will be deserialized and introduced into the declaration
-  /// chain of the identifier. FIXME: if this identifier names a macro,
-  /// deserialize the macro.
+  /// chain of the identifier.
   virtual IdentifierInfo* get(const char *NameStart, const char *NameEnd);
   IdentifierInfo* get(llvm::StringRef Name) {
     return get(Name.begin(), Name.end());
@@ -712,6 +717,9 @@ public:
   /// \brief Reads the macro record located at the given offset.
   void ReadMacroRecord(uint64_t Offset);
 
+  /// \brief Read the set of macros defined by this external macro source.
+  virtual void ReadDefinedMacros();
+
   /// \brief Retrieve the AST context that this PCH reader
   /// supplements.
   ASTContext *getContext() { return Context; }

clang/include/clang/Lex/ExternalPreprocessorSource.h

@@ -0,0 +1,34 @@
+//===- ExternalPreprocessorSource.h - Abstract Macro Interface --*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the ExternalPreprocessorSource interface, which enables
+//  construction of macro definitions from some external source.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_CLANG_LEX_EXTERNAL_PREPROCESSOR_SOURCE_H
+#define LLVM_CLANG_LEX_EXTERNAL_PREPROCESSOR_SOURCE_H
+
+namespace clang {
+  
+/// \brief Abstract interface for external sources of preprocessor 
+/// information.
+///
+/// This abstract class allows an external sources (such as the \c PCHReader) 
+/// to provide additional macro definitions.
+class ExternalPreprocessorSource {
+public:
+  virtual ~ExternalPreprocessorSource();
+  
+  /// \brief Read the set of macros defined by this external macro source.
+  virtual void ReadDefinedMacros() = 0;
+};
+  
+}
+
+#endif // LLVM_CLANG_LEX_EXTERNAL_PREPROCESSOR_SOURCE_H

clang/include/clang/Lex/Preprocessor.h

@@ -32,6 +32,7 @@
 namespace clang {
 
 class SourceManager;
+class ExternalPreprocessorSource;
 class FileManager;
 class FileEntry;
 class HeaderSearch;
@@ -42,7 +43,7 @@ class ScratchBuffer;
 class TargetInfo;
 class PPCallbacks;
 class DirectoryLookup;
-
+  
 /// Preprocessor - This object engages in a tight little dance with the lexer to
 /// efficiently preprocess tokens.  Lexers know only about tokens within a
 /// single source file, and don't know anything about preprocessor-level issues
@@ -57,6 +58,9 @@ class Preprocessor {
   ScratchBuffer     *ScratchBuf;
   HeaderSearch      &HeaderInfo;
 
+  /// \brief External source of macros.
+  ExternalPreprocessorSource *ExternalSource;
+  
   /// PTH - An optional PTHManager object used for getting tokens from
   ///  a token cache rather than lexing the original source file.
   llvm::OwningPtr<PTHManager> PTH;
@@ -99,6 +103,9 @@ class Preprocessor {
   /// DisableMacroExpansion - True if macro expansion is disabled.
   bool DisableMacroExpansion : 1;
 
+  /// \brief Whether we have already loaded macros from the external source.
+  mutable bool ReadMacrosFromExternalSource : 1;
+  
   /// Identifiers - This is mapping/lookup information for all identifiers in
   /// the program, including program keywords.
   mutable IdentifierTable Identifiers;
@@ -247,6 +254,14 @@ public:
 
   PTHManager *getPTHManager() { return PTH.get(); }
 
+  void setExternalSource(ExternalPreprocessorSource *Source) {
+    ExternalSource = Source;
+  }
+  
+  ExternalPreprocessorSource *getExternalSource() const {
+    return ExternalSource;
+  }
+  
   /// SetCommentRetentionState - Control whether or not the preprocessor retains
   /// comments in output.
   void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
@@ -296,11 +311,9 @@ public:
   /// state of the macro table.  This visits every currently-defined macro.
   typedef llvm::DenseMap<IdentifierInfo*,
                          MacroInfo*>::const_iterator macro_iterator;
-  macro_iterator macro_begin() const { return Macros.begin(); }
-  macro_iterator macro_end() const { return Macros.end(); }
-
-
-
+  macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
+  macro_iterator macro_end(bool IncludeExternalMacros = true) const;
+  
   const std::string &getPredefines() const { return Predefines; }
   /// setPredefines - Set the predefines for this Preprocessor.  These
   /// predefines are automatically injected when parsing the main file.

clang/lib/Frontend/PCHReader.cpp

@@ -1079,6 +1079,61 @@ void PCHReader::ReadMacroRecord(uint64_t Offset) {
   }
 }
 
+void PCHReader::ReadDefinedMacros() {
+  // If there was no preprocessor block, do nothing.
+  if (!MacroCursor.getBitStreamReader())
+    return;
+  
+  llvm::BitstreamCursor Cursor = MacroCursor;
+  if (Cursor.EnterSubBlock(pch::PREPROCESSOR_BLOCK_ID)) {
+    Error("malformed preprocessor block record in PCH file");
+    return;
+  }
+  
+  RecordData Record;
+  while (true) {
+    unsigned Code = Cursor.ReadCode();
+    if (Code == llvm::bitc::END_BLOCK) {
+      if (Cursor.ReadBlockEnd())
+        Error("error at end of preprocessor block in PCH file");
+      return;
+    }
+    
+    if (Code == llvm::bitc::ENTER_SUBBLOCK) {
+      // No known subblocks, always skip them.
+      Cursor.ReadSubBlockID();
+      if (Cursor.SkipBlock()) {
+        Error("malformed block record in PCH file");
+        return;
+      }
+      continue;
+    }
+    
+    if (Code == llvm::bitc::DEFINE_ABBREV) {
+      Cursor.ReadAbbrevRecord();
+      continue;
+    }
+    
+    // Read a record.
+    const char *BlobStart;
+    unsigned BlobLen;
+    Record.clear();
+    switch (Cursor.ReadRecord(Code, Record, &BlobStart, &BlobLen)) {
+    default:  // Default behavior: ignore.
+      break;
+        
+    case pch::PP_MACRO_OBJECT_LIKE:
+    case pch::PP_MACRO_FUNCTION_LIKE:
+        DecodeIdentifierInfo(Record[0]);
+      break;
+
+    case pch::PP_TOKEN:
+      // Ignore tokens.
+      break;
+    }
+  }
+}
+
 /// \brief If we are loading a relocatable PCH file, and the filename is
 /// not an absolute path, add the system root to the beginning of the file
 /// name.
@@ -1140,6 +1195,10 @@ PCHReader::ReadPCHBlock() {
         break;
 
       case pch::PREPROCESSOR_BLOCK_ID:
+        MacroCursor = Stream;
+        if (PP)
+          PP->setExternalSource(this);
+
         if (Stream.SkipBlock()) {
           Error("malformed block record in PCH file");
           return Failure;
@@ -1494,7 +1553,8 @@ void PCHReader::InitializeContext(ASTContext &Ctx) {
   assert(PP && "Forgot to set Preprocessor ?");
   PP->getIdentifierTable().setExternalIdentifierLookup(this);
   PP->getHeaderSearchInfo().SetExternalLookup(this);
-
+  PP->setExternalSource(this);
+  
   // Load the translation unit declaration
   ReadDeclRecord(DeclOffsets[0], 0);
 

clang/lib/Frontend/PCHWriter.cpp

@@ -1148,7 +1148,6 @@ void PCHWriter::WritePreprocessor(const Preprocessor &PP) {
 
   // Loop over all the macro definitions that are live at the end of the file,
   // emitting each to the PP section.
-  // FIXME: Make sure that this sees macros defined in included PCH files.
   for (Preprocessor::macro_iterator I = PP.macro_begin(), E = PP.macro_end();
        I != E; ++I) {
     // FIXME: This emits macros in hash table order, we should do it in a stable
@@ -1160,7 +1159,6 @@ void PCHWriter::WritePreprocessor(const Preprocessor &PP) {
     if (MI->isBuiltinMacro())
       continue;
 
-    // FIXME: Remove this identifier reference?
     AddIdentifierRef(I->first, Record);
     MacroOffsets[I->first] = Stream.GetCurrentBitNo();
     Record.push_back(MI->getDefinitionLoc().getRawEncoding());

clang/lib/Lex/PPLexerChange.cpp

@@ -249,7 +249,8 @@ bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) {
   // diagnostic is enabled, look for macros that have not been used.
   if (getDiagnostics().getDiagnosticLevel(diag::pp_macro_not_used) !=
         Diagnostic::Ignored) {
-    for (macro_iterator I = macro_begin(), E = macro_end(); I != E; ++I)
+    for (macro_iterator I = macro_begin(false), E = macro_end(false); 
+         I != E; ++I)
       if (!I->second->isUsed())
         Diag(I->second->getDefinitionLoc(), diag::pp_macro_not_used);
   }

clang/lib/Lex/Preprocessor.cpp

@@ -27,6 +27,7 @@
 
 #include "clang/Lex/Preprocessor.h"
 #include "MacroArgs.h"
+#include "clang/Lex/ExternalPreprocessorSource.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/Pragma.h"
@@ -43,6 +44,7 @@
 using namespace clang;
 
 //===----------------------------------------------------------------------===//
+ExternalPreprocessorSource::~ExternalPreprocessorSource() { }
 
 Preprocessor::Preprocessor(Diagnostic &diags, const LangOptions &opts,
                            const TargetInfo &target, SourceManager &SM,
@@ -50,9 +52,9 @@ Preprocessor::Preprocessor(Diagnostic &diags, const LangOptions &opts,
                            IdentifierInfoLookup* IILookup,
                            bool OwnsHeaders)
   : Diags(&diags), Features(opts), Target(target),FileMgr(Headers.getFileMgr()),
-    SourceMgr(SM), HeaderInfo(Headers), Identifiers(opts, IILookup),
-    BuiltinInfo(Target), CodeCompletionFile(0), CurPPLexer(0), CurDirLookup(0),
-    Callbacks(0), MacroArgCache(0) {
+    SourceMgr(SM), HeaderInfo(Headers), ExternalSource(0),
+    Identifiers(opts, IILookup), BuiltinInfo(Target), CodeCompletionFile(0), 
+    CurPPLexer(0), CurDirLookup(0), Callbacks(0), MacroArgCache(0) {
   ScratchBuf = new ScratchBuffer(SourceMgr);
   CounterValue = 0; // __COUNTER__ starts at 0.
   OwnsHeaderSearch = OwnsHeaders;
@@ -77,6 +79,9 @@ Preprocessor::Preprocessor(Diagnostic &diags, const LangOptions &opts,
 
   CachedLexPos = 0;
 
+  // We haven't read anything from the external source.
+  ReadMacrosFromExternalSource = false;
+      
   // "Poison" __VA_ARGS__, which can only appear in the expansion of a macro.
   // This gets unpoisoned where it is allowed.
   (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
@@ -194,6 +199,28 @@ void Preprocessor::PrintStats() {
              << NumFastTokenPaste << " on the fast path.\n";
 }
 
+Preprocessor::macro_iterator 
+Preprocessor::macro_begin(bool IncludeExternalMacros) const { 
+  if (IncludeExternalMacros && ExternalSource && 
+      !ReadMacrosFromExternalSource) {
+    ReadMacrosFromExternalSource = true;
+    ExternalSource->ReadDefinedMacros();
+  }
+  
+  return Macros.begin(); 
+}
+
+Preprocessor::macro_iterator 
+Preprocessor::macro_end(bool IncludeExternalMacros) const { 
+  if (IncludeExternalMacros && ExternalSource && 
+      !ReadMacrosFromExternalSource) {
+    ReadMacrosFromExternalSource = true;
+    ExternalSource->ReadDefinedMacros();
+  }
+  
+  return Macros.end(); 
+}
+
 bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File, 
                                           unsigned TruncateAtLine, 
                                           unsigned TruncateAtColumn) {

clang/test/CodeCompletion/Inputs/macros.h

@@ -0,0 +1,4 @@
+#define FOO
+#define BAR(X, Y) X, Y
+#define IDENTITY(X) X
+#define WIBBLE(...)

clang/test/CodeCompletion/macros.c

@@ -1,8 +1,3 @@
-#define FOO
-#define BAR(X, Y) X, Y
-#define IDENTITY(X) X
-#define WIBBLE(...)
-
 enum Color {
   Red, Green, Blue
 };
@@ -13,11 +8,17 @@ struct Point {
 };
 
 void test(struct Point *p) {
-  // RUN: %clang_cc1 -fsyntax-only -code-completion-macros -code-completion-at=%s:17:14 %s -o - | FileCheck -check-prefix=CC1 %s
+  // RUN: %clang_cc1 -include %S/Inputs/macros.h -fsyntax-only -code-completion-macros -code-completion-at=%s:12:14 %s -o - | FileCheck -check-prefix=CC1 %s
   switch (p->IDENTITY(color)) {
-  // RUN: %clang_cc1 -fsyntax-only -code-completion-macros -code-completion-at=%s:19:9 %s -o - | FileCheck -check-prefix=CC2 %s
+  // RUN: %clang_cc1 -include %S/Inputs/macros.h -fsyntax-only -code-completion-macros -code-completion-at=%s:14:9 %s -o - | FileCheck -check-prefix=CC2 %s
     case 
   }
+
+  // Run the same tests, this time with macros loaded from the PCH file.
+  // RUN: %clang_cc1 -emit-pch -o %t %S/Inputs/macros.h
+  // RUN: %clang_cc1 -include-pch %t -fsyntax-only -code-completion-macros -code-completion-at=%s:12:14 %s -o - | FileCheck -check-prefix=CC1 %s
+  // RUN: %clang_cc1 -include-pch %t -fsyntax-only -code-completion-macros -code-completion-at=%s:14:9 %s -o - | FileCheck -check-prefix=CC2 %s
+
   // CC1: color
   // CC1: x
   // CC1: y