PCH serialization/deserialization of the source manager. With this

improvement, source locations read from the PCH file will properly
resolve to the source files that were used to build the PCH file
itself.

Once we have the preprocessor state stored in the PCH file, source
locations that refer to macro instantiations that occur in the PCH
file should have the appropriate instantiation information.

llvm-svn: 68758

clang/include/clang/Basic/SourceManager.h

@@ -183,7 +183,7 @@ namespace SrcMgr {
     unsigned SpellingLoc;
     
     /// InstantiationLocStart/InstantiationLocEnd - In a macro expansion, these
-    /// indicate the start and end of the instantiation.  In object-line macros,
+    /// indicate the start and end of the instantiation.  In object-like macros,
     /// these will be the same.  In a function-like macro instantiation, the
     /// start will be the identifier and the end will be the ')'.
     unsigned InstantiationLocStart, InstantiationLocEnd;
@@ -610,7 +610,20 @@ public:
   /// Read - Reconstitute a SourceManager from Bitcode.
   static SourceManager* CreateAndRegister(llvm::Deserializer& S,
                                           FileManager &FMgr);
-  
+
+  // Iteration over the source location entry table.  
+  typedef std::vector<SrcMgr::SLocEntry>::const_iterator sloc_entry_iterator;
+
+  sloc_entry_iterator sloc_entry_begin() const { 
+    return SLocEntryTable.begin(); 
+  }
+
+  sloc_entry_iterator sloc_entry_end() const { 
+    return SLocEntryTable.end(); 
+  }
+
+  unsigned sloc_entry_size() const { return SLocEntryTable.size(); }
+
 private:
   friend class SrcMgr::ContentCache; // Used for deserialization.
   

clang/include/clang/Frontend/PCHBitCodes.h

@@ -31,7 +31,19 @@ namespace clang {
       /// \brief The PCH block, which acts as a container around the
       /// full PCH block.
       PCH_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID,
-      
+
+      /// \brief The block containing information about the language
+      /// options used to build this precompiled header.
+      LANGUAGE_OPTIONS_BLOCK_ID,
+
+      /// \brief The block containing information about the source
+      /// manager.
+      SOURCE_MANAGER_BLOCK_ID,
+
+      /// \brief The block containing information about the
+      /// preprocessor.
+      PREPROCESSOR_BLOCK_ID,
+
       /// \brief The block containing the definitions of all of the
       /// types used within the PCH file.
       TYPES_BLOCK_ID,
@@ -55,6 +67,30 @@ namespace clang {
       DECL_OFFSETS_BLOCK_ID
     };
 
+    /// \brief Record types used within a source manager block.
+    enum SourceManagerRecordTypes {
+      /// \brief Describes a source location entry (SLocEntry) for a
+      /// file.
+      SM_SLOC_FILE_ENTRY = 1,
+      /// \brief Describes a source location entry (SLocEntry) for a
+      /// buffer.
+      SM_SLOC_BUFFER_ENTRY = 2,
+      /// \brief Describes a blob that contains the data for a buffer
+      /// entry. This kind of record always directly follows a
+      /// SM_SLOC_BUFFER_ENTRY record.
+      SM_SLOC_BUFFER_BLOB = 3,
+      /// \brief Describes a source location entry (SLocEntry) for a
+      /// macro instantiation.
+      SM_SLOC_INSTANTIATION_ENTRY = 4
+    };
+
+    /// \defgroup PCHAST Precompiled header AST constants
+    ///
+    /// The constants in this group describe various components of the
+    /// abstract syntax tree within a precompiled header.
+    ///
+    /// @{
+
     /// \brief Predefined type IDs.
     ///
     /// These type IDs correspond to predefined types in the AST
@@ -233,6 +269,8 @@ namespace clang {
     enum DeclOffsetCode {
       DECL_OFFSET = 1
     };
+
+    /// @}
   }
 } // end namespace clang
 

clang/include/clang/Frontend/PCHReader.h

@@ -34,6 +34,7 @@ namespace clang {
 class ASTContext;
 class Decl;
 class DeclContext;
+class Preprocessor;
 
 /// \brief Reads a precompiled head containing the contents of a
 /// translation unit.
@@ -48,6 +49,9 @@ class DeclContext;
 /// required when traversing the AST. Only those AST nodes that are
 /// actually required will be de-serialized.
 class PCHReader : public ExternalASTSource {
+  /// \brief The preprocessor that will be loading the source file.
+  Preprocessor &PP;
+
   /// \brief The AST context into which we'll read the PCH file.
   ASTContext &Context;
 
@@ -95,6 +99,7 @@ class PCHReader : public ExternalASTSource {
   DeclContextOffsetsMap DeclContextOffsets;
 
   bool ReadPCHBlock();
+  bool ReadSourceManagerBlock();
   bool ReadTypeOffsets();
   bool ReadDeclOffsets();
 
@@ -108,7 +113,9 @@ class PCHReader : public ExternalASTSource {
 public:
   typedef llvm::SmallVector<uint64_t, 64> RecordData;
 
-  PCHReader(ASTContext &Context) : Context(Context), Buffer() { }
+  PCHReader(Preprocessor &PP, ASTContext &Context) 
+    : PP(PP), Context(Context), Buffer() { }
+
   ~PCHReader();
 
   bool ReadPCH(const std::string &FileName);

clang/include/clang/Frontend/PCHWriter.h

@@ -30,6 +30,7 @@ namespace llvm {
 namespace clang {
 
 class ASTContext;
+class SourceManager;
 
 /// \brief Writes a precompiled header containing the contents of a
 /// translation unit.
@@ -74,6 +75,7 @@ class PCHWriter {
   /// \brief The type ID that will be assigned to the next new type.
   unsigned NextTypeID;
 
+  void WriteSourceManagerBlock(SourceManager &SourceMgr);
   void WriteType(const Type *T);
   void WriteTypesBlock(ASTContext &Context);
   uint64_t WriteDeclContextLexicalBlock(ASTContext &Context, DeclContext *DC);

clang/include/clang/Lex/Preprocessor.h

@@ -782,6 +782,7 @@ class PreprocessorFactory {
 public:
   virtual ~PreprocessorFactory();
   virtual Preprocessor* CreatePreprocessor() = 0;  
+  virtual bool FinishInitialization(Preprocessor *PP);
 };
   
 }  // end namespace clang

clang/lib/Frontend/PCHReader.cpp

@@ -15,6 +15,9 @@
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/Type.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Basic/FileManager.h"
 #include "llvm/Bitcode/BitstreamReader.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -111,6 +114,84 @@ static bool Error(const char *Str) {
   return true;
 }
 
+/// \brief Read the source manager block
+bool PCHReader::ReadSourceManagerBlock() {
+  using namespace SrcMgr;
+  if (Stream.EnterSubBlock(pch::SOURCE_MANAGER_BLOCK_ID))
+    return Error("Malformed source manager block record");
+
+  SourceManager &SourceMgr = Context.getSourceManager();
+  RecordData Record;
+  while (true) {
+    unsigned Code = Stream.ReadCode();
+    if (Code == llvm::bitc::END_BLOCK) {
+      if (Stream.ReadBlockEnd())
+        return Error("Error at end of Source Manager block");
+      return false;
+    }
+    
+    if (Code == llvm::bitc::ENTER_SUBBLOCK) {
+      // No known subblocks, always skip them.
+      Stream.ReadSubBlockID();
+      if (Stream.SkipBlock())
+        return Error("Malformed block record");
+      continue;
+    }
+    
+    if (Code == llvm::bitc::DEFINE_ABBREV) {
+      Stream.ReadAbbrevRecord();
+      continue;
+    }
+    
+    // Read a record.
+    const char *BlobStart;
+    unsigned BlobLen;
+    Record.clear();
+    switch (Stream.ReadRecord(Code, Record, &BlobStart, &BlobLen)) {
+    default:  // Default behavior: ignore.
+      break;
+
+    case pch::SM_SLOC_FILE_ENTRY: {
+      // FIXME: We would really like to delay the creation of this
+      // FileEntry until it is actually required, e.g., when producing
+      // a diagnostic with a source location in this file.
+      const FileEntry *File 
+        = PP.getFileManager().getFile(BlobStart, BlobStart + BlobLen);
+      // FIXME: Error recovery if file cannot be found.
+      SourceMgr.createFileID(File,
+                             SourceLocation::getFromRawEncoding(Record[1]),
+                             (CharacteristicKind)Record[2]);
+      break;
+    }
+
+    case pch::SM_SLOC_BUFFER_ENTRY: {
+      const char *Name = BlobStart;
+      unsigned Code = Stream.ReadCode();
+      Record.clear();
+      unsigned RecCode = Stream.ReadRecord(Code, Record, &BlobStart, &BlobLen);
+      assert(RecCode == pch::SM_SLOC_BUFFER_BLOB && "Ill-formed PCH file");
+      SourceMgr.createFileIDForMemBuffer(
+          llvm::MemoryBuffer::getMemBuffer(BlobStart, BlobStart + BlobLen - 1,
+                                           Name));
+      break;
+    }
+
+    case pch::SM_SLOC_INSTANTIATION_ENTRY: {
+      SourceLocation SpellingLoc 
+        = SourceLocation::getFromRawEncoding(Record[1]);
+      SourceMgr.createInstantiationLoc(
+                              SpellingLoc,
+                              SourceLocation::getFromRawEncoding(Record[2]),
+                              SourceLocation::getFromRawEncoding(Record[3]),
+                              Lexer::MeasureTokenLength(SpellingLoc, 
+                                                        SourceMgr));
+      break;
+    }
+
+    }
+  }
+}
+
 /// \brief Read the type-offsets block.
 bool PCHReader::ReadTypeOffsets() {
   if (Stream.EnterSubBlock(pch::TYPE_OFFSETS_BLOCK_ID))
@@ -217,6 +298,10 @@ bool PCHReader::ReadPCHBlock() {
           return Error("Malformed block record");
         break;
 
+      case pch::SOURCE_MANAGER_BLOCK_ID:
+        if (ReadSourceManagerBlock())
+          return Error("Malformed source manager block");
+        break;
 
       case pch::TYPE_OFFSETS_BLOCK_ID:
         if (ReadTypeOffsets())

clang/lib/Frontend/PCHWriter.cpp

@@ -17,8 +17,11 @@
 #include "clang/AST/DeclContextInternals.h"
 #include "clang/AST/DeclVisitor.h"
 #include "clang/AST/Type.h"
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/SourceManager.h"
 #include "llvm/Bitcode/BitstreamWriter.h"
 #include "llvm/Support/Compiler.h"
+#include "llvm/Support/MemoryBuffer.h"
 
 using namespace clang;
 
@@ -323,6 +326,154 @@ void PCHDeclWriter::VisitDeclContext(DeclContext *DC, uint64_t LexicalOffset,
 // PCHWriter Implementation
 //===----------------------------------------------------------------------===//
 
+//===----------------------------------------------------------------------===//
+// Source Manager Serialization
+//===----------------------------------------------------------------------===//
+
+/// \brief Create an abbreviation for the SLocEntry that refers to a
+/// file.
+static unsigned CreateSLocFileAbbrev(llvm::BitstreamWriter &S) {
+  using namespace llvm;
+  BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
+  Abbrev->Add(BitCodeAbbrevOp(pch::SM_SLOC_FILE_ENTRY));
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Offset
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Include location
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 2)); // Characteristic
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // Line directives
+  // FIXME: Need an actual encoding for the line directives; maybe
+  // this should be an array?
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // File name
+  return S.EmitAbbrev(Abbrev);
+}
+
+/// \brief Create an abbreviation for the SLocEntry that refers to a
+/// buffer.
+static unsigned CreateSLocBufferAbbrev(llvm::BitstreamWriter &S) {
+  using namespace llvm;
+  BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
+  Abbrev->Add(BitCodeAbbrevOp(pch::SM_SLOC_BUFFER_ENTRY));
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Offset
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Include location
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 2)); // Characteristic
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // Line directives
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Buffer name blob
+  return S.EmitAbbrev(Abbrev);
+}
+
+/// \brief Create an abbreviation for the SLocEntry that refers to a
+/// buffer's blob.
+static unsigned CreateSLocBufferBlobAbbrev(llvm::BitstreamWriter &S) {
+  using namespace llvm;
+  BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
+  Abbrev->Add(BitCodeAbbrevOp(pch::SM_SLOC_BUFFER_BLOB));
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Blob
+  return S.EmitAbbrev(Abbrev);
+}
+
+/// \brief Create an abbreviation for the SLocEntry that refers to an
+/// buffer.
+static unsigned CreateSLocInstantiationAbbrev(llvm::BitstreamWriter &S) {
+  using namespace llvm;
+  BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
+  Abbrev->Add(BitCodeAbbrevOp(pch::SM_SLOC_INSTANTIATION_ENTRY));
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Offset
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Spelling location
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Start location
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // End location
+  return S.EmitAbbrev(Abbrev);
+}
+
+/// \brief Writes the block containing the serialized form of the
+/// source manager.
+///
+/// TODO: We should probably use an on-disk hash table (stored in a
+/// blob), indexed based on the file name, so that we only create
+/// entries for files that we actually need. In the common case (no
+/// errors), we probably won't have to create file entries for any of
+/// the files in the AST.
+void PCHWriter::WriteSourceManagerBlock(SourceManager &SourceMgr) {
+  // Enter the types block
+  S.EnterSubblock(pch::SOURCE_MANAGER_BLOCK_ID, 3);
+
+  // Abbreviations for the various kinds of source-location entries.
+  int SLocFileAbbrv = -1;
+  int SLocBufferAbbrv = -1;
+  int SLocBufferBlobAbbrv = -1;
+  int SLocInstantiationAbbrv = -1;
+
+  // Write out the source location entry table. We skip the first
+  // entry, which is always the same dummy entry.
+  RecordData Record;
+  for (SourceManager::sloc_entry_iterator 
+         SLoc = SourceMgr.sloc_entry_begin() + 1,
+         SLocEnd = SourceMgr.sloc_entry_end();
+       SLoc != SLocEnd; ++SLoc) {
+    // Figure out which record code to use.
+    unsigned Code;
+    if (SLoc->isFile()) {
+      if (SLoc->getFile().getContentCache()->Entry)
+        Code = pch::SM_SLOC_FILE_ENTRY;
+      else
+        Code = pch::SM_SLOC_BUFFER_ENTRY;
+    } else
+      Code = pch::SM_SLOC_INSTANTIATION_ENTRY;
+    Record.push_back(Code);
+
+    Record.push_back(SLoc->getOffset());
+    if (SLoc->isFile()) {
+      const SrcMgr::FileInfo &File = SLoc->getFile();
+      Record.push_back(File.getIncludeLoc().getRawEncoding());
+      Record.push_back(File.getFileCharacteristic()); // FIXME: stable encoding
+      Record.push_back(File.hasLineDirectives()); // FIXME: encode the
+                                                  // line directives?
+
+      const SrcMgr::ContentCache *Content = File.getContentCache();
+      if (Content->Entry) {
+        // The source location entry is a file. The blob associated
+        // with this entry is the file name.
+        if (SLocFileAbbrv == -1)
+          SLocFileAbbrv = CreateSLocFileAbbrev(S);
+        S.EmitRecordWithBlob(SLocFileAbbrv, Record,
+                             Content->Entry->getName(),
+                             strlen(Content->Entry->getName()));
+      } else {
+        // The source location entry is a buffer. The blob associated
+        // with this entry contains the contents of the buffer.
+        if (SLocBufferAbbrv == -1) {
+          SLocBufferAbbrv = CreateSLocBufferAbbrev(S);
+          SLocBufferBlobAbbrv = CreateSLocBufferBlobAbbrev(S);
+        }
+
+        // We add one to the size so that we capture the trailing NULL
+        // that is required by llvm::MemoryBuffer::getMemBuffer (on
+        // the reader side).
+        const llvm::MemoryBuffer *Buffer = Content->getBuffer();
+        const char *Name = Buffer->getBufferIdentifier();
+        S.EmitRecordWithBlob(SLocBufferAbbrv, Record, Name, strlen(Name) + 1);
+        Record.clear();
+        Record.push_back(pch::SM_SLOC_BUFFER_BLOB);
+        S.EmitRecordWithBlob(SLocBufferBlobAbbrv, Record,
+                             Buffer->getBufferStart(),
+                             Buffer->getBufferSize() + 1);
+      }
+    } else {
+      // The source location entry is an instantiation.
+      const SrcMgr::InstantiationInfo &Inst = SLoc->getInstantiation();
+      Record.push_back(Inst.getSpellingLoc().getRawEncoding());
+      Record.push_back(Inst.getInstantiationLocStart().getRawEncoding());
+      Record.push_back(Inst.getInstantiationLocEnd().getRawEncoding());
+
+      if (SLocInstantiationAbbrv == -1)
+        SLocInstantiationAbbrv = CreateSLocInstantiationAbbrev(S);
+      S.EmitRecordWithAbbrev(SLocInstantiationAbbrv, Record);
+    }
+
+    Record.clear();
+  }
+
+  S.ExitBlock();
+}
+
 /// \brief Write the representation of a type to the PCH stream.
 void PCHWriter::WriteType(const Type *T) {
   pch::ID &ID = TypeIDs[T];
@@ -521,6 +672,7 @@ void PCHWriter::WritePCH(ASTContext &Context) {
 
   // Write the remaining PCH contents.
   S.EnterSubblock(pch::PCH_BLOCK_ID, 2);
+  WriteSourceManagerBlock(Context.getSourceManager());
   WriteTypesBlock(Context);
   WriteDeclsBlock(Context);
   S.ExitBlock();

clang/lib/Lex/Preprocessor.cpp

@@ -45,6 +45,10 @@ using namespace clang;
 
 PreprocessorFactory::~PreprocessorFactory() {}
 
+bool PreprocessorFactory::FinishInitialization(Preprocessor *PP) {
+  return false;
+}
+
 Preprocessor::Preprocessor(Diagnostic &diags, const LangOptions &opts,
                            TargetInfo &target, SourceManager &SM, 
                            HeaderSearch &Headers,

clang/test/PCH/variables.c

@@ -3,4 +3,8 @@
 
 int *ip2 = &x;
 float *fp = &ip; // expected-warning{{incompatible pointer types}}
+// FIXME:variables.h expected-note{{previous}}
+double z; // expected-error{{redefinition}}
+
+//double VeryHappy; // FIXME: xpected-error{{redefinition}}
 

clang/test/PCH/variables.h

@@ -1,5 +1,12 @@
 // RUN: clang-cc -emit-pch -o variables.h.pch variables.h
-extern int x;
+// Do not mess with the whitespace in this file. It's important.
 extern float y;
-extern int *ip;
+extern int *ip, x;
+
 float z;
+
+
+
+#define MAKE_HAPPY(X) X##Happy
+int MAKE_HAPPY(Very);
+

clang/tools/clang-cc/clang-cc.cpp

@@ -1379,22 +1379,26 @@ public:
       PP->setPTHManager(PTHMgr.take());
     }
     
+    return PP.take();
+  }
+
+  virtual bool FinishInitialization(Preprocessor *PP) {
     if (InitializePreprocessor(*PP, InitializeSourceMgr, InFile)) {
-      return NULL;
+      return true;
     }
     
     /// FIXME: PP can only handle one callback
     if (ProgAction != PrintPreprocessedInput) {
       std::string ErrStr;
-      bool DFG = CreateDependencyFileGen(PP.get(), ErrStr);
+      bool DFG = CreateDependencyFileGen(PP, ErrStr);
       if (!DFG && !ErrStr.empty()) {
         fprintf(stderr, "%s", ErrStr.c_str());
-        return NULL;
+        return true;
       }
     }
 
     InitializeSourceMgr = false;
-    return PP.take();
+    return false;
   }
 };
 }
@@ -1697,13 +1701,20 @@ static void ProcessInputFile(Preprocessor &PP, PreprocessorFactory &PPF,
     if (!ImplicitIncludePCH.empty()) {
       // The user has asked us to include a precompiled header. Load
       // the precompiled header into the AST context.
-      llvm::OwningPtr<PCHReader> Reader(
-                                   new clang::PCHReader(*ContextOwner.get()));
+      llvm::OwningPtr<PCHReader> 
+        Reader(new clang::PCHReader(PP, *ContextOwner.get()));
       if (Reader->ReadPCH(ImplicitIncludePCH))
         return;
 
       llvm::OwningPtr<ExternalASTSource> Source(Reader.take());
       ContextOwner->setExternalSource(Source);
+
+      // Finish preprocessor initialization. We do this now (rather
+      // than earlier) because this initialization creates new source
+      // location entries in the source manager, which must come after
+      // the source location entries for the PCH file.
+      if (PPF.FinishInitialization(&PP))
+        return;
     }
 
     ParseAST(PP, Consumer.get(), *ContextOwner.get(), Stats);
@@ -1912,6 +1923,10 @@ int main(int argc, char **argv) {
     if (!PP)
       continue;
 
+    if (ImplicitIncludePCH.empty()
+        && PPFactory.FinishInitialization(PP.get()))
+      continue;
+
     // Create the HTMLDiagnosticsClient if we are using one.  Otherwise,
     // always reset to using TextDiagClient.
     llvm::OwningPtr<DiagnosticClient> TmpClient;