- Fix recording of offsets of types in dependent PCHs.

- Stop reading in (and thus deserializing) every declaration in the TU when creating a dependent PCH.
- Switch the storage of a decl context's lexical declarations to a blob containing the IDs instead of a record. This is the only sane way of supporting update records later on.

llvm-svn: 109474

clang/include/clang/AST/DeclBase.h

@@ -840,6 +840,12 @@ public:
   decl_iterator decls_end() const;
   bool decls_empty() const;
 
+  /// noload_decls_begin/end - Iterate over the declarations stored in this
+  /// context that are currently loaded; don't attempt to retrieve anything
+  /// from an external source.
+  decl_iterator noload_decls_begin() const;
+  decl_iterator noload_decls_end() const;
+
   /// specific_decl_iterator - Iterates over a subrange of
   /// declarations stored in a DeclContext, providing only those that
   /// are of type SpecificDecl (or a class derived from it). This

clang/include/clang/Frontend/PCHBitCodes.h

@@ -238,7 +238,7 @@ namespace clang {
       /// PCH version and the name of the PCH this is chained to.
       CHAINED_METADATA = 26,
       
-      /// \brief Record  code for referenced selector pool.
+      /// \brief Record code for referenced selector pool.
       REFERENCED_SELECTOR_POOL = 27
     };
 
@@ -547,8 +547,8 @@ namespace clang {
       /// \brief A record that stores the set of declarations that are
       /// lexically stored within a given DeclContext.
       ///
-      /// The record itself is an array of declaration IDs, in the
-      /// order in which those declarations were added to the
+      /// The record itself is a blob that is an array of declaration IDs,
+      /// in the order in which those declarations were added to the
       /// declaration context. This data is used when iterating over
       /// the contents of a DeclContext, e.g., via
       /// DeclContext::decls_begin()/DeclContext::decls_end().

clang/include/clang/Frontend/PCHReader.h

@@ -307,8 +307,9 @@ private:
   /// \brief Information about the contents of a DeclContext.
   struct DeclContextInfo {
     llvm::BitstreamCursor *Stream;
-    uint64_t OffsetToLexicalDecls;
     uint64_t OffsetToVisibleDecls;
+    const pch::DeclID *LexicalDecls;
+    unsigned NumLexicalDecls;
   };
   typedef llvm::SmallVector<DeclContextInfo, 1> DeclContextInfos;
   typedef llvm::DenseMap<const DeclContext *, DeclContextInfos>
@@ -318,6 +319,11 @@ private:
   /// DeclContext.
   DeclContextOffsetsMap DeclContextOffsets;
 
+  /// \brief Read the records that describe the contents of declcontexts.
+  bool ReadDeclContextStorage(llvm::BitstreamCursor &Cursor,
+                              const std::pair<uint64_t, uint64_t> &Offsets,
+                              DeclContextInfo &Info);
+
   /// \brief A vector containing identifiers that have already been
   /// loaded.
   ///

clang/include/clang/Frontend/PCHWriter.h

@@ -256,6 +256,7 @@ private:
   void WriteAttributeRecord(const Attr *Attr);
 
   unsigned ParmVarDeclAbbrev;
+  unsigned DeclContextLexicalAbbrev;
   void WriteDeclsBlockAbbrevs();
   void WriteDecl(ASTContext &Context, Decl *D);
 
@@ -345,6 +346,9 @@ public:
   /// \brief Emit a reference to a declaration.
   void AddDeclRef(const Decl *D, RecordData &Record);
 
+  /// \brief Force a declaration to be emitted and get its ID.
+  pch::DeclID GetDeclRef(const Decl *D);
+
   /// \brief Determine the declaration ID of an already-emitted
   /// declaration.
   pch::DeclID getDeclID(const Decl *D);

clang/lib/AST/DeclBase.cpp

@@ -715,6 +715,14 @@ void ExternalASTSource::SetExternalVisibleDecls(const DeclContext *DC,
   }
 }
 
+DeclContext::decl_iterator DeclContext::noload_decls_begin() const {
+  return decl_iterator(FirstDecl);
+}
+
+DeclContext::decl_iterator DeclContext::noload_decls_end() const {
+  return decl_iterator();
+}
+
 DeclContext::decl_iterator DeclContext::decls_begin() const {
   if (hasExternalLexicalStorage())
     LoadLexicalDeclsFromExternalStorage();

clang/lib/Frontend/PCHReader.cpp

@@ -1634,8 +1634,8 @@ PCHReader::ReadPCHBlock(PerFileData &F) {
       unsigned int numEl = Record[0]*2;
       for (unsigned int i = 1; i <= numEl; i++)
         F.ReferencedSelectorsData.push_back(Record[i]);
-    }
       break;
+    }
 
     case pch::PP_COUNTER_VALUE:
       if (!Record.empty() && Listener)
@@ -2896,30 +2896,15 @@ bool PCHReader::FindExternalLexicalDecls(const DeclContext *DC,
   DeclContextInfos &Infos = DeclContextOffsets[DC];
   for (DeclContextInfos::iterator I = Infos.begin(), E = Infos.end();
        I != E; ++I) {
-    uint64_t Offset = I->OffsetToLexicalDecls;
-    // Offset can be 0 if this file only contains visible decls.
-    if (Offset == 0)
+    // IDs can be 0 if this context doesn't contain declarations.
+    if (!I->LexicalDecls)
       continue;
-    llvm::BitstreamCursor &DeclsCursor = *I->Stream;
-
-    // Keep track of where we are in the stream, then jump back there
-    // after reading this context.
-    SavedStreamPosition SavedPosition(DeclsCursor);
-
-    // Load the record containing all of the declarations lexically in
-    // this context.
-    DeclsCursor.JumpToBit(Offset);
-    RecordData Record;
-    unsigned Code = DeclsCursor.ReadCode();
-    unsigned RecCode = DeclsCursor.ReadRecord(Code, Record);
-    if (RecCode != pch::DECL_CONTEXT_LEXICAL) {
-      Error("Expected lexical block");
-      return true;
-    }
 
     // Load all of the declaration IDs
-    for (RecordData::iterator J = Record.begin(), F = Record.end(); J != F; ++J)
-      Decls.push_back(GetDecl(*J));
+    for (const pch::DeclID *ID = I->LexicalDecls,
+                           *IDE = ID + I->NumLexicalDecls;
+         ID != IDE; ++ID)
+      Decls.push_back(GetDecl(*ID));
   }
 
   ++NumLexicalDeclContextsRead;

clang/lib/Frontend/PCHReaderDecl.cpp

@@ -1517,11 +1517,9 @@ Decl *PCHReader::ReadDeclRecord(unsigned Index) {
     if (Offsets.first || Offsets.second) {
       DC->setHasExternalLexicalStorage(Offsets.first != 0);
       DC->setHasExternalVisibleStorage(Offsets.second != 0);
-      PCHReader::DeclContextInfo Info = {
-        Loc.first,
-        Offsets.first,
-        Offsets.second
-      };
+      DeclContextInfo Info;
+      if (ReadDeclContextStorage(DeclsCursor, Offsets, Info))
+        return 0;
       DeclContextOffsets[DC].push_back(Info);
     }
   }
@@ -1536,3 +1534,35 @@ Decl *PCHReader::ReadDeclRecord(unsigned Index) {
 
   return D;
 }
+
+bool PCHReader::ReadDeclContextStorage(llvm::BitstreamCursor &Cursor,
+                                   const std::pair<uint64_t, uint64_t> &Offsets,
+                                       DeclContextInfo &Info) {
+  SavedStreamPosition SavedPosition(Cursor);
+  // First the lexical decls.
+  if (Offsets.first != 0) {
+    Cursor.JumpToBit(Offsets.first);
+
+    RecordData Record;
+    const char *Blob;
+    unsigned BlobLen;
+    unsigned Code = Cursor.ReadCode();
+    unsigned RecCode = Cursor.ReadRecord(Code, Record, &Blob, &BlobLen);
+    if (RecCode != pch::DECL_CONTEXT_LEXICAL) {
+      Error("Expected lexical block");
+      return true;
+    }
+
+    Info.LexicalDecls = reinterpret_cast<const pch::DeclID*>(Blob);
+    Info.NumLexicalDecls = BlobLen / sizeof(pch::DeclID);
+  } else {
+    Info.LexicalDecls = 0;
+    Info.NumLexicalDecls = 0;
+  }
+
+  // Now the visible decls.
+  Info.Stream = &Cursor;
+  Info.OffsetToVisibleDecls = Offsets.second;
+
+  return false;
+}

clang/lib/Frontend/PCHWriter.cpp

@@ -1391,11 +1391,12 @@ void PCHWriter::WriteType(QualType T) {
     ID = NextTypeID++;
 
   // Record the offset for this type.
-  if (TypeOffsets.size() == ID - pch::NUM_PREDEF_TYPE_IDS)
+  unsigned Index = ID - FirstTypeID;
+  if (TypeOffsets.size() == Index)
     TypeOffsets.push_back(Stream.GetCurrentBitNo());
-  else if (TypeOffsets.size() < ID - pch::NUM_PREDEF_TYPE_IDS) {
-    TypeOffsets.resize(ID + 1 - pch::NUM_PREDEF_TYPE_IDS);
-    TypeOffsets[ID - pch::NUM_PREDEF_TYPE_IDS] = Stream.GetCurrentBitNo();
+  else if (TypeOffsets.size() < Index) {
+    TypeOffsets.resize(Index + 1);
+    TypeOffsets[Index] = Stream.GetCurrentBitNo();
   }
 
   RecordData Record;
@@ -1442,12 +1443,15 @@ uint64_t PCHWriter::WriteDeclContextLexicalBlock(ASTContext &Context,
 
   uint64_t Offset = Stream.GetCurrentBitNo();
   RecordData Record;
+  Record.push_back(pch::DECL_CONTEXT_LEXICAL);
+  llvm::SmallVector<pch::DeclID, 64> Decls;
   for (DeclContext::decl_iterator D = DC->decls_begin(), DEnd = DC->decls_end();
          D != DEnd; ++D)
-    AddDeclRef(*D, Record);
+    Decls.push_back(GetDeclRef(*D));
 
   ++NumLexicalDeclContexts;
-  Stream.EmitRecord(pch::DECL_CONTEXT_LEXICAL, Record);
+  Stream.EmitRecordWithBlob(DeclContextLexicalAbbrev, Record,
+     reinterpret_cast<char*>(Decls.data()), Decls.size() * sizeof(pch::DeclID));
   return Offset;
 }
 
@@ -2332,7 +2336,6 @@ void PCHWriter::WritePCHChain(Sema &SemaRef, MemorizeStatCalls *StatCalls,
 
   ASTContext &Context = SemaRef.Context;
   Preprocessor &PP = SemaRef.PP;
-  (void)PP;
 
   RecordData Record;
   Stream.EnterSubblock(pch::PCH_BLOCK_ID, 5);
@@ -2351,16 +2354,15 @@ void PCHWriter::WritePCHChain(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   // The TU was loaded before we managed to register ourselves as a listener.
   // Thus we need to add it manually.
   DeclIDs[TU] = 1;
-  // FIXME: We don't want to iterate over everything here, because it needlessly
-  // deserializes the entire original PCH. Instead we only want to iterate over
-  // the stuff that's already there.
-  // All in good time, though.
-  for (DeclContext::decl_iterator I = TU->decls_begin(), E = TU->decls_end();
+  Record.clear();
+  for (DeclContext::decl_iterator I = TU->noload_decls_begin(),
+                                  E = TU->noload_decls_end();
        I != E; ++I) {
     if ((*I)->getPCHLevel() == 0) {
-      DeclTypesToEmit.push(*I);
+      AddDeclRef(*I, Record);
     }
   }
+  // We also need to write a lexical updates block for the TU.
 
   Stream.EnterSubblock(pch::DECLTYPES_BLOCK_ID, 3);
   WriteDeclsBlockAbbrevs();
@@ -2584,9 +2586,12 @@ void PCHWriter::AddTypeRef(QualType T, RecordData &Record) {
 }
 
 void PCHWriter::AddDeclRef(const Decl *D, RecordData &Record) {
+  Record.push_back(GetDeclRef(D));
+}
+
+pch::DeclID PCHWriter::GetDeclRef(const Decl *D) {
   if (D == 0) {
-    Record.push_back(0);
-    return;
+    return 0;
   }
 
   pch::DeclID &ID = DeclIDs[D];
@@ -2597,7 +2602,7 @@ void PCHWriter::AddDeclRef(const Decl *D, RecordData &Record) {
     DeclTypesToEmit.push(const_cast<Decl *>(D));
   }
 
-  Record.push_back(ID);
+  return ID;
 }
 
 pch::DeclID PCHWriter::getDeclID(const Decl *D) {

clang/lib/Frontend/PCHWriterDecl.cpp

@@ -1059,6 +1059,11 @@ void PCHWriter::WriteDeclsBlockAbbrevs() {
   Abv->Add(BitCodeAbbrevOp(0));                   // HasUninstantiatedDefaultArg
 
   ParmVarDeclAbbrev = Stream.EmitAbbrev(Abv);
+
+  Abv = new BitCodeAbbrev();
+  Abv->Add(BitCodeAbbrevOp(pch::DECL_CONTEXT_LEXICAL));
+  Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
+  DeclContextLexicalAbbrev = Stream.EmitAbbrev(Abv);
 }
 
 /// isRequiredDecl - Check if this is a "required" Decl, which must be seen by