Resubmit "[pdb] Allow zero-copy read support for symbol streams.""

Due to differences in template instantiation rules, it is not
portable to static_assert(false) inside of an invalid specialization
of a template.  Instead I just =delete the method so that it can't
be used, and leave a comment that it must be explicitly specialized.

llvm-svn: 271027

llvm/include/llvm/DebugInfo/CodeView/CVSymbolVisitor.h

@@ -46,7 +46,7 @@ public:
 #define SYMBOL_RECORD_ALIAS(EnumName, EnumVal, Name, AliasName)
 #include "CVSymbolTypes.def"
 
-  void visitSymbolRecord(const SymbolIterator::Record &Record) {
+  void visitSymbolRecord(const CVRecord<SymbolKind> &Record) {
     ArrayRef<uint8_t> Data = Record.Data;
     auto *DerivedThis = static_cast<Derived *>(this);
     DerivedThis->visitSymbolBegin(Record.Type, Data);

llvm/include/llvm/DebugInfo/CodeView/CVTypeVisitor.h

@@ -51,7 +51,7 @@ public:
 #define MEMBER_RECORD_ALIAS(EnumName, EnumVal, Name, AliasName)
 #include "TypeRecords.def"
 
-  void visitTypeRecord(const TypeIterator::Record &Record) {
+  void visitTypeRecord(const CVRecord<TypeLeafKind> &Record) {
     ArrayRef<uint8_t> LeafData = Record.Data;
     ArrayRef<uint8_t> RecordData = LeafData;
     auto *DerivedThis = static_cast<Derived *>(this);

llvm/include/llvm/DebugInfo/CodeView/RecordIterator.h

@@ -13,19 +13,41 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/DebugInfo/CodeView/RecordSerialization.h"
+#include "llvm/DebugInfo/CodeView/StreamInterface.h"
+#include "llvm/DebugInfo/CodeView/StreamReader.h"
 #include "llvm/Support/Endian.h"
 
 namespace llvm {
 namespace codeview {
 
+template <typename Kind> struct CVRecord {
+  uint32_t Length;
+  Kind Type;
+  ArrayRef<uint8_t> Data;
+};
+
+template <typename Kind> struct VarStreamArrayExtractor<CVRecord<Kind>> {
+  uint32_t operator()(const StreamInterface &Stream,
+                      CVRecord<Kind> &Item) const {
+    const RecordPrefix *Prefix = nullptr;
+    StreamReader Reader(Stream);
+    if (auto EC = Reader.readObject(Prefix)) {
+      consumeError(std::move(EC));
+      return 0;
+    }
+    Item.Length = Prefix->RecordLen;
+    Item.Type = static_cast<Kind>(uint16_t(Prefix->RecordKind));
+    if (auto EC = Reader.readBytes(Item.Length - 2, Item.Data)) {
+      consumeError(std::move(EC));
+      return 0;
+    }
+    return Prefix->RecordLen + 2;
+  }
+};
+
 // A const input iterator interface to the CodeView record stream.
 template <typename Kind> class RecordIterator {
 public:
-  struct Record {
-    std::size_t Length;
-    Kind Type;
-    ArrayRef<uint8_t> Data;
-  };
 
   explicit RecordIterator(const ArrayRef<uint8_t> &RecordBytes, bool *HadError)
       : HadError(HadError), Data(RecordBytes), AtEnd(false) {
@@ -46,12 +68,12 @@ public:
     return !(lhs == rhs);
   }
 
-  const Record &operator*() const {
+  const CVRecord<Kind> &operator*() const {
     assert(!AtEnd);
     return Current;
   }
 
-  const Record *operator->() const {
+  const CVRecord<Kind> *operator->() const {
     assert(!AtEnd);
     return &Current;
   }
@@ -106,7 +128,7 @@ private:
 
   bool *HadError;
   ArrayRef<uint8_t> Data;
-  Record Current;
+  CVRecord<Kind> Current;
   bool AtEnd;
 };
 

llvm/include/llvm/DebugInfo/CodeView/StreamArray.h

@@ -18,6 +18,17 @@
 namespace llvm {
 namespace codeview {
 
+/// VarStreamArrayExtractor is intended to be specialized to provide customized
+/// extraction logic.  It should return the total number of bytes of the next
+/// record (so that the array knows how much data to skip to get to the next
+/// record, and it should initialize the second parameter with the desired
+/// value type.
+template <typename T> struct VarStreamArrayExtractor {
+  // Method intentionally deleted.  You must provide an explicit specialization
+  // with the following method implemented.
+  uint32_t operator()(const StreamInterface &Stream, T &t) const = delete;
+};
+
 /// VarStreamArray represents an array of variable length records backed by a
 /// stream.  This could be a contiguous sequence of bytes in memory, it could
 /// be a file on disk, or it could be a PDB stream where bytes are stored as
@@ -27,33 +38,39 @@ namespace codeview {
 /// re-ordering of stream data to be contiguous before iterating over it.  By
 /// abstracting this out, we need not duplicate this memory, and we can
 /// iterate over arrays in arbitrarily formatted streams.
-class VarStreamArrayIterator;
+template <typename ValueType, typename Extractor> class VarStreamArrayIterator;
 
+template <typename ValueType,
+          typename Extractor = VarStreamArrayExtractor<ValueType>>
 class VarStreamArray {
-  friend class VarStreamArrayIterator;
-  typedef std::function<uint32_t(const StreamInterface &)> LengthFuncType;
+  friend class VarStreamArrayIterator<ValueType, Extractor>;
 
 public:
-  template <typename LengthFunc>
-  VarStreamArray(StreamRef Stream, const LengthFunc &Len)
-      : Stream(Stream), Len(Len) {}
+  typedef VarStreamArrayIterator<ValueType, Extractor> Iterator;
 
-  VarStreamArrayIterator begin() const;
-  VarStreamArrayIterator end() const;
+  VarStreamArray() {}
+
+  VarStreamArray(StreamRef Stream) : Stream(Stream) {}
+
+  Iterator begin() const { return Iterator(*this); }
+
+  Iterator end() const { return Iterator(); }
 
 private:
   StreamRef Stream;
-  LengthFuncType Len; // Function used to calculate legth of a record
 };
 
-class VarStreamArrayIterator {
+template <typename ValueType, typename Extractor> class VarStreamArrayIterator {
+  typedef VarStreamArrayIterator<ValueType, Extractor> IterType;
+  typedef VarStreamArray<ValueType, Extractor> ArrayType;
+
 public:
-  VarStreamArrayIterator(const VarStreamArray &Array)
+  VarStreamArrayIterator(const ArrayType &Array)
       : Array(&Array), IterRef(Array.Stream) {
-    ThisLen = Array.Len(IterRef);
+    ThisLen = Extract(IterRef, ThisValue);
   }
   VarStreamArrayIterator() : Array(nullptr), IterRef() {}
-  bool operator==(const VarStreamArrayIterator &R) const {
+  bool operator==(const IterType &R) const {
     if (Array && R.Array) {
       // Both have a valid array, make sure they're same.
       assert(Array == R.Array);
@@ -68,45 +85,37 @@ public:
     return false;
   }
 
-  bool operator!=(const VarStreamArrayIterator &R) { return !(*this == R); }
+  bool operator!=(const IterType &R) { return !(*this == R); }
 
-  StreamRef operator*() const {
-    ArrayRef<uint8_t> Result;
-    return IterRef.keep_front(ThisLen);
-  }
+  const ValueType &operator*() const { return ThisValue; }
 
-  VarStreamArrayIterator &operator++() {
-    if (!Array || IterRef.getLength() == 0)
+  IterType &operator++() {
+    if (!Array || IterRef.getLength() == 0 || ThisLen == 0)
       return *this;
     IterRef = IterRef.drop_front(ThisLen);
     if (IterRef.getLength() == 0) {
       Array = nullptr;
       ThisLen = 0;
     } else {
-      ThisLen = Array->Len(IterRef);
+      ThisLen = Extract(IterRef, ThisValue);
     }
     return *this;
   }
 
-  VarStreamArrayIterator operator++(int) {
-    VarStreamArrayIterator Original = *this;
+  IterType operator++(int) {
+    IterType Original = *this;
     ++*this;
     return Original;
   }
 
 private:
-  const VarStreamArray *Array;
+  const ArrayType *Array;
   uint32_t ThisLen;
+  ValueType ThisValue;
   StreamRef IterRef;
+  Extractor Extract;
 };
 
-inline VarStreamArrayIterator VarStreamArray::begin() const {
-  return VarStreamArrayIterator(*this);
-}
-inline VarStreamArrayIterator VarStreamArray::end() const {
-  return VarStreamArrayIterator();
-}
-
 template <typename T> class FixedStreamArrayIterator;
 
 template <typename T> class FixedStreamArray {

llvm/include/llvm/DebugInfo/CodeView/StreamReader.h

@@ -45,6 +45,15 @@ public:
     return Error::success();
   }
 
+  template <typename T>
+  Error readArray(VarStreamArray<T> &Array, uint32_t Size) {
+    StreamRef S;
+    if (auto EC = readStreamRef(S, Size))
+      return EC;
+    Array = VarStreamArray<T>(S);
+    return Error::success();
+  }
+
   template <typename T>
   Error readArray(FixedStreamArray<T> &Array, uint32_t NumItems) {
     if (NumItems == 0) {

llvm/include/llvm/DebugInfo/CodeView/SymbolDumper.h

@@ -35,7 +35,7 @@ public:
   /// and true otherwise.  This should be called in order, since the dumper
   /// maintains state about previous records which are necessary for cross
   /// type references.
-  bool dump(const SymbolIterator::Record &Record);
+  bool dump(const CVRecord<SymbolKind> &Record);
 
   /// Dumps the type records in Data. Returns false if there was a type stream
   /// parse error, and true otherwise.

llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h

@@ -15,6 +15,8 @@
 #include "llvm/DebugInfo/CodeView/CodeView.h"
 #include "llvm/DebugInfo/CodeView/RecordIterator.h"
 #include "llvm/DebugInfo/CodeView/RecordSerialization.h"
+#include "llvm/DebugInfo/CodeView/StreamArray.h"
+#include "llvm/DebugInfo/CodeView/StreamInterface.h"
 #include "llvm/DebugInfo/CodeView/TypeIndex.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
@@ -1441,6 +1443,8 @@ public:
 };
 
 typedef RecordIterator<SymbolKind> SymbolIterator;
+typedef CVRecord<SymbolKind> CVSymbol;
+typedef VarStreamArray<CVSymbol> CVSymbolArray;
 
 inline iterator_range<SymbolIterator> makeSymbolRange(ArrayRef<uint8_t> Data,
                                                       bool *HadError) {

llvm/include/llvm/DebugInfo/CodeView/TypeDumper.h

@@ -34,7 +34,7 @@ public:
   /// and true otherwise.  This should be called in order, since the dumper
   /// maintains state about previous records which are necessary for cross
   /// type references.
-  bool dump(const TypeIterator::Record &Record);
+  bool dump(const CVRecord<TypeLeafKind> &Record);
 
   /// Dumps the type records in Data. Returns false if there was a type stream
   /// parse error, and true otherwise.

llvm/include/llvm/DebugInfo/PDB/Raw/DbiStream.h

@@ -10,7 +10,6 @@
 #ifndef LLVM_DEBUGINFO_PDB_RAW_PDBDBISTREAM_H
 #define LLVM_DEBUGINFO_PDB_RAW_PDBDBISTREAM_H
 
-#include "llvm/DebugInfo/CodeView/ByteStream.h"
 #include "llvm/DebugInfo/CodeView/StreamArray.h"
 #include "llvm/DebugInfo/CodeView/StreamRef.h"
 #include "llvm/DebugInfo/PDB/PDBTypes.h"

llvm/include/llvm/DebugInfo/PDB/Raw/ModInfo.h

@@ -11,6 +11,7 @@
 #define LLVM_DEBUGINFO_PDB_RAW_MODINFO_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/DebugInfo/CodeView/StreamArray.h"
 #include "llvm/DebugInfo/CodeView/StreamRef.h"
 #include <cstdint>
 #include <vector>
@@ -23,6 +24,7 @@ private:
   struct FileLayout;
 
 public:
+  ModInfo();
   ModInfo(codeview::StreamRef Stream);
   ModInfo(const ModInfo &Info);
   ~ModInfo();
@@ -50,6 +52,7 @@ private:
 
 struct ModuleInfoEx {
   ModuleInfoEx(codeview::StreamRef Stream) : Info(Stream) {}
+  ModuleInfoEx(const ModInfo &Info) : Info(Info) {}
   ModuleInfoEx(const ModuleInfoEx &Ex)
       : Info(Ex.Info), SourceFiles(Ex.SourceFiles) {}
 
@@ -57,11 +60,17 @@ struct ModuleInfoEx {
   std::vector<StringRef> SourceFiles;
 };
 
-inline uint32_t ModInfoRecordLength(const codeview::StreamInterface &Stream) {
-  return ModInfo(Stream).getRecordLength();
+} // end namespace pdb
+
+namespace codeview {
+template <> struct VarStreamArrayExtractor<pdb::ModInfo> {
+  uint32_t operator()(const StreamInterface &Stream, pdb::ModInfo &Info) const {
+    Info = pdb::ModInfo(Stream);
+    return Info.getRecordLength();
+  }
+};
 }
 
-} // end namespace pdb
 } // end namespace llvm
 
 #endif // LLVM_DEBUGINFO_PDB_RAW_MODINFO_H

llvm/include/llvm/DebugInfo/PDB/Raw/ModStream.h

@@ -11,7 +11,7 @@
 #define LLVM_DEBUGINFO_PDB_RAW_MODSTREAM_H
 
 #include "llvm/ADT/iterator_range.h"
-#include "llvm/DebugInfo/CodeView/ByteStream.h"
+#include "llvm/DebugInfo/CodeView/RecordIterator.h"
 #include "llvm/DebugInfo/CodeView/StreamRef.h"
 #include "llvm/DebugInfo/CodeView/SymbolRecord.h"
 #include "llvm/DebugInfo/PDB/Raw/MappedBlockStream.h"
@@ -29,14 +29,14 @@ public:
 
   Error reload();
 
-  iterator_range<codeview::SymbolIterator> symbols() const;
+  iterator_range<codeview::CVSymbolArray::Iterator> symbols() const;
 
 private:
   const ModInfo &Mod;
 
   MappedBlockStream Stream;
 
-  codeview::ByteStream SymbolsSubstream;
+  codeview::CVSymbolArray SymbolsSubstream;
   codeview::StreamRef LinesSubstream;
   codeview::StreamRef C13LinesSubstream;
   codeview::StreamRef GlobalRefsSubstream;

llvm/lib/DebugInfo/CodeView/SymbolDumper.cpp

@@ -866,7 +866,7 @@ void CVSymbolDumperImpl::visitUnknownSymbol(SymbolKind Kind,
   W.printNumber("Length", uint32_t(Data.size()));
 }
 
-bool CVSymbolDumper::dump(const SymbolIterator::Record &Record) {
+bool CVSymbolDumper::dump(const CVRecord<SymbolKind> &Record) {
   CVSymbolDumperImpl Dumper(CVTD, ObjDelegate.get(), W, PrintRecordBytes);
   Dumper.visitSymbolRecord(Record);
   return !Dumper.hadError();

llvm/lib/DebugInfo/CodeView/TypeDumper.cpp

@@ -676,7 +676,7 @@ void CVTypeDumper::printTypeIndex(StringRef FieldName, TypeIndex TI) {
     W->printHex(FieldName, TI.getIndex());
 }
 
-bool CVTypeDumper::dump(const TypeIterator::Record &Record) {
+bool CVTypeDumper::dump(const CVRecord<TypeLeafKind> &Record) {
   assert(W && "printer should not be null");
   CVTypeDumperImpl Dumper(*this, *W, PrintRecordBytes);
   Dumper.visitTypeRecord(Record);

llvm/lib/DebugInfo/PDB/Raw/DbiStream.cpp

@@ -136,14 +136,12 @@ Error DbiStream::reload() {
     return make_error<RawError>(raw_error_code::corrupt_file,
                                 "DBI type server substream not aligned.");
 
-  if (auto EC =
-          Reader.readStreamRef(ModInfoSubstream, Header->ModiSubstreamSize))
-    return EC;
-
   // Since each ModInfo in the stream is a variable length, we have to iterate
   // them to know how many there actually are.
-  codeview::VarStreamArray ModInfoArray(ModInfoSubstream, ModInfoRecordLength);
-  for (auto Info : ModInfoArray) {
+  codeview::VarStreamArray<ModInfo> ModInfoArray;
+  if (auto EC = Reader.readArray(ModInfoArray, Header->ModiSubstreamSize))
+    return EC;
+  for (auto &Info : ModInfoArray) {
     ModuleInfos.emplace_back(Info);
   }
 

llvm/lib/DebugInfo/PDB/Raw/ModInfo.cpp

@@ -67,6 +67,8 @@ struct ModInfo::FileLayout {
                              // Null terminated Obj File Name
 };
 
+ModInfo::ModInfo() : Layout(nullptr) {}
+
 ModInfo::ModInfo(codeview::StreamRef Stream) : Layout(nullptr) {
   codeview::StreamReader Reader(Stream);
   if (auto EC = Reader.readObject(Layout)) {

llvm/lib/DebugInfo/PDB/Raw/ModStream.cpp

@@ -9,6 +9,7 @@
 
 #include "llvm/DebugInfo/PDB/Raw/ModStream.h"
 
+#include "llvm/DebugInfo/CodeView/RecordIterator.h"
 #include "llvm/DebugInfo/CodeView/StreamReader.h"
 #include "llvm/DebugInfo/PDB/Raw/ModInfo.h"
 #include "llvm/DebugInfo/PDB/Raw/RawError.h"
@@ -32,8 +33,14 @@ Error ModStream::reload() {
     return llvm::make_error<RawError>(raw_error_code::corrupt_file,
                                       "Module has both C11 and C13 line info");
 
-  if (auto EC = SymbolsSubstream.load(Reader, SymbolSize))
+  codeview::StreamRef S;
+
+  uint32_t SymbolSubstreamSig = 0;
+  if (auto EC = Reader.readInteger(SymbolSubstreamSig))
     return EC;
+  if (auto EC = Reader.readArray(SymbolsSubstream, SymbolSize - 4))
+    return EC;
+
   if (auto EC = Reader.readStreamRef(LinesSubstream, C11Size))
     return EC;
   if (auto EC = Reader.readStreamRef(C13LinesSubstream, C13Size))
@@ -51,6 +58,6 @@ Error ModStream::reload() {
   return Error::success();
 }
 
-iterator_range<codeview::SymbolIterator> ModStream::symbols() const {
-  return codeview::makeSymbolRange(SymbolsSubstream.data().slice(4), nullptr);
+iterator_range<codeview::CVSymbolArray::Iterator> ModStream::symbols() const {
+  return llvm::make_range(SymbolsSubstream.begin(), SymbolsSubstream.end());
 }