Resubmit "[pdb] Actually write a PDB to disk from YAML.""

Reviewed By: ruiu
Differential Revision: http://reviews.llvm.org/D21220

llvm-svn: 272708

llvm/include/llvm/DebugInfo/CodeView/StreamRef.h

@@ -89,8 +89,6 @@ public:
     return true;
   }
 
-  bool operator!=(const StreamRef &Other) const { return !(*this == Other); }
-
 private:
   const StreamInterface *Stream;
   uint32_t ViewOffset;

llvm/include/llvm/DebugInfo/CodeView/StreamWriter.h

@@ -42,6 +42,10 @@ public:
   }
 
   template <typename T> Error writeObject(const T &Obj) {
+    static_assert(!std::is_pointer<T>::value,
+                  "writeObject should not be used with pointers, to write "
+                  "the pointed-to value dereference the pointer before calling "
+                  "writeObject");
     return writeBytes(
         ArrayRef<uint8_t>(reinterpret_cast<const uint8_t *>(&Obj), sizeof(T)));
   }

llvm/include/llvm/DebugInfo/PDB/Raw/PDBFile.h

@@ -82,12 +82,18 @@ public:
   uint32_t getStreamByteSize(uint32_t StreamIndex) const override;
   ArrayRef<support::ulittle32_t>
   getStreamBlockList(uint32_t StreamIndex) const override;
+  size_t getFileSize() const;
 
   ArrayRef<uint8_t> getBlockData(uint32_t BlockIndex,
                                  uint32_t NumBytes) const override;
   Error setBlockData(uint32_t BlockIndex, uint32_t Offset,
                      ArrayRef<uint8_t> Data) const override;
 
+  ArrayRef<support::ulittle32_t> getStreamSizes() const { return StreamSizes; }
+  ArrayRef<ArrayRef<support::ulittle32_t>> getStreamMap() const {
+    return StreamMap;
+  }
+
   ArrayRef<support::ulittle32_t> getDirectoryBlockArray() const;
 
   Error parseFileHeaders();
@@ -109,6 +115,11 @@ public:
   Expected<SymbolStream &> getPDBSymbolStream();
   Expected<NameHashTable &> getStringTable();
 
+  void setSuperBlock(const SuperBlock *Block);
+  void setStreamSizes(ArrayRef<support::ulittle32_t> Sizes);
+  void setStreamMap(ArrayRef<ArrayRef<support::ulittle32_t>> Blocks);
+  void commit();
+
 private:
   std::unique_ptr<codeview::StreamInterface> Buffer;
   const PDBFile::SuperBlock *SB;

llvm/lib/DebugInfo/PDB/Raw/PDBFile.cpp

@@ -23,6 +23,7 @@
 #include "llvm/DebugInfo/PDB/Raw/SymbolStream.h"
 #include "llvm/DebugInfo/PDB/Raw/TpiStream.h"
 #include "llvm/Support/Endian.h"
+#include "llvm/Support/FileOutputBuffer.h"
 #include "llvm/Support/MemoryBuffer.h"
 
 using namespace llvm;
@@ -69,6 +70,8 @@ PDBFile::getStreamBlockList(uint32_t StreamIndex) const {
   return StreamMap[StreamIndex];
 }
 
+size_t PDBFile::getFileSize() const { return Buffer->getLength(); }
+
 ArrayRef<uint8_t> PDBFile::getBlockData(uint32_t BlockIndex,
                                         uint32_t NumBytes) const {
   uint64_t StreamBlockOffset = blockToOffset(BlockIndex, getBlockSize());
@@ -317,3 +320,15 @@ Expected<NameHashTable &> PDBFile::getStringTable() {
   }
   return *StringTable;
 }
+
+void PDBFile::setSuperBlock(const SuperBlock *Block) { SB = Block; }
+
+void PDBFile::setStreamSizes(ArrayRef<support::ulittle32_t> Sizes) {
+  StreamSizes = Sizes;
+}
+
+void PDBFile::setStreamMap(ArrayRef<ArrayRef<support::ulittle32_t>> Blocks) {
+  StreamMap = Blocks;
+}
+
+void PDBFile::commit() {}

llvm/test/DebugInfo/PDB/pdbdump-yaml.test

@@ -2,16 +2,18 @@
 ; RUN:               | FileCheck -check-prefix=YAML %s
 
 ; YAML:      ---
-; YAML-NEXT: MSF:
+; YAML-NEXT: MSF:
-; YAML-NEXT:   BlockSize:       4096
+; YAML-NEXT:   SuperBlock:
-; YAML-NEXT:   Unknown0:        2
+; YAML-NEXT:     BlockSize:       4096
-; YAML-NEXT:   NumBlocks:       25
+; YAML-NEXT:     Unknown0:        2
-; YAML-NEXT:   NumDirectoryBytes: 136
+; YAML-NEXT:     NumBlocks:       25
-; YAML-NEXT:   Unknown1:        0
+; YAML-NEXT:     NumDirectoryBytes: 136
-; YAML-NEXT:   BlockMapAddr:    24
+; YAML-NEXT:     Unknown1:        0
+; YAML-NEXT:     BlockMapAddr:    24
 ; YAML-NEXT:   NumDirectoryBlocks: 1
 ; YAML-NEXT:   BlockMapOffset:  98304
 ; YAML-NEXT:   DirectoryBlocks:
 ; YAML-NEXT:     - 23
-; YAML-NEXT:   NumStreams:      17
+; YAML-NEXT:   NumStreams:      17
+; YAML-NEXT:   FileSize:        102400
 ; YAML-NEXT: ...

llvm/tools/llvm-pdbdump/PdbYaml.cpp

@@ -16,17 +16,31 @@ using namespace llvm::yaml;
 using namespace llvm::pdb;
 using namespace llvm::pdb::yaml;
 
+void MappingTraits<PDBFile::SuperBlock>::mapping(IO &IO,
+                                                 PDBFile::SuperBlock &SB) {
+  if (!IO.outputting()) {
+    ::memcpy(SB.MagicBytes, MsfMagic, sizeof(MsfMagic));
+  }
+
+  IO.mapRequired("BlockSize", SB.BlockSize);
+  IO.mapRequired("Unknown0", SB.Unknown0);
+  IO.mapRequired("NumBlocks", SB.NumBlocks);
+  IO.mapRequired("NumDirectoryBytes", SB.NumDirectoryBytes);
+  IO.mapRequired("Unknown1", SB.Unknown1);
+  IO.mapRequired("BlockMapAddr", SB.BlockMapAddr);
+}
+
+void MappingTraits<StreamBlockList>::mapping(IO &IO, StreamBlockList &SB) {
+  IO.mapRequired("Stream", SB.Blocks);
+}
+
 void MappingTraits<MsfHeaders>::mapping(IO &IO, MsfHeaders &Obj) {
-  IO.mapRequired("BlockSize", Obj.BlockSize);
+  IO.mapRequired("SuperBlock", Obj.SuperBlock);
-  IO.mapRequired("Unknown0", Obj.Unknown0);
-  IO.mapRequired("NumBlocks", Obj.BlockCount);
-  IO.mapRequired("NumDirectoryBytes", Obj.NumDirectoryBytes);
-  IO.mapRequired("Unknown1", Obj.Unknown1);
-  IO.mapRequired("BlockMapAddr", Obj.BlockMapIndex);
   IO.mapRequired("NumDirectoryBlocks", Obj.NumDirectoryBlocks);
   IO.mapRequired("BlockMapOffset", Obj.BlockMapOffset);
   IO.mapRequired("DirectoryBlocks", Obj.DirectoryBlocks);
   IO.mapRequired("NumStreams", Obj.NumStreams);
+  IO.mapRequired("FileSize", Obj.FileSize);
 }
 
 void MappingTraits<PdbObject>::mapping(IO &IO, PdbObject &Obj) {
@@ -34,11 +48,3 @@ void MappingTraits<PdbObject>::mapping(IO &IO, PdbObject &Obj) {
   IO.mapOptional("StreamSizes", Obj.StreamSizes);
   IO.mapOptional("StreamMap", Obj.StreamMap);
 }
-
-void MappingTraits<StreamSizeEntry>::mapping(IO &IO, StreamSizeEntry &Obj) {
-  IO.mapRequired("Size", Obj.Size);
-}
-
-void MappingTraits<StreamMapEntry>::mapping(IO &IO, StreamMapEntry &Obj) {
-  IO.mapRequired("Blocks", Obj.Blocks);
-}

llvm/tools/llvm-pdbdump/PdbYaml.h

@@ -13,6 +13,7 @@
 #include "OutputStyle.h"
 
 #include "llvm/ADT/Optional.h"
+#include "llvm/DebugInfo/PDB/Raw/PDBFile.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/YAMLTraits.h"
 
@@ -20,34 +21,25 @@
 
 namespace llvm {
 namespace pdb {
-class PDBFile;
 
 namespace yaml {
 struct MsfHeaders {
-  uint32_t BlockSize;
+  PDBFile::SuperBlock SuperBlock;
-  uint32_t Unknown0;
-  uint32_t BlockCount;
-  uint32_t NumDirectoryBytes;
-  uint32_t Unknown1;
-  uint32_t BlockMapIndex;
   uint32_t NumDirectoryBlocks;
   uint32_t BlockMapOffset;
-  std::vector<uint32_t> DirectoryBlocks;
+  std::vector<support::ulittle32_t> DirectoryBlocks;
   uint32_t NumStreams;
+  uint32_t FileSize;
 };
 
-struct StreamSizeEntry {
+struct StreamBlockList {
-  uint32_t Size;
+  std::vector<support::ulittle32_t> Blocks;
-};
-
-struct StreamMapEntry {
-  std::vector<uint32_t> Blocks;
 };
 
 struct PdbObject {
-  Optional<MsfHeaders> Headers;
+  MsfHeaders Headers;
-  Optional<std::vector<StreamSizeEntry>> StreamSizes;
+  Optional<std::vector<support::ulittle32_t>> StreamSizes;
-  Optional<std::vector<StreamMapEntry>> StreamMap;
+  Optional<std::vector<StreamBlockList>> StreamMap;
 };
 }
 }
@@ -55,25 +47,26 @@ struct PdbObject {
 
 namespace llvm {
 namespace yaml {
-template <> struct MappingTraits<pdb::yaml::StreamSizeEntry> {
+
-  static void mapping(IO &IO, pdb::yaml::StreamSizeEntry &Obj);
+template <> struct MappingTraits<pdb::PDBFile::SuperBlock> {
+  static void mapping(IO &IO, pdb::PDBFile::SuperBlock &SB);
 };
 
-template <> struct MappingTraits<pdb::yaml::StreamMapEntry> {
+template <> struct MappingTraits<pdb::yaml::StreamBlockList> {
-  static void mapping(IO &IO, pdb::yaml::StreamMapEntry &Obj);
+  static void mapping(IO &IO, pdb::yaml::StreamBlockList &SB);
 };
 
 template <> struct MappingTraits<pdb::yaml::MsfHeaders> {
   static void mapping(IO &IO, pdb::yaml::MsfHeaders &Obj);
 };
+
 template <> struct MappingTraits<pdb::yaml::PdbObject> {
   static void mapping(IO &IO, pdb::yaml::PdbObject &Obj);
 };
 }
 }
 
-LLVM_YAML_IS_SEQUENCE_VECTOR(uint32_t)
+LLVM_YAML_IS_SEQUENCE_VECTOR(support::ulittle32_t)
-LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::pdb::yaml::StreamSizeEntry)
+LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::pdb::yaml::StreamBlockList)
-LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::pdb::yaml::StreamMapEntry)
 
 #endif // LLVM_TOOLS_LLVMPDBDUMP_PDBYAML_H

llvm/tools/llvm-pdbdump/YAMLOutputStyle.cpp

@@ -24,19 +24,18 @@ Error YAMLOutputStyle::dumpFileHeaders() {
     return Error::success();
 
   yaml::MsfHeaders Headers;
-  Headers.BlockCount = File.getBlockCount();
+  Obj.Headers.SuperBlock.NumBlocks = File.getBlockCount();
-  Headers.BlockMapIndex = File.getBlockMapIndex();
+  Obj.Headers.SuperBlock.BlockMapAddr = File.getBlockMapIndex();
-  Headers.BlockMapOffset = File.getBlockMapOffset();
+  Obj.Headers.BlockMapOffset = File.getBlockMapOffset();
-  Headers.BlockSize = File.getBlockSize();
+  Obj.Headers.SuperBlock.BlockSize = File.getBlockSize();
   auto Blocks = File.getDirectoryBlockArray();
-  Headers.DirectoryBlocks.assign(Blocks.begin(), Blocks.end());
+  Obj.Headers.DirectoryBlocks.assign(Blocks.begin(), Blocks.end());
-  Headers.NumDirectoryBlocks = File.getNumDirectoryBlocks();
+  Obj.Headers.NumDirectoryBlocks = File.getNumDirectoryBlocks();
-  Headers.NumDirectoryBytes = File.getNumDirectoryBytes();
+  Obj.Headers.SuperBlock.NumDirectoryBytes = File.getNumDirectoryBytes();
-  Headers.NumStreams = File.getNumStreams();
+  Obj.Headers.NumStreams = File.getNumStreams();
-  Headers.Unknown0 = File.getUnknown0();
+  Obj.Headers.SuperBlock.Unknown0 = File.getUnknown0();
-  Headers.Unknown1 = File.getUnknown1();
+  Obj.Headers.SuperBlock.Unknown1 = File.getUnknown1();
-
+  Obj.Headers.FileSize = File.getFileSize();
-  Obj.Headers.emplace(Headers);
 
   return Error::success();
 }
@@ -45,13 +44,7 @@ Error YAMLOutputStyle::dumpStreamSummary() {
   if (!opts::DumpStreamSummary)
     return Error::success();
 
-  std::vector<yaml::StreamSizeEntry> Sizes;
+  Obj.StreamSizes = File.getStreamSizes();
-  for (uint32_t I = 0; I < File.getNumStreams(); ++I) {
-    yaml::StreamSizeEntry Entry;
-    Entry.Size = File.getStreamByteSize(I);
-    Sizes.push_back(Entry);
-  }
-  Obj.StreamSizes.emplace(Sizes);
   return Error::success();
 }
 
@@ -59,14 +52,13 @@ Error YAMLOutputStyle::dumpStreamBlocks() {
   if (!opts::DumpStreamBlocks)
     return Error::success();
 
-  std::vector<yaml::StreamMapEntry> Blocks;
+  auto StreamMap = File.getStreamMap();
-  for (uint32_t I = 0; I < File.getNumStreams(); ++I) {
+  Obj.StreamMap.emplace();
-    yaml::StreamMapEntry Entry;
+  for (auto &Stream : StreamMap) {
-    auto BlockList = File.getStreamBlockList(I);
+    pdb::yaml::StreamBlockList BlockList;
-    Entry.Blocks.assign(BlockList.begin(), BlockList.end());
+    BlockList.Blocks = Stream;
-    Blocks.push_back(Entry);
+    Obj.StreamMap->push_back(BlockList);
   }
-  Obj.StreamMap.emplace(Blocks);
 
   return Error::success();
 }

llvm/tools/llvm-pdbdump/llvm-pdbdump.cpp

@@ -29,6 +29,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Config/config.h"
+#include "llvm/DebugInfo/CodeView/ByteStream.h"
 #include "llvm/DebugInfo/PDB/GenericError.h"
 #include "llvm/DebugInfo/PDB/IPDBEnumChildren.h"
 #include "llvm/DebugInfo/PDB/IPDBRawSymbol.h"
@@ -46,6 +47,7 @@
 #include "llvm/Support/COM.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ConvertUTF.h"
+#include "llvm/Support/FileOutputBuffer.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/ManagedStatic.h"
@@ -60,6 +62,21 @@ using namespace llvm;
 using namespace llvm::codeview;
 using namespace llvm::pdb;
 
+namespace {
+// A simple adapter that acts like a ByteStream but holds ownership over
+// and underlying FileOutputBuffer.
+class FileBufferByteStream : public ByteStream<true> {
+public:
+  FileBufferByteStream(std::unique_ptr<FileOutputBuffer> Buffer)
+      : ByteStream(MutableArrayRef<uint8_t>(Buffer->getBufferStart(),
+                                            Buffer->getBufferEnd())),
+        FileBuffer(std::move(Buffer)) {}
+
+private:
+  std::unique_ptr<FileOutputBuffer> FileBuffer;
+};
+}
+
 namespace opts {
 
 enum class PDB_DumpType { ByType, ByObjFile, Both };
@@ -163,6 +180,14 @@ cl::opt<bool>
            cl::desc("Implies most other options in 'Native Options' category"),
            cl::cat(NativeOptions));
 
+cl::opt<bool>
+    YamlToPdb("yaml-to-pdb",
+              cl::desc("The input file is yaml, and the tool outputs a pdb"),
+              cl::cat(NativeOptions));
+cl::opt<std::string> YamlPdbOutputFile(
+    "pdb-output", cl::desc("When yaml-to-pdb is specified, the output file"),
+    cl::cat(NativeOptions));
+
 cl::list<std::string>
     ExcludeTypes("exclude-types",
                  cl::desc("Exclude types by regular expression"),
@@ -307,6 +332,45 @@ bool isRawDumpEnabled() {
   return false;
 }
 
+static void yamlToPdb(StringRef Path) {
+  ErrorOr<std::unique_ptr<MemoryBuffer>> ErrorOrBuffer =
+      MemoryBuffer::getFileOrSTDIN(Path, /*FileSize=*/-1,
+                                   /*RequiresNullTerminator=*/false);
+
+  if (ErrorOrBuffer.getError()) {
+    ExitOnErr(make_error<GenericError>(generic_error_code::invalid_path, Path));
+  }
+
+  std::unique_ptr<MemoryBuffer> &Buffer = ErrorOrBuffer.get();
+
+  llvm::yaml::Input In(Buffer->getBuffer());
+  pdb::yaml::PdbObject YamlObj;
+  In >> YamlObj;
+
+  auto OutFileOrError = FileOutputBuffer::create(opts::YamlPdbOutputFile,
+                                                 YamlObj.Headers.FileSize);
+  if (OutFileOrError.getError())
+    ExitOnErr(make_error<GenericError>(generic_error_code::invalid_path,
+                                       opts::YamlPdbOutputFile));
+
+  auto FileByteStream =
+      llvm::make_unique<FileBufferByteStream>(std::move(*OutFileOrError));
+  PDBFile Pdb(std::move(FileByteStream));
+  Pdb.setSuperBlock(&YamlObj.Headers.SuperBlock);
+  if (YamlObj.StreamMap.hasValue()) {
+    std::vector<ArrayRef<support::ulittle32_t>> StreamMap;
+    for (auto &E : YamlObj.StreamMap.getValue()) {
+      StreamMap.push_back(E.Blocks);
+    }
+    Pdb.setStreamMap(StreamMap);
+  }
+  if (YamlObj.StreamSizes.hasValue()) {
+    Pdb.setStreamSizes(YamlObj.StreamSizes.getValue());
+  }
+
+  Pdb.commit();
+}
+
 static void dumpInput(StringRef Path) {
   std::unique_ptr<IPDBSession> Session;
   if (isRawDumpEnabled()) {
@@ -494,8 +558,13 @@ int main(int argc_, const char *argv_[]) {
 
   llvm::sys::InitializeCOMRAII COM(llvm::sys::COMThreadingMode::MultiThreaded);
 
-  std::for_each(opts::InputFilenames.begin(), opts::InputFilenames.end(),
+  if (opts::YamlToPdb) {
-                dumpInput);
+    std::for_each(opts::InputFilenames.begin(), opts::InputFilenames.end(),
+                  yamlToPdb);
+  } else {
+    std::for_each(opts::InputFilenames.begin(), opts::InputFilenames.end(),
+                  dumpInput);
+  }
 
   outs().flush();
   return 0;