From 1dc9fd3c4a260883b1559f7b92ad9156ecc2418c Mon Sep 17 00:00:00 2001 From: Zachary Turner Date: Tue, 14 Jun 2016 20:48:36 +0000 Subject: [PATCH] Resubmit "[pdb] Actually write a PDB to disk from YAML."" Reviewed By: ruiu Differential Revision: http://reviews.llvm.org/D21220 llvm-svn: 272708 --- .../llvm/DebugInfo/CodeView/StreamRef.h | 2 - .../llvm/DebugInfo/CodeView/StreamWriter.h | 4 + llvm/include/llvm/DebugInfo/PDB/Raw/PDBFile.h | 11 +++ llvm/lib/DebugInfo/PDB/Raw/PDBFile.cpp | 15 ++++ llvm/test/DebugInfo/PDB/pdbdump-yaml.test | 18 +++-- llvm/tools/llvm-pdbdump/PdbYaml.cpp | 34 +++++---- llvm/tools/llvm-pdbdump/PdbYaml.h | 41 +++++------ llvm/tools/llvm-pdbdump/YAMLOutputStyle.cpp | 44 +++++------ llvm/tools/llvm-pdbdump/llvm-pdbdump.cpp | 73 ++++++++++++++++++- 9 files changed, 166 insertions(+), 76 deletions(-) diff --git a/llvm/include/llvm/DebugInfo/CodeView/StreamRef.h b/llvm/include/llvm/DebugInfo/CodeView/StreamRef.h index 93b116b78c85..f6e3da59c28c 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/StreamRef.h +++ b/llvm/include/llvm/DebugInfo/CodeView/StreamRef.h @@ -89,8 +89,6 @@ public: return true; } - bool operator!=(const StreamRef &Other) const { return !(*this == Other); } - private: const StreamInterface *Stream; uint32_t ViewOffset; diff --git a/llvm/include/llvm/DebugInfo/CodeView/StreamWriter.h b/llvm/include/llvm/DebugInfo/CodeView/StreamWriter.h index ad9277b8e225..4d393d2ef790 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/StreamWriter.h +++ b/llvm/include/llvm/DebugInfo/CodeView/StreamWriter.h @@ -42,6 +42,10 @@ public: } template Error writeObject(const T &Obj) { + static_assert(!std::is_pointer::value, + "writeObject should not be used with pointers, to write " + "the pointed-to value dereference the pointer before calling " + "writeObject"); return writeBytes( ArrayRef(reinterpret_cast(&Obj), sizeof(T))); } diff --git a/llvm/include/llvm/DebugInfo/PDB/Raw/PDBFile.h b/llvm/include/llvm/DebugInfo/PDB/Raw/PDBFile.h index b79717d40f0e..966716fbc56c 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Raw/PDBFile.h +++ b/llvm/include/llvm/DebugInfo/PDB/Raw/PDBFile.h @@ -82,12 +82,18 @@ public: uint32_t getStreamByteSize(uint32_t StreamIndex) const override; ArrayRef getStreamBlockList(uint32_t StreamIndex) const override; + size_t getFileSize() const; ArrayRef getBlockData(uint32_t BlockIndex, uint32_t NumBytes) const override; Error setBlockData(uint32_t BlockIndex, uint32_t Offset, ArrayRef Data) const override; + ArrayRef getStreamSizes() const { return StreamSizes; } + ArrayRef> getStreamMap() const { + return StreamMap; + } + ArrayRef getDirectoryBlockArray() const; Error parseFileHeaders(); @@ -109,6 +115,11 @@ public: Expected getPDBSymbolStream(); Expected getStringTable(); + void setSuperBlock(const SuperBlock *Block); + void setStreamSizes(ArrayRef Sizes); + void setStreamMap(ArrayRef> Blocks); + void commit(); + private: std::unique_ptr Buffer; const PDBFile::SuperBlock *SB; diff --git a/llvm/lib/DebugInfo/PDB/Raw/PDBFile.cpp b/llvm/lib/DebugInfo/PDB/Raw/PDBFile.cpp index 6186a4770010..a5c383e5f49e 100644 --- a/llvm/lib/DebugInfo/PDB/Raw/PDBFile.cpp +++ b/llvm/lib/DebugInfo/PDB/Raw/PDBFile.cpp @@ -23,6 +23,7 @@ #include "llvm/DebugInfo/PDB/Raw/SymbolStream.h" #include "llvm/DebugInfo/PDB/Raw/TpiStream.h" #include "llvm/Support/Endian.h" +#include "llvm/Support/FileOutputBuffer.h" #include "llvm/Support/MemoryBuffer.h" using namespace llvm; @@ -69,6 +70,8 @@ PDBFile::getStreamBlockList(uint32_t StreamIndex) const { return StreamMap[StreamIndex]; } +size_t PDBFile::getFileSize() const { return Buffer->getLength(); } + ArrayRef PDBFile::getBlockData(uint32_t BlockIndex, uint32_t NumBytes) const { uint64_t StreamBlockOffset = blockToOffset(BlockIndex, getBlockSize()); @@ -317,3 +320,15 @@ Expected PDBFile::getStringTable() { } return *StringTable; } + +void PDBFile::setSuperBlock(const SuperBlock *Block) { SB = Block; } + +void PDBFile::setStreamSizes(ArrayRef Sizes) { + StreamSizes = Sizes; +} + +void PDBFile::setStreamMap(ArrayRef> Blocks) { + StreamMap = Blocks; +} + +void PDBFile::commit() {} \ No newline at end of file diff --git a/llvm/test/DebugInfo/PDB/pdbdump-yaml.test b/llvm/test/DebugInfo/PDB/pdbdump-yaml.test index c629b1fd1306..99df5dd92b15 100644 --- a/llvm/test/DebugInfo/PDB/pdbdump-yaml.test +++ b/llvm/test/DebugInfo/PDB/pdbdump-yaml.test @@ -2,16 +2,18 @@ ; RUN: | FileCheck -check-prefix=YAML %s ; YAML: --- -; YAML-NEXT: MSF: -; YAML-NEXT: BlockSize: 4096 -; YAML-NEXT: Unknown0: 2 -; YAML-NEXT: NumBlocks: 25 -; YAML-NEXT: NumDirectoryBytes: 136 -; YAML-NEXT: Unknown1: 0 -; YAML-NEXT: BlockMapAddr: 24 +; YAML-NEXT: MSF: +; YAML-NEXT: SuperBlock: +; YAML-NEXT: BlockSize: 4096 +; YAML-NEXT: Unknown0: 2 +; YAML-NEXT: NumBlocks: 25 +; YAML-NEXT: NumDirectoryBytes: 136 +; YAML-NEXT: Unknown1: 0 +; YAML-NEXT: BlockMapAddr: 24 ; YAML-NEXT: NumDirectoryBlocks: 1 ; YAML-NEXT: BlockMapOffset: 98304 ; YAML-NEXT: DirectoryBlocks: ; YAML-NEXT: - 23 -; YAML-NEXT: NumStreams: 17 +; YAML-NEXT: NumStreams: 17 +; YAML-NEXT: FileSize: 102400 ; YAML-NEXT: ... diff --git a/llvm/tools/llvm-pdbdump/PdbYaml.cpp b/llvm/tools/llvm-pdbdump/PdbYaml.cpp index efaa986a8178..fa15ff2b39d3 100644 --- a/llvm/tools/llvm-pdbdump/PdbYaml.cpp +++ b/llvm/tools/llvm-pdbdump/PdbYaml.cpp @@ -16,17 +16,31 @@ using namespace llvm::yaml; using namespace llvm::pdb; using namespace llvm::pdb::yaml; +void MappingTraits::mapping(IO &IO, + PDBFile::SuperBlock &SB) { + if (!IO.outputting()) { + ::memcpy(SB.MagicBytes, MsfMagic, sizeof(MsfMagic)); + } + + IO.mapRequired("BlockSize", SB.BlockSize); + IO.mapRequired("Unknown0", SB.Unknown0); + IO.mapRequired("NumBlocks", SB.NumBlocks); + IO.mapRequired("NumDirectoryBytes", SB.NumDirectoryBytes); + IO.mapRequired("Unknown1", SB.Unknown1); + IO.mapRequired("BlockMapAddr", SB.BlockMapAddr); +} + +void MappingTraits::mapping(IO &IO, StreamBlockList &SB) { + IO.mapRequired("Stream", SB.Blocks); +} + void MappingTraits::mapping(IO &IO, MsfHeaders &Obj) { - IO.mapRequired("BlockSize", Obj.BlockSize); - IO.mapRequired("Unknown0", Obj.Unknown0); - IO.mapRequired("NumBlocks", Obj.BlockCount); - IO.mapRequired("NumDirectoryBytes", Obj.NumDirectoryBytes); - IO.mapRequired("Unknown1", Obj.Unknown1); - IO.mapRequired("BlockMapAddr", Obj.BlockMapIndex); + IO.mapRequired("SuperBlock", Obj.SuperBlock); IO.mapRequired("NumDirectoryBlocks", Obj.NumDirectoryBlocks); IO.mapRequired("BlockMapOffset", Obj.BlockMapOffset); IO.mapRequired("DirectoryBlocks", Obj.DirectoryBlocks); IO.mapRequired("NumStreams", Obj.NumStreams); + IO.mapRequired("FileSize", Obj.FileSize); } void MappingTraits::mapping(IO &IO, PdbObject &Obj) { @@ -34,11 +48,3 @@ void MappingTraits::mapping(IO &IO, PdbObject &Obj) { IO.mapOptional("StreamSizes", Obj.StreamSizes); IO.mapOptional("StreamMap", Obj.StreamMap); } - -void MappingTraits::mapping(IO &IO, StreamSizeEntry &Obj) { - IO.mapRequired("Size", Obj.Size); -} - -void MappingTraits::mapping(IO &IO, StreamMapEntry &Obj) { - IO.mapRequired("Blocks", Obj.Blocks); -} diff --git a/llvm/tools/llvm-pdbdump/PdbYaml.h b/llvm/tools/llvm-pdbdump/PdbYaml.h index a7c942a3bd8e..f6e070f8398d 100644 --- a/llvm/tools/llvm-pdbdump/PdbYaml.h +++ b/llvm/tools/llvm-pdbdump/PdbYaml.h @@ -13,6 +13,7 @@ #include "OutputStyle.h" #include "llvm/ADT/Optional.h" +#include "llvm/DebugInfo/PDB/Raw/PDBFile.h" #include "llvm/Support/Endian.h" #include "llvm/Support/YAMLTraits.h" @@ -20,34 +21,25 @@ namespace llvm { namespace pdb { -class PDBFile; namespace yaml { struct MsfHeaders { - uint32_t BlockSize; - uint32_t Unknown0; - uint32_t BlockCount; - uint32_t NumDirectoryBytes; - uint32_t Unknown1; - uint32_t BlockMapIndex; + PDBFile::SuperBlock SuperBlock; uint32_t NumDirectoryBlocks; uint32_t BlockMapOffset; - std::vector DirectoryBlocks; + std::vector DirectoryBlocks; uint32_t NumStreams; + uint32_t FileSize; }; -struct StreamSizeEntry { - uint32_t Size; -}; - -struct StreamMapEntry { - std::vector Blocks; +struct StreamBlockList { + std::vector Blocks; }; struct PdbObject { - Optional Headers; - Optional> StreamSizes; - Optional> StreamMap; + MsfHeaders Headers; + Optional> StreamSizes; + Optional> StreamMap; }; } } @@ -55,25 +47,26 @@ struct PdbObject { namespace llvm { namespace yaml { -template <> struct MappingTraits { - static void mapping(IO &IO, pdb::yaml::StreamSizeEntry &Obj); + +template <> struct MappingTraits { + static void mapping(IO &IO, pdb::PDBFile::SuperBlock &SB); }; -template <> struct MappingTraits { - static void mapping(IO &IO, pdb::yaml::StreamMapEntry &Obj); +template <> struct MappingTraits { + static void mapping(IO &IO, pdb::yaml::StreamBlockList &SB); }; template <> struct MappingTraits { static void mapping(IO &IO, pdb::yaml::MsfHeaders &Obj); }; + template <> struct MappingTraits { static void mapping(IO &IO, pdb::yaml::PdbObject &Obj); }; } } -LLVM_YAML_IS_SEQUENCE_VECTOR(uint32_t) -LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::pdb::yaml::StreamSizeEntry) -LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::pdb::yaml::StreamMapEntry) +LLVM_YAML_IS_SEQUENCE_VECTOR(support::ulittle32_t) +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::pdb::yaml::StreamBlockList) #endif // LLVM_TOOLS_LLVMPDBDUMP_PDBYAML_H diff --git a/llvm/tools/llvm-pdbdump/YAMLOutputStyle.cpp b/llvm/tools/llvm-pdbdump/YAMLOutputStyle.cpp index d60ccd7ebbd7..a0cabb7a3669 100644 --- a/llvm/tools/llvm-pdbdump/YAMLOutputStyle.cpp +++ b/llvm/tools/llvm-pdbdump/YAMLOutputStyle.cpp @@ -24,19 +24,18 @@ Error YAMLOutputStyle::dumpFileHeaders() { return Error::success(); yaml::MsfHeaders Headers; - Headers.BlockCount = File.getBlockCount(); - Headers.BlockMapIndex = File.getBlockMapIndex(); - Headers.BlockMapOffset = File.getBlockMapOffset(); - Headers.BlockSize = File.getBlockSize(); + Obj.Headers.SuperBlock.NumBlocks = File.getBlockCount(); + Obj.Headers.SuperBlock.BlockMapAddr = File.getBlockMapIndex(); + Obj.Headers.BlockMapOffset = File.getBlockMapOffset(); + Obj.Headers.SuperBlock.BlockSize = File.getBlockSize(); auto Blocks = File.getDirectoryBlockArray(); - Headers.DirectoryBlocks.assign(Blocks.begin(), Blocks.end()); - Headers.NumDirectoryBlocks = File.getNumDirectoryBlocks(); - Headers.NumDirectoryBytes = File.getNumDirectoryBytes(); - Headers.NumStreams = File.getNumStreams(); - Headers.Unknown0 = File.getUnknown0(); - Headers.Unknown1 = File.getUnknown1(); - - Obj.Headers.emplace(Headers); + Obj.Headers.DirectoryBlocks.assign(Blocks.begin(), Blocks.end()); + Obj.Headers.NumDirectoryBlocks = File.getNumDirectoryBlocks(); + Obj.Headers.SuperBlock.NumDirectoryBytes = File.getNumDirectoryBytes(); + Obj.Headers.NumStreams = File.getNumStreams(); + Obj.Headers.SuperBlock.Unknown0 = File.getUnknown0(); + Obj.Headers.SuperBlock.Unknown1 = File.getUnknown1(); + Obj.Headers.FileSize = File.getFileSize(); return Error::success(); } @@ -45,13 +44,7 @@ Error YAMLOutputStyle::dumpStreamSummary() { if (!opts::DumpStreamSummary) return Error::success(); - std::vector Sizes; - for (uint32_t I = 0; I < File.getNumStreams(); ++I) { - yaml::StreamSizeEntry Entry; - Entry.Size = File.getStreamByteSize(I); - Sizes.push_back(Entry); - } - Obj.StreamSizes.emplace(Sizes); + Obj.StreamSizes = File.getStreamSizes(); return Error::success(); } @@ -59,14 +52,13 @@ Error YAMLOutputStyle::dumpStreamBlocks() { if (!opts::DumpStreamBlocks) return Error::success(); - std::vector Blocks; - for (uint32_t I = 0; I < File.getNumStreams(); ++I) { - yaml::StreamMapEntry Entry; - auto BlockList = File.getStreamBlockList(I); - Entry.Blocks.assign(BlockList.begin(), BlockList.end()); - Blocks.push_back(Entry); + auto StreamMap = File.getStreamMap(); + Obj.StreamMap.emplace(); + for (auto &Stream : StreamMap) { + pdb::yaml::StreamBlockList BlockList; + BlockList.Blocks = Stream; + Obj.StreamMap->push_back(BlockList); } - Obj.StreamMap.emplace(Blocks); return Error::success(); } diff --git a/llvm/tools/llvm-pdbdump/llvm-pdbdump.cpp b/llvm/tools/llvm-pdbdump/llvm-pdbdump.cpp index 1f6b778c4b2f..a7d43ed064cf 100644 --- a/llvm/tools/llvm-pdbdump/llvm-pdbdump.cpp +++ b/llvm/tools/llvm-pdbdump/llvm-pdbdump.cpp @@ -29,6 +29,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/StringExtras.h" #include "llvm/Config/config.h" +#include "llvm/DebugInfo/CodeView/ByteStream.h" #include "llvm/DebugInfo/PDB/GenericError.h" #include "llvm/DebugInfo/PDB/IPDBEnumChildren.h" #include "llvm/DebugInfo/PDB/IPDBRawSymbol.h" @@ -46,6 +47,7 @@ #include "llvm/Support/COM.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/ConvertUTF.h" +#include "llvm/Support/FileOutputBuffer.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Format.h" #include "llvm/Support/ManagedStatic.h" @@ -60,6 +62,21 @@ using namespace llvm; using namespace llvm::codeview; using namespace llvm::pdb; +namespace { +// A simple adapter that acts like a ByteStream but holds ownership over +// and underlying FileOutputBuffer. +class FileBufferByteStream : public ByteStream { +public: + FileBufferByteStream(std::unique_ptr Buffer) + : ByteStream(MutableArrayRef(Buffer->getBufferStart(), + Buffer->getBufferEnd())), + FileBuffer(std::move(Buffer)) {} + +private: + std::unique_ptr FileBuffer; +}; +} + namespace opts { enum class PDB_DumpType { ByType, ByObjFile, Both }; @@ -163,6 +180,14 @@ cl::opt cl::desc("Implies most other options in 'Native Options' category"), cl::cat(NativeOptions)); +cl::opt + YamlToPdb("yaml-to-pdb", + cl::desc("The input file is yaml, and the tool outputs a pdb"), + cl::cat(NativeOptions)); +cl::opt YamlPdbOutputFile( + "pdb-output", cl::desc("When yaml-to-pdb is specified, the output file"), + cl::cat(NativeOptions)); + cl::list ExcludeTypes("exclude-types", cl::desc("Exclude types by regular expression"), @@ -307,6 +332,45 @@ bool isRawDumpEnabled() { return false; } +static void yamlToPdb(StringRef Path) { + ErrorOr> ErrorOrBuffer = + MemoryBuffer::getFileOrSTDIN(Path, /*FileSize=*/-1, + /*RequiresNullTerminator=*/false); + + if (ErrorOrBuffer.getError()) { + ExitOnErr(make_error(generic_error_code::invalid_path, Path)); + } + + std::unique_ptr &Buffer = ErrorOrBuffer.get(); + + llvm::yaml::Input In(Buffer->getBuffer()); + pdb::yaml::PdbObject YamlObj; + In >> YamlObj; + + auto OutFileOrError = FileOutputBuffer::create(opts::YamlPdbOutputFile, + YamlObj.Headers.FileSize); + if (OutFileOrError.getError()) + ExitOnErr(make_error(generic_error_code::invalid_path, + opts::YamlPdbOutputFile)); + + auto FileByteStream = + llvm::make_unique(std::move(*OutFileOrError)); + PDBFile Pdb(std::move(FileByteStream)); + Pdb.setSuperBlock(&YamlObj.Headers.SuperBlock); + if (YamlObj.StreamMap.hasValue()) { + std::vector> StreamMap; + for (auto &E : YamlObj.StreamMap.getValue()) { + StreamMap.push_back(E.Blocks); + } + Pdb.setStreamMap(StreamMap); + } + if (YamlObj.StreamSizes.hasValue()) { + Pdb.setStreamSizes(YamlObj.StreamSizes.getValue()); + } + + Pdb.commit(); +} + static void dumpInput(StringRef Path) { std::unique_ptr Session; if (isRawDumpEnabled()) { @@ -494,8 +558,13 @@ int main(int argc_, const char *argv_[]) { llvm::sys::InitializeCOMRAII COM(llvm::sys::COMThreadingMode::MultiThreaded); - std::for_each(opts::InputFilenames.begin(), opts::InputFilenames.end(), - dumpInput); + if (opts::YamlToPdb) { + std::for_each(opts::InputFilenames.begin(), opts::InputFilenames.end(), + yamlToPdb); + } else { + std::for_each(opts::InputFilenames.begin(), opts::InputFilenames.end(), + dumpInput); + } outs().flush(); return 0;