[pdb] Teach MsfBuilder and other classes about the Free Page Map.

Block 1 and 2 of an MSF file are bit vectors that represent the
list of blocks allocated and free in the file.  We had been using
these blocks to write stream data and other data, so we mark them
as the free page map now.  We don't yet serialize these pages to
the disk, but at least we make a note of what it is, and avoid
writing random data to them.

Doing this also necessitated cleaning up some of the tests to be
more general and hardcode fewer values, which is nice.

llvm-svn: 275629

llvm/include/llvm/DebugInfo/PDB/Raw/MsfBuilder.h

@@ -64,7 +64,7 @@ public:
   /// when editing a PDB and you want the layout to be as stable as possible.
   Error setBlockMapAddr(uint32_t Addr);
   Error setDirectoryBlocksHint(ArrayRef<uint32_t> DirBlocks);
-  void setUnknown0(uint32_t Unk0);
+  void setFreePageMap(uint32_t Fpm);
   void setUnknown1(uint32_t Unk1);
 
   /// Add a stream to the MSF file with the given size, occupying the given
@@ -126,7 +126,7 @@ private:
   BumpPtrAllocator &Allocator;
 
   bool IsGrowable;
-  uint32_t Unknown0;
+  uint32_t FreePageMap;
   uint32_t Unknown1;
   uint32_t BlockSize;
   uint32_t MininumBlocks;

llvm/include/llvm/DebugInfo/PDB/Raw/MsfCommon.h

@@ -35,8 +35,8 @@ struct SuperBlock {
   // These elements are referred to as blocks.  The size of a block may vary
   // from system to system.
   support::ulittle32_t BlockSize;
-  // This field's purpose is not yet known.
-  support::ulittle32_t Unknown0;
+  // The index of the free block map.
+  support::ulittle32_t FreeBlockMapBlock;
   // This contains the number of blocks resident in the file system.  In
   // practice, NumBlocks * BlockSize is equivalent to the size of the PDB
   // file.
@@ -67,6 +67,13 @@ inline bool isValidBlockSize(uint32_t Size) {
   return false;
 }
 
+// Super Block, Fpm0, Fpm1, and Block Map
+inline uint32_t getMinimumBlockCount() { return 4; }
+
+// Super Block, Fpm0, and Fpm1 are reserved.  The Block Map, although required
+// need not be at block 3.
+inline uint32_t getFirstUnreservedBlock() { return 3; }
+
 inline uint64_t bytesToBlocks(uint64_t NumBytes, uint64_t BlockSize) {
   return alignTo(NumBytes, BlockSize) / BlockSize;
 }

llvm/include/llvm/DebugInfo/PDB/Raw/PDBFile.h

@@ -45,7 +45,7 @@ public:
   explicit PDBFile(std::unique_ptr<codeview::StreamInterface> PdbFileBuffer);
   ~PDBFile() override;
 
-  uint32_t getUnknown0() const;
+  uint32_t getFreeBlockMapBlock() const;
   uint32_t getUnknown1() const;
 
   uint32_t getBlockSize() const override;

llvm/lib/DebugInfo/PDB/Raw/MsfBuilder.cpp

@@ -17,15 +17,21 @@ using namespace llvm::support;
 
 namespace {
 const uint32_t kSuperBlockBlock = 0;
-const uint32_t kDefaultBlockMapAddr = 1;
+const uint32_t kFreePageMap0Block = 1;
+const uint32_t kFreePageMap1Block = 2;
+const uint32_t kNumReservedPages = 3;
+
+const uint32_t kDefaultBlockMapAddr = kNumReservedPages;
 }
 
 MsfBuilder::MsfBuilder(uint32_t BlockSize, uint32_t MinBlockCount, bool CanGrow,
                        BumpPtrAllocator &Allocator)
     : Allocator(Allocator), IsGrowable(CanGrow), BlockSize(BlockSize),
       MininumBlocks(MinBlockCount), BlockMapAddr(kDefaultBlockMapAddr),
-      FreeBlocks(std::max(MinBlockCount, 2U), true) {
+      FreeBlocks(MinBlockCount, true) {
   FreeBlocks[kSuperBlockBlock] = false;
+  FreeBlocks[kFreePageMap0Block] = false;
+  FreeBlocks[kFreePageMap1Block] = false;
   FreeBlocks[BlockMapAddr] = false;
 }
 
@@ -36,7 +42,9 @@ Expected<MsfBuilder> MsfBuilder::create(BumpPtrAllocator &Allocator,
     return make_error<RawError>(raw_error_code::unspecified,
                                 "The requested block size is unsupported");
 
-  return MsfBuilder(BlockSize, MinBlockCount, CanGrow, Allocator);
+  return MsfBuilder(BlockSize,
+                    std::max(MinBlockCount, msf::getMinimumBlockCount()),
+                    CanGrow, Allocator);
 }
 
 Error MsfBuilder::setBlockMapAddr(uint32_t Addr) {
@@ -59,7 +67,7 @@ Error MsfBuilder::setBlockMapAddr(uint32_t Addr) {
   return Error::success();
 }
 
-void MsfBuilder::setUnknown0(uint32_t Unk0) { Unknown0 = Unk0; }
+void MsfBuilder::setFreePageMap(uint32_t Fpm) { FreePageMap = Fpm; }
 
 void MsfBuilder::setUnknown1(uint32_t Unk1) { Unknown1 = Unk1; }
 
@@ -217,7 +225,7 @@ Expected<Layout> MsfBuilder::build() {
   L.SB->BlockMapAddr = BlockMapAddr;
   L.SB->BlockSize = BlockSize;
   L.SB->NumDirectoryBytes = computeDirectoryByteSize();
-  L.SB->Unknown0 = Unknown0;
+  L.SB->FreeBlockMapBlock = FreePageMap;
   L.SB->Unknown1 = Unknown1;
 
   uint32_t NumDirectoryBlocks =

llvm/lib/DebugInfo/PDB/Raw/PDBFile.cpp

@@ -42,7 +42,7 @@ PDBFile::~PDBFile() {}
 
 uint32_t PDBFile::getBlockSize() const { return SB->BlockSize; }
 
-uint32_t PDBFile::getUnknown0() const { return SB->Unknown0; }
+uint32_t PDBFile::getFreeBlockMapBlock() const { return SB->FreeBlockMapBlock; }
 
 uint32_t PDBFile::getBlockCount() const { return SB->NumBlocks; }
 

llvm/lib/DebugInfo/PDB/Raw/PDBFileBuilder.cpp

@@ -37,9 +37,9 @@ Error PDBFileBuilder::initialize(const msf::SuperBlock &Super) {
   auto &MsfResult = *ExpectedMsf;
   if (auto EC = MsfResult.setBlockMapAddr(Super.BlockMapAddr))
     return EC;
-  MsfResult.setUnknown0(Super.Unknown0);
-  MsfResult.setUnknown1(Super.Unknown1);
   Msf = llvm::make_unique<MsfBuilder>(std::move(MsfResult));
+  Msf->setFreePageMap(Super.FreeBlockMapBlock);
+  Msf->setUnknown1(Super.Unknown1);
   return Error::success();
 }
 

llvm/test/DebugInfo/PDB/pdbdump-headers.test

@@ -10,7 +10,7 @@
 
 ; EMPTY:      FileHeaders {
 ; EMPTY-NEXT:   BlockSize: 4096
-; EMPTY-NEXT:   Unknown0: 2
+; EMPTY-NEXT:   FreeBlockMap: 2
 ; EMPTY-NEXT:   NumBlocks: 25
 ; EMPTY-NEXT:   NumDirectoryBytes: 136
 ; EMPTY-NEXT:   Unknown1: 0
@@ -945,7 +945,7 @@
 
 ; ALL: FileHeaders {
 ; ALL:   BlockSize: 4096
-; ALL:   Unknown0: 2
+; ALL:   FreeBlockMap: 2
 ; ALL:   NumBlocks: 25
 ; ALL:   NumDirectoryBytes: 136
 ; ALL:   Unknown1: 0
@@ -1660,7 +1660,7 @@
 
 ; BIG:      FileHeaders {
 ; BIG-NEXT:   BlockSize: 4096
-; BIG-NEXT:   Unknown0: 2
+; BIG-NEXT:   FreeBlockMap: 2
 ; BIG-NEXT:   NumBlocks: 99
 ; BIG-NEXT:   NumDirectoryBytes: 616
 ; BIG-NEXT:   Unknown1: 0

llvm/test/DebugInfo/PDB/pdbdump-yaml.test

@@ -7,7 +7,7 @@
 ; YAML-NEXT: MSF:
 ; YAML-NEXT:   SuperBlock:
 ; YAML-NEXT:     BlockSize:       4096
-; YAML-NEXT:     Unknown0:        2
+; YAML-NEXT:     FreeBlockMap:     2
 ; YAML-NEXT:     NumBlocks:       25
 ; YAML-NEXT:     NumDirectoryBytes: 136
 ; YAML-NEXT:     Unknown1:        0

llvm/tools/llvm-pdbdump/LLVMOutputStyle.cpp

@@ -95,7 +95,7 @@ Error LLVMOutputStyle::dumpFileHeaders() {
 
   DictScope D(P, "FileHeaders");
   P.printNumber("BlockSize", File.getBlockSize());
-  P.printNumber("Unknown0", File.getUnknown0());
+  P.printNumber("FreeBlockMap", File.getFreeBlockMapBlock());
   P.printNumber("NumBlocks", File.getBlockCount());
   P.printNumber("NumDirectoryBytes", File.getNumDirectoryBytes());
   P.printNumber("Unknown1", File.getUnknown1());

llvm/tools/llvm-pdbdump/PdbYaml.cpp

@@ -126,7 +126,7 @@ void MappingTraits<msf::SuperBlock>::mapping(IO &IO, msf::SuperBlock &SB) {
   }
 
   IO.mapRequired("BlockSize", SB.BlockSize);
-  IO.mapRequired("Unknown0", SB.Unknown0);
+  IO.mapRequired("FreeBlockMap", SB.FreeBlockMapBlock);
   IO.mapRequired("NumBlocks", SB.NumBlocks);
   IO.mapRequired("NumDirectoryBytes", SB.NumDirectoryBytes);
   IO.mapRequired("Unknown1", SB.Unknown1);

llvm/tools/llvm-pdbdump/YAMLOutputStyle.cpp

@@ -60,7 +60,7 @@ Error YAMLOutputStyle::dumpFileHeaders() {
   Obj.Headers->SuperBlock.NumDirectoryBytes = File.getNumDirectoryBytes();
   Obj.Headers->NumStreams =
       opts::pdb2yaml::StreamMetadata ? File.getNumStreams() : 0;
-  Obj.Headers->SuperBlock.Unknown0 = File.getUnknown0();
+  Obj.Headers->SuperBlock.FreeBlockMapBlock = File.getFreeBlockMapBlock();
   Obj.Headers->SuperBlock.Unknown1 = File.getUnknown1();
   Obj.Headers->FileSize = File.getFileSize();
 

llvm/unittests/DebugInfo/PDB/MsfBuilderTest.cpp

@@ -81,11 +81,11 @@ TEST_F(MsfBuilderTest, TestUsedBlocksMarkedAsUsed) {
   // are correctly marked as used after adding, but no other incorrect blocks
   // are accidentally marked as used.
 
+  std::vector<uint32_t> Blocks = {4, 5, 6, 7, 8, 9, 10, 11, 12};
   // Allocate some extra blocks at the end so we can verify that they're free
   // after the initialization.
-  std::vector<uint32_t> Blocks = {2, 3, 4, 5, 6, 7, 8, 9, 10};
-  auto ExpectedMsf =
-      MsfBuilder::create(Allocator, 4096, 2 + Blocks.size() + 10);
+  uint32_t NumBlocks = msf::getMinimumBlockCount() + Blocks.size() + 10;
+  auto ExpectedMsf = MsfBuilder::create(Allocator, 4096, NumBlocks);
   EXPECT_EXPECTED(ExpectedMsf);
   auto &Msf = *ExpectedMsf;
 
@@ -94,7 +94,9 @@ TEST_F(MsfBuilderTest, TestUsedBlocksMarkedAsUsed) {
   for (auto B : Blocks) {
     EXPECT_FALSE(Msf.isBlockFree(B));
   }
-  for (int I = 11; I < 21; ++I) {
+
+  uint32_t FreeBlockStart = Blocks.back() + 1;
+  for (uint32_t I = FreeBlockStart; I < NumBlocks; ++I) {
     EXPECT_TRUE(Msf.isBlockFree(I));
   }
 }
@@ -256,7 +258,7 @@ TEST_F(MsfBuilderTest, TestBlockCountsWhenAddingStreams) {
 
   // one for the super block, one for the directory block map
   uint32_t NumUsedBlocks = Msf.getNumUsedBlocks();
-  EXPECT_EQ(2U, NumUsedBlocks);
+  EXPECT_EQ(msf::getMinimumBlockCount(), NumUsedBlocks);
   EXPECT_EQ(0U, Msf.getNumFreeBlocks());
 
   const uint32_t StreamSizes[] = {4000, 6193, 189723};
@@ -276,7 +278,7 @@ TEST_F(MsfBuilderTest, BuildMsfLayout) {
   auto &Msf = *ExpectedMsf;
 
   const uint32_t StreamSizes[] = {4000, 6193, 189723};
-  uint32_t ExpectedNumBlocks = 2;
+  uint32_t ExpectedNumBlocks = msf::getMinimumBlockCount();
   for (int I = 0; I < 3; ++I) {
     EXPECT_NO_ERROR(Msf.addStream(StreamSizes[I]));
     ExpectedNumBlocks += bytesToBlocks(StreamSizes[I], 4096);
@@ -301,31 +303,33 @@ TEST_F(MsfBuilderTest, BuildMsfLayout) {
 }
 
 TEST_F(MsfBuilderTest, UseDirectoryBlockHint) {
-  Expected<MsfBuilder> ExpectedMsf =
-      MsfBuilder::create(Allocator, 4096, 4, false);
+  Expected<MsfBuilder> ExpectedMsf = MsfBuilder::create(
+      Allocator, 4096, msf::getMinimumBlockCount() + 1, false);
   EXPECT_EXPECTED(ExpectedMsf);
   auto &Msf = *ExpectedMsf;
 
-  EXPECT_NO_ERROR(Msf.setDirectoryBlocksHint({2}));
-  EXPECT_NO_ERROR(Msf.addStream(2048, {3}));
+  uint32_t B = msf::getFirstUnreservedBlock();
+  EXPECT_NO_ERROR(Msf.setDirectoryBlocksHint({B + 1}));
+  EXPECT_NO_ERROR(Msf.addStream(2048, {B + 2}));
 
   auto ExpectedLayout = Msf.build();
   EXPECT_EXPECTED(ExpectedLayout);
   Layout &L = *ExpectedLayout;
-  EXPECT_EQ(4U, L.SB->NumBlocks);
+  EXPECT_EQ(msf::getMinimumBlockCount() + 2, L.SB->NumBlocks);
   EXPECT_EQ(1U, L.DirectoryBlocks.size());
   EXPECT_EQ(1U, L.StreamMap[0].size());
 
-  EXPECT_EQ(2U, L.DirectoryBlocks[0]);
-  EXPECT_EQ(3U, L.StreamMap[0].front());
+  EXPECT_EQ(B + 1, L.DirectoryBlocks[0]);
+  EXPECT_EQ(B + 2, L.StreamMap[0].front());
 }
 
 TEST_F(MsfBuilderTest, DirectoryBlockHintInsufficient) {
-  Expected<MsfBuilder> ExpectedMsf = MsfBuilder::create(Allocator, 4096, 4);
+  Expected<MsfBuilder> ExpectedMsf =
+      MsfBuilder::create(Allocator, 4096, msf::getMinimumBlockCount() + 2);
   EXPECT_EXPECTED(ExpectedMsf);
   auto &Msf = *ExpectedMsf;
-
-  EXPECT_NO_ERROR(Msf.setDirectoryBlocksHint({2}));
+  uint32_t B = msf::getFirstUnreservedBlock();
+  EXPECT_NO_ERROR(Msf.setDirectoryBlocksHint({B + 1}));
 
   uint32_t Size = 4096 * 4096 / 4;
   EXPECT_NO_ERROR(Msf.addStream(Size));
@@ -334,15 +338,17 @@ TEST_F(MsfBuilderTest, DirectoryBlockHintInsufficient) {
   EXPECT_EXPECTED(ExpectedLayout);
   Layout &L = *ExpectedLayout;
   EXPECT_EQ(2U, L.DirectoryBlocks.size());
-  EXPECT_EQ(2U, L.DirectoryBlocks[0]);
+  EXPECT_EQ(B + 1, L.DirectoryBlocks[0]);
 }
 
 TEST_F(MsfBuilderTest, DirectoryBlockHintOverestimated) {
-  Expected<MsfBuilder> ExpectedMsf = MsfBuilder::create(Allocator, 4096, 4);
+  Expected<MsfBuilder> ExpectedMsf =
+      MsfBuilder::create(Allocator, 4096, msf::getMinimumBlockCount() + 2);
   EXPECT_EXPECTED(ExpectedMsf);
   auto &Msf = *ExpectedMsf;
 
-  EXPECT_NO_ERROR(Msf.setDirectoryBlocksHint({2, 3}));
+  uint32_t B = msf::getFirstUnreservedBlock();
+  EXPECT_NO_ERROR(Msf.setDirectoryBlocksHint({B + 1, B + 2}));
 
   EXPECT_NO_ERROR(Msf.addStream(2048));
 
@@ -350,5 +356,5 @@ TEST_F(MsfBuilderTest, DirectoryBlockHintOverestimated) {
   EXPECT_EXPECTED(ExpectedLayout);
   Layout &L = *ExpectedLayout;
   EXPECT_EQ(1U, L.DirectoryBlocks.size());
-  EXPECT_EQ(2U, L.DirectoryBlocks[0]);
+  EXPECT_EQ(B + 1, L.DirectoryBlocks[0]);
 }