Support embedding natvis files in PDBs.

Natvis is a debug language supported by Visual Studio for
specifying custom visualizers.  The /NATVIS option is an
undocumented link.exe flag which will take a .natvis file
and "inject" it into the PDB.  This way, you can ship the
debug visualizers for a program along with the PDB, which
is very useful for postmortem debugging.

This is implemented by adding a new "named stream" to the
PDB with a special name of /src/files/<natvis file name>
and simply copying the contents of the xml into this file.

Additionally, we need to emit a single stream named
/src/headerblock which contains a hash table of embedded
files to records describing them.

This patch adds this functionality, including the /NATVIS
option to lld-link.

Differential Revision: https://reviews.llvm.org/D44328

llvm-svn: 327895

lld/COFF/Config.h

@@ -99,6 +99,7 @@ struct Configuration {
   bool DebugGHashes = false;
   bool ShowTiming = false;
   unsigned DebugTypes = static_cast<unsigned>(DebugType::None);
+  std::vector<std::string> NatvisFiles;
   llvm::SmallString<128> PDBPath;
   std::vector<llvm::StringRef> Argv;
 

lld/COFF/Driver.cpp

@@ -933,9 +933,12 @@ void LinkerDriver::link(ArrayRef<const char *> ArgsArr) {
 
   // Handle /pdb
   bool ShouldCreatePDB = Args.hasArg(OPT_debug, OPT_debug_ghash);
-  if (ShouldCreatePDB)
+  if (ShouldCreatePDB) {
     if (auto *Arg = Args.getLastArg(OPT_pdb))
       Config->PDBPath = Arg->getValue();
+    if (Args.hasArg(OPT_natvis))
+      Config->NatvisFiles = Args.getAllArgValues(OPT_natvis);
+  }
 
   // Handle /noentry
   if (Args.hasArg(OPT_noentry)) {

lld/COFF/Options.td

@@ -45,6 +45,7 @@ def nodefaultlib : P<"nodefaultlib", "Remove a default library">;
 def opt     : P<"opt", "Control optimizations">;
 def order   : P<"order", "Put functions in order">;
 def out     : P<"out", "Path to file to write output">;
+def natvis : P<"natvis", "Path to natvis file to embed in the PDB">;
 def pdb : P<"pdb", "PDB file path">;
 def section : P<"section", "Specify section attributes">;
 def stack   : P<"stack", "Size of the stack">;
@@ -162,7 +163,6 @@ def delay : QF<"delay">;
 def errorreport : QF<"errorreport">;
 def idlout : QF<"idlout">;
 def maxilksize : QF<"maxilksize">;
-def natvis : QF<"natvis">;
 def pdbaltpath : QF<"pdbaltpath">;
 def tlbid : QF<"tlbid">;
 def tlbout : QF<"tlbout">;

lld/COFF/PDB.cpp

@@ -90,6 +90,9 @@ public:
   /// Emit the basic PDB structure: initial streams, headers, etc.
   void initialize(const llvm::codeview::DebugInfo &BuildId);
 
+  /// Add natvis files specified on the command line.
+  void addNatvisFiles();
+
   /// Link CodeView from each object file in the symbol table into the PDB.
   void addObjectsToPDB();
 
@@ -961,6 +964,18 @@ void PDBLinker::addObjectsToPDB() {
   }
 }
 
+void PDBLinker::addNatvisFiles() {
+  for (StringRef File : Config->NatvisFiles) {
+    ErrorOr<std::unique_ptr<MemoryBuffer>> DataOrErr =
+        MemoryBuffer::getFile(File);
+    if (!DataOrErr) {
+      warn("Cannot open input file: " + File);
+      continue;
+    }
+    Builder.addInjectedSource(File, std::move(*DataOrErr));
+  }
+}
+
 static void addCommonLinkerModuleSymbols(StringRef Path,
                                          pdb::DbiModuleDescriptorBuilder &Mod,
                                          BumpPtrAllocator &Allocator) {
@@ -1041,6 +1056,7 @@ void coff::createPDB(SymbolTable *Symtab,
   PDB.initialize(BuildId);
   PDB.addObjectsToPDB();
   PDB.addSections(OutputSections, SectionTable);
+  PDB.addNatvisFiles();
 
   ScopedTimer T2(DiskCommitTimer);
   PDB.commit();

lld/test/COFF/Inputs/generic.yaml

@@ -0,0 +1,282 @@
+--- !COFF
+header:
+  Machine:         IMAGE_FILE_MACHINE_AMD64
+  Characteristics: [  ]
+sections:
+  - Name:            .text
+    Characteristics: [ IMAGE_SCN_CNT_CODE, IMAGE_SCN_MEM_EXECUTE, IMAGE_SCN_MEM_READ ]
+    Alignment:       16
+    SectionData:     4883EC1831C0C7442414000000004889542408894C24044883C418C3
+  - Name:            .data
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_READ, IMAGE_SCN_MEM_WRITE ]
+    Alignment:       4
+    SectionData:     ''
+  - Name:            .bss
+    Characteristics: [ IMAGE_SCN_CNT_UNINITIALIZED_DATA, IMAGE_SCN_MEM_READ, IMAGE_SCN_MEM_WRITE ]
+    Alignment:       4
+    SectionData:     ''
+  - Name:            .xdata
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_READ ]
+    Alignment:       4
+    SectionData:     '0104010004220000'
+  - Name:            .drectve
+    Characteristics: [ IMAGE_SCN_LNK_INFO, IMAGE_SCN_LNK_REMOVE ]
+    Alignment:       1
+    SectionData:     202F44454641554C544C49423A6C6962636D742E6C6962202F44454641554C544C49423A6F6C646E616D65732E6C6962
+  - Name:            '.debug$S'
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    Alignment:       4
+    SectionData:     04000000F10000002F0000002D003C1101000000D0000700000000000000581B000000000000636C616E672076657273696F6E20372E302E30200000F1000000760000002A0047110000000000000000000000001C000000000000000000000003100000000000000000006D61696E000D003E117400000001006172676300120045114F0100000400000017000000000005000D003E110010000001006172677600120045114F01000008000000170000000000050002004F110000F20000002800000000000000000000001C00000000000000020000001C00000000000000020000001700000003000000F40000001800000001000000100139E9A066A1995A99DD01F5A392F26D7C0000F30000003000000000443A5C7372635C6C6C766D6275696C645C636C5C52656C656173655C7836345C67656E657269632E63707000000000
+    Subsections:
+      - !Symbols
+        Records:
+          - Kind:            S_COMPILE3
+            Compile3Sym:
+              Flags:           [  ]
+              Machine:         X64
+              FrontendMajor:   7
+              FrontendMinor:   0
+              FrontendBuild:   0
+              FrontendQFE:     0
+              BackendMajor:    7000
+              BackendMinor:    0
+              BackendBuild:    0
+              BackendQFE:      0
+              Version:         'clang version 7.0.0 '
+      - !Symbols
+        Records:
+          - Kind:            S_GPROC32_ID
+            ProcSym:
+              CodeSize:        28
+              DbgStart:        0
+              DbgEnd:          0
+              FunctionType:    4099
+              Flags:           [  ]
+              DisplayName:     main
+          - Kind:            S_LOCAL
+            LocalSym:
+              Type:            116
+              Flags:           [ IsParameter ]
+              VarName:         argc
+          - Kind:            S_DEFRANGE_REGISTER_REL
+            DefRangeRegisterRelSym:
+              Register:        335
+              Flags:           0
+              BasePointerOffset: 4
+              Range:
+                OffsetStart:     23
+                ISectStart:      0
+                Range:           5
+              Gaps:
+          - Kind:            S_LOCAL
+            LocalSym:
+              Type:            4096
+              Flags:           [ IsParameter ]
+              VarName:         argv
+          - Kind:            S_DEFRANGE_REGISTER_REL
+            DefRangeRegisterRelSym:
+              Register:        335
+              Flags:           0
+              BasePointerOffset: 8
+              Range:
+                OffsetStart:     23
+                ISectStart:      0
+                Range:           5
+              Gaps:
+          - Kind:            S_PROC_ID_END
+            ScopeEndSym:
+      - !Lines
+        CodeSize:        28
+        Flags:           [  ]
+        RelocOffset:     0
+        RelocSegment:    0
+        Blocks:
+          - FileName:        'D:\src\llvmbuild\cl\Release\x64\generic.cpp'
+            Lines:
+              - Offset:          0
+                LineStart:       2
+                IsStatement:     false
+                EndDelta:        0
+              - Offset:          23
+                LineStart:       3
+                IsStatement:     false
+                EndDelta:        0
+            Columns:
+      - !FileChecksums
+        Checksums:
+          - FileName:        'D:\src\llvmbuild\cl\Release\x64\generic.cpp'
+            Kind:            MD5
+            Checksum:        39E9A066A1995A99DD01F5A392F26D7C
+      - !StringTable
+        Strings:
+          - 'D:\src\llvmbuild\cl\Release\x64\generic.cpp'
+          - ''
+          - ''
+          - ''
+    Relocations:
+      - VirtualAddress:  100
+        SymbolName:      main
+        Type:            IMAGE_REL_AMD64_SECREL
+      - VirtualAddress:  104
+        SymbolName:      main
+        Type:            IMAGE_REL_AMD64_SECTION
+      - VirtualAddress:  139
+        SymbolName:      .text
+        Type:            IMAGE_REL_AMD64_SECREL
+      - VirtualAddress:  143
+        SymbolName:      .text
+        Type:            IMAGE_REL_AMD64_SECTION
+      - VirtualAddress:  174
+        SymbolName:      .text
+        Type:            IMAGE_REL_AMD64_SECREL
+      - VirtualAddress:  178
+        SymbolName:      .text
+        Type:            IMAGE_REL_AMD64_SECTION
+      - VirtualAddress:  196
+        SymbolName:      main
+        Type:            IMAGE_REL_AMD64_SECREL
+      - VirtualAddress:  200
+        SymbolName:      main
+        Type:            IMAGE_REL_AMD64_SECTION
+  - Name:            '.debug$T'
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    Alignment:       4
+    SectionData:     040000000A000210700600000C0001000E0001120200000074000000001000000E0008107400000000000200011000001200011600000000021000006D61696E00F3F2F1
+    Types:
+      - Kind:            LF_POINTER
+        Pointer:
+          ReferentType:    1648
+          Attrs:           65548
+      - Kind:            LF_ARGLIST
+        ArgList:
+          ArgIndices:      [ 116, 4096 ]
+      - Kind:            LF_PROCEDURE
+        Procedure:
+          ReturnType:      116
+          CallConv:        NearC
+          Options:         [ None ]
+          ParameterCount:  2
+          ArgumentList:    4097
+      - Kind:            LF_FUNC_ID
+        FuncId:
+          ParentScope:     0
+          FunctionType:    4098
+          Name:            main
+  - Name:            .pdata
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_READ ]
+    Alignment:       4
+    SectionData:     000000001C00000000000000
+    Relocations:
+      - VirtualAddress:  0
+        SymbolName:      main
+        Type:            IMAGE_REL_AMD64_ADDR32NB
+      - VirtualAddress:  4
+        SymbolName:      main
+        Type:            IMAGE_REL_AMD64_ADDR32NB
+      - VirtualAddress:  8
+        SymbolName:      .xdata
+        Type:            IMAGE_REL_AMD64_ADDR32NB
+symbols:
+  - Name:            .text
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+    SectionDefinition:
+      Length:          28
+      NumberOfRelocations: 0
+      NumberOfLinenumbers: 0
+      CheckSum:        594448369
+      Number:          1
+  - Name:            .data
+    Value:           0
+    SectionNumber:   2
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+    SectionDefinition:
+      Length:          0
+      NumberOfRelocations: 0
+      NumberOfLinenumbers: 0
+      CheckSum:        0
+      Number:          2
+  - Name:            .bss
+    Value:           0
+    SectionNumber:   3
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+    SectionDefinition:
+      Length:          0
+      NumberOfRelocations: 0
+      NumberOfLinenumbers: 0
+      CheckSum:        0
+      Number:          3
+  - Name:            .xdata
+    Value:           0
+    SectionNumber:   4
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+    SectionDefinition:
+      Length:          8
+      NumberOfRelocations: 0
+      NumberOfLinenumbers: 0
+      CheckSum:        1192424177
+      Number:          4
+  - Name:            .drectve
+    Value:           0
+    SectionNumber:   5
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+    SectionDefinition:
+      Length:          48
+      NumberOfRelocations: 0
+      NumberOfLinenumbers: 0
+      CheckSum:        149686238
+      Number:          5
+  - Name:            '.debug$S'
+    Value:           0
+    SectionNumber:   6
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+    SectionDefinition:
+      Length:          324
+      NumberOfRelocations: 8
+      NumberOfLinenumbers: 0
+      CheckSum:        4196717433
+      Number:          6
+  - Name:            '.debug$T'
+    Value:           0
+    SectionNumber:   7
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+    SectionDefinition:
+      Length:          68
+      NumberOfRelocations: 0
+      NumberOfLinenumbers: 0
+      CheckSum:        485351071
+      Number:          7
+  - Name:            .pdata
+    Value:           0
+    SectionNumber:   8
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+    SectionDefinition:
+      Length:          12
+      NumberOfRelocations: 3
+      NumberOfLinenumbers: 0
+      CheckSum:        722740324
+      Number:          8
+  - Name:            main
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_FUNCTION
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+...

lld/test/COFF/Inputs/natvis-1.natvis

@@ -0,0 +1 @@
+1st Natvis Test

lld/test/COFF/Inputs/natvis-2.natvis

@@ -0,0 +1 @@
+Second Natvis Test

lld/test/COFF/Inputs/natvis-3.natvis

@@ -0,0 +1 @@
+Third Natvis Test

lld/test/COFF/pdb-natvis.test

@@ -0,0 +1,26 @@
+REQUIRES: diasdk
+
+RUN: yaml2obj %p/Inputs/generic.yaml > %t.obj
+RUN: lld-link /DEBUG %t.obj /nodefaultlib /entry:main /NATVIS:%p/Inputs/natvis-1.natvis \
+RUN:   /NATVIS:%p/Inputs/natvis-2.natvis /NATVIS:%p/Inputs/natvis-3.natvis /OUT:%t.exe \
+RUN:   /PDB:%t.pdb
+RUN: llvm-pdbutil pretty -injected-sources -injected-source-content %t.pdb | FileCheck \
+RUN:   --check-prefix=CHECK-FIRST %s
+RUN: llvm-pdbutil pretty -injected-sources -injected-source-content %t.pdb | FileCheck \
+RUN:   --check-prefix=CHECK-SECOND %s
+RUN: llvm-pdbutil pretty -injected-sources -injected-source-content %t.pdb | FileCheck \
+RUN:   --check-prefix=CHECK-THIRD %s
+
+RUN: lld-link /DEBUG %t.obj /nodefaultlib /entry:main /NATVIS:%p/Inputs/test2.natvis \
+RUN:   /OUT:%t.exe /PDB:%t.pdb 2>&1 | FileCheck --check-prefix=CHECK-MISSING %s
+
+CHECK-FIRST:      {{.*}}natvis-1.natvis (16 bytes): obj=<null>, vname={{.*}}natvis-1.natvis, crc=355285096, compression=None
+CHECK-FIRST-NEXT: 1st Natvis Test
+
+CHECK-SECOND:      {{.*}}natvis-2.natvis (19 bytes): obj=<null>, vname={{.*}}natvis-2.natvis, crc=4252640062, compression=None
+CHECK-SECOND-NEXT: Second Natvis Test
+
+CHECK-THIRD:      {{.*}}natvis-3.natvis (18 bytes): obj=<null>, vname={{.*}}natvis-3.natvis, crc=2069719849, compression=None
+CHECK-THIRD-NEXT: Third Natvis Test
+
+CHECK-MISSING: Cannot open input file: {{.*}}test2.natvis

lld/test/lit.cfg.py

@@ -85,6 +85,9 @@ if (lit.util.which('cvtres', config.environment['PATH'])) or \
 if (config.llvm_libxml2_enabled == '1'):
     config.available_features.add('libxml2')
 
+if config.have_dia_sdk:
+    config.available_features.add("diasdk")
+
 tar_executable = lit.util.which('tar', config.environment['PATH'])
 if tar_executable:
     tar_version = subprocess.Popen(

lld/test/lit.site.cfg.py.in

@@ -1,5 +1,6 @@
 @LIT_SITE_CFG_IN_HEADER@
 
+config.have_dia_sdk = "@LLVM_ENABLE_DIA_SDK@"
 config.llvm_src_root = "@LLVM_SOURCE_DIR@"
 config.llvm_obj_root = "@LLVM_BINARY_DIR@"
 config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"

llvm/include/llvm/DebugInfo/CodeView/DebugStringTableSubsection.h

@@ -10,6 +10,7 @@
 #ifndef LLVM_DEBUGINFO_CODEVIEW_DEBUGSTRINGTABLESUBSECTION_H
 #define LLVM_DEBUGINFO_CODEVIEW_DEBUGSTRINGTABLESUBSECTION_H
 
+#include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/DebugInfo/CodeView/CodeView.h"
@@ -66,19 +67,24 @@ public:
   uint32_t insert(StringRef S);
 
   // Return the ID for string S.  Assumes S exists in the table.
-  uint32_t getStringId(StringRef S) const;
+  uint32_t getIdForString(StringRef S) const;
+
+  StringRef getStringForId(uint32_t Id) const;
 
   uint32_t calculateSerializedSize() const override;
   Error commit(BinaryStreamWriter &Writer) const override;
 
   uint32_t size() const;
 
-  StringMap<uint32_t>::const_iterator begin() const { return Strings.begin(); }
+  StringMap<uint32_t>::const_iterator begin() const {
+    return StringToId.begin();
+  }
 
-  StringMap<uint32_t>::const_iterator end() const { return Strings.end(); }
+  StringMap<uint32_t>::const_iterator end() const { return StringToId.end(); }
 
 private:
-  StringMap<uint32_t> Strings;
+  DenseMap<uint32_t, StringRef> IdToString;
+  StringMap<uint32_t> StringToId;
   uint32_t StringSize = 1;
 };
 

llvm/include/llvm/DebugInfo/PDB/Native/HashTable.h

@@ -213,6 +213,7 @@ public:
     Deleted.clear();
   }
 
+  bool empty() const { return size() == 0; }
   uint32_t capacity() const { return Buckets.size(); }
   uint32_t size() const { return Present.count(); }
 
@@ -303,12 +304,12 @@ private:
 
   void grow() {
     uint32_t S = size();
+    uint32_t MaxLoad = maxLoad(capacity());
     if (S < maxLoad(capacity()))
       return;
     assert(capacity() != UINT32_MAX && "Can't grow Hash table!");
 
-    uint32_t NewCapacity =
+    uint32_t NewCapacity = (capacity() <= INT32_MAX) ? MaxLoad * 2 : UINT32_MAX;
-        (capacity() <= INT32_MAX) ? capacity() * 2 : UINT32_MAX;
 
     // Growing requires rebuilding the table and re-hashing every item.  Make a
     // copy with a larger capacity, insert everything into the copy, then swap

llvm/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h

@@ -17,9 +17,11 @@
 #include "llvm/DebugInfo/PDB/Native/PDBFile.h"
 #include "llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h"
 #include "llvm/DebugInfo/PDB/Native/RawConstants.h"
+#include "llvm/DebugInfo/PDB/Native/RawTypes.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBuffer.h"
 
 #include <memory>
 #include <vector>
@@ -54,12 +56,34 @@ public:
   Error commit(StringRef Filename);
 
   Expected<uint32_t> getNamedStreamIndex(StringRef Name) const;
-  Error addNamedStream(StringRef Name, uint32_t Size);
+  Error addNamedStream(StringRef Name, StringRef Data);
+  void addInjectedSource(StringRef Name, std::unique_ptr<MemoryBuffer> Buffer);
 
 private:
+  struct InjectedSourceDescriptor {
+    // The full name of the stream that contains the contents of this injected
+    // source.  This is built as a concatenation of the literal "/src/files"
+    // plus the "vname".
+    std::string StreamName;
+
+    // The exact name of the file name as specified by the user.
+    uint32_t NameIndex;
+
+    // The string table index of the "vname" of the file.  As far as we
+    // understand, this is the same as the name, except it is lowercased and
+    // forward slashes are converted to backslashes.
+    uint32_t VNameIndex;
+    std::unique_ptr<MemoryBuffer> Content;
+  };
+
   Expected<msf::MSFLayout> finalizeMsfLayout();
+  Expected<uint32_t> allocateNamedStream(StringRef Name, uint32_t Size);
 
   void commitFpm(WritableBinaryStream &MsfBuffer, const msf::MSFLayout &Layout);
+  void commitInjectedSources(WritableBinaryStream &MsfBuffer,
+                             const msf::MSFLayout &Layout);
+  void commitSrcHeaderBlock(WritableBinaryStream &MsfBuffer,
+                            const msf::MSFLayout &Layout);
 
   BumpPtrAllocator &Allocator;
 
@@ -71,7 +95,13 @@ private:
   std::unique_ptr<TpiStreamBuilder> Ipi;
 
   PDBStringTableBuilder Strings;
+  StringTableHashTraits InjectedSourceHashTraits;
+  HashTable<SrcHeaderBlockEntry, StringTableHashTraits> InjectedSourceTable;
+
+  SmallVector<InjectedSourceDescriptor, 2> InjectedSources;
+
   NamedStreamMap NamedStreams;
+  DenseMap<uint32_t, std::string> NamedStreamData;
 };
 }
 }

llvm/include/llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h

@@ -31,6 +31,16 @@ struct MSFLayout;
 namespace pdb {
 
 class PDBFileBuilder;
+class PDBStringTableBuilder;
+
+struct StringTableHashTraits {
+  PDBStringTableBuilder *Table;
+
+  explicit StringTableHashTraits(PDBStringTableBuilder &Table);
+  uint32_t hashLookupKey(StringRef S) const;
+  StringRef storageKeyToLookupKey(uint32_t Offset) const;
+  uint32_t lookupKeyToStorageKey(StringRef S);
+};
 
 class PDBStringTableBuilder {
 public:
@@ -38,6 +48,9 @@ public:
   // Returns the ID for S.
   uint32_t insert(StringRef S);
 
+  uint32_t getIdForString(StringRef S) const;
+  StringRef getStringForId(uint32_t Id) const;
+
   uint32_t calculateSerializedSize() const;
   Error commit(BinaryStreamWriter &Writer) const;
 

llvm/include/llvm/DebugInfo/PDB/Native/RawConstants.h

@@ -32,6 +32,8 @@ enum PdbRaw_ImplVer : uint32_t {
   PdbImplVC140 = 20140508,
 };
 
+enum class PdbRaw_SrcHeaderBlockVer : uint32_t { SrcVerOne = 19980827 };
+
 enum class PdbRaw_FeatureSig : uint32_t {
   VC110 = PdbImplVC110,
   VC140 = PdbImplVC140,

llvm/include/llvm/DebugInfo/PDB/Native/RawTypes.h

@@ -328,6 +328,34 @@ struct PDBStringTableHeader {
 
 const uint32_t PDBStringTableSignature = 0xEFFEEFFE;
 
+/// The header preceding the /src/headerblock stream.
+struct SrcHeaderBlockHeader {
+  support::ulittle32_t Version; // PdbRaw_SrcHeaderBlockVer enumeration.
+  support::ulittle32_t Size;    // Size of entire stream.
+  uint64_t FileTime;            // Time stamp (Windows FILETIME format).
+  support::ulittle32_t Age;     // Age
+  uint8_t Padding[44];          // Pad to 64 bytes.
+};
+static_assert(sizeof(SrcHeaderBlockHeader) == 64, "Incorrect struct size!");
+
+/// A single file record entry within the /src/headerblock stream.
+struct SrcHeaderBlockEntry {
+  support::ulittle32_t Size;     // Record Length.
+  support::ulittle32_t Version;  // PdbRaw_SrcHeaderBlockVer enumeration.
+  support::ulittle32_t CRC;      // CRC of the original file contents.
+  support::ulittle32_t FileSize; // Size of original source file.
+  support::ulittle32_t FileNI;   // String table index of file name.
+  support::ulittle32_t ObjNI;    // String table index of object name.
+  support::ulittle32_t VFileNI;  // String table index of virtual file name.
+  uint8_t Compression;           // PDB_SourceCompression enumeration.
+  uint8_t IsVirtual;             // Is this a virtual file (injected)?
+  short Padding;                 // Pad to 4 bytes.
+  char Reserved[8];
+};
+
+constexpr int I = sizeof(SrcHeaderBlockEntry);
+static_assert(sizeof(SrcHeaderBlockEntry) == 40, "Incorrect struct size!");
+
 } // namespace pdb
 } // namespace llvm
 

llvm/lib/DebugInfo/CodeView/DebugChecksumsSubsection.cpp

@@ -109,7 +109,7 @@ Error DebugChecksumsSubsection::commit(BinaryStreamWriter &Writer) const {
 }
 
 uint32_t DebugChecksumsSubsection::mapChecksumOffset(StringRef FileName) const {
-  uint32_t Offset = Strings.getStringId(FileName);
+  uint32_t Offset = Strings.getIdForString(FileName);
   auto Iter = OffsetMap.find(Offset);
   assert(Iter != OffsetMap.end());
   return Iter->second;

llvm/lib/DebugInfo/CodeView/DebugCrossImpSubsection.cpp

@@ -80,13 +80,13 @@ Error DebugCrossModuleImportsSubsection::commit(
     Ids.push_back(&M);
 
   std::sort(Ids.begin(), Ids.end(), [this](const T &L1, const T &L2) {
-    return Strings.getStringId(L1->getKey()) <
+    return Strings.getIdForString(L1->getKey()) <
-           Strings.getStringId(L2->getKey());
+           Strings.getIdForString(L2->getKey());
   });
 
   for (const auto &Item : Ids) {
     CrossModuleImport Imp;
-    Imp.ModuleNameOffset = Strings.getStringId(Item->getKey());
+    Imp.ModuleNameOffset = Strings.getIdForString(Item->getKey());
     Imp.Count = Item->getValue().size();
     if (auto EC = Writer.writeObject(Imp))
       return EC;

llvm/lib/DebugInfo/CodeView/DebugStringTableSubsection.cpp

@@ -46,12 +46,15 @@ DebugStringTableSubsection::DebugStringTableSubsection()
     : DebugSubsection(DebugSubsectionKind::StringTable) {}
 
 uint32_t DebugStringTableSubsection::insert(StringRef S) {
-  auto P = Strings.insert({S, StringSize});
+  auto P = StringToId.insert({S, StringSize});
 
   // If a given string didn't exist in the string table, we want to increment
-  // the string table size.
+  // the string table size and insert it into the reverse lookup.
-  if (P.second)
+  if (P.second) {
+    IdToString.insert({P.first->getValue(), P.first->getKey()});
     StringSize += S.size() + 1; // +1 for '\0'
+  }
+
   return P.first->second;
 }
 
@@ -67,7 +70,7 @@ Error DebugStringTableSubsection::commit(BinaryStreamWriter &Writer) const {
   if (auto EC = Writer.writeCString(StringRef()))
     return EC;
 
-  for (auto &Pair : Strings) {
+  for (auto &Pair : StringToId) {
     StringRef S = Pair.getKey();
     uint32_t Offset = Begin + Pair.getValue();
     Writer.setOffset(Offset);
@@ -81,10 +84,16 @@ Error DebugStringTableSubsection::commit(BinaryStreamWriter &Writer) const {
   return Error::success();
 }
 
-uint32_t DebugStringTableSubsection::size() const { return Strings.size(); }
+uint32_t DebugStringTableSubsection::size() const { return StringToId.size(); }
 
-uint32_t DebugStringTableSubsection::getStringId(StringRef S) const {
+uint32_t DebugStringTableSubsection::getIdForString(StringRef S) const {
-  auto Iter = Strings.find(S);
+  auto Iter = StringToId.find(S);
-  assert(Iter != Strings.end());
+  assert(Iter != StringToId.end());
+  return Iter->second;
+}
+
+StringRef DebugStringTableSubsection::getStringForId(uint32_t Id) const {
+  auto Iter = IdToString.find(Id);
+  assert(Iter != IdToString.end());
   return Iter->second;
 }

llvm/lib/DebugInfo/PDB/Native/InfoStreamBuilder.cpp

@@ -73,5 +73,6 @@ Error InfoStreamBuilder::commit(const msf::MSFLayout &Layout,
     if (auto EC = Writer.writeEnum(E))
       return EC;
   }
+  assert(Writer.bytesRemaining() == 0);
   return Error::success();
 }

llvm/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp

@@ -24,6 +24,8 @@
 #include "llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h"
 #include "llvm/Support/BinaryStream.h"
 #include "llvm/Support/BinaryStreamWriter.h"
+#include "llvm/Support/JamCRC.h"
+#include "llvm/Support/Path.h"
 
 using namespace llvm;
 using namespace llvm::codeview;
@@ -32,7 +34,8 @@ using namespace llvm::pdb;
 using namespace llvm::support;
 
 PDBFileBuilder::PDBFileBuilder(BumpPtrAllocator &Allocator)
-    : Allocator(Allocator) {}
+    : Allocator(Allocator), InjectedSourceHashTraits(Strings),
+      InjectedSourceTable(2, InjectedSourceHashTraits) {}
 
 PDBFileBuilder::~PDBFileBuilder() {}
 
@@ -80,14 +83,45 @@ GSIStreamBuilder &PDBFileBuilder::getGsiBuilder() {
   return *Gsi;
 }
 
-Error PDBFileBuilder::addNamedStream(StringRef Name, uint32_t Size) {
+Expected<uint32_t> PDBFileBuilder::allocateNamedStream(StringRef Name,
+                                                       uint32_t Size) {
   auto ExpectedStream = Msf->addStream(Size);
-  if (!ExpectedStream)
+  if (ExpectedStream)
-    return ExpectedStream.takeError();
+    NamedStreams.set(Name, *ExpectedStream);
-  NamedStreams.set(Name, *ExpectedStream);
+  return ExpectedStream;
+}
+
+Error PDBFileBuilder::addNamedStream(StringRef Name, StringRef Data) {
+  Expected<uint32_t> ExpectedIndex = allocateNamedStream(Name, Data.size());
+  if (!ExpectedIndex)
+    return ExpectedIndex.takeError();
+  assert(NamedStreamData.count(*ExpectedIndex) == 0);
+  NamedStreamData[*ExpectedIndex] = Data;
   return Error::success();
 }
 
+void PDBFileBuilder::addInjectedSource(StringRef Name,
+                                       std::unique_ptr<MemoryBuffer> Buffer) {
+  // Stream names must be exact matches, since they get looked up in a hash
+  // table and the hash value is dependent on the exact contents of the string.
+  // link.exe lowercases a path and converts / to \, so we must do the same.
+  SmallString<64> VName;
+  sys::path::native(Name.lower(), VName);
+
+  uint32_t NI = getStringTableBuilder().insert(Name);
+  uint32_t VNI = getStringTableBuilder().insert(VName);
+
+  InjectedSourceDescriptor Desc;
+  Desc.Content = std::move(Buffer);
+  Desc.NameIndex = NI;
+  Desc.VNameIndex = VNI;
+  Desc.StreamName = "/src/files/";
+
+  Desc.StreamName += VName;
+
+  InjectedSources.push_back(std::move(Desc));
+}
+
 Expected<msf::MSFLayout> PDBFileBuilder::finalizeMsfLayout() {
 
   if (Ipi && Ipi->getRecordCount() > 0) {
@@ -101,15 +135,13 @@ Expected<msf::MSFLayout> PDBFileBuilder::finalizeMsfLayout() {
 
   uint32_t StringsLen = Strings.calculateSerializedSize();
 
-  if (auto EC = addNamedStream("/names", StringsLen))
+  Expected<uint32_t> SN = allocateNamedStream("/names", StringsLen);
-    return std::move(EC);
+  if (!SN)
-  if (auto EC = addNamedStream("/LinkInfo", 0))
+    return SN.takeError();
-    return std::move(EC);
+  SN = allocateNamedStream("/LinkInfo", 0);
+  if (!SN)
+    return SN.takeError();
 
-  if (Info) {
-    if (auto EC = Info->finalizeMsfLayout())
-      return std::move(EC);
-  }
   if (Dbi) {
     if (auto EC = Dbi->finalizeMsfLayout())
       return std::move(EC);
@@ -132,6 +164,46 @@ Expected<msf::MSFLayout> PDBFileBuilder::finalizeMsfLayout() {
     }
   }
 
+  if (!InjectedSources.empty()) {
+    for (const auto &IS : InjectedSources) {
+      JamCRC CRC(0);
+      CRC.update(makeArrayRef(IS.Content->getBufferStart(),
+                              IS.Content->getBufferSize()));
+
+      SrcHeaderBlockEntry Entry;
+      ::memset(&Entry, 0, sizeof(SrcHeaderBlockEntry));
+      Entry.Size = sizeof(SrcHeaderBlockEntry);
+      Entry.FileSize = IS.Content->getBufferSize();
+      Entry.FileNI = IS.NameIndex;
+      Entry.VFileNI = IS.VNameIndex;
+      Entry.IsVirtual = 0;
+      Entry.Version =
+          static_cast<uint32_t>(PdbRaw_SrcHeaderBlockVer::SrcVerOne);
+      Entry.CRC = CRC.getCRC();
+      StringRef VName = getStringTableBuilder().getStringForId(IS.VNameIndex);
+      InjectedSourceTable.set_as(VName, std::move(Entry));
+    }
+
+    uint32_t SrcHeaderBlockSize =
+        sizeof(SrcHeaderBlockHeader) +
+        InjectedSourceTable.calculateSerializedLength();
+    SN = allocateNamedStream("/src/headerblock", SrcHeaderBlockSize);
+    if (!SN)
+      return SN.takeError();
+    for (const auto &IS : InjectedSources) {
+      SN = allocateNamedStream(IS.StreamName, IS.Content->getBufferSize());
+      if (!SN)
+        return SN.takeError();
+    }
+  }
+
+  // Do this last, since it relies on the named stream map being complete, and
+  // that can be updated by previous steps in the finalization.
+  if (Info) {
+    if (auto EC = Info->finalizeMsfLayout())
+      return std::move(EC);
+  }
+
   return Msf->build();
 }
 
@@ -167,6 +239,45 @@ void PDBFileBuilder::commitFpm(WritableBinaryStream &MsfBuffer,
   assert(FpmWriter.bytesRemaining() == 0);
 }
 
+void PDBFileBuilder::commitSrcHeaderBlock(WritableBinaryStream &MsfBuffer,
+                                          const msf::MSFLayout &Layout) {
+  assert(!InjectedSourceTable.empty());
+
+  uint32_t SN = cantFail(getNamedStreamIndex("/src/headerblock"));
+  auto Stream = WritableMappedBlockStream::createIndexedStream(
+      Layout, MsfBuffer, SN, Allocator);
+  BinaryStreamWriter Writer(*Stream);
+
+  SrcHeaderBlockHeader Header;
+  ::memset(&Header, 0, sizeof(Header));
+  Header.Version = static_cast<uint32_t>(PdbRaw_SrcHeaderBlockVer::SrcVerOne);
+  Header.Size = Writer.bytesRemaining();
+
+  cantFail(Writer.writeObject(Header));
+  cantFail(InjectedSourceTable.commit(Writer));
+
+  assert(Writer.bytesRemaining() == 0);
+}
+
+void PDBFileBuilder::commitInjectedSources(WritableBinaryStream &MsfBuffer,
+                                           const msf::MSFLayout &Layout) {
+  if (InjectedSourceTable.empty())
+    return;
+
+  commitSrcHeaderBlock(MsfBuffer, Layout);
+
+  for (const auto &IS : InjectedSources) {
+    uint32_t SN = cantFail(getNamedStreamIndex(IS.StreamName));
+
+    auto SourceStream = WritableMappedBlockStream::createIndexedStream(
+        Layout, MsfBuffer, SN, Allocator);
+    BinaryStreamWriter SourceWriter(*SourceStream);
+    assert(SourceWriter.bytesRemaining() == IS.Content->getBufferSize());
+    cantFail(SourceWriter.writeBytes(
+        arrayRefFromStringRef(IS.Content->getBuffer())));
+  }
+}
+
 Error PDBFileBuilder::commit(StringRef Filename) {
   assert(!Filename.empty());
   auto ExpectedLayout = finalizeMsfLayout();
@@ -219,6 +330,17 @@ Error PDBFileBuilder::commit(StringRef Filename) {
   if (auto EC = Strings.commit(NSWriter))
     return EC;
 
+  for (const auto &NSE : NamedStreamData) {
+    if (NSE.second.empty())
+      continue;
+
+    auto NS = WritableMappedBlockStream::createIndexedStream(
+        Layout, Buffer, NSE.first, Allocator);
+    BinaryStreamWriter NSW(*NS);
+    if (auto EC = NSW.writeBytes(arrayRefFromStringRef(NSE.second)))
+      return EC;
+  }
+
   if (Info) {
     if (auto EC = Info->commit(Layout, Buffer))
       return EC;
@@ -251,6 +373,8 @@ Error PDBFileBuilder::commit(StringRef Filename) {
   InfoStreamHeader *H = reinterpret_cast<InfoStreamHeader *>(
       FOB->getBufferStart() + InfoStreamFileOffset);
 
+  commitInjectedSources(Buffer, Layout);
+
   // Set the build id at the very end, after every other byte of the PDB
   // has been written.
   // FIXME: Use a hash of the PDB rather than time(nullptr) for the signature.

llvm/lib/DebugInfo/PDB/Native/PDBStringTableBuilder.cpp

@@ -21,10 +21,33 @@ using namespace llvm::support;
 using namespace llvm::support::endian;
 using namespace llvm::pdb;
 
+StringTableHashTraits::StringTableHashTraits(PDBStringTableBuilder &Table)
+    : Table(&Table) {}
+
+uint32_t StringTableHashTraits::hashLookupKey(StringRef S) const {
+  return Table->getIdForString(S);
+}
+
+StringRef StringTableHashTraits::storageKeyToLookupKey(uint32_t Offset) const {
+  return Table->getStringForId(Offset);
+}
+
+uint32_t StringTableHashTraits::lookupKeyToStorageKey(StringRef S) {
+  return Table->insert(S);
+}
+
 uint32_t PDBStringTableBuilder::insert(StringRef S) {
   return Strings.insert(S);
 }
 
+uint32_t PDBStringTableBuilder::getIdForString(StringRef S) const {
+  return Strings.getIdForString(S);
+}
+
+StringRef PDBStringTableBuilder::getStringForId(uint32_t Id) const {
+  return Strings.getStringForId(Id);
+}
+
 static uint32_t computeBucketCount(uint32_t NumStrings) {
   // The /names stream is basically an on-disk open-addressing hash table.
   // Hash collisions are resolved by linear probing. We cannot make

llvm/tools/llvm-pdbutil/DumpOutputStyle.cpp

@@ -90,6 +90,12 @@ Error DumpOutputStyle::dump() {
     P.NewLine();
   }
 
+  if (opts::dump::DumpNamedStreams) {
+    if (auto EC = dumpNamedStreams())
+      return EC;
+    P.NewLine();
+  }
+
   if (opts::dump::DumpStringTable) {
     if (auto EC = dumpStringTable())
       return EC;
@@ -909,6 +915,29 @@ Error DumpOutputStyle::dumpStringTableFromObj() {
   return Error::success();
 }
 
+Error DumpOutputStyle::dumpNamedStreams() {
+  printHeader(P, "Named Streams");
+  AutoIndent Indent(P, 2);
+
+  if (File.isObj()) {
+    P.formatLine("Dumping Named Streams is only supported for PDB files.");
+    return Error::success();
+  }
+  ExitOnError Err("Invalid PDB File: ");
+
+  auto &IS = Err(File.pdb().getPDBInfoStream());
+  const NamedStreamMap &NS = IS.getNamedStreams();
+  for (const auto &Entry : NS.entries()) {
+    P.printLine(Entry.getKey());
+    AutoIndent Indent2(P, 2);
+    P.formatLine("Index: {0}", Entry.getValue());
+    P.formatLine("Size in bytes: {0}",
+                 File.pdb().getStreamByteSize(Entry.getValue()));
+  }
+
+  return Error::success();
+}
+
 Error DumpOutputStyle::dumpStringTable() {
   printHeader(P, "String Table");
 

llvm/tools/llvm-pdbutil/DumpOutputStyle.h

@@ -74,6 +74,7 @@ private:
   Error dumpStreamSummary();
   Error dumpSymbolStats();
   Error dumpUdtStats();
+  Error dumpNamedStreams();
   Error dumpStringTable();
   Error dumpStringTableFromPdb();
   Error dumpStringTableFromObj();

llvm/tools/llvm-pdbutil/llvm-pdbutil.cpp

@@ -534,6 +534,10 @@ cl::opt<bool> JustMyCode("jmc", cl::Optional,
                          cl::cat(FileOptions), cl::sub(DumpSubcommand));
 
 // MISCELLANEOUS OPTIONS
+cl::opt<bool> DumpNamedStreams("named-streams",
+                               cl::desc("dump PDB named stream table"),
+                               cl::cat(MiscOptions), cl::sub(DumpSubcommand));
+
 cl::opt<bool> DumpStringTable("string-table", cl::desc("dump PDB String Table"),
                               cl::cat(MiscOptions), cl::sub(DumpSubcommand));
 

llvm/tools/llvm-pdbutil/llvm-pdbutil.h

@@ -142,6 +142,7 @@ extern llvm::cl::opt<bool> DumpLines;
 extern llvm::cl::opt<bool> DumpInlineeLines;
 extern llvm::cl::opt<bool> DumpXmi;
 extern llvm::cl::opt<bool> DumpXme;
+extern llvm::cl::opt<bool> DumpNamedStreams;
 extern llvm::cl::opt<bool> DumpStringTable;
 extern llvm::cl::opt<bool> DumpTypes;
 extern llvm::cl::opt<bool> DumpTypeData;