[Dsymutil][Debuginfo][NFC] #3 Refactor dsymutil to separate DWARF optimizing part.

Summary:
This is the next portion of patches for dsymutil.

Create DwarfEmitter interface to generate all debug info tables.
Put DwarfEmitter into DwarfLinker library and make tools/dsymutil/DwarfStreamer
to be child of DwarfEmitter.

It passes check-all testing. MD5 checksum for clang .dSYM bundle matches
for the dsymutil with/without that patch.

Reviewers: JDevlieghere, friss, dblaikie, aprantl

Reviewed By: JDevlieghere

Subscribers: merge_guards_bot, hiraditya, thegameg, probinson, llvm-commits

Tags: #llvm, #debug-info

Differential Revision: https://reviews.llvm.org/D72476

llvm/include/llvm/DWARFLinker/DWARFLinker.h

@@ -9,10 +9,12 @@
 #ifndef LLVM_DWARFLINKER_DWARFLINKER_H
 #define LLVM_DWARFLINKER_DWARFLINKER_H
 
+#include "llvm/CodeGen/AccelTable.h"
 #include "llvm/CodeGen/NonRelocatableStringpool.h"
 #include "llvm/DWARFLinker/DWARFLinkerDeclContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFCompileUnit.h"
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
+#include "llvm/MC/MCDwarf.h"
 #include <map>
 
 namespace llvm {
@@ -81,6 +83,116 @@ public:
   virtual void clear() = 0;
 };
 
+/// DwarfEmitter presents interface to generate all debug info tables.
+class DwarfEmitter {
+public:
+  virtual ~DwarfEmitter();
+
+  /// Emit DIE containing warnings.
+  virtual void emitPaperTrailWarningsDie(const Triple &Triple, DIE &Die) = 0;
+
+  /// Emit section named SecName with content equals to
+  /// corresponding section in Obj.
+  virtual void emitSectionContents(const object::ObjectFile &Obj,
+                                   StringRef SecName) = 0;
+
+  /// Emit the abbreviation table \p Abbrevs to the debug_abbrev section.
+  virtual void
+  emitAbbrevs(const std::vector<std::unique_ptr<DIEAbbrev>> &Abbrevs,
+              unsigned DwarfVersion) = 0;
+
+  /// Emit the string table described by \p Pool.
+  virtual void emitStrings(const NonRelocatableStringpool &Pool) = 0;
+
+  /// Emit DWARF debug names.
+  virtual void
+  emitDebugNames(AccelTable<DWARF5AccelTableStaticData> &Table) = 0;
+
+  /// Emit Apple namespaces accelerator table.
+  virtual void
+  emitAppleNamespaces(AccelTable<AppleAccelTableStaticOffsetData> &Table) = 0;
+
+  /// Emit Apple names accelerator table.
+  virtual void
+  emitAppleNames(AccelTable<AppleAccelTableStaticOffsetData> &Table) = 0;
+
+  /// Emit Apple Objective-C accelerator table.
+  virtual void
+  emitAppleObjc(AccelTable<AppleAccelTableStaticOffsetData> &Table) = 0;
+
+  /// Emit Apple type accelerator table.
+  virtual void
+  emitAppleTypes(AccelTable<AppleAccelTableStaticTypeData> &Table) = 0;
+
+  /// Emit debug_ranges for \p FuncRange by translating the
+  /// original \p Entries.
+  virtual void emitRangesEntries(
+      int64_t UnitPcOffset, uint64_t OrigLowPc,
+      const FunctionIntervals::const_iterator &FuncRange,
+      const std::vector<DWARFDebugRangeList::RangeListEntry> &Entries,
+      unsigned AddressSize) = 0;
+
+  /// Emit debug_aranges entries for \p Unit and if \p DoRangesSection is true,
+  /// also emit the debug_ranges entries for the DW_TAG_compile_unit's
+  /// DW_AT_ranges attribute.
+  virtual void emitUnitRangesEntries(CompileUnit &Unit,
+                                     bool DoRangesSection) = 0;
+
+  /// Copy the debug_line over to the updated binary while unobfuscating the
+  /// file names and directories.
+  virtual void translateLineTable(DataExtractor LineData, uint64_t Offset) = 0;
+
+  /// Emit the line table described in \p Rows into the debug_line section.
+  virtual void emitLineTableForUnit(MCDwarfLineTableParams Params,
+                                    StringRef PrologueBytes,
+                                    unsigned MinInstLength,
+                                    std::vector<DWARFDebugLine::Row> &Rows,
+                                    unsigned AdddressSize) = 0;
+
+  /// Emit the .debug_pubnames contribution for \p Unit.
+  virtual void emitPubNamesForUnit(const CompileUnit &Unit) = 0;
+
+  /// Emit the .debug_pubtypes contribution for \p Unit.
+  virtual void emitPubTypesForUnit(const CompileUnit &Unit) = 0;
+
+  /// Emit a CIE.
+  virtual void emitCIE(StringRef CIEBytes) = 0;
+
+  /// Emit an FDE with data \p Bytes.
+  virtual void emitFDE(uint32_t CIEOffset, uint32_t AddreSize, uint32_t Address,
+                       StringRef Bytes) = 0;
+
+  /// Emit the debug_loc contribution for \p Unit by copying the entries from
+  /// \p Dwarf and offsetting them. Update the location attributes to point to
+  /// the new entries.
+  virtual void emitLocationsForUnit(
+      const CompileUnit &Unit, DWARFContext &Dwarf,
+      std::function<void(StringRef, SmallVectorImpl<uint8_t> &)>
+          ProcessExpr) = 0;
+
+  /// Emit the compilation unit header for \p Unit in the
+  /// debug_info section.
+  ///
+  /// As a side effect, this also switches the current Dwarf version
+  /// of the MC layer to the one of U.getOrigUnit().
+  virtual void emitCompileUnitHeader(CompileUnit &Unit) = 0;
+
+  /// Recursively emit the DIE tree rooted at \p Die.
+  virtual void emitDIE(DIE &Die) = 0;
+
+  /// Returns size of generated .debug_line section.
+  virtual uint64_t getLineSectionSize() const = 0;
+
+  /// Returns size of generated .debug_frame section.
+  virtual uint64_t getFrameSectionSize() const = 0;
+
+  /// Returns size of generated .debug_ranges section.
+  virtual uint64_t getRangesSectionSize() const = 0;
+
+  /// Returns size of generated .debug_info section.
+  virtual uint64_t getDebugInfoSectionSize() const = 0;
+};
+
 } // end namespace llvm
 
 #endif // LLVM_DWARFLINKER_DWARFLINKER_H

llvm/lib/DWARFLinker/DWARFLinker.cpp

@@ -12,4 +12,6 @@ namespace llvm {
 
 AddressesMap::~AddressesMap() {}
 
+DwarfEmitter::~DwarfEmitter() {}
+
 } // namespace llvm

llvm/tools/dsymutil/DwarfLinkerForBinary.cpp

@@ -2464,11 +2464,12 @@ void DwarfLinkerForBinary::DIECloner::cloneAllCompileUnits(
   if (!Linker.Streamer)
     return;
 
+  uint64_t OutputDebugInfoSize = Linker.Streamer->getDebugInfoSectionSize();
   for (auto &CurrentUnit : CompileUnits) {
     auto InputDIE = CurrentUnit->getOrigUnit().getUnitDIE();
-    CurrentUnit->setStartOffset(Linker.OutputDebugInfoSize);
+    CurrentUnit->setStartOffset(OutputDebugInfoSize);
     if (!InputDIE) {
-      Linker.OutputDebugInfoSize = CurrentUnit->computeNextUnitOffset();
+      OutputDebugInfoSize = CurrentUnit->computeNextUnitOffset();
       continue;
     }
     if (CurrentUnit->getInfo(0).Keep) {
@@ -2480,7 +2481,7 @@ void DwarfLinkerForBinary::DIECloner::cloneAllCompileUnits(
                CurrentUnit->getOutputUnitDIE());
     }
 
-    Linker.OutputDebugInfoSize = CurrentUnit->computeNextUnitOffset();
+    OutputDebugInfoSize = CurrentUnit->computeNextUnitOffset();
 
     if (Linker.Options.NoOutput)
       continue;
@@ -2522,8 +2523,12 @@ void DwarfLinkerForBinary::DIECloner::cloneAllCompileUnits(
     if (!CurrentUnit->getOutputUnitDIE())
       continue;
 
+    assert(Linker.Streamer->getDebugInfoSectionSize() ==
+           CurrentUnit->getStartOffset());
     Linker.Streamer->emitCompileUnitHeader(*CurrentUnit);
     Linker.Streamer->emitDIE(*CurrentUnit->getOutputUnitDIE());
+    assert(Linker.Streamer->getDebugInfoSectionSize() ==
+           CurrentUnit->computeNextUnitOffset());
   }
 }
 
@@ -2593,13 +2598,7 @@ bool DwarfLinkerForBinary::emitPaperTrailWarnings(
     Size += getULEB128Size(Abbrev.getNumber());
   }
   CUDie->setSize(Size);
-  auto &Asm = Streamer->getAsmPrinter();
+  Streamer->emitPaperTrailWarningsDie(Map.getTriple(), *CUDie);
-  Asm.emitInt32(11 + CUDie->getSize() - 4);
-  Asm.emitInt16(2);
-  Asm.emitInt32(0);
-  Asm.emitInt8(Map.getTriple().isArch64Bit() ? 8 : 4);
-  Streamer->emitDIE(*CUDie);
-  OutputDebugInfoSize += 11 /* Header */ + Size;
 
   return true;
 }
@@ -2680,7 +2679,6 @@ bool DwarfLinkerForBinary::link(const DebugMap &Map) {
     return false;
 
   // Size of the DIEs (and headers) generated for the linked output.
-  OutputDebugInfoSize = 0;
   // A unique ID that identifies each compile unit.
   unsigned UnitID = 0;
   DebugMap ModuleMap(Map.getTriple(), Map.getBinaryPath());
@@ -2819,7 +2817,8 @@ bool DwarfLinkerForBinary::link(const DebugMap &Map) {
   // is already emitted, without being affected by canonical die offsets set
   // later. This prevents undeterminism when analyze and clone execute
   // concurrently, as clone set the canonical DIE offset and analyze reads it.
-  const uint64_t ModulesEndOffset = OutputDebugInfoSize;
+  const uint64_t ModulesEndOffset =
+      Options.NoOutput ? 0 : Streamer->getDebugInfoSectionSize();
 
   // These variables manage the list of processed object files.
   // The mutex and condition variable are to ensure that this is thread safe.

llvm/tools/dsymutil/DwarfLinkerForBinary.h

@@ -499,7 +499,6 @@ private:
   BinaryHolder &BinHolder;
   LinkOptions Options;
   std::unique_ptr<DwarfStreamer> Streamer;
-  uint64_t OutputDebugInfoSize;
 
   unsigned MaxDwarfVersion = 0;
   unsigned MinDwarfVersion = std::numeric_limits<unsigned>::max();

llvm/tools/dsymutil/DwarfStreamer.cpp

@@ -123,6 +123,7 @@ bool DwarfStreamer::init(Triple TheTriple) {
   LocSectionSize = 0;
   LineSectionSize = 0;
   FrameSectionSize = 0;
+  DebugInfoSectionSize = 0;
 
   return true;
 }
@@ -169,6 +170,7 @@ void DwarfStreamer::emitCompileUnitHeader(CompileUnit &Unit) {
   // start of the section.
   Asm->emitInt32(0);
   Asm->emitInt8(Unit.getOrigUnit().getAddressByteSize());
+  DebugInfoSectionSize += 11;
 
   // Remember this CU.
   EmittedUnits.push_back({Unit.getUniqueID(), Unit.getLabelBegin()});
@@ -188,6 +190,45 @@ void DwarfStreamer::emitAbbrevs(
 void DwarfStreamer::emitDIE(DIE &Die) {
   MS->SwitchSection(MOFI->getDwarfInfoSection());
   Asm->emitDwarfDIE(Die);
+  DebugInfoSectionSize += Die.getSize();
+}
+
+/// Emit contents of section SecName From Obj.
+void DwarfStreamer::emitSectionContents(const object::ObjectFile &Obj,
+                                        StringRef SecName) {
+  MCSection *Section =
+      StringSwitch<MCSection *>(SecName)
+          .Case("debug_line", MC->getObjectFileInfo()->getDwarfLineSection())
+          .Case("debug_loc", MC->getObjectFileInfo()->getDwarfLocSection())
+          .Case("debug_ranges",
+                MC->getObjectFileInfo()->getDwarfRangesSection())
+          .Case("debug_frame", MC->getObjectFileInfo()->getDwarfFrameSection())
+          .Case("debug_aranges",
+                MC->getObjectFileInfo()->getDwarfARangesSection())
+          .Default(nullptr);
+
+  if (Section) {
+    MS->SwitchSection(Section);
+
+    if (auto Sec = getSectionByName(Obj, SecName)) {
+      if (Expected<StringRef> E = Sec->getContents())
+        MS->EmitBytes(*E);
+      else
+        consumeError(E.takeError());
+    }
+  }
+}
+
+/// Emit DIE containing warnings.
+void DwarfStreamer::emitPaperTrailWarningsDie(const Triple &Triple, DIE &Die) {
+  switchToDebugInfoSection(/* Version */ 2);
+  auto &Asm = getAsmPrinter();
+  Asm.emitInt32(11 + Die.getSize() - 4);
+  Asm.emitInt16(2);
+  Asm.emitInt32(0);
+  Asm.emitInt8(Triple.isArch64Bit() ? 8 : 4);
+  DebugInfoSectionSize += 11;
+  emitDIE(Die);
 }
 
 /// Emit the debug_str section stored in \p Pool.
@@ -680,33 +721,23 @@ void DwarfStreamer::translateLineTable(DataExtractor Data, uint64_t Offset) {
   Offset = UnitEnd;
 }
 
-static void emitSectionContents(const object::ObjectFile &Obj,
-                                StringRef SecName, MCStreamer *MS) {
-  if (auto Sec = getSectionByName(Obj, SecName)) {
-    if (Expected<StringRef> E = Sec->getContents())
-      MS->EmitBytes(*E);
-    else
-      consumeError(E.takeError());
-  }
-}
-
 void DwarfStreamer::copyInvariantDebugSection(const object::ObjectFile &Obj) {
   if (!Options.Translator) {
     MS->SwitchSection(MC->getObjectFileInfo()->getDwarfLineSection());
-    emitSectionContents(Obj, "debug_line", MS);
+    emitSectionContents(Obj, "debug_line");
   }
 
   MS->SwitchSection(MC->getObjectFileInfo()->getDwarfLocSection());
-  emitSectionContents(Obj, "debug_loc", MS);
+  emitSectionContents(Obj, "debug_loc");
 
   MS->SwitchSection(MC->getObjectFileInfo()->getDwarfRangesSection());
-  emitSectionContents(Obj, "debug_ranges", MS);
+  emitSectionContents(Obj, "debug_ranges");
 
   MS->SwitchSection(MC->getObjectFileInfo()->getDwarfFrameSection());
-  emitSectionContents(Obj, "debug_frame", MS);
+  emitSectionContents(Obj, "debug_frame");
 
   MS->SwitchSection(MC->getObjectFileInfo()->getDwarfARangesSection());
-  emitSectionContents(Obj, "debug_aranges", MS);
+  emitSectionContents(Obj, "debug_aranges");
 }
 
 /// Emit the pubnames or pubtypes section contribution for \p

llvm/tools/dsymutil/DwarfStreamer.h

@@ -14,6 +14,7 @@
 #include "llvm/CodeGen/AccelTable.h"
 #include "llvm/CodeGen/AsmPrinter.h"
 #include "llvm/CodeGen/NonRelocatableStringpool.h"
+#include "llvm/DWARFLinker/DWARFLinker.h"
 #include "llvm/DWARFLinker/DWARFLinkerCompileUnit.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugLine.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugRangeList.h"
@@ -41,7 +42,7 @@ namespace dsymutil {
 ///
 /// All interactions with the MC layer that is used to build the debug
 /// information binary representation are handled in this class.
-class DwarfStreamer {
+class DwarfStreamer : public DwarfEmitter {
 public:
   DwarfStreamer(raw_fd_ostream &OutFile, LinkOptions Options)
       : OutFile(OutFile), Options(std::move(Options)) {}
@@ -62,17 +63,24 @@ public:
   ///
   /// As a side effect, this also switches the current Dwarf version
   /// of the MC layer to the one of U.getOrigUnit().
-  void emitCompileUnitHeader(CompileUnit &Unit);
+  void emitCompileUnitHeader(CompileUnit &Unit) override;
 
   /// Recursively emit the DIE tree rooted at \p Die.
-  void emitDIE(DIE &Die);
+  void emitDIE(DIE &Die) override;
 
   /// Emit the abbreviation table \p Abbrevs to the debug_abbrev section.
   void emitAbbrevs(const std::vector<std::unique_ptr<DIEAbbrev>> &Abbrevs,
-                   unsigned DwarfVersion);
+                   unsigned DwarfVersion) override;
+
+  /// Emit DIE containing warnings.
+  void emitPaperTrailWarningsDie(const Triple &Triple, DIE &Die) override;
+
+  /// Emit contents of section SecName From Obj.
+  void emitSectionContents(const object::ObjectFile &Obj,
+                           StringRef SecName) override;
 
   /// Emit the string table described by \p Pool.
-  void emitStrings(const NonRelocatableStringpool &Pool);
+  void emitStrings(const NonRelocatableStringpool &Pool) override;
 
   /// Emit the swift_ast section stored in \p Buffer.
   void emitSwiftAST(StringRef Buffer);
@@ -83,66 +91,75 @@ public:
       int64_t UnitPcOffset, uint64_t OrigLowPc,
       const FunctionIntervals::const_iterator &FuncRange,
       const std::vector<DWARFDebugRangeList::RangeListEntry> &Entries,
-      unsigned AddressSize);
+      unsigned AddressSize) override;
 
   /// Emit debug_aranges entries for \p Unit and if \p DoRangesSection is true,
   /// also emit the debug_ranges entries for the DW_TAG_compile_unit's
   /// DW_AT_ranges attribute.
-  void emitUnitRangesEntries(CompileUnit &Unit, bool DoRangesSection);
+  void emitUnitRangesEntries(CompileUnit &Unit, bool DoRangesSection) override;
 
-  uint32_t getRangesSectionSize() const { return RangesSectionSize; }
+  uint64_t getRangesSectionSize() const override { return RangesSectionSize; }
 
   /// Emit the debug_loc contribution for \p Unit by copying the entries from
   /// \p Dwarf and offsetting them. Update the location attributes to point to
   /// the new entries.
   void emitLocationsForUnit(
       const CompileUnit &Unit, DWARFContext &Dwarf,
-      std::function<void(StringRef, SmallVectorImpl<uint8_t> &)> ProcessExpr);
+      std::function<void(StringRef, SmallVectorImpl<uint8_t> &)> ProcessExpr)
+      override;
 
   /// Emit the line table described in \p Rows into the debug_line section.
   void emitLineTableForUnit(MCDwarfLineTableParams Params,
                             StringRef PrologueBytes, unsigned MinInstLength,
                             std::vector<DWARFDebugLine::Row> &Rows,
-                            unsigned AdddressSize);
+                            unsigned AdddressSize) override;
 
   /// Copy the debug_line over to the updated binary while unobfuscating the
   /// file names and directories.
-  void translateLineTable(DataExtractor LineData, uint64_t Offset);
+  void translateLineTable(DataExtractor LineData, uint64_t Offset) override;
 
   /// Copy over the debug sections that are not modified when updating.
   void copyInvariantDebugSection(const object::ObjectFile &Obj);
 
-  uint64_t getLineSectionSize() const { return LineSectionSize; }
+  uint64_t getLineSectionSize() const override { return LineSectionSize; }
 
   /// Emit the .debug_pubnames contribution for \p Unit.
-  void emitPubNamesForUnit(const CompileUnit &Unit);
+  void emitPubNamesForUnit(const CompileUnit &Unit) override;
 
   /// Emit the .debug_pubtypes contribution for \p Unit.
-  void emitPubTypesForUnit(const CompileUnit &Unit);
+  void emitPubTypesForUnit(const CompileUnit &Unit) override;
 
   /// Emit a CIE.
-  void emitCIE(StringRef CIEBytes);
+  void emitCIE(StringRef CIEBytes) override;
 
   /// Emit an FDE with data \p Bytes.
   void emitFDE(uint32_t CIEOffset, uint32_t AddreSize, uint32_t Address,
-               StringRef Bytes);
+               StringRef Bytes) override;
 
   /// Emit DWARF debug names.
-  void emitDebugNames(AccelTable<DWARF5AccelTableStaticData> &Table);
+  void emitDebugNames(AccelTable<DWARF5AccelTableStaticData> &Table) override;
 
   /// Emit Apple namespaces accelerator table.
-  void emitAppleNamespaces(AccelTable<AppleAccelTableStaticOffsetData> &Table);
+  void emitAppleNamespaces(
+      AccelTable<AppleAccelTableStaticOffsetData> &Table) override;
 
   /// Emit Apple names accelerator table.
-  void emitAppleNames(AccelTable<AppleAccelTableStaticOffsetData> &Table);
+  void
+  emitAppleNames(AccelTable<AppleAccelTableStaticOffsetData> &Table) override;
 
   /// Emit Apple Objective-C accelerator table.
-  void emitAppleObjc(AccelTable<AppleAccelTableStaticOffsetData> &Table);
+  void
+  emitAppleObjc(AccelTable<AppleAccelTableStaticOffsetData> &Table) override;
 
   /// Emit Apple type accelerator table.
-  void emitAppleTypes(AccelTable<AppleAccelTableStaticTypeData> &Table);
+  void
+  emitAppleTypes(AccelTable<AppleAccelTableStaticTypeData> &Table) override;
 
-  uint32_t getFrameSectionSize() const { return FrameSectionSize; }
+  uint64_t getFrameSectionSize() const override { return FrameSectionSize; }
+
+  uint64_t getDebugInfoSectionSize() const override {
+    return DebugInfoSectionSize;
+  }
 
 private:
   /// \defgroup MCObjects MC layer objects constructed by the streamer
@@ -166,10 +183,11 @@ private:
 
   LinkOptions Options;
 
-  uint32_t RangesSectionSize;
+  uint64_t RangesSectionSize = 0;
-  uint32_t LocSectionSize;
+  uint64_t LocSectionSize = 0;
-  uint64_t LineSectionSize;
+  uint64_t LineSectionSize = 0;
-  uint32_t FrameSectionSize;
+  uint64_t FrameSectionSize = 0;
+  uint64_t DebugInfoSectionSize = 0;
 
   /// Keep track of emitted CUs and their Unique ID.
   struct EmittedUnit {