[dsymutil] Implement the --threads option

This patch adds the --threads option to dsymutil to process
architectures in parallel. The feature is already present in the version
distributed with Xcode, but was not yet upstreamed.

This is NFC as far as the linking behavior is concerned. As threads are
used automatically, the current tests cover the change in
implementation.

Differential revision: https://reviews.llvm.org/D39355

llvm-svn: 316999

llvm/test/tools/dsymutil/cmdline.test

@@ -0,0 +1,21 @@
+RUN: llvm-dsymutil -help 2>&1 | FileCheck --check-prefix=HELP %s
+HELP: OVERVIEW: manipulate archived DWARF debug symbol files.
+HELP: USAGE: llvm-dsymutil [options] <input files>
+HELP-NOT: -reverse-iterate
+HELP: Specific Options:
+HELP: -arch=<string>
+HELP: -dump-debug-map
+HELP: -flat
+HELP: -no-odr
+HELP: -no-output
+HELP: -no-swiftmodule-timestamp
+HELP: -o=<filename>
+HELP: -oso-prepend-path=<path>
+HELP: -symtab
+HELP: -threads=<n>
+HELP: -verbose
+HELP: -y
+HELP-NOT: -reverse-iterate
+
+RUN: llvm-dsymutil --version 2>&1 | FileCheck --check-prefix=VERSION %s
+VERSION: {{ version }}

llvm/tools/dsymutil/DwarfLinker.cpp

@@ -35,6 +35,7 @@
 #include "llvm/Object/MachO.h"
 #include "llvm/Support/LEB128.h"
 #include "llvm/Support/TargetRegistry.h"
+#include "llvm/Support/ToolOutputFile.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
 #include <memory>
@@ -479,7 +480,7 @@ class DwarfStreamer {
   /// @}
 
   /// The file we stream the linked Dwarf to.
-  std::unique_ptr<raw_fd_ostream> OutFile;
+  std::unique_ptr<ToolOutputFile> OutFile;
 
   uint32_t RangesSectionSize;
   uint32_t LocSectionSize;
@@ -617,13 +618,13 @@ bool DwarfStreamer::init(Triple TheTriple, StringRef OutputFilename) {
   // Create the output file.
   std::error_code EC;
   OutFile =
-      llvm::make_unique<raw_fd_ostream>(OutputFilename, EC, sys::fs::F_None);
+      llvm::make_unique<ToolOutputFile>(OutputFilename, EC, sys::fs::F_None);
   if (EC)
     return error(Twine(OutputFilename) + ": " + EC.message(), Context);
 
   MCTargetOptions MCOptions = InitMCTargetOptionsFromFlags();
   MS = TheTarget->createMCObjectStreamer(
-      TheTriple, *MC, std::unique_ptr<MCAsmBackend>(MAB), *OutFile,
+      TheTriple, *MC, std::unique_ptr<MCAsmBackend>(MAB), OutFile->os(),
       std::unique_ptr<MCCodeEmitter>(MCE), *MSTI, MCOptions.MCRelaxAll,
       MCOptions.MCIncrementalLinkerCompatible,
       /*DWARFMustBeAtTheEnd*/ false);
@@ -649,11 +650,16 @@ bool DwarfStreamer::init(Triple TheTriple, StringRef OutputFilename) {
 }
 
 bool DwarfStreamer::finish(const DebugMap &DM) {
+  bool Result = true;
   if (DM.getTriple().isOSDarwin() && !DM.getBinaryPath().empty())
-    return MachOUtils::generateDsymCompanion(DM, *MS, *OutFile);
-
+    Result = MachOUtils::generateDsymCompanion(DM, *MS, OutFile->os());
+  else
     MS->Finish();
-  return true;
+
+  // Declare success.
+  OutFile->keep();
+
+  return Result;
 }
 
 /// Set the current output section to debug_info and change

llvm/tools/dsymutil/dsymutil.cpp

@@ -12,9 +12,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "dsymutil.h"
 #include "DebugMap.h"
 #include "MachOUtils.h"
-#include "dsymutil.h"
 #include "llvm/Object/MachO.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/FileUtilities.h"
@@ -22,8 +22,9 @@
 #include "llvm/Support/Options.h"
 #include "llvm/Support/PrettyStackTrace.h"
 #include "llvm/Support/Signals.h"
-#include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/TargetSelect.h"
+#include "llvm/Support/ThreadPool.h"
+#include "llvm/Support/raw_ostream.h"
 #include <cstdint>
 #include <string>
 
@@ -61,6 +62,13 @@ static opt<bool> FlatOut("flat",
                          init(false), cat(DsymCategory));
 static alias FlatOutA("f", desc("Alias for --flat"), aliasopt(FlatOut));
 
+static opt<unsigned> Threads(
+    "threads",
+    desc("Specifies the maximum number (n) of simultaneous threads to use\n"
+         "when linking multiple architectures."),
+    value_desc("n"), init(0), cat(DsymCategory));
+static alias ThreadsA("t", desc("Alias for --threads"), aliasopt(Threads));
+
 static opt<bool> Verbose("verbose", desc("Verbosity level"), init(false),
                          cat(DsymCategory));
 
@@ -316,6 +324,15 @@ int main(int argc, char **argv) {
       exitDsymutil(1);
     }
 
+    unsigned NumThreads = Threads;
+    if (!NumThreads)
+      NumThreads = llvm::thread::hardware_concurrency();
+    if (DumpDebugMap || Verbose)
+      NumThreads = 1;
+    NumThreads = std::min(NumThreads, (unsigned)DebugMapPtrsOrErr->size());
+
+    llvm::ThreadPool Threads(NumThreads);
+
     // If there is more than one link to execute, we need to generate
     // temporary files.
     bool NeedsTempFiles = !DumpDebugMap && (*DebugMapPtrsOrErr).size() != 1;
@@ -333,14 +350,27 @@ int main(int argc, char **argv) {
                      << ")\n";
 
       std::string OutputFile = getOutputFileName(InputFile, NeedsTempFiles);
+
+      auto LinkLambda = [OutputFile, Options, &Map]() {
         if (OutputFile.empty() || !linkDwarf(OutputFile, *Map, Options))
           exitDsymutil(1);
+      };
+
+      // FIXME: The DwarfLinker can have some very deep recursion that can max
+      // out the (significantly smaller) stack when using threads. We don't
+      // want this limitation when we only have a single thread.
+      if (NumThreads == 1)
+        LinkLambda();
+      else
+        Threads.async(LinkLambda);
 
       if (NeedsTempFiles)
         TempFiles.emplace_back(Map->getTriple().getArchName().str(),
                                OutputFile);
     }
 
+    Threads.wait();
+
     if (NeedsTempFiles &&
         !MachOUtils::generateUniversalBinary(
             TempFiles, getOutputFileName(InputFile), Options, SDKPath))