[mach-o] Support -filelist option in darwin driver

The darwin linker has an option, heavily used by Xcode, in which, instead
of listing all input files on the command line, the input file paths are
written to a text file and the path of that text file is passed to the linker
with the -filelist option (similar to @file).

In order to make test cases for this, I generalized the -test_libresolution
option to become -test_file_usage.

llvm-svn: 215762

lld/include/lld/ReaderWriter/MachOLinkingContext.h

@@ -83,7 +83,7 @@ public:
   void setDoNothing(bool value) { _doNothing = value; }
   bool doNothing() const { return _doNothing; }
   bool printAtoms() const { return _printAtoms; }
-  bool testingLibResolution() const { return _testingLibResolution; }
+  bool testingFileUsage() const { return _testingFileUsage; }
   const StringRefVector &searchDirs() const { return _searchDirs; }
   const StringRefVector &frameworkDirs() const { return _frameworkDirs; }
   void setSysLibRoots(const StringRefVector &paths);
@@ -91,7 +91,7 @@ public:
 
   /// \brief Checks whether a given path on the filesystem exists.
   ///
-  /// When running in -test_libresolution mode, this method consults an
+  /// When running in -test_file_usage mode, this method consults an
   /// internally maintained list of files that exist (provided by -path_exists)
   /// instead of the actual filesystem.
   bool pathExists(StringRef path) const;
@@ -171,8 +171,8 @@ public:
   }
   void setBundleLoader(StringRef loader) { _bundleLoader = loader; }
   void setPrintAtoms(bool value=true) { _printAtoms = value; }
-  void setTestingLibResolution(bool value = true) {
+  void setTestingFileUsage(bool value = true) {
-    _testingLibResolution = value;
+    _testingFileUsage = value;
   }
   void addExistingPathForDebug(StringRef path) {
     _existingPaths.insert(path);
@@ -203,6 +203,9 @@ public:
   /// search paths to allow indirect dylibs to be overridden.
   mach_o::MachODylibFile* findIndirectDylib(StringRef path) const;
 
+  /// Creates a copy (owned by this MachOLinkingContext) of a string.
+  StringRef copy(StringRef str) { return str.copy(_allocator); }
+
   static Arch archFromCpuType(uint32_t cputype, uint32_t cpusubtype);
   static Arch archFromName(StringRef archName);
   static StringRef nameFromArch(Arch arch);
@@ -254,7 +257,7 @@ private:
   StringRef _installName;
   bool _deadStrippableDylib;
   bool _printAtoms;
-  bool _testingLibResolution;
+  bool _testingFileUsage;
   StringRef _bundleLoader;
   mutable std::unique_ptr<mach_o::ArchHandler> _archHandler;
   mutable std::unique_ptr<Writer> _writer;

lld/lib/Driver/DarwinLdDriver.cpp

@@ -34,6 +34,8 @@
 
 #include <algorithm>
 
+using namespace lld;
+
 namespace {
 
 // Create enum with OPT_xxx values for each option in DarwinLdOptions.td
@@ -67,6 +69,78 @@ public:
   DarwinLdOptTable() : OptTable(infoTable, llvm::array_lengthof(infoTable)){}
 };
 
+// Test may be running on Windows. Canonicalize the path
+// separator to '/' to get consistent outputs for tests.
+std::string canonicalizePath(StringRef path) {
+  char sep = llvm::sys::path::get_separator().front();
+  if (sep != '/') {
+    std::string fixedPath = path;
+    std::replace(fixedPath.begin(), fixedPath.end(), sep, '/');
+    return fixedPath;
+  } else {
+    return path;
+  }
+}
+
+void addFile(StringRef path, std::unique_ptr<InputGraph> &inputGraph,
+             bool forceLoad) {
+   inputGraph->addInputElement(std::unique_ptr<InputElement>(
+                                          new MachOFileNode(path, forceLoad)));
+}
+
+//
+// There are two variants of the  -filelist option:
+//
+//   -filelist <path>
+// In this variant, the path is to a text file which contains one file path
+// per line.  There are no comments or trimming of whitespace.
+//
+//   -fileList <path>,<dir>
+// In this variant, the path is to a text file which contains a partial path
+// per line. The <dir> prefix is prepended to each partial path.
+//
+std::error_code parseFileList(StringRef fileListPath,
+                              std::unique_ptr<InputGraph> &inputGraph,
+                              MachOLinkingContext &ctx, bool forceLoad,
+                              raw_ostream &diagnostics) {
+  // If there is a comma, split off <dir>.
+  std::pair<StringRef, StringRef> opt = fileListPath.split(',');
+  StringRef filePath = opt.first;
+  StringRef dirName = opt.second;
+  // Map in file list file.
+  ErrorOr<std::unique_ptr<MemoryBuffer>> mb =
+                                        MemoryBuffer::getFileOrSTDIN(filePath);
+  if (std::error_code ec = mb.getError())
+    return ec;
+  StringRef buffer = mb->get()->getBuffer();
+  while (!buffer.empty()) {
+    // Split off each line in the file.
+    std::pair<StringRef, StringRef> lineAndRest = buffer.split('\n');
+    StringRef line = lineAndRest.first;
+    StringRef path;
+    if (!dirName.empty()) {
+      // If there is a <dir> then prepend dir to each line.
+      SmallString<256> fullPath;
+      fullPath.assign(dirName);
+      llvm::sys::path::append(fullPath, Twine(line));
+      path = ctx.copy(fullPath.str());
+    } else {
+      // No <dir> use whole line as input file path.
+      path = ctx.copy(line);
+    }
+    if (!ctx.pathExists(path)) {
+      return make_dynamic_error_code(Twine("File not found '")
+                                     + path
+                                     + "'");
+    }
+    if (ctx.testingFileUsage()) {
+      diagnostics << "Found filelist entry " << canonicalizePath(path) << '\n';
+    }
+    addFile(path, inputGraph, forceLoad);
+    buffer = lineAndRest.second;
+  }
+  return std::error_code();
+}
 
 } // namespace anonymous
 
@@ -293,12 +367,12 @@ bool DarwinLdDriver::parse(int argc, const char *argv[],
   if (parsedArgs->getLastArg(OPT_t))
     ctx.setLogInputFiles(true);
 
-  // In -test_libresolution mode, we'll be given an explicit list of paths that
+  // In -test_file_usage mode, we'll be given an explicit list of paths that
   // exist. We'll also be expected to print out information about how we located
   // libraries and so on that the user specified, but not to actually do any
   // linking.
-  if (parsedArgs->getLastArg(OPT_test_libresolution)) {
+  if (parsedArgs->getLastArg(OPT_test_file_usage)) {
-    ctx.setTestingLibResolution();
+    ctx.setTestingFileUsage();
 
     // With paths existing by fiat, linking is not going to end well.
     ctx.setDoNothing(true);
@@ -350,9 +424,9 @@ bool DarwinLdDriver::parse(int argc, const char *argv[],
     ctx.addFrameworkSearchDir("/System/Library/Frameworks", true);
   }
 
-  // Now that we've constructed the final set of search paths, print out what
+  // Now that we've constructed the final set of search paths, print out those
-  // we'll be using for testing purposes.
+  // search paths in verbose mode.
-  if (ctx.testingLibResolution() || parsedArgs->getLastArg(OPT_v)) {
+  if (parsedArgs->getLastArg(OPT_v)) {
     diagnostics << "Library search paths:\n";
     for (auto path : ctx.searchDirs()) {
       diagnostics << "    " << path << '\n';
@@ -365,46 +439,44 @@ bool DarwinLdDriver::parse(int argc, const char *argv[],
 
   // Handle input files
   for (auto &arg : *parsedArgs) {
-    StringRef inputPath;
+    ErrorOr<StringRef> resolvedPath = StringRef();
     switch (arg->getOption().getID()) {
     default:
       continue;
     case OPT_INPUT:
-      inputPath = arg->getValue();
+      addFile(arg->getValue(), inputGraph, globalWholeArchive);
       break;
-    case OPT_l: {
+    case OPT_l:
-      ErrorOr<StringRef> resolvedPath = ctx.searchLibrary(arg->getValue());
+      resolvedPath = ctx.searchLibrary(arg->getValue());
       if (!resolvedPath) {
         diagnostics << "Unable to find library -l" << arg->getValue() << "\n";
         return false;
-      } else if (ctx.testingLibResolution()) {
+      } else if (ctx.testingFileUsage()) {
-       // Test may be running on Windows. Canonicalize the path
+       diagnostics << "Found library " << canonicalizePath(resolvedPath.get()) << '\n';
-       // separator to '/' to get consistent outputs for tests.
-       std::string path = resolvedPath.get();
-       std::replace(path.begin(), path.end(), '\\', '/');
-       diagnostics << "Found library " << path << '\n';
       }
-      inputPath = resolvedPath.get();
+      addFile(resolvedPath.get(), inputGraph, globalWholeArchive);
       break;
-    }
+    case OPT_framework:
-    case OPT_framework: {
+      resolvedPath = ctx.findPathForFramework(arg->getValue());
-      ErrorOr<StringRef> fullPath = ctx.findPathForFramework(arg->getValue());
+      if (!resolvedPath) {
-      if (!fullPath) {
         diagnostics << "Unable to find -framework " << arg->getValue() << "\n";
         return false;
-      } else if (ctx.testingLibResolution()) {
+      } else if (ctx.testingFileUsage()) {
-       // Test may be running on Windows. Canonicalize the path
+        diagnostics << "Found framework " << canonicalizePath(resolvedPath.get()) << '\n';
-       // separator to '/' to get consistent outputs for tests.
+      }
-       std::string path = fullPath.get();
+      addFile(resolvedPath.get(), inputGraph, globalWholeArchive);
-       std::replace(path.begin(), path.end(), '\\', '/');
+      break;
-       diagnostics << "Found framework " << path << '\n';
+    case OPT_filelist:
+      if (std::error_code ec = parseFileList(arg->getValue(), inputGraph,
+                                             ctx, globalWholeArchive,
+                                             diagnostics)) {
+        diagnostics << "error: " << ec.message()
+                    << ", processing '-filelist " << arg->getValue()
+                    << "'\n";
+        return false;
       }
-      inputPath = fullPath.get();
       break;
     }
-    }
-    inputGraph->addInputElement(std::unique_ptr<InputElement>(
-        new MachOFileNode(inputPath, globalWholeArchive)));
   }
 
   if (!inputGraph->size()) {
@@ -418,4 +490,5 @@ bool DarwinLdDriver::parse(int argc, const char *argv[],
   return ctx.validate(diagnostics);
 }
 
+
 } // namespace lld

lld/lib/Driver/DarwinLdOptions.td

@@ -82,15 +82,18 @@ def l : Joined<["-"], "l">,
         HelpText<"Base name of library searched for in -L directories">;
 def framework : Separate<["-"], "framework">,
     HelpText<"Base name of framework searched for in -F directories">;
+def filelist : Separate<["-"], "filelist">,
+    HelpText<"file containing paths to input files">;
+
 
 // test case options
 def print_atoms : Flag<["-"], "print_atoms">,
         HelpText<"Emit output as yaml atoms">;
-def test_libresolution : Flag<["-"], "test_libresolution">,
+def test_file_usage : Flag<["-"], "test_file_usage">,
         HelpText<"Only files specified by -file_exists are considered to exist."
-                 " Print debugging information during resolution">;
+                 " Print which files would be used.">;
 def path_exists : Separate<["-"], "path_exists">,
-        HelpText<"When used with -test_libresolution, only these paths exist">;
+        HelpText<"When used with -test_file_usage, only these paths exist">;
 
 
 // general options

lld/lib/ReaderWriter/MachO/MachOLinkingContext.cpp

@@ -130,7 +130,7 @@ MachOLinkingContext::MachOLinkingContext()
       _doNothing(false), _arch(arch_unknown), _os(OS::macOSX), _osMinVersion(0),
       _pageZeroSize(0), _pageSize(4096), _compatibilityVersion(0),
       _currentVersion(0), _deadStrippableDylib(false), _printAtoms(false),
-      _testingLibResolution(false), _archHandler(nullptr) {}
+      _testingFileUsage(false), _archHandler(nullptr) {}
 
 MachOLinkingContext::~MachOLinkingContext() {}
 
@@ -297,7 +297,7 @@ bool MachOLinkingContext::addUnixThreadLoadCommand() const {
 }
 
 bool MachOLinkingContext::pathExists(StringRef path) const {
-  if (!testingLibResolution())
+  if (!_testingFileUsage)
     return llvm::sys::fs::exists(path.str());
 
   // Otherwise, we're in test mode: only files explicitly provided on the

lld/test/mach-o/Inputs/full.filelist

@@ -0,0 +1,3 @@
+/foo/bar/a.o
+/foo/bar/b.o
+/foo/x.a

lld/test/mach-o/Inputs/partial.filelist

@@ -0,0 +1,3 @@
+bar/a.o
+bar/b.o
+x.a

lld/test/mach-o/filelist.yaml

@@ -0,0 +1,18 @@
+# RUN: lld -flavor darwin -test_file_usage  \
+# RUN:    -filelist %p/Inputs/full.filelist \
+# RUN:        -path_exists /foo/bar/a.o \
+# RUN:        -path_exists /foo/bar/b.o \
+# RUN:        -path_exists /foo/x.a \
+# RUN: 2>&1 | FileCheck %s
+#
+# RUN: lld -flavor darwin -test_file_usage -t \
+# RUN:    -filelist %p/Inputs/partial.filelist,/foo \
+# RUN:        -path_exists /foo/bar/a.o \
+# RUN:        -path_exists /foo/bar/b.o \
+# RUN:        -path_exists /foo/x.a \
+# RUN: 2>&1 | FileCheck %s
+
+
+# CHECK: Found filelist entry /foo/bar/a.o
+# CHECK: Found filelist entry /foo/bar/b.o
+# CHECK: Found filelist entry /foo/x.a

lld/test/mach-o/framework-user-paths.yaml

@@ -5,7 +5,7 @@
 #   /opt/Frameworks should not be found in SDK
 #   /System/Library/Frameworks is implicit and should be in SDK
 #
-# RUN: lld -flavor darwin -arch x86_64 -r -test_libresolution \
+# RUN: lld -flavor darwin -arch x86_64 -r -test_file_usage -v \
 # RUN:        -path_exists myFrameworks \
 # RUN:        -path_exists myFrameworks/my.framework/my \
 # RUN:        -path_exists /opt/Frameworks \

lld/test/mach-o/libresolve-bizarre-root-override.yaml

@@ -1,4 +1,4 @@
-# RUN: not lld -flavor darwin -test_libresolution \
+# RUN: not lld -flavor darwin -test_file_usage -v \
 # RUN:        -path_exists /usr/lib \
 # RUN:        -path_exists /Applications/MySDK/usr/local/lib \
 # RUN:        -path_exists /Applications/MySDK/usr/lib \

lld/test/mach-o/libresolve-multiple-syslibroots.yaml

@@ -1,4 +1,4 @@
-# RUN: lld -flavor darwin -test_libresolution \
+# RUN: lld -flavor darwin -test_file_usage -v \
 # RUN:        -path_exists /usr/lib \
 # RUN:        -path_exists /Applications/MyFirstSDK/usr/local/lib \
 # RUN:        -path_exists /Applications/MySecondSDK/usr/local/lib \

lld/test/mach-o/libresolve-one-syslibroot.yaml

@@ -1,4 +1,4 @@
-# RUN: lld -flavor darwin -test_libresolution \
+# RUN: lld -flavor darwin -test_file_usage -v \
 # RUN:        -path_exists /usr/lib \
 # RUN:        -path_exists /Applications/MySDK/usr/local/lib \
 # RUN:        -path_exists /Applications/MySDK/usr/local/lib/libSystem.a \

lld/test/mach-o/libresolve-simple.yaml

@@ -1,4 +1,4 @@
-# RUN: lld -flavor darwin -arch x86_64 -r -test_libresolution \
+# RUN: lld -flavor darwin -arch x86_64 -r -test_file_usage -v \
 # RUN:        -path_exists /usr/lib \
 # RUN:        -path_exists /usr/local/lib \
 # RUN:        -path_exists /usr/lib/libSystem.dylib \

lld/test/mach-o/libresolve-user-paths.yaml

@@ -1,4 +1,4 @@
-# RUN: lld -flavor darwin -arch x86_64 -r -test_libresolution \
+# RUN: lld -flavor darwin -arch x86_64 -r -test_file_usage -v \
 # RUN:        -path_exists hasFoo \
 # RUN:        -path_exists hasFoo/libFoo.dylib \
 # RUN:        -path_exists /hasBar \

lld/test/mach-o/libresolve-z.yaml

@@ -1,4 +1,4 @@
-# RUN: lld -flavor darwin -arch x86_64 -r -test_libresolution \
+# RUN: lld -flavor darwin -arch x86_64 -r -test_file_usage -v \
 # RUN:        -path_exists /usr/lib \
 # RUN:        -path_exists /usr/local/lib \
 # RUN:        -path_exists /usr/lib/libSystem.dylib \