[lld-macho] Handle TAPI and regular re-exports uniformly

The re-exports list in a TAPI document can either refer to other inlined
TAPI documents, or to on-disk files (which may themselves be TBD or
regular files.) Similarly, the re-exports of a regular dylib can refer
to a TBD file.

Differential Revision: https://reviews.llvm.org/D85404

lld/MachO/CMakeLists.txt

@@ -5,6 +5,7 @@ add_public_tablegen_target(MachOOptionsTableGen)
 add_lld_library(lldMachO2
   Arch/X86_64.cpp
   Driver.cpp
+  DriverUtils.cpp
   ExportTrie.cpp
   InputFiles.cpp
   InputSection.cpp

lld/MachO/Config.h

@@ -39,6 +39,7 @@ struct Configuration {
   llvm::MachO::Architecture arch;
   PlatformInfo platform;
   llvm::MachO::HeaderFileType outputType;
+  std::vector<llvm::StringRef> systemLibraryRoots;
   std::vector<llvm::StringRef> librarySearchPaths;
   std::vector<llvm::StringRef> frameworkSearchPaths;
   std::vector<llvm::StringRef> runtimePaths;

lld/MachO/Driver.cpp

@@ -8,6 +8,7 @@
 
 #include "Driver.h"
 #include "Config.h"
+#include "DriverUtils.h"
 #include "InputFiles.h"
 #include "OutputSection.h"
 #include "OutputSegment.h"
@@ -129,7 +130,7 @@ static Optional<std::string> findFramework(StringRef name) {
       // Suffix lookup failed, fall through to the no-suffix case.
     }
 
-    if (Optional<std::string> path = findWithExtension(symlink, {".tbd", ""}))
+    if (Optional<std::string> path = resolveDylibPath(symlink))
       return path;
   }
   return {};
@@ -233,13 +234,10 @@ static void addFile(StringRef path) {
     inputFiles.push_back(make<DylibFile>(mbref));
     break;
   case file_magic::tapi_file: {
-    Expected<std::unique_ptr<InterfaceFile>> result = TextAPIReader::get(mbref);
-    if (!result) {
-      error("could not load TAPI file at " + mbref.getBufferIdentifier() +
-            ": " + toString(result.takeError()));
+    Optional<DylibFile *> dylibFile = makeDylibFromTAPI(mbref);
+    if (!dylibFile)
       return;
-    }
-    inputFiles.push_back(make<DylibFile>(**result));
+    inputFiles.push_back(*dylibFile);
     break;
   }
   default:
@@ -506,7 +504,7 @@ bool macho::link(llvm::ArrayRef<const char *> argsArr, bool canExitEarly,
   config->outputType = args.hasArg(OPT_dylib) ? MH_DYLIB : MH_EXECUTE;
   config->runtimePaths = args::getStrings(args, OPT_rpath);
 
-  std::vector<StringRef> roots;
+  std::vector<StringRef> &roots = config->systemLibraryRoots;
   for (const Arg *arg : args.filtered(OPT_syslibroot))
     roots.push_back(arg->getValue());
   // NOTE: the final `-syslibroot` being `/` will ignore all roots

lld/MachO/DriverUtils.cpp

@@ -0,0 +1,46 @@
+//===- DriverUtils.cpp ----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "DriverUtils.h"
+#include "InputFiles.h"
+
+#include "lld/Common/ErrorHandler.h"
+#include "lld/Common/Memory.h"
+#include "llvm/Support/Path.h"
+#include "llvm/TextAPI/MachO/TextAPIReader.h"
+
+using namespace llvm;
+using namespace llvm::MachO;
+using namespace llvm::sys;
+using namespace lld;
+using namespace lld::macho;
+
+Optional<std::string> macho::resolveDylibPath(StringRef path) {
+  // TODO: if a tbd and dylib are both present, we should check to make sure
+  // they are consistent.
+  if (fs::exists(path))
+    return std::string(path);
+
+  SmallString<261> location = path;
+  path::replace_extension(location, ".tbd");
+  if (fs::exists(location))
+    return std::string(location);
+
+  return {};
+}
+
+Optional<DylibFile *> macho::makeDylibFromTAPI(MemoryBufferRef mbref,
+                                               DylibFile *umbrella) {
+  Expected<std::unique_ptr<InterfaceFile>> result = TextAPIReader::get(mbref);
+  if (!result) {
+    error("could not load TAPI file at " + mbref.getBufferIdentifier() + ": " +
+          toString(result.takeError()));
+    return {};
+  }
+  return make<DylibFile>(**result, umbrella);
+}

lld/MachO/DriverUtils.h

@@ -0,0 +1,31 @@
+//===- DriverUtils.h --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLD_MACHO_DRIVER_UTILS_H
+#define LLD_MACHO_DRIVER_UTILS_H
+
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/MemoryBuffer.h"
+
+namespace lld {
+namespace macho {
+
+class DylibFile;
+
+// Check for both libfoo.dylib and libfoo.tbd (in that order).
+llvm::Optional<std::string> resolveDylibPath(llvm::StringRef path);
+
+llvm::Optional<DylibFile *> makeDylibFromTAPI(llvm::MemoryBufferRef mbref,
+                                              DylibFile *umbrella = nullptr);
+
+} // namespace macho
+} // namespace lld
+
+#endif

lld/MachO/InputFiles.cpp

@@ -43,6 +43,7 @@
 
 #include "InputFiles.h"
 #include "Config.h"
+#include "DriverUtils.h"
 #include "ExportTrie.h"
 #include "InputSection.h"
 #include "MachOStructs.h"
@@ -53,6 +54,7 @@
 
 #include "lld/Common/ErrorHandler.h"
 #include "lld/Common/Memory.h"
+#include "llvm/ADT/iterator.h"
 #include "llvm/BinaryFormat/MachO.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -340,6 +342,60 @@ ObjFile::ObjFile(MemoryBufferRef mb) : InputFile(ObjKind, mb) {
     parseRelocations(sectionHeaders[i], subsections[i]);
 }
 
+// The path can point to either a dylib or a .tbd file.
+static Optional<DylibFile *> loadDylib(StringRef path, DylibFile *umbrella) {
+  Optional<MemoryBufferRef> mbref = readFile(path);
+  if (!mbref) {
+    error("could not read dylib file at " + path);
+    return {};
+  }
+
+  file_magic magic = identify_magic(mbref->getBuffer());
+  if (magic == file_magic::tapi_file)
+    return makeDylibFromTAPI(*mbref, umbrella);
+  assert(magic == file_magic::macho_dynamically_linked_shared_lib);
+  return make<DylibFile>(*mbref, umbrella);
+}
+
+// TBD files are parsed into a series of TAPI documents (InterfaceFiles), with
+// the first document storing child pointers to the rest of them. When we are
+// processing a given TBD file, we store that top-level document here. When
+// processing re-exports, we search its children for potentially matching
+// documents in the same TBD file. Note that the children themselves don't
+// point to further documents, i.e. this is a two-level tree.
+//
+// ld64 allows a TAPI re-export to reference documents nested within other TBD
+// files, but that seems like a strange design, so this is an intentional
+// deviation.
+const InterfaceFile *currentTopLevelTapi = nullptr;
+
+// Re-exports can either refer to on-disk files, or to documents within .tbd
+// files.
+static Optional<DylibFile *> loadReexport(StringRef path, DylibFile *umbrella) {
+  if (path::is_absolute(path, path::Style::posix))
+    for (StringRef root : config->systemLibraryRoots)
+      if (Optional<std::string> dylibPath =
+              resolveDylibPath((root + path).str()))
+        return loadDylib(*dylibPath, umbrella);
+
+  // TODO: Expand @loader_path, @executable_path etc
+
+  if (currentTopLevelTapi != nullptr) {
+    for (InterfaceFile &child :
+         make_pointee_range(currentTopLevelTapi->documents())) {
+      if (path == child.getInstallName())
+        return make<DylibFile>(child, umbrella);
+      assert(child.documents().empty());
+    }
+  }
+
+  if (Optional<std::string> dylibPath = resolveDylibPath(path))
+    return loadDylib(*dylibPath, umbrella);
+
+  error("unable to locate re-export with install name " + path);
+  return {};
+}
+
 DylibFile::DylibFile(MemoryBufferRef mb, DylibFile *umbrella)
     : InputFile(DylibKind, mb) {
   if (umbrella == nullptr)
@@ -358,6 +414,9 @@ DylibFile::DylibFile(MemoryBufferRef mb, DylibFile *umbrella)
   }
 
   // Initialize symbols.
+  // TODO: if a re-exported dylib is public (lives in /usr/lib or
+  // /System/Library/Frameworks), we should bind to its symbols directly
+  // instead of the re-exporting umbrella library.
   if (const load_command *cmd = findCommand(hdr, LC_DYLD_INFO_ONLY)) {
     auto *c = reinterpret_cast<const dyld_info_command *>(cmd);
     parseTrie(buf + c->export_off, c->export_size,
@@ -386,13 +445,8 @@ DylibFile::DylibFile(MemoryBufferRef mb, DylibFile *umbrella)
     auto *c = reinterpret_cast<const dylib_command *>(cmd);
     StringRef reexportPath =
         reinterpret_cast<const char *>(c) + read32le(&c->dylib.name);
-    // TODO: Expand @loader_path, @executable_path etc in reexportPath
-    Optional<MemoryBufferRef> buffer = readFile(reexportPath);
-    if (!buffer) {
-      error("unable to read re-exported dylib at " + reexportPath);
-      return;
-    }
-    reexported.push_back(make<DylibFile>(*buffer, umbrella));
+    if (Optional<DylibFile *> reexport = loadReexport(reexportPath, umbrella))
+      reexported.push_back(*reexport);
   }
 }
 
@@ -431,11 +485,20 @@ DylibFile::DylibFile(const InterfaceFile &interface, DylibFile *umbrella)
       break;
     }
   }
-  // TODO(compnerd) properly represent the hierarchy of the documents as it is
-  // in theory possible to have re-exported dylibs from re-exported dylibs which
-  // should be parent'ed to the child.
-  for (const std::shared_ptr<InterfaceFile> &intf : interface.documents())
-    reexported.push_back(make<DylibFile>(*intf, umbrella));
+
+  bool isTopLevelTapi = false;
+  if (currentTopLevelTapi == nullptr) {
+    currentTopLevelTapi = &interface;
+    isTopLevelTapi = true;
+  }
+
+  for (InterfaceFileRef intfRef : interface.reexportedLibraries())
+    if (Optional<DylibFile *> reexport =
+            loadReexport(intfRef.getInstallName(), umbrella))
+      reexported.push_back(*reexport);
+
+  if (isTopLevelTapi)
+    currentTopLevelTapi = nullptr;
 }
 
 ArchiveFile::ArchiveFile(std::unique_ptr<llvm::object::Archive> &&f)

lld/test/MachO/Inputs/MacOSX.sdk/usr/lib/libc++.tbd

@@ -1,10 +1,10 @@
 --- !tapi-tbd-v3
 archs:            [ i386, x86_64 ]
-uuids:            [ 'i386: 00000000-0000-0000-0000-000000000000', 'x86_64: 00000000-0000-0000-0000-0
-0000000001' ]
+uuids:            [ 'i386: 00000000-0000-0000-0000-000000000000', 'x86_64: 00000000-0000-0000-0000-000000000001' ]
 platform:         macosx
 install-name:     '/usr/lib/libc++.dylib'
 current-version:  1281
 exports:
   - archs:        [ i386, x86_64 ]
+    re-exports:   [ '/usr/lib/libc++abi.dylib' ]
 ...

lld/test/MachO/Inputs/MacOSX.sdk/usr/lib/libc++abi.tbd

@@ -0,0 +1,10 @@
+--- !tapi-tbd-v3
+archs:            [ i386, x86_64 ]
+uuids:            [ 'i386: 00000000-0000-0000-0000-000000000000', 'x86_64: 00000000-0000-0000-0000-000000000001' ]
+platform:         macosx
+install-name:     '/usr/lib/libc++abi.dylib'
+current-version:  1281
+exports:
+  - archs:        [ i386, x86_64 ]
+    symbols:      [ ___gxx_personality_v0 ]
+...

lld/test/MachO/Inputs/iPhoneSimulator.sdk/usr/lib/libSystem.tbd

@@ -20,4 +20,15 @@ exports:
     symbols:    [ __cache_handle_memory_pressure_event ]
   - archs:      [ i386, x86_64 ]
     symbols:    [ _cache_create, _cache_destroy, _cache_get ]
+
+# The following TAPI document is not re-exported by any other document in this
+# TBD file, and should therefore be inaccessible.
+--- !tapi-tbd-v3
+archs:            [ i386, x86_64 ]
+uuids:            [ 'i386: 00000000-0000-0000-0000-000000000003', 'x86_64: 00000000-0000-0000-0000-000000000004' ]
+platform:         ios
+install-name:     '/usr/lib/libnotreexported.dylib'
+exports:
+  - archs:      [ i386, x86_64 ]
+    symbols:    [ _from_non_reexported_tapi_dylib ]
 ...

lld/test/MachO/invalid/stub-link.s

@@ -5,11 +5,13 @@
 # RUN: llvm-mc -filetype obj -triple x86_64-apple-ios %s -o %t/test.o
 # RUN: not lld -flavor darwinnew -o %t/test -Z -L%S/../Inputs/iPhoneSimulator.sdk/usr/lib -lSystem %t/test.o 2>&1 | FileCheck %s
 
-# CHECK: error: undefined symbol __cache_handle_memory_pressure_event
+# CHECK-DAG: error: undefined symbol __cache_handle_memory_pressure_event
+# CHECK-DAG: error: undefined symbol _from_non_reexported_tapi_dylib
 
 .section __TEXT,__text
 .global _main
 
 _main:
   movq __cache_handle_memory_pressure_event@GOTPCREL(%rip), %rax
+  movq _from_non_reexported_tapi_dylib@GOTPCREL(%rip), %rax
   ret

lld/test/MachO/reexport-stub.s

@@ -10,3 +10,19 @@
 # DYLIB-HEADERS:     cmd     LC_REEXPORT_DYLIB
 # DYLIB-HEADERS-NOT: Load command
 # DYLIB-HEADERS:     name    /usr/lib/libc++.dylib
+
+# RUN: llvm-mc -filetype obj -triple x86_64-apple-darwin %s -o %t/test.o
+# RUN: lld -flavor darwinnew -o %t/test -syslibroot %S/Inputs/MacOSX.sdk -lSystem -L%t -lreexporter %t/test.o
+# RUN: llvm-objdump --bind --no-show-raw-insn -d %t/test | FileCheck %s
+
+# CHECK: Bind table:
+# CHECK-DAG: __DATA __data {{.*}} pointer 0 libreexporter ___gxx_personality_v0
+
+.text
+.globl _main
+
+_main:
+  ret
+
+.data
+  .quad ___gxx_personality_v0

lld/test/MachO/stub-link.s

@@ -3,7 +3,7 @@
 # RUN: mkdir -p %t
 #
 # RUN: llvm-mc -filetype obj -triple x86_64-apple-darwin %s -o %t/test.o
-# RUN: lld -flavor darwinnew -o %t/test -syslibroot %S/Inputs/MacOSX.sdk -lSystem -framework CoreFoundation %t/test.o
+# RUN: lld -flavor darwinnew -o %t/test -syslibroot %S/Inputs/MacOSX.sdk -lSystem -lc++ -framework CoreFoundation %t/test.o
 #
 # RUN: llvm-objdump --bind --no-show-raw-insn -d -r %t/test | FileCheck %s
 
@@ -16,11 +16,13 @@
 # CHECK-DAG: __DATA __data {{.*}} pointer 0 CoreFoundation _OBJC_METACLASS_$_NSObject
 # CHECK-DAG: __DATA __data {{.*}} pointer 0 CoreFoundation _OBJC_IVAR_$_NSConstantArray._count
 # CHECK-DAG: __DATA __data {{.*}} pointer 0 CoreFoundation _OBJC_EHTYPE_$_NSException
+# CHECK-DAG: __DATA __data {{.*}} pointer 0 libc++         ___gxx_personality_v0
 
 .section __TEXT,__text
 .global _main
 
 _main:
+## This symbol is defined in an inner TAPI document within libSystem.tbd.
   movq ___nan@GOTPCREL(%rip), %rax
   ret
 
@@ -29,3 +31,9 @@ _main:
   .quad _OBJC_METACLASS_$_NSObject
   .quad _OBJC_IVAR_$_NSConstantArray._count
   .quad _OBJC_EHTYPE_$_NSException
+
+## This symbol is defined in libc++abi.tbd, but we are linking test.o against
+## libc++.tbd (which re-exports libc++abi). Linking against this symbol verifies
+## that .tbd file re-exports can refer not just to TAPI documents within the
+## same .tbd file, but to other on-disk files as well.
+  .quad ___gxx_personality_v0

lld/test/MachO/sub-library.s

@@ -52,7 +52,7 @@
 # RUN: rm -f %t/libgoodbye.dylib
 # RUN: not lld -flavor darwinnew -o %t/sub-library -Z -L%t -lsuper %t/sub-library.o 2>&1 \
 # RUN:  | FileCheck %s --check-prefix=MISSING-REEXPORT -DDIR=%t
-# MISSING-REEXPORT: error: unable to read re-exported dylib at [[DIR]]/libgoodbye.dylib
+# MISSING-REEXPORT: error: unable to locate re-export with install name [[DIR]]/libgoodbye.dylib
 
 .text
 .globl _main