[lld/mac] Add support for -flat_namespace

-flat_namespace makes lld emit binaries that use name lookup that's more in
line with other POSIX systems: Instead of looking up symbols as (dylib,name)
pairs by dyld, they're instead looked up just by name.

-flat_namespace has three effects:

1. MH_TWOLEVEL and MH_NNOUNDEFS are no longer set in the Mach-O header
2. All symbols use BIND_SPECIAL_DYLIB_FLAT_LOOKUP as ordinal
3. When a dylib is added to the link, its dependent dylibs are also added,
   so that lld can verify that no undefined symbols remain at the end of
   a link with -flat_namespace. These transitive dylibs are added for symbol
   resolution, but they are not emitted in LC_LOAD_COMMANDs.

-undefined with -flat_namespace still isn't implemented. Before this change,
it was impossible to hit that combination because -flat_namespace caused a
diagnostic. Now that it no longer does, emit a dedicated temporary diagnostic
when both flags are used.

Differential Revision: https://reviews.llvm.org/D97641

lld/MachO/Driver.cpp

@@ -796,11 +796,6 @@ bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly,
     config->namespaceKind = arg->getOption().getID() == OPT_twolevel_namespace
                                 ? NamespaceKind::twolevel
                                 : NamespaceKind::flat;
-    if (config->namespaceKind == NamespaceKind::flat) {
-      warn("Option '" + arg->getOption().getPrefixedName() +
-           "' is not yet implemented. Stay tuned...");
-      config->namespaceKind = NamespaceKind::twolevel;
-    }
   }
 
   config->systemLibraryRoots = getSystemLibraryRoots(args);

lld/MachO/InputFiles.cpp

@@ -587,15 +587,14 @@ const InterfaceFile *currentTopLevelTapi = nullptr;
 
 // Re-exports can either refer to on-disk files, or to documents within .tbd
 // files.
-static Optional<DylibFile *> loadReexportHelper(StringRef path,
+static Optional<DylibFile *> findDylib(StringRef path, DylibFile *umbrella) {
-                                                DylibFile *umbrella) {
   if (path::is_absolute(path, path::Style::posix))
     for (StringRef root : config->systemLibraryRoots)
       if (Optional<std::string> dylibPath =
               resolveDylibPath((root + path).str()))
         return loadDylib(*dylibPath, umbrella);
 
-  // TODO: Expand @loader_path, @executable_path etc
+  // TODO: Expand @loader_path, @executable_path, @rpath etc, handle -dylib_path
 
   if (currentTopLevelTapi) {
     for (InterfaceFile &child :
@@ -609,7 +608,6 @@ static Optional<DylibFile *> loadReexportHelper(StringRef path,
   if (Optional<std::string> dylibPath = resolveDylibPath(path))
     return loadDylib(*dylibPath, umbrella);
 
-  error("unable to locate re-export with install name " + path);
   return {};
 }
 
@@ -634,8 +632,10 @@ static bool isImplicitlyLinked(StringRef path) {
 }
 
 void loadReexport(StringRef path, DylibFile *umbrella) {
-  Optional<DylibFile *> reexport = loadReexportHelper(path, umbrella);
+  Optional<DylibFile *> reexport = findDylib(path, umbrella);
-  if (reexport && isImplicitlyLinked(path))
+  if (!reexport)
+    error("unable to locate re-export with install name " + path);
+  else if (isImplicitlyLinked(path))
     inputFiles.insert(*reexport);
 }
 
@@ -679,21 +679,33 @@ DylibFile::DylibFile(MemoryBufferRef mb, DylibFile *umbrella,
     return;
   }
 
-  if (hdr->flags & MH_NO_REEXPORTED_DYLIBS)
-    return;
-
   const uint8_t *p =
       reinterpret_cast<const uint8_t *>(hdr) + sizeof(mach_header_64);
   for (uint32_t i = 0, n = hdr->ncmds; i < n; ++i) {
     auto *cmd = reinterpret_cast<const load_command *>(p);
     p += cmd->cmdsize;
-    if (cmd->cmd != LC_REEXPORT_DYLIB)
-      continue;
 
-    auto *c = reinterpret_cast<const dylib_command *>(cmd);
+    if (!(hdr->flags & MH_NO_REEXPORTED_DYLIBS) &&
-    StringRef reexportPath =
+        cmd->cmd == LC_REEXPORT_DYLIB) {
-        reinterpret_cast<const char *>(c) + read32le(&c->dylib.name);
+      const auto *c = reinterpret_cast<const dylib_command *>(cmd);
-    loadReexport(reexportPath, umbrella);
+      StringRef reexportPath =
+          reinterpret_cast<const char *>(c) + read32le(&c->dylib.name);
+      loadReexport(reexportPath, umbrella);
+    }
+
+    // FIXME: What about LC_LOAD_UPWARD_DYLIB, LC_LAZY_LOAD_DYLIB,
+    // LC_LOAD_WEAK_DYLIB, LC_REEXPORT_DYLIB (..are reexports from dylibs with
+    // MH_NO_REEXPORTED_DYLIBS loaded for -flat_namespace)?
+    if (config->namespaceKind == NamespaceKind::flat &&
+        cmd->cmd == LC_LOAD_DYLIB) {
+      const auto *c = reinterpret_cast<const dylib_command *>(cmd);
+      StringRef dylibPath =
+          reinterpret_cast<const char *>(c) + read32le(&c->dylib.name);
+      Optional<DylibFile *> dylib = findDylib(dylibPath, umbrella);
+      if (!dylib)
+        error(Twine("unable to locate library '") + dylibPath +
+              "' loaded from '" + toString(this) + "' for -flat_namespace");
+    }
   }
 }
 

lld/MachO/Options.td

@@ -448,7 +448,6 @@ def alias_list : Separate<["-"], "alias_list">,
      Group<grp_resolve>;
 def flat_namespace : Flag<["-"], "flat_namespace">,
      HelpText<"Resolve symbols from all dylibs, both direct and transitive. Do not record source libraries: dyld must re-search at runtime and use the first definition found">,
-     Flags<[HelpHidden]>,
      Group<grp_resolve>;
 def twolevel_namespace : Flag<["-"], "twolevel_namespace">,
      HelpText<"Make dyld look up symbols by (dylib,name) pairs (default)">,

lld/MachO/SymbolTable.cpp

@@ -181,12 +181,14 @@ void lld::macho::treatUndefinedSymbol(const Undefined &sym) {
     message += "\n>>> referenced by " + fileName;
   switch (config->undefinedSymbolTreatment) {
   case UndefinedSymbolTreatment::suppress:
+    error("-undefined suppress unimplemented");
     break;
   case UndefinedSymbolTreatment::error:
     error(message);
     break;
   case UndefinedSymbolTreatment::warning:
     warn(message);
+    error("-undefined warning unimplemented");
     break;
   case UndefinedSymbolTreatment::dynamic_lookup:
     error("dynamic_lookup unimplemented for " + message);

lld/MachO/SyntheticSections.cpp

@@ -78,7 +78,10 @@ void MachHeaderSection::writeTo(uint8_t *buf) const {
   hdr->filetype = config->outputType;
   hdr->ncmds = loadCommands.size();
   hdr->sizeofcmds = sizeOfCmds;
-  hdr->flags = MachO::MH_NOUNDEFS | MachO::MH_DYLDLINK | MachO::MH_TWOLEVEL;
+  hdr->flags = MachO::MH_DYLDLINK;
+
+  if (config->namespaceKind == NamespaceKind::twolevel)
+    hdr->flags |= MachO::MH_NOUNDEFS | MachO::MH_TWOLEVEL;
 
   if (config->outputType == MachO::MH_DYLIB && !config->hasReexports)
     hdr->flags |= MachO::MH_NO_REEXPORTED_DYLIBS;
@@ -280,8 +283,9 @@ static void encodeBinding(const Symbol *sym, const OutputSection *osec,
 
 // Non-weak bindings need to have their dylib ordinal encoded as well.
 static int16_t ordinalForDylibSymbol(const DylibSymbol &dysym) {
-  return dysym.isDynamicLookup() ? MachO::BIND_SPECIAL_DYLIB_FLAT_LOOKUP
+  return config->namespaceKind == NamespaceKind::flat || dysym.isDynamicLookup()
-                                 : dysym.getFile()->ordinal;
+             ? MachO::BIND_SPECIAL_DYLIB_FLAT_LOOKUP
+             : dysym.getFile()->ordinal;
 }
 
 static void encodeDylibOrdinal(int16_t ordinal, raw_svector_ostream &os) {
@@ -816,13 +820,15 @@ void SymtabSection::writeTo(uint8_t *buf) const {
       nList->n_desc |= defined->isExternalWeakDef() ? MachO::N_WEAK_DEF : 0;
     } else if (auto *dysym = dyn_cast<DylibSymbol>(entry.sym)) {
       uint16_t n_desc = nList->n_desc;
-      if (dysym->isDynamicLookup())
+      int16_t ordinal = ordinalForDylibSymbol(*dysym);
+      if (ordinal == MachO::BIND_SPECIAL_DYLIB_FLAT_LOOKUP)
         MachO::SET_LIBRARY_ORDINAL(n_desc, MachO::DYNAMIC_LOOKUP_ORDINAL);
-      else if (dysym->getFile()->isBundleLoader)
+      else if (ordinal == MachO::BIND_SPECIAL_DYLIB_MAIN_EXECUTABLE)
         MachO::SET_LIBRARY_ORDINAL(n_desc, MachO::EXECUTABLE_ORDINAL);
-      else
+      else {
-        MachO::SET_LIBRARY_ORDINAL(
+        assert(ordinal > 0);
-            n_desc, static_cast<uint8_t>(dysym->getFile()->ordinal));
+        MachO::SET_LIBRARY_ORDINAL(n_desc, static_cast<uint8_t>(ordinal));
+      }
 
       nList->n_type = MachO::N_EXT;
       n_desc |= dysym->isWeakDef() ? MachO::N_WEAK_DEF : 0;

lld/test/MachO/flat-namespace.s

@@ -0,0 +1,87 @@
+# REQUIRES: x86
+# RUN: rm -rf %t
+# RUN: split-file %s %t
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos -o %t/foo.o %t/foo.s
+# RUN: %lld -dylib -o %t/foo.dylib %t/foo.o
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos -o %t/bar.o %t/bar.s
+# RUN: %lld -lSystem -dylib -o %t/bar.dylib %t/bar.o %t/foo.dylib
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos -o %t/baz.o %t/baz.s
+# RUN: %lld -lSystem -dylib -o %t/baz.dylib %t/baz.o %t/bar.dylib
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos -o %t/main.o %t/main.s
+
+# With flat_namespace, the linker automatically looks in foo.dylib and
+# bar.dylib too, but it doesn't add a LC_LOAD_DYLIB for it.
+# RUN: %lld -flat_namespace -lSystem %t/main.o %t/baz.dylib -o %t/out
+# RUN: llvm-objdump --macho --all-headers %t/out \
+# RUN:     | FileCheck --check-prefix=HEADERBITS %s
+# RUN: llvm-objdump --macho --bind --lazy-bind --weak-bind %t/out \
+# RUN:     | FileCheck --check-prefix=FLAT %s
+# RUN: llvm-nm -m %t/out | FileCheck --check-prefix=FLATSYM %s
+# RUN: llvm-readobj --syms %t/out | FileCheck --check-prefix=FLATSYM-READOBJ %s
+
+# HEADERBITS-NOT: NOUNDEFS
+# HEADERBITS-NOT: TWOLEVEL
+# HEADERBITS: DYLDLINK
+# HEADERBITS-NOT: foo.dylib
+# HEADERBITS-NOT: bar.dylib
+
+# FLAT: Bind table:
+# FLAT: __DATA_CONST __got          0x{{[0-9a-f]*}} pointer         0 flat-namespace   dyld_stub_binder
+# FLAT: Lazy bind table:
+# FLAT-DAG: __DATA   __la_symbol_ptr    0x{{[0-9a-f]*}} flat-namespace   _bar
+# FLAT-DAG: __DATA   __la_symbol_ptr    0x{{[0-9a-f]*}} flat-namespace   _baz
+# FLAT-DAG: __DATA   __la_symbol_ptr    0x{{[0-9a-f]*}} flat-namespace   _foo
+
+# No "(dynamically looked up)" because llvm-nm -m doesn't print that
+# for files without MH_TWOLEVEL for some reason.
+# FLATSYM: (undefined) external _bar
+# FLATSYM: (undefined) external _baz
+# FLATSYM: (undefined) external _foo
+
+# ...but `llvm-readobj --syms` does, so verify we put the right thing there.
+# FLATSYM-READOBJ: Flags [ (0xFE00)
+
+# Undefined symbols should still cause errors by default.
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos \
+# RUN:     -o %t/main-with-undef.o %t/main-with-undef.s
+# RUN: not %lld -flat_namespace -lSystem %t/main-with-undef.o %t/bar.dylib \
+# RUN:     -o %t/out 2>&1 | FileCheck --check-prefix=UNDEF %s
+# UNDEF: error: undefined symbol: _quux
+
+#--- foo.s
+.globl _foo
+_foo:
+  ret
+
+#--- bar.s
+.globl _bar
+_bar:
+  callq _foo
+  ret
+
+#--- baz.s
+.globl _baz
+_baz:
+  callq _bar
+  ret
+
+#--- main.s
+.globl _main
+_main:
+  callq _foo
+  callq _bar
+  callq _baz
+  ret
+
+#--- main-with-undef.s
+.globl _main
+_main:
+  callq _foo
+  callq _bar
+  callq _baz
+  callq _quux
+  ret

lld/test/MachO/header.s

@@ -12,8 +12,8 @@
 # RUN: llvm-objdump --macho --all-headers %t/x86-64-dylib | FileCheck %s -DCAPS=0x00
 # RUN: llvm-objdump --macho --all-headers %t/arm64-dylib | FileCheck %s -DCAPS=0x00
 
-# CHECK:      magic        cputype cpusubtype  caps    filetype
+# CHECK:      magic        cputype cpusubtype  caps     filetype {{.*}} flags
-# CHECK-NEXT: MH_MAGIC_64  {{.*}}         ALL  [[CAPS]]   {{.*}}
+# CHECK-NEXT: MH_MAGIC_64  {{.*}}         ALL  [[CAPS]] {{.*}}          NOUNDEFS {{.*}} TWOLEVEL
 
 .globl _main
 _main: