[ThinLTO] Emit files for distributed builds for all modules

With the new LTO API in r278338, we stopped emitting the individual
index files and imports files for some modules in the distributed backend
case (thinlto-index-only plugin option).

Specifically, this is when the linker decides not to include a module in the
link, because it was in an archive library and did not have a strong
reference to it. Not creating the expected output files makes the
distributed build system implementation more difficult, in terms of
checking for the expected outputs of the thin link, and scheduling the
backend jobs. To address this, the gold-plugin will write dummy empty
.thinlto.bc and .imports files for modules not included in the link
(which LTO never sees).

Augmented a gold v1.12+ test, since that version of gold has the handling
for notifying on modules not being included in the link.

llvm-svn: 282100

llvm/include/llvm/LTO/LTO.h

@@ -85,6 +85,13 @@ void thinLTOInternalizeAndPromoteInIndex(
 
 namespace lto {
 
+/// Given the original \p Path to an output file, replace any path
+/// prefix matching \p OldPrefix with \p NewPrefix. Also, create the
+/// resulting directory if it does not yet exist.
+std::string getThinLTOOutputFile(const std::string &Path,
+                                 const std::string &OldPrefix,
+                                 const std::string &NewPrefix);
+
 class LTO;
 struct SymbolResolution;
 class ThinBackendProc;

llvm/lib/LTO/LTO.cpp

@@ -609,6 +609,26 @@ ThinBackend lto::createInProcessThinBackend(unsigned ParallelismLevel) {
   };
 }
 
+// Given the original \p Path to an output file, replace any path
+// prefix matching \p OldPrefix with \p NewPrefix. Also, create the
+// resulting directory if it does not yet exist.
+std::string lto::getThinLTOOutputFile(const std::string &Path,
+                                      const std::string &OldPrefix,
+                                      const std::string &NewPrefix) {
+  if (OldPrefix.empty() && NewPrefix.empty())
+    return Path;
+  SmallString<128> NewPath(Path);
+  llvm::sys::path::replace_path_prefix(NewPath, OldPrefix, NewPrefix);
+  StringRef ParentPath = llvm::sys::path::parent_path(NewPath.str());
+  if (!ParentPath.empty()) {
+    // Make sure the new directory exists, creating it if necessary.
+    if (std::error_code EC = llvm::sys::fs::create_directories(ParentPath))
+      llvm::errs() << "warning: could not create directory '" << ParentPath
+                   << "': " << EC.message() << '\n';
+  }
+  return NewPath.str();
+}
+
 class WriteIndexesThinBackend : public ThinBackendProc {
   std::string OldPrefix, NewPrefix;
   bool ShouldEmitImportsFiles;
@@ -627,26 +647,6 @@ public:
         ShouldEmitImportsFiles(ShouldEmitImportsFiles),
         LinkedObjectsFileName(LinkedObjectsFileName) {}
 
-  /// Given the original \p Path to an output file, replace any path
-  /// prefix matching \p OldPrefix with \p NewPrefix. Also, create the
-  /// resulting directory if it does not yet exist.
-  std::string getThinLTOOutputFile(const std::string &Path,
-                                   const std::string &OldPrefix,
-                                   const std::string &NewPrefix) {
-    if (OldPrefix.empty() && NewPrefix.empty())
-      return Path;
-    SmallString<128> NewPath(Path);
-    llvm::sys::path::replace_path_prefix(NewPath, OldPrefix, NewPrefix);
-    StringRef ParentPath = llvm::sys::path::parent_path(NewPath.str());
-    if (!ParentPath.empty()) {
-      // Make sure the new directory exists, creating it if necessary.
-      if (std::error_code EC = llvm::sys::fs::create_directories(ParentPath))
-        llvm::errs() << "warning: could not create directory '" << ParentPath
-                     << "': " << EC.message() << '\n';
-    }
-    return NewPath.str();
-  }
-
   Error start(
       unsigned Task, MemoryBufferRef MBRef,
       const FunctionImporter::ImportMapTy &ImportList,

llvm/test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll

@@ -9,14 +9,26 @@
 ; be included in the link, and not %t2.o since it is within
 ; a library (--start-lib/--end-lib pair) and not strongly referenced.
 ; Note that the support for detecting this is in gold v1.12.
+; RUN: rm -f %t.o.thinlto.bc
+; RUN: rm -f %t2.o.thinlto.bc
+; RUN: rm -f %t.o.imports
+; RUN: rm -f %t2.o.imports
 ; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \
 ; RUN:    --plugin-opt=thinlto \
 ; RUN:    --plugin-opt=thinlto-index-only=%t3 \
+; RUN:    --plugin-opt=thinlto-emit-imports-files \
 ; RUN:    -m elf_x86_64 \
 ; RUN:    -o %t4 \
 ; RUN:    %t.o \
 ; RUN:    --start-lib %t2.o --end-lib
 
+; Ensure that the expected output files are created, even for the file
+; the linker decided not to include in the link.
+; RUN: ls %t.o.thinlto.bc
+; RUN: ls %t2.o.thinlto.bc
+; RUN: ls %t.o.imports
+; RUN: ls %t2.o.imports
+
 ; RUN: cat %t3 | FileCheck %s
 ; CHECK: thinlto_emit_linked_objects.ll.tmp.o
 ; CHECK-NOT: thinlto_emit_linked_objects.ll.tmp2.o

llvm/tools/gold/gold-plugin.cpp

@@ -752,6 +752,34 @@ static std::unique_ptr<LTO> createLTO() {
                                 ParallelCodeGenParallelismLevel);
 }
 
+// Write empty files that may be expected by a distributed build
+// system when invoked with thinlto_index_only. This is invoked when
+// the linker has decided not to include the given module in the
+// final link. Frequently the distributed build system will want to
+// confirm that all expected outputs are created based on all of the
+// modules provided to the linker.
+static void writeEmptyDistributedBuildOutputs(std::string &ModulePath,
+                                              std::string &OldPrefix,
+                                              std::string &NewPrefix) {
+  std::string NewModulePath =
+      getThinLTOOutputFile(ModulePath, OldPrefix, NewPrefix);
+  std::error_code EC;
+  {
+    raw_fd_ostream OS(NewModulePath + ".thinlto.bc", EC,
+                      sys::fs::OpenFlags::F_None);
+    if (EC)
+      message(LDPL_FATAL, "Failed to write '%s': %s",
+              (NewModulePath + ".thinlto.bc").c_str(), EC.message().c_str());
+  }
+  if (options::thinlto_emit_imports_files) {
+    raw_fd_ostream OS(NewModulePath + ".imports", EC,
+                      sys::fs::OpenFlags::F_None);
+    if (EC)
+      message(LDPL_FATAL, "Failed to write '%s': %s",
+              (NewModulePath + ".imports").c_str(), EC.message().c_str());
+  }
+}
+
 /// gold informs us that all symbols have been read. At this point, we use
 /// get_symbols to see if any of our definitions have been overridden by a
 /// native object file. Then, perform optimization and codegen.
@@ -771,13 +799,22 @@ static ld_plugin_status allSymbolsReadHook() {
 
   std::unique_ptr<LTO> Lto = createLTO();
 
+  std::string OldPrefix, NewPrefix;
+  if (options::thinlto_index_only)
+    getThinLTOOldAndNewPrefix(OldPrefix, NewPrefix);
+
   for (claimed_file &F : Modules) {
     if (options::thinlto && !HandleToInputFile.count(F.leader_handle))
       HandleToInputFile.insert(std::make_pair(
           F.leader_handle, llvm::make_unique<PluginInputFile>(F.handle)));
     const void *View = getSymbolsAndView(F);
-    if (!View)
+    if (!View) {
+      if (options::thinlto_index_only)
+        // Write empty output files that may be expected by the distributed
+        // build system.
+        writeEmptyDistributedBuildOutputs(F.name, OldPrefix, NewPrefix);
       continue;
+    }
     addModule(*Lto, F, View);
   }