COFF: Implement ThinLTO cache and cache pruning support.

Differential Revision: https://reviews.llvm.org/D37607

llvm-svn: 312770

clang/docs/ThinLTO.rst

@@ -119,23 +119,27 @@ Incremental
 ThinLTO supports fast incremental builds through the use of a cache,
 which currently must be enabled through a linker option.
 
-- gold (as of LLVM r279883):
+- gold (as of LLVM 4.0):
   ``-Wl,-plugin-opt,cache-dir=/path/to/cache``
 - ld64 (support in clang 3.9 and Xcode 8):
   ``-Wl,-cache_path_lto,/path/to/cache``
-- lld (as of LLVM r296702):
+- ELF lld (as of LLVM 5.0):
   ``-Wl,--thinlto-cache-dir=/path/to/cache``
+- COFF lld (as of LLVM 6.0):
+  ``/lldltocache:/path/to/cache``
 
 Cache Pruning
 -------------
 
 To help keep the size of the cache under control, ThinLTO supports cache
-pruning. Cache pruning is supported with ld64 and ELF lld, but currently only
-ELF lld allows you to control the policy with a policy string.  The cache
-policy must be specified with a linker option.
+pruning. Cache pruning is supported with ld64 and ELF and COFF lld, but
+currently only ELF and COFF lld allow you to control the policy with a
+policy string. The cache policy must be specified with a linker option.
 
-- ELF lld (as of LLVM r298036):
+- ELF lld (as of LLVM 5.0):
   ``-Wl,--thinlto-cache-policy,POLICY``
+- COFF lld (as of LLVM 6.0):
+  ``/lldltocachepolicy:POLICY``
 
 A policy string is a series of key-value pairs separated by ``:`` characters.
 Possible key-value pairs are:

lld/COFF/Config.h

@@ -12,6 +12,7 @@
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Object/COFF.h"
+#include "llvm/Support/CachePruning.h"
 #include <cstdint>
 #include <map>
 #include <set>
@@ -123,6 +124,11 @@ struct Configuration {
   // Used for /opt:lldltopartitions=N
   unsigned LTOPartitions = 1;
 
+  // Used for /opt:lldltocache=path
+  StringRef LTOCache;
+  // Used for /opt:lldltocachepolicy=policy
+  llvm::CachePruningPolicy LTOCachePolicy;
+
   // Used for /merge:from=to (e.g. /merge:.rdata=.text)
   std::map<StringRef, StringRef> Merge;
 

lld/COFF/Driver.cpp

@@ -877,6 +877,16 @@ void LinkerDriver::link(ArrayRef<const char *> ArgsArr) {
   if (Args.hasArg(OPT_lldsavetemps))
     Config->SaveTemps = true;
 
+  // Handle /lldltocache
+  if (auto *Arg = Args.getLastArg(OPT_lldltocache))
+    Config->LTOCache = Arg->getValue();
+
+  // Handle /lldsavecachepolicy
+  if (auto *Arg = Args.getLastArg(OPT_lldltocachepolicy))
+    Config->LTOCachePolicy = check(
+        parseCachePruningPolicy(Arg->getValue()),
+        Twine("/lldltocachepolicy: invalid cache policy: ") + Arg->getValue());
+
   // Handle /failifmismatch
   for (auto *Arg : Args.filtered(OPT_failifmismatch))
     checkFailIfMismatch(Arg->getValue());

lld/COFF/LTO.cpp

@@ -18,6 +18,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/IR/DiagnosticPrinter.h"
+#include "llvm/LTO/Caching.h"
 #include "llvm/LTO/Config.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Object/SymbolicFile.h"
@@ -118,11 +119,27 @@ void BitcodeCompiler::add(BitcodeFile &F) {
 std::vector<StringRef> BitcodeCompiler::compile() {
   unsigned MaxTasks = LTOObj->getMaxTasks();
   Buff.resize(MaxTasks);
+  Files.resize(MaxTasks);
 
-  checkError(LTOObj->run([&](size_t Task) {
-    return llvm::make_unique<lto::NativeObjectStream>(
-        llvm::make_unique<raw_svector_ostream>(Buff[Task]));
-  }));
+  // The /lldltocache option specifies the path to a directory in which to cache
+  // native object files for ThinLTO incremental builds. If a path was
+  // specified, configure LTO to use it as the cache directory.
+  lto::NativeObjectCache Cache;
+  if (!Config->LTOCache.empty())
+    Cache = check(
+        lto::localCache(Config->LTOCache,
+                        [&](size_t Task, std::unique_ptr<MemoryBuffer> MB,
+                            StringRef Path) { Files[Task] = std::move(MB); }));
+
+  checkError(LTOObj->run(
+      [&](size_t Task) {
+        return llvm::make_unique<lto::NativeObjectStream>(
+            llvm::make_unique<raw_svector_ostream>(Buff[Task]));
+      },
+      Cache));
+
+  if (!Config->LTOCache.empty())
+    pruneCache(Config->LTOCache, Config->LTOCachePolicy);
 
   std::vector<StringRef> Ret;
   for (unsigned I = 0; I != MaxTasks; ++I) {
@@ -136,5 +153,10 @@ std::vector<StringRef> BitcodeCompiler::compile() {
     }
     Ret.emplace_back(Buff[I].data(), Buff[I].size());
   }
+
+  for (std::unique_ptr<MemoryBuffer> &File : Files)
+    if (File)
+      Ret.push_back(File->getBuffer());
+
   return Ret;
 }

lld/COFF/LTO.h

@@ -49,6 +49,7 @@ public:
 private:
   std::unique_ptr<llvm::lto::LTO> LTOObj;
   std::vector<SmallString<0>> Buff;
+  std::vector<std::unique_ptr<MemoryBuffer>> Files;
 };
 }
 }

lld/COFF/Options.td

@@ -31,6 +31,8 @@ def heap    : P<"heap", "Size of the heap">;
 def implib  : P<"implib", "Import library name">;
 def libpath : P<"libpath", "Additional library search path">;
 def linkrepro : P<"linkrepro", "Dump linker invocation and input files for debugging">;
+def lldltocache : P<"lldltocache", "Path to ThinLTO cached object file directory">;
+def lldltocachepolicy : P<"lldltocachepolicy", "Pruning policy for the ThinLTO cache">;
 def lldsavetemps : F<"lldsavetemps">,
     HelpText<"Save temporary files instead of deleting them">;
 def machine : P<"machine", "Specify target platform">;

lld/test/COFF/Inputs/lto-cache.ll

@@ -0,0 +1,10 @@
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+define i32 @main() {
+entry:
+  call void (...) @globalfunc()
+  ret i32 0
+}
+
+declare void @globalfunc(...)

lld/test/COFF/lto-cache.ll

@@ -0,0 +1,21 @@
+; REQUIRES: x86
+
+; RUN: opt -module-hash -module-summary %s -o %t.o
+; RUN: opt -module-hash -module-summary %p/Inputs/lto-cache.ll -o %t2.o
+
+; RUN: rm -Rf %t.cache && mkdir %t.cache
+; Create two files that would be removed by cache pruning due to age.
+; We should only remove files matching the pattern "llvmcache-*".
+; RUN: touch -t 197001011200 %t.cache/llvmcache-foo %t.cache/foo
+; RUN: lld-link /lldltocache:%t.cache /lldltocachepolicy:prune_after=1h /out:%t3 /entry:main %t2.o %t.o
+
+; Two cached objects, plus a timestamp file and "foo", minus the file we removed.
+; RUN: ls %t.cache | count 4
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+define void @globalfunc() #0 {
+entry:
+  ret void
+}