Don't ever call materializeAllPermanently during LTO.

To do this, change the representation of lazy loaded functions.

The previous representation cannot differentiate between a function whose body
has been removed and one whose body hasn't been read from the .bc file. That
means that in order to drop a function, the entire body had to be read.

llvm-svn: 220580

llvm/include/llvm/IR/Function.h

@@ -143,6 +143,9 @@ public:
   /// arguments.
   bool isVarArg() const;
 
+  bool isMaterializable() const;
+  void setIsMaterializable(bool V);
+
   /// getIntrinsicID - This method returns the ID number of the specified
   /// function, or Intrinsic::not_intrinsic if the function is not an
   /// intrinsic, or if the pointer is null.  This value is always defined to be

llvm/include/llvm/IR/GVMaterializer.h

@@ -32,10 +32,6 @@ protected:
 public:
   virtual ~GVMaterializer();
 
-  /// True if GV can be materialized from whatever backing store this
-  /// GVMaterializer uses and has not been materialized yet.
-  virtual bool isMaterializable(const GlobalValue *GV) const = 0;
-
   /// True if GV has been materialized and can be dematerialized back to
   /// whatever backing store this GVMaterializer uses.
   virtual bool isDematerializable(const GlobalValue *GV) const = 0;

llvm/include/llvm/IR/GlobalObject.h

@@ -35,12 +35,24 @@ protected:
 
   std::string Section;     // Section to emit this into, empty means default
   Comdat *ObjComdat;
+  static const unsigned AlignmentBits = 5;
+  static const unsigned GlobalObjectSubClassDataBits =
+      GlobalValueSubClassDataBits - AlignmentBits;
+
+private:
+  static const unsigned AlignmentMask = (1 << AlignmentBits) - 1;
+
 public:
   unsigned getAlignment() const {
-    return (1u << getGlobalValueSubClassData()) >> 1;
+    unsigned Data = getGlobalValueSubClassData();
+    unsigned AlignmentData = Data & AlignmentMask;
+    return (1u << AlignmentData) >> 1;
   }
   void setAlignment(unsigned Align);
 
+  unsigned getGlobalObjectSubClassData() const;
+  void setGlobalObjectSubClassData(unsigned Val);
+
   bool hasSection() const { return !StringRef(getSection()).empty(); }
   const char *getSection() const { return Section.c_str(); }
   void setSection(StringRef S);

llvm/include/llvm/IR/GlobalValue.h

@@ -84,6 +84,7 @@ private:
   // (19 + 3 + 2 + 1 + 2 + 5) == 32.
   unsigned SubClassData : 19;
 protected:
+  static const unsigned GlobalValueSubClassDataBits = 19;
   unsigned getGlobalValueSubClassData() const {
     return SubClassData;
   }
@@ -326,6 +327,13 @@ public:
   /// the current translation unit.
   bool isDeclaration() const;
 
+  bool isDeclarationForLinker() const {
+    if (hasAvailableExternallyLinkage())
+      return true;
+
+    return isDeclaration();
+  }
+
   /// This method unlinks 'this' from the containing module, but does not delete
   /// it.
   virtual void removeFromParent() = 0;

llvm/include/llvm/IR/Module.h

@@ -469,9 +469,6 @@ public:
   /// Retrieves the GVMaterializer, if any, for this Module.
   GVMaterializer *getMaterializer() const { return Materializer.get(); }
 
-  /// True if the definition of GV has yet to be materializedfrom the
-  /// GVMaterializer.
-  bool isMaterializable(const GlobalValue *GV) const;
   /// Returns true if this GV was loaded from this Module's GVMaterializer and
   /// the GVMaterializer knows how to dematerialize the GV.
   bool isDematerializable(const GlobalValue *GV) const;

llvm/lib/Bitcode/Reader/BitcodeReader.cpp

@@ -2070,6 +2070,7 @@ std::error_code BitcodeReader::ParseModule(bool Resume) {
       // If this is a function with a body, remember the prototype we are
       // creating now, so that we can match up the body with them later.
       if (!isProto) {
+        Func->setIsMaterializable(true);
         FunctionsWithBodies.push_back(Func);
         if (LazyStreamer)
           DeferredFunctionInfo[Func] = 0;
@@ -3281,14 +3282,6 @@ std::error_code BitcodeReader::FindFunctionInStream(
 
 void BitcodeReader::releaseBuffer() { Buffer.release(); }
 
-bool BitcodeReader::isMaterializable(const GlobalValue *GV) const {
-  if (const Function *F = dyn_cast<Function>(GV)) {
-    return F->isDeclaration() &&
-      DeferredFunctionInfo.count(const_cast<Function*>(F));
-  }
-  return false;
-}
-
 std::error_code BitcodeReader::Materialize(GlobalValue *GV) {
   Function *F = dyn_cast<Function>(GV);
   // If it's not a function or is already material, ignore the request.
@@ -3308,6 +3301,7 @@ std::error_code BitcodeReader::Materialize(GlobalValue *GV) {
 
   if (std::error_code EC = ParseFunctionBody(F))
     return EC;
+  F->setIsMaterializable(false);
 
   // Upgrade any old intrinsic calls in the function.
   for (UpgradedIntrinsicMap::iterator I = UpgradedIntrinsics.begin(),
@@ -3349,6 +3343,7 @@ void BitcodeReader::Dematerialize(GlobalValue *GV) {
 
   // Just forget the function body, we can remat it later.
   F->dropAllReferences();
+  F->setIsMaterializable(true);
 }
 
 std::error_code BitcodeReader::MaterializeModule(Module *M) {

llvm/lib/Bitcode/Reader/BitcodeReader.h

@@ -223,7 +223,6 @@ public:
 
   void releaseBuffer();
 
-  bool isMaterializable(const GlobalValue *GV) const override;
   bool isDematerializable(const GlobalValue *GV) const override;
   std::error_code Materialize(GlobalValue *GV) override;
   std::error_code MaterializeModule(Module *M) override;

llvm/lib/IR/Function.cpp

@@ -213,6 +213,12 @@ void Argument::removeAttr(AttributeSet AS) {
 // Helper Methods in Function
 //===----------------------------------------------------------------------===//
 
+bool Function::isMaterializable() const {
+  return getGlobalObjectSubClassData();
+}
+
+void Function::setIsMaterializable(bool V) { setGlobalObjectSubClassData(V); }
+
 LLVMContext &Function::getContext() const {
   return getType()->getContext();
 }
@@ -247,6 +253,7 @@ Function::Function(FunctionType *Ty, LinkageTypes Linkage, const Twine &name,
                    Linkage, name) {
   assert(FunctionType::isValidReturnType(getReturnType()) &&
          "invalid return type");
+  setIsMaterializable(false);
   SymTab = new ValueSymbolTable();
 
   // If the function has arguments, mark them as lazily built.
@@ -318,6 +325,8 @@ void Function::setParent(Module *parent) {
 // delete.
 //
 void Function::dropAllReferences() {
+  setIsMaterializable(false);
+
   for (iterator I = begin(), E = end(); I != E; ++I)
     I->dropAllReferences();
 

llvm/lib/IR/Globals.cpp

@@ -29,7 +29,9 @@ using namespace llvm;
 //===----------------------------------------------------------------------===//
 
 bool GlobalValue::isMaterializable() const {
-  return getParent() && getParent()->isMaterializable(this);
+  if (const Function *F = dyn_cast<Function>(this))
+    return F->isMaterializable();
+  return false;
 }
 bool GlobalValue::isDematerializable() const {
   return getParent() && getParent()->isDematerializable(this);
@@ -77,10 +79,24 @@ void GlobalObject::setAlignment(unsigned Align) {
   assert((Align & (Align-1)) == 0 && "Alignment is not a power of 2!");
   assert(Align <= MaximumAlignment &&
          "Alignment is greater than MaximumAlignment!");
-  setGlobalValueSubClassData(Log2_32(Align) + 1);
+  unsigned AlignmentData = Log2_32(Align) + 1;
+  unsigned OldData = getGlobalValueSubClassData();
+  setGlobalValueSubClassData((OldData & ~AlignmentMask) | AlignmentData);
   assert(getAlignment() == Align && "Alignment representation error!");
 }
 
+unsigned GlobalObject::getGlobalObjectSubClassData() const {
+  unsigned ValueData = getGlobalValueSubClassData();
+  return ValueData >> AlignmentBits;
+}
+
+void GlobalObject::setGlobalObjectSubClassData(unsigned Val) {
+  unsigned OldData = getGlobalValueSubClassData();
+  setGlobalValueSubClassData((OldData & AlignmentMask) |
+                             (Val << AlignmentBits));
+  assert(getGlobalObjectSubClassData() == Val && "representation error");
+}
+
 void GlobalObject::copyAttributesFrom(const GlobalValue *Src) {
   const auto *GV = cast<GlobalObject>(Src);
   GlobalValue::copyAttributesFrom(GV);
@@ -117,7 +133,7 @@ bool GlobalValue::isDeclaration() const {
 
   // Functions are definitions if they have a body.
   if (const Function *F = dyn_cast<Function>(this))
-    return F->empty();
+    return F->empty() && !F->isMaterializable();
 
   // Aliases are always definitions.
   assert(isa<GlobalAlias>(this));

llvm/lib/IR/Module.cpp

@@ -389,12 +389,6 @@ void Module::setMaterializer(GVMaterializer *GVM) {
   Materializer.reset(GVM);
 }
 
-bool Module::isMaterializable(const GlobalValue *GV) const {
-  if (Materializer)
-    return Materializer->isMaterializable(GV);
-  return false;
-}
-
 bool Module::isDematerializable(const GlobalValue *GV) const {
   if (Materializer)
     return Materializer->isDematerializable(GV);

llvm/lib/IR/Verifier.cpp

@@ -2632,7 +2632,7 @@ bool llvm::verifyModule(const Module &M, raw_ostream *OS) {
 
   bool Broken = false;
   for (Module::const_iterator I = M.begin(), E = M.end(); I != E; ++I)
-    if (!I->isDeclaration())
+    if (!I->isDeclaration() && !I->isMaterializable())
       Broken |= !V.verify(*I);
 
   // Note that this function's return value is inverted from what you would

llvm/lib/Linker/LinkModules.cpp

@@ -672,21 +672,10 @@ bool ModuleLinker::getComdatResult(const Comdat *SrcC,
                                        LinkFromSrc);
 }
 
-// FIXME: Duplicated from the gold plugin. This should be refactored somewhere.
-static bool isDeclaration(const GlobalValue &V) {
-  if (V.hasAvailableExternallyLinkage())
-    return true;
-
-  if (V.isMaterializable())
-    return false;
-
-  return V.isDeclaration();
-}
-
 bool ModuleLinker::shouldLinkFromSource(const GlobalValue &Dest,
                                         const GlobalValue &Src) {
-  bool SrcIsDeclaration = isDeclaration(Src);
-  bool DestIsDeclaration = isDeclaration(Dest);
+  bool SrcIsDeclaration = Src.isDeclarationForLinker();
+  bool DestIsDeclaration = Dest.isDeclarationForLinker();
 
   // FIXME: Make datalayout mandatory and just use getDataLayout().
   DataLayout DL(Dest.getParent());
@@ -1635,14 +1624,16 @@ bool ModuleLinker::run() {
           SF->getPrefixData(), ValueMap, RF_None, &TypeMap, &ValMaterializer));
     }
 
-    // Skip if no body (function is external) or materialize.
-    if (SF->isDeclaration()) {
-      if (!SF->isMaterializable())
-        continue;
+    // Materialize if needed.
+    if (SF->isMaterializable()) {
       if (SF->Materialize(&ErrorMsg))
         return true;
     }
 
+    // Skip if no body (function is external).
+    if (SF->isDeclaration())
+      continue;
+
     linkFunctionBody(DF, SF);
     SF->Dematerialize();
   }
@@ -1684,14 +1675,16 @@ bool ModuleLinker::run() {
                                    &ValMaterializer));
       }
 
-      // Materialize if necessary.
-      if (SF->isDeclaration()) {
-        if (!SF->isMaterializable())
-          continue;
+      // Materialize if needed.
+      if (SF->isMaterializable()) {
         if (SF->Materialize(&ErrorMsg))
           return true;
       }
 
+      // Skip if no body (function is external).
+      if (SF->isDeclaration())
+        continue;
+
       // Erase from vector *before* the function body is linked - linkFunctionBody could
       // invalidate I.
       LazilyLinkFunctions.erase(I);

llvm/lib/Object/IRObjectFile.cpp

@@ -204,16 +204,6 @@ std::error_code IRObjectFile::printSymbolName(raw_ostream &OS,
   return object_error::success;
 }
 
-static bool isDeclaration(const GlobalValue &V) {
-  if (V.hasAvailableExternallyLinkage())
-    return true;
-
-  if (V.isMaterializable())
-    return false;
-
-  return V.isDeclaration();
-}
-
 uint32_t IRObjectFile::getSymbolFlags(DataRefImpl Symb) const {
   const GlobalValue *GV = getGV(Symb);
 
@@ -224,7 +214,7 @@ uint32_t IRObjectFile::getSymbolFlags(DataRefImpl Symb) const {
   }
 
   uint32_t Res = BasicSymbolRef::SF_None;
-  if (isDeclaration(*GV))
+  if (GV->isDeclarationForLinker())
     Res |= BasicSymbolRef::SF_Undefined;
   if (GV->hasPrivateLinkage())
     Res |= BasicSymbolRef::SF_FormatSpecific;

llvm/tools/gold/gold-plugin.cpp

@@ -418,18 +418,8 @@ static void keepGlobalValue(GlobalValue &GV,
   assert(!GV.isDiscardableIfUnused());
 }
 
-static bool isDeclaration(const GlobalValue &V) {
-  if (V.hasAvailableExternallyLinkage())
-    return true;
-
-  if (V.isMaterializable())
-    return false;
-
-  return V.isDeclaration();
-}
-
 static void internalize(GlobalValue &GV) {
-  if (isDeclaration(GV))
+  if (GV.isDeclarationForLinker())
     return; // We get here if there is a matching asm definition.
   if (!GV.hasLocalLinkage())
     GV.setLinkage(GlobalValue::InternalLinkage);
@@ -492,6 +482,9 @@ static GlobalObject *makeInternalReplacement(GlobalObject *GO) {
   Module *M = GO->getParent();
   GlobalObject *Ret;
   if (auto *F = dyn_cast<Function>(GO)) {
+    if (F->isMaterializable())
+      F->Materialize();
+
     auto *NewF = Function::Create(F->getFunctionType(), F->getLinkage(),
                                   F->getName(), M);
 
@@ -620,7 +613,7 @@ getModuleForFile(LLVMContext &Context, claimed_file &F, raw_fd_ostream *ApiFile,
     case LDPR_RESOLVED_EXEC:
     case LDPR_RESOLVED_DYN:
     case LDPR_UNDEF:
-      assert(isDeclaration(*GV));
+      assert(GV->isDeclarationForLinker());
       break;
 
     case LDPR_PREVAILING_DEF_IRONLY: {
@@ -667,12 +660,6 @@ getModuleForFile(LLVMContext &Context, claimed_file &F, raw_fd_ostream *ApiFile,
     Sym.comdat_key = nullptr;
   }
 
-  if (!Drop.empty())
-    // This is horrible. Given how lazy loading is implemented, dropping
-    // the body while there is a materializer present doesn't work, the
-    // linker will just read the body back.
-    M->materializeAllPermanently();
-
   ValueToValueMapTy VM;
   LocalValueMaterializer Materializer(Drop);
   for (GlobalAlias *GA : KeptAliases) {