[BOLT] Free memory for CFG after emission

Summary:
Once we emit function code, we no longer need CFG for next phases
that use basic blocks for address-translation and symbol update
purposes. We free memory used by CFG and instructions. The freed
memory gets reused by later phases resulting in overall memory usage
reduction.

We can probably improve memory consumption even further by replacing
BinaryBasicBlocks with more compact data structures.

(cherry picked from FBD18408954)

bolt/src/BinaryBasicBlock.cpp

@@ -33,6 +33,14 @@ bool operator<(const BinaryBasicBlock &LHS, const BinaryBasicBlock &RHS) {
   return LHS.Index < RHS.Index;
 }
 
+bool BinaryBasicBlock::hasCFG() const {
+  return getParent()->hasCFG();
+}
+
+bool BinaryBasicBlock::hasInstructions() const {
+  return getParent()->hasInstructions();
+}
+
 void BinaryBasicBlock::adjustNumPseudos(const MCInst &Inst, int Sign) {
   auto &BC = Function->getBinaryContext();
   if (BC.MII->get(Inst.getOpcode()).isPseudo())

bolt/src/BinaryBasicBlock.h

@@ -176,21 +176,35 @@ public:
   using reverse_iterator       = std::reverse_iterator<iterator>;
   using const_reverse_iterator = std::reverse_iterator<const_iterator>;
 
-  bool         empty()            const { return Instructions.empty(); }
-  size_t       size()             const { return Instructions.size(); }
-  MCInst       &front()                 { return Instructions.front();  }
-  MCInst       &back()                  { return Instructions.back();   }
-  const MCInst &front()           const { return Instructions.front();  }
-  const MCInst &back()            const { return Instructions.back();   }
+  bool         empty()            const { assert(hasInstructions());
+                                          return Instructions.empty(); }
+  size_t       size()             const { assert(hasInstructions());
+                                          return Instructions.size(); }
+  MCInst       &front()                 { assert(hasInstructions());
+                                          return Instructions.front();  }
+  MCInst       &back()                  { assert(hasInstructions());
+                                          return Instructions.back();   }
+  const MCInst &front()           const { assert(hasInstructions());
+                                          return Instructions.front();  }
+  const MCInst &back()            const { assert(hasInstructions());
+                                          return Instructions.back();   }
 
-  iterator                begin()       { return Instructions.begin();  }
-  const_iterator          begin() const { return Instructions.begin();  }
-  iterator                end  ()       { return Instructions.end();    }
-  const_iterator          end  () const { return Instructions.end();    }
-  reverse_iterator       rbegin()       { return Instructions.rbegin(); }
-  const_reverse_iterator rbegin() const { return Instructions.rbegin(); }
-  reverse_iterator       rend  ()       { return Instructions.rend();   }
-  const_reverse_iterator rend  () const { return Instructions.rend();   }
+  iterator                begin()       { assert(hasInstructions());
+                                          return Instructions.begin();  }
+  const_iterator          begin() const { assert(hasInstructions());
+                                          return Instructions.begin();  }
+  iterator                end  ()       { assert(hasInstructions());
+                                          return Instructions.end();    }
+  const_iterator          end  () const { assert(hasInstructions());
+                                          return Instructions.end();    }
+  reverse_iterator       rbegin()       { assert(hasInstructions());
+                                          return Instructions.rbegin(); }
+  const_reverse_iterator rbegin() const { assert(hasInstructions());
+                                          return Instructions.rbegin(); }
+  reverse_iterator       rend  ()       { assert(hasInstructions());
+                                          return Instructions.rend();   }
+  const_reverse_iterator rend  () const { assert(hasInstructions());
+                                          return Instructions.rend();   }
 
   // CFG iterators.
   using pred_iterator        = std::vector<BinaryBasicBlock *>::iterator;
@@ -263,33 +277,43 @@ public:
   bool               lp_empty() const { return LandingPads.empty();   }
 
   inline iterator_range<iterator> instructions() {
+    assert(hasInstructions());
     return iterator_range<iterator>(begin(), end());
   }
   inline iterator_range<const_iterator> instructions() const {
+    assert(hasInstructions());
     return iterator_range<const_iterator>(begin(), end());
   }
   inline iterator_range<pred_iterator> predecessors() {
+    assert(hasCFG());
     return iterator_range<pred_iterator>(pred_begin(), pred_end());
   }
   inline iterator_range<const_pred_iterator> predecessors() const {
+    assert(hasCFG());
     return iterator_range<const_pred_iterator>(pred_begin(), pred_end());
   }
   inline iterator_range<succ_iterator> successors() {
+    assert(hasCFG());
     return iterator_range<succ_iterator>(succ_begin(), succ_end());
   }
   inline iterator_range<const_succ_iterator> successors() const {
+    assert(hasCFG());
     return iterator_range<const_succ_iterator>(succ_begin(), succ_end());
   }
   inline iterator_range<throw_iterator> throwers() {
+    assert(hasCFG());
     return iterator_range<throw_iterator>(throw_begin(), throw_end());
   }
   inline iterator_range<const_throw_iterator> throwers() const {
+    assert(hasCFG());
     return iterator_range<const_throw_iterator>(throw_begin(), throw_end());
   }
   inline iterator_range<lp_iterator> landing_pads() {
+    assert(hasCFG());
     return iterator_range<lp_iterator>(lp_begin(), lp_end());
   }
   inline iterator_range<const_lp_iterator> landing_pads() const {
+    assert(hasCFG());
     return iterator_range<const_lp_iterator>(lp_begin(), lp_end());
   }
 
@@ -936,6 +960,12 @@ public:
     return getFunction();
   }
 
+  /// Return true if the containing function is in CFG state.
+  bool hasCFG() const;
+
+  /// Return true if the containing function is in a state with instructions.
+  bool hasInstructions() const;
+
 private:
   void adjustNumPseudos(const MCInst &Inst, int Sign);
 
@@ -976,6 +1006,21 @@ private:
   void setIndex(unsigned I) {
     Index = I;
   }
+
+  template<typename T> void clearList(T& List) {
+    T TempList;
+    TempList.swap(List);
+  }
+
+  /// Release memory taken by CFG edges and instructions.
+  void releaseCFG() {
+    clearList(Predecessors);
+    clearList(Successors);
+    clearList(Throwers);
+    clearList(LandingPads);
+    clearList(BranchInfo);
+    clearList(Instructions);
+  }
 };
 
 /// Keep the size of the BinaryBasicBlock within a reasonable size class.

bolt/src/BinaryContext.cpp

@@ -1750,6 +1750,7 @@ BinaryContext::createInjectedBinaryFunction(const std::string &Name,
   InjectedBinaryFunctions.push_back(new BinaryFunction(Name, *this, IsSimple));
   auto *BF = InjectedBinaryFunctions.back();
   setSymbolToFunctionMap(BF->getSymbol(), BF);
+  BF->CurrentState = BinaryFunction::State::CFG;
   return BF;
 }
 

bolt/src/BinaryFunction.cpp

@@ -256,7 +256,7 @@ std::string BinaryFunction::getDemangledName() const {
   char *const Name =
       abi::__cxa_demangle(MangledName.str().c_str(), 0, 0, &Status);
   const std::string NameStr(Status == 0 ? Name : MangledName);
-  free(Name);
+  ::free(Name);
   return NameStr;
 }
 
@@ -4219,7 +4219,8 @@ DebugAddressRangesVector BinaryFunction::translateInputToOutputRanges(
       continue;
     }
     auto InputOffset = Range.LowPC - getAddress();
-    const auto InputEndOffset = std::min(Range.HighPC - getAddress(), getSize());
+    const auto InputEndOffset = std::min(Range.HighPC - getAddress(),
+                                         getSize());
 
     auto BBI = std::upper_bound(BasicBlockOffsets.begin(),
                                 BasicBlockOffsets.end(),

bolt/src/BinaryFunction.h

@@ -123,7 +123,8 @@ public:
     Disassembled,     /// Function have been disassembled.
     CFG,              /// Control flow graph has been built.
     CFG_Finalized,    /// CFG is finalized. No optimizations allowed.
-    Emitted,          /// Instructions have been emitted to output.
+    EmittedCFG,       /// Instructions have been emitted to output.
+    Emitted,          /// Same as above plus CFG is destroyed.
   };
 
   /// Types of profile the function can use. Could be a combination.
@@ -964,11 +965,18 @@ public:
   bool hasCFG() const {
     return getState() == State::CFG ||
            getState() == State::CFG_Finalized ||
-           getState() == State::Emitted;
+           getState() == State::EmittedCFG;
+  }
+
+  /// Return true if the function state implies that it includes instructions.
+  bool hasInstructions() const {
+    return getState() == State::Disassembled ||
+           hasCFG();
   }
 
   bool isEmitted() const {
-    return getState() == State::Emitted;
+    return getState() == State::EmittedCFG ||
+           getState() == State::Emitted;
   }
 
   BinarySection &getSection() const {
@@ -2124,8 +2132,12 @@ public:
     CurrentState = State::CFG_Finalized;
   }
 
-  void setEmitted() {
-    CurrentState = State::Emitted;
+  void setEmitted(bool KeepCFG = false) {
+    CurrentState = State::EmittedCFG;
+    if (!KeepCFG) {
+      releaseCFG();
+      CurrentState = State::Emitted;
+    }
   }
 
   /// Process LSDA information for the function.
@@ -2338,6 +2350,17 @@ public:
   FragmentInfo &cold() { return ColdFragment; }
 
   const FragmentInfo &cold() const { return ColdFragment; }
+
+  /// Release memory allocated for CFG and instructions.
+  /// We still keep basic blocks for address translation/mapping purposes.
+  void releaseCFG() {
+    for (auto *BB : BasicBlocks) {
+      BB->releaseCFG();
+    }
+    for (auto BB : DeletedBasicBlocks) {
+      BB->releaseCFG();
+    }
+  }
 };
 
 inline raw_ostream &operator<<(raw_ostream &OS,
@@ -2353,6 +2376,7 @@ inline raw_ostream &operator<<(raw_ostream &OS,
   case BinaryFunction::State::Disassembled: OS << "disassembled";  break;
   case BinaryFunction::State::CFG:          OS << "CFG constructed";  break;
   case BinaryFunction::State::CFG_Finalized:OS << "CFG finalized";  break;
+  case BinaryFunction::State::EmittedCFG:   OS << "emitted with CFG";  break;
   case BinaryFunction::State::Emitted:      OS << "emitted";  break;
   }
 

bolt/src/RewriteInstance.cpp

@@ -2823,14 +2823,14 @@ void RewriteInstance::runOptimizationPasses() {
 }
 
 // Helper function to emit the contents of a function via a MCStreamer object.
-void RewriteInstance::emitFunction(MCStreamer &Streamer,
+bool RewriteInstance::emitFunction(MCStreamer &Streamer,
                                    BinaryFunction &Function,
                                    bool EmitColdPart) {
   if (Function.size() == 0)
-    return;
+    return false;
 
   if (Function.getState() == BinaryFunction::State::Empty)
-    return;
+    return false;
 
   auto *Section =
       BC->getCodeSection(EmitColdPart ? Function.getColdCodeSectionName()
@@ -2928,7 +2928,7 @@ void RewriteInstance::emitFunction(MCStreamer &Streamer,
   if (!EmitColdPart && opts::JumpTables > JTS_NONE)
     Function.emitJumpTables(&Streamer);
 
-  Function.setEmitted();
+  return true;
 }
 
 namespace {
@@ -3206,10 +3206,14 @@ void RewriteInstance::emitFunctions(MCStreamer *Streamer) {
                    << *Function << "\" : "
                    << Function->getFunctionNumber() << '\n');
 
-      emitFunction(*Streamer, *Function, /*EmitColdPart=*/false);
+      bool Emitted{false};
+      Emitted |= emitFunction(*Streamer, *Function, /*EmitColdPart=*/false);
 
       if (Function->isSplit())
-        emitFunction(*Streamer, *Function, /*EmitColdPart=*/true);
+        Emitted |= emitFunction(*Streamer, *Function, /*EmitColdPart=*/true);
+
+      if (Emitted)
+        Function->setEmitted(/*KeepCFG=*/opts::PrintCacheMetrics);
     }
   };
 

bolt/src/RewriteInstance.h

@@ -163,7 +163,7 @@ public:
 
 private:
   /// Emit a single function.
-  void emitFunction(MCStreamer &Streamer, BinaryFunction &Function,
+  bool emitFunction(MCStreamer &Streamer, BinaryFunction &Function,
                     bool EmitColdPart);
 
   /// Detect addresses and offsets available in the binary for allocating