updating cache metrics

Summary:
This is a replacement of a previous diff. The implemented metric
('graph distance') is not very useful at the moment but I plan to add
more relevant metrics in the subsequent diff. This diff fixes some
obvious problems and moves the call of CalcMetrics::printAll to the
right place.

(cherry picked from FBD6072312)

bolt/CMakeLists.txt

@@ -64,7 +64,7 @@ add_llvm_tool(llvm-bolt
   BinaryContext.cpp
   BinaryFunction.cpp
   BinaryPassManager.cpp
-  CalcCacheMetrics.cpp
+  CacheMetrics.cpp
   DataAggregator.cpp
   DataReader.cpp
   DebugData.cpp

bolt/CacheMetrics.cpp

@@ -1,4 +1,4 @@
-//===------ CalcCacheMetrics.cpp - Calculate metrics of cache lines -------===//
+//===------ CacheMetrics.cpp - Calculate metrics for instruction cache ----===//
 //
 //                     Functions to show metrics of cache lines
 //
@@ -7,31 +7,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-
-#include "BinaryBasicBlock.h"
-#include "BinaryContext.h"
-#include "BinaryFunction.h"
-#include "BinaryPassManager.h"
-#include "CalcCacheMetrics.h"
-#include "Exceptions.h"
-#include "RewriteInstance.h"
-#include "llvm/MC/MCAsmLayout.h"
-#include "llvm/MC/MCObjectStreamer.h"
-#include "llvm/MC/MCSectionELF.h"
-#include <fstream>
+#include "CacheMetrics.h"
 
 using namespace llvm;
-using namespace object;
 using namespace bolt;
 using Traversal = std::vector<BinaryBasicBlock *>;
 
-namespace opts {
-
-extern cl::OptionCategory BoltOptCategory;
-
-} // namespace opts
-
-
 namespace {
 
 /// Initialize and return a position map for binary basic blocks.
@@ -47,10 +28,8 @@ getPositionMap(const BinaryFunction &Function) {
   return DistMap;
 }
 
-/// Initialize and return a vector of traversals for a given function and its
-/// entry point
-std::vector<Traversal> getTraversals(const BinaryFunction &Function,
-                                     BinaryBasicBlock *EntryBB) {
+/// Initialize and return a vector of traversals for a given entry block
+std::vector<Traversal> getTraversals(BinaryBasicBlock *EntryBB) {
   std::vector<Traversal> AllTraversals;
   std::stack<std::pair<BinaryBasicBlock *, Traversal>> Stack;
   Stack.push(std::make_pair(EntryBB, Traversal()));
@@ -105,10 +84,6 @@ std::vector<Traversal> getTraversals(const BinaryFunction &Function,
 double
 getTraversalLength(std::unordered_map<BinaryBasicBlock *, double> &DistMap,
                    Traversal const &Path) {
-  if (Path.size() <= 1) {
-    return 0.0;
-  }
-
   double Length = 0.0;
   BinaryBasicBlock *PrevBB = Path.front();
   for (auto BBI = std::next(Path.begin()); BBI != Path.end(); ++BBI) {
@@ -119,56 +94,62 @@ getTraversalLength(std::unordered_map<BinaryBasicBlock *, double> &DistMap,
   return Length;
 }
 
-/// Helper function of calcGraphDistance to go through the call traversals of
-/// certain function and to calculate and record the length of each
-/// traversal.
-void graphDistHelper(std::vector<Traversal> &AllTraversals,
-                     const BinaryFunction &Function,
-                     std::unordered_map<uint64_t, double> &TraversalMap,
-                     uint64_t &TraversalCount) {
-  auto DistMap = getPositionMap(Function);
-
-  for (auto const &Path : AllTraversals) {
-    TraversalMap[++TraversalCount] = getTraversalLength(DistMap, Path);
-  }
-}
-}
-
-void CalcCacheMetrics::calcGraphDistance(
-    const std::map<uint64_t, BinaryFunction> &BinaryFunctions) {
-
-  double TotalFuncValue = 0;
-  uint64_t FuncCount = 0;
-  for (auto &BFI : BinaryFunctions) {
-    auto &Function = BFI.second;
+/// Calculate average number of call distance for every graph traversal
+double calcGraphDistance(const std::vector<BinaryFunction *> &BinaryFunctions) {
+  double TotalTraversalLength = 0;
+  double NumTraversals = 0;
+  for (auto BF : BinaryFunctions) {
     // Only consider functions which are known to be executed
-    if (Function.getKnownExecutionCount() == 0)
+    if (BF->getKnownExecutionCount() == 0)
       continue;
 
-    std::unordered_map<uint64_t, double> TraversalMap;
-    uint64_t TraversalCount = 0;
-    for (auto *BB : Function.layout()) {
+    for (auto BB : BF->layout()) {
       if (BB->isEntryPoint()) {
-        auto AllTraversals = getTraversals(Function, BB);
-        graphDistHelper(AllTraversals, Function, TraversalMap, TraversalCount);
+        auto AllTraversals = getTraversals(BB);
+        auto DistMap = getPositionMap(*BF);
+        for (auto const &Path : AllTraversals) {
+          // Ignore short traversals
+          if (Path.size() <= 1)
+            continue;
+          TotalTraversalLength += getTraversalLength(DistMap, Path);
+          NumTraversals++;
+        }
       }
     }
-
-    double TotalValue = 0;
-    for (auto const &Entry : TraversalMap) {
-      TotalValue += Entry.second;
-    }
-
-    double AverageValue =
-        TraversalMap.empty() ? 0 : (TotalValue * 1.0 / TraversalMap.size());
-    TotalFuncValue += AverageValue;
-    FuncCount += TraversalMap.empty() ? 0 : 1;
   }
 
-  outs() << format("           Sum of averages of traversal distance for all "
-                   "functions is: %.2f\n",
-                   TotalFuncValue)
-         << format("           There are %u functions in total\n", FuncCount)
-         << format("           On average, every traversal is %.2f long\n\n",
-                   TotalFuncValue / FuncCount);
+  return TotalTraversalLength / NumTraversals;
+}
+
+}
+
+void CacheMetrics::printAll(
+  const std::vector<BinaryFunction *> &BinaryFunctions) {
+
+  size_t NumFunctions = 0;
+  size_t NumHotFunctions = 0;
+  size_t NumBlocks = 0;
+  size_t NumHotBlocks = 0;
+
+  for (auto BF : BinaryFunctions) {
+    NumFunctions++;
+    if (BF->getKnownExecutionCount() > 0)
+      NumHotFunctions++;
+    for (auto BB : BF->layout()) {
+      NumBlocks++;
+      if (BB->getKnownExecutionCount() > 0)
+        NumHotBlocks++;
+    }
+  }
+
+  outs() << format("  There are %zu functions;", NumFunctions)
+         << format(" %zu (%.2lf%%) have non-empty execution count\n",
+                   NumHotFunctions, 100.0 * NumHotFunctions / NumFunctions);
+  outs() << format("  There are %zu basic blocks;", NumBlocks)
+         << format(" %zu (%.2lf%%) have non-empty execution count\n",
+                  NumHotBlocks, 100.0 * NumHotBlocks / NumBlocks);
+
+  const auto GraphDistance = calcGraphDistance(BinaryFunctions);
+  outs() << "  An average length of graph traversal is "
+         << format("%.2lf\n", GraphDistance);
 }

bolt/CacheMetrics.h

@@ -0,0 +1,27 @@
+//===- CacheMetrics.h - Interface for instruction cache evaluation       --===//
+//
+//                     Functions to show metrics of cache lines
+//
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TOOLS_LLVM_BOLT_CACHEMETRICS_H
+#define LLVM_TOOLS_LLVM_BOLT_CACHEMETRICS_H
+
+#include "BinaryFunction.h"
+#include <map>
+
+namespace llvm {
+namespace bolt {
+namespace CacheMetrics {
+
+/// Calculate and print various metrics related to instruction cache performance
+void printAll(const std::vector<BinaryFunction *> &BinaryFunctions);
+
+} // namespace CacheMetrics
+} // namespace bolt
+} // namespace llvm
+
+#endif //LLVM_CACHEMETRICS_H

bolt/CalcCacheMetrics.h

@@ -1,27 +0,0 @@
-//===- CalcCacheMetrics.h - Interface for metrics printing of cache lines --===//
-//
-//                     Functions to show metrics of cache lines
-//
-//
-//===----------------------------------------------------------------------===//
-//
-//===----------------------------------------------------------------------===//
-
-
-#ifndef LLVM_CALCCACHEMETRICS_H
-#define LLVM_CALCCACHEMETRICS_H
-
-#include "BinaryFunction.h"
-#include <map>
-
-using namespace llvm;
-using namespace object;
-using namespace bolt;
-
-namespace CalcCacheMetrics {
-/// Calculate average number of call distance for every graph traversal.
-void calcGraphDistance(
-    const std::map<uint64_t, BinaryFunction> &BinaryFunctions);
-}
-
-#endif //LLVM_CALCCACHEMETRICS_H

bolt/RewriteInstance.cpp

@@ -14,7 +14,7 @@
 #include "BinaryContext.h"
 #include "BinaryFunction.h"
 #include "BinaryPassManager.h"
-#include "CalcCacheMetrics.h"
+#include "CacheMetrics.h"
 #include "DataAggregator.h"
 #include "DataReader.h"
 #include "Exceptions.h"
@@ -79,8 +79,8 @@ extern cl::opt<JumpTableSupportLevel> JumpTables;
 extern cl::opt<BinaryFunction::ReorderType> ReorderFunctions;
 
 static cl::opt<bool>
-CalcCacheMetrics("calc-cache-metrics",
-  cl::desc("calculate metrics of cache lines"),
+PrintCacheMetrics("print-cache-metrics",
+  cl::desc("calculate and print various metrics for instruction cache"),
   cl::init(false),
   cl::ZeroOrMore,
   cl::cat(BoltOptCategory));
@@ -920,12 +920,6 @@ void RewriteInstance::run() {
     executeRewritePass(LargeFunctions);
   }
 
-  if (opts::CalcCacheMetrics) {
-    outs() << "\nBOLT-INFO: After Optimization CFG Graph Statistics: Jump "
-            "Distance \n\n";
-    CalcCacheMetrics::calcGraphDistance(BinaryFunctions);
-  }
-
   if (opts::UpdateDebugSections)
     updateDebugInfo();
 
@@ -2096,12 +2090,6 @@ void RewriteInstance::disassembleFunctions() {
       }
     }
   }
-
-  if (opts::CalcCacheMetrics) {
-    outs() << "\nBOLT-INFO: Before Optimization CFG Graph Statistics: Jump "
-            "Distance \n\n";
-    CalcCacheMetrics::calcGraphDistance(BinaryFunctions);
-  }
 }
 
 void RewriteInstance::runOptimizationPasses() {
@@ -2424,6 +2412,11 @@ void RewriteInstance::emitFunctions() {
 
   OLT.emitAndFinalize(ObjectsHandle);
 
+  if (opts::PrintCacheMetrics) {
+    outs() << "BOLT-INFO: cache metrics after optimization\n";
+    CacheMetrics::printAll(SortedFunctions);
+  }
+
   if (opts::KeepTmp)
     TempOut->keep();
 }