[libFuzzer] Implement stat::stability_rate based on the percentage of unstable edges.

Summary:
Created a -print_unstable_stats flag.
When -print_unstable_stats=1, we run it 2 more times on interesting inputs poisoning unstable edges in an array.
On program termination, we run PrintUnstableStats() which will print a line with a stability percentage like AFL does.

Patch by Kyungtak Woo (@kevinwkt).

Reviewers: metzman, Dor1s, kcc, morehouse

Reviewed By: metzman, Dor1s, morehouse

Subscribers: delcypher, llvm-commits, #sanitizers, kcc, morehouse, Dor1s

Differential Revision: https://reviews.llvm.org/D49212

llvm-svn: 337175

compiler-rt/lib/fuzzer/FuzzerDriver.cpp

@@ -102,14 +102,14 @@ static void PrintHelp() {
     Printf("%d\t%s\n", D.Default, D.Description);
   }
   Printf("\nFlags starting with '--' will be ignored and "
-            "will be passed verbatim to subprocesses.\n");
+         "will be passed verbatim to subprocesses.\n");
 }
 
 static const char *FlagValue(const char *Param, const char *Name) {
   size_t Len = strlen(Name);
   if (Param[0] == '-' && strstr(Param + 1, Name) == Param + 1 &&
       Param[Len + 1] == '=')
-      return &Param[Len + 2];
+    return &Param[Len + 2];
   return nullptr;
 }
 
@@ -302,10 +302,10 @@ static std::string GetDedupTokenFromFile(const std::string &Path) {
 }
 
 int CleanseCrashInput(const Vector<std::string> &Args,
-                       const FuzzingOptions &Options) {
+                      const FuzzingOptions &Options) {
   if (Inputs->size() != 1 || !Flags.exact_artifact_path) {
     Printf("ERROR: -cleanse_crash should be given one input file and"
-          " -exact_artifact_path\n");
+           " -exact_artifact_path\n");
     exit(1);
   }
   std::string InputFilePath = Inputs->at(0);
@@ -520,7 +520,7 @@ int AnalyzeDictionary(Fuzzer *F, const Vector<Unit>& Dict,
   for (size_t i = 0; i < Dict.size(); ++i) {
     // Dictionary units with positive score are treated as useful ones.
     if (Scores[i] > 0)
-       continue;
+      continue;
 
     Printf("\"");
     PrintASCII(Dict[i].data(), Dict[i].size(), "\"");
@@ -617,6 +617,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
   Options.PrintFinalStats = Flags.print_final_stats;
   Options.PrintCorpusStats = Flags.print_corpus_stats;
   Options.PrintCoverage = Flags.print_coverage;
+  Options.PrintUnstableStats = Flags.print_unstable_stats;
   Options.DumpCoverage = Flags.dump_coverage;
   if (Flags.exit_on_src_pos)
     Options.ExitOnSrcPos = Flags.exit_on_src_pos;

compiler-rt/lib/fuzzer/FuzzerFlags.def

@@ -110,6 +110,8 @@ FUZZER_FLAG_INT(print_coverage, 0, "If 1, print coverage information as text"
 FUZZER_FLAG_INT(dump_coverage, 0, "Deprecated."
                                   " If 1, dump coverage information as a"
                                   " .sancov file at exit.")
+FUZZER_FLAG_INT(print_unstable_stats, 0, "Experimental."
+				  " If 1, print unstable statistics at exit.")
 FUZZER_FLAG_INT(handle_segv, 1, "If 1, try to intercept SIGSEGV.")
 FUZZER_FLAG_INT(handle_bus, 1, "If 1, try to intercept SIGBUS.")
 FUZZER_FLAG_INT(handle_abrt, 1, "If 1, try to intercept SIGABRT.")

compiler-rt/lib/fuzzer/FuzzerInternal.h

@@ -67,6 +67,7 @@ public:
   static void StaticGracefulExitCallback();
 
   void ExecuteCallback(const uint8_t *Data, size_t Size);
+  void CheckForUnstableCounters(const uint8_t *Data, size_t Size);
   bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false,
               InputInfo *II = nullptr, bool *FoundUniqFeatures = nullptr);
 

compiler-rt/lib/fuzzer/FuzzerLoop.cpp

@@ -352,6 +352,8 @@ void Fuzzer::PrintStats(const char *Where, const char *End, size_t Units) {
 void Fuzzer::PrintFinalStats() {
   if (Options.PrintCoverage)
     TPC.PrintCoverage();
+  if (Options.PrintUnstableStats)
+    TPC.PrintUnstableStats();
   if (Options.DumpCoverage)
     TPC.DumpCoverage();
   if (Options.PrintCorpusStats)
@@ -444,6 +446,29 @@ void Fuzzer::PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size) {
   }
 }
 
+void Fuzzer::CheckForUnstableCounters(const uint8_t *Data, size_t Size) {
+  auto CBSetupAndRun = [&]() {
+    ScopedEnableMsanInterceptorChecks S;
+    UnitStartTime = system_clock::now();
+    TPC.ResetMaps();
+    RunningCB = true;
+    CB(Data, Size);
+    RunningCB = false;
+    UnitStopTime = system_clock::now();
+  };
+
+  // Copy original run counters into our unstable counters
+  TPC.InitializeUnstableCounters();
+
+  // First Rerun
+  CBSetupAndRun();
+  TPC.UpdateUnstableCounters();
+
+  // Second Rerun
+  CBSetupAndRun();
+  TPC.UpdateUnstableCounters();
+}
+
 bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile,
                     InputInfo *II, bool *FoundUniqFeatures) {
   if (!Size)
@@ -466,6 +491,12 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile,
     *FoundUniqFeatures = FoundUniqFeaturesOfII;
   PrintPulseAndReportSlowInput(Data, Size);
   size_t NumNewFeatures = Corpus.NumFeatureUpdates() - NumUpdatesBefore;
+
+  // If print_unstable_stats, execute the same input two more times to detect
+  // unstable edges.
+  if (NumNewFeatures && Options.PrintUnstableStats)
+    CheckForUnstableCounters(Data, Size);
+
   if (NumNewFeatures) {
     TPC.UpdateObservedPCs();
     Corpus.AddToCorpus({Data, Data + Size}, NumNewFeatures, MayDeleteFile,
@@ -669,7 +700,7 @@ void Fuzzer::MutateAndTestOne() {
       break;  // We will mutate this input more in the next rounds.
     }
     if (Options.ReduceDepth && !FoundUniqFeatures)
-        break;
+      break;
   }
 }
 

compiler-rt/lib/fuzzer/FuzzerOptions.h

@@ -54,6 +54,7 @@ struct FuzzingOptions {
   bool PrintFinalStats = false;
   bool PrintCorpusStats = false;
   bool PrintCoverage = false;
+  bool PrintUnstableStats = false;
   bool DumpCoverage = false;
   bool DetectLeaks = true;
   int PurgeAllocatorIntervalSec = 1;

compiler-rt/lib/fuzzer/FuzzerTracePC.cpp

@@ -59,6 +59,37 @@ size_t TracePC::GetTotalPCCoverage() {
   return Res;
 }
 
+// Initializes unstable counters by copying Inline8bitCounters to unstable
+// counters.
+void TracePC::InitializeUnstableCounters() {
+  if (NumInline8bitCounters && NumInline8bitCounters == NumPCsInPCTables) {
+    size_t UnstableIdx = 0;
+    for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) {
+      uint8_t *Beg = ModuleCounters[i].Start;
+      size_t Size = ModuleCounters[i].Stop - Beg;
+      assert(Size == (size_t)(ModulePCTable[i].Stop - ModulePCTable[i].Start));
+      for (size_t j = 0; j < Size; j++, UnstableIdx++)
+        if (UnstableCounters[UnstableIdx] != kUnstableCounter)
+          UnstableCounters[UnstableIdx] = Beg[j];
+    }
+  }
+}
+
+// Compares the current counters with counters from previous runs
+// and records differences as unstable edges.
+void TracePC::UpdateUnstableCounters() {
+  if (NumInline8bitCounters && NumInline8bitCounters == NumPCsInPCTables) {
+    size_t UnstableIdx = 0;
+    for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) {
+      uint8_t *Beg = ModuleCounters[i].Start;
+      size_t Size = ModuleCounters[i].Stop - Beg;
+      assert(Size == (size_t)(ModulePCTable[i].Stop - ModulePCTable[i].Start));
+      for (size_t j = 0; j < Size; j++, UnstableIdx++)
+        if (Beg[j] != UnstableCounters[UnstableIdx])
+          UnstableCounters[UnstableIdx] = kUnstableCounter;
+    }
+  }
+}
 
 void TracePC::HandleInline8bitCountersInit(uint8_t *Start, uint8_t *Stop) {
   if (Start == Stop) return;
@@ -310,6 +341,15 @@ void TracePC::DumpCoverage() {
   }
 }
 
+void TracePC::PrintUnstableStats() {
+  size_t count = 0;
+  for (size_t i = 0; i < NumInline8bitCounters; i++)
+    if (UnstableCounters[i] == kUnstableCounter)
+      count++;
+  Printf("stat::stability_rate: %.2f\n",
+         100 - static_cast<float>(count * 100) / NumInline8bitCounters);
+}
+
 // Value profile.
 // We keep track of various values that affect control flow.
 // These values are inserted into a bit-set-based hash map.
@@ -356,9 +396,9 @@ void TracePC::HandleCmp(uintptr_t PC, T Arg1, T Arg2) {
   uint64_t ArgDistance = __builtin_popcountll(ArgXor) + 1; // [1,65]
   uintptr_t Idx = ((PC & 4095) + 1) * ArgDistance;
   if (sizeof(T) == 4)
-      TORC4.Insert(ArgXor, Arg1, Arg2);
+    TORC4.Insert(ArgXor, Arg1, Arg2);
   else if (sizeof(T) == 8)
-      TORC8.Insert(ArgXor, Arg1, Arg2);
+    TORC8.Insert(ArgXor, Arg1, Arg2);
   // TODO: remove these flags and instead use all metrics at once.
   if (UseValueProfileMask & 1)
     ValueProfileMap.AddValue(Idx);
@@ -589,7 +629,7 @@ void __sanitizer_weak_hook_strncmp(void *caller_pc, const char *s1,
 
 ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY
 void __sanitizer_weak_hook_strcmp(void *caller_pc, const char *s1,
-                                   const char *s2, int result) {
+                                  const char *s2, int result) {
   if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return;
   if (result == 0) return;  // No reason to mutate.
   size_t N = fuzzer::InternalStrnlen2(s1, s2);

compiler-rt/lib/fuzzer/FuzzerTracePC.h

@@ -68,7 +68,7 @@ struct MemMemTable {
 };
 
 class TracePC {
- public:
+public:
   static const size_t kNumPCs = 1 << 21;
   // How many bits of PC are used from __sanitizer_cov_trace_pc.
   static const size_t kTracePcBits = 18;
@@ -103,6 +103,7 @@ class TracePC {
 
   void PrintCoverage();
   void DumpCoverage();
+  void PrintUnstableStats();
 
   template<class CallBack>
   void IterateCoveredFunctions(CallBack CB);
@@ -135,7 +136,17 @@ class TracePC {
   void SetFocusFunction(const std::string &FuncName);
   bool ObservedFocusFunction();
 
+  void InitializeUnstableCounters();
+  void UpdateUnstableCounters();
+
 private:
+  // Value used to represent unstable edge.
+  static constexpr int16_t kUnstableCounter = -1;
+
+  // Uses 16-bit signed type to be able to accommodate any possible value from
+  // uint8_t counter and -1 constant as well.
+  int16_t UnstableCounters[kNumPCs];
+
   bool UseCounters = false;
   uint32_t UseValueProfileMask = false;
   bool DoPrintNewPCs = false;
@@ -204,27 +215,27 @@ void ForEachNonZeroByte(const uint8_t *Begin, const uint8_t *End,
 // Given a non-zero Counter returns a number in the range [0,7].
 template<class T>
 unsigned CounterToFeature(T Counter) {
-    // Returns a feature number by placing Counters into buckets as illustrated
-    // below.
-    //
-    // Counter bucket: [1] [2] [3] [4-7] [8-15] [16-31] [32-127] [128+]
-    // Feature number:  0   1   2    3     4       5       6       7
-    //
-    // This is a heuristic taken from AFL (see
-    // http://lcamtuf.coredump.cx/afl/technical_details.txt).
-    //
-    // This implementation may change in the future so clients should
-    // not rely on it.
-    assert(Counter);
-    unsigned Bit = 0;
-    /**/ if (Counter >= 128) Bit = 7;
-    else if (Counter >= 32) Bit = 6;
-    else if (Counter >= 16) Bit = 5;
-    else if (Counter >= 8) Bit = 4;
-    else if (Counter >= 4) Bit = 3;
-    else if (Counter >= 3) Bit = 2;
-    else if (Counter >= 2) Bit = 1;
-    return Bit;
+  // Returns a feature number by placing Counters into buckets as illustrated
+  // below.
+  //
+  // Counter bucket: [1] [2] [3] [4-7] [8-15] [16-31] [32-127] [128+]
+  // Feature number:  0   1   2    3     4       5       6       7
+  //
+  // This is a heuristic taken from AFL (see
+  // http://lcamtuf.coredump.cx/afl/technical_details.txt).
+  //
+  // This implementation may change in the future so clients should
+  // not rely on it.
+  assert(Counter);
+  unsigned Bit = 0;
+   /**/ if (Counter >= 128) Bit = 7;
+   else if (Counter >= 32) Bit = 6;
+   else if (Counter >= 16) Bit = 5;
+   else if (Counter >= 8) Bit = 4;
+   else if (Counter >= 4) Bit = 3;
+   else if (Counter >= 3) Bit = 2;
+   else if (Counter >= 2) Bit = 1;
+  return Bit;
 }
 
 template <class Callback>  // void Callback(size_t Feature)

compiler-rt/test/fuzzer/PrintUnstableStatsTest.cpp

@@ -0,0 +1,69 @@
+#include <assert.h>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+
+int x = 0;
+bool skip0 = false;
+bool skip1 = false;
+bool skip2 = false;
+
+__attribute__((noinline)) void det0() { x++; }
+__attribute__((noinline)) void det1() { x++; }
+__attribute__((noinline)) void det2() { x++; }
+__attribute__((noinline)) void det3() { x++; }
+__attribute__((noinline)) void det4() { x++; }
+
+__attribute__((noinline)) void ini0() { x++; }
+__attribute__((noinline)) void ini1() { x++; }
+__attribute__((noinline)) void ini2() { x++; }
+
+__attribute__((noinline)) void t0() { x++; }
+__attribute__((noinline)) void t1() { x++; }
+__attribute__((noinline)) void t2() { x++; }
+__attribute__((noinline)) void t3() { x++; }
+__attribute__((noinline)) void t4() { x++; }
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
+  if (Size == 1 && Data[0] == 'A' && !skip0) {
+    skip0 = true;
+    ini0();
+  }
+  if (Size == 1 && Data[0] == 'B' && !skip1) {
+    skip1 = true;
+    ini1();
+  }
+  if (Size == 1 && Data[0] == 'C' && !skip2) {
+    skip2 = true;
+    ini2();
+  }
+
+  det0();
+  det1();
+  int a = rand();
+  det2();
+
+  switch (a % 5) {
+  case 0:
+    t0();
+    break;
+  case 1:
+    t1();
+    break;
+  case 2:
+    t2();
+    break;
+  case 3:
+    t3();
+    break;
+  case 4:
+    t4();
+    break;
+  default:
+    assert(false);
+  }
+
+  det3();
+  det4();
+  return 0;
+}

compiler-rt/test/fuzzer/print_unstable_stats.test

@@ -0,0 +1,3 @@
+RUN: %cpp_compiler %S/PrintUnstableStatsTest.cpp -o %t-PrintUnstableStatsTest
+RUN: %run %t-PrintUnstableStatsTest -print_unstable_stats=1 -runs=100000 2>&1 | FileCheck %s --check-prefix=LONG
+LONG: stat::stability_rate: 27.59