[libFuzzer] add exeprimental -rss_limit_mb flag to fight against OOMs

llvm-svn: 268807

llvm/lib/Fuzzer/FuzzerDriver.cpp

@@ -234,6 +234,21 @@ static int RunInMultipleProcesses(const std::vector<std::string> &Args,
   return HasErrors ? 1 : 0;
 }
 
+static void RssThread(Fuzzer *F, size_t RssLimitMb) {
+  while (true) {
+    std::this_thread::sleep_for(std::chrono::seconds(1));
+    size_t Peak = GetPeakRSSMb();
+    if (Peak > RssLimitMb)
+      F->RssLimitCallback(Peak, RssLimitMb);
+  }
+}
+
+static void StartRssThread(Fuzzer *F, size_t RssLimitMb) {
+  if (!RssLimitMb) return;
+  std::thread T(RssThread, F, RssLimitMb);
+  T.detach();
+}
+
 int RunOneTest(Fuzzer *F, const char *InputFilePath) {
   Unit U = FileToVector(InputFilePath);
   Unit PreciseSizedU(U);
@@ -331,6 +346,8 @@ static int FuzzerDriver(const std::vector<std::string> &Args,
     if (U.size() <= Word::GetMaxSize())
       MD.AddWordToManualDictionary(Word(U.data(), U.size()));
 
+  StartRssThread(&F, Flags.rss_limit_mb);
+
   // Timer
   if (Flags.timeout > 0)
     SetTimer(Flags.timeout / 2 + 1);

llvm/lib/Fuzzer/FuzzerFlags.def

@@ -81,6 +81,8 @@ FUZZER_FLAG_INT(close_fd_mask, 0, "If 1, close stdout at startup; "
     "Be careful, this will also close e.g. asan's stderr/stdout.")
 FUZZER_FLAG_INT(detect_leaks, 1, "If 1, and if LeakSanitizer is enabled "
     "try to detect memory leaks during fuzzing (i.e. not only at shut down).")
+FUZZER_FLAG_INT(rss_limit_mb, 0, "If non-zero, the fuzzer will exit upon"
+    "reaching this limit of RSS memory usage.")
 
 FUZZER_DEPRECATED_FLAG(exit_on_first)
 FUZZER_DEPRECATED_FLAG(save_minimized_corpus)

llvm/lib/Fuzzer/FuzzerInternal.h

@@ -353,6 +353,7 @@ public:
   MutationDispatcher &GetMD() { return MD; }
   void PrintFinalStats();
   void SetMaxLen(size_t MaxLen);
+  void RssLimitCallback(size_t RssPeakMb, size_t RssLimitMb);
 
 private:
   void AlarmCallback();

llvm/lib/Fuzzer/FuzzerLoop.cpp

@@ -176,6 +176,17 @@ void Fuzzer::AlarmCallback() {
   }
 }
 
+void Fuzzer::RssLimitCallback(size_t RssPeakMb, size_t RssLimitMb) {
+  Printf("==%d== ERROR: libFuzzer: out-of-memory (used: %zdMb; limit: %zdMb)\n",
+         GetPid(), RssPeakMb, RssLimitMb);
+  Printf("*****************************************************************\n");
+  Printf("** Experimental! TODO: dump the stack trace and the reproducer **\n");
+  Printf("*****************************************************************\n");
+  Printf("SUMMARY: libFuzzer: out-of-memory\n");
+  PrintFinalStats();
+  _Exit(Options.ErrorExitCode); // Stop right now.
+}
+
 void Fuzzer::PrintStats(const char *Where, const char *End) {
   size_t ExecPerSec = execPerSec();
   if (Options.OutputCSV) {