[LibFuzzer] Fix `-jobs=<N>` where <N> > 1 and the number of workers is > 1 on macOS.

The original `ExecuteCommand()` called `system()` from the C library.
The C library implementation of this on macOS contains a mutex which
serializes calls to `system()`. This prevented the `-jobs=` flag
from running copies of the fuzzing binary in parallel which is
the opposite of what is intended.

To fix this on macOS an alternative implementation of `ExecuteCommand()`
is provided that can be used concurrently. This is provided in
`FuzzerUtilDarwin.cpp` which is guarded to only compile code on Apple
platforms. The existing implementation has been moved to a new file
`FuzzerUtilLinux.cpp` which is guarded to only compile code on Linux.

This commit includes a simple test to check that LibFuzzer is being
executed in parallel when requested.

Differential Revision: https://reviews.llvm.org/D22742

llvm-svn: 278544

llvm/lib/Fuzzer/CMakeLists.txt

@@ -20,6 +20,8 @@ if( LLVM_USE_SANITIZE_COVERAGE )
     FuzzerSHA1.cpp
     FuzzerTracePC.cpp
     FuzzerUtil.cpp
+    FuzzerUtilDarwin.cpp
+    FuzzerUtilLinux.cpp
     )
   add_library(LLVMFuzzerNoMain STATIC
     $<TARGET_OBJECTS:LLVMFuzzerNoMainObjects>

llvm/lib/Fuzzer/FuzzerUtil.cpp

@@ -146,10 +146,6 @@ int NumberOfCpuCores() {
   return N;
 }
 
-int ExecuteCommand(const std::string &Command) {
-  return system(Command.c_str());
-}
-
 bool ToASCII(uint8_t *Data, size_t Size) {
   bool Changed = false;
   for (size_t i = 0; i < Size; i++) {

llvm/lib/Fuzzer/FuzzerUtilDarwin.cpp

@@ -0,0 +1,148 @@
+//===- FuzzerUtilDarwin.cpp - Misc utils ----------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// Misc utils for Darwin.
+//===----------------------------------------------------------------------===//
+#include "FuzzerInternal.h"
+#if LIBFUZZER_APPLE
+#include <mutex>
+#include <signal.h>
+#include <spawn.h>
+#include <sys/wait.h>
+
+// There is no header for this on macOS so declare here
+extern "C" char **environ;
+
+namespace fuzzer {
+
+static std::mutex SignalMutex;
+// Global variables used to keep track of how signal handling should be
+// restored. They should **not** be accessed without holding `SignalMutex`.
+static int ActiveThreadCount = 0;
+static struct sigaction OldSigIntAction;
+static struct sigaction OldSigQuitAction;
+static sigset_t OldBlockedSignalsSet;
+
+// This is a reimplementation of Libc's `system()`. On Darwin the Libc
+// implementation contains a mutex which prevents it from being used
+// concurrently. This implementation **can** be used concurrently. It sets the
+// signal handlers when the first thread enters and restores them when the last
+// thread finishes execution of the function and ensures this is not racey by
+// using a mutex.
+int ExecuteCommand(const std::string &Command) {
+  posix_spawnattr_t SpawnAttributes;
+  if (posix_spawnattr_init(&SpawnAttributes))
+    return -1;
+  // Block and ignore signals of the current process when the first thread
+  // enters.
+  {
+    std::lock_guard<std::mutex> Lock(SignalMutex);
+    if (ActiveThreadCount == 0) {
+      static struct sigaction IgnoreSignalAction;
+      sigset_t BlockedSignalsSet;
+      memset(&IgnoreSignalAction, 0, sizeof(IgnoreSignalAction));
+      IgnoreSignalAction.sa_handler = SIG_IGN;
+
+      if (sigaction(SIGINT, &IgnoreSignalAction, &OldSigIntAction) == -1) {
+        Printf("Failed to ignore SIGINT\n");
+        (void)posix_spawnattr_destroy(&SpawnAttributes);
+        return -1;
+      }
+      if (sigaction(SIGQUIT, &IgnoreSignalAction, &OldSigQuitAction) == -1) {
+        Printf("Failed to ignore SIGQUIT\n");
+        // Try our best to restore the signal handlers.
+        (void)sigaction(SIGINT, &OldSigIntAction, NULL);
+        (void)posix_spawnattr_destroy(&SpawnAttributes);
+        return -1;
+      }
+
+      (void)sigemptyset(&BlockedSignalsSet);
+      (void)sigaddset(&BlockedSignalsSet, SIGCHLD);
+      if (sigprocmask(SIG_BLOCK, &BlockedSignalsSet, &OldBlockedSignalsSet) ==
+          -1) {
+        Printf("Failed to block SIGCHLD\n");
+        // Try our best to restore the signal handlers.
+        (void)sigaction(SIGQUIT, &OldSigQuitAction, NULL);
+        (void)sigaction(SIGINT, &OldSigIntAction, NULL);
+        (void)posix_spawnattr_destroy(&SpawnAttributes);
+        return -1;
+      }
+    }
+    ++ActiveThreadCount;
+  }
+
+  // NOTE: Do not introduce any new `return` statements past this
+  // point. It is important that `ActiveThreadCount` always be decremented
+  // when leaving this function.
+
+  // Make sure the child process uses the default handlers for the
+  // following signals rather than inheriting what the parent has.
+  sigset_t DefaultSigSet;
+  (void)sigemptyset(&DefaultSigSet);
+  (void)sigaddset(&DefaultSigSet, SIGQUIT);
+  (void)sigaddset(&DefaultSigSet, SIGINT);
+  (void)posix_spawnattr_setsigdefault(&SpawnAttributes, &DefaultSigSet);
+  // Make sure the child process doesn't block SIGCHLD
+  (void)posix_spawnattr_setsigmask(&SpawnAttributes, &OldBlockedSignalsSet);
+  short SpawnFlags = POSIX_SPAWN_SETSIGDEF | POSIX_SPAWN_SETSIGMASK;
+  (void)posix_spawnattr_setflags(&SpawnAttributes, SpawnFlags);
+
+  pid_t Pid;
+  char **Environ = environ; // Read from global
+  const char *CommandCStr = Command.c_str();
+  const char *Argv[] = {"sh", "-c", CommandCStr, NULL};
+  int ErrorCode = 0, ProcessStatus = 0;
+  // FIXME: We probably shouldn't hardcode the shell path.
+  ErrorCode = posix_spawn(&Pid, "/bin/sh", NULL, &SpawnAttributes,
+                          (char *const *)Argv, Environ);
+  (void)posix_spawnattr_destroy(&SpawnAttributes);
+  if (!ErrorCode) {
+    pid_t SavedPid = Pid;
+    do {
+      // Repeat until call completes uninterrupted.
+      Pid = waitpid(SavedPid, &ProcessStatus, /*options=*/0);
+    } while (Pid == -1 && errno == EINTR);
+    if (Pid == -1) {
+      // Fail for some other reason.
+      ProcessStatus = -1;
+    }
+  } else if (ErrorCode == ENOMEM || ErrorCode == EAGAIN) {
+    // Fork failure.
+    ProcessStatus = -1;
+  } else {
+    // Shell execution failure.
+    ProcessStatus = W_EXITCODE(127, 0);
+  }
+
+  // Restore the signal handlers of the current process when the last thread
+  // using this function finishes.
+  {
+    std::lock_guard<std::mutex> Lock(SignalMutex);
+    --ActiveThreadCount;
+    if (ActiveThreadCount == 0) {
+      bool FailedRestore = false;
+      if (sigaction(SIGINT, &OldSigIntAction, NULL) == -1) {
+        Printf("Failed to restore SIGINT handling\n");
+        FailedRestore = true;
+      }
+      if (sigaction(SIGQUIT, &OldSigQuitAction, NULL) == -1) {
+        Printf("Failed to restore SIGQUIT handling\n");
+        FailedRestore = true;
+      }
+      if (sigprocmask(SIG_BLOCK, &OldBlockedSignalsSet, NULL) == -1) {
+        Printf("Failed to unblock SIGCHLD\n");
+        FailedRestore = true;
+      }
+      if (FailedRestore)
+        ProcessStatus = -1;
+    }
+  }
+  return ProcessStatus;
+}
+}
+#endif // LIBFUZZER_APPLE

llvm/lib/Fuzzer/FuzzerUtilLinux.cpp

@@ -0,0 +1,19 @@
+//===- FuzzerUtilLinux.cpp - Misc utils -----------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// Misc utils for Linux.
+//===----------------------------------------------------------------------===//
+#include "FuzzerInternal.h"
+#if LIBFUZZER_LINUX
+#include <stdlib.h>
+namespace fuzzer {
+int ExecuteCommand(const std::string &Command) {
+  return system(Command.c_str());
+}
+}
+#endif // LIBFUZZER_LINUX

llvm/lib/Fuzzer/test/fuzzer-jobs.test

@@ -0,0 +1,29 @@
+RUN: rm -rf %tmp
+RUN: mkdir %tmp && cd %tmp
+# Create a shared corpus directory
+RUN: rm -rf FuzzerJobsTestCORPUS
+RUN: mkdir FuzzerJobsTestCORPUS
+RUN: rm -f fuzz-{0,1}.log
+# Start fuzzer and in parallel check that the output files
+# that should be created exist.
+RUN: LLVMFuzzer-EmptyTest -max_total_time=4 -jobs=2 -workers=2 FuzzerJobsTestCORPUS > %t-fuzzer-jobs-test.log 2>&1 & export FUZZER_PID=$!
+# Wait a short while to give time for the child processes
+# to start fuzzing
+RUN: sleep 1
+# If the instances are running in parallel they should have created their log
+# files by now.
+RUN: ls fuzz-0.log
+RUN: ls fuzz-1.log
+# Wait for libfuzzer to finish.
+# This probably isn't portable but we need a way to block until
+# the fuzzer is done otherwise we might remove the files while
+# they are being used.
+RUN: while kill -0 ${FUZZER_PID}; do : ; done
+RUN: rm -f fuzz-{0,1}.log
+RUN: rm -rf FuzzerJobsTestCORPUS
+RUN: FileCheck -input-file=%t-fuzzer-jobs-test.log %s
+RUN: rm %t-fuzzer-jobs-test.log
+RUN: cd ../
+
+CHECK-DAG: Job 0 exited with exit code 0
+CHECK-DAG: Job 1 exited with exit code 0