[asan] properly report mmap failure

llvm-svn: 165214

compiler-rt/lib/asan/lit_tests/Linux/rlimit_mmap_test.cc

@@ -0,0 +1,16 @@
+// Check that we properly report mmap failure.
+// RUN: %clangxx_asan %s -o %t && %t 2>&1 | FileCheck %s
+#include <stdlib.h>
+#include <assert.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+
+static volatile void *x;
+
+int main(int argc, char **argv) {
+  struct rlimit mmap_resource_limit = { 0, 0 };
+  assert(0 == setrlimit(RLIMIT_AS, &mmap_resource_limit));
+  x = malloc(10000000);
+// CHECK: AddressSanitizer is unable to mmap
+  return 0;
+}

compiler-rt/lib/sanitizer_common/sanitizer_posix.cc

@@ -47,6 +47,14 @@ void *MmapOrDie(uptr size, const char *mem_type) {
                             PROT_READ | PROT_WRITE,
                             MAP_PRIVATE | MAP_ANON, -1, 0);
   if (res == (void*)-1) {
+    static int recursion_count;
+    if (recursion_count) {
+      // The Report() and CHECK calls below may call mmap recursively and fail.
+      // If we went into recursion, just die.
+      RawWrite("AddressSanitizer is unable to mmap\n");
+      Die();
+    }
+    recursion_count++;
     Report("ERROR: Failed to allocate 0x%zx (%zd) bytes of %s: %s\n",
            size, size, mem_type, strerror(errno));
     DumpProcessMap();