Make detect_invalid_pointer_pairs option to be tristate.

Summary:
With the change, one can choose not to report comparison (or subtraction)
of a pointer with nullptr pointer.

Reviewers: kcc, jakubjelinek, alekseyshl

Reviewed By: alekseyshl

Subscribers: kubamracek

Differential Revision: https://reviews.llvm.org/D41479

llvm-svn: 323995

compiler-rt/lib/asan/asan_flags.inc

@@ -136,9 +136,9 @@ ASAN_FLAG(
     "Android. ")
 ASAN_FLAG(
     int, detect_invalid_pointer_pairs, 0,
-    "If non-zero, try to detect operations like <, <=, >, >= and - on "
-    "invalid pointer pairs (e.g. when pointers belong to different objects). "
-    "The bigger the value the harder we try.")
+    "If >= 2, detect operations like <, <=, >, >= and - on invalid pointer "
+    "pairs (e.g. when pointers belong to different objects); "
+    "If == 1, detect invalid operations only when both pointers are non-null.")
 ASAN_FLAG(
     bool, detect_container_overflow, true,
     "If true, honor the container overflow annotations. See "

compiler-rt/lib/asan/asan_report.cc

@@ -343,7 +343,11 @@ static bool IsInvalidPointerPair(uptr a1, uptr a2) {
 }
 
 static INLINE void CheckForInvalidPointerPair(void *p1, void *p2) {
-  if (!flags()->detect_invalid_pointer_pairs) return;
+  switch (flags()->detect_invalid_pointer_pairs) {
+    case 0 : return;
+    case 1 : if (p1 == nullptr || p2 == nullptr) return; break;
+  }
+
   uptr a1 = reinterpret_cast<uptr>(p1);
   uptr a2 = reinterpret_cast<uptr>(p2);
 

compiler-rt/test/asan/TestCases/Posix/invalid-pointer-pairs-threads.cc

@@ -1,7 +1,7 @@
 // RUN: %clangxx_asan -O0 %s -pthread -o %t -mllvm -asan-detect-invalid-pointer-pair
 
-// RUN: %env_asan_opts=detect_invalid_pointer_pairs=1 %run %t a 2>&1 | FileCheck %s -check-prefix=OK -allow-empty
-// RUN: %env_asan_opts=detect_invalid_pointer_pairs=1 not %run %t b 2>&1 | FileCheck %s -check-prefix=B
+// RUN: %env_asan_opts=detect_invalid_pointer_pairs=2 %run %t a 2>&1 | FileCheck %s -check-prefix=OK -allow-empty
+// RUN: %env_asan_opts=detect_invalid_pointer_pairs=2 not %run %t b 2>&1 | FileCheck %s -check-prefix=B
 
 // pthread barriers are not available on OS X
 // UNSUPPORTED: darwin

compiler-rt/test/asan/TestCases/invalid-pointer-pairs-compare-errors.cc

@@ -1,6 +1,6 @@
 // RUN: %clangxx_asan -O0 %s -o %t -mllvm -asan-detect-invalid-pointer-pair
 
-// RUN: %env_asan_opts=detect_invalid_pointer_pairs=1:halt_on_error=0 %run %t 2>&1 | FileCheck %s
+// RUN: %env_asan_opts=detect_invalid_pointer_pairs=2:halt_on_error=0 %run %t 2>&1 | FileCheck %s
 
 #include <assert.h>
 #include <stdlib.h>

compiler-rt/test/asan/TestCases/invalid-pointer-pairs-compare-null.cc

@@ -0,0 +1,42 @@
+// RUN: %clangxx_asan -O0 %s -o %t -mllvm -asan-detect-invalid-pointer-pair
+
+// RUN: %env_asan_opts=detect_invalid_pointer_pairs=1 %run %t
+
+#include <assert.h>
+#include <stdlib.h>
+
+int foo(char *p, char *q) {
+  return p <= q;
+}
+
+char global[8192] = {};
+char small_global[7] = {};
+
+int main() {
+  // Heap allocated memory.
+  char *p = (char *)malloc(42);
+  int r = foo(p, nullptr);
+  free(p);
+
+  p = (char *)malloc(1024);
+  foo(nullptr, p);
+  free(p);
+
+  p = (char *)malloc(4096);
+  foo(p, nullptr);
+  free(p);
+
+  // Global variable.
+  foo(&global[0], nullptr);
+  foo(&global[1000], nullptr);
+
+  p = &small_global[0];
+  foo(p, nullptr);
+
+  // Stack variable.
+  char stack[10000];
+  foo(&stack[0], nullptr);
+  foo(nullptr, &stack[9000]);
+
+  return 0;
+}

compiler-rt/test/asan/TestCases/invalid-pointer-pairs-compare-success.cc

@@ -1,6 +1,6 @@
 // RUN: %clangxx_asan -O0 %s -o %t -mllvm -asan-detect-invalid-pointer-pair
 
-// RUN: %env_asan_opts=detect_invalid_pointer_pairs=1 %run %t
+// RUN: %env_asan_opts=detect_invalid_pointer_pairs=2 %run %t
 
 #include <assert.h>
 #include <stdlib.h>

compiler-rt/test/asan/TestCases/invalid-pointer-pairs-subtract-errors.cc

@@ -1,6 +1,6 @@
 // RUN: %clangxx_asan -O0 %s -o %t -mllvm -asan-detect-invalid-pointer-pair
 
-// RUN: %env_asan_opts=detect_invalid_pointer_pairs=1:halt_on_error=0 %run %t 2>&1 | FileCheck %s
+// RUN: %env_asan_opts=detect_invalid_pointer_pairs=2:halt_on_error=0 %run %t 2>&1 | FileCheck %s
 
 #include <assert.h>
 #include <stdlib.h>

compiler-rt/test/asan/TestCases/invalid-pointer-pairs-subtract-success.cc

@@ -1,6 +1,6 @@
 // RUN: %clangxx_asan -O0 %s -o %t -mllvm -asan-detect-invalid-pointer-pair
 
-// RUN: %env_asan_opts=detect_invalid_pointer_pairs=1 %run %t
+// RUN: %env_asan_opts=detect_invalid_pointer_pairs=2 %run %t
 
 #include <assert.h>
 #include <stdlib.h>