[analyzer] UnixAPI: Check that the third argument to open(2) (if present) is an integer.

Patch by Daniel Fahlgren.

llvm-svn: 216079

clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp

@@ -95,6 +95,15 @@ void UnixAPIChecker::CheckOpen(CheckerContext &C, const CallExpr *CE) const {
     // The frontend should issue a warning for this case, so this is a sanity
     // check.
     return;
+  } else if (CE->getNumArgs() == 3) {
+    const Expr *Arg = CE->getArg(2);
+    QualType QT = Arg->getType();
+    if (!QT->isIntegerType()) {
+      ReportOpenBug(C, state,
+                    "Third argument to 'open' is not an integer",
+                    Arg->getSourceRange());
+      return;
+    }
   } else if (CE->getNumArgs() > 3) {
     ReportOpenBug(C, state,
                   "Call to 'open' with more than three arguments",

clang/test/Analysis/unix-api.c

@@ -25,3 +25,51 @@ void open_2(const char *path) {
   if (fd > -1)
     close(fd);
 }
+
+void open_3(const char *path) {
+  int fd;
+  fd = open(path, O_RDONLY, NULL); // expected-warning{{Third argument to 'open' is not an integer}}
+  if (fd > -1)
+    close(fd);
+}
+
+void open_4(const char *path) {
+  int fd;
+  fd = open(path, O_RDONLY, ""); // expected-warning{{Third argument to 'open' is not an integer}}
+  if (fd > -1)
+    close(fd);
+}
+
+void open_5(const char *path) {
+  int fd;
+  struct {
+    int val;
+  } st = {0};
+  fd = open(path, O_RDONLY, st); // expected-warning{{Third argument to 'open' is not an integer}}
+  if (fd > -1)
+    close(fd);
+}
+
+void open_6(const char *path) {
+  int fd;
+  struct {
+    int val;
+  } st = {0};
+  fd = open(path, O_RDONLY, st.val); // no-warning
+  if (fd > -1)
+    close(fd);
+}
+
+void open_7(const char *path) {
+  int fd;
+  fd = open(path, O_RDONLY, &open); // expected-warning{{Third argument to 'open' is not an integer}}
+  if (fd > -1)
+    close(fd);
+}
+
+void open_8(const char *path) {
+  int fd;
+  fd = open(path, O_RDONLY, 0.0f); // expected-warning{{Third argument to 'open' is not an integer}}
+  if (fd > -1)
+    close(fd);
+}