From c240169ff2e416f2652a5d3a80a83ef68ab7fa23 Mon Sep 17 00:00:00 2001
From: Sanjay Patel <spatel@rotateright.com>
Date: Wed, 22 Sep 2021 13:24:51 -0400
Subject: [PATCH] [Analysis] improve function matching for strlen libcall

The return type of strlen is size_t, not just any integer.

This is a partial fix for an example based on:
https://llvm.org/PR50836

There's another bug here because we can still crash
processing a real strlen or something that looks like it.
---
 llvm/lib/Analysis/TargetLibraryInfo.cpp              |  4 ++--
 llvm/test/Transforms/InstCombine/call-callconv.ll    |  5 +++++
 .../test/Transforms/InstCombine/simplify-libcalls.ll | 12 ++++++++++++
 3 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Analysis/TargetLibraryInfo.cpp b/llvm/lib/Analysis/TargetLibraryInfo.cpp
index 0a2031de5b89..a7027bcb0631 100644
--- a/llvm/lib/Analysis/TargetLibraryInfo.cpp
+++ b/llvm/lib/Analysis/TargetLibraryInfo.cpp
@@ -760,8 +760,8 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy,
       return false;
     LLVM_FALLTHROUGH;
   case LibFunc_strlen:
-    return (NumParams == 1 && FTy.getParamType(0)->isPointerTy() &&
-            FTy.getReturnType()->isIntegerTy());
+    return NumParams == 1 && FTy.getParamType(0)->isPointerTy() &&
+           IsSizeTTy(FTy.getReturnType());
 
   case LibFunc_strchr:
   case LibFunc_strrchr:
diff --git a/llvm/test/Transforms/InstCombine/call-callconv.ll b/llvm/test/Transforms/InstCombine/call-callconv.ll
index 58a0cf21b24e..48fdea4bb274 100644
--- a/llvm/test/Transforms/InstCombine/call-callconv.ll
+++ b/llvm/test/Transforms/InstCombine/call-callconv.ll
@@ -2,6 +2,11 @@
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 ; Verify that the non-default calling conv doesn't prevent the libcall simplification
 
+; Layout specifies type of pointer which determines "size_t"
+; which is used to identify libcalls such as "size_t strlen(char*)".
+
+target datalayout = "p:32:32"
+
 @.str = private unnamed_addr constant [4 x i8] c"abc\00", align 1
 
 define arm_aapcscc i32 @_abs(i32 %i) nounwind readnone {
diff --git a/llvm/test/Transforms/InstCombine/simplify-libcalls.ll b/llvm/test/Transforms/InstCombine/simplify-libcalls.ll
index f80286a8cc7e..e4db9d7a31e2 100644
--- a/llvm/test/Transforms/InstCombine/simplify-libcalls.ll
+++ b/llvm/test/Transforms/InstCombine/simplify-libcalls.ll
@@ -230,5 +230,17 @@ define i32 @fake_snprintf(i32 %buf, double %len, i32 * %str) {
   ret i32 %call
 }
 
+; Wrong return type for the real strlen.
+; https://llvm.org/PR50836
+
+define i4 @strlen(i8* %s) {
+; CHECK-LABEL: @strlen(
+; CHECK-NEXT:    [[R:%.*]] = call i4 @strlen(i8* [[S:%.*]])
+; CHECK-NEXT:    ret i4 0
+;
+  %r = call i4 @strlen(i8* %s)
+  ret i4 0
+}
+
 attributes #0 = { nobuiltin }
 attributes #1 = { builtin }