maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity
llvm-project/llvm/test/Transforms/LoopPredication/predicate-exits.ll

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

1111 lines
48 KiB
LLVM
Raw Normal View History

[LoopPred/WC] Use a dominating widenable condition to remove analyze loop exits This implements a version of the predicateLoopExits transform from IndVarSimplify extended to exploit widenable conditions - and thus be much wider in scope of legality. The code structure ends up being almost entirely different, so I chose to duplicate this into the LoopPredication pass instead of trying to reuse the code in the IndVars. The core notions of the transform are as follows: If we have a widenable condition which controls entry into the loop, we're allowed to widen it arbitrarily. Given that, it's simply a *profitability* question as to what conditions to fold into the widenable branch. To avoid pass ordering issues, we want to avoid widening cases that would otherwise be dischargeable. Or... widen in a form which can still be discharged. Thus, we phrase the transform as selecting one analyzeable exit from the set of analyzeable exits to keep. This avoids creating pass ordering complexities. Since none of the above proves that we actually exit through our analyzeable exits - we might exit through something else entirely - we limit ourselves to cases where a) the latch is analyzeable and b) the latch is predicted taken, and c) the exit being removed is statically cold. Differential Revision: https://reviews.llvm.org/D69830
2019-11-19 03:21:53 +08:00
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -loop-predication -S | FileCheck %s
declare void @prevent_merging()
; Base case - with side effects in loop
define i32 @test1(i32* %array, i32 %length, i32 %n, i1 %cond_0) {
; CHECK-LABEL: @test1(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt i32 [[N:%.*]], 1
; CHECK-NEXT: [[UMAX:%.*]] = select i1 [[TMP0]], i32 [[N]], i32 1
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[UMAX]], -1
; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i32 [[LENGTH:%.*]], [[TMP1]]
; CHECK-NEXT: [[UMIN:%.*]] = select i1 [[TMP2]], i32 [[LENGTH]], i32 [[TMP1]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ugt i32 [[LENGTH]], [[UMIN]]
; CHECK-NEXT: [[TMP4:%.*]] = freeze i1 [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = and i1 [[TMP4]], [[COND_0:%.*]]
; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[TMP5]], [[WIDENABLE_COND]]
; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
; CHECK: deopt:
; CHECK-NEXT: [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET]]
; CHECK: loop.preheader:
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH]]
; CHECK-NEXT: br i1 true, label [[GUARDED]], label [[DEOPT2:%.*]], !prof !0
; CHECK: deopt2:
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET2]]
; CHECK: guarded:
; CHECK-NEXT: [[I_I64:%.*]] = zext i32 [[I]] to i64
; CHECK-NEXT: [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
; CHECK-NEXT: [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
; CHECK-NEXT: store i32 0, i32* [[ARRAY_I_PTR]]
; CHECK-NEXT: [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
; CHECK-NEXT: [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
; CHECK: exit:
; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
; CHECK-NEXT: ret i32 [[RESULT]]
;
entry:
%widenable_cond = call i1 @llvm.experimental.widenable.condition()
%exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
br i1 %exiplicit_guard_cond, label %loop.preheader, label %deopt, !prof !0
deopt:
%deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret
loop.preheader:
br label %loop
loop:
%loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
%i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
call void @unknown()
%within.bounds = icmp ult i32 %i, %length
br i1 %within.bounds, label %guarded, label %deopt2, !prof !0
deopt2:
call void @unknown()
%deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret2
guarded:
%i.i64 = zext i32 %i to i64
%array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
%array.i = load i32, i32* %array.i.ptr, align 4
store i32 0, i32* %array.i.ptr
%loop.acc.next = add i32 %loop.acc, %array.i
%i.next = add nuw i32 %i, 1
%continue = icmp ult i32 %i.next, %n
br i1 %continue, label %loop, label %exit
exit:
%result = phi i32 [ %loop.acc.next, %guarded ]
ret i32 %result
}
define i32 @test_non_canonical(i32* %array, i32 %length, i1 %cond_0) {
; CHECK-LABEL: @test_non_canonical(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt i32 [[LENGTH:%.*]], 1
; CHECK-NEXT: [[UMAX:%.*]] = select i1 [[TMP0]], i32 [[LENGTH]], i32 1
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[UMAX]], -1
; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i32 [[LENGTH]], [[TMP1]]
; CHECK-NEXT: [[UMIN:%.*]] = select i1 [[TMP2]], i32 [[LENGTH]], i32 [[TMP1]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ugt i32 [[LENGTH]], [[UMIN]]
; CHECK-NEXT: [[TMP4:%.*]] = freeze i1 [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = and i1 [[TMP4]], [[COND_0:%.*]]
; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[TMP5]], [[WIDENABLE_COND]]
; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
; CHECK: deopt:
; CHECK-NEXT: [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET]]
; CHECK: loop.preheader:
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH]]
; CHECK-NEXT: br i1 true, label [[GUARDED]], label [[DEOPT2:%.*]], !prof !0
; CHECK: deopt2:
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET2]]
; CHECK: guarded:
; CHECK-NEXT: [[I_I64:%.*]] = zext i32 [[I]] to i64
; CHECK-NEXT: [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
; CHECK-NEXT: [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
; CHECK-NEXT: store i32 0, i32* [[ARRAY_I_PTR]]
; CHECK-NEXT: [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
; CHECK-NEXT: [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[LENGTH]]
; CHECK-NEXT: br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
; CHECK: exit:
; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
; CHECK-NEXT: ret i32 [[RESULT]]
;
entry:
%widenable_cond = call i1 @llvm.experimental.widenable.condition()
%exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
br i1 %exiplicit_guard_cond, label %loop.preheader, label %deopt, !prof !0
deopt:
%deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret
loop.preheader:
br label %loop
loop:
%loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
%i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
call void @unknown()
%within.bounds = icmp ult i32 %i, %length
br i1 %within.bounds, label %guarded, label %deopt2, !prof !0
deopt2:
call void @unknown()
%deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret2
guarded:
%i.i64 = zext i32 %i to i64
%array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
%array.i = load i32, i32* %array.i.ptr, align 4
store i32 0, i32* %array.i.ptr
%loop.acc.next = add i32 %loop.acc, %array.i
%i.next = add nuw i32 %i, 1
%continue = icmp ult i32 %i.next, %length
br i1 %continue, label %loop, label %exit
exit:
%result = phi i32 [ %loop.acc.next, %guarded ]
ret i32 %result
}
define i32 @test_two_range_checks(i32* %array, i32 %length.1, i32 %length.2, i32 %n, i1 %cond_0) {
; CHECK-LABEL: @test_two_range_checks(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
; CHECK-NEXT: [[TMP0:%.*]] = icmp ult i32 [[LENGTH_2:%.*]], [[LENGTH_1:%.*]]
; CHECK-NEXT: [[UMIN:%.*]] = select i1 [[TMP0]], i32 [[LENGTH_2]], i32 [[LENGTH_1]]
; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt i32 [[N:%.*]], 1
; CHECK-NEXT: [[UMAX:%.*]] = select i1 [[TMP1]], i32 [[N]], i32 1
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[UMAX]], -1
; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i32 [[UMIN]], [[TMP2]]
; CHECK-NEXT: [[UMIN1:%.*]] = select i1 [[TMP3]], i32 [[UMIN]], i32 [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt i32 [[LENGTH_1]], [[UMIN1]]
; CHECK-NEXT: [[TMP5:%.*]] = freeze i1 [[TMP4]]
; CHECK-NEXT: [[TMP6:%.*]] = and i1 [[TMP5]], [[COND_0:%.*]]
; CHECK-NEXT: [[TMP7:%.*]] = icmp ugt i32 [[LENGTH_2]], [[UMIN1]]
; CHECK-NEXT: [[TMP8:%.*]] = freeze i1 [[TMP7]]
; CHECK-NEXT: [[TMP9:%.*]] = and i1 [[TMP8]], [[TMP6]]
; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[TMP9]], [[WIDENABLE_COND]]
; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
; CHECK: deopt:
; CHECK-NEXT: [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET]]
; CHECK: loop.preheader:
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED2:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED2]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH_1]]
; CHECK-NEXT: br i1 true, label [[GUARDED:%.*]], label [[DEOPT2:%.*]], !prof !0
; CHECK: deopt2:
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET2]]
; CHECK: guarded:
; CHECK-NEXT: [[WITHIN_BOUNDS2:%.*]] = icmp ult i32 [[I]], [[LENGTH_2]]
; CHECK-NEXT: br i1 true, label [[GUARDED2]], label [[DEOPT3:%.*]], !prof !0
; CHECK: deopt3:
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[DEOPTRET3:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET3]]
; CHECK: guarded2:
; CHECK-NEXT: [[I_I64:%.*]] = zext i32 [[I]] to i64
; CHECK-NEXT: [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
; CHECK-NEXT: [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
; CHECK-NEXT: store i32 0, i32* [[ARRAY_I_PTR]]
; CHECK-NEXT: [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
; CHECK-NEXT: [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
; CHECK: exit:
; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED2]] ]
; CHECK-NEXT: ret i32 [[RESULT]]
;
entry:
%widenable_cond = call i1 @llvm.experimental.widenable.condition()
%exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
br i1 %exiplicit_guard_cond, label %loop.preheader, label %deopt, !prof !0
deopt:
%deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret
loop.preheader:
br label %loop
loop:
%loop.acc = phi i32 [ %loop.acc.next, %guarded2 ], [ 0, %loop.preheader ]
%i = phi i32 [ %i.next, %guarded2 ], [ 0, %loop.preheader ]
call void @unknown()
%within.bounds = icmp ult i32 %i, %length.1
br i1 %within.bounds, label %guarded, label %deopt2, !prof !0
deopt2:
call void @unknown()
%deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret2
guarded:
%within.bounds2 = icmp ult i32 %i, %length.2
br i1 %within.bounds2, label %guarded2, label %deopt3, !prof !0
deopt3:
call void @unknown()
%deoptret3 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret3
guarded2:
%i.i64 = zext i32 %i to i64
%array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
%array.i = load i32, i32* %array.i.ptr, align 4
store i32 0, i32* %array.i.ptr
%loop.acc.next = add i32 %loop.acc, %array.i
%i.next = add nuw i32 %i, 1
%continue = icmp ult i32 %i.next, %n
br i1 %continue, label %loop, label %exit
exit:
%result = phi i32 [ %loop.acc.next, %guarded2 ]
ret i32 %result
}
@G = external global i32
define i32 @test_unanalyzeable_exit(i32* %array, i32 %length, i32 %n, i1 %cond_0) {
; CHECK-LABEL: @test_unanalyzeable_exit(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
; CHECK: deopt:
; CHECK-NEXT: [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET]]
; CHECK: loop.preheader:
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED2:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED2]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[VOL:%.*]] = load volatile i32, i32* @G
; CHECK-NEXT: [[UNKNOWN:%.*]] = icmp eq i32 [[VOL]], 0
; CHECK-NEXT: br i1 [[UNKNOWN]], label [[GUARDED2]], label [[DEOPT3:%.*]], !prof !0
; CHECK: deopt3:
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[DEOPTRET3:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET3]]
; CHECK: guarded2:
; CHECK-NEXT: [[I_I64:%.*]] = zext i32 [[I]] to i64
; CHECK-NEXT: [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
; CHECK-NEXT: [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
; CHECK-NEXT: store i32 0, i32* [[ARRAY_I_PTR]]
; CHECK-NEXT: [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
; CHECK-NEXT: [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N:%.*]]
; CHECK-NEXT: br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
; CHECK: exit:
; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED2]] ]
; CHECK-NEXT: ret i32 [[RESULT]]
;
entry:
%widenable_cond = call i1 @llvm.experimental.widenable.condition()
%exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
br i1 %exiplicit_guard_cond, label %loop.preheader, label %deopt, !prof !0
deopt:
%deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret
loop.preheader:
br label %loop
loop:
%loop.acc = phi i32 [ %loop.acc.next, %guarded2 ], [ 0, %loop.preheader ]
%i = phi i32 [ %i.next, %guarded2 ], [ 0, %loop.preheader ]
call void @unknown()
%vol = load volatile i32, i32* @G
%unknown = icmp eq i32 %vol, 0
br i1 %unknown, label %guarded2, label %deopt3, !prof !0
deopt3:
call void @unknown()
%deoptret3 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret3
guarded2:
%i.i64 = zext i32 %i to i64
%array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
%array.i = load i32, i32* %array.i.ptr, align 4
store i32 0, i32* %array.i.ptr
%loop.acc.next = add i32 %loop.acc, %array.i
%i.next = add nuw i32 %i, 1
%continue = icmp ult i32 %i.next, %n
br i1 %continue, label %loop, label %exit
exit:
%result = phi i32 [ %loop.acc.next, %guarded2 ]
ret i32 %result
}
define i32 @test_unanalyzeable_exit2(i32* %array, i32 %length, i32 %n, i1 %cond_0) {
; CHECK-LABEL: @test_unanalyzeable_exit2(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt i32 [[N:%.*]], 1
; CHECK-NEXT: [[UMAX:%.*]] = select i1 [[TMP0]], i32 [[N]], i32 1
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[UMAX]], -1
; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i32 [[LENGTH:%.*]], [[TMP1]]
; CHECK-NEXT: [[UMIN:%.*]] = select i1 [[TMP2]], i32 [[LENGTH]], i32 [[TMP1]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ugt i32 [[LENGTH]], [[UMIN]]
; CHECK-NEXT: [[TMP4:%.*]] = freeze i1 [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = and i1 [[TMP4]], [[COND_0:%.*]]
; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[TMP5]], [[WIDENABLE_COND]]
; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
; CHECK: deopt:
; CHECK-NEXT: [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET]]
; CHECK: loop.preheader:
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED2:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED2]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH]]
; CHECK-NEXT: br i1 true, label [[GUARDED:%.*]], label [[DEOPT2:%.*]], !prof !0
; CHECK: deopt2:
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET2]]
; CHECK: guarded:
; CHECK-NEXT: [[VOL:%.*]] = load volatile i32, i32* @G
; CHECK-NEXT: [[UNKNOWN:%.*]] = icmp eq i32 [[VOL]], 0
; CHECK-NEXT: br i1 [[UNKNOWN]], label [[GUARDED2]], label [[DEOPT3:%.*]], !prof !0
; CHECK: deopt3:
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[DEOPTRET3:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET3]]
; CHECK: guarded2:
; CHECK-NEXT: [[I_I64:%.*]] = zext i32 [[I]] to i64
; CHECK-NEXT: [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
; CHECK-NEXT: [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
; CHECK-NEXT: store i32 0, i32* [[ARRAY_I_PTR]]
; CHECK-NEXT: [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
; CHECK-NEXT: [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
; CHECK: exit:
; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED2]] ]
; CHECK-NEXT: ret i32 [[RESULT]]
;
entry:
%widenable_cond = call i1 @llvm.experimental.widenable.condition()
%exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
br i1 %exiplicit_guard_cond, label %loop.preheader, label %deopt, !prof !0
deopt:
%deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret
loop.preheader:
br label %loop
loop:
%loop.acc = phi i32 [ %loop.acc.next, %guarded2 ], [ 0, %loop.preheader ]
%i = phi i32 [ %i.next, %guarded2 ], [ 0, %loop.preheader ]
call void @unknown()
%within.bounds = icmp ult i32 %i, %length
br i1 %within.bounds, label %guarded, label %deopt2, !prof !0
deopt2:
call void @unknown()
%deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret2
guarded:
%vol = load volatile i32, i32* @G
%unknown = icmp eq i32 %vol, 0
br i1 %unknown, label %guarded2, label %deopt3, !prof !0
deopt3:
call void @unknown()
%deoptret3 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret3
guarded2:
%i.i64 = zext i32 %i to i64
%array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
%array.i = load i32, i32* %array.i.ptr, align 4
store i32 0, i32* %array.i.ptr
%loop.acc.next = add i32 %loop.acc, %array.i
%i.next = add nuw i32 %i, 1
%continue = icmp ult i32 %i.next, %n
br i1 %continue, label %loop, label %exit
exit:
%result = phi i32 [ %loop.acc.next, %guarded2 ]
ret i32 %result
}
define i32 @test_unanalyzeable_latch(i32* %array, i32 %length, i32 %n, i1 %cond_0) {
; CHECK-LABEL: @test_unanalyzeable_latch(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
; CHECK: deopt:
; CHECK-NEXT: [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET]]
; CHECK: loop.preheader:
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH:%.*]]
; CHECK-NEXT: br i1 [[WITHIN_BOUNDS]], label [[GUARDED]], label [[DEOPT2:%.*]], !prof !0
; CHECK: deopt2:
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET2]]
; CHECK: guarded:
; CHECK-NEXT: [[I_I64:%.*]] = zext i32 [[I]] to i64
; CHECK-NEXT: [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
; CHECK-NEXT: [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
; CHECK-NEXT: store i32 0, i32* [[ARRAY_I_PTR]]
; CHECK-NEXT: [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
; CHECK-NEXT: [[VOL:%.*]] = load volatile i32, i32* @G
; CHECK-NEXT: [[UNKNOWN:%.*]] = icmp eq i32 [[VOL]], 0
; CHECK-NEXT: br i1 [[UNKNOWN]], label [[LOOP]], label [[EXIT:%.*]]
; CHECK: exit:
; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
; CHECK-NEXT: ret i32 [[RESULT]]
;
entry:
%widenable_cond = call i1 @llvm.experimental.widenable.condition()
%exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
br i1 %exiplicit_guard_cond, label %loop.preheader, label %deopt, !prof !0
deopt:
%deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret
loop.preheader:
br label %loop
loop:
%loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
%i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
call void @unknown()
%within.bounds = icmp ult i32 %i, %length
br i1 %within.bounds, label %guarded, label %deopt2, !prof !0
deopt2:
call void @unknown()
%deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret2
guarded:
%i.i64 = zext i32 %i to i64
%array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
%array.i = load i32, i32* %array.i.ptr, align 4
store i32 0, i32* %array.i.ptr
%loop.acc.next = add i32 %loop.acc, %array.i
%i.next = add nuw i32 %i, 1
%vol = load volatile i32, i32* @G
%unknown = icmp eq i32 %vol, 0
br i1 %unknown, label %loop, label %exit
exit:
%result = phi i32 [ %loop.acc.next, %guarded ]
ret i32 %result
}
define i32 @provably_taken(i32* %array, i1 %cond_0) {
; CHECK-LABEL: @provably_taken(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
; CHECK-NEXT: [[TMP0:%.*]] = freeze i1 false
; CHECK-NEXT: [[TMP1:%.*]] = and i1 [[TMP0]], [[COND_0:%.*]]
; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[TMP1]], [[WIDENABLE_COND]]
; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
; CHECK: deopt:
; CHECK-NEXT: [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET]]
; CHECK: loop.preheader:
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], 198
; CHECK-NEXT: br i1 true, label [[GUARDED]], label [[DEOPT2:%.*]], !prof !0
; CHECK: deopt2:
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET2]]
; CHECK: guarded:
; CHECK-NEXT: [[I_I64:%.*]] = zext i32 [[I]] to i64
; CHECK-NEXT: [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
; CHECK-NEXT: [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
; CHECK-NEXT: store i32 0, i32* [[ARRAY_I_PTR]]
; CHECK-NEXT: [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
; CHECK-NEXT: [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], 200
; CHECK-NEXT: br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
; CHECK: exit:
; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
; CHECK-NEXT: ret i32 [[RESULT]]
;
entry:
%widenable_cond = call i1 @llvm.experimental.widenable.condition()
%exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
br i1 %exiplicit_guard_cond, label %loop.preheader, label %deopt, !prof !0
deopt:
%deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret
loop.preheader:
br label %loop
loop:
%loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
%i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
call void @unknown()
%within.bounds = icmp ult i32 %i, 198
br i1 %within.bounds, label %guarded, label %deopt2, !prof !0
deopt2:
call void @unknown()
%deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret2
guarded:
%i.i64 = zext i32 %i to i64
%array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
%array.i = load i32, i32* %array.i.ptr, align 4
store i32 0, i32* %array.i.ptr
%loop.acc.next = add i32 %loop.acc, %array.i
%i.next = add nuw i32 %i, 1
%continue = icmp ult i32 %i.next, 200
br i1 %continue, label %loop, label %exit
exit:
%result = phi i32 [ %loop.acc.next, %guarded ]
ret i32 %result
}
define i32 @provably_not_taken(i32* %array, i1 %cond_0) {
; CHECK-LABEL: @provably_not_taken(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
; CHECK-NEXT: [[TMP0:%.*]] = freeze i1 true
; CHECK-NEXT: [[TMP1:%.*]] = and i1 [[TMP0]], [[COND_0:%.*]]
; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[TMP1]], [[WIDENABLE_COND]]
; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
; CHECK: deopt:
; CHECK-NEXT: [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET]]
; CHECK: loop.preheader:
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], 205
; CHECK-NEXT: br i1 true, label [[GUARDED]], label [[DEOPT2:%.*]], !prof !0
; CHECK: deopt2:
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET2]]
; CHECK: guarded:
; CHECK-NEXT: [[I_I64:%.*]] = zext i32 [[I]] to i64
; CHECK-NEXT: [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
; CHECK-NEXT: [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
; CHECK-NEXT: store i32 0, i32* [[ARRAY_I_PTR]]
; CHECK-NEXT: [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
; CHECK-NEXT: [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], 200
; CHECK-NEXT: br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
; CHECK: exit:
; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
; CHECK-NEXT: ret i32 [[RESULT]]
;
entry:
%widenable_cond = call i1 @llvm.experimental.widenable.condition()
%exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
br i1 %exiplicit_guard_cond, label %loop.preheader, label %deopt, !prof !0
deopt:
%deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret
loop.preheader:
br label %loop
loop:
%loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
%i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
call void @unknown()
%within.bounds = icmp ult i32 %i, 205
br i1 %within.bounds, label %guarded, label %deopt2, !prof !0
deopt2:
call void @unknown()
%deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret2
guarded:
%i.i64 = zext i32 %i to i64
%array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
%array.i = load i32, i32* %array.i.ptr, align 4
store i32 0, i32* %array.i.ptr
%loop.acc.next = add i32 %loop.acc, %array.i
%i.next = add nuw i32 %i, 1
%continue = icmp ult i32 %i.next, 200
br i1 %continue, label %loop, label %exit
exit:
%result = phi i32 [ %loop.acc.next, %guarded ]
ret i32 %result
}
[LoopPred] Generalize profitability check to handle unswitch output Unswitch (and other loop transforms) like to generate loop exit blocks with unconditional successors, and phi nodes (LCSSA, or simple multiple exiting blocks sharing an exit). Generalize the "likely very rare exit" check slightly to handle this form.
2019-11-20 06:06:36 +08:00
;; Unswitch likes to produce some ugly exit blocks without simplifications
;; being applied. Make sure we can handle that form.
define i32 @unswitch_exit_form(i32* %array, i32 %length, i32 %n, i1 %cond_0) {
; CHECK-LABEL: @unswitch_exit_form(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt i32 [[N:%.*]], 1
; CHECK-NEXT: [[UMAX:%.*]] = select i1 [[TMP0]], i32 [[N]], i32 1
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[UMAX]], -1
; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i32 [[LENGTH:%.*]], [[TMP1]]
; CHECK-NEXT: [[UMIN:%.*]] = select i1 [[TMP2]], i32 [[LENGTH]], i32 [[TMP1]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ugt i32 [[LENGTH]], [[UMIN]]
; CHECK-NEXT: [[TMP4:%.*]] = freeze i1 [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = and i1 [[TMP4]], [[COND_0:%.*]]
; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[TMP5]], [[WIDENABLE_COND]]
; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
; CHECK: deopt.loopexit:
; CHECK-NEXT: br label [[DEOPT]]
; CHECK: deopt:
; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ 1, [[DEOPT_LOOPEXIT:%.*]] ]
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: br label [[ACTUAL_DEOPT:%.*]]
; CHECK: actual_deopt:
; CHECK-NEXT: [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"(i32 [[PHI]]) ]
; CHECK-NEXT: ret i32 [[DEOPTRET]]
; CHECK: loop.preheader:
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH]]
; CHECK-NEXT: br i1 true, label [[GUARDED]], label [[DEOPT_LOOPEXIT]], !prof !0
; CHECK: guarded:
; CHECK-NEXT: [[I_I64:%.*]] = zext i32 [[I]] to i64
; CHECK-NEXT: [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
; CHECK-NEXT: [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
; CHECK-NEXT: store i32 0, i32* [[ARRAY_I_PTR]]
; CHECK-NEXT: [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
; CHECK-NEXT: [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
; CHECK: exit:
; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
; CHECK-NEXT: ret i32 [[RESULT]]
;
entry:
%widenable_cond = call i1 @llvm.experimental.widenable.condition()
%exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
br i1 %exiplicit_guard_cond, label %loop.preheader, label %deopt, !prof !0
[LoopPred/WC] Use a dominating widenable condition to remove analyze loop exits This implements a version of the predicateLoopExits transform from IndVarSimplify extended to exploit widenable conditions - and thus be much wider in scope of legality. The code structure ends up being almost entirely different, so I chose to duplicate this into the LoopPredication pass instead of trying to reuse the code in the IndVars. The core notions of the transform are as follows: If we have a widenable condition which controls entry into the loop, we're allowed to widen it arbitrarily. Given that, it's simply a *profitability* question as to what conditions to fold into the widenable branch. To avoid pass ordering issues, we want to avoid widening cases that would otherwise be dischargeable. Or... widen in a form which can still be discharged. Thus, we phrase the transform as selecting one analyzeable exit from the set of analyzeable exits to keep. This avoids creating pass ordering complexities. Since none of the above proves that we actually exit through our analyzeable exits - we might exit through something else entirely - we limit ourselves to cases where a) the latch is analyzeable and b) the latch is predicted taken, and c) the exit being removed is statically cold. Differential Revision: https://reviews.llvm.org/D69830
2019-11-19 03:21:53 +08:00
[LoopPred] Generalize profitability check to handle unswitch output Unswitch (and other loop transforms) like to generate loop exit blocks with unconditional successors, and phi nodes (LCSSA, or simple multiple exiting blocks sharing an exit). Generalize the "likely very rare exit" check slightly to handle this form.
2019-11-20 06:06:36 +08:00
deopt:
;; This is written to look like an unsimplified loop exit after unswitch
;; (i.e. phis, merge, and branch to actual block)
%phi = phi i32 [0, %entry], [1, %loop]
call void @unknown() ;; it's okay to skip possible throws
br label %actual_deopt
actual_deopt:
%deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"(i32 %phi) ]
ret i32 %deoptret
[LoopPred/WC] Use a dominating widenable condition to remove analyze loop exits This implements a version of the predicateLoopExits transform from IndVarSimplify extended to exploit widenable conditions - and thus be much wider in scope of legality. The code structure ends up being almost entirely different, so I chose to duplicate this into the LoopPredication pass instead of trying to reuse the code in the IndVars. The core notions of the transform are as follows: If we have a widenable condition which controls entry into the loop, we're allowed to widen it arbitrarily. Given that, it's simply a *profitability* question as to what conditions to fold into the widenable branch. To avoid pass ordering issues, we want to avoid widening cases that would otherwise be dischargeable. Or... widen in a form which can still be discharged. Thus, we phrase the transform as selecting one analyzeable exit from the set of analyzeable exits to keep. This avoids creating pass ordering complexities. Since none of the above proves that we actually exit through our analyzeable exits - we might exit through something else entirely - we limit ourselves to cases where a) the latch is analyzeable and b) the latch is predicted taken, and c) the exit being removed is statically cold. Differential Revision: https://reviews.llvm.org/D69830
2019-11-19 03:21:53 +08:00
[LoopPred] Generalize profitability check to handle unswitch output Unswitch (and other loop transforms) like to generate loop exit blocks with unconditional successors, and phi nodes (LCSSA, or simple multiple exiting blocks sharing an exit). Generalize the "likely very rare exit" check slightly to handle this form.
2019-11-20 06:06:36 +08:00
loop.preheader:
br label %loop
[LoopPred/WC] Use a dominating widenable condition to remove analyze loop exits This implements a version of the predicateLoopExits transform from IndVarSimplify extended to exploit widenable conditions - and thus be much wider in scope of legality. The code structure ends up being almost entirely different, so I chose to duplicate this into the LoopPredication pass instead of trying to reuse the code in the IndVars. The core notions of the transform are as follows: If we have a widenable condition which controls entry into the loop, we're allowed to widen it arbitrarily. Given that, it's simply a *profitability* question as to what conditions to fold into the widenable branch. To avoid pass ordering issues, we want to avoid widening cases that would otherwise be dischargeable. Or... widen in a form which can still be discharged. Thus, we phrase the transform as selecting one analyzeable exit from the set of analyzeable exits to keep. This avoids creating pass ordering complexities. Since none of the above proves that we actually exit through our analyzeable exits - we might exit through something else entirely - we limit ourselves to cases where a) the latch is analyzeable and b) the latch is predicted taken, and c) the exit being removed is statically cold. Differential Revision: https://reviews.llvm.org/D69830
2019-11-19 03:21:53 +08:00
[LoopPred] Generalize profitability check to handle unswitch output Unswitch (and other loop transforms) like to generate loop exit blocks with unconditional successors, and phi nodes (LCSSA, or simple multiple exiting blocks sharing an exit). Generalize the "likely very rare exit" check slightly to handle this form.
2019-11-20 06:06:36 +08:00
loop:
%loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
%i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
call void @unknown()
%within.bounds = icmp ult i32 %i, %length
br i1 %within.bounds, label %guarded, label %deopt, !prof !0
[LoopPred/WC] Use a dominating widenable condition to remove analyze loop exits This implements a version of the predicateLoopExits transform from IndVarSimplify extended to exploit widenable conditions - and thus be much wider in scope of legality. The code structure ends up being almost entirely different, so I chose to duplicate this into the LoopPredication pass instead of trying to reuse the code in the IndVars. The core notions of the transform are as follows: If we have a widenable condition which controls entry into the loop, we're allowed to widen it arbitrarily. Given that, it's simply a *profitability* question as to what conditions to fold into the widenable branch. To avoid pass ordering issues, we want to avoid widening cases that would otherwise be dischargeable. Or... widen in a form which can still be discharged. Thus, we phrase the transform as selecting one analyzeable exit from the set of analyzeable exits to keep. This avoids creating pass ordering complexities. Since none of the above proves that we actually exit through our analyzeable exits - we might exit through something else entirely - we limit ourselves to cases where a) the latch is analyzeable and b) the latch is predicted taken, and c) the exit being removed is statically cold. Differential Revision: https://reviews.llvm.org/D69830
2019-11-19 03:21:53 +08:00
[LoopPred] Generalize profitability check to handle unswitch output Unswitch (and other loop transforms) like to generate loop exit blocks with unconditional successors, and phi nodes (LCSSA, or simple multiple exiting blocks sharing an exit). Generalize the "likely very rare exit" check slightly to handle this form.
2019-11-20 06:06:36 +08:00
guarded:
%i.i64 = zext i32 %i to i64
%array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
%array.i = load i32, i32* %array.i.ptr, align 4
store i32 0, i32* %array.i.ptr
%loop.acc.next = add i32 %loop.acc, %array.i
%i.next = add nuw i32 %i, 1
%continue = icmp ult i32 %i.next, %n
br i1 %continue, label %loop, label %exit
[LoopPred/WC] Use a dominating widenable condition to remove analyze loop exits This implements a version of the predicateLoopExits transform from IndVarSimplify extended to exploit widenable conditions - and thus be much wider in scope of legality. The code structure ends up being almost entirely different, so I chose to duplicate this into the LoopPredication pass instead of trying to reuse the code in the IndVars. The core notions of the transform are as follows: If we have a widenable condition which controls entry into the loop, we're allowed to widen it arbitrarily. Given that, it's simply a *profitability* question as to what conditions to fold into the widenable branch. To avoid pass ordering issues, we want to avoid widening cases that would otherwise be dischargeable. Or... widen in a form which can still be discharged. Thus, we phrase the transform as selecting one analyzeable exit from the set of analyzeable exits to keep. This avoids creating pass ordering complexities. Since none of the above proves that we actually exit through our analyzeable exits - we might exit through something else entirely - we limit ourselves to cases where a) the latch is analyzeable and b) the latch is predicted taken, and c) the exit being removed is statically cold. Differential Revision: https://reviews.llvm.org/D69830
2019-11-19 03:21:53 +08:00
[LoopPred] Generalize profitability check to handle unswitch output Unswitch (and other loop transforms) like to generate loop exit blocks with unconditional successors, and phi nodes (LCSSA, or simple multiple exiting blocks sharing an exit). Generalize the "likely very rare exit" check slightly to handle this form.
2019-11-20 06:06:36 +08:00
exit:
%result = phi i32 [ %loop.acc.next, %guarded ]
ret i32 %result
}
[LoopPred/WC] Use a dominating widenable condition to remove analyze loop exits This implements a version of the predicateLoopExits transform from IndVarSimplify extended to exploit widenable conditions - and thus be much wider in scope of legality. The code structure ends up being almost entirely different, so I chose to duplicate this into the LoopPredication pass instead of trying to reuse the code in the IndVars. The core notions of the transform are as follows: If we have a widenable condition which controls entry into the loop, we're allowed to widen it arbitrarily. Given that, it's simply a *profitability* question as to what conditions to fold into the widenable branch. To avoid pass ordering issues, we want to avoid widening cases that would otherwise be dischargeable. Or... widen in a form which can still be discharged. Thus, we phrase the transform as selecting one analyzeable exit from the set of analyzeable exits to keep. This avoids creating pass ordering complexities. Since none of the above proves that we actually exit through our analyzeable exits - we might exit through something else entirely - we limit ourselves to cases where a) the latch is analyzeable and b) the latch is predicted taken, and c) the exit being removed is statically cold. Differential Revision: https://reviews.llvm.org/D69830
2019-11-19 03:21:53 +08:00
Broaden the definition of a "widenable branch" As a reminder, a "widenable branch" is the pattern "br i1 (and i1 X, WC()), label %taken, label %untaken" where "WC" is the widenable condition intrinsics. The semantics of such a branch (derived from the semantics of WC) is that a new condition can be added into the condition arbitrarily without violating legality. Broaden the definition in two ways: Allow swapped operands to the br (and X, WC()) form Allow widenable branch w/trivial condition (i.e. true) which takes form of br i1 WC() The former is just general robustness (e.g. for X = non-instruction this is what instcombine produces). The later is specifically important as partial unswitching of a widenable range check produces exactly this form above the loop. Differential Revision: https://reviews.llvm.org/D70502
2019-11-22 02:44:13 +08:00
define i32 @swapped_wb(i32* %array, i32 %length, i32 %n, i1 %cond_0) {
; CHECK-LABEL: @swapped_wb(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt i32 [[N:%.*]], 1
; CHECK-NEXT: [[UMAX:%.*]] = select i1 [[TMP0]], i32 [[N]], i32 1
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[UMAX]], -1
; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i32 [[LENGTH:%.*]], [[TMP1]]
; CHECK-NEXT: [[UMIN:%.*]] = select i1 [[TMP2]], i32 [[LENGTH]], i32 [[TMP1]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ugt i32 [[LENGTH]], [[UMIN]]
; CHECK-NEXT: [[TMP4:%.*]] = freeze i1 [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = and i1 [[TMP4]], [[COND_0:%.*]]
; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WIDENABLE_COND]], [[TMP5]]
; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
; CHECK: deopt:
; CHECK-NEXT: [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET]]
; CHECK: loop.preheader:
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH]]
; CHECK-NEXT: br i1 true, label [[GUARDED]], label [[DEOPT2:%.*]], !prof !0
; CHECK: deopt2:
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET2]]
; CHECK: guarded:
; CHECK-NEXT: [[I_I64:%.*]] = zext i32 [[I]] to i64
; CHECK-NEXT: [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
; CHECK-NEXT: [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
; CHECK-NEXT: store i32 0, i32* [[ARRAY_I_PTR]]
; CHECK-NEXT: [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
; CHECK-NEXT: [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
; CHECK: exit:
; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
; CHECK-NEXT: ret i32 [[RESULT]]
;
entry:
%widenable_cond = call i1 @llvm.experimental.widenable.condition()
%exiplicit_guard_cond = and i1 %widenable_cond, %cond_0
br i1 %exiplicit_guard_cond, label %loop.preheader, label %deopt, !prof !0
deopt:
%deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret
loop.preheader:
br label %loop
loop:
%loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
%i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
call void @unknown()
%within.bounds = icmp ult i32 %i, %length
br i1 %within.bounds, label %guarded, label %deopt2, !prof !0
deopt2:
call void @unknown()
%deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret2
guarded:
%i.i64 = zext i32 %i to i64
%array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
%array.i = load i32, i32* %array.i.ptr, align 4
store i32 0, i32* %array.i.ptr
%loop.acc.next = add i32 %loop.acc, %array.i
%i.next = add nuw i32 %i, 1
%continue = icmp ult i32 %i.next, %n
br i1 %continue, label %loop, label %exit
exit:
%result = phi i32 [ %loop.acc.next, %guarded ]
ret i32 %result
}
define i32 @trivial_wb(i32* %array, i32 %length, i32 %n) {
; CHECK-LABEL: @trivial_wb(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt i32 [[N:%.*]], 1
; CHECK-NEXT: [[UMAX:%.*]] = select i1 [[TMP0]], i32 [[N]], i32 1
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[UMAX]], -1
; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i32 [[LENGTH:%.*]], [[TMP1]]
; CHECK-NEXT: [[UMIN:%.*]] = select i1 [[TMP2]], i32 [[LENGTH]], i32 [[TMP1]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ugt i32 [[LENGTH]], [[UMIN]]
; CHECK-NEXT: [[TMP4:%.*]] = freeze i1 [[TMP3]]
; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
; CHECK-NEXT: [[TMP5:%.*]] = and i1 [[TMP4]], [[WIDENABLE_COND]]
; CHECK-NEXT: br i1 [[TMP5]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
; CHECK: deopt:
; CHECK-NEXT: [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET]]
; CHECK: loop.preheader:
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH]]
; CHECK-NEXT: br i1 true, label [[GUARDED]], label [[DEOPT2:%.*]], !prof !0
; CHECK: deopt2:
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET2]]
; CHECK: guarded:
; CHECK-NEXT: [[I_I64:%.*]] = zext i32 [[I]] to i64
; CHECK-NEXT: [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
; CHECK-NEXT: [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
; CHECK-NEXT: store i32 0, i32* [[ARRAY_I_PTR]]
; CHECK-NEXT: [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
; CHECK-NEXT: [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
; CHECK: exit:
; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
; CHECK-NEXT: ret i32 [[RESULT]]
;
entry:
%widenable_cond = call i1 @llvm.experimental.widenable.condition()
br i1 %widenable_cond, label %loop.preheader, label %deopt, !prof !0
deopt:
%deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret
loop.preheader:
br label %loop
loop:
%loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
%i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
call void @unknown()
%within.bounds = icmp ult i32 %i, %length
br i1 %within.bounds, label %guarded, label %deopt2, !prof !0
deopt2:
call void @unknown()
%deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret2
guarded:
%i.i64 = zext i32 %i to i64
%array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
%array.i = load i32, i32* %array.i.ptr, align 4
store i32 0, i32* %array.i.ptr
%loop.acc.next = add i32 %loop.acc, %array.i
%i.next = add nuw i32 %i, 1
%continue = icmp ult i32 %i.next, %n
br i1 %continue, label %loop, label %exit
exit:
%result = phi i32 [ %loop.acc.next, %guarded ]
ret i32 %result
}
[LoopPred/WC] Use a dominating widenable condition to remove analyze loop exits This implements a version of the predicateLoopExits transform from IndVarSimplify extended to exploit widenable conditions - and thus be much wider in scope of legality. The code structure ends up being almost entirely different, so I chose to duplicate this into the LoopPredication pass instead of trying to reuse the code in the IndVars. The core notions of the transform are as follows: If we have a widenable condition which controls entry into the loop, we're allowed to widen it arbitrarily. Given that, it's simply a *profitability* question as to what conditions to fold into the widenable branch. To avoid pass ordering issues, we want to avoid widening cases that would otherwise be dischargeable. Or... widen in a form which can still be discharged. Thus, we phrase the transform as selecting one analyzeable exit from the set of analyzeable exits to keep. This avoids creating pass ordering complexities. Since none of the above proves that we actually exit through our analyzeable exits - we might exit through something else entirely - we limit ourselves to cases where a) the latch is analyzeable and b) the latch is predicted taken, and c) the exit being removed is statically cold. Differential Revision: https://reviews.llvm.org/D69830
2019-11-19 03:21:53 +08:00
; TODO: Non-latch exits can still be predicated
; This is currently prevented by an overly restrictive profitability check.
define i32 @todo_unconditional_latch(i32* %array, i32 %length, i1 %cond_0) {
; CHECK-LABEL: @todo_unconditional_latch(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
; CHECK: deopt:
; CHECK-NEXT: [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET]]
; CHECK: loop.preheader:
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH:%.*]]
; CHECK-NEXT: br i1 [[WITHIN_BOUNDS]], label [[GUARDED]], label [[DEOPT2:%.*]], !prof !0
; CHECK: deopt2:
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET2]]
; CHECK: guarded:
; CHECK-NEXT: [[I_I64:%.*]] = zext i32 [[I]] to i64
; CHECK-NEXT: [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
; CHECK-NEXT: [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
; CHECK-NEXT: store i32 0, i32* [[ARRAY_I_PTR]]
; CHECK-NEXT: [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
; CHECK-NEXT: br label [[LOOP]]
;
entry:
%widenable_cond = call i1 @llvm.experimental.widenable.condition()
%exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
br i1 %exiplicit_guard_cond, label %loop.preheader, label %deopt, !prof !0
deopt:
%deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret
loop.preheader:
br label %loop
loop:
%loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
%i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
call void @unknown()
%within.bounds = icmp ult i32 %i, %length
br i1 %within.bounds, label %guarded, label %deopt2, !prof !0
deopt2:
call void @unknown()
%deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret2
guarded:
%i.i64 = zext i32 %i to i64
%array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
%array.i = load i32, i32* %array.i.ptr, align 4
store i32 0, i32* %array.i.ptr
%loop.acc.next = add i32 %loop.acc, %array.i
%i.next = add nuw i32 %i, 1
br label %loop
}
[LoopPred] Robustly handle partially unswitched loops We may end up with a case where we have a widenable branch above the loop, but not all widenable branches within the loop have been removed. Since a widenable branch inhibit SCEVs ability to reason about exit counts (by design), we have a tradeoff between effectiveness of this optimization and allowing future widening of the branches within the loop. LoopPred is thought to be one of the most important optimizations for range check elimination, so let's pay the cost.
2019-11-22 07:26:58 +08:00
; If we have a stray widenable branch in the loop, we should still be able to
; run. This can happen when unswitching's cost model avoids unswitching some
; branches.
define i32 @wb_in_loop(i32* %array, i32 %length, i32 %n, i1 %cond_0) {
; CHECK-LABEL: @wb_in_loop(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
; CHECK-NEXT: [[WC2:%.*]] = call i1 @llvm.experimental.widenable.condition()
; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt i32 [[N:%.*]], 1
; CHECK-NEXT: [[UMAX:%.*]] = select i1 [[TMP0]], i32 [[N]], i32 1
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[UMAX]], -1
; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i32 [[LENGTH:%.*]], [[TMP1]]
; CHECK-NEXT: [[UMIN:%.*]] = select i1 [[TMP2]], i32 [[LENGTH]], i32 [[TMP1]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ugt i32 [[LENGTH]], [[UMIN]]
; CHECK-NEXT: [[TMP4:%.*]] = freeze i1 [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = and i1 [[TMP4]], [[COND_0:%.*]]
; CHECK-NEXT: [[TMP6:%.*]] = icmp ugt i32 [[LENGTH]], [[UMIN]]
; CHECK-NEXT: [[TMP7:%.*]] = freeze i1 [[TMP6]]
; CHECK-NEXT: [[TMP8:%.*]] = and i1 [[TMP7]], [[TMP5]]
; CHECK-NEXT: [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[TMP8]], [[WIDENABLE_COND]]
; CHECK-NEXT: br i1 [[EXIPLICIT_GUARD_COND]], label [[LOOP_PREHEADER:%.*]], label [[DEOPT:%.*]], !prof !0
; CHECK: deopt:
; CHECK-NEXT: [[DEOPTRET:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET]]
; CHECK: loop.preheader:
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED2:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED2]] ], [ 0, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH]]
; CHECK-NEXT: br i1 true, label [[GUARDED:%.*]], label [[DEOPT2:%.*]], !prof !0
; CHECK: deopt2:
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[DEOPTRET2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET2]]
; CHECK: guarded:
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[WITHIN_BOUNDS2:%.*]] = icmp ult i32 [[I]], [[LENGTH]]
; CHECK-NEXT: [[WB_COND:%.*]] = and i1 [[WITHIN_BOUNDS2]], true
; CHECK-NEXT: br i1 true, label [[GUARDED2]], label [[DEOPT3:%.*]], !prof !0
; CHECK: deopt3:
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: [[DEOPTRET3:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
; CHECK-NEXT: ret i32 [[DEOPTRET3]]
; CHECK: guarded2:
; CHECK-NEXT: [[I_I64:%.*]] = zext i32 [[I]] to i64
; CHECK-NEXT: [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
; CHECK-NEXT: [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
; CHECK-NEXT: store i32 0, i32* [[ARRAY_I_PTR]]
; CHECK-NEXT: [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
; CHECK-NEXT: [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
; CHECK: exit:
; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED2]] ]
; CHECK-NEXT: ret i32 [[RESULT]]
;
entry:
%widenable_cond = call i1 @llvm.experimental.widenable.condition()
%wc2 = call i1 @llvm.experimental.widenable.condition()
%exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
br i1 %exiplicit_guard_cond, label %loop.preheader, label %deopt, !prof !0
deopt:
%deoptret = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret
loop.preheader:
br label %loop
loop:
%loop.acc = phi i32 [ %loop.acc.next, %guarded2 ], [ 0, %loop.preheader ]
%i = phi i32 [ %i.next, %guarded2 ], [ 0, %loop.preheader ]
call void @unknown()
%within.bounds = icmp ult i32 %i, %length
br i1 %within.bounds, label %guarded, label %deopt2, !prof !0
deopt2:
call void @unknown()
%deoptret2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret2
guarded:
call void @unknown()
%within.bounds2 = icmp ult i32 %i, %length
%wb_cond = and i1 %within.bounds2, %wc2
br i1 %wb_cond, label %guarded2, label %deopt3, !prof !0
deopt3:
call void @unknown()
%deoptret3 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"() ]
ret i32 %deoptret3
guarded2:
%i.i64 = zext i32 %i to i64
%array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
%array.i = load i32, i32* %array.i.ptr, align 4
store i32 0, i32* %array.i.ptr
%loop.acc.next = add i32 %loop.acc, %array.i
%i.next = add nuw i32 %i, 1
%continue = icmp ult i32 %i.next, %n
br i1 %continue, label %loop, label %exit
exit:
%result = phi i32 [ %loop.acc.next, %guarded2 ]
ret i32 %result
}
[LoopPred/WC] Use a dominating widenable condition to remove analyze loop exits This implements a version of the predicateLoopExits transform from IndVarSimplify extended to exploit widenable conditions - and thus be much wider in scope of legality. The code structure ends up being almost entirely different, so I chose to duplicate this into the LoopPredication pass instead of trying to reuse the code in the IndVars. The core notions of the transform are as follows: If we have a widenable condition which controls entry into the loop, we're allowed to widen it arbitrarily. Given that, it's simply a *profitability* question as to what conditions to fold into the widenable branch. To avoid pass ordering issues, we want to avoid widening cases that would otherwise be dischargeable. Or... widen in a form which can still be discharged. Thus, we phrase the transform as selecting one analyzeable exit from the set of analyzeable exits to keep. This avoids creating pass ordering complexities. Since none of the above proves that we actually exit through our analyzeable exits - we might exit through something else entirely - we limit ourselves to cases where a) the latch is analyzeable and b) the latch is predicted taken, and c) the exit being removed is statically cold. Differential Revision: https://reviews.llvm.org/D69830
2019-11-19 03:21:53 +08:00
declare void @unknown()
declare i1 @llvm.experimental.widenable.condition()
declare i32 @llvm.experimental.deoptimize.i32(...)
!0 = !{!"branch_weights", i32 1048576, i32 1}
!1 = !{i32 1, i32 -2147483648}
!2 = !{i32 0, i32 50}