forked from OSchip/llvm-project
[AARCH64][NEON] Allow to sink operands for aarch64_neon_pmull
This teaches AArch64TargetLowering::shouldSinkOperands to sink the operands of aarch64_neon_pmull intrinsic. Differential Revision: https://reviews.llvm.org/D117944
This commit is contained in:
Sunho Kim
David Green
parent
6cda6d2f61
commit
44601f4956
|
|
@ -12197,6 +12197,12 @@ bool AArch64TargetLowering::shouldSinkOperands(
|
|||
Ops.push_back(&II->getOperandUse(1));
|
||||
return !Ops.empty();
|
||||
|
||||
case Intrinsic::aarch64_neon_pmull:
|
||||
if (!areExtractShuffleVectors(II->getOperand(0), II->getOperand(1)))
|
||||
return false;
|
||||
Ops.push_back(&II->getOperandUse(0));
|
||||
Ops.push_back(&II->getOperandUse(1));
|
||||
return true;
|
||||
case Intrinsic::aarch64_neon_pmull64:
|
||||
if (!areOperandsOfVmullHighP64(II->getArgOperand(0),
|
||||
II->getArgOperand(1)))
|
||||
|
|
|
|||
|
|
@ -0,0 +1,81 @@
|
|||
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
|
||||
; RUN: llc < %s -asm-verbose=0 -mtriple=aarch64-none-linux-gnu -mattr=+neon | FileCheck %s
|
||||
|
||||
; Check that pmull2 instruction is used for vmull_high_p8 intrinsic
|
||||
; even if shufflevector instructions are located in different basic blocks,
|
||||
; which can happen when vmull_high_p8 is used inside a loop body.
|
||||
;
|
||||
|
||||
define <8 x i16> @test_pmull2_sink(<16 x i8> %a, <16 x i8> %b, <8 x i16> %c, i1 %t) {
|
||||
; CHECK-LABEL: test_pmull2_sink:
|
||||
; CHECK: tbz w0, #0, .LBB0_2
|
||||
; CHECK-NEXT: pmull2 v0.8h, v0.16b, v1.16b
|
||||
; CHECK-NEXT: ret
|
||||
; CHECK-NEXT: .LBB0_2:
|
||||
; CHECK-NEXT: mov v0.16b, v2.16b
|
||||
; CHECK-NEXT: ret
|
||||
entry:
|
||||
%0 = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
|
||||
br i1 %t, label %if.then, label %cleanup
|
||||
|
||||
if.then:
|
||||
%1 = shufflevector <16 x i8> %b, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
|
||||
%res = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %0, <8 x i8> %1)
|
||||
br label %cleanup
|
||||
|
||||
cleanup:
|
||||
%retval = phi <8 x i16> [ %res, %if.then ], [ %c, %entry ]
|
||||
ret <8 x i16> %retval
|
||||
}
|
||||
|
||||
define <8 x i16> @test_pmull2_sink2(<16 x i8> %a, <16 x i8> %b, <8 x i16> %c, i1 %t) {
|
||||
; CHECK-LABEL: test_pmull2_sink2:
|
||||
; CHECK: tbz w0, #0, .LBB1_2
|
||||
; CHECK-NEXT: pmull2 v0.8h, v0.16b, v0.16b
|
||||
; CHECK-NEXT: ret
|
||||
; CHECK-NEXT: .LBB1_2:
|
||||
; CHECK-NEXT: mov v0.16b, v2.16b
|
||||
; CHECK-NEXT: ret
|
||||
entry:
|
||||
%0 = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
|
||||
%1 = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
|
||||
br i1 %t, label %if.then, label %cleanup
|
||||
|
||||
if.then:
|
||||
%res = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %0, <8 x i8> %1)
|
||||
br label %cleanup
|
||||
|
||||
cleanup:
|
||||
%retval = phi <8 x i16> [ %res, %if.then ], [ %c, %entry ]
|
||||
ret <8 x i16> %retval
|
||||
}
|
||||
|
||||
define <8 x i16> @test_pmull2_sink3(<16 x i8> %a, <16 x i8> %b, <8 x i16> %c, i1 %t, i1 %t2) {
|
||||
; CHECK-LABEL: test_pmull2_sink3:
|
||||
; CHECK: tbz w0, #0, .LBB2_2
|
||||
; CHECK-NEXT: tbz w1, #0, .LBB2_3
|
||||
; CHECK-NEXT: .LBB2_2:
|
||||
; CHECK-NEXT: pmull2 v0.8h, v0.16b, v1.16b
|
||||
; CHECK-NEXT: ret
|
||||
; CHECK-NEXT: .LBB2_3:
|
||||
; CHECK-NEXT: pmull2 v0.8h, v0.16b, v0.16b
|
||||
; CHECK-NEXT: ret
|
||||
entry:
|
||||
%0 = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
|
||||
br i1 %t, label %if.then, label %if.then.2
|
||||
|
||||
if.then:
|
||||
%res = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %0, <8 x i8> %0)
|
||||
br i1 %t2, label %if.then.2, label %cleanup
|
||||
|
||||
if.then.2:
|
||||
%1 = shufflevector <16 x i8> %b, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
|
||||
%res2 = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %0, <8 x i8> %1)
|
||||
br label %cleanup
|
||||
|
||||
cleanup:
|
||||
%retval = phi <8 x i16> [ %res2, %if.then.2 ], [ %res, %if.then ]
|
||||
ret <8 x i16> %retval
|
||||
}
|
||||
|
||||
declare <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8>, <8 x i8>)
|
||||
|
|
@ -397,3 +397,100 @@ for.cond4.preheader.us.preheader: ; preds = %for.cond4.preheader
|
|||
for.cond4.preheader.preheader: ; preds = %for.cond4.preheader.lr.ph
|
||||
ret <4 x i32> zeroinitializer
|
||||
}
|
||||
|
||||
declare <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8>, <8 x i8>)
|
||||
|
||||
define <8 x i16> @sink_shufflevector_pmull(<16 x i8> %a, <16 x i8> %b) {
|
||||
; CHECK-LABEL: @sink_shufflevector_pmull(
|
||||
; CHECK-NEXT: entry:
|
||||
; CHECK-NEXT: br i1 poison, label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
|
||||
; CHECK: if.then:
|
||||
; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <16 x i8> [[A:%.*]], <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
|
||||
; CHECK-NEXT: [[S2:%.*]] = shufflevector <16 x i8> [[B:%.*]], <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
|
||||
; CHECK-NEXT: [[VMULL0:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> [[TMP0]], <8 x i8> [[S2]])
|
||||
; CHECK-NEXT: ret <8 x i16> [[VMULL0]]
|
||||
; CHECK: if.else:
|
||||
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i8> [[A]], <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
|
||||
; CHECK-NEXT: [[S4:%.*]] = shufflevector <16 x i8> [[B]], <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
|
||||
; CHECK-NEXT: [[VMULL1:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> [[TMP1]], <8 x i8> [[S4]])
|
||||
; CHECK-NEXT: ret <8 x i16> [[VMULL1]]
|
||||
;
|
||||
entry:
|
||||
%s1 = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
|
||||
%s3 = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
|
||||
br i1 poison, label %if.then, label %if.else
|
||||
|
||||
if.then:
|
||||
%s2 = shufflevector <16 x i8> %b, <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
|
||||
%vmull0 = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %s1, <8 x i8> %s2)
|
||||
ret <8 x i16> %vmull0
|
||||
|
||||
if.else:
|
||||
%s4 = shufflevector <16 x i8> %b, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
|
||||
%vmull1 = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %s3, <8 x i8> %s4)
|
||||
ret <8 x i16> %vmull1
|
||||
}
|
||||
|
||||
; Indexed pmull is not available on aarch64. Shuffle vector should not be sunk here.
|
||||
define <8 x i16> @no_sink_splatvector_pmull(<16 x i8> %a, <16 x i8> %b) {
|
||||
; CHECK-LABEL: @no_sink_splatvector_pmull(
|
||||
; CHECK-NEXT: entry:
|
||||
; CHECK-NEXT: [[S1:%.*]] = shufflevector <16 x i8> [[A:%.*]], <16 x i8> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
|
||||
; CHECK-NEXT: [[S3:%.*]] = shufflevector <16 x i8> [[A]], <16 x i8> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
|
||||
; CHECK-NEXT: br i1 poison, label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
|
||||
; CHECK: if.then:
|
||||
; CHECK-NEXT: [[S2:%.*]] = shufflevector <16 x i8> [[B:%.*]], <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
|
||||
; CHECK-NEXT: [[VMULL0:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> [[S1]], <8 x i8> [[S2]])
|
||||
; CHECK-NEXT: ret <8 x i16> [[VMULL0]]
|
||||
; CHECK: if.else:
|
||||
; CHECK-NEXT: [[S4:%.*]] = shufflevector <16 x i8> [[B]], <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
|
||||
; CHECK-NEXT: [[VMULL1:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> [[S3]], <8 x i8> [[S4]])
|
||||
; CHECK-NEXT: ret <8 x i16> [[VMULL1]]
|
||||
;
|
||||
entry:
|
||||
%s1 = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
|
||||
%s3 = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
|
||||
br i1 poison, label %if.then, label %if.else
|
||||
|
||||
if.then:
|
||||
%s2 = shufflevector <16 x i8> %b, <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
|
||||
%vmull0 = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %s1, <8 x i8> %s2)
|
||||
ret <8 x i16> %vmull0
|
||||
|
||||
if.else:
|
||||
%s4 = shufflevector <16 x i8> %b, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
|
||||
%vmull1 = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %s3, <8 x i8> %s4)
|
||||
ret <8 x i16> %vmull1
|
||||
}
|
||||
|
||||
; Mask used are not suitable for pmull. Shuffle vector should not be sunk here.
|
||||
define <8 x i16> @no_sink_shufflevector_pmull(<16 x i8> %a, <16 x i8> %b) {
|
||||
; CHECK-LABEL: @no_sink_shufflevector_pmull(
|
||||
; CHECK-NEXT: entry:
|
||||
; CHECK-NEXT: [[S1:%.*]] = shufflevector <16 x i8> [[A:%.*]], <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
|
||||
; CHECK-NEXT: [[S3:%.*]] = shufflevector <16 x i8> [[A]], <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
|
||||
; CHECK-NEXT: br i1 poison, label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
|
||||
; CHECK: if.then:
|
||||
; CHECK-NEXT: [[S2:%.*]] = shufflevector <16 x i8> [[B:%.*]], <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
|
||||
; CHECK-NEXT: [[VMULL0:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> [[S1]], <8 x i8> [[S2]])
|
||||
; CHECK-NEXT: ret <8 x i16> [[VMULL0]]
|
||||
; CHECK: if.else:
|
||||
; CHECK-NEXT: [[S4:%.*]] = shufflevector <16 x i8> [[B]], <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
|
||||
; CHECK-NEXT: [[VMULL1:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> [[S3]], <8 x i8> [[S4]])
|
||||
; CHECK-NEXT: ret <8 x i16> [[VMULL1]]
|
||||
;
|
||||
entry:
|
||||
%s1 = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
|
||||
%s3 = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
|
||||
br i1 poison, label %if.then, label %if.else
|
||||
|
||||
if.then:
|
||||
%s2 = shufflevector <16 x i8> %b, <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
|
||||
%vmull0 = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %s1, <8 x i8> %s2)
|
||||
ret <8 x i16> %vmull0
|
||||
|
||||
if.else:
|
||||
%s4 = shufflevector <16 x i8> %b, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
|
||||
%vmull1 = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %s3, <8 x i8> %s4)
|
||||
ret <8 x i16> %vmull1
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue