forked from OSchip/llvm-project
147 lines
5.7 KiB
C++
147 lines
5.7 KiB
C++
//===-- AArch64SelectionDAGInfo.cpp - AArch64 SelectionDAG Info -----------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// This file implements the AArch64SelectionDAGInfo class.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "AArch64TargetMachine.h"
|
|
using namespace llvm;
|
|
|
|
#define DEBUG_TYPE "aarch64-selectiondag-info"
|
|
|
|
SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemset(
|
|
SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
|
|
SDValue Size, unsigned Align, bool isVolatile,
|
|
MachinePointerInfo DstPtrInfo) const {
|
|
// Check to see if there is a specialized entry-point for memory zeroing.
|
|
ConstantSDNode *V = dyn_cast<ConstantSDNode>(Src);
|
|
ConstantSDNode *SizeValue = dyn_cast<ConstantSDNode>(Size);
|
|
const AArch64Subtarget &STI =
|
|
DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
|
|
const char *bzeroName = (V && V->isNullValue())
|
|
? DAG.getTargetLoweringInfo().getLibcallName(RTLIB::BZERO) : nullptr;
|
|
// For small size (< 256), it is not beneficial to use bzero
|
|
// instead of memset.
|
|
if (bzeroName && (!SizeValue || SizeValue->getZExtValue() > 256)) {
|
|
const AArch64TargetLowering &TLI = *STI.getTargetLowering();
|
|
|
|
EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout());
|
|
Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
|
|
TargetLowering::ArgListTy Args;
|
|
TargetLowering::ArgListEntry Entry;
|
|
Entry.Node = Dst;
|
|
Entry.Ty = IntPtrTy;
|
|
Args.push_back(Entry);
|
|
Entry.Node = Size;
|
|
Args.push_back(Entry);
|
|
TargetLowering::CallLoweringInfo CLI(DAG);
|
|
CLI.setDebugLoc(dl)
|
|
.setChain(Chain)
|
|
.setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
|
|
DAG.getExternalSymbol(bzeroName, IntPtr),
|
|
std::move(Args))
|
|
.setDiscardResult();
|
|
std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
|
|
return CallResult.second;
|
|
}
|
|
return SDValue();
|
|
}
|
|
bool AArch64SelectionDAGInfo::generateFMAsInMachineCombiner(
|
|
CodeGenOpt::Level OptLevel) const {
|
|
return OptLevel >= CodeGenOpt::Aggressive;
|
|
}
|
|
|
|
static const int kSetTagLoopThreshold = 176;
|
|
|
|
static SDValue EmitUnrolledSetTag(SelectionDAG &DAG, const SDLoc &dl,
|
|
SDValue Chain, SDValue Ptr, uint64_t ObjSize,
|
|
const MachineMemOperand *BaseMemOperand,
|
|
bool ZeroData) {
|
|
MachineFunction &MF = DAG.getMachineFunction();
|
|
unsigned ObjSizeScaled = ObjSize / 16;
|
|
|
|
SDValue TagSrc = Ptr;
|
|
if (Ptr.getOpcode() == ISD::FrameIndex) {
|
|
int FI = cast<FrameIndexSDNode>(Ptr)->getIndex();
|
|
Ptr = DAG.getTargetFrameIndex(FI, MVT::i64);
|
|
// A frame index operand may end up as [SP + offset] => it is fine to use SP
|
|
// register as the tag source.
|
|
TagSrc = DAG.getRegister(AArch64::SP, MVT::i64);
|
|
}
|
|
|
|
const unsigned OpCode1 = ZeroData ? AArch64ISD::STZG : AArch64ISD::STG;
|
|
const unsigned OpCode2 = ZeroData ? AArch64ISD::STZ2G : AArch64ISD::ST2G;
|
|
|
|
SmallVector<SDValue, 8> OutChains;
|
|
unsigned OffsetScaled = 0;
|
|
while (OffsetScaled < ObjSizeScaled) {
|
|
if (ObjSizeScaled - OffsetScaled >= 2) {
|
|
SDValue AddrNode = DAG.getMemBasePlusOffset(Ptr, OffsetScaled * 16, dl);
|
|
SDValue St = DAG.getMemIntrinsicNode(
|
|
OpCode2, dl, DAG.getVTList(MVT::Other),
|
|
{Chain, TagSrc, AddrNode},
|
|
MVT::v4i64,
|
|
MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16 * 2));
|
|
OffsetScaled += 2;
|
|
OutChains.push_back(St);
|
|
continue;
|
|
}
|
|
|
|
if (ObjSizeScaled - OffsetScaled > 0) {
|
|
SDValue AddrNode = DAG.getMemBasePlusOffset(Ptr, OffsetScaled * 16, dl);
|
|
SDValue St = DAG.getMemIntrinsicNode(
|
|
OpCode1, dl, DAG.getVTList(MVT::Other),
|
|
{Chain, TagSrc, AddrNode},
|
|
MVT::v2i64,
|
|
MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16));
|
|
OffsetScaled += 1;
|
|
OutChains.push_back(St);
|
|
}
|
|
}
|
|
|
|
SDValue Res = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
|
|
return Res;
|
|
}
|
|
|
|
SDValue AArch64SelectionDAGInfo::EmitTargetCodeForSetTag(
|
|
SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr,
|
|
SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const {
|
|
uint64_t ObjSize = cast<ConstantSDNode>(Size)->getZExtValue();
|
|
assert(ObjSize % 16 == 0);
|
|
|
|
MachineFunction &MF = DAG.getMachineFunction();
|
|
MachineMemOperand *BaseMemOperand = MF.getMachineMemOperand(
|
|
DstPtrInfo, MachineMemOperand::MOStore, ObjSize, 16);
|
|
|
|
bool UseSetTagRangeLoop =
|
|
kSetTagLoopThreshold >= 0 && (int)ObjSize >= kSetTagLoopThreshold;
|
|
if (!UseSetTagRangeLoop)
|
|
return EmitUnrolledSetTag(DAG, dl, Chain, Addr, ObjSize, BaseMemOperand,
|
|
ZeroData);
|
|
|
|
if (ObjSize % 32 != 0) {
|
|
SDNode *St1 = DAG.getMachineNode(
|
|
ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex, dl,
|
|
{MVT::i64, MVT::Other},
|
|
{Addr, Addr, DAG.getTargetConstant(1, dl, MVT::i64), Chain});
|
|
DAG.setNodeMemRefs(cast<MachineSDNode>(St1), {BaseMemOperand});
|
|
ObjSize -= 16;
|
|
Addr = SDValue(St1, 0);
|
|
Chain = SDValue(St1, 1);
|
|
}
|
|
|
|
const EVT ResTys[] = {MVT::i64, MVT::i64, MVT::Other};
|
|
SDValue Ops[] = {DAG.getConstant(ObjSize, dl, MVT::i64), Addr, Chain};
|
|
SDNode *St = DAG.getMachineNode(
|
|
ZeroData ? AArch64::STZGloop : AArch64::STGloop, dl, ResTys, Ops);
|
|
|
|
DAG.setNodeMemRefs(cast<MachineSDNode>(St), {BaseMemOperand});
|
|
return SDValue(St, 2);
|
|
}
|