llvm-project/llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp

//===- AArch64InstructionSelector.cpp ----------------------------*- C++ -*-==//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
/// \file
/// This file implements the targeting of the InstructionSelector class for
/// AArch64.
/// \todo This should be generated by TableGen.
//===----------------------------------------------------------------------===//

#include "AArch64InstrInfo.h"
#include "AArch64MachineFunctionInfo.h"
#include "AArch64RegisterBankInfo.h"
#include "AArch64RegisterInfo.h"
#include "AArch64Subtarget.h"
#include "AArch64TargetMachine.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
#include "llvm/CodeGen/GlobalISel/Utils.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/IR/Type.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"

#define DEBUG_TYPE "aarch64-isel"

#include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h"

using namespace llvm;

namespace {

#define GET_GLOBALISEL_PREDICATE_BITSET
#include "AArch64GenGlobalISel.inc"
#undef GET_GLOBALISEL_PREDICATE_BITSET

class AArch64InstructionSelector : public InstructionSelector {
public:
  AArch64InstructionSelector(const AArch64TargetMachine &TM,
                             const AArch64Subtarget &STI,
                             const AArch64RegisterBankInfo &RBI);

  bool select(MachineInstr &I) const override;

private:
  /// tblgen-erated 'select' implementation, used as the initial selector for
  /// the patterns that don't require complex C++.
  bool selectImpl(MachineInstr &I) const;

  bool selectVaStartAAPCS(MachineInstr &I, MachineFunction &MF,
                          MachineRegisterInfo &MRI) const;
  bool selectVaStartDarwin(MachineInstr &I, MachineFunction &MF,
                           MachineRegisterInfo &MRI) const;

  bool selectCompareBranch(MachineInstr &I, MachineFunction &MF,
                           MachineRegisterInfo &MRI) const;

  ComplexRendererFn selectArithImmed(MachineOperand &Root) const;

  ComplexRendererFn selectAddrModeUnscaled(MachineOperand &Root,
                                           unsigned Size) const;

  ComplexRendererFn selectAddrModeUnscaled8(MachineOperand &Root) const {
    return selectAddrModeUnscaled(Root, 1);
  }
  ComplexRendererFn selectAddrModeUnscaled16(MachineOperand &Root) const {
    return selectAddrModeUnscaled(Root, 2);
  }
  ComplexRendererFn selectAddrModeUnscaled32(MachineOperand &Root) const {
    return selectAddrModeUnscaled(Root, 4);
  }
  ComplexRendererFn selectAddrModeUnscaled64(MachineOperand &Root) const {
    return selectAddrModeUnscaled(Root, 8);
  }
  ComplexRendererFn selectAddrModeUnscaled128(MachineOperand &Root) const {
    return selectAddrModeUnscaled(Root, 16);
  }

  ComplexRendererFn selectAddrModeIndexed(MachineOperand &Root,
                                          unsigned Size) const;
  template <int Width>
  ComplexRendererFn selectAddrModeIndexed(MachineOperand &Root) const {
    return selectAddrModeIndexed(Root, Width / 8);
  }

  const AArch64TargetMachine &TM;
  const AArch64Subtarget &STI;
  const AArch64InstrInfo &TII;
  const AArch64RegisterInfo &TRI;
  const AArch64RegisterBankInfo &RBI;

#define GET_GLOBALISEL_PREDICATES_DECL
#include "AArch64GenGlobalISel.inc"
#undef GET_GLOBALISEL_PREDICATES_DECL

// We declare the temporaries used by selectImpl() in the class to minimize the
// cost of constructing placeholder values.
#define GET_GLOBALISEL_TEMPORARIES_DECL
#include "AArch64GenGlobalISel.inc"
#undef GET_GLOBALISEL_TEMPORARIES_DECL
};

} // end anonymous namespace

#define GET_GLOBALISEL_IMPL
#include "AArch64GenGlobalISel.inc"
#undef GET_GLOBALISEL_IMPL

AArch64InstructionSelector::AArch64InstructionSelector(
    const AArch64TargetMachine &TM, const AArch64Subtarget &STI,
    const AArch64RegisterBankInfo &RBI)
    : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()),
      TRI(*STI.getRegisterInfo()), RBI(RBI),
#define GET_GLOBALISEL_PREDICATES_INIT
#include "AArch64GenGlobalISel.inc"
#undef GET_GLOBALISEL_PREDICATES_INIT
#define GET_GLOBALISEL_TEMPORARIES_INIT
#include "AArch64GenGlobalISel.inc"
#undef GET_GLOBALISEL_TEMPORARIES_INIT
{
}

// FIXME: This should be target-independent, inferred from the types declared
// for each class in the bank.
static const TargetRegisterClass *
getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB,
                         const RegisterBankInfo &RBI) {
  if (RB.getID() == AArch64::GPRRegBankID) {
    if (Ty.getSizeInBits() <= 32)
      return &AArch64::GPR32RegClass;
    if (Ty.getSizeInBits() == 64)
      return &AArch64::GPR64RegClass;
    return nullptr;
  }

  if (RB.getID() == AArch64::FPRRegBankID) {
    if (Ty.getSizeInBits() == 32)
      return &AArch64::FPR32RegClass;
    if (Ty.getSizeInBits() == 64)
      return &AArch64::FPR64RegClass;
    if (Ty.getSizeInBits() == 128)
      return &AArch64::FPR128RegClass;
    return nullptr;
  }

  return nullptr;
}

/// Check whether \p I is a currently unsupported binary operation:
/// - it has an unsized type
/// - an operand is not a vreg
/// - all operands are not in the same bank
/// These are checks that should someday live in the verifier, but right now,
/// these are mostly limitations of the aarch64 selector.
static bool unsupportedBinOp(const MachineInstr &I,
                             const AArch64RegisterBankInfo &RBI,
                             const MachineRegisterInfo &MRI,
                             const AArch64RegisterInfo &TRI) {
  LLT Ty = MRI.getType(I.getOperand(0).getReg());
  if (!Ty.isValid()) {
    DEBUG(dbgs() << "Generic binop register should be typed\n");
    return true;
  }

  const RegisterBank *PrevOpBank = nullptr;
  for (auto &MO : I.operands()) {
    // FIXME: Support non-register operands.
    if (!MO.isReg()) {
      DEBUG(dbgs() << "Generic inst non-reg operands are unsupported\n");
      return true;
    }

    // FIXME: Can generic operations have physical registers operands? If
    // so, this will need to be taught about that, and we'll need to get the
    // bank out of the minimal class for the register.
    // Either way, this needs to be documented (and possibly verified).
    if (!TargetRegisterInfo::isVirtualRegister(MO.getReg())) {
      DEBUG(dbgs() << "Generic inst has physical register operand\n");
      return true;
    }

    const RegisterBank *OpBank = RBI.getRegBank(MO.getReg(), MRI, TRI);
    if (!OpBank) {
      DEBUG(dbgs() << "Generic register has no bank or class\n");
      return true;
    }

    if (PrevOpBank && OpBank != PrevOpBank) {
      DEBUG(dbgs() << "Generic inst operands have different banks\n");
      return true;
    }
    PrevOpBank = OpBank;
  }
  return false;
}

/// Select the AArch64 opcode for the basic binary operation \p GenericOpc
/// (such as G_OR or G_SDIV), appropriate for the register bank \p RegBankID
/// and of size \p OpSize.
/// \returns \p GenericOpc if the combination is unsupported.
static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID,
                               unsigned OpSize) {
  switch (RegBankID) {
  case AArch64::GPRRegBankID:
    if (OpSize == 32) {
      switch (GenericOpc) {
      case TargetOpcode::G_SHL:
        return AArch64::LSLVWr;
      case TargetOpcode::G_LSHR:
        return AArch64::LSRVWr;
      case TargetOpcode::G_ASHR:
        return AArch64::ASRVWr;
      default:
        return GenericOpc;
      }
    } else if (OpSize == 64) {
      switch (GenericOpc) {
      case TargetOpcode::G_GEP:
        return AArch64::ADDXrr;
      case TargetOpcode::G_SHL:
        return AArch64::LSLVXr;
      case TargetOpcode::G_LSHR:
        return AArch64::LSRVXr;
      case TargetOpcode::G_ASHR:
        return AArch64::ASRVXr;
      default:
        return GenericOpc;
      }
    }
    break;
  case AArch64::FPRRegBankID:
    switch (OpSize) {
    case 32:
      switch (GenericOpc) {
      case TargetOpcode::G_FADD:
        return AArch64::FADDSrr;
      case TargetOpcode::G_FSUB:
        return AArch64::FSUBSrr;
      case TargetOpcode::G_FMUL:
        return AArch64::FMULSrr;
      case TargetOpcode::G_FDIV:
        return AArch64::FDIVSrr;
      default:
        return GenericOpc;
      }
    case 64:
      switch (GenericOpc) {
      case TargetOpcode::G_FADD:
        return AArch64::FADDDrr;
      case TargetOpcode::G_FSUB:
        return AArch64::FSUBDrr;
      case TargetOpcode::G_FMUL:
        return AArch64::FMULDrr;
      case TargetOpcode::G_FDIV:
        return AArch64::FDIVDrr;
      case TargetOpcode::G_OR:
        return AArch64::ORRv8i8;
      default:
        return GenericOpc;
      }
    }
    break;
  }
  return GenericOpc;
}

/// Select the AArch64 opcode for the G_LOAD or G_STORE operation \p GenericOpc,
/// appropriate for the (value) register bank \p RegBankID and of memory access
/// size \p OpSize.  This returns the variant with the base+unsigned-immediate
/// addressing mode (e.g., LDRXui).
/// \returns \p GenericOpc if the combination is unsupported.
static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID,
                                    unsigned OpSize) {
  const bool isStore = GenericOpc == TargetOpcode::G_STORE;
  switch (RegBankID) {
  case AArch64::GPRRegBankID:
    switch (OpSize) {
    case 8:
      return isStore ? AArch64::STRBBui : AArch64::LDRBBui;
    case 16:
      return isStore ? AArch64::STRHHui : AArch64::LDRHHui;
    case 32:
      return isStore ? AArch64::STRWui : AArch64::LDRWui;
    case 64:
      return isStore ? AArch64::STRXui : AArch64::LDRXui;
    }
    break;
  case AArch64::FPRRegBankID:
    switch (OpSize) {
    case 8:
      return isStore ? AArch64::STRBui : AArch64::LDRBui;
    case 16:
      return isStore ? AArch64::STRHui : AArch64::LDRHui;
    case 32:
      return isStore ? AArch64::STRSui : AArch64::LDRSui;
    case 64:
      return isStore ? AArch64::STRDui : AArch64::LDRDui;
    }
    break;
  }
  return GenericOpc;
}

static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
                       MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
                       const RegisterBankInfo &RBI) {

  unsigned DstReg = I.getOperand(0).getReg();
  if (TargetRegisterInfo::isPhysicalRegister(DstReg)) {
    assert(I.isCopy() && "Generic operators do not allow physical registers");
    return true;
  }

  const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
  const unsigned DstSize = MRI.getType(DstReg).getSizeInBits();
  unsigned SrcReg = I.getOperand(1).getReg();
  const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);
  (void)SrcSize;
  assert((!TargetRegisterInfo::isPhysicalRegister(SrcReg) || I.isCopy()) &&
         "No phys reg on generic operators");
  assert(
      (DstSize == SrcSize ||
       // Copies are a mean to setup initial types, the number of
       // bits may not exactly match.
       (TargetRegisterInfo::isPhysicalRegister(SrcReg) &&
        DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI)) ||
       // Copies are a mean to copy bits around, as long as we are
       // on the same register class, that's fine. Otherwise, that
       // means we need some SUBREG_TO_REG or AND & co.
       (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) &&
      "Copy with different width?!");
  assert((DstSize <= 64 || RegBank.getID() == AArch64::FPRRegBankID) &&
         "GPRs cannot get more than 64-bit width values");
  const TargetRegisterClass *RC = nullptr;

  if (RegBank.getID() == AArch64::FPRRegBankID) {
    if (DstSize <= 16)
      RC = &AArch64::FPR16RegClass;
    else if (DstSize <= 32)
      RC = &AArch64::FPR32RegClass;
    else if (DstSize <= 64)
      RC = &AArch64::FPR64RegClass;
    else if (DstSize <= 128)
      RC = &AArch64::FPR128RegClass;
    else {
      DEBUG(dbgs() << "Unexpected bitcast size " << DstSize << '\n');
      return false;
    }
  } else {
    assert(RegBank.getID() == AArch64::GPRRegBankID &&
           "Bitcast for the flags?");
    RC =
        DstSize <= 32 ? &AArch64::GPR32allRegClass : &AArch64::GPR64allRegClass;
  }

  // No need to constrain SrcReg. It will get constrained when
  // we hit another of its use or its defs.
  // Copies do not have constraints.
  if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) {
    DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
                 << " operand\n");
    return false;
  }
  I.setDesc(TII.get(AArch64::COPY));
  return true;
}

static unsigned selectFPConvOpc(unsigned GenericOpc, LLT DstTy, LLT SrcTy) {
  if (!DstTy.isScalar() || !SrcTy.isScalar())
    return GenericOpc;

  const unsigned DstSize = DstTy.getSizeInBits();
  const unsigned SrcSize = SrcTy.getSizeInBits();

  switch (DstSize) {
  case 32:
    switch (SrcSize) {
    case 32:
      switch (GenericOpc) {
      case TargetOpcode::G_SITOFP:
        return AArch64::SCVTFUWSri;
      case TargetOpcode::G_UITOFP:
        return AArch64::UCVTFUWSri;
      case TargetOpcode::G_FPTOSI:
        return AArch64::FCVTZSUWSr;
      case TargetOpcode::G_FPTOUI:
        return AArch64::FCVTZUUWSr;
      default:
        return GenericOpc;
      }
    case 64:
      switch (GenericOpc) {
      case TargetOpcode::G_SITOFP:
        return AArch64::SCVTFUXSri;
      case TargetOpcode::G_UITOFP:
        return AArch64::UCVTFUXSri;
      case TargetOpcode::G_FPTOSI:
        return AArch64::FCVTZSUWDr;
      case TargetOpcode::G_FPTOUI:
        return AArch64::FCVTZUUWDr;
      default:
        return GenericOpc;
      }
    default:
      return GenericOpc;
    }
  case 64:
    switch (SrcSize) {
    case 32:
      switch (GenericOpc) {
      case TargetOpcode::G_SITOFP:
        return AArch64::SCVTFUWDri;
      case TargetOpcode::G_UITOFP:
        return AArch64::UCVTFUWDri;
      case TargetOpcode::G_FPTOSI:
        return AArch64::FCVTZSUXSr;
      case TargetOpcode::G_FPTOUI:
        return AArch64::FCVTZUUXSr;
      default:
        return GenericOpc;
      }
    case 64:
      switch (GenericOpc) {
      case TargetOpcode::G_SITOFP:
        return AArch64::SCVTFUXDri;
      case TargetOpcode::G_UITOFP:
        return AArch64::UCVTFUXDri;
      case TargetOpcode::G_FPTOSI:
        return AArch64::FCVTZSUXDr;
      case TargetOpcode::G_FPTOUI:
        return AArch64::FCVTZUUXDr;
      default:
        return GenericOpc;
      }
    default:
      return GenericOpc;
    }
  default:
    return GenericOpc;
  };
  return GenericOpc;
}

static AArch64CC::CondCode changeICMPPredToAArch64CC(CmpInst::Predicate P) {
  switch (P) {
  default:
    llvm_unreachable("Unknown condition code!");
  case CmpInst::ICMP_NE:
    return AArch64CC::NE;
  case CmpInst::ICMP_EQ:
    return AArch64CC::EQ;
  case CmpInst::ICMP_SGT:
    return AArch64CC::GT;
  case CmpInst::ICMP_SGE:
    return AArch64CC::GE;
  case CmpInst::ICMP_SLT:
    return AArch64CC::LT;
  case CmpInst::ICMP_SLE:
    return AArch64CC::LE;
  case CmpInst::ICMP_UGT:
    return AArch64CC::HI;
  case CmpInst::ICMP_UGE:
    return AArch64CC::HS;
  case CmpInst::ICMP_ULT:
    return AArch64CC::LO;
  case CmpInst::ICMP_ULE:
    return AArch64CC::LS;
  }
}

static void changeFCMPPredToAArch64CC(CmpInst::Predicate P,
                                      AArch64CC::CondCode &CondCode,
                                      AArch64CC::CondCode &CondCode2) {
  CondCode2 = AArch64CC::AL;
  switch (P) {
  default:
    llvm_unreachable("Unknown FP condition!");
  case CmpInst::FCMP_OEQ:
    CondCode = AArch64CC::EQ;
    break;
  case CmpInst::FCMP_OGT:
    CondCode = AArch64CC::GT;
    break;
  case CmpInst::FCMP_OGE:
    CondCode = AArch64CC::GE;
    break;
  case CmpInst::FCMP_OLT:
    CondCode = AArch64CC::MI;
    break;
  case CmpInst::FCMP_OLE:
    CondCode = AArch64CC::LS;
    break;
  case CmpInst::FCMP_ONE:
    CondCode = AArch64CC::MI;
    CondCode2 = AArch64CC::GT;
    break;
  case CmpInst::FCMP_ORD:
    CondCode = AArch64CC::VC;
    break;
  case CmpInst::FCMP_UNO:
    CondCode = AArch64CC::VS;
    break;
  case CmpInst::FCMP_UEQ:
    CondCode = AArch64CC::EQ;
    CondCode2 = AArch64CC::VS;
    break;
  case CmpInst::FCMP_UGT:
    CondCode = AArch64CC::HI;
    break;
  case CmpInst::FCMP_UGE:
    CondCode = AArch64CC::PL;
    break;
  case CmpInst::FCMP_ULT:
    CondCode = AArch64CC::LT;
    break;
  case CmpInst::FCMP_ULE:
    CondCode = AArch64CC::LE;
    break;
  case CmpInst::FCMP_UNE:
    CondCode = AArch64CC::NE;
    break;
  }
}

bool AArch64InstructionSelector::selectCompareBranch(
    MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {

  const unsigned CondReg = I.getOperand(0).getReg();
  MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
  MachineInstr *CCMI = MRI.getVRegDef(CondReg);
  if (CCMI->getOpcode() == TargetOpcode::G_TRUNC)
    CCMI = MRI.getVRegDef(CCMI->getOperand(1).getReg());
  if (CCMI->getOpcode() != TargetOpcode::G_ICMP)
    return false;

  unsigned LHS = CCMI->getOperand(2).getReg();
  unsigned RHS = CCMI->getOperand(3).getReg();
  if (!getConstantVRegVal(RHS, MRI))
    std::swap(RHS, LHS);

  const auto RHSImm = getConstantVRegVal(RHS, MRI);
  if (!RHSImm || *RHSImm != 0)
    return false;

  const RegisterBank &RB = *RBI.getRegBank(LHS, MRI, TRI);
  if (RB.getID() != AArch64::GPRRegBankID)
    return false;

  const auto Pred = (CmpInst::Predicate)CCMI->getOperand(1).getPredicate();
  if (Pred != CmpInst::ICMP_NE && Pred != CmpInst::ICMP_EQ)
    return false;

  const unsigned CmpWidth = MRI.getType(LHS).getSizeInBits();
  unsigned CBOpc = 0;
  if (CmpWidth <= 32)
    CBOpc = (Pred == CmpInst::ICMP_EQ ? AArch64::CBZW : AArch64::CBNZW);
  else if (CmpWidth == 64)
    CBOpc = (Pred == CmpInst::ICMP_EQ ? AArch64::CBZX : AArch64::CBNZX);
  else
    return false;

  auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CBOpc))
                 .addUse(LHS)
                 .addMBB(DestMBB);

  constrainSelectedInstRegOperands(*MIB.getInstr(), TII, TRI, RBI);
  I.eraseFromParent();
  return true;
}

bool AArch64InstructionSelector::selectVaStartAAPCS(
    MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
  return false;
}

bool AArch64InstructionSelector::selectVaStartDarwin(
    MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
  AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
  unsigned ListReg = I.getOperand(0).getReg();

  unsigned ArgsAddrReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);

  auto MIB =
      BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::ADDXri))
          .addDef(ArgsAddrReg)
          .addFrameIndex(FuncInfo->getVarArgsStackIndex())
          .addImm(0)
          .addImm(0);

  constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);

  MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::STRXui))
            .addUse(ArgsAddrReg)
            .addUse(ListReg)
            .addImm(0)
            .addMemOperand(*I.memoperands_begin());

  constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
  I.eraseFromParent();
  return true;
}

bool AArch64InstructionSelector::select(MachineInstr &I) const {
  assert(I.getParent() && "Instruction should be in a basic block!");
  assert(I.getParent()->getParent() && "Instruction should be in a function!");

  MachineBasicBlock &MBB = *I.getParent();
  MachineFunction &MF = *MBB.getParent();
  MachineRegisterInfo &MRI = MF.getRegInfo();

  unsigned Opcode = I.getOpcode();
  // G_PHI requires same handling as PHI
  if (!isPreISelGenericOpcode(Opcode) || Opcode == TargetOpcode::G_PHI) {
    // Certain non-generic instructions also need some special handling.

    if (Opcode ==  TargetOpcode::LOAD_STACK_GUARD)
      return constrainSelectedInstRegOperands(I, TII, TRI, RBI);

    if (Opcode == TargetOpcode::PHI || Opcode == TargetOpcode::G_PHI) {
      const unsigned DefReg = I.getOperand(0).getReg();
      const LLT DefTy = MRI.getType(DefReg);

      const TargetRegisterClass *DefRC = nullptr;
      if (TargetRegisterInfo::isPhysicalRegister(DefReg)) {
        DefRC = TRI.getRegClass(DefReg);
      } else {
        const RegClassOrRegBank &RegClassOrBank =
            MRI.getRegClassOrRegBank(DefReg);

        DefRC = RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
        if (!DefRC) {
          if (!DefTy.isValid()) {
            DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
            return false;
          }
          const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>();
          DefRC = getRegClassForTypeOnBank(DefTy, RB, RBI);
          if (!DefRC) {
            DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
            return false;
          }
        }
      }
      I.setDesc(TII.get(TargetOpcode::PHI));

      return RBI.constrainGenericRegister(DefReg, *DefRC, MRI);
    }

    if (I.isCopy())
      return selectCopy(I, TII, MRI, TRI, RBI);

    return true;
  }


  if (I.getNumOperands() != I.getNumExplicitOperands()) {
    DEBUG(dbgs() << "Generic instruction has unexpected implicit operands\n");
    return false;
  }

  if (selectImpl(I))
    return true;

  LLT Ty =
      I.getOperand(0).isReg() ? MRI.getType(I.getOperand(0).getReg()) : LLT{};

  switch (Opcode) {
  case TargetOpcode::G_BRCOND: {
    if (Ty.getSizeInBits() > 32) {
      // We shouldn't need this on AArch64, but it would be implemented as an
      // EXTRACT_SUBREG followed by a TBNZW because TBNZX has no encoding if the
      // bit being tested is < 32.
      DEBUG(dbgs() << "G_BRCOND has type: " << Ty
                   << ", expected at most 32-bits");
      return false;
    }

    const unsigned CondReg = I.getOperand(0).getReg();
    MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();

    if (selectCompareBranch(I, MF, MRI))
      return true;

    auto MIB = BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::TBNZW))
                   .addUse(CondReg)
                   .addImm(/*bit offset=*/0)
                   .addMBB(DestMBB);

    I.eraseFromParent();
    return constrainSelectedInstRegOperands(*MIB.getInstr(), TII, TRI, RBI);
  }

  case TargetOpcode::G_BRINDIRECT: {
    I.setDesc(TII.get(AArch64::BR));
    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
  }

  case TargetOpcode::G_FCONSTANT:
  case TargetOpcode::G_CONSTANT: {
    const bool isFP = Opcode == TargetOpcode::G_FCONSTANT;

    const LLT s32 = LLT::scalar(32);
    const LLT s64 = LLT::scalar(64);
    const LLT p0 = LLT::pointer(0, 64);

    const unsigned DefReg = I.getOperand(0).getReg();
    const LLT DefTy = MRI.getType(DefReg);
    const unsigned DefSize = DefTy.getSizeInBits();
    const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);

    // FIXME: Redundant check, but even less readable when factored out.
    if (isFP) {
      if (Ty != s32 && Ty != s64) {
        DEBUG(dbgs() << "Unable to materialize FP " << Ty
                     << " constant, expected: " << s32 << " or " << s64
                     << '\n');
        return false;
      }

      if (RB.getID() != AArch64::FPRRegBankID) {
        DEBUG(dbgs() << "Unable to materialize FP " << Ty
                     << " constant on bank: " << RB << ", expected: FPR\n");
        return false;
      }

      // The case when we have 0.0 is covered by tablegen. Reject it here so we
      // can be sure tablegen works correctly and isn't rescued by this code.
      if (I.getOperand(1).getFPImm()->getValueAPF().isExactlyValue(0.0))
        return false;
    } else {
      // s32 and s64 are covered by tablegen.
      if (Ty != p0) {
        DEBUG(dbgs() << "Unable to materialize integer " << Ty
                     << " constant, expected: " << s32 << ", " << s64 << ", or "
                     << p0 << '\n');
        return false;
      }

      if (RB.getID() != AArch64::GPRRegBankID) {
        DEBUG(dbgs() << "Unable to materialize integer " << Ty
                     << " constant on bank: " << RB << ", expected: GPR\n");
        return false;
      }
    }

    const unsigned MovOpc =
        DefSize == 32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;

    I.setDesc(TII.get(MovOpc));

    if (isFP) {
      const TargetRegisterClass &GPRRC =
          DefSize == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass;
      const TargetRegisterClass &FPRRC =
          DefSize == 32 ? AArch64::FPR32RegClass : AArch64::FPR64RegClass;

      const unsigned DefGPRReg = MRI.createVirtualRegister(&GPRRC);
      MachineOperand &RegOp = I.getOperand(0);
      RegOp.setReg(DefGPRReg);

      BuildMI(MBB, std::next(I.getIterator()), I.getDebugLoc(),
              TII.get(AArch64::COPY))
          .addDef(DefReg)
          .addUse(DefGPRReg);

      if (!RBI.constrainGenericRegister(DefReg, FPRRC, MRI)) {
        DEBUG(dbgs() << "Failed to constrain G_FCONSTANT def operand\n");
        return false;
      }

      MachineOperand &ImmOp = I.getOperand(1);
      // FIXME: Is going through int64_t always correct?
      ImmOp.ChangeToImmediate(
          ImmOp.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
    } else if (I.getOperand(1).isCImm()) {
      uint64_t Val = I.getOperand(1).getCImm()->getZExtValue();
      I.getOperand(1).ChangeToImmediate(Val);
    } else if (I.getOperand(1).isImm()) {
      uint64_t Val = I.getOperand(1).getImm();
      I.getOperand(1).ChangeToImmediate(Val);
    }

    constrainSelectedInstRegOperands(I, TII, TRI, RBI);
    return true;
  }
  case TargetOpcode::G_EXTRACT: {
    LLT SrcTy = MRI.getType(I.getOperand(1).getReg());
    // Larger extracts are vectors, same-size extracts should be something else
    // by now (either split up or simplified to a COPY).
    if (SrcTy.getSizeInBits() > 64 || Ty.getSizeInBits() > 32)
      return false;

    I.setDesc(TII.get(AArch64::UBFMXri));
    MachineInstrBuilder(MF, I).addImm(I.getOperand(2).getImm() +
                                      Ty.getSizeInBits() - 1);

    unsigned DstReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
    BuildMI(MBB, std::next(I.getIterator()), I.getDebugLoc(),
            TII.get(AArch64::COPY))
        .addDef(I.getOperand(0).getReg())
        .addUse(DstReg, 0, AArch64::sub_32);
    RBI.constrainGenericRegister(I.getOperand(0).getReg(),
                                 AArch64::GPR32RegClass, MRI);
    I.getOperand(0).setReg(DstReg);

    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
  }

  case TargetOpcode::G_INSERT: {
    LLT SrcTy = MRI.getType(I.getOperand(2).getReg());
    // Larger inserts are vectors, same-size ones should be something else by
    // now (split up or turned into COPYs).
    if (Ty.getSizeInBits() > 64 || SrcTy.getSizeInBits() > 32)
      return false;

    I.setDesc(TII.get(AArch64::BFMXri));
    unsigned LSB = I.getOperand(3).getImm();
    unsigned Width = MRI.getType(I.getOperand(2).getReg()).getSizeInBits();
    I.getOperand(3).setImm((64 - LSB) % 64);
    MachineInstrBuilder(MF, I).addImm(Width - 1);

    unsigned SrcReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
    BuildMI(MBB, I.getIterator(), I.getDebugLoc(),
            TII.get(AArch64::SUBREG_TO_REG))
        .addDef(SrcReg)
        .addImm(0)
        .addUse(I.getOperand(2).getReg())
        .addImm(AArch64::sub_32);
    RBI.constrainGenericRegister(I.getOperand(2).getReg(),
                                 AArch64::GPR32RegClass, MRI);
    I.getOperand(2).setReg(SrcReg);

    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
  }
  case TargetOpcode::G_FRAME_INDEX: {
    // allocas and G_FRAME_INDEX are only supported in addrspace(0).
    if (Ty != LLT::pointer(0, 64)) {
      DEBUG(dbgs() << "G_FRAME_INDEX pointer has type: " << Ty
            << ", expected: " << LLT::pointer(0, 64) << '\n');
      return false;
    }
    I.setDesc(TII.get(AArch64::ADDXri));

    // MOs for a #0 shifted immediate.
    I.addOperand(MachineOperand::CreateImm(0));
    I.addOperand(MachineOperand::CreateImm(0));

    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
  }

  case TargetOpcode::G_GLOBAL_VALUE: {
    auto GV = I.getOperand(1).getGlobal();
    if (GV->isThreadLocal()) {
      // FIXME: we don't support TLS yet.
      return false;
    }
    unsigned char OpFlags = STI.ClassifyGlobalReference(GV, TM);
    if (OpFlags & AArch64II::MO_GOT) {
      I.setDesc(TII.get(AArch64::LOADgot));
      I.getOperand(1).setTargetFlags(OpFlags);
    } else {
      I.setDesc(TII.get(AArch64::MOVaddr));
      I.getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_PAGE);
      MachineInstrBuilder MIB(MF, I);
      MIB.addGlobalAddress(GV, I.getOperand(1).getOffset(),
                           OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
    }
    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
  }

  case TargetOpcode::G_LOAD:
  case TargetOpcode::G_STORE: {
    LLT MemTy = Ty;
    LLT PtrTy = MRI.getType(I.getOperand(1).getReg());

    if (PtrTy != LLT::pointer(0, 64)) {
      DEBUG(dbgs() << "Load/Store pointer has type: " << PtrTy
                   << ", expected: " << LLT::pointer(0, 64) << '\n');
      return false;
    }

    auto &MemOp = **I.memoperands_begin();
    if (MemOp.getOrdering() != AtomicOrdering::NotAtomic) {
      DEBUG(dbgs() << "Atomic load/store not supported yet\n");
      return false;
    }

    const unsigned PtrReg = I.getOperand(1).getReg();
#ifndef NDEBUG
    const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, MRI, TRI);
    // Sanity-check the pointer register.
    assert(PtrRB.getID() == AArch64::GPRRegBankID &&
           "Load/Store pointer operand isn't a GPR");
    assert(MRI.getType(PtrReg).isPointer() &&
           "Load/Store pointer operand isn't a pointer");
#endif

    const unsigned ValReg = I.getOperand(0).getReg();
    const RegisterBank &RB = *RBI.getRegBank(ValReg, MRI, TRI);

    const unsigned NewOpc =
        selectLoadStoreUIOp(I.getOpcode(), RB.getID(), MemTy.getSizeInBits());
    if (NewOpc == I.getOpcode())
      return false;

    I.setDesc(TII.get(NewOpc));

    uint64_t Offset = 0;
    auto *PtrMI = MRI.getVRegDef(PtrReg);

    // Try to fold a GEP into our unsigned immediate addressing mode.
    if (PtrMI->getOpcode() == TargetOpcode::G_GEP) {
      if (auto COff = getConstantVRegVal(PtrMI->getOperand(2).getReg(), MRI)) {
        int64_t Imm = *COff;
        const unsigned Size = MemTy.getSizeInBits() / 8;
        const unsigned Scale = Log2_32(Size);
        if ((Imm & (Size - 1)) == 0 && Imm >= 0 && Imm < (0x1000 << Scale)) {
          unsigned Ptr2Reg = PtrMI->getOperand(1).getReg();
          I.getOperand(1).setReg(Ptr2Reg);
          PtrMI = MRI.getVRegDef(Ptr2Reg);
          Offset = Imm / Size;
        }
      }
    }

    // If we haven't folded anything into our addressing mode yet, try to fold
    // a frame index into the base+offset.
    if (!Offset && PtrMI->getOpcode() == TargetOpcode::G_FRAME_INDEX)
      I.getOperand(1).ChangeToFrameIndex(PtrMI->getOperand(1).getIndex());

    I.addOperand(MachineOperand::CreateImm(Offset));

    // If we're storing a 0, use WZR/XZR.
    if (auto CVal = getConstantVRegVal(ValReg, MRI)) {
      if (*CVal == 0 && Opcode == TargetOpcode::G_STORE) {
        if (I.getOpcode() == AArch64::STRWui)
          I.getOperand(0).setReg(AArch64::WZR);
        else if (I.getOpcode() == AArch64::STRXui)
          I.getOperand(0).setReg(AArch64::XZR);
      }
    }

    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
  }

  case TargetOpcode::G_SMULH:
  case TargetOpcode::G_UMULH: {
    // Reject the various things we don't support yet.
    if (unsupportedBinOp(I, RBI, MRI, TRI))
      return false;

    const unsigned DefReg = I.getOperand(0).getReg();
    const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);

    if (RB.getID() != AArch64::GPRRegBankID) {
      DEBUG(dbgs() << "G_[SU]MULH on bank: " << RB << ", expected: GPR\n");
      return false;
    }

    if (Ty != LLT::scalar(64)) {
      DEBUG(dbgs() << "G_[SU]MULH has type: " << Ty
                   << ", expected: " << LLT::scalar(64) << '\n');
      return false;
    }

    unsigned NewOpc = I.getOpcode() == TargetOpcode::G_SMULH ? AArch64::SMULHrr
                                                             : AArch64::UMULHrr;
    I.setDesc(TII.get(NewOpc));

    // Now that we selected an opcode, we need to constrain the register
    // operands to use appropriate classes.
    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
  }
  case TargetOpcode::G_FADD:
  case TargetOpcode::G_FSUB:
  case TargetOpcode::G_FMUL:
  case TargetOpcode::G_FDIV:

  case TargetOpcode::G_OR:
  case TargetOpcode::G_SHL:
  case TargetOpcode::G_LSHR:
  case TargetOpcode::G_ASHR:
  case TargetOpcode::G_GEP: {
    // Reject the various things we don't support yet.
    if (unsupportedBinOp(I, RBI, MRI, TRI))
      return false;

    const unsigned OpSize = Ty.getSizeInBits();

    const unsigned DefReg = I.getOperand(0).getReg();
    const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);

    const unsigned NewOpc = selectBinaryOp(I.getOpcode(), RB.getID(), OpSize);
    if (NewOpc == I.getOpcode())
      return false;

    I.setDesc(TII.get(NewOpc));
    // FIXME: Should the type be always reset in setDesc?

    // Now that we selected an opcode, we need to constrain the register
    // operands to use appropriate classes.
    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
  }

  case TargetOpcode::G_PTR_MASK: {
    uint64_t Align = I.getOperand(2).getImm();
    if (Align >= 64 || Align == 0)
      return false;

    uint64_t Mask = ~((1ULL << Align) - 1);
    I.setDesc(TII.get(AArch64::ANDXri));
    I.getOperand(2).setImm(AArch64_AM::encodeLogicalImmediate(Mask, 64));

    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
  }
  case TargetOpcode::G_PTRTOINT:
  case TargetOpcode::G_TRUNC: {
    const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
    const LLT SrcTy = MRI.getType(I.getOperand(1).getReg());

    const unsigned DstReg = I.getOperand(0).getReg();
    const unsigned SrcReg = I.getOperand(1).getReg();

    const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
    const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);

    if (DstRB.getID() != SrcRB.getID()) {
      DEBUG(dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n");
      return false;
    }

    if (DstRB.getID() == AArch64::GPRRegBankID) {
      const TargetRegisterClass *DstRC =
          getRegClassForTypeOnBank(DstTy, DstRB, RBI);
      if (!DstRC)
        return false;

      const TargetRegisterClass *SrcRC =
          getRegClassForTypeOnBank(SrcTy, SrcRB, RBI);
      if (!SrcRC)
        return false;

      if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
          !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
        DEBUG(dbgs() << "Failed to constrain G_TRUNC/G_PTRTOINT\n");
        return false;
      }

      if (DstRC == SrcRC) {
        // Nothing to be done
      } else if (Opcode == TargetOpcode::G_TRUNC && DstTy == LLT::scalar(32) &&
                 SrcTy == LLT::scalar(64)) {
        llvm_unreachable("TableGen can import this case");
        return false;
      } else if (DstRC == &AArch64::GPR32RegClass &&
                 SrcRC == &AArch64::GPR64RegClass) {
        I.getOperand(1).setSubReg(AArch64::sub_32);
      } else {
        DEBUG(dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n");
        return false;
      }

      I.setDesc(TII.get(TargetOpcode::COPY));
      return true;
    } else if (DstRB.getID() == AArch64::FPRRegBankID) {
      if (DstTy == LLT::vector(4, 16) && SrcTy == LLT::vector(4, 32)) {
        I.setDesc(TII.get(AArch64::XTNv4i16));
        constrainSelectedInstRegOperands(I, TII, TRI, RBI);
        return true;
      }
    }

    return false;
  }

  case TargetOpcode::G_ANYEXT: {
    const unsigned DstReg = I.getOperand(0).getReg();
    const unsigned SrcReg = I.getOperand(1).getReg();

    const RegisterBank &RBDst = *RBI.getRegBank(DstReg, MRI, TRI);
    if (RBDst.getID() != AArch64::GPRRegBankID) {
      DEBUG(dbgs() << "G_ANYEXT on bank: " << RBDst << ", expected: GPR\n");
      return false;
    }

    const RegisterBank &RBSrc = *RBI.getRegBank(SrcReg, MRI, TRI);
    if (RBSrc.getID() != AArch64::GPRRegBankID) {
      DEBUG(dbgs() << "G_ANYEXT on bank: " << RBSrc << ", expected: GPR\n");
      return false;
    }

    const unsigned DstSize = MRI.getType(DstReg).getSizeInBits();

    if (DstSize == 0) {
      DEBUG(dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n");
      return false;
    }

    if (DstSize != 64 && DstSize > 32) {
      DEBUG(dbgs() << "G_ANYEXT to size: " << DstSize
                   << ", expected: 32 or 64\n");
      return false;
    }
    // At this point G_ANYEXT is just like a plain COPY, but we need
    // to explicitly form the 64-bit value if any.
    if (DstSize > 32) {
      unsigned ExtSrc = MRI.createVirtualRegister(&AArch64::GPR64allRegClass);
      BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG))
          .addDef(ExtSrc)
          .addImm(0)
          .addUse(SrcReg)
          .addImm(AArch64::sub_32);
      I.getOperand(1).setReg(ExtSrc);
    }
    return selectCopy(I, TII, MRI, TRI, RBI);
  }

  case TargetOpcode::G_ZEXT:
  case TargetOpcode::G_SEXT: {
    unsigned Opcode = I.getOpcode();
    const LLT DstTy = MRI.getType(I.getOperand(0).getReg()),
              SrcTy = MRI.getType(I.getOperand(1).getReg());
    const bool isSigned = Opcode == TargetOpcode::G_SEXT;
    const unsigned DefReg = I.getOperand(0).getReg();
    const unsigned SrcReg = I.getOperand(1).getReg();
    const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);

    if (RB.getID() != AArch64::GPRRegBankID) {
      DEBUG(dbgs() << TII.getName(I.getOpcode()) << " on bank: " << RB
                   << ", expected: GPR\n");
      return false;
    }

    MachineInstr *ExtI;
    if (DstTy == LLT::scalar(64)) {
      // FIXME: Can we avoid manually doing this?
      if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass, MRI)) {
        DEBUG(dbgs() << "Failed to constrain " << TII.getName(Opcode)
                     << " operand\n");
        return false;
      }

      const unsigned SrcXReg =
          MRI.createVirtualRegister(&AArch64::GPR64RegClass);
      BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG))
          .addDef(SrcXReg)
          .addImm(0)
          .addUse(SrcReg)
          .addImm(AArch64::sub_32);

      const unsigned NewOpc = isSigned ? AArch64::SBFMXri : AArch64::UBFMXri;
      ExtI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(NewOpc))
                 .addDef(DefReg)
                 .addUse(SrcXReg)
                 .addImm(0)
                 .addImm(SrcTy.getSizeInBits() - 1);
    } else if (DstTy.isScalar() && DstTy.getSizeInBits() <= 32) {
      const unsigned NewOpc = isSigned ? AArch64::SBFMWri : AArch64::UBFMWri;
      ExtI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(NewOpc))
                 .addDef(DefReg)
                 .addUse(SrcReg)
                 .addImm(0)
                 .addImm(SrcTy.getSizeInBits() - 1);
    } else {
      return false;
    }

    constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI);

    I.eraseFromParent();
    return true;
  }

  case TargetOpcode::G_SITOFP:
  case TargetOpcode::G_UITOFP:
  case TargetOpcode::G_FPTOSI:
  case TargetOpcode::G_FPTOUI: {
    const LLT DstTy = MRI.getType(I.getOperand(0).getReg()),
              SrcTy = MRI.getType(I.getOperand(1).getReg());
    const unsigned NewOpc = selectFPConvOpc(Opcode, DstTy, SrcTy);
    if (NewOpc == Opcode)
      return false;

    I.setDesc(TII.get(NewOpc));
    constrainSelectedInstRegOperands(I, TII, TRI, RBI);

    return true;
  }


  case TargetOpcode::G_INTTOPTR:
    // The importer is currently unable to import pointer types since they
    // didn't exist in SelectionDAG.
    return selectCopy(I, TII, MRI, TRI, RBI);

  case TargetOpcode::G_BITCAST:
    // Imported SelectionDAG rules can handle every bitcast except those that
    // bitcast from a type to the same type. Ideally, these shouldn't occur
    // but we might not run an optimizer that deletes them.
    if (MRI.getType(I.getOperand(0).getReg()) ==
        MRI.getType(I.getOperand(1).getReg()))
      return selectCopy(I, TII, MRI, TRI, RBI);
    return false;

  case TargetOpcode::G_SELECT: {
    if (MRI.getType(I.getOperand(1).getReg()) != LLT::scalar(1)) {
      DEBUG(dbgs() << "G_SELECT cond has type: " << Ty
                   << ", expected: " << LLT::scalar(1) << '\n');
      return false;
    }

    const unsigned CondReg = I.getOperand(1).getReg();
    const unsigned TReg = I.getOperand(2).getReg();
    const unsigned FReg = I.getOperand(3).getReg();

    unsigned CSelOpc = 0;

    if (Ty == LLT::scalar(32)) {
      CSelOpc = AArch64::CSELWr;
    } else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) {
      CSelOpc = AArch64::CSELXr;
    } else {
      return false;
    }

    MachineInstr &TstMI =
        *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ANDSWri))
             .addDef(AArch64::WZR)
             .addUse(CondReg)
             .addImm(AArch64_AM::encodeLogicalImmediate(1, 32));

    MachineInstr &CSelMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CSelOpc))
                                .addDef(I.getOperand(0).getReg())
                                .addUse(TReg)
                                .addUse(FReg)
                                .addImm(AArch64CC::NE);

    constrainSelectedInstRegOperands(TstMI, TII, TRI, RBI);
    constrainSelectedInstRegOperands(CSelMI, TII, TRI, RBI);

    I.eraseFromParent();
    return true;
  }
  case TargetOpcode::G_ICMP: {
    if (Ty != LLT::scalar(32)) {
      DEBUG(dbgs() << "G_ICMP result has type: " << Ty
                   << ", expected: " << LLT::scalar(32) << '\n');
      return false;
    }

    unsigned CmpOpc = 0;
    unsigned ZReg = 0;

    LLT CmpTy = MRI.getType(I.getOperand(2).getReg());
    if (CmpTy == LLT::scalar(32)) {
      CmpOpc = AArch64::SUBSWrr;
      ZReg = AArch64::WZR;
    } else if (CmpTy == LLT::scalar(64) || CmpTy.isPointer()) {
      CmpOpc = AArch64::SUBSXrr;
      ZReg = AArch64::XZR;
    } else {
      return false;
    }

    // CSINC increments the result by one when the condition code is false.
    // Therefore, we have to invert the predicate to get an increment by 1 when
    // the predicate is true.
    const AArch64CC::CondCode invCC =
        changeICMPPredToAArch64CC(CmpInst::getInversePredicate(
            (CmpInst::Predicate)I.getOperand(1).getPredicate()));

    MachineInstr &CmpMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CmpOpc))
                               .addDef(ZReg)
                               .addUse(I.getOperand(2).getReg())
                               .addUse(I.getOperand(3).getReg());

    MachineInstr &CSetMI =
        *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
             .addDef(I.getOperand(0).getReg())
             .addUse(AArch64::WZR)
             .addUse(AArch64::WZR)
             .addImm(invCC);

    constrainSelectedInstRegOperands(CmpMI, TII, TRI, RBI);
    constrainSelectedInstRegOperands(CSetMI, TII, TRI, RBI);

    I.eraseFromParent();
    return true;
  }

  case TargetOpcode::G_FCMP: {
    if (Ty != LLT::scalar(32)) {
      DEBUG(dbgs() << "G_FCMP result has type: " << Ty
                   << ", expected: " << LLT::scalar(32) << '\n');
      return false;
    }

    unsigned CmpOpc = 0;
    LLT CmpTy = MRI.getType(I.getOperand(2).getReg());
    if (CmpTy == LLT::scalar(32)) {
      CmpOpc = AArch64::FCMPSrr;
    } else if (CmpTy == LLT::scalar(64)) {
      CmpOpc = AArch64::FCMPDrr;
    } else {
      return false;
    }

    // FIXME: regbank

    AArch64CC::CondCode CC1, CC2;
    changeFCMPPredToAArch64CC(
        (CmpInst::Predicate)I.getOperand(1).getPredicate(), CC1, CC2);

    MachineInstr &CmpMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CmpOpc))
                               .addUse(I.getOperand(2).getReg())
                               .addUse(I.getOperand(3).getReg());

    const unsigned DefReg = I.getOperand(0).getReg();
    unsigned Def1Reg = DefReg;
    if (CC2 != AArch64CC::AL)
      Def1Reg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);

    MachineInstr &CSetMI =
        *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
             .addDef(Def1Reg)
             .addUse(AArch64::WZR)
             .addUse(AArch64::WZR)
             .addImm(getInvertedCondCode(CC1));

    if (CC2 != AArch64CC::AL) {
      unsigned Def2Reg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
      MachineInstr &CSet2MI =
          *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
               .addDef(Def2Reg)
               .addUse(AArch64::WZR)
               .addUse(AArch64::WZR)
               .addImm(getInvertedCondCode(CC2));
      MachineInstr &OrMI =
          *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ORRWrr))
               .addDef(DefReg)
               .addUse(Def1Reg)
               .addUse(Def2Reg);
      constrainSelectedInstRegOperands(OrMI, TII, TRI, RBI);
      constrainSelectedInstRegOperands(CSet2MI, TII, TRI, RBI);
    }

    constrainSelectedInstRegOperands(CmpMI, TII, TRI, RBI);
    constrainSelectedInstRegOperands(CSetMI, TII, TRI, RBI);

    I.eraseFromParent();
    return true;
  }
  case TargetOpcode::G_VASTART:
    return STI.isTargetDarwin() ? selectVaStartDarwin(I, MF, MRI)
                                : selectVaStartAAPCS(I, MF, MRI);
  case TargetOpcode::G_IMPLICIT_DEF:
    I.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
    return true;
  }

  return false;
}

/// SelectArithImmed - Select an immediate value that can be represented as
/// a 12-bit value shifted left by either 0 or 12.  If so, return true with
/// Val set to the 12-bit value and Shift set to the shifter operand.
InstructionSelector::ComplexRendererFn
AArch64InstructionSelector::selectArithImmed(MachineOperand &Root) const {
  MachineInstr &MI = *Root.getParent();
  MachineBasicBlock &MBB = *MI.getParent();
  MachineFunction &MF = *MBB.getParent();
  MachineRegisterInfo &MRI = MF.getRegInfo();

  // This function is called from the addsub_shifted_imm ComplexPattern,
  // which lists [imm] as the list of opcode it's interested in, however
  // we still need to check whether the operand is actually an immediate
  // here because the ComplexPattern opcode list is only used in
  // root-level opcode matching.
  uint64_t Immed;
  if (Root.isImm())
    Immed = Root.getImm();
  else if (Root.isCImm())
    Immed = Root.getCImm()->getZExtValue();
  else if (Root.isReg()) {
    MachineInstr *Def = MRI.getVRegDef(Root.getReg());
    if (Def->getOpcode() != TargetOpcode::G_CONSTANT)
      return None;
    MachineOperand &Op1 = Def->getOperand(1);
    if (!Op1.isCImm() || Op1.getCImm()->getBitWidth() > 64)
      return None;
    Immed = Op1.getCImm()->getZExtValue();
  } else
    return None;

  unsigned ShiftAmt;

  if (Immed >> 12 == 0) {
    ShiftAmt = 0;
  } else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
    ShiftAmt = 12;
    Immed = Immed >> 12;
  } else
    return None;

  unsigned ShVal = AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftAmt);
  return {{
      [=](MachineInstrBuilder &MIB) { MIB.addImm(Immed); },
      [=](MachineInstrBuilder &MIB) { MIB.addImm(ShVal); },
  }};
}

/// Select a "register plus unscaled signed 9-bit immediate" address.  This
/// should only match when there is an offset that is not valid for a scaled
/// immediate addressing mode.  The "Size" argument is the size in bytes of the
/// memory reference, which is needed here to know what is valid for a scaled
/// immediate.
InstructionSelector::ComplexRendererFn
AArch64InstructionSelector::selectAddrModeUnscaled(MachineOperand &Root,
                                                   unsigned Size) const {
  MachineRegisterInfo &MRI =
      Root.getParent()->getParent()->getParent()->getRegInfo();

  if (!Root.isReg())
    return None;

  if (!isBaseWithConstantOffset(Root, MRI))
    return None;

  MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
  if (!RootDef)
    return None;

  MachineOperand &OffImm = RootDef->getOperand(2);
  if (!OffImm.isReg())
    return None;
  MachineInstr *RHS = MRI.getVRegDef(OffImm.getReg());
  if (!RHS || RHS->getOpcode() != TargetOpcode::G_CONSTANT)
    return None;
  int64_t RHSC;
  MachineOperand &RHSOp1 = RHS->getOperand(1);
  if (!RHSOp1.isCImm() || RHSOp1.getCImm()->getBitWidth() > 64)
    return None;
  RHSC = RHSOp1.getCImm()->getSExtValue();

  // If the offset is valid as a scaled immediate, don't match here.
  if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Log2_32(Size)))
    return None;
  if (RHSC >= -256 && RHSC < 256) {
    MachineOperand &Base = RootDef->getOperand(1);
    return {{
        [=](MachineInstrBuilder &MIB) { MIB.add(Base); },
        [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
    }};
  }
  return None;
}

/// Select a "register plus scaled unsigned 12-bit immediate" address.  The
/// "Size" argument is the size in bytes of the memory reference, which
/// determines the scale.
InstructionSelector::ComplexRendererFn
AArch64InstructionSelector::selectAddrModeIndexed(MachineOperand &Root,
                                                  unsigned Size) const {
  MachineRegisterInfo &MRI =
      Root.getParent()->getParent()->getParent()->getRegInfo();

  if (!Root.isReg())
    return None;

  MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
  if (!RootDef)
    return None;

  if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
    return {{
        [=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); },
        [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
    }};
  }

  if (isBaseWithConstantOffset(Root, MRI)) {
    MachineOperand &LHS = RootDef->getOperand(1);
    MachineOperand &RHS = RootDef->getOperand(2);
    MachineInstr *LHSDef = MRI.getVRegDef(LHS.getReg());
    MachineInstr *RHSDef = MRI.getVRegDef(RHS.getReg());
    if (LHSDef && RHSDef) {
      int64_t RHSC = (int64_t)RHSDef->getOperand(1).getCImm()->getZExtValue();
      unsigned Scale = Log2_32(Size);
      if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Scale)) {
        if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
          LHSDef = MRI.getVRegDef(LHSDef->getOperand(1).getReg());
        return {{
            [=](MachineInstrBuilder &MIB) { MIB.add(LHS); },
            [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); },
        }};
      }
    }
  }

  // Before falling back to our general case, check if the unscaled
  // instructions can handle this. If so, that's preferable.
  if (selectAddrModeUnscaled(Root, Size).hasValue())
    return None;

  return {{
      [=](MachineInstrBuilder &MIB) { MIB.add(Root); },
      [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
  }};
}

namespace llvm {
InstructionSelector *
createAArch64InstructionSelector(const AArch64TargetMachine &TM,
                                 AArch64Subtarget &Subtarget,
                                 AArch64RegisterBankInfo &RBI) {
  return new AArch64InstructionSelector(TM, Subtarget, RBI);
}
}