IR+AArch64: add a "swiftasync" argument attribute.

This extends any frame record created in the function to include that parameter, passed in X22. The new record looks like [X22, FP, LR] in memory, and FP is stored with 0b0001 in bits 63:60 (CodeGen assumes they are 0b0000 in normal operation). The effect of this is that tools walking the stack should expect to see one of three values there: * 0b0000 => a normal, non-extended record with just [FP, LR] * 0b0001 => the extended record [X22, FP, LR] * 0b1111 => kernel space, and a non-extended record. All other values are currently reserved. If compiling for arm64e this context pointer is address-discriminated with the discriminator 0xc31a and the DB (process-specific) key. There is also an "i8** @llvm.swift.async.context.addr()" intrinsic providing front-ends access to this slot (and forcing its creation initialized to nullptr if necessary).
2021-01-20 10:14:03 +00:00 · 2021-01-20 10:14:03 +00:00 · ea0eec69f1
parent 079bbea2b2
commit ea0eec69f1
41 changed files with 575 additions and 30 deletions
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@ -1278,6 +1278,12 @@ Currently, only the following parameter attributes are defined:
    a valid attribute for return values and can only be applied to one
    parameter.

+``swiftasync``
+    This indicates that the parameter is the asynchronous context parameter and
+    triggers the creation of a target-specific extended frame record to store
+    this pointer. This is not a valid attribute for return values and can only
+    be applied to one parameter.
+
 ``swifterror``
    This attribute is motivated to model and optimize Swift error handling. It
    can be applied to a parameter with pointer to pointer type or a
@ -12360,6 +12366,29 @@ Note that calling this intrinsic does not prevent function inlining or
 other aggressive transformations, so the value returned may not be that
 of the obvious source-language caller.

+'``llvm.swift.async.context.addr``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+::
+
+      declare i8** @llvm.swift.async.context.addr()
+
+Overview:
+"""""""""
+
+The '``llvm.swift.async.context.addr``' intrinsic returns a pointer to
+the part of the extended frame record containing the asynchronous
+context of a Swift execution.
+
+Semantics:
+""""""""""
+
+If the function has a ``swiftasync`` parameter, that argument will initially
+be stored at the returned address. If not, it will be initialized to null.
+
 '``llvm.localescape``' and '``llvm.localrecover``' Intrinsics
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

--- a/llvm/include/llvm/AsmParser/LLToken.h
+++ b/llvm/include/llvm/AsmParser/LLToken.h
@ -239,6 +239,7 @@ enum Kind {
  kw_strictfp,
  kw_swifterror,
  kw_swiftself,
+  kw_swiftasync,
  kw_uwtable,
  kw_vscale_range,
  kw_willreturn,
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@ -665,6 +665,7 @@ enum AttributeKindCodes {
  ATTR_KIND_HOT = 72,
  ATTR_KIND_NO_PROFILE = 73,
  ATTR_KIND_VSCALE_RANGE = 74,
+  ATTR_KIND_SWIFT_ASYNC = 75,
 };

 enum ComdatSelectionKindCodes {
--- a/llvm/include/llvm/CodeGen/TargetCallingConv.h
+++ b/llvm/include/llvm/CodeGen/TargetCallingConv.h
@ -39,6 +39,7 @@ namespace ISD {
    unsigned IsPreallocated : 1; ///< ByVal without the copy
    unsigned IsSplitEnd : 1;   ///< Last part of a split
    unsigned IsSwiftSelf : 1;  ///< Swift self parameter
+    unsigned IsSwiftAsync : 1;  ///< Swift async context parameter
    unsigned IsSwiftError : 1; ///< Swift error parameter
    unsigned IsCFGuardTarget : 1; ///< Control Flow Guard target
    unsigned IsHva : 1;        ///< HVA field for
@ -58,11 +59,12 @@ namespace ISD {

  public:
    ArgFlagsTy()
-      : IsZExt(0), IsSExt(0), IsInReg(0), IsSRet(0), IsByVal(0), IsByRef(0),
-          IsNest(0), IsReturned(0), IsSplit(0), IsInAlloca(0), IsPreallocated(0),
-          IsSplitEnd(0), IsSwiftSelf(0), IsSwiftError(0), IsCFGuardTarget(0),
-          IsHva(0), IsHvaStart(0), IsSecArgPass(0), MemAlign(0),
-          OrigAlign(0), IsInConsecutiveRegsLast(0), IsInConsecutiveRegs(0),
+        : IsZExt(0), IsSExt(0), IsInReg(0), IsSRet(0), IsByVal(0), IsByRef(0),
+          IsNest(0), IsReturned(0), IsSplit(0), IsInAlloca(0),
+          IsPreallocated(0), IsSplitEnd(0), IsSwiftSelf(0), IsSwiftAsync(0),
+          IsSwiftError(0), IsCFGuardTarget(0), IsHva(0), IsHvaStart(0),
+          IsSecArgPass(0), MemAlign(0), OrigAlign(0),
+          IsInConsecutiveRegsLast(0), IsInConsecutiveRegs(0),
          IsCopyElisionCandidate(0), IsPointer(0), ByValOrByRefSize(0),
          PointerAddrSpace(0) {
      static_assert(sizeof(*this) == 3 * sizeof(unsigned), "flags are too big");
@ -95,6 +97,9 @@ namespace ISD {
    bool isSwiftSelf() const { return IsSwiftSelf; }
    void setSwiftSelf() { IsSwiftSelf = 1; }

+    bool isSwiftAsync() const { return IsSwiftAsync; }
+    void setSwiftAsync() { IsSwiftAsync = 1; }
+
    bool isSwiftError() const { return IsSwiftError; }
    void setSwiftError() { IsSwiftError = 1; }

--- a/llvm/include/llvm/CodeGen/TargetFrameLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetFrameLowering.h
@ -149,6 +149,14 @@ public:
  /// returns false, spill slots will be assigned using generic implementation.
  /// assignCalleeSavedSpillSlots() may add, delete or rearrange elements of
  /// CSI.
+  virtual bool assignCalleeSavedSpillSlots(MachineFunction &MF,
+                                           const TargetRegisterInfo *TRI,
+                                           std::vector<CalleeSavedInfo> &CSI,
+                                           unsigned &MinCSFrameIndex,
+                                           unsigned &MaxCSFrameIndex) const {
+    return assignCalleeSavedSpillSlots(MF, TRI, CSI);
+  }
+
  virtual bool
  assignCalleeSavedSpillSlots(MachineFunction &MF,
                              const TargetRegisterInfo *TRI,
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@ -284,6 +284,7 @@ public:
    bool IsPreallocated : 1;
    bool IsReturned : 1;
    bool IsSwiftSelf : 1;
+    bool IsSwiftAsync : 1;
    bool IsSwiftError : 1;
    bool IsCFGuardTarget : 1;
    MaybeAlign Alignment = None;
@ -294,7 +295,7 @@ public:
        : IsSExt(false), IsZExt(false), IsInReg(false), IsSRet(false),
          IsNest(false), IsByVal(false), IsByRef(false), IsInAlloca(false),
          IsPreallocated(false), IsReturned(false), IsSwiftSelf(false),
-          IsSwiftError(false), IsCFGuardTarget(false) {}
+          IsSwiftAsync(false), IsSwiftError(false), IsCFGuardTarget(false) {}

    void setAttributes(const CallBase *Call, unsigned ArgIdx);
  };
--- a/llvm/include/llvm/IR/Attributes.td
+++ b/llvm/include/llvm/IR/Attributes.td
@ -244,6 +244,9 @@ def SwiftError : EnumAttr<"swifterror">;
 /// Argument is swift self/context.
 def SwiftSelf : EnumAttr<"swiftself">;

+/// Argument is swift async context.
+def SwiftAsync : EnumAttr<"swiftasync">;
+
 /// Function must be in a unwind table.
 def UWTable : EnumAttr<"uwtable">;

--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@ -479,7 +479,12 @@ def int_objc_arc_annotation_bottomup_bbstart  : Intrinsic<[],
 def int_objc_arc_annotation_bottomup_bbend  : Intrinsic<[],
                                                        [llvm_ptrptr_ty,
                                                         llvm_ptrptr_ty]>;
+//===--------------- Swift asynchronous context intrinsics ----------------===//

+// Returns the location of the Swift asynchronous context (usually stored just
+// before the frame pointer), and triggers the creation of a null context if it
+// would otherwise be unneeded.
+def int_swift_async_context_addr : Intrinsic<[llvm_ptrptr_ty], [], [IntrNoMem]>;

 //===--------------------- Code Generator Intrinsics ----------------------===//
 //
--- a/llvm/include/llvm/Target/TargetCallingConv.td
+++ b/llvm/include/llvm/Target/TargetCallingConv.td
@ -51,6 +51,11 @@ class CCIfPreallocated<CCAction A> : CCIf<"ArgFlags.isPreallocated()", A> {
 class CCIfSwiftSelf<CCAction A> : CCIf<"ArgFlags.isSwiftSelf()", A> {
 }

+/// CCIfSwiftAsync - If the current argument has swiftasync parameter attribute,
+/// apply Action A.
+class CCIfSwiftAsync<CCAction A> : CCIf<"ArgFlags.isSwiftAsync()", A> {
+}
+
 /// CCIfSwiftError - If the current argument has swifterror parameter attribute,
 /// apply Action A.
 class CCIfSwiftError<CCAction A> : CCIf<"ArgFlags.isSwiftError()", A> {
--- a/llvm/lib/AsmParser/LLLexer.cpp
+++ b/llvm/lib/AsmParser/LLLexer.cpp
@ -696,6 +696,7 @@ lltok::Kind LLLexer::LexIdentifier() {
  KEYWORD(speculative_load_hardening);
  KEYWORD(swifterror);
  KEYWORD(swiftself);
+  KEYWORD(swiftasync);
  KEYWORD(uwtable);
  KEYWORD(vscale_range);
  KEYWORD(willreturn);
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@ -1460,6 +1460,7 @@ bool LLParser::parseFnAttributeValuePairs(AttrBuilder &B,
    case lltok::kw_sret:
    case lltok::kw_swifterror:
    case lltok::kw_swiftself:
+    case lltok::kw_swiftasync:
    case lltok::kw_immarg:
    case lltok::kw_byref:
      HaveError |=
@ -1799,6 +1800,7 @@ bool LLParser::parseOptionalParamAttrs(AttrBuilder &B) {
    case lltok::kw_signext:         B.addAttribute(Attribute::SExt); break;
    case lltok::kw_swifterror:      B.addAttribute(Attribute::SwiftError); break;
    case lltok::kw_swiftself:       B.addAttribute(Attribute::SwiftSelf); break;
+    case lltok::kw_swiftasync:      B.addAttribute(Attribute::SwiftAsync); break;
    case lltok::kw_writeonly:       B.addAttribute(Attribute::WriteOnly); break;
    case lltok::kw_zeroext:         B.addAttribute(Attribute::ZExt); break;
    case lltok::kw_immarg:          B.addAttribute(Attribute::ImmArg); break;
@ -1905,6 +1907,7 @@ bool LLParser::parseOptionalReturnAttrs(AttrBuilder &B) {
    case lltok::kw_sret:
    case lltok::kw_swifterror:
    case lltok::kw_swiftself:
+    case lltok::kw_swiftasync:
    case lltok::kw_immarg:
    case lltok::kw_byref:
      HaveError |=
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@ -1524,6 +1524,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
    return Attribute::SwiftError;
  case bitc::ATTR_KIND_SWIFT_SELF:
    return Attribute::SwiftSelf;
+  case bitc::ATTR_KIND_SWIFT_ASYNC:
+    return Attribute::SwiftAsync;
  case bitc::ATTR_KIND_UW_TABLE:
    return Attribute::UWTable;
  case bitc::ATTR_KIND_VSCALE_RANGE:
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@ -736,6 +736,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
    return bitc::ATTR_KIND_SWIFT_ERROR;
  case Attribute::SwiftSelf:
    return bitc::ATTR_KIND_SWIFT_SELF;
+  case Attribute::SwiftAsync:
+    return bitc::ATTR_KIND_SWIFT_ASYNC;
  case Attribute::UWTable:
    return bitc::ATTR_KIND_UW_TABLE;
  case Attribute::VScaleRange:
--- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
@ -54,6 +54,8 @@ addFlagsUsingAttrFn(ISD::ArgFlagsTy &Flags,
    Flags.setReturned();
  if (AttrFn(Attribute::SwiftSelf))
    Flags.setSwiftSelf();
+  if (AttrFn(Attribute::SwiftAsync))
+    Flags.setSwiftAsync();
  if (AttrFn(Attribute::SwiftError))
    Flags.setSwiftError();
 }
--- a/llvm/lib/CodeGen/PrologEpilogInserter.cpp
+++ b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
@ -399,7 +399,8 @@ static void assignCalleeSavedSpillSlots(MachineFunction &F,

  const TargetFrameLowering *TFI = F.getSubtarget().getFrameLowering();
  MachineFrameInfo &MFI = F.getFrameInfo();
-  if (!TFI->assignCalleeSavedSpillSlots(F, RegInfo, CSI)) {
+  if (!TFI->assignCalleeSavedSpillSlots(F, RegInfo, CSI, MinCSFrameIndex,
+                                        MaxCSFrameIndex)) {
    // If target doesn't implement this, use generic code.

    if (CSI.empty())
@ -677,10 +678,12 @@ computeFreeStackSlots(MachineFrameInfo &MFI, bool StackGrowsDown,
    // StackSlot scavenging is only implemented for the default stack.
    if (MFI.getStackID(i) == TargetStackID::Default)
      AllocatedFrameSlots.push_back(i);
-  // Add callee-save objects.
-  for (int i = MinCSFrameIndex; i <= (int)MaxCSFrameIndex; ++i)
-    if (MFI.getStackID(i) == TargetStackID::Default)
-      AllocatedFrameSlots.push_back(i);
+  // Add callee-save objects if there are any.
+  if (MinCSFrameIndex <= MaxCSFrameIndex) {
+    for (int i = MinCSFrameIndex; i <= (int)MaxCSFrameIndex; ++i)
+      if (MFI.getStackID(i) == TargetStackID::Default)
+        AllocatedFrameSlots.push_back(i);
+  }

  for (int i : AllocatedFrameSlots) {
    // These are converted from int64_t, but they should always fit in int
@ -833,7 +836,7 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &MF) {

  // First assign frame offsets to stack objects that are used to spill
  // callee saved registers.
-  if (StackGrowsDown) {
+  if (StackGrowsDown && MaxCSFrameIndex >= MinCSFrameIndex) {
    for (unsigned i = MinCSFrameIndex; i <= MaxCSFrameIndex; ++i) {
      if (MFI.getStackID(i) !=
          TargetStackID::Default) // Only allocate objects on the default stack.
--- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
@ -1048,6 +1048,8 @@ bool FastISel::lowerCallTo(CallLoweringInfo &CLI) {
      Flags.setSRet();
    if (Arg.IsSwiftSelf)
      Flags.setSwiftSelf();
+    if (Arg.IsSwiftAsync)
+      Flags.setSwiftAsync();
    if (Arg.IsSwiftError)
      Flags.setSwiftError();
    if (Arg.IsCFGuardTarget)
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@ -9385,6 +9385,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
    Entry.IsByRef = false;
    Entry.IsReturned = false;
    Entry.IsSwiftSelf = false;
+    Entry.IsSwiftAsync = false;
    Entry.IsSwiftError = false;
    Entry.IsCFGuardTarget = false;
    Entry.Alignment = Alignment;
@ -9498,6 +9499,8 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
        Flags.setSRet();
      if (Args[i].IsSwiftSelf)
        Flags.setSwiftSelf();
+      if (Args[i].IsSwiftAsync)
+        Flags.setSwiftAsync();
      if (Args[i].IsSwiftError)
        Flags.setSwiftError();
      if (Args[i].IsCFGuardTarget)
@ -10035,6 +10038,8 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
        Flags.setSRet();
      if (Arg.hasAttribute(Attribute::SwiftSelf))
        Flags.setSwiftSelf();
+      if (Arg.hasAttribute(Attribute::SwiftAsync))
+        Flags.setSwiftAsync();
      if (Arg.hasAttribute(Attribute::SwiftError))
        Flags.setSwiftError();
      if (Arg.hasAttribute(Attribute::ByVal))
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@ -115,6 +115,7 @@ void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,
  IsNest = Attrs.hasParamAttribute(ArgIdx, Attribute::Nest);
  IsReturned = Attrs.hasParamAttribute(ArgIdx, Attribute::Returned);
  IsSwiftSelf = Attrs.hasParamAttribute(ArgIdx, Attribute::SwiftSelf);
+  IsSwiftAsync = Attrs.hasParamAttr(ArgIdx, Attribute::SwiftAsync);
  IsSwiftError = Attrs.hasParamAttribute(ArgIdx, Attribute::SwiftError);
  Alignment = Attrs.getParamStackAlignment(ArgIdx);

--- a/llvm/lib/IR/Attributes.cpp
+++ b/llvm/lib/IR/Attributes.cpp
@ -384,6 +384,8 @@ std::string Attribute::getAsString(bool InAttrGrp) const {
    return "swifterror";
  if (hasAttribute(Attribute::SwiftSelf))
    return "swiftself";
+  if (hasAttribute(Attribute::SwiftAsync))
+    return "swiftasync";
  if (hasAttribute(Attribute::InaccessibleMemOnly))
    return "inaccessiblememonly";
  if (hasAttribute(Attribute::InaccessibleMemOrArgMemOnly))
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@ -1910,6 +1910,7 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs,
  bool SawReturned = false;
  bool SawSRet = false;
  bool SawSwiftSelf = false;
+  bool SawSwiftAsync = false;
  bool SawSwiftError = false;

  // Verify return value attributes.
@ -1924,11 +1925,12 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs,
          !RetAttrs.hasAttribute(Attribute::Preallocated) &&
          !RetAttrs.hasAttribute(Attribute::ByRef) &&
          !RetAttrs.hasAttribute(Attribute::SwiftSelf) &&
+          !RetAttrs.hasAttribute(Attribute::SwiftAsync) &&
          !RetAttrs.hasAttribute(Attribute::SwiftError)),
         "Attributes 'byval', 'inalloca', 'preallocated', 'byref', "
         "'nest', 'sret', 'nocapture', 'nofree', "
-         "'returned', 'swiftself', and 'swifterror' do not apply to return "
-         "values!",
+         "'returned', 'swiftself', 'swiftasync', and 'swifterror'"
+         " do not apply to return values!",
         V);
  Assert((!RetAttrs.hasAttribute(Attribute::ReadOnly) &&
          !RetAttrs.hasAttribute(Attribute::WriteOnly) &&
@ -1976,6 +1978,11 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs,
      SawSwiftSelf = true;
    }

+    if (ArgAttrs.hasAttribute(Attribute::SwiftAsync)) {
+      Assert(!SawSwiftAsync, "Cannot have multiple 'swiftasync' parameters!", V);
+      SawSwiftAsync = true;
+    }
+
    if (ArgAttrs.hasAttribute(Attribute::SwiftError)) {
      Assert(!SawSwiftError, "Cannot have multiple 'swifterror' parameters!",
             V);
@ -3370,9 +3377,10 @@ static bool isTypeCongruent(Type *L, Type *R) {

 static AttrBuilder getParameterABIAttributes(int I, AttributeList Attrs) {
  static const Attribute::AttrKind ABIAttrs[] = {
-      Attribute::StructRet,    Attribute::ByVal,     Attribute::InAlloca,
-      Attribute::InReg,        Attribute::SwiftSelf, Attribute::SwiftError,
-      Attribute::Preallocated, Attribute::ByRef,     Attribute::StackAlignment};
+      Attribute::StructRet,  Attribute::ByVal,      Attribute::InAlloca,
+      Attribute::InReg,      Attribute::SwiftSelf,  Attribute::SwiftAsync,
+      Attribute::SwiftError, Attribute::SwiftAsync, Attribute::Preallocated,
+      Attribute::ByRef};
  AttrBuilder Copy;
  for (auto AK : ABIAttrs) {
    if (Attrs.hasParamAttribute(I, AK))
--- a/llvm/lib/Target/AArch64/AArch64CallingConvention.td
+++ b/llvm/lib/Target/AArch64/AArch64CallingConvention.td
@ -69,6 +69,10 @@ def CC_AArch64_AAPCS : CallingConv<[
  // A SwiftError is passed in X21.
  CCIfSwiftError<CCIfType<[i64], CCAssignToRegWithShadow<[X21], [W21]>>>,

+  // Pass SwiftAsync in an otherwise callee saved register so that it will be
+  // preserved for normal function calls.
+  CCIfSwiftAsync<CCIfType<[i64], CCAssignToRegWithShadow<[X22], [W22]>>>,
+
  CCIfConsecutiveRegs<CCCustom<"CC_AArch64_Custom_Block">>,

  CCIfType<[nxv16i8, nxv8i16, nxv4i32, nxv2i64, nxv2f16, nxv4f16, nxv8f16,
@ -203,6 +207,10 @@ def CC_AArch64_DarwinPCS : CallingConv<[
  // A SwiftError is passed in X21.
  CCIfSwiftError<CCIfType<[i64], CCAssignToRegWithShadow<[X21], [W21]>>>,

+  // Pass SwiftAsync in an otherwise callee saved register so that it will be
+  // preserved for normal function calls.
+  CCIfSwiftAsync<CCIfType<[i64], CCAssignToRegWithShadow<[X22], [W22]>>>,
+
  CCIfConsecutiveRegs<CCCustom<"CC_AArch64_Custom_Block">>,

  // Handle i1, i8, i16, i32, i64, f32, f64 and v2f64 by passing in registers,
--- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@ -86,6 +86,8 @@ private:
                          unsigned N);
  bool expandCALL_RVMARKER(MachineBasicBlock &MBB,
                           MachineBasicBlock::iterator MBBI);
+  bool expandStoreSwiftAsyncContext(MachineBasicBlock &MBB,
+                                    MachineBasicBlock::iterator MBBI);
 };

 } // end anonymous namespace
@ -696,6 +698,63 @@ bool AArch64ExpandPseudo::expandCALL_RVMARKER(
  return true;
 }

+bool AArch64ExpandPseudo::expandStoreSwiftAsyncContext(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) {
+  Register CtxReg = MBBI->getOperand(0).getReg();
+  Register BaseReg = MBBI->getOperand(1).getReg();
+  int Offset = MBBI->getOperand(2).getImm();
+  DebugLoc DL(MBBI->getDebugLoc());
+  auto &STI = MBB.getParent()->getSubtarget<AArch64Subtarget>();
+
+  if (STI.getTargetTriple().getArchName() != "arm64e") {
+    BuildMI(MBB, MBBI, DL, TII->get(AArch64::STRXui))
+        .addUse(CtxReg)
+        .addUse(BaseReg)
+        .addImm(Offset / 8)
+        .setMIFlag(MachineInstr::FrameSetup);
+    MBBI->eraseFromParent();
+    return true;
+  }
+
+  // We need to sign the context in an address-discriminated way. 0xc31a is a
+  // fixed random value, chosen as part of the ABI.
+  //     add x16, xBase, #Offset
+  //     movk x16, #0xc31a, lsl #48
+  //     mov x17, x22/xzr
+  //     pacdb x17, x16
+  //     str x17, [xBase, #Offset]
+  unsigned Opc = Offset >= 0 ? AArch64::ADDXri : AArch64::SUBXri;
+  BuildMI(MBB, MBBI, DL, TII->get(Opc), AArch64::X16)
+      .addUse(BaseReg)
+      .addImm(abs(Offset))
+      .addImm(0)
+      .setMIFlag(MachineInstr::FrameSetup);
+  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVKXi), AArch64::X16)
+      .addUse(AArch64::X16)
+      .addImm(0xc31a)
+      .addImm(48)
+      .setMIFlag(MachineInstr::FrameSetup);
+  // We're not allowed to clobber X22 (and couldn't clobber XZR if we tried), so
+  // move it somewhere before signing.
+  BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrs), AArch64::X17)
+      .addUse(AArch64::XZR)
+      .addUse(CtxReg)
+      .addImm(0)
+      .setMIFlag(MachineInstr::FrameSetup);
+  BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACDB), AArch64::X17)
+      .addUse(AArch64::X17)
+      .addUse(AArch64::X16)
+      .setMIFlag(MachineInstr::FrameSetup);
+  BuildMI(MBB, MBBI, DL, TII->get(AArch64::STRXui))
+      .addUse(AArch64::X17)
+      .addUse(BaseReg)
+      .addImm(Offset / 8)
+      .setMIFlag(MachineInstr::FrameSetup);
+
+  MBBI->eraseFromParent();
+  return true;
+}
+
 /// If MBBI references a pseudo instruction that should be expanded here,
 /// do the expansion and return true.  Otherwise return false.
 bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB,
@ -1110,6 +1169,8 @@ bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB,
     return expandSVESpillFill(MBB, MBBI, AArch64::LDR_ZXI, 2);
   case AArch64::BLR_RVMARKER:
     return expandCALL_RVMARKER(MBB, MBBI);
+   case AArch64::StoreSwiftAsyncContext:
+     return expandStoreSwiftAsyncContext(MBB, MBBI);
  }
  return false;
 }
--- a/llvm/lib/Target/AArch64/AArch64FastISel.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FastISel.cpp
@ -2899,6 +2899,7 @@ bool AArch64FastISel::fastLowerArguments() {
        Arg.hasAttribute(Attribute::InReg) ||
        Arg.hasAttribute(Attribute::StructRet) ||
        Arg.hasAttribute(Attribute::SwiftSelf) ||
+        Arg.hasAttribute(Attribute::SwiftAsync) ||
        Arg.hasAttribute(Attribute::SwiftError) ||
        Arg.hasAttribute(Attribute::Nest))
      return false;
@ -3157,7 +3158,7 @@ bool AArch64FastISel::fastLowerCall(CallLoweringInfo &CLI) {

  for (auto Flag : CLI.OutFlags)
    if (Flag.isInReg() || Flag.isSRet() || Flag.isNest() || Flag.isByVal() ||
-        Flag.isSwiftSelf() || Flag.isSwiftError())
+        Flag.isSwiftSelf() || Flag.isSwiftAsync() || Flag.isSwiftError())
      return false;

  // Set up the argument vectors.
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@ -47,8 +47,9 @@
 // | callee-saved gpr registers        | <--.
 // |                                   |    | On Darwin platforms these
 // |- - - - - - - - - - - - - - - - - -|    | callee saves are swapped,
-// |                                   |    | (frame record first)
-// | prev_fp, prev_lr                  | <--'
+// | prev_lr                           |    | (frame record first)
+// | prev_fp                           | <--'
+// | async context if needed           |
 // | (a.k.a. "frame record")           |
 // |-----------------------------------| <- fp(=x29)
 // |                                   |
@ -940,6 +941,16 @@ static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
      SEH->eraseFromParent();
  }

+  // If the first store isn't right where we want SP then we can't fold the
+  // update in so create a normal arithmetic instruction instead.
+  if (MBBI->getOperand(MBBI->getNumOperands() - 1).getImm() != 0) {
+    emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
+                    StackOffset::getFixed(CSStackSizeInc), TII,
+                    InProlog ? MachineInstr::FrameSetup
+                             : MachineInstr::FrameDestroy);
+    return std::prev(MBBI);
+  }
+
  MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
  MIB.addReg(AArch64::SP, RegState::Define);

@ -1128,6 +1139,18 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
        .setMIFlags(MachineInstr::FrameSetup);
  }

+  // We signal the presence of a Swift extended frame to external tools by
+  // storing FP with 0b0001 in bits 63:60. In normal userland operation a simple
+  // ORR is sufficient, it is assumed a Swift kernel would initialize the TBI
+  // bits so that is still true.
+  if (HasFP && AFI->hasSwiftAsyncContext()) {
+    // ORR x29, x29, #0x1000_0000_0000_0000
+    BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXri), AArch64::FP)
+        .addUse(AArch64::FP)
+        .addImm(0x1100)
+        .setMIFlag(MachineInstr::FrameSetup);
+  }
+
  // All calls are tail calls in GHC calling conv, and functions have no
  // prologue/epilogue.
  if (MF.getFunction().getCallingConv() == CallingConv::GHC)
@ -1234,6 +1257,20 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
    if (CombineSPBump)
      FPOffset += AFI->getLocalStackSize();

+    if (AFI->hasSwiftAsyncContext()) {
+      // Before we update the live FP we have to ensure there's a valid (or
+      // null) asynchronous context in its slot just before FP in the frame
+      // record, so store it now.
+      const auto &Attrs = MF.getFunction().getAttributes();
+      bool HaveInitialContext = Attrs.hasAttrSomewhere(Attribute::SwiftAsync);
+
+      BuildMI(MBB, MBBI, DL, TII->get(AArch64::StoreSwiftAsyncContext))
+          .addUse(HaveInitialContext ? AArch64::X22 : AArch64::XZR)
+          .addUse(AArch64::SP)
+          .addImm(FPOffset - 8)
+          .setMIFlags(MachineInstr::FrameSetup);
+    }
+
    if (HomPrologEpilog) {
      auto Prolog = MBBI;
      --Prolog;
@ -1745,6 +1782,18 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
        .setMIFlag(MachineInstr::FrameDestroy);
  }

+  if (hasFP(MF) && AFI->hasSwiftAsyncContext()) {
+    // We need to reset FP to its untagged state on return. Bit 60 is currently
+    // used to show the presence of an extended frame.
+
+    // BIC x29, x29, #0x1000_0000_0000_0000
+    BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::ANDXri),
+            AArch64::FP)
+        .addUse(AArch64::FP)
+        .addImm(0x10fe)
+        .setMIFlag(MachineInstr::FrameDestroy);
+  }
+
  const StackOffset &SVEStackSize = getSVEStackSize(MF);

  // If there is a single SP update, insert it before the ret and we're done.
@ -2309,6 +2358,12 @@ static void computeCalleeSaveRegisterPairs(
    else
      ByteOffset += StackFillDir * (RPI.isPaired() ? 2 * Scale : Scale);

+    // Swift's async context is directly before FP, so allocate an extra
+    // 8 bytes for it.
+    if (NeedsFrameRecord && AFI->hasSwiftAsyncContext() &&
+        RPI.Reg2 == AArch64::FP)
+      ByteOffset += StackFillDir * 8;
+
    assert(!(RPI.isScalable() && RPI.isPaired()) &&
           "Paired spill/fill instructions don't exist for SVE vectors");

@ -2332,6 +2387,12 @@ static void computeCalleeSaveRegisterPairs(
    // If filling top down (default), we want the offset after incrementing it.
    // If fillibg bootom up (WinCFI) we need the original offset.
    int Offset = NeedsWinCFI ? OffsetPre : OffsetPost;
+
+    // The FP, LR pair goes 8 bytes into our expanded 24-byte slot so that the
+    // Swift context can directly precede FP.
+    if (NeedsFrameRecord && AFI->hasSwiftAsyncContext() &&
+        RPI.Reg2 == AArch64::FP)
+      Offset += 8;
    RPI.Offset = Offset / Scale;

    assert(((!RPI.isScalable() && RPI.Offset >= -64 && RPI.Offset <= 63) ||
@ -2800,6 +2861,12 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,

  // Adding the size of additional 64bit GPR saves.
  CSStackSize += 8 * (SavedRegs.count() - NumSavedRegs);
+
+  // A Swift asynchronous context extends the frame record with a pointer
+  // directly before FP.
+  if (hasFP(MF) && AFI->hasSwiftAsyncContext())
+    CSStackSize += 8;
+
  uint64_t AlignedCSStackSize = alignTo(CSStackSize, 16);
  LLVM_DEBUG(dbgs() << "Estimated stack frame size: "
               << EstimatedStackSize + AlignedCSStackSize
@ -2817,8 +2884,9 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
 }

 bool AArch64FrameLowering::assignCalleeSavedSpillSlots(
-    MachineFunction &MF, const TargetRegisterInfo *TRI,
-    std::vector<CalleeSavedInfo> &CSI) const {
+    MachineFunction &MF, const TargetRegisterInfo *RegInfo,
+    std::vector<CalleeSavedInfo> &CSI, unsigned &MinCSFrameIndex,
+    unsigned &MaxCSFrameIndex) const {
  bool NeedsWinCFI = needsWinCFI(MF);
  // To match the canonical windows frame layout, reverse the list of
  // callee saved registers to get them laid out by PrologEpilogInserter
@ -2827,8 +2895,35 @@ bool AArch64FrameLowering::assignCalleeSavedSpillSlots(
  // the top, thus have the CSI array start from the highest registers.)
  if (NeedsWinCFI)
    std::reverse(CSI.begin(), CSI.end());
-  // Let the generic code do the rest of the setup.
-  return false;
+
+  if (CSI.empty())
+    return true; // Early exit if no callee saved registers are modified!
+
+  // Now that we know which registers need to be saved and restored, allocate
+  // stack slots for them.
+  MachineFrameInfo &MFI = MF.getFrameInfo();
+  auto *AFI = MF.getInfo<AArch64FunctionInfo>();
+  for (auto &CS : CSI) {
+    Register Reg = CS.getReg();
+    const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg);
+
+    unsigned Size = RegInfo->getSpillSize(*RC);
+    Align Alignment(RegInfo->getSpillAlign(*RC));
+    int FrameIdx = MFI.CreateStackObject(Size, Alignment, true);
+    CS.setFrameIdx(FrameIdx);
+
+    if ((unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx;
+    if ((unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx;
+
+    // Grab 8 bytes below FP for the extended asynchronous frame info.
+    if (hasFP(MF) && AFI->hasSwiftAsyncContext() && Reg == AArch64::FP) {
+      FrameIdx = MFI.CreateStackObject(8, Alignment, true);
+      AFI->setSwiftAsyncContextFrameIdx(FrameIdx);
+      if ((unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx;
+      if ((unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx;
+    }
+  }
+  return true;
 }

 bool AArch64FrameLowering::enableStackSlotScavenging(
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.h
@ -67,10 +67,13 @@ public:
  bool hasFP(const MachineFunction &MF) const override;
  bool hasReservedCallFrame(const MachineFunction &MF) const override;

-  bool
-  assignCalleeSavedSpillSlots(MachineFunction &MF,
-                              const TargetRegisterInfo *TRI,
-                              std::vector<CalleeSavedInfo> &CSI) const override;
+  bool hasSwiftExtendedFrame(const MachineFunction &MF) const;
+
+  bool assignCalleeSavedSpillSlots(MachineFunction &MF,
+                                   const TargetRegisterInfo *TRI,
+                                   std::vector<CalleeSavedInfo> &CSI,
+                                   unsigned &MinCSFrameIndex,
+                                   unsigned &MaxCSFrameIndex) const override;

  void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs,
                            RegScavenger *RS) const override;
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@ -3901,6 +3901,18 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
      if (tryMULLV64LaneV128(IntNo, Node))
        return;
      break;
+    case Intrinsic::swift_async_context_addr: {
+      SDLoc DL(Node);
+      CurDAG->SelectNodeTo(Node, AArch64::SUBXri, MVT::i64,
+                           CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL,
+                                                  AArch64::FP, MVT::i64),
+                           CurDAG->getTargetConstant(8, DL, MVT::i32),
+                           CurDAG->getTargetConstant(0, DL, MVT::i32));
+      auto &MF = CurDAG->getMachineFunction();
+      MF.getFrameInfo().setFrameAddressIsTaken(true);
+      MF.getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
+      return;
+    }
    }
    break;
  }
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@ -4810,6 +4810,9 @@ SDValue AArch64TargetLowering::LowerFormalArguments(
      continue;
    }

+    if (Ins[i].Flags.isSwiftAsync())
+      MF.getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
+
    SDValue ArgValue;
    if (VA.isRegLoc()) {
      // Arguments stored in registers.
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@ -1957,7 +1957,8 @@ class OneXRegData<bits<3> opc, string asm, SDPatternOperator node>
 }

 class SignAuthOneData<bits<3> opcode_prefix, bits<2> opcode, string asm>
-  : I<(outs GPR64:$Rd), (ins GPR64sp:$Rn), asm, "\t$Rd, $Rn", "",
+  : I<(outs GPR64:$Rd), (ins GPR64:$src, GPR64sp:$Rn), asm, "\t$Rd, $Rn",
+      "$Rd = $src",
      []>,
    Sched<[WriteI, ReadI]> {
  bits<5> Rd;
@ -1970,7 +1971,8 @@ class SignAuthOneData<bits<3> opcode_prefix, bits<2> opcode, string asm>
 }

 class SignAuthZero<bits<3> opcode_prefix, bits<2> opcode, string asm>
-  : I<(outs GPR64:$Rd), (ins), asm, "\t$Rd", "", []>, Sched<[]> {
+  : I<(outs GPR64:$Rd), (ins GPR64:$src), asm, "\t$Rd", "$Rd = $src",
+      []>, Sched<[]> {
  bits<5> Rd;
  let Inst{31-15} = 0b11011010110000010;
  let Inst{14-12} = opcode_prefix;
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@ -135,6 +135,9 @@ unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
  case AArch64::SPACE:
    NumBytes = MI.getOperand(1).getImm();
    break;
+  case AArch64::StoreSwiftAsyncContext:
+    NumBytes = 20;
+    break;
  case TargetOpcode::BUNDLE:
    NumBytes = getInstBundleLength(MI);
    break;
@ -2640,6 +2643,13 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
    MinOffset = 0;
    MaxOffset = 4095;
    break;
+  case AArch64::StoreSwiftAsyncContext:
+    // Store is an STRXui, but there might be an ADDXri in the expansion too.
+    Scale = TypeSize::Fixed(1);
+    Width = 8;
+    MinOffset = 0;
+    MaxOffset = 4095;
+    break;
  case AArch64::LDPWi:
  case AArch64::LDPSi:
  case AArch64::LDNPWi:
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@ -7979,6 +7979,11 @@ let Predicates = [HasLS64] in {
  def : ST64BPattern<int_aarch64_st64bv0, ST64BV0>;
 }

+let Defs = [X16, X17], mayStore = 1, isCodeGenOnly = 1 in
+def StoreSwiftAsyncContext
+      : Pseudo<(outs), (ins GPR64:$ctx, GPR64sp:$base, simm9:$offset),
+               []>, Sched<[]>;
+
 include "AArch64InstrAtomics.td"
 include "AArch64SVEInstrInfo.td"

--- a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
@ -159,6 +159,14 @@ class AArch64FunctionInfo final : public MachineFunctionInfo {
  /// indirect branch destinations.
  bool BranchTargetEnforcement = false;

+  /// Whether this function has an extended frame record [Ctx, FP, LR]. If so,
+  /// bit 60 of the in-memory FP will be 1 to enable other tools to detect the
+  /// extended record.
+  bool HasSwiftAsyncContext = false;
+
+  /// The stack slot where the Swift asynchronous context is stored.
+  int SwiftAsyncContextFrameIdx = std::numeric_limits<int>::max();
+
 public:
  explicit AArch64FunctionInfo(MachineFunction &MF);

@ -239,6 +247,13 @@ public:
        MaxOffset = std::max<int64_t>(Offset + ObjSize, MaxOffset);
      }

+      if (SwiftAsyncContextFrameIdx != std::numeric_limits<int>::max()) {
+        int64_t Offset = MFI.getObjectOffset(getSwiftAsyncContextFrameIdx());
+        int64_t ObjSize = MFI.getObjectSize(getSwiftAsyncContextFrameIdx());
+        MinOffset = std::min<int64_t>(Offset, MinOffset);
+        MaxOffset = std::max<int64_t>(Offset + ObjSize, MaxOffset);
+      }
+
      unsigned Size = alignTo(MaxOffset - MinOffset, 16);
      assert((!HasCalleeSavedStackSize || getCalleeSavedStackSize() == Size) &&
             "Invalid size calculated for callee saves");
@ -372,6 +387,16 @@ public:

  bool branchTargetEnforcement() const { return BranchTargetEnforcement; }

+  void setHasSwiftAsyncContext(bool HasContext) {
+    HasSwiftAsyncContext = HasContext;
+  }
+  bool hasSwiftAsyncContext() const { return HasSwiftAsyncContext; }
+
+  void setSwiftAsyncContextFrameIdx(int FI) {
+    SwiftAsyncContextFrameIdx = FI;
+  }
+  int getSwiftAsyncContextFrameIdx() const { return SwiftAsyncContextFrameIdx; }
+
 private:
  // Hold the lists of LOHs.
  MILOHContainer LOHContainerSet;
--- a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
@ -537,6 +537,9 @@ bool AArch64CallLowering::lowerFormalArguments(
    ArgInfo OrigArg{VRegs[i], Arg};
    setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, F);

+    if (Arg.hasAttribute(Attribute::SwiftAsync))
+      MF.getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
+
    splitToValueTypes(OrigArg, SplitArgs, DL, F.getCallingConv());
    ++i;
  }
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@ -5042,6 +5042,17 @@ bool AArch64InstructionSelector::selectIntrinsic(MachineInstr &I,
    I.eraseFromParent();
    return true;
  }
+  case Intrinsic::swift_async_context_addr:
+    auto Sub = MIB.buildInstr(AArch64::SUBXri, {I.getOperand(0).getReg()},
+                              {Register(AArch64::FP)})
+                   .addImm(8)
+                   .addImm(0);
+    constrainSelectedInstRegOperands(*Sub, TII, TRI, RBI);
+
+    MF->getFrameInfo().setFrameAddressIsTaken(true);
+    MF->getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
+    I.eraseFromParent();
+    return true;
  }
  return false;
 }
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
@ -574,6 +574,7 @@ public:
    unsigned StackSize = 0;

    uint32_t CompactUnwindEncoding = 0;
+    int CurOffset = 0;
    for (size_t i = 0, e = Instrs.size(); i != e; ++i) {
      const MCCFIInstruction &Inst = Instrs[i];

@ -603,6 +604,9 @@ public:
        assert(FPPush.getOperation() == MCCFIInstruction::OpOffset &&
               "Frame pointer not pushed!");

+        assert(FPPush.getOffset() + 8 == LRPush.getOffset());
+        CurOffset = FPPush.getOffset();
+
        unsigned LRReg = *MRI.getLLVMRegNum(LRPush.getRegister(), true);
        unsigned FPReg = *MRI.getLLVMRegNum(FPPush.getRegister(), true);

@ -629,11 +633,19 @@ public:
        if (i + 1 == e)
          return CU::UNWIND_ARM64_MODE_DWARF;

+        if (CurOffset != 0 && Inst.getOffset() != CurOffset - 8)
+          return CU::UNWIND_ARM64_MODE_DWARF;
+        CurOffset = Inst.getOffset();
+
        const MCCFIInstruction &Inst2 = Instrs[++i];
        if (Inst2.getOperation() != MCCFIInstruction::OpOffset)
          return CU::UNWIND_ARM64_MODE_DWARF;
        unsigned Reg2 = *MRI.getLLVMRegNum(Inst2.getRegister(), true);

+        if (Inst2.getOffset() != CurOffset - 8)
+          return CU::UNWIND_ARM64_MODE_DWARF;
+        CurOffset = Inst2.getOffset();
+
        // N.B. The encodings must be in register number order, and the X
        // registers before the D registers.

--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@ -929,6 +929,7 @@ Function *CodeExtractor::constructFunction(const ValueSet &inputs,
      case Attribute::StructRet:
      case Attribute::SwiftError:
      case Attribute::SwiftSelf:
+      case Attribute::SwiftAsync:
      case Attribute::WillReturn:
      case Attribute::WriteOnly:
      case Attribute::ZExt:
--- a/llvm/test/Bitcode/attributes.ll
+++ b/llvm/test/Bitcode/attributes.ll
@ -447,6 +447,12 @@ define void @f75() vscale_range(0,0)
  ret void
 }

+; CHECK: define void @f76(i8* swiftasync %0)
+define void @f76(i8* swiftasync %0)
+{
+  ret void;
+}
+
 ; CHECK: attributes #0 = { noreturn }
 ; CHECK: attributes #1 = { nounwind }
 ; CHECK: attributes #2 = { readnone }
--- a/llvm/test/Bitcode/compatibility.ll
+++ b/llvm/test/Bitcode/compatibility.ll
@ -552,6 +552,12 @@ declare void @f.param.dereferenceable_or_null(i8* dereferenceable_or_null(4))
 ; CHECK: declare void @f.param.dereferenceable_or_null(i8* dereferenceable_or_null(4))
 declare void @f.param.stack_align([2 x double] alignstack(16))
 ; CHECK: declare void @f.param.stack_align([2 x double] alignstack(16))
+declare void @f.param.swiftself(i8* swiftself)
+; CHECK: declare void @f.param.swiftself(i8* swiftself)
+declare void @f.param.swiftasync(i8* swiftasync)
+; CHECK: declare void @f.param.swiftasync(i8* swiftasync)
+declare void @f.param.swifterror(i8** swifterror)
+; CHECK: declare void @f.param.swifterror(i8** swifterror)

 ; Functions -- unnamed_addr and local_unnamed_addr
 declare void @f.unnamed_addr() unnamed_addr
--- a/llvm/test/CodeGen/AArch64/swift-async-reg.ll
+++ b/llvm/test/CodeGen/AArch64/swift-async-reg.ll
@ -0,0 +1,18 @@
+; RUN: llc -mtriple=arm64-apple-ios %s -o - | FileCheck %s
+; RUN: llc -mtriple=arm64-apple-ios %s -o - -global-isel | FileCheck %s
+; RUN: llc -mtriple=arm64-apple-ios %s -o - -fast-isel | FileCheck %s
+
+define i8* @argument(i8* swiftasync %in) {
+; CHECK-LABEL: argument:
+; CHECK: mov x0, x22
+
+  ret i8* %in
+}
+
+define void @call(i8* %in) {
+; CHECK-LABEL: call:
+; CHECK: mov x22, x0
+
+  call i8* @argument(i8* swiftasync %in)
+  ret void
+}
--- a/llvm/test/CodeGen/AArch64/swift-async-unwind.ll
+++ b/llvm/test/CodeGen/AArch64/swift-async-unwind.ll
@ -0,0 +1,13 @@
+; RUN: llc -mtriple=arm64-apple-ios %s -filetype=obj -o - | llvm-objdump --unwind-info - | FileCheck %s
+
+; Swift asynchronous context is incompatible with the compact unwind encoding
+; that currently exists and assumes callee-saved registers are right next to FP
+; in a particular order. This isn't a problem now because C++ exceptions aren't
+; allowed to unwind through Swift code, but at least make sure the compact info
+; says to use DWARF correctly.
+
+; CHECK: compact encoding: 0x03000000
+define void @foo(i8* swiftasync %in) "frame-pointer"="all" {
+  call void asm sideeffect "", "~{x23}"()
+  ret void
+}
--- a/llvm/test/CodeGen/AArch64/swift-async.ll
+++ b/llvm/test/CodeGen/AArch64/swift-async.ll
@ -0,0 +1,157 @@
+; RUN: llc -mtriple=arm64-apple-ios %s -o - | FileCheck %s --check-prefixes=CHECK-NOAUTH,CHECK
+; RUN: llc -mtriple=arm64-apple-ios -mcpu=apple-a13 %s -o - | FileCheck %s --check-prefixes=CHECK-NOAUTH,CHECK
+; RUN: llc -mtriple=arm64e-apple-ios %s -o - | FileCheck %s --check-prefixes=CHECK-AUTH,CHECK
+
+; Important details in prologue:
+;   * x22 is stored just below x29
+;   * Enough stack space is allocated for everything
+define void @simple(i8* swiftasync %ctx) "frame-pointer"="all" {
+; CHECK-LABEL: simple:
+; CHECK: orr x29, x29, #0x100000000000000
+; CHECK: sub sp, sp, #32
+; CHECK: stp x29, x30, [sp, #16]
+
+; CHECK-NOAUTH: str x22, [sp, #8]
+; CHECK-AUTH: add x16, sp, #8
+; CHECK-AUTH: movk x16, #49946, lsl #48
+; CHECK-AUTH: mov x17, x22
+; CHECK-AUTH: pacdb x17, x16
+; CHECK-AUTH: str x17, [sp, #8]
+
+; CHECK: add x29, sp, #16
+; CHECK: .cfi_def_cfa w29, 16
+; CHECK: .cfi_offset w30, -8
+; CHECK: .cfi_offset w29, -16
+
+;[...]
+
+; CHECK: ldp x29, x30, [sp, #16]
+; CHECK: and x29, x29, #0xefffffffffffffff
+; CHECK: add sp, sp, #32
+
+  ret void
+}
+
+define void @more_csrs(i8* swiftasync %ctx) "frame-pointer"="all" {
+; CHECK-LABEL: more_csrs:
+; CHECK: orr x29, x29, #0x100000000000000
+; CHECK: sub sp, sp, #48
+; CHECK: stp x24, x23, [sp, #8]
+; CHECK: stp x29, x30, [sp, #32]
+
+; CHECK-NOAUTH: str x22, [sp, #24]
+; CHECK-AUTH: add x16, sp, #24
+; CHECK-AUTH: movk x16, #49946, lsl #48
+; CHECK-AUTH: mov x17, x22
+; CHECK-AUTH: pacdb x17, x16
+; CHECK-AUTH: str x17, [sp, #24]
+
+; CHECK: add x29, sp, #32
+; CHECK: .cfi_def_cfa w29, 16
+; CHECK: .cfi_offset w30, -8
+; CHECK: .cfi_offset w29, -16
+; CHECK: .cfi_offset w23, -32
+
+; [...]
+
+; CHECK: ldp x29, x30, [sp, #32]
+; CHECK: ldp x24, x23, [sp, #8]
+; CHECK: and x29, x29, #0xefffffffffffffff
+; CHECK: add sp, sp, #48
+  call void asm sideeffect "", "~{x23}"()
+  ret void
+}
+
+define void @locals(i8* swiftasync %ctx) "frame-pointer"="all" {
+; CHECK-LABEL: locals:
+; CHECK: orr x29, x29, #0x100000000000000
+; CHECK: sub sp, sp, #64
+; CHECK: stp x29, x30, [sp, #48]
+
+; CHECK-NOAUTH: str x22, [sp, #40]
+; CHECK-AUTH: add x16, sp, #40
+; CHECK-AUTH: movk x16, #49946, lsl #48
+; CHECK-AUTH: mov x17, x22
+; CHECK-AUTH: pacdb x17, x16
+; CHECK-AUTH: str x17, [sp, #40]
+
+; CHECK: add x29, sp, #48
+; CHECK: .cfi_def_cfa w29, 16
+; CHECK: .cfi_offset w30, -8
+; CHECK: .cfi_offset w29, -16
+
+; CHECK: mov x0, sp
+; CHECK: bl _bar
+
+; [...]
+
+; CHECK: ldp x29, x30, [sp, #48]
+; CHECK: and x29, x29, #0xefffffffffffffff
+; CHECK: add sp, sp, #64
+  %var = alloca i32, i32 10
+  call void @bar(i32* %var)
+  ret void
+}
+
+define void @use_input_context(i8* swiftasync %ctx, i8** %ptr) "frame-pointer"="all" {
+; CHECK-LABEL: use_input_context:
+
+; CHECK-NOAUTH: str x22, [sp
+; CHECK-AUTH: mov x17, x22
+
+; CHECK-NOT: x22
+; CHECK: str x22, [x0]
+
+  store i8* %ctx, i8** %ptr
+  ret void
+}
+
+define i8** @context_in_func() "frame-pointer"="non-leaf" {
+; CHECK-LABEL: context_in_func:
+
+; CHECK-NOAUTH: str xzr, [sp, #8]
+; CHECK-AUTH: add x16, sp, #8
+; CHECK-AUTH: movk x16, #49946, lsl #48
+; CHECK-AUTH: mov x17, xzr
+; CHECK-AUTH: pacdb x17, x16
+; CHECK-AUTH: str x17, [sp, #8]
+
+  %ptr = call i8** @llvm.swift.async.context.addr()
+  ret i8** %ptr
+}
+
+define void @write_frame_context(i8* swiftasync %ctx, i8* %newctx) "frame-pointer"="non-leaf" {
+; CHECK-LABEL: write_frame_context:
+; CHECK: sub x[[ADDR:[0-9]+]], x29, #8
+; CHECK: str x0, [x[[ADDR]]]
+  %ptr = call i8** @llvm.swift.async.context.addr()
+  store i8* %newctx, i8** %ptr
+  ret void
+}
+
+define void @simple_fp_elim(i8* swiftasync %ctx) "frame-pointer"="non-leaf" {
+; CHECK-LABEL: simple_fp_elim:
+; CHECK-NOT: orr x29, x29, #0x100000000000000
+
+  ret void
+}
+
+define void @large_frame(i8* swiftasync %ctx) "frame-pointer"="all" {
+; CHECK-LABEL: large_frame:
+; CHECK: sub sp, sp, #48
+; CHECK: stp x28, x27, [sp, #8]
+; CHECK: stp x29, x30, [sp, #32]
+; CHECK-NOAUTH: str x22, [sp, #24]
+; CHECK: add x29, sp, #32
+; CHECK: sub sp, sp, #1024
+; [...]
+; CHECK: add sp, sp, #1024
+; CHECK: ldp x29, x30, [sp, #32]
+; CHECK: ldp x28, x27, [sp, #8]
+; CHECK: ret
+  %var = alloca i8, i32 1024
+  ret void
+}
+
+declare void @bar(i32*)
+declare i8** @llvm.swift.async.context.addr()
--- a/llvm/test/Verifier/swiftasync.ll
+++ b/llvm/test/Verifier/swiftasync.ll
@ -0,0 +1,4 @@
+; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
+
+declare void @a(i32* swiftasync %a, i32* swiftasync %b)
+; CHECK: Cannot have multiple 'swiftasync' parameters!