[coroutines] Add codegen for await and yield expressions

Details:

Emit suspend expression which roughly looks like:

auto && x = CommonExpr();
if (!x.await_ready()) {
   llvm_coro_save();
   x.await_suspend(...);     (*)
   llvm_coro_suspend(); (**)
}
x.await_resume();
where the result of the entire expression is the result of x.await_resume()

(*) If x.await_suspend return type is bool, it allows to veto a suspend:
if (x.await_suspend(...))
   llvm_coro_suspend();
(**) llvm_coro_suspend() encodes three possible continuations as a switch instruction:

%where-to = call i8 @llvm.coro.suspend(...)
switch i8 %where-to, label %coro.ret [ ; jump to epilogue to suspend
  i8 0, label %yield.ready   ; go here when resumed
  i8 1, label %yield.cleanup ; go here when destroyed
]

llvm-svn: 298784

clang/lib/CodeGen/CGCoroutine.cpp

@@ -12,35 +12,58 @@
 //===----------------------------------------------------------------------===//
 
 #include "CodeGenFunction.h"
+#include "llvm/ADT/ScopeExit.h"
 #include "clang/AST/StmtCXX.h"
 
 using namespace clang;
 using namespace CodeGen;
 
-namespace clang {
-namespace CodeGen {
+using llvm::Value;
+using llvm::BasicBlock;
 
-struct CGCoroData {
+namespace {
+enum class AwaitKind { Init, Normal, Yield, Final };
+static constexpr llvm::StringLiteral AwaitKindStr[] = {"init", "await", "yield",
+                                                       "final"};
+}
 
-  // Stores the jump destination just before the final suspend. Coreturn
+struct clang::CodeGen::CGCoroData {
+  // What is the current await expression kind and how many
+  // await/yield expressions were encountered so far.
+  // These are used to generate pretty labels for await expressions in LLVM IR.
+  AwaitKind CurrentAwaitKind = AwaitKind::Init;
+  unsigned AwaitNum = 0;
+  unsigned YieldNum = 0;
+
+  // How many co_return statements are in the coroutine. Used to decide whether
+  // we need to add co_return; equivalent at the end of the user authored body.
+  unsigned CoreturnCount = 0;
+
+  // A branch to this block is emitted when coroutine needs to suspend.
+  llvm::BasicBlock *SuspendBB = nullptr;
+
+  // Stores the jump destination just before the coroutine memory is freed.
+  // This is the destination that every suspend point jumps to for the cleanup
+  // branch.
+  CodeGenFunction::JumpDest CleanupJD;
+
+  // Stores the jump destination just before the final suspend. The co_return
   // statements jumps to this point after calling return_xxx promise member.
   CodeGenFunction::JumpDest FinalJD;
 
-  unsigned CoreturnCount = 0;
-
   // Stores the llvm.coro.id emitted in the function so that we can supply it
   // as the first argument to coro.begin, coro.alloc and coro.free intrinsics.
   // Note: llvm.coro.id returns a token that cannot be directly expressed in a
   // builtin.
   llvm::CallInst *CoroId = nullptr;
+
   // If coro.id came from the builtin, remember the expression to give better
   // diagnostic. If CoroIdExpr is nullptr, the coro.id was created by
   // EmitCoroutineBody.
   CallExpr const *CoroIdExpr = nullptr;
 };
-}
-}
 
+// Defining these here allows to keep CGCoroData private to this file.
 clang::CodeGen::CodeGenFunction::CGCoroInfo::CGCoroInfo() {}
 CodeGenFunction::CGCoroInfo::~CGCoroInfo() {}
 
@@ -66,6 +89,126 @@ static void createCoroData(CodeGenFunction &CGF,
   CurCoro.Data->CoroIdExpr = CoroIdExpr;
 }
 
+// Synthesize a pretty name for a suspend point.
+static SmallString<32> buildSuspendPrefixStr(CGCoroData &Coro, AwaitKind Kind) {
+  unsigned No = 0;
+  switch (Kind) {
+  case AwaitKind::Init:
+  case AwaitKind::Final:
+    break;
+  case AwaitKind::Normal:
+    No = ++Coro.AwaitNum;
+    break;
+  case AwaitKind::Yield:
+    No = ++Coro.YieldNum;
+    break;
+  }
+  SmallString<32> Prefix(AwaitKindStr[static_cast<unsigned>(Kind)]);
+  if (No > 1) {
+    Twine(No).toVector(Prefix);
+  }
+  return Prefix;
+}
+
+// Emit suspend expression which roughly looks like:
+//
+//   auto && x = CommonExpr();
+//   if (!x.await_ready()) {
+//      llvm_coro_save();
+//      x.await_suspend(...);     (*)
+//      llvm_coro_suspend(); (**)
+//   }
+//   x.await_resume();
+//
+// where the result of the entire expression is the result of x.await_resume()
+//
+//   (*) If x.await_suspend return type is bool, it allows to veto a suspend:
+//      if (x.await_suspend(...))
+//        llvm_coro_suspend();
+//
+//  (**) llvm_coro_suspend() encodes three possible continuations as
+//       a switch instruction:
+//
+//  %where-to = call i8 @llvm.coro.suspend(...)
+//  switch i8 %where-to, label %coro.ret [ ; jump to epilogue to suspend
+//    i8 0, label %yield.ready   ; go here when resumed
+//    i8 1, label %yield.cleanup ; go here when destroyed
+//  ]
+//
+//  See llvm's docs/Coroutines.rst for more details.
+//
+static RValue emitSuspendExpression(CodeGenFunction &CGF, CGCoroData &Coro,
+                                    CoroutineSuspendExpr const &S,
+                                    AwaitKind Kind, AggValueSlot aggSlot,
+                                    bool ignoreResult) {
+  auto *E = S.getCommonExpr();
+  auto Binder =
+      CodeGenFunction::OpaqueValueMappingData::bind(CGF, S.getOpaqueValue(), E);
+  auto UnbindOnExit = llvm::make_scope_exit([&] { Binder.unbind(CGF); });
+
+  auto Prefix = buildSuspendPrefixStr(Coro, Kind);
+  BasicBlock *ReadyBlock = CGF.createBasicBlock(Prefix + Twine(".ready"));
+  BasicBlock *SuspendBlock = CGF.createBasicBlock(Prefix + Twine(".suspend"));
+  BasicBlock *CleanupBlock = CGF.createBasicBlock(Prefix + Twine(".cleanup"));
+
+  // If expression is ready, no need to suspend.
+  CGF.EmitBranchOnBoolExpr(S.getReadyExpr(), ReadyBlock, SuspendBlock, 0);
+
+  // Otherwise, emit suspend logic.
+  CGF.EmitBlock(SuspendBlock);
+
+  auto &Builder = CGF.Builder;
+  llvm::Function *CoroSave = CGF.CGM.getIntrinsic(llvm::Intrinsic::coro_save);
+  auto *NullPtr = llvm::ConstantPointerNull::get(CGF.CGM.Int8PtrTy);
+  auto *SaveCall = Builder.CreateCall(CoroSave, {NullPtr});
+
+  auto *SuspendRet = CGF.EmitScalarExpr(S.getSuspendExpr());
+  if (SuspendRet != nullptr) {
+    // Veto suspension if requested by bool returning await_suspend.
+    assert(SuspendRet->getType()->isIntegerTy(1) &&
+           "Sema should have already checked that it is void or bool");
+    BasicBlock *RealSuspendBlock =
+        CGF.createBasicBlock(Prefix + Twine(".suspend.bool"));
+    CGF.Builder.CreateCondBr(SuspendRet, RealSuspendBlock, ReadyBlock);
+    SuspendBlock = RealSuspendBlock;
+    CGF.EmitBlock(RealSuspendBlock);
+  }
+
+  // Emit the suspend point.
+  const bool IsFinalSuspend = (Kind == AwaitKind::Final);
+  llvm::Function *CoroSuspend =
+      CGF.CGM.getIntrinsic(llvm::Intrinsic::coro_suspend);
+  auto *SuspendResult = Builder.CreateCall(
+      CoroSuspend, {SaveCall, Builder.getInt1(IsFinalSuspend)});
+
+  // Create a switch capturing three possible continuations.
+  auto *Switch = Builder.CreateSwitch(SuspendResult, Coro.SuspendBB, 2);
+  Switch->addCase(Builder.getInt8(0), ReadyBlock);
+  Switch->addCase(Builder.getInt8(1), CleanupBlock);
+
+  // Emit cleanup for this suspend point.
+  CGF.EmitBlock(CleanupBlock);
+  CGF.EmitBranchThroughCleanup(Coro.CleanupJD);
+
+  // Emit await_resume expression.
+  CGF.EmitBlock(ReadyBlock);
+  return CGF.EmitAnyExpr(S.getResumeExpr(), aggSlot, ignoreResult);
+}
+
+RValue CodeGenFunction::EmitCoawaitExpr(const CoawaitExpr &E,
+                                        AggValueSlot aggSlot,
+                                        bool ignoreResult) {
+  return emitSuspendExpression(*this, *CurCoro.Data, E,
+                               CurCoro.Data->CurrentAwaitKind, aggSlot,
+                               ignoreResult);
+}
+RValue CodeGenFunction::EmitCoyieldExpr(const CoyieldExpr &E,
+                                        AggValueSlot aggSlot,
+                                        bool ignoreResult) {
+  return emitSuspendExpression(*this, *CurCoro.Data, E, AwaitKind::Yield,
+                               aggSlot, ignoreResult);
+}
+
 void CodeGenFunction::EmitCoreturnStmt(CoreturnStmt const &S) {
   ++CurCoro.Data->CoreturnCount;
   EmitStmt(S.getPromiseCall());
@@ -78,11 +221,13 @@ void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) {
   unsigned NewAlign = TI.getNewAlign() / TI.getCharWidth();
 
   auto *FinalBB = createBasicBlock("coro.final");
+  auto *RetBB = createBasicBlock("coro.ret");
 
   auto *CoroId = Builder.CreateCall(
       CGM.getIntrinsic(llvm::Intrinsic::coro_id),
       {Builder.getInt32(NewAlign), NullPtr, NullPtr, NullPtr});
   createCoroData(*this, CurCoro, CoroId);
+  CurCoro.Data->SuspendBB = RetBB;
 
   EmitScalarExpr(S.getAllocate());
 
@@ -90,10 +235,12 @@ void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) {
 
   EmitStmt(S.getPromiseDeclStmt());
 
+  CurCoro.Data->CleanupJD = getJumpDestInCurrentScope(RetBB);
   CurCoro.Data->FinalJD = getJumpDestInCurrentScope(FinalBB);
 
   // FIXME: Emit initial suspend and more before the body.
 
+  CurCoro.Data->CurrentAwaitKind = AwaitKind::Normal;
   EmitStmt(S.getBody());
 
   // See if we need to generate final suspend.
@@ -105,6 +252,8 @@ void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) {
   }
   EmitStmt(S.getDeallocate());
 
+  EmitBlock(RetBB);
+
   // FIXME: Emit return for the coroutine return object.
 }
 

clang/lib/CodeGen/CGExprAgg.cpp

@@ -111,6 +111,13 @@ public:
   void VisitGenericSelectionExpr(GenericSelectionExpr *GE) {
     Visit(GE->getResultExpr());
   }
+  void VisitCoawaitExpr(CoawaitExpr *E) {
+    CGF.EmitCoawaitExpr(*E, Dest, IsResultUnused);
+  }
+  void VisitCoyieldExpr(CoyieldExpr *E) {
+    CGF.EmitCoyieldExpr(*E, Dest, IsResultUnused);
+  }
+  void VisitUnaryCoawait(UnaryOperator *E) { Visit(E->getSubExpr()); }
   void VisitUnaryExtension(UnaryOperator *E) { Visit(E->getSubExpr()); }
   void VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *E) {
     return Visit(E->getReplacement());

clang/lib/CodeGen/CGExprComplex.cpp

@@ -110,6 +110,16 @@ public:
   VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *PE) {
     return Visit(PE->getReplacement());
   }
+  ComplexPairTy VisitCoawaitExpr(CoawaitExpr *S) {
+    return CGF.EmitCoawaitExpr(*S).getComplexVal();
+  }
+  ComplexPairTy VisitCoyieldExpr(CoyieldExpr *S) {
+    return CGF.EmitCoyieldExpr(*S).getComplexVal();
+  }
+  ComplexPairTy VisitUnaryCoawait(const UnaryOperator *E) {
+    return Visit(E->getSubExpr());
+  }
+
 
   // l-values.
   ComplexPairTy VisitDeclRefExpr(DeclRefExpr *E) {

clang/lib/CodeGen/CGExprScalar.cpp

@@ -276,6 +276,15 @@ public:
   Value *VisitGenericSelectionExpr(GenericSelectionExpr *GE) {
     return Visit(GE->getResultExpr());
   }
+  Value *VisitCoawaitExpr(CoawaitExpr *S) {
+    return CGF.EmitCoawaitExpr(*S).getScalarVal();
+  }
+  Value *VisitCoyieldExpr(CoyieldExpr *S) {
+    return CGF.EmitCoyieldExpr(*S).getScalarVal();
+  }
+  Value *VisitUnaryCoawait(const UnaryOperator *E) {
+    return Visit(E->getSubExpr());
+  }
 
   // Leaves.
   Value *VisitIntegerLiteral(const IntegerLiteral *E) {

clang/lib/CodeGen/CodeGenFunction.h

@@ -1986,7 +1986,7 @@ public:
   /// pointer to a char.
   Address EmitMSVAListRef(const Expr *E);
 
-  /// EmitAnyExprToTemp - Similary to EmitAnyExpr(), however, the result will
+  /// EmitAnyExprToTemp - Similarly to EmitAnyExpr(), however, the result will
   /// always be accessible even if no aggregate location is provided.
   RValue EmitAnyExprToTemp(const Expr *E);
 
@@ -2528,6 +2528,12 @@ public:
 
   void EmitCoroutineBody(const CoroutineBodyStmt &S);
   void EmitCoreturnStmt(const CoreturnStmt &S);
+  RValue EmitCoawaitExpr(const CoawaitExpr &E,
+                         AggValueSlot aggSlot = AggValueSlot::ignored(),
+                         bool ignoreResult = false);
+  RValue EmitCoyieldExpr(const CoyieldExpr &E,
+                         AggValueSlot aggSlot = AggValueSlot::ignored(),
+                         bool ignoreResult = false);
   RValue EmitCoroutineIntrinsic(const CallExpr *E, unsigned int IID);
 
   void EnterCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false);

clang/test/CodeGenCoroutines/coro-await.cpp

@@ -0,0 +1,230 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fcoroutines-ts -std=c++14 -emit-llvm %s -o - -disable-llvm-passes | FileCheck %s
+
+namespace std {
+namespace experimental {
+template <typename... T>
+struct coroutine_traits;
+
+template <typename Promise = void> struct coroutine_handle;
+
+template <>
+struct coroutine_handle<void> {
+  void *ptr;
+  static coroutine_handle from_address(void *);
+};
+
+template <typename Promise>
+struct coroutine_handle : coroutine_handle<> {
+  static coroutine_handle from_address(void *);
+};
+
+}
+}
+
+struct suspend_always {
+  int stuff;
+  bool await_ready();
+  void await_suspend(std::experimental::coroutine_handle<>);
+  void await_resume();
+};
+
+template<>
+struct std::experimental::coroutine_traits<void> {
+  struct promise_type {
+    void get_return_object();
+    suspend_always initial_suspend();
+    suspend_always final_suspend();
+    void return_void();
+  };
+};
+
+// CHECK-LABEL: f0(
+extern "C" void f0() {
+
+  co_await suspend_always{};
+  // See if we need to suspend:
+  // --------------------------
+  // CHECK: %[[READY:.+]] = call zeroext i1 @_ZN14suspend_always11await_readyEv(%struct.suspend_always* %[[AWAITABLE:.+]])
+  // CHECK: br i1 %[[READY]], label %[[READY_BB:.+]], label %[[SUSPEND_BB:.+]]
+
+  // If we are suspending:
+  // ---------------------
+  // CHECK: [[SUSPEND_BB]]:
+  // CHECK: %[[SUSPEND_ID:.+]] = call token @llvm.coro.save(
+  // ---------------------------
+  // Build the coroutine handle and pass it to await_suspend
+  // ---------------------------
+  // CHECK: %[[FRAME:.+]] = call i8* @llvm.coro.frame()
+  // CHECK: call i8* @_ZNSt12experimental16coroutine_handleINS_16coroutine_traitsIJvEE12promise_typeEE12from_addressEPv(i8* %[[FRAME]])
+  //   ... many lines of code to coerce coroutine_handle into an i8* scalar
+  // CHECK: %[[CH:.+]] = load i8*, i8** %{{.+}}
+  // CHECK: call void @_ZN14suspend_always13await_suspendENSt12experimental16coroutine_handleIvEE(%struct.suspend_always* %[[AWAITABLE]], i8* %[[CH]])
+  // -------------------------
+  // Generate a suspend point:
+  // -------------------------
+  // CHECK: %[[OUTCOME:.+]] = call i8 @llvm.coro.suspend(token %[[SUSPEND_ID]], i1 false)
+  // CHECK: switch i8 %[[OUTCOME]], label %[[RET_BB:.+]] [
+  // CHECK:   i8 0, label %[[READY_BB]]
+  // CHECK:   i8 1, label %[[CLEANUP_BB:.+]]
+  // CHECK: ]
+
+  // Cleanup code goes here:
+  // -----------------------
+  // CHECK: [[CLEANUP_BB]]:
+
+  // When coroutine is resumed, call await_resume
+  // --------------------------
+  // CHECK: [[READY_BB]]:
+  // CHECK:  call void @_ZN14suspend_always12await_resumeEv(%struct.suspend_always* %[[AWAITABLE]])
+}
+
+struct suspend_maybe {
+  float stuff;
+  ~suspend_maybe();
+  bool await_ready();
+  bool await_suspend(std::experimental::coroutine_handle<>);
+  void await_resume();
+};
+
+
+template<>
+struct std::experimental::coroutine_traits<void,int> {
+  struct promise_type {
+    void get_return_object();
+    suspend_always initial_suspend();
+    suspend_always final_suspend();
+    void return_void();
+    suspend_maybe yield_value(int);
+  };
+};
+
+// CHECK-LABEL: f1(
+extern "C" void f1(int) {
+  co_yield 42;
+  // CHECK: %[[PROMISE:.+]] = alloca %"struct.std::experimental::coroutine_traits<void, int>::promise_type"
+  // CHECK: call void @_ZNSt12experimental16coroutine_traitsIJviEE12promise_type11yield_valueEi(%struct.suspend_maybe* sret %[[AWAITER:.+]], %"struct.std::experimental::coroutine_traits<void, int>::promise_type"* %[[PROMISE]], i32 42)
+
+  // See if we need to suspend:
+  // --------------------------
+  // CHECK: %[[READY:.+]] = call zeroext i1 @_ZN13suspend_maybe11await_readyEv(%struct.suspend_maybe* %[[AWAITABLE]])
+  // CHECK: br i1 %[[READY]], label %[[READY_BB:.+]], label %[[SUSPEND_BB:.+]]
+
+  // If we are suspending:
+  // ---------------------
+  // CHECK: [[SUSPEND_BB]]:
+  // CHECK: %[[SUSPEND_ID:.+]] = call token @llvm.coro.save(
+  // ---------------------------
+  // Build the coroutine handle and pass it to await_suspend
+  // ---------------------------
+  // CHECK: %[[FRAME:.+]] = call i8* @llvm.coro.frame()
+  // CHECK: call i8* @_ZNSt12experimental16coroutine_handleINS_16coroutine_traitsIJviEE12promise_typeEE12from_addressEPv(i8* %[[FRAME]])
+  //   ... many lines of code to coerce coroutine_handle into an i8* scalar
+  // CHECK: %[[CH:.+]] = load i8*, i8** %{{.+}}
+  // CHECK: %[[YES:.+]] = call zeroext i1 @_ZN13suspend_maybe13await_suspendENSt12experimental16coroutine_handleIvEE(%struct.suspend_maybe* %[[AWAITABLE]], i8* %[[CH]])
+  // -------------------------------------------
+  // See if await_suspend decided not to suspend
+  // -------------------------------------------
+  // CHECK: br i1 %[[YES]], label %[[SUSPEND_PLEASE:.+]], label %[[READY_BB]]
+
+  // CHECK: [[SUSPEND_PLEASE]]:
+  // CHECK:    call i8 @llvm.coro.suspend(token %[[SUSPEND_ID]], i1 false)
+
+  // CHECK: [[READY_BB]]:
+  // CHECK:     call void @_ZN13suspend_maybe12await_resumeEv(%struct.suspend_maybe* %[[AWAITABLE]])
+}
+
+struct ComplexAwaiter {
+  template <typename F> void await_suspend(F);
+  bool await_ready();
+  _Complex float await_resume();
+};
+extern "C" void UseComplex(_Complex float);
+
+// CHECK-LABEL: @TestComplex(
+extern "C" void TestComplex() {
+  UseComplex(co_await ComplexAwaiter{});
+  // CHECK: call <2 x float> @_ZN14ComplexAwaiter12await_resumeEv(%struct.ComplexAwaiter*
+  // CHECK: call void @UseComplex(<2 x float> %{{.+}})
+
+  co_await ComplexAwaiter{};
+  // CHECK: call <2 x float> @_ZN14ComplexAwaiter12await_resumeEv(%struct.ComplexAwaiter*
+
+  _Complex float Val = co_await ComplexAwaiter{};
+  // CHECK: call <2 x float> @_ZN14ComplexAwaiter12await_resumeEv(%struct.ComplexAwaiter*
+}
+
+struct Aggr { int X, Y, Z; ~Aggr(); };
+struct AggrAwaiter {
+  template <typename F> void await_suspend(F);
+  bool await_ready();
+  Aggr await_resume();
+};
+
+extern "C" void Whatever();
+extern "C" void UseAggr(Aggr&&);
+
+// FIXME: Once the cleanup code is in, add testing that destructors for Aggr
+// are invoked properly on the cleanup branches.
+
+// CHECK-LABEL: @TestAggr(
+extern "C" void TestAggr() {
+  UseAggr(co_await AggrAwaiter{});
+  Whatever();
+  // CHECK: call void @_ZN11AggrAwaiter12await_resumeEv(%struct.Aggr* sret %[[AwaitResume:.+]],
+  // CHECK: call void @UseAggr(%struct.Aggr* dereferenceable(12) %[[AwaitResume]])
+  // CHECK: call void @_ZN4AggrD1Ev(%struct.Aggr* %[[AwaitResume]])
+  // CHECK: call void @Whatever()
+
+  co_await AggrAwaiter{};
+  Whatever();
+  // CHECK: call void @_ZN11AggrAwaiter12await_resumeEv(%struct.Aggr* sret %[[AwaitResume2:.+]],
+  // CHECK: call void @_ZN4AggrD1Ev(%struct.Aggr* %[[AwaitResume2]])
+  // CHECK: call void @Whatever()
+
+  Aggr Val = co_await AggrAwaiter{};
+  Whatever();
+  // CHECK: call void @_ZN11AggrAwaiter12await_resumeEv(%struct.Aggr* sret %[[AwaitResume3:.+]],
+  // CHECK: call void @Whatever()
+  // CHECK: call void @_ZN4AggrD1Ev(%struct.Aggr* %[[AwaitResume3]])
+}
+
+struct ScalarAwaiter {
+  template <typename F> void await_suspend(F);
+  bool await_ready();
+  int await_resume();
+};
+
+extern "C" void UseScalar(int);
+
+// CHECK-LABEL: @TestScalar(
+extern "C" void TestScalar() {
+  UseScalar(co_await ScalarAwaiter{});
+  // CHECK: %[[Result:.+]] = call i32 @_ZN13ScalarAwaiter12await_resumeEv(%struct.ScalarAwaiter*
+  // CHECK: call void @UseScalar(i32 %[[Result]])
+
+  int Val = co_await ScalarAwaiter{};
+  // CHECK: %[[Result2:.+]] = call i32 @_ZN13ScalarAwaiter12await_resumeEv(%struct.ScalarAwaiter*
+  // CHECK: store i32 %[[Result2]], i32* %Val
+
+  co_await ScalarAwaiter{};
+  // CHECK: call i32 @_ZN13ScalarAwaiter12await_resumeEv(%struct.ScalarAwaiter*
+}
+
+// Test operator co_await codegen.
+enum class MyInt: int {};
+ScalarAwaiter operator co_await(MyInt);
+
+struct MyAgg {
+  AggrAwaiter operator co_await();
+};
+
+// CHECK-LABEL: @TestOpAwait(
+extern "C" void TestOpAwait() {
+  co_await MyInt(42);
+  // CHECK: call void @_Zaw5MyInt(i32 42)
+  // CHECK: call i32 @_ZN13ScalarAwaiter12await_resumeEv(%struct.ScalarAwaiter* %
+
+  co_await MyAgg{};
+  // CHECK: call void @_ZN5MyAggawEv(%struct.MyAgg* %
+  // CHECK: call void @_ZN11AggrAwaiter12await_resumeEv(%struct.Aggr* sret %
+}