15#include "llvm/ADT/ScopeExit.h"
20using namespace CodeGen;
23using llvm::BasicBlock;
26enum class AwaitKind { Init,
Normal, Yield, Final };
27static constexpr llvm::StringLiteral AwaitKindStr[] = {
"init",
"await",
"yield",
90 CodeGenFunction::CGCoroInfo &
CurCoro,
91 llvm::CallInst *CoroId,
92 CallExpr const *CoroIdExpr =
nullptr) {
95 CGF.
CGM.
Error(CoroIdExpr->getBeginLoc(),
96 "only one __builtin_coro_id can be used in a function");
98 CGF.
CGM.
Error(CoroIdExpr->getBeginLoc(),
99 "__builtin_coro_id shall not be used in a C++ coroutine");
101 llvm_unreachable(
"EmitCoroutineBodyStatement called twice?");
115 case AwaitKind::Init:
116 case AwaitKind::Final:
118 case AwaitKind::Normal:
121 case AwaitKind::Yield:
127 Twine(
No).toVector(Prefix);
133 if (
const auto *CE = dyn_cast<CXXMemberCallExpr>(E))
134 if (
const auto *Proto =
170 struct LValueOrRValue {
178 bool ignoreResult,
bool forLValue) {
179 auto *E = S.getCommonExpr();
183 auto UnbindOnExit = llvm::make_scope_exit([&] { Binder.unbind(CGF); });
187 BasicBlock *SuspendBlock = CGF.
createBasicBlock(Prefix + Twine(
".suspend"));
188 BasicBlock *CleanupBlock = CGF.
createBasicBlock(Prefix + Twine(
".cleanup"));
197 llvm::Function *CoroSave = CGF.
CGM.
getIntrinsic(llvm::Intrinsic::coro_save);
198 auto *NullPtr = llvm::ConstantPointerNull::get(CGF.
CGM.
Int8PtrTy);
199 auto *SaveCall =
Builder.CreateCall(CoroSave, {NullPtr});
204 if (SuspendRet !=
nullptr && SuspendRet->getType()->isIntegerTy(1)) {
206 BasicBlock *RealSuspendBlock =
208 CGF.
Builder.CreateCondBr(SuspendRet, RealSuspendBlock, ReadyBlock);
213 const bool IsFinalSuspend = (Kind == AwaitKind::Final);
214 llvm::Function *CoroSuspend =
216 auto *SuspendResult =
Builder.CreateCall(
217 CoroSuspend, {SaveCall,
Builder.getInt1(IsFinalSuspend)});
240 auto Loc = S.getResumeExpr()->getExprLoc();
253 Res.RV = CGF.
EmitAnyExpr(S.getResumeExpr(), aggSlot, ignoreResult);
268 ignoreResult,
false).RV;
274 aggSlot, ignoreResult,
false).RV;
279 const Expr *RV = S.getOperand();
283 RunCleanupsScope cleanupScope(*
this);
297 assert(isa<CallExpr>(RE) &&
"unexpected suspend expression type");
298 return cast<CallExpr>(RE)->getCallReturnType(Ctx);
305 "Can't have a scalar return unless the return type is a "
315 "Can't have a scalar return unless the return type is a "
324struct GetParamRef :
public StmtVisitor<GetParamRef> {
329 assert(
Expr ==
nullptr &&
"multilple declref in param move");
332 void VisitStmt(
Stmt *S) {
333 for (
auto *
C : S->children()) {
346 struct ParamReferenceReplacerRAII {
347 CodeGenFunction::DeclMapTy SavedLocals;
348 CodeGenFunction::DeclMapTy& LocalDeclMap;
350 ParamReferenceReplacerRAII(CodeGenFunction::DeclMapTy &LocalDeclMap)
351 : LocalDeclMap(LocalDeclMap) {}
360 Visitor.Visit(
const_cast<Expr*
>(InitExpr));
361 assert(Visitor.Expr);
365 auto it = LocalDeclMap.find(PD);
366 assert(it != LocalDeclMap.end() &&
"parameter is not found");
367 SavedLocals.insert({ PD, it->second });
369 auto copyIt = LocalDeclMap.find(VD);
370 assert(copyIt != LocalDeclMap.end() &&
"parameter copy is not found");
371 it->second = copyIt->getSecond();
374 ~ParamReferenceReplacerRAII() {
375 for (
auto&& SavedLocal : SavedLocals) {
376 LocalDeclMap.insert({SavedLocal.first, SavedLocal.second});
389 BundleList.emplace_back(
"funclet", EHPad);
401 auto *NullPtr = llvm::ConstantPointerNull::get(CGF.
Int8PtrTy);
402 llvm::Function *CoroEndFn = CGM.getIntrinsic(llvm::Intrinsic::coro_end);
405 auto *CoroEnd = CGF.
Builder.CreateCall(
406 CoroEndFn, {NullPtr, CGF.
Builder.getTrue()}, Bundles);
407 if (Bundles.empty()) {
412 CGF.
Builder.CreateCondBr(CoroEnd, ResumeBB, CleanupContBB);
435 BasicBlock *SaveInsertBlock = CGF.
Builder.GetInsertBlock();
447 CGF.
CGM.
Error(Deallocate->getBeginLoc(),
448 "Deallocation expressoin does not refer to coro.free");
453 auto *InsertPt = SaveInsertBlock->getTerminator();
454 CoroFree->moveBefore(InsertPt);
455 CGF.
Builder.SetInsertPoint(InsertPt);
458 auto *NullPtr = llvm::ConstantPointerNull::get(CGF.
Int8PtrTy);
459 auto *Cond = CGF.
Builder.CreateICmpNE(CoroFree, NullPtr);
460 CGF.
Builder.CreateCondBr(Cond, FreeBB, AfterFreeBB);
463 InsertPt->eraseFromParent();
464 CGF.
Builder.SetInsertPoint(AfterFreeBB);
466 explicit CallCoroDelete(
Stmt *DeallocStmt) : Deallocate(DeallocStmt) {}
471struct GetReturnObjectManager {
476 bool DirectEmit =
false;
479 CodeGenFunction::AutoVarEmission GroEmission;
482 : CGF(CGF), Builder(CGF.Builder), S(S), GroActiveFlag(
Address::invalid()),
504 auto *RVI = S.getReturnValueInit();
505 assert(RVI &&
"expected RVI");
506 auto GroType = RVI->getType();
516 void EmitGroAlloca() {
520 auto *GroDeclStmt = dyn_cast_or_null<DeclStmt>(S.getResultDecl());
526 auto *GroVarDecl = cast<VarDecl>(GroDeclStmt->getSingleDecl());
531 Builder.CreateStore(Builder.getFalse(), GroActiveFlag);
543 if (
auto *Cleanup = dyn_cast<EHCleanupScope>(&*
b)) {
544 assert(!Cleanup->hasActiveFlag() &&
"cleanup already has active flag?");
545 Cleanup->setActiveFlag(GroActiveFlag);
546 Cleanup->setTestFlagInEHCleanup();
547 Cleanup->setTestFlagInNormalCleanup();
568 S.getReturnValue()->getType().getQualifiers(),
574 if (!GroActiveFlag.
isValid()) {
582 Builder.CreateStore(Builder.getTrue(), GroActiveFlag);
590 const bool CanFallthrough = CGF.
Builder.GetInsertBlock();
592 if (
Stmt *OnFallthrough = S.getFallthroughHandler())
597 auto *NullPtr = llvm::ConstantPointerNull::get(
Builder.getInt8PtrTy());
599 unsigned NewAlign = TI.
getNewAlign() / TI.getCharWidth();
601 auto *EntryBB =
Builder.GetInsertBlock();
607 auto *CoroId =
Builder.CreateCall(
609 {Builder.getInt32(NewAlign), NullPtr, NullPtr, NullPtr});
612 assert(ShouldEmitLifetimeMarkers &&
613 "Must emit lifetime intrinsics for coroutines");
617 auto *CoroAlloc =
Builder.CreateCall(
620 Builder.CreateCondBr(CoroAlloc, AllocBB, InitBB);
624 auto *AllocOrInvokeContBB =
Builder.GetInsertBlock();
627 if (
auto *RetOnAllocFailure = S.getReturnStmtOnAllocFailure()) {
631 auto *NullPtr = llvm::ConstantPointerNull::get(
Int8PtrTy);
632 auto *Cond =
Builder.CreateICmpNE(AllocateCall, NullPtr);
633 Builder.CreateCondBr(Cond, InitBB, RetOnFailureBB);
647 Phi->addIncoming(NullPtr, EntryBB);
648 Phi->addIncoming(AllocateCall, AllocOrInvokeContBB);
649 auto *CoroBegin =
Builder.CreateCall(
653 GetReturnObjectManager GroManager(*
this, S);
654 GroManager.EmitGroAlloca();
659 ParamReferenceReplacerRAII ParamReplacer(LocalDeclMap);
660 CodeGenFunction::RunCleanupsScope ResumeScope(*
this);
666 (ParamMoves.size() == 0 || (ParamMoves.size() ==
FnArgs.size())) &&
667 "ParamMoves and FnArgs should be the same size for coroutine function");
668 if (ParamMoves.size() ==
FnArgs.size() && DI)
669 for (
const auto Pair : llvm::zip(
FnArgs, ParamMoves))
671 {std::get<0>(Pair), std::get<1>(Pair)});
676 for (
auto *PM : S.getParamMoves()) {
678 ParamReplacer.addCopy(cast<DeclStmt>(PM));
687 auto *PromiseAddrVoidPtr =
691 CoroId->setArgOperand(1, PromiseAddrVoidPtr);
694 GroManager.EmitGroInit();
699 CurCoro.
Data->ExceptionHandler = S.getExceptionHandler();
703 CurCoro.
Data->CurrentAwaitKind = AwaitKind::Normal;
711 BasicBlock *ContBB =
nullptr;
717 Builder.CreateCondBr(SkipBody, ContBB, BodyBB);
721 auto Loc = S.getBeginLoc();
739 const bool CanFallthrough =
Builder.GetInsertBlock();
740 const bool HasCoreturns =
CurCoro.
Data->CoreturnCount > 0;
741 if (CanFallthrough || HasCoreturns) {
757 if (
Stmt *Ret = S.getReturnStmt()) {
760 if (GroManager.DirectEmit)
761 cast<ReturnStmt>(Ret)->setRetValue(
nullptr);
766 CurFn->setPresplitCoroutine();
778 case llvm::Intrinsic::coro_frame: {
783 "has been used earlier in this function");
784 auto *NullPtr = llvm::ConstantPointerNull::get(
Builder.getInt8PtrTy());
787 case llvm::Intrinsic::coro_size: {
794 case llvm::Intrinsic::coro_align: {
804 case llvm::Intrinsic::coro_alloc:
805 case llvm::Intrinsic::coro_begin:
806 case llvm::Intrinsic::coro_free: {
812 " been used earlier in this function");
818 case llvm::Intrinsic::coro_suspend:
826 llvm::CallInst *
Call =
Builder.CreateCall(F, Args);
832 if (IID == llvm::Intrinsic::coro_id) {
835 else if (IID == llvm::Intrinsic::coro_begin) {
839 else if (IID == llvm::Intrinsic::coro_free) {
static LValueOrRValue emitSuspendExpression(CodeGenFunction &CGF, CGCoroData &Coro, CoroutineSuspendExpr const &S, AwaitKind Kind, AggValueSlot aggSlot, bool ignoreResult, bool forLValue)
static bool memberCallExpressionCanThrow(const Expr *E)
static SmallString< 32 > buildSuspendPrefixStr(CGCoroData &Coro, AwaitKind Kind)
static QualType getCoroutineSuspendExprReturnType(const ASTContext &Ctx, const CoroutineSuspendExpr *E)
static void createCoroData(CodeGenFunction &CGF, CodeGenFunction::CGCoroInfo &CurCoro, llvm::CallInst *CoroId, CallExpr const *CoroIdExpr=nullptr)
static void emitBodyAndFallthrough(CodeGenFunction &CGF, const CoroutineBodyStmt &S, Stmt *Body)
static SmallVector< llvm::OperandBundleDef, 1 > getBundlesForCoroEnd(CodeGenFunction &CGF)
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
bool hasSameType(QualType T1, QualType T2) const
Determine whether the given types T1 and T2 are equivalent.
CanQualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
const TargetInfo & getTargetInfo() const
CXXCatchStmt - This represents a C++ catch block.
CXXTryStmt - A C++ try block, including all handlers.
static CXXTryStmt * Create(const ASTContext &C, SourceLocation tryLoc, CompoundStmt *tryBlock, ArrayRef< Stmt * > handlers)
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
SourceLocation getBeginLoc() const LLVM_READONLY
static CharUnits One()
One - Construct a CharUnits quantity of one.
Represents a 'co_await' expression.
llvm::Value * getPointer() const
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored.
llvm::StoreInst * CreateFlagStore(bool Value, llvm::Value *Addr)
Emit a store to an i1 flag variable.
llvm::LoadInst * CreateFlagLoad(llvm::Value *Addr, const llvm::Twine &Name="")
Emit a load from an i1 flag variable.
This class gathers all debug information during compilation and is responsible for emitting to llvm g...
ParamDecl2StmtTy & getCoroutineParameterMappings()
static OpaqueValueMappingData bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const Expr *e)
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target)
The given basic block lies in the current EH scope, but may be a target of a potentially scope-crossi...
LValue EmitCoawaitLValue(const CoawaitExpr *E)
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
llvm::BasicBlock * getEHResumeBlock(bool isCleanup)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
llvm::AllocaInst * CreateTempAlloca(llvm::Type *Ty, const Twine &Name="tmp", llvm::Value *ArraySize=nullptr)
CreateTempAlloca - This creates an alloca and inserts it into the entry block if ArraySize is nullptr...
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
void EmitAutoVarInit(const AutoVarEmission &emission)
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount, Stmt::Likelihood LH=Stmt::LH_None)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
LValue EmitCoyieldLValue(const CoyieldExpr *E)
RValue EmitCoroutineIntrinsic(const CallExpr *E, unsigned int IID)
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
EmitAnyExpr - Emit code to compute the specified expression which can have any type.
RValue EmitCoyieldExpr(const CoyieldExpr &E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
CGDebugInfo * getDebugInfo()
void EnterCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock=false)
void EmitCoroutineBody(const CoroutineBodyStmt &S)
ASTContext & getContext() const
void EmitBranchThroughCleanup(JumpDest Dest)
EmitBranchThroughCleanup - Emit a branch from the current insert block through the normal cleanup han...
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
void EmitAutoVarCleanups(const AutoVarEmission &emission)
llvm::SmallVector< const ParmVarDecl *, 4 > FnArgs
Save Parameter Decl for coroutine.
RValue EmitCoawaitExpr(const CoawaitExpr &E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
void EmitCoreturnStmt(const CoreturnStmt &S)
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
void ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock=false)
llvm::Instruction * CurrentFuncletPad
void EmitStmt(const Stmt *S, ArrayRef< const Attr * > Attrs=std::nullopt)
EmitStmt - Emit the code for the statement.
llvm::LLVMContext & getLLVMContext()
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
void Error(SourceLocation loc, StringRef error)
Emit a general error that something can't be done.
ASTContext & getContext() const
llvm::Function * getIntrinsic(unsigned IID, ArrayRef< llvm::Type * > Tys=std::nullopt)
Information for lazily generating a cleanup.
stable_iterator stable_begin() const
Create a stable reference to the top of the EH stack.
iterator find(stable_iterator save) const
Turn a stable reference to a scope depth into a unstable pointer to the EH stack.
LValue - This represents an lvalue references.
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
static RValue get(llvm::Value *V)
static CompoundStmt * Create(const ASTContext &C, ArrayRef< Stmt * > Stmts, FPOptionsOverride FPFeatures, SourceLocation LB, SourceLocation RB)
Represents a 'co_return' statement in the C++ Coroutines TS.
Represents the body of a coroutine.
Represents an expression that might suspend coroutine execution; either a co_await or co_yield expres...
Expr * getResumeExpr() const
Represents a 'co_yield' expression.
A reference to a declared variable, function, enum, etc.
DeclStmt - Adaptor class for mixing declarations with statements and expressions.
bool isSingleDecl() const
isSingleDecl - This method returns true if this DeclStmt refers to a single Decl.
const Decl * getSingleDecl() const
This represents one expression.
Represents difference between two FPOptions values.
Represents a prototype with parameter type info, e.g.
A (possibly-)qualified type.
StmtVisitor - This class implements a simple visitor for Stmt subclasses.
Stmt - This represents one statement.
unsigned getNewAlign() const
Return the largest alignment for which a suitably-sized allocation with '::operator new(size_t)' is g...
Represents a variable declaration or definition.
const Expr * getInit() const
@ EHCleanup
Denotes a cleanup that should run when a scope is exited using exceptional control flow (a throw stat...
bool Call(InterpState &S, CodePtr OpPC, const Function *Func)
@ C
Languages that the frontend can parse and compile.
bool isNoexceptExceptionSpec(ExceptionSpecificationType ESpecType)
llvm::Value * ResumeEHVar
CodeGenFunction::JumpDest FinalJD
CallExpr const * CoroIdExpr
CodeGenFunction::JumpDest CleanupJD
llvm::BasicBlock * SuspendBB
AwaitKind CurrentAwaitKind
llvm::CallInst * CoroBegin
llvm::CallInst * LastCoroFree
std::unique_ptr< CGCoroData > Data
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
llvm::PointerType * VoidPtrTy
llvm::IntegerType * SizeTy
llvm::PointerType * Int8PtrTy