31#include "llvm/ADT/SmallSet.h"
32#include "llvm/BinaryFormat/Dwarf.h"
33#include "llvm/Frontend/OpenMP/OMPConstants.h"
34#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DebugInfoMetadata.h"
37#include "llvm/IR/Instructions.h"
38#include "llvm/IR/Metadata.h"
39#include "llvm/Support/AtomicOrdering.h"
40#include "llvm/Support/Debug.h"
44using namespace llvm::omp;
46#define TTL_CODEGEN_TYPE "target-teams-loop-codegen"
62 if (DKind == OMPD_teams_loop)
64 if (
const auto *TTLD = dyn_cast<OMPTargetTeamsGenericLoopDirective>(&S);
65 TTLD && !TTLD->canBeParallelFor())
71 S.hasClausesOfKind<OMPReductionClause>() &&
73 !S.getSingleClause<OMPScheduleClause>() &&
74 !S.getSingleClause<OMPOrderedClause>();
81 void emitPreInitStmt(CodeGenFunction &CGF,
const OMPExecutableDirective &S) {
82 for (
const auto *
C : S.clauses()) {
83 if (
const auto *CPI = OMPClauseWithPreInit::get(
C)) {
84 if (
const auto *PreInit =
85 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
86 for (
const auto *I : PreInit->decls()) {
87 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
90 CodeGenFunction::AutoVarEmission Emission =
99 CodeGenFunction::OMPPrivateScope InlinedShareds;
101 static bool isCapturedVar(CodeGenFunction &CGF,
const VarDecl *VD) {
110 CodeGenFunction &CGF,
const OMPExecutableDirective &S,
111 const std::optional<OpenMPDirectiveKind> CapturedRegion = std::nullopt,
112 const bool EmitPreInitStmt =
true)
114 InlinedShareds(CGF) {
116 emitPreInitStmt(CGF, S);
119 assert(S.hasAssociatedStmt() &&
120 "Expected associated statement for inlined directive.");
121 const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
123 if (
C.capturesVariable() ||
C.capturesVariableByCopy()) {
124 auto *VD =
C.getCapturedVar();
126 "Canonical decl must be captured.");
130 InlinedShareds.isGlobalVarCaptured(VD)),
135 (void)InlinedShareds.Privatize();
141class OMPParallelScope final :
public OMPLexicalScope {
142 bool EmitPreInitStmt(
const OMPExecutableDirective &S) {
150 OMPParallelScope(CodeGenFunction &CGF,
const OMPExecutableDirective &S)
151 : OMPLexicalScope(CGF, S, std::nullopt,
152 EmitPreInitStmt(S)) {}
157class OMPTeamsScope final :
public OMPLexicalScope {
158 bool EmitPreInitStmt(
const OMPExecutableDirective &S) {
165 OMPTeamsScope(CodeGenFunction &CGF,
const OMPExecutableDirective &S)
166 : OMPLexicalScope(CGF, S, std::nullopt,
167 EmitPreInitStmt(S)) {}
173 void emitPreInitStmt(CodeGenFunction &CGF,
const OMPLoopBasedDirective &S) {
174 const Stmt *PreInits;
175 CodeGenFunction::OMPMapVars PreCondVars;
176 SmallVector<const Decl *, 4> DoacrossOnlyDecls;
177 if (
auto *LD = dyn_cast<OMPLoopDirective>(&S)) {
179 (void)OMPLoopBasedDirective::doForAllLoops(
180 LD->getInnermostCapturedStmt()->getCapturedStmt(),
181 true, LD->getLoopsNumber(),
182 [&CGF](
unsigned Cnt,
const Stmt *CurStmt) {
183 if (const auto *CXXFor = dyn_cast<CXXForRangeStmt>(CurStmt)) {
184 if (const Stmt *Init = CXXFor->getInit())
186 CGF.EmitStmt(CXXFor->getRangeStmt());
187 CGF.EmitStmt(CXXFor->getBeginStmt());
188 CGF.EmitStmt(CXXFor->getEndStmt());
192 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
193 for (
const auto *E : LD->counters()) {
200 for (
const auto *
C : LD->getClausesOfKind<OMPPrivateClause>()) {
201 for (
const Expr *IRef :
C->varlist()) {
203 const auto *OrigVD = dyn_cast<VarDecl>(OrigDecl);
206 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
207 QualType OrigVDTy = OrigVD->getType().getNonReferenceType();
217 (void)PreCondVars.
apply(CGF);
222 if (
const auto *
C = LD->getSingleClause<OMPOrderedClause>())
223 (void)OMPLoopBasedDirective::doForAllLoops(
224 LD->getInnermostCapturedStmt()->getCapturedStmt(),
226 C->getLoopNumIterations().size(),
227 [&CGF, LD, &DoacrossOnlyDecls](
unsigned Cnt,
const Stmt *CurStmt) {
228 const auto *CXXFor = dyn_cast<CXXForRangeStmt>(CurStmt);
229 if (!CXXFor || Cnt < LD->getLoopsNumber())
231 for (const Stmt *Helper : std::initializer_list<const Stmt *>{
232 CXXFor->getInit(), CXXFor->getRangeStmt(),
233 CXXFor->getBeginStmt(), CXXFor->getEndStmt()}) {
237 if (
const auto *DS = dyn_cast<DeclStmt>(Helper))
238 llvm::append_range(DoacrossOnlyDecls, DS->decls());
242 PreInits = LD->getPreInits();
243 }
else if (
const auto *
Tile = dyn_cast<OMPTileDirective>(&S)) {
244 PreInits =
Tile->getPreInits();
245 }
else if (
const auto *Stripe = dyn_cast<OMPStripeDirective>(&S)) {
246 PreInits = Stripe->getPreInits();
247 }
else if (
const auto *Unroll = dyn_cast<OMPUnrollDirective>(&S)) {
248 PreInits = Unroll->getPreInits();
249 }
else if (
const auto *Reverse = dyn_cast<OMPReverseDirective>(&S)) {
250 PreInits = Reverse->getPreInits();
251 }
else if (
const auto *Split = dyn_cast<OMPSplitDirective>(&S)) {
252 PreInits =
Split->getPreInits();
253 }
else if (
const auto *Interchange =
254 dyn_cast<OMPInterchangeDirective>(&S)) {
255 PreInits = Interchange->getPreInits();
256 }
else if (
const auto *Flatten = dyn_cast<OMPFlattenDirective>(&S)) {
257 PreInits = Flatten->getPreInits();
259 llvm_unreachable(
"Unknown loop-based directive kind.");
261 doEmitPreinits(PreInits);
262 for (
const Decl *D : DoacrossOnlyDecls)
270 const Stmt *PreInits;
271 if (
const auto *Fuse = dyn_cast<OMPFuseDirective>(&S)) {
272 PreInits = Fuse->getPreInits();
275 "Unknown canonical loop sequence transform directive kind.");
277 doEmitPreinits(PreInits);
280 void doEmitPreinits(
const Stmt *PreInits) {
286 if (
auto *PreInitCompound = dyn_cast<CompoundStmt>(PreInits))
287 llvm::append_range(PreInitStmts, PreInitCompound->body());
289 PreInitStmts.push_back(PreInits);
291 for (
const Stmt *S : PreInitStmts) {
294 if (
auto *PreInitDecl = dyn_cast<DeclStmt>(S)) {
295 for (
Decl *I : PreInitDecl->decls())
307 emitPreInitStmt(CGF, S);
312 emitPreInitStmt(CGF, S);
317 CodeGenFunction::OMPPrivateScope InlinedShareds;
319 static bool isCapturedVar(CodeGenFunction &CGF,
const VarDecl *VD) {
327 OMPSimdLexicalScope(CodeGenFunction &CGF,
const OMPExecutableDirective &S)
329 InlinedShareds(CGF) {
330 for (
const auto *
C : S.clauses()) {
331 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
332 if (const auto *PreInit =
333 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
334 for (const auto *I : PreInit->decls()) {
335 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
336 CGF.EmitVarDecl(cast<VarDecl>(*I));
338 CodeGenFunction::AutoVarEmission Emission =
339 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
340 CGF.EmitAutoVarCleanups(Emission);
344 }
else if (
const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(
C)) {
345 for (
const Expr *E : UDP->varlist()) {
347 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
350 }
else if (
const auto *UDP = dyn_cast<OMPUseDeviceAddrClause>(
C)) {
351 for (
const Expr *E : UDP->varlist()) {
353 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
360 if (
const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
361 if (
const Expr *E = TG->getReductionRef())
366 llvm::DenseSet<CanonicalDeclPtr<const Decl>> CopyArrayTemps;
367 for (
const auto *
C : S.getClausesOfKind<OMPReductionClause>()) {
368 if (C->getModifier() != OMPC_REDUCTION_inscan)
370 for (const Expr *E : C->copy_array_temps())
371 CopyArrayTemps.insert(cast<DeclRefExpr>(E)->getDecl());
373 const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
376 if (
C.capturesVariable() ||
C.capturesVariableByCopy()) {
377 auto *VD =
C.getCapturedVar();
378 if (CopyArrayTemps.contains(VD))
381 "Canonical decl must be captured.");
382 DeclRefExpr DRE(CGF.
getContext(),
const_cast<VarDecl *
>(VD),
383 isCapturedVar(CGF, VD) ||
385 InlinedShareds.isGlobalVarCaptured(VD)),
393 (void)InlinedShareds.Privatize();
405 if (Kind == OMPD_teams_loop)
406 return OMPD_teams_distribute;
407 if (Kind == OMPD_target_loop)
408 return OMPD_target_simd;
409 if (Kind == OMPD_target_teams_loop)
411 ? OMPD_target_teams_distribute_parallel_for
412 : OMPD_target_teams_distribute;
414 if (Kind != OMPD_loop)
419 BindKind =
C->getBindKind();
422 case OMPC_BIND_parallel:
424 case OMPC_BIND_teams:
425 return OMPD_distribute;
426 case OMPC_BIND_thread:
453 if (
const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
454 if (
const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
455 OrigVD = OrigVD->getCanonicalDecl();
461 OrigDRE->getType(),
VK_LValue, OrigDRE->getExprLoc());
464 if (
const auto *OrigBD = dyn_cast<BindingDecl>(OrigDRE->getDecl())) {
471 IsCaptured, OrigDRE->getType(),
VK_LValue,
472 OrigDRE->getExprLoc());
481 llvm::Value *Size =
nullptr;
482 auto SizeInChars =
C.getTypeSizeInChars(Ty);
483 if (SizeInChars.isZero()) {
491 SizeInChars =
C.getTypeSizeInChars(Ty);
492 if (SizeInChars.isZero())
493 return llvm::ConstantInt::get(
SizeTy, 0);
494 return Builder.CreateNUWMul(Size,
CGM.getSize(SizeInChars));
496 return CGM.getSize(SizeInChars);
506 I != E; ++I, ++CurField, ++CurCap) {
507 if (CurField->hasCapturedVLAType()) {
510 CapturedVars.push_back(Val);
511 }
else if (CurCap->capturesThis()) {
512 CapturedVars.push_back(CXXThisValue);
513 }
else if (CurCap->capturesVariableByCopy()) {
518 if (!CurField->getType()->isAnyPointerType()) {
522 Twine(CurCap->getCapturedVar()->getName(),
".casted"));
538 CapturedVars.push_back(CV);
540 assert(CurCap->capturesVariable() &&
"Expected capture by reference.");
543 llvm::Type *ArgTy =
ConvertType(CurField->getType());
544 if (
Addr->getType() != ArgTy)
546 CapturedVars.push_back(
Addr);
566 if (
T->isLValueReferenceType())
567 return C.getLValueReferenceType(
570 if (
T->isPointerType())
572 if (
const ArrayType *A =
T->getAsArrayTypeUnsafe()) {
573 if (
const auto *VLA = dyn_cast<VariableArrayType>(A))
575 if (!A->isVariablyModifiedType())
576 return C.getCanonicalType(
T);
578 return C.getCanonicalParamType(
T);
583struct FunctionOptions {
585 const CapturedStmt *S =
nullptr;
588 const bool UIntPtrCastRequired =
true;
591 const bool RegisterCastedArgsOnly =
false;
593 const StringRef FunctionName;
596 const bool IsDeviceKernel =
false;
597 explicit FunctionOptions(
const CapturedStmt *S,
bool UIntPtrCastRequired,
598 bool RegisterCastedArgsOnly, StringRef FunctionName,
599 SourceLocation Loc,
bool IsDeviceKernel)
600 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
601 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
602 FunctionName(FunctionName), Loc(Loc), IsDeviceKernel(IsDeviceKernel) {}
608 llvm::MapVector<
const Decl *, std::pair<const VarDecl *, Address>>
610 llvm::DenseMap<
const Decl *, std::pair<const Expr *, llvm::Value *>>
612 llvm::Value *&CXXThisValue,
const FunctionOptions &FO) {
615 assert(CD->
hasBody() &&
"missing CapturedDecl body");
617 CXXThisValue =
nullptr;
629 if (!FO.UIntPtrCastRequired) {
649 if (FO.UIntPtrCastRequired &&
650 ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
651 I->capturesVariableArrayType()))
654 if (I->capturesVariable() || I->capturesVariableByCopy()) {
655 CapVar = I->getCapturedVar();
657 }
else if (I->capturesThis()) {
660 assert(I->capturesVariableArrayType());
663 if (ArgType->isVariablyModifiedType())
670 }
else if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
672 Ctx, DebugFunctionDecl,
673 CapVar ? CapVar->
getBeginLoc() : FD->getBeginLoc(),
674 CapVar ? CapVar->
getLocation() : FD->getLocation(), II, ArgType,
680 Args.emplace_back(Arg);
682 TargetArgs.emplace_back(
683 FO.UIntPtrCastRequired
704 llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
710 if (CGM.
getTriple().isSPIRV() && !FO.IsDeviceKernel)
711 F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
714 F->setDoesNotThrow();
718 F->removeFnAttr(llvm::Attribute::NoInline);
719 F->addFnAttr(llvm::Attribute::AlwaysInline);
722 F->addFnAttr(
"sample-profile-suffix-elision-policy",
"selected");
726 FO.UIntPtrCastRequired ? FO.Loc : FO.S->
getBeginLoc(),
727 FO.UIntPtrCastRequired ? FO.Loc
734 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
742 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
743 const VarDecl *CurVD = I->getCapturedVar();
744 if (!FO.RegisterCastedArgsOnly)
745 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
753 if (FD->hasCapturedVLAType()) {
754 if (FO.UIntPtrCastRequired) {
757 Args[Cnt]->getName(), ArgLVal),
762 VLASizes.try_emplace(Args[Cnt], VAT->
getSizeExpr(), ExprArg);
763 }
else if (I->capturesVariable()) {
764 const VarDecl *Var = I->getCapturedVar();
766 Address ArgAddr = ArgLVal.getAddress();
767 if (ArgLVal.getType()->isLValueReferenceType()) {
770 assert(ArgLVal.getType()->isPointerType());
772 ArgAddr, ArgLVal.getType()->castAs<
PointerType>());
774 if (!FO.RegisterCastedArgsOnly) {
778 }
else if (I->capturesVariableByCopy()) {
779 assert(!FD->getType()->isAnyPointerType() &&
780 "Not expecting a captured pointer.");
781 const VarDecl *Var = I->getCapturedVar();
782 LocalAddrs.insert({Args[Cnt],
783 {Var, FO.UIntPtrCastRequired
785 CGF, I->getLocation(), FD->getType(),
786 Args[Cnt]->getName(), ArgLVal)
787 : ArgLVal.getAddress()}});
790 assert(I->capturesThis());
792 LocalAddrs.insert({Args[Cnt], {
nullptr, ArgLVal.getAddress()}});
803 llvm::MapVector<
const Decl *, std::pair<const VarDecl *, Address>>
805 llvm::DenseMap<
const Decl *, std::pair<const Expr *, llvm::Value *>>
807 llvm::Value *&CXXThisValue, llvm::Value *&ContextV,
const CapturedStmt &CS,
812 CXXThisValue =
nullptr;
822 llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
826 F->setDoesNotThrow();
833 llvm::Type *PtrTy = CGF.
Builder.getPtrTy();
834 llvm::Align PtrAlign = CGM.
getDataLayout().getPointerABIAlignment(0);
837 for (
auto [FD,
C, FieldIdx] :
840 llvm::Value *SlotPtr =
841 CGF.
Builder.CreateConstInBoundsGEP1_32(PtrTy, ContextV, FieldIdx);
847 if (
C.capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
848 const VarDecl *CurVD =
C.getCapturedVar();
849 Slot->setName(CurVD->
getName());
850 Address SlotAddr(Slot, PtrTy, SlotAlign);
851 LocalAddrs.insert({FD, {CurVD, SlotAddr}});
852 }
else if (FD->hasCapturedVLAType()) {
859 VLASizes.try_emplace(FD, VAT->
getSizeExpr(), ExprArg);
860 }
else if (
C.capturesVariable()) {
861 const VarDecl *Var =
C.getCapturedVar();
865 Slot->setName(Var->
getName() +
".addr");
866 Address SlotAddr(Slot, PtrTy, SlotAlign);
867 LocalAddrs.insert({FD, {Var, SlotAddr}});
870 PtrTy, Slot, PtrAlign, Var->
getName());
871 LocalAddrs.insert({FD,
875 }
else if (
C.capturesVariableByCopy()) {
876 assert(!FD->getType()->isAnyPointerType() &&
877 "Not expecting a captured pointer.");
878 const VarDecl *Var =
C.getCapturedVar();
893 LocalAddrs.insert({FD, {Var, CopyAddr}});
895 assert(
C.capturesThis() &&
"Default case expected to be CXX 'this'");
898 Address SlotAddr(Slot, PtrTy, SlotAlign);
899 LocalAddrs.insert({FD, {
nullptr, SlotAddr}});
911 "CapturedStmtInfo should be set when generating the captured function");
914 bool NeedWrapperFunction =
917 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs,
919 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes,
922 llvm::raw_svector_ostream Out(Buffer);
925 bool IsDeviceKernel =
CGM.getOpenMPRuntime().isGPU() &&
927 D.getCapturedStmt(OMPD_target) == &S;
928 CodeGenFunction WrapperCGF(
CGM,
true);
929 llvm::Function *WrapperF =
nullptr;
930 if (NeedWrapperFunction) {
933 FunctionOptions WrapperFO(&S,
true,
940 WrapperCGF.CXXThisValue, WrapperFO);
943 FunctionOptions FO(&S, !NeedWrapperFunction,
false,
944 Out.str(), Loc, !NeedWrapperFunction && IsDeviceKernel);
946 *
this, WrapperArgs, WrapperLocalAddrs, WrapperVLASizes, CXXThisValue, FO);
948 for (
const auto &LocalAddrPair : WrapperLocalAddrs) {
949 if (LocalAddrPair.second.first) {
950 LocalScope.addPrivate(LocalAddrPair.second.first,
951 LocalAddrPair.second.second);
954 (void)LocalScope.Privatize();
955 for (
const auto &VLASizePair : WrapperVLASizes)
956 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
959 LocalScope.ForceCleanup();
961 if (!NeedWrapperFunction)
965 WrapperF->removeFromParent();
966 F->getParent()->getFunctionList().insertAfter(F->getIterator(), WrapperF);
969 auto *PI = F->arg_begin();
970 for (
const auto *Arg : Args) {
972 auto I = LocalAddrs.find(Arg);
973 if (I != LocalAddrs.end()) {
976 I->second.first ? I->second.first->getType() : Arg->getType(),
982 auto EI = VLASizes.find(Arg);
983 if (EI != VLASizes.end()) {
995 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, Loc, F, CallArgs);
1005 "CapturedStmtInfo should be set when generating the captured function");
1009 bool NeedWrapperFunction =
1012 CodeGenFunction WrapperCGF(
CGM,
true);
1013 llvm::Function *WrapperF =
nullptr;
1014 llvm::Value *WrapperContextV =
nullptr;
1015 if (NeedWrapperFunction) {
1018 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
1020 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
1023 WrapperCGF, WrapperArgs, WrapperLocalAddrs, WrapperVLASizes,
1024 WrapperCGF.CXXThisValue, WrapperContextV, S, Loc, FunctionName);
1028 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
1029 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
1032 if (NeedWrapperFunction) {
1034 llvm::raw_svector_ostream Out(Buffer);
1035 Out << FunctionName <<
"_debug__";
1037 FunctionOptions FO(&S,
false,
1038 false, Out.str(), Loc,
1043 llvm::Value *ContextV =
nullptr;
1045 CXXThisValue, ContextV, S, Loc,
1054 llvm::Align PtrAlign =
CGM.getDataLayout().getPointerABIAlignment(0);
1055 llvm::Value *SlotPtr =
Builder.CreateConstInBoundsGEP1_32(
1056 Builder.getPtrTy(), ContextV, FieldIdx,
1057 Twine(Param->getName()) +
".addr");
1058 llvm::Value *ParamAddr =
1059 Builder.CreateAlignedLoad(
Builder.getPtrTy(), SlotPtr, PtrAlign);
1060 llvm::Value *ParamVal =
Builder.CreateAlignedLoad(
1061 Builder.getPtrTy(), ParamAddr, PtrAlign, Param->getName());
1064 Builder.CreateStore(ParamVal, ParamLocalAddr);
1065 LocalAddrs.insert({Param, {Param, ParamLocalAddr}});
1071 for (
const auto &LocalAddrPair : LocalAddrs) {
1072 if (LocalAddrPair.second.first)
1073 LocalScope.addPrivate(LocalAddrPair.second.first,
1074 LocalAddrPair.second.second);
1076 (void)LocalScope.Privatize();
1077 for (
const auto &VLASizePair : VLASizes)
1078 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
1079 PGO->assignRegionCounters(
GlobalDecl(CD), F);
1081 (void)LocalScope.ForceCleanup();
1084 if (!NeedWrapperFunction)
1088 WrapperF->removeFromParent();
1089 F->getParent()->getFunctionList().insertAfter(F->getIterator(), WrapperF);
1091 llvm::Align PtrAlign =
CGM.getDataLayout().getPointerABIAlignment(0);
1094 "Expected context param at position 0 for target regions");
1095 assert(RD->
getNumFields() + 1 == F->arg_size() &&
"Argument count mismatch");
1097 for (
auto [FD, InnerParam, SlotIdx] : llvm::zip(
1099 llvm::Value *SlotPtr = WrapperCGF.
Builder.CreateConstInBoundsGEP1_32(
1100 WrapperCGF.
Builder.getPtrTy(), WrapperContextV, SlotIdx);
1102 WrapperCGF.
Builder.getPtrTy(), SlotPtr, PtrAlign);
1104 InnerParam.getType(), Slot, PtrAlign, InnerParam.getName());
1105 CallArgs.push_back(Val);
1110 auto InnerParam = F->arg_begin() + SlotIdx;
1111 llvm::Value *SlotPtr = WrapperCGF.
Builder.CreateConstInBoundsGEP1_32(
1112 WrapperCGF.
Builder.getPtrTy(), WrapperContextV, SlotIdx);
1114 WrapperCGF.
Builder.getPtrTy(), SlotPtr, PtrAlign);
1116 InnerParam->getType(), Slot, PtrAlign, InnerParam->getName());
1117 CallArgs.push_back(Val);
1119 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, Loc, F, CallArgs);
1135 llvm::Value *NumElements =
emitArrayLength(ArrayTy, ElementTy, DestAddr);
1142 DestBegin, NumElements);
1147 llvm::Value *IsEmpty =
1148 Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arraycpy.isempty");
1149 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
1152 llvm::BasicBlock *EntryBB =
Builder.GetInsertBlock();
1157 llvm::PHINode *SrcElementPHI =
1158 Builder.CreatePHI(SrcBegin->getType(), 2,
"omp.arraycpy.srcElementPast");
1159 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
1164 llvm::PHINode *DestElementPHI =
Builder.CreatePHI(
1165 DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
1166 DestElementPHI->addIncoming(DestBegin, EntryBB);
1172 CopyGen(DestElementCurrent, SrcElementCurrent);
1175 llvm::Value *DestElementNext =
1177 1,
"omp.arraycpy.dest.element");
1178 llvm::Value *SrcElementNext =
1180 1,
"omp.arraycpy.src.element");
1183 Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
1184 Builder.CreateCondBr(Done, DoneBB, BodyBB);
1185 DestElementPHI->addIncoming(DestElementNext,
Builder.GetInsertBlock());
1186 SrcElementPHI->addIncoming(SrcElementNext,
Builder.GetInsertBlock());
1196 const auto *BO = dyn_cast<BinaryOperator>(
Copy);
1197 if (BO && BO->getOpcode() == BO_Assign) {
1206 DestAddr, SrcAddr, OriginalType,
1234 bool DeviceConstTarget =
getLangOpts().OpenMPIsTargetDevice &&
1236 bool FirstprivateIsLastprivate =
false;
1237 llvm::SmallDenseMap<const Decl *, OpenMPLastprivateModifier> Lastprivates;
1238 for (
const auto *
C : D.getClausesOfKind<OMPLastprivateClause>()) {
1239 for (
const auto *D :
C->varlist()) {
1244 llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
1249 bool MustEmitFirstprivateCopy =
1250 CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
1251 for (
const auto *
C : D.getClausesOfKind<OMPFirstprivateClause>()) {
1252 const auto *IRef =
C->varlist_begin();
1253 const auto *InitsRef =
C->inits().begin();
1254 for (
const Expr *IInit :
C->private_copies()) {
1258 if (
const auto *BD = dyn_cast<BindingDecl>(OrigDecl)) {
1260 bool ThisFirstprivateIsLastprivate =
1261 Lastprivates.count(BD->getCanonicalDecl()) > 0;
1265 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate) {
1267 if (!FD->getType()->isReferenceType()) {
1270 assert(IsRegistered &&
1271 "firstprivate var already registered as firstprivate");
1279 const auto *VDInit =
1286 if (
Type->isArrayType()) {
1301 RunCleanupsScope InitScope(*this);
1303 setAddrOfLocalVar(VDInit, SrcElement);
1304 EmitAnyExprToMem(Init, DestElement,
1305 Init->getType().getQualifiers(),
1307 LocalDeclMap.erase(VDInit);
1316 setAddrOfLocalVar(VDInit, OriginalAddr);
1318 LocalDeclMap.erase(VDInit);
1321 IsRegistered = PrivateScope.
addPrivate(BD, VDAddr);
1324 assert(IsRegistered &&
1325 "firstprivate var already registered as firstprivate");
1327 FirstprivateIsLastprivate =
1328 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
1335 const VarDecl *OrigVD = dyn_cast<VarDecl>(OrigDecl);
1336 assert(OrigVD &&
"Expected VarDecl for non-BindingDecl firstprivate");
1337 bool ThisFirstprivateIsLastprivate =
1340 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
1342 (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())) {
1352 (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())) {
1358 FirstprivateIsLastprivate =
1359 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
1361 const auto *VDInit =
1380 assert(!CE &&
"Expected non-constant firstprivate.");
1387 if (
Type->isArrayType()) {
1403 RunCleanupsScope InitScope(*this);
1405 setAddrOfLocalVar(VDInit, SrcElement);
1406 EmitAnyExprToMem(Init, DestElement,
1407 Init->getType().getQualifiers(),
1409 LocalDeclMap.erase(VDInit);
1420 setAddrOfLocalVar(VDInit, OriginalAddr);
1422 LocalDeclMap.erase(VDInit);
1424 if (ThisFirstprivateIsLastprivate &&
1426 OMPC_LASTPRIVATE_conditional) {
1431 (*IRef)->getExprLoc());
1432 VDAddr =
CGM.getOpenMPRuntime().emitLastprivateConditionalInit(
1436 LocalDeclMap.erase(VD);
1437 setAddrOfLocalVar(VD, VDAddr);
1439 IsRegistered = PrivateScope.
addPrivate(OrigVD, VDAddr);
1441 assert(IsRegistered &&
1442 "firstprivate var already registered as private");
1450 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
1458 llvm::SmallDenseSet<const ValueDecl *> EmittedAsPrivate;
1459 for (
const auto *
C : D.getClausesOfKind<OMPPrivateClause>()) {
1460 auto IRef =
C->varlist_begin();
1461 for (
const Expr *IInit :
C->private_copies()) {
1464 if (EmittedAsPrivate.insert(
cast<ValueDecl>(OrigDecl->getCanonicalDecl()))
1469 assert(IsRegistered &&
"private var already registered as private");
1484 llvm::DenseSet<const VarDecl *> CopiedVars;
1485 llvm::BasicBlock *CopyBegin =
nullptr, *CopyEnd =
nullptr;
1487 auto IRef =
C->varlist_begin();
1488 auto ISrcRef =
C->source_exprs().begin();
1489 auto IDestRef =
C->destination_exprs().begin();
1490 for (
const Expr *AssignOp :
C->assignment_ops()) {
1499 getContext().getTargetInfo().isTLSSupported()) {
1501 "Copyin threadprivates should have been captured!");
1505 LocalDeclMap.erase(VD);
1509 :
CGM.GetAddrOfGlobal(VD),
1510 CGM.getTypes().ConvertTypeForMem(VD->
getType()),
1515 if (CopiedVars.size() == 1) {
1521 auto *MasterAddrInt =
Builder.CreatePtrToInt(
1523 auto *PrivateAddrInt =
Builder.CreatePtrToInt(
1526 Builder.CreateICmpNE(MasterAddrInt, PrivateAddrInt), CopyBegin,
1532 const auto *DestVD =
1553 bool HasAtLeastOneLastprivate =
false;
1555 llvm::DenseSet<const VarDecl *> SIMDLCVs;
1558 for (
const Expr *
C : LoopDirective->counters()) {
1563 llvm::SmallDenseSet<const ValueDecl *> AlreadyEmittedVars;
1564 for (
const auto *
C : D.getClausesOfKind<OMPLastprivateClause>()) {
1565 HasAtLeastOneLastprivate =
true;
1568 const auto *IRef =
C->varlist_begin();
1569 const auto *IDestRef =
C->destination_exprs().begin();
1570 for (
const Expr *IInit :
C->private_copies()) {
1575 if (
const auto *BD = dyn_cast<BindingDecl>(OrigDecl)) {
1576 if (AlreadyEmittedVars.insert(
cast<ValueDecl>(BD->getCanonicalDecl()))
1578 const auto *DestVD =
1590 bool IsRegistered = PrivateScope.
addPrivate(BD, VDAddr);
1591 assert(IsRegistered &&
1592 "lastprivate binding already registered as private");
1603 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
1604 const auto *DestVD =
1609 (*IRef)->getType(),
VK_LValue, (*IRef)->getExprLoc());
1614 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
1617 if (
C->getKind() == OMPC_LASTPRIVATE_conditional) {
1618 VDAddr =
CGM.getOpenMPRuntime().emitLastprivateConditionalInit(
1620 setAddrOfLocalVar(VD, VDAddr);
1626 bool IsRegistered = PrivateScope.
addPrivate(OrigVD, VDAddr);
1627 assert(IsRegistered &&
1628 "lastprivate var already registered as private");
1636 return HasAtLeastOneLastprivate;
1641 llvm::Value *IsLastIterCond) {
1650 llvm::BasicBlock *ThenBB =
nullptr;
1651 llvm::BasicBlock *DoneBB =
nullptr;
1652 if (IsLastIterCond) {
1656 llvm::any_of(D.getClausesOfKind<OMPLastprivateClause>(),
1657 [](
const OMPLastprivateClause *
C) {
1658 return C->getKind() == OMPC_LASTPRIVATE_conditional;
1660 CGM.getOpenMPRuntime().emitBarrierCall(*
this, D.getBeginLoc(),
1667 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1670 llvm::DenseSet<const ValueDecl *> AlreadyEmittedVars;
1671 llvm::SmallDenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1672 if (
const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1673 auto IC = LoopDirective->counters().begin();
1674 for (
const Expr *F : LoopDirective->finals()) {
1678 AlreadyEmittedVars.insert(D);
1680 LoopCountersAndUpdates[D] = F;
1684 for (
const auto *
C : D.getClausesOfKind<OMPLastprivateClause>()) {
1685 auto IRef =
C->varlist_begin();
1686 auto ISrcRef =
C->source_exprs().begin();
1687 auto IDestRef =
C->destination_exprs().begin();
1688 for (
const Expr *AssignOp :
C->assignment_ops()) {
1693 const VarDecl *PrivateVD =
nullptr;
1695 if ((BD = dyn_cast<BindingDecl>(PrivateDecl))) {
1700 if (LocalDeclMap.count(SrcVD)) {
1703 }
else if (OMPPrivatizedBindings.count(BD)) {
1712 const auto *CanonicalVD =
1718 .insert(PrivateVD ?
static_cast<const ValueDecl *
>(CanonicalVD)
1726 if (
const Expr *FinalExpr =
1727 LoopCountersAndUpdates.lookup(CanonicalVD))
1732 const auto *DestVD =
1740 auto It = OMPPrivatizedBindings.find(BD);
1741 assert(It != OMPPrivatizedBindings.end() &&
1742 "BindingDecl should be privatized");
1743 PrivateAddr = It->second;
1748 Builder.CreateLoad(PrivateAddr),
1749 CGM.getTypes().ConvertTypeForMem(RefTy->getPointeeType()),
1750 CGM.getNaturalTypeAlignment(RefTy->getPointeeType()));
1753 if (
C->getKind() == OMPC_LASTPRIVATE_conditional)
1754 CGM.getOpenMPRuntime().emitLastprivateConditionalFinalUpdate(
1756 (*IRef)->getExprLoc());
1759 EmitOMPCopy(
Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1765 if (
const Expr *PostUpdate =
C->getPostUpdateExpr())
1785 for (
const auto *
C : D.getClausesOfKind<OMPReductionClause>()) {
1786 if (ForInscan != (
C->getModifier() == OMPC_REDUCTION_inscan))
1788 Shareds.append(
C->varlist_begin(),
C->varlist_end());
1789 Privates.append(
C->privates().begin(),
C->privates().end());
1790 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
1791 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
1792 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
1793 if (
C->getModifier() == OMPC_REDUCTION_task) {
1794 Data.ReductionVars.append(
C->privates().begin(),
C->privates().end());
1795 Data.ReductionOrigs.append(
C->varlist_begin(),
C->varlist_end());
1796 Data.ReductionCopies.append(
C->privates().begin(),
C->privates().end());
1797 Data.ReductionOps.append(
C->reduction_ops().begin(),
1798 C->reduction_ops().end());
1799 TaskLHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
1800 TaskRHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
1805 auto *ILHS = LHSs.begin();
1806 auto *IRHS = RHSs.begin();
1808 for (
const Expr *IRef : Shareds) {
1815 [&Emission](CodeGenFunction &CGF) {
1816 CGF.EmitAutoVarInit(Emission);
1824 assert(IsRegistered &&
"private var already registered as private");
1832 if (isaOMPArraySectionExpr &&
Type->isVariablyModifiedType()) {
1837 }
else if ((isaOMPArraySectionExpr &&
Type->isScalarType()) ||
1855 PrivateScope.
addPrivate(LHSVD, OriginalAddr);
1866 if (!
Data.ReductionVars.empty()) {
1868 Data.IsReductionWithTaskMod =
true;
1870 llvm::Value *ReductionDesc =
CGM.getOpenMPRuntime().emitTaskReductionInit(
1871 *
this, D.getBeginLoc(), TaskLHSs, TaskRHSs,
Data);
1872 const Expr *TaskRedRef =
nullptr;
1883 case OMPD_parallel_for:
1886 case OMPD_parallel_master:
1890 case OMPD_parallel_sections:
1894 case OMPD_target_parallel:
1898 case OMPD_target_parallel_for:
1902 case OMPD_distribute_parallel_for:
1906 case OMPD_teams_distribute_parallel_for:
1908 .getTaskReductionRefExpr();
1910 case OMPD_target_teams_distribute_parallel_for:
1912 .getTaskReductionRefExpr();
1920 case OMPD_parallel_for_simd:
1922 case OMPD_taskyield:
1926 case OMPD_taskgroup:
1930 case OMPD_ordered_standalone:
1931 case OMPD_ordered_blockassoc:
1935 case OMPD_cancellation_point:
1937 case OMPD_target_data:
1938 case OMPD_target_enter_data:
1939 case OMPD_target_exit_data:
1941 case OMPD_taskloop_simd:
1942 case OMPD_master_taskloop:
1943 case OMPD_master_taskloop_simd:
1944 case OMPD_parallel_master_taskloop:
1945 case OMPD_parallel_master_taskloop_simd:
1946 case OMPD_distribute:
1947 case OMPD_target_update:
1948 case OMPD_distribute_parallel_for_simd:
1949 case OMPD_distribute_simd:
1950 case OMPD_target_parallel_for_simd:
1951 case OMPD_target_simd:
1952 case OMPD_teams_distribute:
1953 case OMPD_teams_distribute_simd:
1954 case OMPD_teams_distribute_parallel_for_simd:
1955 case OMPD_target_teams:
1956 case OMPD_target_teams_distribute:
1957 case OMPD_target_teams_distribute_parallel_for_simd:
1958 case OMPD_target_teams_distribute_simd:
1959 case OMPD_declare_target:
1960 case OMPD_end_declare_target:
1961 case OMPD_threadprivate:
1963 case OMPD_declare_reduction:
1964 case OMPD_declare_mapper:
1965 case OMPD_declare_simd:
1967 case OMPD_declare_variant:
1968 case OMPD_begin_declare_variant:
1969 case OMPD_end_declare_variant:
1972 llvm_unreachable(
"Unexpected directive with task reductions.");
1978 false, TaskRedRef->
getType());
1991 bool HasAtLeastOneReduction =
false;
1992 bool IsReductionWithTaskMod =
false;
1993 for (
const auto *
C : D.getClausesOfKind<OMPReductionClause>()) {
1995 if (
C->getModifier() == OMPC_REDUCTION_inscan)
1997 HasAtLeastOneReduction =
true;
1998 Privates.append(
C->privates().begin(),
C->privates().end());
1999 LHSExprs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
2000 RHSExprs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
2001 IsPrivateVarReduction.append(
C->private_var_reduction_flags().begin(),
2002 C->private_var_reduction_flags().end());
2003 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
2004 IsReductionWithTaskMod =
2005 IsReductionWithTaskMod ||
C->getModifier() == OMPC_REDUCTION_task;
2007 if (HasAtLeastOneReduction) {
2009 if (IsReductionWithTaskMod) {
2010 CGM.getOpenMPRuntime().emitTaskReductionFini(
2013 bool TeamsLoopCanBeParallel =
false;
2014 if (
auto *TTLD = dyn_cast<OMPTargetTeamsGenericLoopDirective>(&D))
2015 TeamsLoopCanBeParallel = TTLD->canBeParallelFor();
2016 bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
2018 TeamsLoopCanBeParallel || ReductionKind == OMPD_simd;
2019 bool SimpleReduction = ReductionKind == OMPD_simd;
2022 CGM.getOpenMPRuntime().emitReduction(
2023 *
this, D.getEndLoc(),
Privates, LHSExprs, RHSExprs, ReductionOps,
2024 {WithNowait, SimpleReduction, IsPrivateVarReduction, ReductionKind});
2033 llvm::BasicBlock *DoneBB =
nullptr;
2034 for (
const auto *
C : D.getClausesOfKind<OMPReductionClause>()) {
2035 if (
const Expr *PostUpdate =
C->getPostUpdateExpr()) {
2037 if (llvm::Value *Cond = CondGen(CGF)) {
2042 CGF.
Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2058 const OMPExecutableDirective &,
2059 llvm::SmallVectorImpl<llvm::Value *> &)>
2060 CodeGenBoundParametersTy;
2068 llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> PrivateDecls;
2069 for (
const auto *
C : S.getClausesOfKind<OMPReductionClause>()) {
2070 for (
const Expr *Ref :
C->varlist()) {
2071 if (!Ref->getType()->isScalarType())
2073 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
2077 const auto *VD = dyn_cast<VarDecl>(DRE->getDecl());
2080 PrivateDecls.insert(VD);
2084 for (
const auto *
C : S.getClausesOfKind<OMPLastprivateClause>()) {
2085 for (
const Expr *Ref :
C->varlist()) {
2086 if (!Ref->getType()->isScalarType())
2088 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
2093 if (
const auto *VD = dyn_cast<VarDecl>(DRE->getDecl())) {
2094 PrivateDecls.insert(VD);
2099 for (
const auto *
C : S.getClausesOfKind<OMPLinearClause>()) {
2100 for (
const Expr *Ref :
C->varlist()) {
2101 if (!Ref->getType()->isScalarType())
2103 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
2106 if (
const auto *VD = dyn_cast<VarDecl>(DRE->getDecl())) {
2107 PrivateDecls.insert(VD);
2116 for (
const auto *
C : S.getClausesOfKind<OMPFirstprivateClause>()) {
2117 for (
const Expr *Ref :
C->varlist()) {
2118 if (!Ref->getType()->isScalarType())
2120 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
2124 if (
const auto *VD = dyn_cast<VarDecl>(DRE->getDecl()))
2125 PrivateDecls.insert(VD);
2129 CGF, S, PrivateDecls);
2135 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
2136 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
2137 llvm::Value *NumThreads =
nullptr;
2146 llvm::Function *OutlinedFn =
2153 NumThreads = CGF.
EmitScalarExpr(NumThreadsClause->getNumThreads().front(),
2155 Modifier = NumThreadsClause->getPrescriptivenessModifier();
2156 if (
const auto *MessageClause = S.getSingleClause<OMPMessageClause>()) {
2157 Message = MessageClause->getMessageString();
2158 MessageLoc = MessageClause->getBeginLoc();
2160 if (
const auto *SeverityClause = S.getSingleClause<OMPSeverityClause>()) {
2161 Severity = SeverityClause->getSeverityKind();
2162 SeverityLoc = SeverityClause->getBeginLoc();
2165 CGF, NumThreads, NumThreadsClause->getBeginLoc(), Modifier, Severity,
2166 SeverityLoc, Message, MessageLoc);
2168 if (
const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
2171 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
2173 const Expr *IfCond =
nullptr;
2174 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
2175 if (
C->getNameModifier() == OMPD_unknown ||
2176 C->getNameModifier() == OMPD_parallel) {
2177 IfCond =
C->getCondition();
2182 OMPParallelScope
Scope(CGF, S);
2188 CodeGenBoundParameters(CGF, S, CapturedVars);
2191 CapturedVars, IfCond, NumThreads,
2192 Modifier, Severity, Message);
2197 if (!CVD->
hasAttr<OMPAllocateDeclAttr>())
2199 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
2201 return !((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||
2202 AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&
2203 !AA->getAllocator());
2218 CGF, S.getBeginLoc(), OMPD_unknown,
false,
2224 CodeGenFunction &CGF,
const VarDecl *VD) {
2226 auto &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
2239 Size = CGF.
Builder.CreateNUWAdd(
2241 Size = CGF.
Builder.CreateUDiv(Size,
CGM.getSize(Align));
2242 Size = CGF.
Builder.CreateNUWMul(Size,
CGM.getSize(Align));
2248 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
2249 assert(AA->getAllocator() &&
2250 "Expected allocator expression for non-default allocator.");
2254 if (Allocator->getType()->isIntegerTy())
2255 Allocator = CGF.
Builder.CreateIntToPtr(Allocator,
CGM.VoidPtrTy);
2256 else if (Allocator->getType()->isPointerTy())
2260 llvm::Value *
Addr = OMPBuilder.createOMPAlloc(
2263 llvm::CallInst *FreeCI =
2264 OMPBuilder.createOMPFree(CGF.
Builder,
Addr, Allocator);
2278 if (
CGM.getLangOpts().OpenMPUseTLS &&
2279 CGM.getContext().getTargetInfo().isTLSSupported())
2282 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
2287 llvm::ConstantInt *Size =
CGM.getSize(
CGM.GetTargetTypeStoreSize(VarTy));
2289 llvm::Twine CacheName = Twine(
CGM.getMangledName(VD)).concat(Suffix);
2291 llvm::CallInst *ThreadPrivateCacheCall =
2292 OMPBuilder.createCachedThreadPrivate(CGF.
Builder,
Data, Size, CacheName);
2300 llvm::raw_svector_ostream OS(Buffer);
2301 StringRef Sep = FirstSeparator;
2302 for (StringRef Part : Parts) {
2306 return OS.str().str();
2314 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
Builder,
false,
2315 "." + RegionName +
".after");
2322 if (
Builder.saveIP().isValid())
2331 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
Builder,
false,
2332 "." + RegionName +
".after");
2339 if (
Builder.saveIP().isValid())
2344 if (
CGM.getLangOpts().OpenMPIRBuilder) {
2345 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
2347 llvm::Value *IfCond =
nullptr;
2352 llvm::Value *NumThreads =
nullptr;
2354 NumThreads =
EmitScalarExpr(NumThreadsClause->getNumThreads().front(),
2357 ProcBindKind ProcBind = OMP_PROC_BIND_default;
2358 if (
const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>())
2359 ProcBind = ProcBindClause->getProcBindKind();
2361 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
2365 auto FiniCB = [
this](InsertPointTy IP) {
2367 return llvm::Error::success();
2374 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
2375 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
2386 auto BodyGenCB = [&,
this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
2389 *
this, ParallelRegionBodyStmt, AllocIP, CodeGenIP,
"parallel");
2390 return llvm::Error::success();
2395 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
2397 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
2398 cantFail(OMPBuilder.createParallel(
2399 Builder, AllocaIP, {}, BodyGenCB, PrivCB, FiniCB,
2400 IfCond, NumThreads, ProcBind, S.hasCancel()));
2414 CGF.
EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
2423 [](CodeGenFunction &) {
return nullptr; });
2436class OMPTransformDirectiveScopeRAII {
2437 OMPLoopScope *
Scope =
nullptr;
2441 OMPTransformDirectiveScopeRAII(
const OMPTransformDirectiveScopeRAII &) =
2443 OMPTransformDirectiveScopeRAII &
2444 operator=(
const OMPTransformDirectiveScopeRAII &) =
delete;
2448 if (
const auto *Dir = dyn_cast<OMPLoopBasedDirective>(S)) {
2449 Scope =
new OMPLoopScope(CGF, *Dir);
2452 }
else if (
const auto *Dir =
2453 dyn_cast<OMPCanonicalLoopSequenceTransformationDirective>(
2458 Scope =
new OMPLoopScope(CGF, *Dir);
2463 ~OMPTransformDirectiveScopeRAII() {
2474 int MaxLevel,
int Level = 0) {
2475 assert(Level < MaxLevel &&
"Too deep lookup during loop body codegen.");
2477 if (
const auto *CS = dyn_cast<CompoundStmt>(SimplifiedS)) {
2480 "LLVM IR generation of compound statement ('{}')");
2484 for (
const Stmt *CurStmt : CS->body())
2485 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level);
2492 if (SimplifiedS == OMPLoopBasedDirective::ignoreIntraTileHint(NextLoop)) {
2493 if (
auto *Dir = dyn_cast<OMPLoopTransformationDirective>(SimplifiedS))
2494 SimplifiedS = Dir->getTransformedStmt();
2495 if (
const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(SimplifiedS))
2496 SimplifiedS = CanonLoop->getLoopStmt();
2497 if (
const auto *For = dyn_cast<ForStmt>(SimplifiedS)) {
2501 "Expected canonical for loop or range-based for loop.");
2503 CGF.
EmitStmt(CXXFor->getLoopVarStmt());
2504 S = CXXFor->getBody();
2506 if (Level + 1 < MaxLevel) {
2507 NextLoop = OMPLoopDirective::tryToFindNextInnerLoop(
2509 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1);
2527 for (
const auto *
C : D.getClausesOfKind<OMPLinearClause>()) {
2528 for (
const Expr *UE :
C->updates())
2535 BreakContinueStack.push_back(BreakContinue(D,
LoopExit, Continue));
2549 bool IsInscanRegion = InscanScope.
Privatize();
2550 if (IsInscanRegion) {
2560 if (EKind != OMPD_simd && !
getLangOpts().OpenMPSimd)
2569 D.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers();
2572 OMPLoopBasedDirective::tryToFindNextInnerLoop(
2574 D.getLoopsNumber());
2582 BreakContinueStack.pop_back();
2593 std::unique_ptr<CodeGenFunction::CGCapturedStmtInfo> CSI =
2594 std::make_unique<CodeGenFunction::CGCapturedStmtInfo>(*S);
2598 return {F, CapStruct.getPointer(
ParentCGF)};
2602static llvm::CallInst *
2607 EffectiveArgs.reserve(Args.size() + 1);
2608 llvm::append_range(EffectiveArgs, Args);
2609 EffectiveArgs.push_back(Cap.second);
2614llvm::CanonicalLoopInfo *
2616 assert(Depth == 1 &&
"Nested loops with OpenMPIRBuilder not yet implemented");
2642 const Stmt *SyntacticalLoop = S->getLoopStmt();
2653 const Stmt *BodyStmt;
2654 if (
const auto *For = dyn_cast<ForStmt>(SyntacticalLoop)) {
2655 if (
const Stmt *InitStmt = For->getInit())
2657 BodyStmt = For->getBody();
2658 }
else if (
const auto *RangeFor =
2659 dyn_cast<CXXForRangeStmt>(SyntacticalLoop)) {
2660 if (
const DeclStmt *RangeStmt = RangeFor->getRangeStmt())
2662 if (
const DeclStmt *BeginStmt = RangeFor->getBeginStmt())
2664 if (
const DeclStmt *EndStmt = RangeFor->getEndStmt())
2666 if (
const DeclStmt *LoopVarStmt = RangeFor->getLoopVarStmt())
2668 BodyStmt = RangeFor->getBody();
2670 llvm_unreachable(
"Expected for-stmt or range-based for-stmt");
2673 const CapturedStmt *DistanceFunc = S->getDistanceFunc();
2686 llvm::Value *DistVal =
Builder.CreateLoad(CountAddr,
".count");
2689 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
2690 auto BodyGen = [&,
this](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP,
2691 llvm::Value *IndVar) {
2696 const DeclRefExpr *LoopVarRef = S->getLoopVarRef();
2704 return llvm::Error::success();
2707 llvm::CanonicalLoopInfo *
CL =
2708 cantFail(OMPBuilder.createCanonicalLoop(
Builder, BodyGen, DistVal));
2722 return S.getSourceRange();
2723 const Stmt *
Loop = S.getRawStmt()->IgnoreContainers();
2726 return Loop->getSourceRange();
2731 const Expr *IncExpr,
2732 const llvm::function_ref<
void(CodeGenFunction &)> BodyGen,
2733 const llvm::function_ref<
void(CodeGenFunction &)> PostIncGen) {
2743 const CapturedStmt *ICS = OMPED.getInnermostCapturedStmt();
2757 llvm::BasicBlock *ExitBlock =
LoopExit.getBlock();
2758 if (RequiresCleanup)
2765 if (ExitBlock !=
LoopExit.getBlock()) {
2775 BreakContinueStack.push_back(BreakContinue(S,
LoopExit, Continue));
2783 BreakContinueStack.pop_back();
2794 bool HasLinears =
false;
2795 for (
const auto *
C : D.getClausesOfKind<OMPLinearClause>()) {
2799 if (
const auto *Ref =
2822 if (
const auto *CS = cast_or_null<BinaryOperator>(
C->getCalcStep()))
2834 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
2837 llvm::BasicBlock *DoneBB =
nullptr;
2839 for (
const auto *
C : D.getClausesOfKind<OMPLinearClause>()) {
2840 auto IC =
C->varlist_begin();
2841 for (
const Expr *F :
C->finals()) {
2843 if (llvm::Value *Cond = CondGen(*
this)) {
2848 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2854 if (
const auto *BD = dyn_cast<BindingDecl>(OrigDecl)) {
2860 (*IC)->getType(),
VK_LValue, (*IC)->getExprLoc());
2869 if (
const Expr *PostUpdate =
C->getPostUpdateExpr())
2881 llvm::APInt ClauseAlignment(64, 0);
2882 if (
const Expr *AlignmentExpr = Clause->getAlignment()) {
2885 ClauseAlignment = AlignmentCI->getValue();
2887 for (
const Expr *E : Clause->varlist()) {
2888 llvm::APInt Alignment(ClauseAlignment);
2889 if (Alignment == 0) {
2896 E->getType()->getPointeeType()))
2899 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2900 "alignment is not power of 2");
2901 if (Alignment != 0) {
2922 LocalDeclMap.erase(PrivateVD);
2928 E->getType(),
VK_LValue, E->getExprLoc());
2936 for (
const auto *
C : S.getClausesOfKind<OMPOrderedClause>()) {
2937 if (!
C->getNumForLoops())
2939 for (
unsigned I = S.getLoopsNumber(), E =
C->getLoopNumIterations().size();
2945 if (DRE->refersToEnclosingVariableOrCapture()) {
2954 const Expr *Cond, llvm::BasicBlock *TrueBlock,
2955 llvm::BasicBlock *FalseBlock,
uint64_t TrueCount) {
2973 assert(!E->getType().getNonReferenceType()->isRecordType() &&
2974 "dependent counter must not be an iterator.");
2978 (void)PreCondVars.
setVarAddr(CGF, VD, CounterAddr);
2980 (void)PreCondVars.
apply(CGF);
2995 llvm::DenseSet<const VarDecl *> SIMDLCVs;
2999 for (
const Expr *
C : LoopDirective->counters()) {
3004 for (
const auto *
C : D.getClausesOfKind<OMPLinearClause>()) {
3005 auto CurPrivate =
C->privates().begin();
3006 for (
const Expr *E :
C->varlist()) {
3008 const auto *PrivateVD =
3010 bool IsSIMDLCV =
false;
3011 if (
const auto *VarD = dyn_cast<VarDecl>(VD))
3012 IsSIMDLCV = SIMDLCVs.count(VarD->getCanonicalDecl());
3017 bool IsRegistered = PrivateScope.
addPrivate(VD, PrivateAddr);
3018 assert(IsRegistered &&
"linear var already registered as private");
3106 if (
const auto *CS = dyn_cast<CapturedStmt>(S))
3124 if (HasOrderedDirective)
3132 const Stmt *AssociatedStmt = D.getAssociatedStmt();
3136 if (
C->getKind() == OMPC_ORDER_concurrent)
3139 if ((EKind == OMPD_simd ||
3141 llvm::any_of(D.getClausesOfKind<OMPReductionClause>(),
3142 [](
const OMPReductionClause *
C) {
3143 return C->getModifier() == OMPC_REDUCTION_inscan;
3151 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
3154 llvm::BasicBlock *DoneBB =
nullptr;
3160 const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
3162 OrigVD->hasGlobalStorage() || CED) {
3164 if (llvm::Value *Cond = CondGen(*
this)) {
3169 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
3212 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](
CodeGenFunction &CGF,
3226 const Expr *IfCond =
nullptr;
3228 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
3230 (
C->getNameModifier() == OMPD_unknown ||
3231 C->getNameModifier() == OMPD_simd)) {
3232 IfCond =
C->getCondition();
3248 OMPLoopScope PreInitScope(CGF, S);
3270 llvm::BasicBlock *ContBlock =
nullptr;
3322 emitOMPLoopBodyWithStopPoint(CGF, S,
3323 CodeGenFunction::JumpDest());
3329 if (HasLastprivateClause)
3358 if (
const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(S.getRawStmt())) {
3359 if (
const Stmt *SyntacticalLoop = CanonLoop->getLoopStmt()) {
3360 for (
const Stmt *SubStmt : SyntacticalLoop->
children()) {
3363 if (
const CompoundStmt *CS = dyn_cast<CompoundStmt>(SubStmt)) {
3378static llvm::MapVector<llvm::Value *, llvm::Value *>
3380 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars;
3382 llvm::APInt ClauseAlignment(64, 0);
3383 if (
const Expr *AlignmentExpr = Clause->getAlignment()) {
3386 ClauseAlignment = AlignmentCI->getValue();
3388 for (
const Expr *E : Clause->varlist()) {
3389 llvm::APInt Alignment(ClauseAlignment);
3390 if (Alignment == 0) {
3397 E->getType()->getPointeeType()))
3400 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
3401 "alignment is not power of 2");
3403 AlignedVars[PtrValue] = CGF.
Builder.getInt64(Alignment.getSExtValue());
3413 bool UseOMPIRBuilder =
3415 if (UseOMPIRBuilder) {
3419 if (UseOMPIRBuilder) {
3420 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars =
3423 const Stmt *Inner = S.getRawStmt();
3424 llvm::CanonicalLoopInfo *CLI =
3425 CGF.EmitOMPCollapsedCanonicalLoopNest(Inner, 1);
3427 llvm::OpenMPIRBuilder &OMPBuilder =
3430 llvm::ConstantInt *Simdlen =
nullptr;
3437 llvm::ConstantInt *Safelen =
nullptr;
3444 llvm::omp::OrderKind Order = llvm::omp::OrderKind::OMP_ORDER_unknown;
3446 if (
C->getKind() == OpenMPOrderClauseKind::OMPC_ORDER_concurrent) {
3447 Order = llvm::omp::OrderKind::OMP_ORDER_concurrent;
3452 OMPBuilder.applySimd(CLI, AlignedVars,
3453 nullptr, Order, Simdlen, Safelen);
3460 OMPLexicalScope
Scope(CGF, S, OMPD_unknown);
3475 OMPLexicalScope
Scope(CGF, S, OMPD_unknown);
3491 OMPTransformDirectiveScopeRAII
Scope(CGF, &S);
3520 OMPTransformDirectiveScopeRAII FlattenScope(*
this, &S);
3527 OMPTransformDirectiveScopeRAII FuseScope(*
this, &S);
3535 bool UseOMPIRBuilder =
CGM.getLangOpts().OpenMPIRBuilder;
3537 if (UseOMPIRBuilder) {
3539 const Stmt *Inner = S.getRawStmt();
3547 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
3550 llvm::CanonicalLoopInfo *UnrolledCLI =
nullptr;
3554 OMPBuilder.unrollLoopFull(DL, CLI);
3556 uint64_t Factor = 0;
3557 if (
Expr *FactorExpr = PartialClause->getFactor()) {
3559 FactorExpr->EvaluateKnownConstInt(
getContext()).getLimitedValue();
3560 assert(Factor >= 1 &&
"Only positive factors are valid");
3562 OMPBuilder.unrollLoopPartial(DL, CLI, Factor,
3563 NeedsUnrolledCLI ? &UnrolledCLI :
nullptr);
3565 OMPBuilder.unrollLoopHeuristic(DL, CLI);
3568 assert((!NeedsUnrolledCLI || UnrolledCLI) &&
3569 "NeedsUnrolledCLI implies UnrolledCLI to be set");
3586 if (
Expr *FactorExpr = PartialClause->getFactor()) {
3588 FactorExpr->EvaluateKnownConstInt(
getContext()).getLimitedValue();
3589 assert(Factor >= 1 &&
"Only positive factors are valid");
3597void CodeGenFunction::EmitOMPOuterLoop(
3600 const CodeGenFunction::OMPLoopArguments &LoopArgs,
3619 llvm::Value *BoolCondVal =
nullptr;
3620 if (!DynamicOrOrdered) {
3631 RT.
emitForNext(*
this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
3632 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
3637 llvm::BasicBlock *ExitBlock =
LoopExit.getBlock();
3642 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
3643 if (ExitBlock !=
LoopExit.getBlock()) {
3651 if (DynamicOrOrdered)
3656 BreakContinueStack.push_back(BreakContinue(S, LoopExit, Continue));
3661 [&S, IsMonotonic, EKind](CodeGenFunction &CGF, PrePostActionTy &) {
3666 if (
const auto *
C = S.getSingleClause<OMPOrderClause>())
3667 if (
C->getKind() == OMPC_ORDER_concurrent)
3673 [&S, &LoopArgs, LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered,
3674 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
3675 SourceLocation Loc = S.getBeginLoc();
3681 CGF.EmitOMPInnerLoop(
3683 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3684 CodeGenLoop(CGF, S, LoopExit);
3686 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
3687 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
3692 BreakContinueStack.pop_back();
3693 if (!DynamicOrOrdered) {
3706 auto &&CodeGen = [DynamicOrOrdered, &S, &LoopArgs](CodeGenFunction &CGF) {
3707 if (!DynamicOrOrdered)
3708 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
3711 OMPCancelStack.emitExit(*
this, EKind, CodeGen);
3714void CodeGenFunction::EmitOMPForOuterLoop(
3715 const OpenMPScheduleTy &ScheduleKind,
bool IsMonotonic,
3717 const OMPLoopArguments &LoopArgs,
3719 CGOpenMPRuntime &RT =
CGM.getOpenMPRuntime();
3725 LoopArgs.Chunk !=
nullptr)) &&
3726 "static non-chunked schedule does not need outer loop");
3784 if (DynamicOrOrdered) {
3785 const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
3786 CGDispatchBounds(*
this, S, LoopArgs.LB, LoopArgs.UB);
3787 llvm::Value *LBVal = DispatchBounds.first;
3788 llvm::Value *UBVal = DispatchBounds.second;
3789 CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
3792 IVSigned, Ordered, DipatchRTInputValues);
3794 CGOpenMPRuntime::StaticRTInput StaticInit(
3795 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
3796 LoopArgs.ST, LoopArgs.Chunk);
3802 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
3803 const unsigned IVSize,
3804 const bool IVSigned) {
3811 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
3812 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
3813 OuterLoopArgs.IncExpr = S.
getInc();
3814 OuterLoopArgs.Init = S.
getInit();
3815 OuterLoopArgs.Cond = S.
getCond();
3818 OuterLoopArgs.DKind = LoopArgs.DKind;
3819 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
3821 if (DynamicOrOrdered) {
3827 const unsigned IVSize,
const bool IVSigned) {}
3829void CodeGenFunction::EmitOMPDistributeOuterLoop(
3834 CGOpenMPRuntime &RT =
CGM.getOpenMPRuntime();
3846 CGOpenMPRuntime::StaticRTInput StaticInit(
3847 IVSize, IVSigned,
false, LoopArgs.IL, LoopArgs.LB,
3848 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
3862 OMPLoopArguments OuterLoopArgs;
3863 OuterLoopArgs.LB = LoopArgs.LB;
3864 OuterLoopArgs.UB = LoopArgs.UB;
3865 OuterLoopArgs.ST = LoopArgs.ST;
3866 OuterLoopArgs.IL = LoopArgs.IL;
3867 OuterLoopArgs.Chunk = LoopArgs.Chunk;
3871 OuterLoopArgs.IncExpr = IncExpr;
3884 OuterLoopArgs.DKind = OMPD_distribute;
3886 EmitOMPOuterLoop(
false,
false, S,
3887 LoopScope, OuterLoopArgs, CodeGenLoopContent,
3891static std::pair<LValue, LValue>
3934static std::pair<llvm::Value *, llvm::Value *>
3945 llvm::Value *LBVal =
3947 llvm::Value *UBVal =
3949 return {LBVal, UBVal};
3958 llvm::Value *LBCast = CGF.
Builder.CreateIntCast(
3960 CapturedVars.push_back(LBCast);
3964 llvm::Value *UBCast = CGF.
Builder.CreateIntCast(
3966 CapturedVars.push_back(UBCast);
3977 bool HasCancel =
false;
3979 if (
const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
3980 HasCancel = D->hasCancel();
3981 else if (
const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
3982 HasCancel = D->hasCancel();
3983 else if (
const auto *D =
3984 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
3985 HasCancel = D->hasCancel();
3995 CGInlinedWorksharingLoop,
4005 OMPLexicalScope
Scope(*
this, S, OMPD_parallel);
4006 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_distribute,
CodeGen);
4015 OMPLexicalScope
Scope(*
this, S, OMPD_parallel);
4016 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_distribute,
CodeGen);
4024 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
4025 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_simd,
CodeGen);
4035 llvm::Constant *
Addr;
4037 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4039 assert(Fn &&
Addr &&
"Target device function emission failed.");
4050struct ScheduleKindModifiersTy {
4057 : Kind(Kind), M1(M1), M2(M2) {}
4081 bool HasLastprivateClause;
4084 OMPLoopScope PreInitScope(*
this, S);
4089 llvm::BasicBlock *ContBlock =
nullptr;
4103 bool Ordered =
false;
4104 if (
const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
4105 if (OrderedClause->getNumForLoops())
4115 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*
this, S);
4116 LValue LB = Bounds.first;
4117 LValue UB = Bounds.second;
4131 CGM.getOpenMPRuntime().emitBarrierCall(
4132 *
this, S.getBeginLoc(), OMPD_unknown,
false,
4144 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*
this, S);
4147 const Expr *ChunkExpr =
nullptr;
4149 if (
const auto *
C = S.getSingleClause<OMPScheduleClause>()) {
4150 ScheduleKind.
Schedule =
C->getScheduleKind();
4151 ScheduleKind.
M1 =
C->getFirstScheduleModifier();
4152 ScheduleKind.
M2 =
C->getSecondScheduleModifier();
4153 ChunkExpr =
C->getChunkSize();
4156 CGM.getOpenMPRuntime().getDefaultScheduleAndChunk(
4157 *
this, S, ScheduleKind.
Schedule, ChunkExpr);
4159 bool HasChunkSizeOne =
false;
4160 llvm::Value *Chunk =
nullptr;
4168 llvm::APSInt EvaluatedChunk =
Result.Val.getInt();
4169 HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
4178 bool StaticChunkedOne =
4180 Chunk !=
nullptr) &&
4192 "fused distribute schedule requires a static chunk-one schedule");
4195 (ScheduleKind.
Schedule == OMPC_SCHEDULE_static &&
4196 !(ScheduleKind.
M1 == OMPC_SCHEDULE_MODIFIER_nonmonotonic ||
4197 ScheduleKind.
M2 == OMPC_SCHEDULE_MODIFIER_nonmonotonic)) ||
4198 ScheduleKind.
M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
4199 ScheduleKind.
M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
4201 Chunk !=
nullptr) ||
4202 StaticChunkedOne) &&
4212 if (
C->getKind() == OMPC_ORDER_concurrent)
4216 [IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk,
4225 IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
4226 UB.getAddress(), ST.getAddress(),
4227 StaticChunkedOne ? Chunk :
nullptr);
4229 CGF, S.getBeginLoc(), EKind, ScheduleKind, StaticInit);
4231 if (!StaticChunkedOne)
4250 StaticChunkedOne ? S.getCombinedParForInDistCond()
4252 StaticChunkedOne ? S.getDistInc() : S.getInc(),
4253 [&S,
LoopExit](CodeGenFunction &CGF) {
4254 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit);
4256 [](CodeGenFunction &) {});
4260 auto &&
CodeGen = [&S](CodeGenFunction &CGF) {
4264 OMPCancelStack.emitExit(*
this, EKind,
CodeGen);
4271 LoopArguments.DKind = OMPD_for;
4272 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
4273 LoopArguments, CGDispatchBounds);
4277 return CGF.
Builder.CreateIsNotNull(
4283 ? OMPD_parallel_for_simd
4287 *
this, S, [IL, &S](CodeGenFunction &CGF) {
4288 return CGF.
Builder.CreateIsNotNull(
4292 if (HasLastprivateClause)
4298 return CGF.
Builder.CreateIsNotNull(
4309 return HasLastprivateClause;
4315static std::pair<LValue, LValue>
4329static std::pair<llvm::Value *, llvm::Value *>
4333 const Expr *IVExpr = LS.getIterationVariable();
4335 llvm::Value *LBVal = CGF.
Builder.getIntN(IVSize, 0);
4337 return {LBVal, UBVal};
4349 llvm::function_ref<llvm::Value *(
CodeGenFunction &)> NumIteratorsGen) {
4350 llvm::Value *OMPScanNumIterations = CGF.
Builder.CreateIntCast(
4351 NumIteratorsGen(CGF), CGF.
SizeTy,
false);
4356 for (
const auto *
C : S.getClausesOfKind<OMPReductionClause>()) {
4357 assert(
C->getModifier() == OMPC_REDUCTION_inscan &&
4358 "Only inscan reductions are expected.");
4359 Shareds.append(
C->varlist_begin(),
C->varlist_end());
4360 Privates.append(
C->privates().begin(),
C->privates().end());
4361 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
4362 CopyArrayTemps.append(
C->copy_array_temps().begin(),
4363 C->copy_array_temps().end());
4371 auto *ITA = CopyArrayTemps.begin();
4376 if (PrivateVD->getType()->isVariablyModifiedType()) {
4401 llvm::function_ref<llvm::Value *(
CodeGenFunction &)> NumIteratorsGen) {
4402 llvm::Value *OMPScanNumIterations = CGF.
Builder.CreateIntCast(
4403 NumIteratorsGen(CGF), CGF.
SizeTy,
false);
4410 for (
const auto *
C : S.getClausesOfKind<OMPReductionClause>()) {
4411 assert(
C->getModifier() == OMPC_REDUCTION_inscan &&
4412 "Only inscan reductions are expected.");
4413 Shareds.append(
C->varlist_begin(),
C->varlist_end());
4414 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
4415 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
4416 Privates.append(
C->privates().begin(),
C->privates().end());
4417 CopyOps.append(
C->copy_ops().begin(),
C->copy_ops().end());
4418 CopyArrayElems.append(
C->copy_array_elems().begin(),
4419 C->copy_array_elems().end());
4423 llvm::Value *OMPLast = CGF.
Builder.CreateNSWSub(
4424 OMPScanNumIterations,
4425 llvm::ConstantInt::get(CGF.
SizeTy, 1,
false));
4426 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
4428 const Expr *OrigExpr = Shareds[I];
4429 const Expr *CopyArrayElem = CopyArrayElems[I];
4436 LValue SrcLVal = CGF.
EmitLValue(CopyArrayElem);
4438 PrivateExpr->
getType(), DestLVal.getAddress(), SrcLVal.getAddress(),
4468 llvm::Value *OMPScanNumIterations = CGF.
Builder.CreateIntCast(
4469 NumIteratorsGen(CGF), CGF.
SizeTy,
false);
4475 for (
const auto *
C : S.getClausesOfKind<OMPReductionClause>()) {
4476 assert(
C->getModifier() == OMPC_REDUCTION_inscan &&
4477 "Only inscan reductions are expected.");
4478 Privates.append(
C->privates().begin(),
C->privates().end());
4479 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
4480 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
4481 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
4482 CopyArrayElems.append(
C->copy_array_elems().begin(),
4483 C->copy_array_elems().end());
4498 auto &&
CodeGen = [&S, OMPScanNumIterations, &LHSs, &RHSs, &CopyArrayElems,
4505 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock();
4506 llvm::BasicBlock *LoopBB = CGF.createBasicBlock(
"omp.outer.log.scan.body");
4507 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
"omp.outer.log.scan.exit");
4509 CGF.CGM.getIntrinsic(llvm::Intrinsic::log2, CGF.DoubleTy);
4511 CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy);
4512 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg);
4513 F = CGF.CGM.getIntrinsic(llvm::Intrinsic::ceil, CGF.DoubleTy);
4514 LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal);
4515 LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy);
4516 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub(
4517 OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1));
4519 CGF.EmitBlock(LoopBB);
4520 auto *Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2);
4522 auto *Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
4523 Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB);
4524 Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB);
4527 llvm::BasicBlock *InnerLoopBB =
4528 CGF.createBasicBlock(
"omp.inner.log.scan.body");
4529 llvm::BasicBlock *InnerExitBB =
4530 CGF.createBasicBlock(
"omp.inner.log.scan.exit");
4531 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K);
4532 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
4533 CGF.EmitBlock(InnerLoopBB);
4534 auto *IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
4535 IVal->addIncoming(NMin1, LoopBB);
4538 auto *ILHS = LHSs.begin();
4539 auto *IRHS = RHSs.begin();
4540 for (
const Expr *CopyArrayElem : CopyArrayElems) {
4550 LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress();
4555 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K);
4561 RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress();
4568 CGF.CGM.getOpenMPRuntime().emitReduction(
4569 CGF, S.getEndLoc(),
Privates, LHSs, RHSs, ReductionOps,
4573 llvm::Value *NextIVal =
4574 CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1));
4575 IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock());
4576 CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K);
4577 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
4578 CGF.EmitBlock(InnerExitBB);
4580 CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1));
4581 Counter->addIncoming(
Next, CGF.Builder.GetInsertBlock());
4583 llvm::Value *NextPow2K =
4584 CGF.Builder.CreateShl(Pow2K, 1,
"",
true);
4585 Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock());
4586 llvm::Value *
Cmp = CGF.Builder.CreateICmpNE(
Next, LogVal);
4587 CGF.Builder.CreateCondBr(
Cmp, LoopBB, ExitBB);
4589 CGF.EmitBlock(ExitBB);
4595 CGF, S.getBeginLoc(), OMPD_unknown,
false,
4598 RegionCodeGenTy RCG(CodeGen);
4609 bool HasLastprivates;
4611 if (llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
4612 [](
const OMPReductionClause *
C) {
4613 return C->getModifier() == OMPC_REDUCTION_inscan;
4617 OMPLoopScope LoopScope(CGF, S);
4620 const auto &&FirstGen = [&S, HasCancel, EKind](
CodeGenFunction &CGF) {
4629 const auto &&SecondGen = [&S, HasCancel, EKind,
4647 return HasLastprivates;
4660 if (
auto *SC = dyn_cast<OMPScheduleClause>(
C)) {
4665 switch (SC->getScheduleKind()) {
4666 case OMPC_SCHEDULE_auto:
4667 case OMPC_SCHEDULE_dynamic:
4668 case OMPC_SCHEDULE_runtime:
4669 case OMPC_SCHEDULE_guided:
4670 case OMPC_SCHEDULE_static:
4683static llvm::omp::ScheduleKind
4685 switch (ScheduleClauseKind) {
4687 return llvm::omp::OMP_SCHEDULE_Default;
4688 case OMPC_SCHEDULE_auto:
4689 return llvm::omp::OMP_SCHEDULE_Auto;
4690 case OMPC_SCHEDULE_dynamic:
4691 return llvm::omp::OMP_SCHEDULE_Dynamic;
4692 case OMPC_SCHEDULE_guided:
4693 return llvm::omp::OMP_SCHEDULE_Guided;
4694 case OMPC_SCHEDULE_runtime:
4695 return llvm::omp::OMP_SCHEDULE_Runtime;
4696 case OMPC_SCHEDULE_static:
4697 return llvm::omp::OMP_SCHEDULE_Static;
4699 llvm_unreachable(
"Unhandled schedule kind");
4706 bool HasLastprivates =
false;
4709 auto &&
CodeGen = [&S, &
CGM, HasCancel, &HasLastprivates,
4712 if (UseOMPIRBuilder) {
4713 bool NeedsBarrier = !S.getSingleClause<OMPNowaitClause>();
4715 llvm::omp::ScheduleKind SchedKind = llvm::omp::OMP_SCHEDULE_Default;
4716 llvm::Value *ChunkSize =
nullptr;
4717 if (
auto *SchedClause = S.getSingleClause<OMPScheduleClause>()) {
4720 if (
const Expr *ChunkSizeExpr = SchedClause->getChunkSize())
4725 const Stmt *Inner = S.getRawStmt();
4726 llvm::CanonicalLoopInfo *CLI =
4729 llvm::OpenMPIRBuilder &OMPBuilder =
4731 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
4733 cantFail(OMPBuilder.applyWorkshareLoop(
4734 CGF.
Builder.getCurrentDebugLocation(), CLI, AllocaIP, NeedsBarrier,
4735 SchedKind, ChunkSize,
false,
4746 OMPLexicalScope
Scope(CGF, S, OMPD_unknown);
4751 if (!UseOMPIRBuilder) {
4753 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
4765 bool HasLastprivates =
false;
4766 auto &&
CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
4773 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
4774 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_simd,
CodeGen);
4778 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
4779 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.getBeginLoc(), OMPD_for);
4786 llvm::Value *
Init =
nullptr) {
4793void CodeGenFunction::EmitSections(
const OMPExecutableDirective &S) {
4794 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4795 const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
4796 bool HasLastprivates =
false;
4798 auto &&CodeGen = [&S, CapturedStmt, CS, EKind,
4799 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
4800 const ASTContext &
C = CGF.getContext();
4801 QualType KmpInt32Ty =
4802 C.getIntTypeForBitwidth(32, 1);
4805 CGF.Builder.getInt32(0));
4806 llvm::ConstantInt *GlobalUBVal = CS !=
nullptr
4807 ? CGF.Builder.getInt32(CS->size() - 1)
4808 : CGF.Builder.getInt32(0);
4812 CGF.Builder.getInt32(1));
4814 CGF.Builder.getInt32(0));
4817 OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty,
VK_LValue);
4818 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
4819 OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty,
VK_LValue);
4820 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
4824 S.getBeginLoc(), FPOptionsOverride());
4828 S.getBeginLoc(),
true, FPOptionsOverride());
4829 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
4841 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
".omp.sections.exit");
4842 llvm::SwitchInst *SwitchStmt =
4843 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.
getBeginLoc()),
4844 ExitBB, CS ==
nullptr ? 1 : CS->size());
4846 unsigned CaseNumber = 0;
4847 for (
const Stmt *SubStmt : CS->
children()) {
4848 auto CaseBB = CGF.createBasicBlock(
".omp.sections.case");
4849 CGF.EmitBlock(CaseBB);
4850 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
4851 CGF.EmitStmt(SubStmt);
4852 CGF.EmitBranch(ExitBB);
4856 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(
".omp.sections.case");
4857 CGF.EmitBlock(CaseBB);
4858 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
4859 CGF.EmitStmt(CapturedStmt);
4860 CGF.EmitBranch(ExitBB);
4862 CGF.EmitBlock(ExitBB,
true);
4865 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
4866 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
4870 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
4874 CGF.EmitOMPPrivateClause(S, LoopScope);
4875 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(CGF, S, IV);
4876 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4877 CGF.EmitOMPReductionClauseInit(S, LoopScope);
4878 (void)LoopScope.Privatize();
4880 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4883 OpenMPScheduleTy ScheduleKind;
4884 ScheduleKind.
Schedule = OMPC_SCHEDULE_static;
4885 CGOpenMPRuntime::StaticRTInput StaticInit(
4886 32,
true,
false, IL.getAddress(),
4887 LB.getAddress(), UB.getAddress(), ST.getAddress());
4888 CGF.CGM.getOpenMPRuntime().emitForStaticInit(CGF, S.
getBeginLoc(), EKind,
4889 ScheduleKind, StaticInit);
4891 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.
getBeginLoc());
4892 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
4893 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
4894 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
4896 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.
getBeginLoc()), IV);
4898 CGF.EmitOMPInnerLoop(S,
false, Cond, Inc, BodyGen,
4899 [](CodeGenFunction &) {});
4901 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
4902 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.
getEndLoc(),
4905 CGF.OMPCancelStack.emitExit(CGF, EKind, CodeGen);
4906 CGF.EmitOMPReductionClauseFinal(S, OMPD_parallel);
4909 return CGF.
Builder.CreateIsNotNull(
4914 if (HasLastprivates)
4921 bool HasCancel =
false;
4922 if (
auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
4923 HasCancel = OSD->hasCancel();
4924 else if (
auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
4925 HasCancel = OPSD->hasCancel();
4927 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_sections, CodeGen,
4932 if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
4950 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4955 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
4956 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_scope,
CodeGen);
4959 if (!S.getSingleClause<OMPNowaitClause>()) {
4960 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.getBeginLoc(), OMPD_scope);
4967 if (
CGM.getLangOpts().OpenMPIRBuilder) {
4968 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
4969 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4970 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4972 auto FiniCB = [](InsertPointTy IP) {
4975 return llvm::Error::success();
4978 const CapturedStmt *ICS = S.getInnermostCapturedStmt();
4979 const Stmt *
CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4984 auto SectionCB = [
this, SubStmt](
4985 InsertPointTy AllocIP, InsertPointTy CodeGenIP,
4988 CodeGenIP,
"section");
4989 return llvm::Error::success();
4991 SectionCBVector.push_back(SectionCB);
4995 [
this,
CapturedStmt](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
4999 return llvm::Error::success();
5001 SectionCBVector.push_back(SectionCB);
5008 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
5009 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
5019 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
5021 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
5022 cantFail(OMPBuilder.createSections(
5024 S.getSingleClause<OMPNowaitClause>()));
5031 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
5035 if (!S.getSingleClause<OMPNowaitClause>()) {
5036 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.getBeginLoc(),
5044 if (
CGM.getLangOpts().OpenMPIRBuilder) {
5045 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
5046 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5048 const Stmt *SectionRegionBodyStmt = S.getAssociatedStmt();
5049 auto FiniCB = [
this](InsertPointTy IP) {
5051 return llvm::Error::success();
5054 auto BodyGenCB = [SectionRegionBodyStmt,
5055 this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
5058 *
this, SectionRegionBodyStmt, AllocIP, CodeGenIP,
"section");
5059 return llvm::Error::success();
5064 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
5065 cantFail(OMPBuilder.createSection(
Builder, BodyGenCB, FiniCB));
5085 CopyprivateVars.append(
C->varlist_begin(),
C->varlist_end());
5086 DestExprs.append(
C->destination_exprs().begin(),
5087 C->destination_exprs().end());
5088 SrcExprs.append(
C->source_exprs().begin(),
C->source_exprs().end());
5089 AssignmentOps.append(
C->assignment_ops().begin(),
5090 C->assignment_ops().end());
5099 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
5104 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
5105 CGM.getOpenMPRuntime().emitSingleRegion(*
this,
CodeGen, S.getBeginLoc(),
5106 CopyprivateVars, DestExprs,
5107 SrcExprs, AssignmentOps);
5111 if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
5112 CGM.getOpenMPRuntime().emitBarrierCall(
5113 *
this, S.getBeginLoc(),
5114 S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
5129 if (
CGM.getLangOpts().OpenMPIRBuilder) {
5130 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
5131 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5133 const Stmt *MasterRegionBodyStmt = S.getAssociatedStmt();
5135 auto FiniCB = [
this](InsertPointTy IP) {
5137 return llvm::Error::success();
5140 auto BodyGenCB = [MasterRegionBodyStmt,
5141 this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
5144 *
this, MasterRegionBodyStmt, AllocIP, CodeGenIP,
"master");
5145 return llvm::Error::success();
5150 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
5151 cantFail(OMPBuilder.createMaster(
Builder, BodyGenCB, FiniCB));
5166 Expr *Filter =
nullptr;
5168 Filter = FilterClause->getThreadID();
5174 if (
CGM.getLangOpts().OpenMPIRBuilder) {
5175 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
5176 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5178 const Stmt *MaskedRegionBodyStmt = S.getAssociatedStmt();
5179 const Expr *Filter =
nullptr;
5181 Filter = FilterClause->getThreadID();
5182 llvm::Value *FilterVal = Filter
5184 : llvm::ConstantInt::get(
CGM.Int32Ty, 0);
5186 auto FiniCB = [
this](InsertPointTy IP) {
5188 return llvm::Error::success();
5191 auto BodyGenCB = [MaskedRegionBodyStmt,
5192 this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
5195 *
this, MaskedRegionBodyStmt, AllocIP, CodeGenIP,
"masked");
5196 return llvm::Error::success();
5201 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
5202 OMPBuilder.createMasked(
Builder, BodyGenCB, FiniCB, FilterVal));
5213 if (
CGM.getLangOpts().OpenMPIRBuilder) {
5214 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
5215 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5217 const Stmt *CriticalRegionBodyStmt = S.getAssociatedStmt();
5218 const Expr *Hint =
nullptr;
5219 if (
const auto *HintClause = S.getSingleClause<
OMPHintClause>())
5220 Hint = HintClause->getHint();
5225 llvm::Value *HintInst =
nullptr;
5230 auto FiniCB = [
this](InsertPointTy IP) {
5232 return llvm::Error::success();
5235 auto BodyGenCB = [CriticalRegionBodyStmt,
5236 this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
5239 *
this, CriticalRegionBodyStmt, AllocIP, CodeGenIP,
"critical");
5240 return llvm::Error::success();
5245 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
5246 cantFail(OMPBuilder.createCritical(
Builder, BodyGenCB, FiniCB,
5256 CGF.
EmitStmt(S.getAssociatedStmt());
5258 const Expr *Hint =
nullptr;
5259 if (
const auto *HintClause = S.getSingleClause<
OMPHintClause>())
5260 Hint = HintClause->getHint();
5263 CGM.getOpenMPRuntime().emitCriticalRegion(*
this,
5265 CodeGen, S.getBeginLoc(), Hint);
5278 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
5282 OMPLoopScope LoopScope(CGF, S);
5285 bool IsInscan = llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
5286 [](
const OMPReductionClause *
C) {
5287 return C->getModifier() == OMPC_REDUCTION_inscan;
5312 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
5316 OMPLoopScope LoopScope(CGF, S);
5319 bool IsInscan = llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
5320 [](
const OMPReductionClause *
C) {
5321 return C->getModifier() == OMPC_REDUCTION_inscan;
5357 [](CodeGenFunction &) {
return nullptr; });
5384 [](CodeGenFunction &) {
return nullptr; });
5397 CGF.EmitSections(S);
5411class CheckVarsEscapingUntiedTaskDeclContext final
5416 explicit CheckVarsEscapingUntiedTaskDeclContext() =
default;
5417 ~CheckVarsEscapingUntiedTaskDeclContext() =
default;
5418 void VisitDeclStmt(
const DeclStmt *S) {
5423 if (
const auto *VD = dyn_cast_or_null<VarDecl>(D))
5425 PrivateDecls.push_back(VD);
5429 void VisitCapturedStmt(
const CapturedStmt *) {}
5431 void VisitBlockExpr(
const BlockExpr *) {}
5432 void VisitStmt(
const Stmt *S) {
5435 for (
const Stmt *Child : S->
children())
5441 ArrayRef<const VarDecl *> getPrivateDecls()
const {
return PrivateDecls; }
5449 bool OmpAllMemory =
false;
5452 return C->getDependencyKind() == OMPC_DEPEND_outallmemory ||
5453 C->getDependencyKind() == OMPC_DEPEND_inoutallmemory;
5455 OmpAllMemory =
true;
5460 Data.Dependences.emplace_back(OMPC_DEPEND_outallmemory,
5469 if (Kind == OMPC_DEPEND_outallmemory || Kind == OMPC_DEPEND_inoutallmemory)
5471 if (OmpAllMemory && (Kind == OMPC_DEPEND_out || Kind == OMPC_DEPEND_inout))
5474 Data.Dependences.emplace_back(
C->getDependencyKind(),
C->getModifier());
5475 DD.
DepExprs.append(
C->varlist_begin(),
C->varlist_end());
5484 const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
5486 auto PartId = std::next(I);
5487 auto TaskT = std::next(I, 4);
5492 const Expr *Cond = Clause->getCondition();
5495 Data.Final.setInt(CondConstant);
5500 Data.Final.setInt(
false);
5504 const Expr *Prio = Clause->getPriority();
5505 Data.Priority.setInt(
true);
5513 llvm::DenseSet<const ValueDecl *> EmittedAsPrivate;
5515 for (
const auto *
C : S.getClausesOfKind<OMPPrivateClause>()) {
5516 auto IRef =
C->varlist_begin();
5517 for (
const Expr *IInit :
C->private_copies()) {
5519 if (EmittedAsPrivate.insert(
cast<ValueDecl>(OrigDecl->getCanonicalDecl()))
5521 Data.PrivateVars.push_back(*IRef);
5522 Data.PrivateCopies.push_back(IInit);
5527 EmittedAsPrivate.clear();
5529 for (
const auto *
C : S.getClausesOfKind<OMPFirstprivateClause>()) {
5530 auto IRef =
C->varlist_begin();
5531 auto IElemInitRef =
C->inits().begin();
5532 for (
const Expr *IInit :
C->private_copies()) {
5534 if (EmittedAsPrivate.insert(
cast<ValueDecl>(OrigDecl->getCanonicalDecl()))
5536 Data.FirstprivateVars.push_back(*IRef);
5537 Data.FirstprivateCopies.push_back(IInit);
5538 Data.FirstprivateInits.push_back(*IElemInitRef);
5545 llvm::MapVector<const ValueDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
5546 llvm::MapVector<const ValueDecl *, const DeclRefExpr *> LastprivateSrcsOrigs;
5547 for (
const auto *
C : S.getClausesOfKind<OMPLastprivateClause>()) {
5548 auto IRef =
C->varlist_begin();
5549 auto ID =
C->destination_exprs().begin();
5550 auto IS =
C->source_exprs().begin();
5551 for (
const Expr *IInit :
C->private_copies()) {
5553 if (EmittedAsPrivate.insert(
cast<ValueDecl>(OrigDecl->getCanonicalDecl()))
5555 Data.LastprivateVars.push_back(*IRef);
5556 Data.LastprivateCopies.push_back(IInit);
5558 LastprivateDstsOrigs.insert(std::make_pair(
5560 LastprivateSrcsOrigs.insert(std::make_pair(
5569 for (
const auto *
C : S.getClausesOfKind<OMPReductionClause>()) {
5570 Data.ReductionVars.append(
C->varlist_begin(),
C->varlist_end());
5571 Data.ReductionOrigs.append(
C->varlist_begin(),
C->varlist_end());
5572 Data.ReductionCopies.append(
C->privates().begin(),
C->privates().end());
5573 Data.ReductionOps.append(
C->reduction_ops().begin(),
5574 C->reduction_ops().end());
5575 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
5576 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
5578 Data.Reductions =
CGM.getOpenMPRuntime().emitTaskReductionInit(
5579 *
this, S.getBeginLoc(), LHSs, RHSs,
Data);
5584 CheckVarsEscapingUntiedTaskDeclContext Checker;
5585 Checker.Visit(S.getInnermostCapturedStmt()->getCapturedStmt());
5586 Data.PrivateLocals.append(Checker.getPrivateDecls().begin(),
5587 Checker.getPrivateDecls().end());
5589 auto &&
CodeGen = [&
Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
5590 &LastprivateSrcsOrigs, CapturedRegion](
5592 llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
5593 std::pair<Address, Address>>
5598 if (
auto *DI = CGF.getDebugInfo()) {
5599 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields =
5600 CGF.CapturedStmtInfo->getCaptureFields();
5601 llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue();
5602 if (CaptureFields.size() && ContextValue) {
5603 unsigned CharWidth = CGF.getContext().getCharWidth();
5617 for (
auto It = CaptureFields.begin(); It != CaptureFields.end(); ++It) {
5618 const VarDecl *SharedVar = It->first;
5621 CGF.getContext().getASTRecordLayout(CaptureRecord);
5624 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
5625 (void)DI->EmitDeclareOfAutoVariable(SharedVar, ContextValue,
5626 CGF.Builder,
false);
5629 auto UpdateExpr = [](llvm::LLVMContext &Ctx,
auto *
Declare,
5634 Ops.push_back(llvm::dwarf::DW_OP_plus_uconst);
5635 Ops.push_back(Offset);
5637 Ops.push_back(llvm::dwarf::DW_OP_deref);
5638 Declare->setExpression(llvm::DIExpression::get(Ctx, Ops));
5640 llvm::Instruction &
Last = CGF.Builder.GetInsertBlock()->back();
5643 assert(!
Last.isTerminator() &&
"unexpected terminator");
5645 CGF.Builder.GetInsertBlock()->getTrailingDbgRecords()) {
5646 for (llvm::DbgVariableRecord &DVR : llvm::reverse(
5647 llvm::filterDbgVars(Marker->getDbgRecordRange()))) {
5648 UpdateExpr(
Last.getContext(), &DVR, Offset);
5657 if (!
Data.PrivateVars.empty() || !
Data.FirstprivateVars.empty() ||
5658 !
Data.LastprivateVars.empty() || !
Data.PrivateLocals.empty()) {
5659 enum { PrivatesParam = 2, CopyFnParam = 3 };
5660 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
5662 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
5668 CallArgs.push_back(PrivatesPtr);
5669 ParamTypes.push_back(PrivatesPtr->getType());
5670 for (
const Expr *E :
Data.PrivateVars) {
5672 RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
5673 CGF.getContext().getPointerType(E->
getType()),
".priv.ptr.addr");
5674 PrivatePtrs.emplace_back(VD, PrivatePtr);
5676 ParamTypes.push_back(PrivatePtr.
getType());
5678 for (
const Expr *E :
Data.FirstprivateVars) {
5680 RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
5681 CGF.getContext().getPointerType(E->
getType()),
5682 ".firstpriv.ptr.addr");
5683 PrivatePtrs.emplace_back(VD, PrivatePtr);
5684 FirstprivatePtrs.emplace_back(VD, PrivatePtr);
5686 ParamTypes.push_back(PrivatePtr.
getType());
5688 for (
const Expr *E :
Data.LastprivateVars) {
5690 RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
5691 CGF.getContext().getPointerType(E->
getType()),
5692 ".lastpriv.ptr.addr");
5693 PrivatePtrs.emplace_back(VD, PrivatePtr);
5695 ParamTypes.push_back(PrivatePtr.
getType());
5700 Ty = CGF.getContext().getPointerType(Ty);
5702 Ty = CGF.getContext().getPointerType(Ty);
5703 RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
5704 CGF.getContext().getPointerType(Ty),
".local.ptr.addr");
5705 auto Result = UntiedLocalVars.insert(
5708 if (
Result.second ==
false)
5709 *
Result.first = std::make_pair(
5712 ParamTypes.push_back(PrivatePtr.
getType());
5714 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
5716 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
5717 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
5718 for (
const auto &Pair : LastprivateDstsOrigs) {
5719 const auto *OrigDecl = Pair.second->getDecl();
5720 if (
const auto *BD = dyn_cast<BindingDecl>(OrigDecl)) {
5723 CGF.EmitOMPBindingOriginalAddr(BD, Pair.second->getExprLoc());
5724 Scope.addPrivate(Pair.first, OrigAddr);
5729 CGF.CapturedStmtInfo->lookup(OrigVD) !=
nullptr,
5731 Pair.second->getExprLoc());
5732 Scope.addPrivate(Pair.first, CGF.EmitLValue(&DRE).getAddress());
5735 for (
const auto &Pair : PrivatePtrs) {
5737 CGF.Builder.CreateLoad(Pair.second),
5738 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
5739 CGF.getContext().getDeclAlign(Pair.first));
5740 Scope.addPrivate(Pair.first, Replacement);
5745 for (
const auto &SrcPair : LastprivateSrcsOrigs) {
5746 if (SrcPair.second->getDecl() == Pair.first) {
5747 Scope.addPrivate(SrcPair.first, Replacement);
5753 if (
auto *DI = CGF.getDebugInfo())
5754 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
5756 if (
const auto *VD = dyn_cast<VarDecl>(Pair.first))
5757 (void)DI->EmitDeclareOfAutoVariable(
5758 VD, Pair.second.getBasePointer(), CGF.Builder,
5763 for (
auto &Pair : UntiedLocalVars) {
5764 QualType VDType = Pair.first->getType().getNonReferenceType();
5765 if (Pair.first->getType()->isLValueReferenceType())
5766 VDType = CGF.getContext().getPointerType(VDType);
5768 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
5771 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(VDType)),
5772 CGF.getPointerAlign());
5773 Pair.second.first = Replacement;
5774 Ptr = CGF.Builder.CreateLoad(Replacement);
5775 Replacement =
Address(Ptr, CGF.ConvertTypeForMem(VDType),
5776 CGF.getContext().getDeclAlign(Pair.first));
5777 Pair.second.second = Replacement;
5779 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
5780 Address Replacement(Ptr, CGF.ConvertTypeForMem(VDType),
5781 CGF.getContext().getDeclAlign(Pair.first));
5782 Pair.second.first = Replacement;
5786 if (
Data.Reductions) {
5788 for (
const auto &Pair : FirstprivatePtrs) {
5790 CGF.Builder.CreateLoad(Pair.second),
5791 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
5792 CGF.getContext().getDeclAlign(Pair.first));
5793 FirstprivateScope.
addPrivate(Pair.first, Replacement);
5796 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
5798 Data.ReductionCopies,
Data.ReductionOps);
5799 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
5801 for (
unsigned Cnt = 0, E =
Data.ReductionVars.size(); Cnt < E; ++Cnt) {
5807 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
5809 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5812 CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF),
5813 CGF.getContext().VoidPtrTy,
5814 CGF.getContext().getPointerType(
5815 Data.ReductionCopies[Cnt]->getType()),
5816 Data.ReductionCopies[Cnt]->getExprLoc()),
5817 CGF.ConvertTypeForMem(
Data.ReductionCopies[Cnt]->getType()),
5818 Replacement.getAlignment());
5824 (void)
Scope.Privatize();
5829 for (
const auto *
C : S.getClausesOfKind<OMPInReductionClause>()) {
5830 auto IPriv =
C->privates().begin();
5831 auto IRed =
C->reduction_ops().begin();
5832 auto ITD =
C->taskgroup_descriptors().begin();
5833 for (
const Expr *Ref :
C->varlist()) {
5834 InRedVars.emplace_back(Ref);
5835 InRedPrivs.emplace_back(*IPriv);
5836 InRedOps.emplace_back(*IRed);
5837 TaskgroupDescriptors.emplace_back(*ITD);
5838 std::advance(IPriv, 1);
5839 std::advance(IRed, 1);
5840 std::advance(ITD, 1);
5846 if (!InRedVars.empty()) {
5848 for (
unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
5856 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
5858 llvm::Value *ReductionsPtr;
5859 if (
const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
5860 ReductionsPtr = CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr),
5861 TRExpr->getExprLoc());
5863 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5865 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5868 CGF.EmitScalarConversion(
5869 Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy,
5870 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
5871 InRedPrivs[Cnt]->getExprLoc()),
5872 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),
5873 Replacement.getAlignment());
5886 llvm::Function *OutlinedFn =
CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
5887 S, *I, *PartId, *TaskT, EKind,
CodeGen,
Data.Tied,
Data.NumberOfParts);
5888 OMPLexicalScope
Scope(*
this, S, std::nullopt,
5891 TaskGen(*
this, OutlinedFn,
Data);
5908 QualType ElemType =
C.getBaseElementType(Ty);
5918 Data.FirstprivateVars.emplace_back(OrigRef);
5919 Data.FirstprivateCopies.emplace_back(PrivateRef);
5920 Data.FirstprivateInits.emplace_back(InitRef);
5933 auto PartId = std::next(I);
5934 auto TaskT = std::next(I, 4);
5937 Data.Final.setInt(
false);
5939 for (
const auto *
C : S.getClausesOfKind<OMPFirstprivateClause>()) {
5940 auto IRef =
C->varlist_begin();
5941 auto IElemInitRef =
C->inits().begin();
5942 for (
auto *IInit :
C->private_copies()) {
5943 Data.FirstprivateVars.push_back(*IRef);
5944 Data.FirstprivateCopies.push_back(IInit);
5945 Data.FirstprivateInits.push_back(*IElemInitRef);
5952 for (
const auto *
C : S.getClausesOfKind<OMPInReductionClause>()) {
5953 Data.ReductionVars.append(
C->varlist_begin(),
C->varlist_end());
5954 Data.ReductionOrigs.append(
C->varlist_begin(),
C->varlist_end());
5955 Data.ReductionCopies.append(
C->privates().begin(),
C->privates().end());
5956 Data.ReductionOps.append(
C->reduction_ops().begin(),
5957 C->reduction_ops().end());
5958 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
5959 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
5974 getContext(),
Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5976 getContext(),
Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5988 if (!isa_and_nonnull<llvm::ConstantPointerNull>(
5991 getContext(),
Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5998 auto &&
CodeGen = [&
Data, &S, CS, &BodyGen, BPVD, PVD, SVD, MVD, EKind,
6002 if (!
Data.FirstprivateVars.empty()) {
6003 enum { PrivatesParam = 2, CopyFnParam = 3 };
6004 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
6006 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
6012 CallArgs.push_back(PrivatesPtr);
6013 ParamTypes.push_back(PrivatesPtr->getType());
6014 for (
const Expr *E :
Data.FirstprivateVars) {
6016 RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
6017 CGF.getContext().getPointerType(E->
getType()),
6018 ".firstpriv.ptr.addr");
6019 PrivatePtrs.emplace_back(VD, PrivatePtr);
6021 ParamTypes.push_back(PrivatePtr.
getType());
6023 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
6025 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
6026 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
6027 for (
const auto &Pair : PrivatePtrs) {
6029 CGF.Builder.CreateLoad(Pair.second),
6030 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
6031 CGF.getContext().getDeclAlign(Pair.first));
6032 Scope.addPrivate(Pair.first, Replacement);
6035 CGF.processInReduction(S,
Data, CGF, CS,
Scope);
6038 CGF.GetAddrOfLocalVar(BPVD), 0);
6040 CGF.GetAddrOfLocalVar(PVD), 0);
6041 InputInfo.
SizesArray = CGF.Builder.CreateConstArrayGEP(
6042 CGF.GetAddrOfLocalVar(SVD), 0);
6045 InputInfo.
MappersArray = CGF.Builder.CreateConstArrayGEP(
6046 CGF.GetAddrOfLocalVar(MVD), 0);
6050 OMPLexicalScope LexScope(CGF, S, OMPD_task,
false);
6052 if (CGF.CGM.getLangOpts().OpenMP >= 51 &&
6057 CGF.CGM.getOpenMPRuntime().emitThreadLimitClause(
6058 CGF, TL->getThreadLimit().front(), S.getBeginLoc());
6062 llvm::Function *OutlinedFn =
CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
6063 S, *I, *PartId, *TaskT, EKind,
CodeGen,
true,
6064 Data.NumberOfParts);
6065 llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
6069 CGM.getOpenMPRuntime().emitTaskCall(*
this, S.getBeginLoc(), S, OutlinedFn,
6070 SharedsTy, CapturedStruct, &IfCond,
Data);
6075 CodeGenFunction &CGF,
6079 if (
Data.Reductions) {
6081 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
6083 Data.ReductionCopies,
Data.ReductionOps);
6086 for (
unsigned Cnt = 0, E =
Data.ReductionVars.size(); Cnt < E; ++Cnt) {
6100 Data.ReductionCopies[Cnt]->getType()),
6101 Data.ReductionCopies[Cnt]->getExprLoc()),
6103 Replacement.getAlignment());
6108 (void)
Scope.Privatize();
6113 for (
const auto *
C : S.getClausesOfKind<OMPInReductionClause>()) {
6114 auto IPriv =
C->privates().begin();
6115 auto IRed =
C->reduction_ops().begin();
6116 auto ITD =
C->taskgroup_descriptors().begin();
6117 for (
const Expr *Ref :
C->varlist()) {
6118 InRedVars.emplace_back(Ref);
6119 InRedPrivs.emplace_back(*IPriv);
6120 InRedOps.emplace_back(*IRed);
6121 TaskgroupDescriptors.emplace_back(*ITD);
6122 std::advance(IPriv, 1);
6123 std::advance(IRed, 1);
6124 std::advance(ITD, 1);
6128 if (!InRedVars.empty()) {
6130 for (
unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
6138 llvm::Value *ReductionsPtr;
6139 if (
const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
6143 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
6151 InRedPrivs[Cnt]->getExprLoc()),
6153 Replacement.getAlignment());
6167 const Expr *IfCond =
nullptr;
6168 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
6169 if (
C->getNameModifier() == OMPD_unknown ||
6170 C->getNameModifier() == OMPD_task) {
6171 IfCond =
C->getCondition();
6178 Data.Tied = !S.getSingleClause<OMPUntiedClause>();
6182 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
6183 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
6186 SharedsTy, CapturedStruct, IfCond,
6196 CGM.getOpenMPRuntime().emitTaskyieldCall(*
this, S.getBeginLoc());
6200 const OMPMessageClause *MC = S.getSingleClause<OMPMessageClause>();
6201 Expr *ME = MC ? MC->getMessageString() :
nullptr;
6202 const OMPSeverityClause *SC = S.getSingleClause<OMPSeverityClause>();
6203 bool IsFatal =
false;
6204 if (!SC || SC->getSeverityKind() == OMPC_SEVERITY_fatal)
6206 CGM.getOpenMPRuntime().emitErrorCall(*
this, S.getBeginLoc(), ME, IsFatal);
6210 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.getBeginLoc(), OMPD_barrier);
6217 Data.HasNowaitClause = S.hasClausesOfKind<OMPNowaitClause>();
6218 CGM.getOpenMPRuntime().emitTaskwaitCall(*
this, S.getBeginLoc(),
Data);
6222 return T.clauses().empty();
6227 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
6229 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
6230 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
6233 auto BodyGenCB = [&,
this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
6236 EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
6237 return llvm::Error::success();
6242 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
6243 cantFail(OMPBuilder.createTaskgroup(
Builder, AllocaIP,
6254 for (
const auto *
C : S.getClausesOfKind<OMPTaskReductionClause>()) {
6255 Data.ReductionVars.append(
C->varlist_begin(),
C->varlist_end());
6256 Data.ReductionOrigs.append(
C->varlist_begin(),
C->varlist_end());
6257 Data.ReductionCopies.append(
C->privates().begin(),
C->privates().end());
6258 Data.ReductionOps.append(
C->reduction_ops().begin(),
6259 C->reduction_ops().end());
6260 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
6261 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
6263 llvm::Value *ReductionDesc =
6271 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
6273 CGM.getOpenMPRuntime().emitTaskgroupRegion(*
this,
CodeGen, S.getBeginLoc());
6278 ? llvm::AtomicOrdering::NotAtomic
6279 : llvm::AtomicOrdering::AcquireRelease;
6280 CGM.getOpenMPRuntime().emitFlush(
6283 if (
const auto *FlushClause = S.getSingleClause<
OMPFlushClause>())
6285 FlushClause->varlist_end());
6288 S.getBeginLoc(), AO);
6298 for (
auto &Dep :
Data.Dependences) {
6299 Address DepAddr =
CGM.getOpenMPRuntime().emitDepobjDependClause(
6300 *
this, Dep, DC->getBeginLoc());
6306 CGM.getOpenMPRuntime().emitDestroyClause(*
this, DOLVal, DC->getBeginLoc());
6309 if (
const auto *UC = S.getSingleClause<OMPUpdateDependObjectsClause>()) {
6310 CGM.getOpenMPRuntime().emitUpdateDependObjectsClause(
6311 *
this, DOLVal, UC->getDependencyKind(), UC->getBeginLoc());
6329 for (
const auto *
C : ParentDir.getClausesOfKind<OMPReductionClause>()) {
6330 if (
C->getModifier() != OMPC_REDUCTION_inscan)
6332 Shareds.append(
C->varlist_begin(),
C->varlist_end());
6333 Privates.append(
C->privates().begin(),
C->privates().end());
6334 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
6335 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
6336 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
6337 CopyOps.append(
C->copy_ops().begin(),
C->copy_ops().end());
6338 CopyArrayTemps.append(
C->copy_array_temps().begin(),
6339 C->copy_array_temps().end());
6340 CopyArrayElems.append(
C->copy_array_elems().begin(),
6341 C->copy_array_elems().end());
6343 if (ParentDir.getDirectiveKind() == OMPD_simd ||
6385 : BreakContinueStack.back().ContinueBlock.getBlock());
6396 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
6398 const Expr *TempExpr = CopyArrayTemps[I];
6410 CGM.getOpenMPRuntime().emitReduction(
6411 *
this, ParentDir.getEndLoc(),
Privates, LHSs, RHSs, ReductionOps,
6414 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
6422 const Expr *TempExpr = CopyArrayTemps[I];
6434 ? BreakContinueStack.back().ContinueBlock.getBlock()
6440 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
6446 .getIterationVariable()
6447 ->IgnoreParenImpCasts();
6450 IdxVal = Builder.CreateIntCast(IdxVal,
SizeTy,
false);
6451 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
6453 const Expr *OrigExpr = Shareds[I];
6454 const Expr *CopyArrayElem = CopyArrayElems[I];
6455 OpaqueValueMapping IdxMapping(
6468 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
6471 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
6477 .getIterationVariable()
6478 ->IgnoreParenImpCasts();
6482 llvm::BasicBlock *ExclusiveExitBB =
nullptr;
6486 llvm::Value *
Cmp =
Builder.CreateIsNull(IdxVal);
6487 Builder.CreateCondBr(
Cmp, ExclusiveExitBB, ContBB);
6490 IdxVal =
Builder.CreateNUWSub(IdxVal, llvm::ConstantInt::get(
SizeTy, 1));
6492 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
6493 const Expr *PrivateExpr =
Privates[I];
6494 const Expr *OrigExpr = Shareds[I];
6495 const Expr *CopyArrayElem = CopyArrayElems[I];
6504 PrivateExpr->
getType(), DestLVal.getAddress(), SrcLVal.getAddress(),
6537 bool HasLastprivateClause =
false;
6540 OMPLoopScope PreInitScope(*
this, S);
6545 llvm::BasicBlock *ContBlock =
nullptr;
6581 CGM.getOpenMPRuntime().emitBarrierCall(
6582 *
this, S.getBeginLoc(), OMPD_unknown,
false,
6594 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*
this, S);
6597 llvm::Value *Chunk =
nullptr;
6600 ScheduleKind =
C->getDistScheduleKind();
6601 if (
const Expr *Ch =
C->getChunkSize()) {
6609 CGM.getOpenMPRuntime().getDefaultDistScheduleAndChunk(
6610 *
this, S, ScheduleKind, Chunk);
6631 bool StaticChunked =
6635 Chunk !=
nullptr) ||
6640 StaticChunked ? Chunk :
nullptr);
6691 [&S, &LoopScope, Cond, IncExpr,
LoopExit, &CodeGenLoop,
6694 S, LoopScope.requiresCleanups(), Cond, IncExpr,
6695 [&S,
LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
6696 CodeGenLoop(CGF, S, LoopExit);
6698 [&S, StaticChunked](CodeGenFunction &CGF) {
6699 if (StaticChunked) {
6700 CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());
6701 CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());
6702 CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());
6703 CGF.EmitIgnoredExpr(S.getCombinedInit());
6713 const OMPLoopArguments LoopArguments = {
6716 EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
6722 return CGF.
Builder.CreateIsNotNull(
6732 *
this, S, [IL, &S](CodeGenFunction &CGF) {
6733 return CGF.
Builder.CreateIsNotNull(
6738 if (HasLastprivateClause) {
6761 OMPLexicalScope
Scope(CGF, S, OMPD_unknown);
6770static llvm::Function *
6777 Fn->setDoesNotRecurse();
6781template <
typename T>
6783 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP,
6784 llvm::OpenMPIRBuilder &OMPBuilder) {
6786 unsigned NumLoops =
C->getNumLoops();
6790 for (
unsigned I = 0; I < NumLoops; I++) {
6791 const Expr *CounterVal =
C->getLoopData(I);
6796 StoreValues.emplace_back(StoreValue);
6798 OMPDoacrossKind<T> ODK;
6799 bool IsDependSource = ODK.isSource(
C);
6801 OMPBuilder.createOrderedDepend(CGF.
Builder, AllocaIP, NumLoops,
6802 StoreValues,
".cnt.addr", IsDependSource));
6809 "Standalone ordered directive should have either depend or doacross "
6812 assert(!S.hasAssociatedStmt() &&
"No associated statement must be in "
6813 "ordered depend|doacross construct.");
6815 if (
CGM.getLangOpts().OpenMPIRBuilder) {
6816 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
6817 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
6829 CGM.getOpenMPRuntime().emitDoacrossOrdered(*
this, DC);
6832 CGM.getOpenMPRuntime().emitDoacrossOrdered(*
this, DC);
6838 if (
CGM.getLangOpts().OpenMPIRBuilder) {
6839 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
6840 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
6846 auto FiniCB = [
this](InsertPointTy IP) {
6848 return llvm::Error::success();
6851 auto BodyGenCB = [&S,
C,
this](InsertPointTy AllocIP,
6852 InsertPointTy CodeGenIP,
6858 llvm::BasicBlock *CodeGenBB = CodeGenIP.getNodeParent();
6859 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
6860 Builder,
false,
".ordered.after");
6864 assert(S.getBeginLoc().isValid() &&
6865 "Outlined function call location must be valid.");
6868 OutlinedFn, CapturedVars);
6873 return llvm::Error::success();
6876 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
6877 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
6878 OMPBuilder.createOrderedThreadsSimd(
Builder, BodyGenCB, FiniCB, !
C));
6884 auto &&
CodeGen = [&S,
C,
this](CodeGenFunction &CGF,
6889 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
6891 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
6892 OutlinedFn, CapturedVars);
6898 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
6899 CGM.getOpenMPRuntime().emitOrderedRegion(*
this,
CodeGen, S.getBeginLoc(), !
C);
6906 "DestType must have scalar evaluation kind.");
6907 assert(!Val.
isAggregate() &&
"Must be a scalar or complex.");
6918 "DestType must have complex evaluation kind.");
6927 ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
6929 assert(Val.
isComplex() &&
"Must be a scalar or complex.");
6934 Val.
getComplexVal().first, SrcElementType, DestElementType, Loc);
6936 Val.
getComplexVal().second, SrcElementType, DestElementType, Loc);
6942 LValue LVal,
RValue RVal) {
6943 if (LVal.isGlobalReg())
6950 llvm::AtomicOrdering AO, LValue LVal,
6952 if (LVal.isGlobalReg())
6955 LVal, Loc, llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO),
6964 *
this, RVal, RValTy, LVal.
getType(), Loc)),
6973 llvm_unreachable(
"Must be a scalar or complex.");
6981 assert(
V->isLValue() &&
"V of 'omp atomic read' is not lvalue");
6982 assert(
X->isLValue() &&
"X of 'omp atomic read' is not lvalue");
6991 case llvm::AtomicOrdering::Acquire:
6992 case llvm::AtomicOrdering::AcquireRelease:
6993 case llvm::AtomicOrdering::SequentiallyConsistent:
6995 llvm::AtomicOrdering::Acquire);
6997 case llvm::AtomicOrdering::Monotonic:
6998 case llvm::AtomicOrdering::Release:
7000 case llvm::AtomicOrdering::NotAtomic:
7001 case llvm::AtomicOrdering::Unordered:
7002 llvm_unreachable(
"Unexpected ordering.");
7009 llvm::AtomicOrdering AO,
const Expr *
X,
7012 assert(
X->isLValue() &&
"X of 'omp atomic write' is not lvalue");
7020 case llvm::AtomicOrdering::Release:
7021 case llvm::AtomicOrdering::AcquireRelease:
7022 case llvm::AtomicOrdering::SequentiallyConsistent:
7024 llvm::AtomicOrdering::Release);
7026 case llvm::AtomicOrdering::Acquire:
7027 case llvm::AtomicOrdering::Monotonic:
7029 case llvm::AtomicOrdering::NotAtomic:
7030 case llvm::AtomicOrdering::Unordered:
7031 llvm_unreachable(
"Unexpected ordering.");
7038 llvm::AtomicOrdering AO,
7039 bool IsXLHSInRHSPart) {
7044 if (BO == BO_Comma || !
Update.isScalar() || !
X.isSimple() ||
7046 (
Update.getScalarVal()->getType() !=
X.getAddress().getElementType())) ||
7047 !Context.getTargetInfo().hasBuiltinAtomic(
7048 Context.getTypeSize(
X.getType()), Context.toBits(
X.getAlignment())))
7049 return std::make_pair(
false,
RValue::get(
nullptr));
7052 if (
T->isIntegerTy())
7055 if (
T->isFloatingPointTy() && (BO == BO_Add || BO == BO_Sub))
7061 if (!CheckAtomicSupport(
Update.getScalarVal()->getType(), BO) ||
7062 !CheckAtomicSupport(
X.getAddress().getElementType(), BO))
7063 return std::make_pair(
false,
RValue::get(
nullptr));
7065 bool IsInteger =
X.getAddress().getElementType()->isIntegerTy();
7066 llvm::AtomicRMWInst::BinOp RMWOp;
7069 RMWOp = IsInteger ? llvm::AtomicRMWInst::Add : llvm::AtomicRMWInst::FAdd;
7072 if (!IsXLHSInRHSPart)
7073 return std::make_pair(
false,
RValue::get(
nullptr));
7074 RMWOp = IsInteger ? llvm::AtomicRMWInst::Sub : llvm::AtomicRMWInst::FSub;
7077 RMWOp = llvm::AtomicRMWInst::And;
7080 RMWOp = llvm::AtomicRMWInst::Or;
7083 RMWOp = llvm::AtomicRMWInst::Xor;
7087 RMWOp =
X.getType()->hasSignedIntegerRepresentation()
7088 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
7089 : llvm::AtomicRMWInst::Max)
7090 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
7091 : llvm::AtomicRMWInst::UMax);
7093 RMWOp = IsXLHSInRHSPart ? llvm::AtomicRMWInst::FMin
7094 : llvm::AtomicRMWInst::FMax;
7098 RMWOp =
X.getType()->hasSignedIntegerRepresentation()
7099 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
7100 : llvm::AtomicRMWInst::Min)
7101 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
7102 : llvm::AtomicRMWInst::UMin);
7104 RMWOp = IsXLHSInRHSPart ? llvm::AtomicRMWInst::FMax
7105 : llvm::AtomicRMWInst::FMin;
7108 RMWOp = llvm::AtomicRMWInst::Xchg;
7117 return std::make_pair(
false,
RValue::get(
nullptr));
7136 llvm_unreachable(
"Unsupported atomic update operation");
7138 llvm::Value *UpdateVal =
Update.getScalarVal();
7139 if (
auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
7141 UpdateVal = CGF.
Builder.CreateIntCast(
7142 IC,
X.getAddress().getElementType(),
7143 X.getType()->hasSignedIntegerRepresentation());
7145 UpdateVal = CGF.
Builder.CreateCast(llvm::Instruction::CastOps::UIToFP, IC,
7146 X.getAddress().getElementType());
7148 llvm::AtomicRMWInst *Res =
7165 if (
X.isGlobalReg()) {
7178 llvm::AtomicOrdering AO,
const Expr *
X,
7182 "Update expr in 'atomic update' must be a binary operator.");
7190 assert(
X->isLValue() &&
"X of 'omp atomic update' is not lvalue");
7197 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](
RValue XRValue) {
7203 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
7210 case llvm::AtomicOrdering::Release:
7211 case llvm::AtomicOrdering::AcquireRelease:
7212 case llvm::AtomicOrdering::SequentiallyConsistent:
7214 llvm::AtomicOrdering::Release);
7216 case llvm::AtomicOrdering::Acquire:
7217 case llvm::AtomicOrdering::Monotonic:
7219 case llvm::AtomicOrdering::NotAtomic:
7220 case llvm::AtomicOrdering::Unordered:
7221 llvm_unreachable(
"Unexpected ordering.");
7239 llvm_unreachable(
"Must be a scalar or complex.");
7243 llvm::AtomicOrdering AO,
7244 bool IsPostfixUpdate,
const Expr *
V,
7246 const Expr *UE,
bool IsXLHSInRHSPart,
7248 assert(
X->isLValue() &&
"X of 'omp atomic capture' is not lvalue");
7249 assert(
V->isLValue() &&
"V of 'omp atomic capture' is not lvalue");
7258 "Update expr in 'atomic capture' must be a binary operator.");
7269 NewVValType = XRValExpr->
getType();
7271 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
7272 IsPostfixUpdate](
RValue XRValue) {
7276 NewVVal = IsPostfixUpdate ? XRValue : Res;
7280 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
7284 if (IsPostfixUpdate) {
7286 NewVVal = Res.second;
7297 NewVValType =
X->getType().getNonReferenceType();
7299 X->getType().getNonReferenceType(), Loc);
7300 auto &&Gen = [&NewVVal, ExprRValue](
RValue XRValue) {
7306 XLValue, ExprRValue, BO_Assign,
false, AO,
7311 NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
7327 case llvm::AtomicOrdering::Release:
7329 llvm::AtomicOrdering::Release);
7331 case llvm::AtomicOrdering::Acquire:
7333 llvm::AtomicOrdering::Acquire);
7335 case llvm::AtomicOrdering::AcquireRelease:
7336 case llvm::AtomicOrdering::SequentiallyConsistent:
7338 CGF, {}, Loc, llvm::AtomicOrdering::AcquireRelease);
7340 case llvm::AtomicOrdering::Monotonic:
7342 case llvm::AtomicOrdering::NotAtomic:
7343 case llvm::AtomicOrdering::Unordered:
7344 llvm_unreachable(
"Unexpected ordering.");
7350 CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO,
7352 const Expr *CE,
bool IsXBinopExpr,
bool IsPostfixUpdate,
bool IsFailOnly,
7354 llvm::OpenMPIRBuilder &OMPBuilder =
7357 OMPAtomicCompareOp Op;
7361 Op = OMPAtomicCompareOp::EQ;
7364 Op = OMPAtomicCompareOp::MIN;
7367 Op = OMPAtomicCompareOp::MAX;
7370 llvm_unreachable(
"unsupported atomic compare binary operator");
7374 Address XAddr = XLVal.getAddress();
7376 auto EmitRValueWithCastIfNeeded = [&CGF, Loc](
const Expr *
X,
const Expr *E) {
7381 if (NewE->
getType() ==
X->getType())
7386 llvm::Value *EVal = EmitRValueWithCastIfNeeded(
X, E);
7387 llvm::Value *DVal = D ? EmitRValueWithCastIfNeeded(
X, D) :
nullptr;
7388 if (
auto *CI = dyn_cast<llvm::ConstantInt>(EVal))
7389 EVal = CGF.
Builder.CreateIntCast(
7390 CI, XLVal.getAddress().getElementType(),
7393 if (
auto *CI = dyn_cast<llvm::ConstantInt>(DVal))
7394 DVal = CGF.
Builder.CreateIntCast(
7395 CI, XLVal.getAddress().getElementType(),
7398 llvm::OpenMPIRBuilder::AtomicOpValue XOpVal{
7400 X->getType()->hasSignedIntegerRepresentation(),
7401 X->getType().isVolatileQualified()};
7402 llvm::OpenMPIRBuilder::AtomicOpValue VOpVal, ROpVal;
7406 VOpVal = {
Addr.emitRawPointer(CGF),
Addr.getElementType(),
7407 V->getType()->hasSignedIntegerRepresentation(),
7408 V->getType().isVolatileQualified()};
7413 ROpVal = {
Addr.emitRawPointer(CGF),
Addr.getElementType(),
7414 R->getType()->hasSignedIntegerRepresentation(),
7415 R->getType().isVolatileQualified()};
7418 if (FailAO == llvm::AtomicOrdering::NotAtomic) {
7421 CGF.
Builder.restoreIP(OMPBuilder.createAtomicCompare(
7422 CGF.
Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
7423 IsPostfixUpdate, IsFailOnly));
7425 CGF.
Builder.restoreIP(OMPBuilder.createAtomicCompare(
7426 CGF.
Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
7427 IsPostfixUpdate, IsFailOnly, FailAO));
7431 llvm::AtomicOrdering AO,
7432 llvm::AtomicOrdering FailAO,
bool IsPostfixUpdate,
7435 const Expr *CE,
bool IsXLHSInRHSPart,
7450 IsXLHSInRHSPart, Loc);
7452 case OMPC_compare: {
7454 IsXLHSInRHSPart, IsPostfixUpdate, IsFailOnly, Loc);
7458 llvm_unreachable(
"Clause is not allowed in 'omp atomic'.");
7463 llvm::AtomicOrdering AO =
CGM.getOpenMPRuntime().getDefaultMemoryOrdering();
7465 llvm::AtomicOrdering FailAO = llvm::AtomicOrdering::NotAtomic;
7466 bool MemOrderingSpecified =
false;
7467 if (S.getSingleClause<OMPSeqCstClause>()) {
7468 AO = llvm::AtomicOrdering::SequentiallyConsistent;
7469 MemOrderingSpecified =
true;
7470 }
else if (S.getSingleClause<OMPAcqRelClause>()) {
7471 AO = llvm::AtomicOrdering::AcquireRelease;
7472 MemOrderingSpecified =
true;
7473 }
else if (S.getSingleClause<OMPAcquireClause>()) {
7474 AO = llvm::AtomicOrdering::Acquire;
7475 MemOrderingSpecified =
true;
7476 }
else if (S.getSingleClause<OMPReleaseClause>()) {
7477 AO = llvm::AtomicOrdering::Release;
7478 MemOrderingSpecified =
true;
7479 }
else if (S.getSingleClause<OMPRelaxedClause>()) {
7480 AO = llvm::AtomicOrdering::Monotonic;
7481 MemOrderingSpecified =
true;
7483 llvm::SmallSet<OpenMPClauseKind, 2> KindsEncountered;
7492 if (K == OMPC_seq_cst || K == OMPC_acq_rel || K == OMPC_acquire ||
7493 K == OMPC_release || K == OMPC_relaxed || K == OMPC_hint)
7496 KindsEncountered.insert(K);
7501 if (KindsEncountered.contains(OMPC_compare) &&
7502 KindsEncountered.contains(OMPC_capture))
7503 Kind = OMPC_compare;
7504 if (!MemOrderingSpecified) {
7505 llvm::AtomicOrdering DefaultOrder =
7506 CGM.getOpenMPRuntime().getDefaultMemoryOrdering();
7507 if (DefaultOrder == llvm::AtomicOrdering::Monotonic ||
7508 DefaultOrder == llvm::AtomicOrdering::SequentiallyConsistent ||
7509 (DefaultOrder == llvm::AtomicOrdering::AcquireRelease &&
7510 Kind == OMPC_capture)) {
7512 }
else if (DefaultOrder == llvm::AtomicOrdering::AcquireRelease) {
7513 if (Kind == OMPC_unknown || Kind == OMPC_update || Kind == OMPC_write) {
7514 AO = llvm::AtomicOrdering::Release;
7515 }
else if (Kind == OMPC_read) {
7516 assert(Kind == OMPC_read &&
"Unexpected atomic kind.");
7517 AO = llvm::AtomicOrdering::Acquire;
7522 if (KindsEncountered.contains(OMPC_compare) &&
7523 KindsEncountered.contains(OMPC_fail)) {
7524 Kind = OMPC_compare;
7525 const auto *FailClause = S.getSingleClause<OMPFailClause>();
7528 if (FailParameter == llvm::omp::OMPC_relaxed)
7529 FailAO = llvm::AtomicOrdering::Monotonic;
7530 else if (FailParameter == llvm::omp::OMPC_acquire)
7531 FailAO = llvm::AtomicOrdering::Acquire;
7532 else if (FailParameter == llvm::omp::OMPC_seq_cst)
7533 FailAO = llvm::AtomicOrdering::SequentiallyConsistent;
7540 if (
const auto *HintClause = S.getSingleClause<
OMPHintClause>()) {
7541 uint64_t Hint = HintClause->getHint()
7544 if (Hint & llvm::to_underlying(OMPSyncHint::OMPXSafeFPAtomics))
7546 else if (Hint & llvm::to_underlying(OMPSyncHint::OMPXFastFPAtomics))
7567 OMPLexicalScope
Scope(CGF, S, OMPD_target);
7570 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
7576 llvm::Function *Fn =
nullptr;
7577 llvm::Constant *FnID =
nullptr;
7579 const Expr *IfCond =
nullptr;
7581 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
7582 if (
C->getNameModifier() == OMPD_unknown ||
7583 C->getNameModifier() == OMPD_target) {
7584 IfCond =
C->getCondition();
7590 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device(
7593 Device.setPointerAndInt(
C->getDevice(),
C->getModifier());
7598 bool IsOffloadEntry =
true;
7602 IsOffloadEntry =
false;
7605 IsOffloadEntry =
false;
7607 if (
CGM.
getLangOpts().OpenMPOffloadMandatory && !IsOffloadEntry) {
7615 while (
const auto *CD = dyn_cast_or_null<CapturedDecl>(CodeDecl))
7617 const auto *ParentFD = dyn_cast_or_null<FunctionDecl>(CodeDecl);
7618 StringRef ParentName =
7620 : CGF.
CurFn->getName();
7625 OMPLexicalScope
Scope(CGF, S, OMPD_task);
7626 auto &&SizeEmitter =
7629 if (IsOffloadEntry) {
7630 OMPLoopScope PreInitScope(CGF, D);
7633 NumIterations = CGF.
Builder.CreateIntCast(NumIterations, CGF.
Int64Ty,
7635 return NumIterations;
7653 CGF.
EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
7658 StringRef ParentName,
7664 llvm::Constant *
Addr;
7666 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7668 assert(Fn &&
Addr &&
"Target device function emission failed.");
7682 const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
7683 llvm::Function *OutlinedFn =
7688 OMPTeamsScope
Scope(CGF, S);
7693 const Expr *NumTeams = NT ? NT->getNumTeams().front() :
nullptr;
7694 const Expr *ThreadLimit = TL ? TL->getThreadLimit().front() :
nullptr;
7701 const Expr *IfCond =
nullptr;
7702 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
7703 if (
C->getNameModifier() == OMPD_unknown ||
7704 C->getNameModifier() == OMPD_teams) {
7705 IfCond =
C->getCondition();
7714 const llvm::APInt One(32, 1);
7721 const Expr *ThreadLimit = TL ? TL->getThreadLimit().front() :
nullptr;
7747 CGF.
EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
7752 [](CodeGenFunction &) {
return nullptr; });
7757 auto *CS = S.getCapturedStmt(OMPD_teams);
7784 llvm::Constant *
Addr;
7786 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7788 assert(Fn &&
Addr &&
"Target device function emission failed.");
7830 llvm::Constant *
Addr;
7832 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7834 assert(Fn &&
Addr &&
"Target device function emission failed.");
7876 llvm::Constant *
Addr;
7878 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7880 assert(Fn &&
Addr &&
"Target device function emission failed.");
7894 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF,
PrePostActionTy &) {
7899 auto &&
CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7911 [](CodeGenFunction &) {
return nullptr; });
7916 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF,
PrePostActionTy &) {
7921 auto &&
CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7933 [](CodeGenFunction &) {
return nullptr; });
7938 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF,
PrePostActionTy &) {
7944 auto &&
CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7956 [](CodeGenFunction &) {
return nullptr; });
7961 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF,
PrePostActionTy &) {
7967 auto &&
CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7974 CGF, OMPD_distribute, CodeGenDistribute,
false);
7980 [](CodeGenFunction &) {
return nullptr; });
7984 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
7985 llvm::Value *
Device =
nullptr;
7986 llvm::Value *NumDependences =
nullptr;
7987 llvm::Value *DependenceList =
nullptr;
7995 if (!
Data.Dependences.empty()) {
7997 std::tie(NumDependences, DependenciesArray) =
7998 CGM.getOpenMPRuntime().emitDependClause(*
this,
Data.Dependences,
8002 Data.HasNowaitClause = S.hasClausesOfKind<OMPNowaitClause>();
8007 "OMPNowaitClause clause is used separately in OMPInteropDirective.");
8010 if (!ItOMPInitClause.empty()) {
8013 llvm::Value *InteropvarPtr =
8015 llvm::omp::OMPInteropType InteropType =
8016 llvm::omp::OMPInteropType::Unknown;
8017 if (
C->getIsTarget()) {
8018 InteropType = llvm::omp::OMPInteropType::Target;
8020 assert(
C->getIsTargetSync() &&
8021 "Expected interop-type target/targetsync");
8022 InteropType = llvm::omp::OMPInteropType::TargetSync;
8024 OMPBuilder.createOMPInteropInit(
Builder, InteropvarPtr, InteropType,
8025 Device, NumDependences, DependenceList,
8026 Data.HasNowaitClause);
8030 if (!ItOMPDestroyClause.empty()) {
8033 llvm::Value *InteropvarPtr =
8035 OMPBuilder.createOMPInteropDestroy(
Builder, InteropvarPtr,
Device,
8036 NumDependences, DependenceList,
8037 Data.HasNowaitClause);
8040 auto ItOMPUseClause = S.getClausesOfKind<
OMPUseClause>();
8041 if (!ItOMPUseClause.empty()) {
8044 llvm::Value *InteropvarPtr =
8046 OMPBuilder.createOMPInteropUse(
Builder, InteropvarPtr,
Device,
8047 NumDependences, DependenceList,
8048 Data.HasNowaitClause);
8070 CGF, OMPD_distribute, CodeGenDistribute,
false);
8089 llvm::Constant *
Addr;
8091 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8093 assert(Fn &&
Addr &&
"Target device function emission failed.");
8122 CGF, OMPD_distribute, CodeGenDistribute,
false);
8141 llvm::Constant *
Addr;
8143 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8145 assert(Fn &&
Addr &&
"Target device function emission failed.");
8158 CGM.getOpenMPRuntime().emitCancellationPointCall(*
this, S.getBeginLoc(),
8163 const Expr *IfCond =
nullptr;
8164 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
8165 if (
C->getNameModifier() == OMPD_unknown ||
8166 C->getNameModifier() == OMPD_cancel) {
8167 IfCond =
C->getCondition();
8171 if (
CGM.getLangOpts().OpenMPIRBuilder) {
8172 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
8178 llvm::Value *IfCondition =
nullptr;
8182 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
8184 return Builder.restoreIP(AfterIP);
8188 CGM.getOpenMPRuntime().emitCancelCall(*
this, S.getBeginLoc(), IfCond,
8194 if (Kind == OMPD_parallel || Kind == OMPD_target_parallel ||
8197 assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
8198 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
8199 Kind == OMPD_distribute_parallel_for ||
8200 Kind == OMPD_target_parallel_for ||
8201 Kind == OMPD_teams_distribute_parallel_for ||
8202 Kind == OMPD_target_teams_distribute_parallel_for);
8203 return OMPCancelStack.getExitBlock();
8208 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
8209 CaptureDeviceAddrMap) {
8210 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
8211 for (
const Expr *OrigVarIt :
C.varlist()) {
8213 if (!Processed.insert(OrigVD).second)
8220 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
8225 "Base should be the current struct!");
8226 MatchingVD = ME->getMemberDecl();
8231 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
8232 if (InitAddrIt == CaptureDeviceAddrMap.end())
8240 Address(InitAddrIt->second, Ty,
8242 assert(IsRegistered &&
"firstprivate var already registered as private");
8250 while (
const auto *OASE = dyn_cast<ArraySectionExpr>(
Base))
8251 Base = OASE->getBase()->IgnoreParenImpCasts();
8252 while (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(
Base))
8253 Base = ASE->getBase()->IgnoreParenImpCasts();
8259 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
8260 CaptureDeviceAddrMap) {
8261 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
8262 for (
const Expr *Ref :
C.varlist()) {
8264 if (!Processed.insert(OrigVD).second)
8270 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
8275 "Base should be the current struct!");
8276 MatchingVD = ME->getMemberDecl();
8281 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
8282 if (InitAddrIt == CaptureDeviceAddrMap.end())
8288 Address(InitAddrIt->second, Ty,
8301 (void)PrivateScope.
addPrivate(OrigVD, PrivAddr);
8309 if (!
CGM.getLangOpts().OpenMPIsTargetDevice)
8310 CGM.getOpenMPRuntime().registerVTable(S);
8318 bool PrivatizeDevicePointers =
false;
8320 bool &PrivatizeDevicePointers;
8323 explicit DevicePointerPrivActionTy(
bool &PrivatizeDevicePointers)
8324 : PrivatizeDevicePointers(PrivatizeDevicePointers) {}
8325 void Enter(CodeGenFunction &CGF)
override {
8326 PrivatizeDevicePointers =
true;
8329 DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
8332 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF,
PrePostActionTy &) {
8333 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
8337 auto &&PrivCodeGen = [&](CodeGenFunction &CGF,
PrePostActionTy &Action) {
8339 PrivatizeDevicePointers =
false;
8345 if (PrivatizeDevicePointers) {
8359 std::optional<OpenMPDirectiveKind> CaptureRegion;
8360 if (
CGM.getLangOpts().OMPTargetTriples.empty()) {
8363 for (
const Expr *E :
C->varlist()) {
8365 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
8369 for (
const Expr *E :
C->varlist()) {
8371 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
8375 CaptureRegion = OMPD_unknown;
8378 OMPLexicalScope
Scope(CGF, S, CaptureRegion);
8390 OMPLexicalScope
Scope(CGF, S);
8399 if (
CGM.getLangOpts().OMPTargetTriples.empty()) {
8405 const Expr *IfCond =
nullptr;
8407 IfCond =
C->getCondition();
8418 CGM.getOpenMPRuntime().emitTargetDataCalls(*
this, S, IfCond,
Device, RCG,
8426 if (
CGM.getLangOpts().OMPTargetTriples.empty())
8430 const Expr *IfCond =
nullptr;
8432 IfCond =
C->getCondition();
8439 OMPLexicalScope
Scope(*
this, S, OMPD_task);
8440 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*
this, S, IfCond,
Device);
8447 if (
CGM.getLangOpts().OMPTargetTriples.empty())
8451 const Expr *IfCond =
nullptr;
8453 IfCond =
C->getCondition();
8460 OMPLexicalScope
Scope(*
this, S, OMPD_task);
8461 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*
this, S, IfCond,
Device);
8468 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
8496 llvm::Constant *
Addr;
8498 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8500 assert(Fn &&
Addr &&
"Target device function emission failed.");
8520 CGF, OMPD_target_parallel_for, S.
hasCancel());
8536 llvm::Constant *
Addr;
8538 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8540 assert(Fn &&
Addr &&
"Target device function emission failed.");
8575 llvm::Constant *
Addr;
8577 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8579 assert(Fn &&
Addr &&
"Target device function emission failed.");
8601 const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);
8604 OMPLexicalScope
Scope(*
this, S, OMPD_taskloop,
false);
8609 const Expr *IfCond =
nullptr;
8610 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
8611 if (
C->getNameModifier() == OMPD_unknown ||
8612 C->getNameModifier() == OMPD_taskloop) {
8613 IfCond =
C->getCondition();
8626 Data.Schedule.setInt(
false);
8629 (Clause->getModifier() == OMPC_GRAINSIZE_strict) ?
true :
false;
8632 Data.Schedule.setInt(
true);
8635 (Clause->getModifier() == OMPC_NUMTASKS_strict) ?
true :
false;
8649 llvm::BasicBlock *ContBlock =
nullptr;
8650 OMPLoopScope PreInitScope(CGF, S);
8651 if (CGF.ConstantFoldsToSimpleInteger(S.
getPreCond(), CondConstant)) {
8655 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock(
"taskloop.if.then");
8656 ContBlock = CGF.createBasicBlock(
"taskloop.if.end");
8658 CGF.getProfileCount(&S));
8659 CGF.EmitBlock(ThenBlock);
8660 CGF.incrementProfileCounter(&S);
8663 (void)CGF.EmitOMPLinearClauseInit(S);
8667 enum { LowerBound = 5, UpperBound, Stride, LastIter };
8669 auto *LBP = std::next(I, LowerBound);
8670 auto *UBP = std::next(I, UpperBound);
8671 auto *STP = std::next(I, Stride);
8672 auto *LIP = std::next(I, LastIter);
8680 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
8681 CGF.EmitOMPLinearClause(S, LoopScope);
8682 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
8687 CGF.EmitVarDecl(*IVDecl);
8688 CGF.EmitIgnoredExpr(S.
getInit());
8700 OMPLexicalScope
Scope(CGF, S, OMPD_taskloop,
false);
8710 [&S](CodeGenFunction &CGF) {
8711 emitOMPLoopBodyWithStopPoint(CGF, S,
8712 CodeGenFunction::JumpDest());
8714 [](CodeGenFunction &) {});
8719 CGF.EmitBranch(ContBlock);
8720 CGF.EmitBlock(ContBlock,
true);
8723 if (HasLastprivateClause) {
8724 CGF.EmitOMPLastprivateClauseFinal(
8726 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
8727 CGF.GetAddrOfLocalVar(*LIP),
false,
8728 (*LIP)->getType(), S.getBeginLoc())));
8731 CGF.EmitOMPLinearClauseFinal(S, [LIP, &S](CodeGenFunction &CGF) {
8732 return CGF.
Builder.CreateIsNotNull(
8734 (*LIP)->
getType(), S.getBeginLoc()));
8737 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
8738 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
8740 auto &&
CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
8742 OMPLoopScope PreInitScope(CGF, S);
8743 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
8744 OutlinedFn, SharedsTy,
8745 CapturedStruct, IfCond,
Data);
8747 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
8753 CGM.getOpenMPRuntime().emitTaskgroupRegion(
8755 [&S, &BodyGen, &TaskGen, &
Data](CodeGenFunction &CGF,
8775 OMPLexicalScope
Scope(*
this, S);
8787 OMPLexicalScope
Scope(*
this, S, std::nullopt,
false);
8788 CGM.getOpenMPRuntime().emitMasterRegion(*
this,
CodeGen, S.getBeginLoc());
8799 OMPLexicalScope
Scope(*
this, S, std::nullopt,
false);
8800 CGM.getOpenMPRuntime().emitMaskedRegion(*
this,
CodeGen, S.getBeginLoc());
8811 OMPLexicalScope
Scope(*
this, S);
8812 CGM.getOpenMPRuntime().emitMasterRegion(*
this,
CodeGen, S.getBeginLoc());
8823 OMPLexicalScope
Scope(*
this, S);
8824 CGM.getOpenMPRuntime().emitMaskedRegion(*
this,
CodeGen, S.getBeginLoc());
8830 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8835 OMPLexicalScope
Scope(CGF, S, OMPD_parallel,
false);
8836 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
8848 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8853 OMPLexicalScope
Scope(CGF, S, OMPD_parallel,
false);
8854 CGM.getOpenMPRuntime().emitMaskedRegion(CGF, TaskLoopCodeGen,
8866 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8871 OMPLexicalScope
Scope(CGF, S, OMPD_parallel,
false);
8872 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
8884 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8889 OMPLexicalScope
Scope(CGF, S, OMPD_parallel,
false);
8890 CGM.getOpenMPRuntime().emitMaskedRegion(CGF, TaskLoopCodeGen,
8904 if (
CGM.getLangOpts().OMPTargetTriples.empty())
8908 const Expr *IfCond =
nullptr;
8910 IfCond =
C->getCondition();
8917 OMPLexicalScope
Scope(*
this, S, OMPD_task);
8918 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*
this, S, IfCond,
Device);
8928 BindKind =
C->getBindKind();
8931 case OMPC_BIND_parallel:
8933 case OMPC_BIND_teams:
8935 case OMPC_BIND_thread:
8946 const auto *ForS = dyn_cast<ForStmt>(CS);
8957 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
8958 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_loop,
CodeGen);
8984 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF,
PrePostActionTy &) {
8989 auto &&
CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
9001 [](CodeGenFunction &) {
return nullptr; });
9006 std::string StatusMsg,
9010 StatusMsg +=
": DEVICE";
9012 StatusMsg +=
": HOST";
9019 llvm::dbgs() << StatusMsg <<
": " <<
FileName <<
": " << LineNo <<
"\n";
9042 CGF, OMPD_distribute, CodeGenDistribute,
false);
9071 CGF, OMPD_distribute, CodeGenDistribute,
false);
9104 llvm::Constant *
Addr;
9106 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
9108 assert(Fn &&
Addr &&
9109 "Target device function emission failed for 'target teams loop'.");
9120 CGF, OMPD_target_parallel_loop,
false);
9136 llvm::Constant *
Addr;
9138 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
9140 assert(Fn &&
Addr &&
"Target device function emission failed.");
9155 if (
const auto *SD = dyn_cast<OMPScanDirective>(&D)) {
9160 if (
const auto *MD = dyn_cast<OMPMetaDirective>(&D)) {
9164 if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
9170 for (
const auto *
C : D.getClausesOfKind<OMPFirstprivateClause>()) {
9171 for (
const Expr *Ref :
C->varlist()) {
9175 const auto *VD = dyn_cast<VarDecl>(DRE->getDecl());
9178 if (!CGF.LocalDeclMap.count(VD)) {
9190 if (
const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
9191 for (
const Expr *E : LD->counters()) {
9199 if (!CGF.LocalDeclMap.count(VD))
9203 for (
const auto *
C : D.getClausesOfKind<OMPOrderedClause>()) {
9204 if (!
C->getNumForLoops())
9206 for (
unsigned I = LD->getLoopsNumber(),
9207 E =
C->getLoopNumIterations().size();
9209 if (
const auto *VD = dyn_cast<OMPCapturedExprDecl>(
9212 if (!CGF.LocalDeclMap.count(VD))
9219 CGF.
EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt());
9222 if (D.getDirectiveKind() == OMPD_atomic ||
9223 D.getDirectiveKind() == OMPD_critical ||
9224 D.getDirectiveKind() == OMPD_section ||
9225 D.getDirectiveKind() == OMPD_master ||
9226 D.getDirectiveKind() == OMPD_masked ||
9227 D.getDirectiveKind() == OMPD_assume ||
9233 OMPSimdLexicalScope
Scope(*
this, D);
9234 CGM.getOpenMPRuntime().emitInlinedDirective(
9237 : D.getDirectiveKind(),
9245 for (
const auto *
C : S.getClausesOfKind<OMPHoldsClause>()) {
9246 const Expr *E =
C->getExpr();
9247 assert(E &&
"holds clause requires an expression");
Defines the clang::ASTContext interface.
static bool isAllocatableDecl(const VarDecl *VD)
static const VarDecl * getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE)
static SourceRange getAssociatedLoopRange(const OMPExecutableDirective &S)
A directive's own source range covers only the pragma, so use the range of the associated loop,...
static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, PrePostActionTy &Action)
static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, PrePostActionTy &Action)
static const VarDecl * getBaseDecl(const Expr *Ref)
static void emitTargetTeamsGenericLoopRegionAsParallel(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsGenericLoopDirective &S)
static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *V, SourceLocation Loc)
static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, bool IsPostfixUpdate, const Expr *V, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
static void emitScanBasedDirective(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen, llvm::function_ref< void(CodeGenFunction &)> FirstGen, llvm::function_ref< void(CodeGenFunction &)> SecondGen)
Emits the code for the directive with inscan reductions.
static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO, LValue LVal, RValue RVal)
static bool isSupportedByOpenMPIRBuilder(const OMPTaskgroupDirective &T)
static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, QualType DstType, StringRef Name, LValue AddrLV)
static void emitLoopNestTransformation(CodeGenFunction &CGF, const OMPCanonicalLoopNestTransformationDirective &S)
Emit a loop-nest transformation followed by its loop variable finalization, which restores the loop-e...
static bool canEmitGPUFusedDistSchedule(const CodeGenModule &CGM, const OMPLoopDirective &S, OpenMPDirectiveKind DKind)
Whether a combined distribute parallel for may use the fused distr_static_chunk + static_chunkone sch...
static void emitDistributeParallelForDistributeInnerBoundParams(CodeGenFunction &CGF, const OMPExecutableDirective &S, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars)
static void emitScanBasedDirectiveFinals(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen)
Copies final inscan reductions values to the original variables.
static void checkForLastprivateConditionalUpdate(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static std::pair< LValue, LValue > emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
The following two functions generate expressions for the loop lower and upper bounds in case of stati...
static void emitTargetParallelForRegion(CodeGenFunction &CGF, const OMPTargetParallelForDirective &S, PrePostActionTy &Action)
static llvm::Function * emitOutlinedFunctionPrologueAggregate(CodeGenFunction &CGF, FunctionArgList &Args, llvm::MapVector< const Decl *, std::pair< const VarDecl *, Address > > &LocalAddrs, llvm::DenseMap< const Decl *, std::pair< const Expr *, llvm::Value * > > &VLASizes, llvm::Value *&CXXThisValue, llvm::Value *&ContextV, const CapturedStmt &CS, SourceLocation Loc, StringRef FunctionName)
static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper)
Emit a helper variable and return corresponding lvalue.
static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
static llvm::Value * convertToScalarValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static llvm::Function * emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S, const OMPExecutableDirective &D)
static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
static std::pair< bool, RValue > emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, RValue Update, BinaryOperatorKind BO, llvm::AtomicOrdering AO, bool IsXLHSInRHSPart)
static std::pair< LValue, LValue > emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitTargetTeamsGenericLoopRegionAsDistribute(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsGenericLoopDirective &S)
static void emitTargetParallelRegion(CodeGenFunction &CGF, const OMPTargetParallelDirective &S, PrePostActionTy &Action)
static std::pair< llvm::Value *, llvm::Value * > emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
When dealing with dispatch schedules (e.g.
static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitRestoreIP(CodeGenFunction &CGF, const T *C, llvm::OpenMPIRBuilder::InsertPointTy AllocaIP, llvm::OpenMPIRBuilder &OMPBuilder)
static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, const RegionCodeGenTy &CodeGen)
static void emitSimdlenSafelenClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
static void emitAlignedClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
static bool isSimdSupportedByOpenMPIRBuilder(const OMPLoopDirective &S)
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters)
static void applyConservativeSimdOrderedDirective(const Stmt &AssociatedStmt, LoopInfoStack &LoopStack)
static bool emitWorksharingDirective(CodeGenFunction &CGF, const OMPLoopDirective &S, bool HasCancel)
static void emitPostUpdateForReductionClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc, const unsigned IVSize, const bool IVSigned)
static void emitTargetTeamsLoopCodegenStatus(CodeGenFunction &CGF, std::string StatusMsg, const OMPExecutableDirective &D)
static bool isForSupportedByOpenMPIRBuilder(const OMPLoopDirective &S, bool HasCancel)
static RValue emitSimpleAtomicLoad(CodeGenFunction &CGF, llvm::AtomicOrdering AO, LValue LVal, SourceLocation Loc)
static std::pair< llvm::Value *, llvm::Value * > emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
if the 'for' loop has a dispatch schedule (e.g.
static bool hasOrderedBlockAssocDirective(const Stmt *S)
static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *UE, const Expr *D, const Expr *CE, bool IsXLHSInRHSPart, bool IsFailOnly, SourceLocation Loc)
static CodeGenFunction::ComplexPairTy convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static ImplicitParamDecl * createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data, QualType Ty, CapturedDecl *CD, SourceLocation Loc)
static EmittedClosureTy emitCapturedStmtFunc(CodeGenFunction &ParentCGF, const CapturedStmt *S)
Emit a captured statement and return the function as well as its captured closure context.
static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
static void emitOMPDistributeDirective(const OMPLoopDirective &S, CodeGenFunction &CGF, CodeGenModule &CGM)
static void emitOMPCopyinClause(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitTargetTeamsDistributeParallelForRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, PrePostActionTy &Action)
static llvm::CallInst * emitCapturedStmtCall(CodeGenFunction &ParentCGF, EmittedClosureTy Cap, llvm::ArrayRef< llvm::Value * > Args)
Emit a call to a previously captured closure.
static void emitMasked(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop, int MaxLevel, int Level=0)
static void emitOMPForDirective(const OMPLoopDirective &S, CodeGenFunction &CGF, CodeGenModule &CGM, bool HasCancel)
static void emitEmptyBoundParameters(CodeGenFunction &, const OMPExecutableDirective &, llvm::SmallVectorImpl< llvm::Value * > &)
static void emitTargetParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetParallelForSimdDirective &S, PrePostActionTy &Action)
static void emitOMPSimdDirective(const OMPLoopDirective &S, CodeGenFunction &CGF, CodeGenModule &CGM)
static void emitOMPAtomicCompareExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO, const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *D, const Expr *CE, bool IsXBinopExpr, bool IsPostfixUpdate, bool IsFailOnly, SourceLocation Loc)
std::pair< llvm::Function *, llvm::Value * > EmittedClosureTy
static OpenMPDirectiveKind getEffectiveDirectiveKind(const OMPExecutableDirective &S)
static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDirective &S)
static void buildDependences(const OMPExecutableDirective &S, OMPTaskDataTy &Data)
static RValue convertToType(CodeGenFunction &CGF, RValue Value, QualType SourceType, QualType ResType, SourceLocation Loc)
static void emitScanBasedDirectiveDecls(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen)
Emits internal temp array declarations for the directive with inscan reductions.
static void emitTargetTeamsDistributeParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForSimdDirective &S, PrePostActionTy &Action)
static void emitTargetTeamsDistributeSimdRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeSimdDirective &S)
static llvm::MapVector< llvm::Value *, llvm::Value * > GetAlignedMapping(const OMPLoopDirective &S, CodeGenFunction &CGF)
static llvm::omp::ScheduleKind convertClauseKindToSchedKind(OpenMPScheduleClauseKind ScheduleClauseKind)
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
static void emitTargetParallelGenericLoopRegion(CodeGenFunction &CGF, const OMPTargetParallelGenericLoopDirective &S, PrePostActionTy &Action)
static QualType getCanonicalParamType(ASTContext &C, QualType T)
static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S, const RegionCodeGenTy &SimdInitGen, const RegionCodeGenTy &BodyCodeGen)
static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, const Twine &Name, llvm::Value *Init=nullptr)
static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *E, SourceLocation Loc)
static llvm::Function * emitOutlinedFunctionPrologue(CodeGenFunction &CGF, FunctionArgList &Args, llvm::MapVector< const Decl *, std::pair< const VarDecl *, Address > > &LocalAddrs, llvm::DenseMap< const Decl *, std::pair< const Expr *, llvm::Value * > > &VLASizes, llvm::Value *&CXXThisValue, const FunctionOptions &FO)
static void emitInnerParallelForWhenCombined(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
static void emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeDirective &S)
This file defines OpenMP nodes for declarative directives.
static const Decl * getCanonicalDecl(const Decl *D)
This file defines OpenMP AST classes for clauses.
Defines some OpenMP-specific enums and functions.
Defines the PrettyStackTraceEntry class, which is used to make crashes give more contextual informati...
Defines the SourceManager interface.
This file defines OpenMP AST classes for executable directives and clauses.
This represents clause 'aligned' in the 'pragma omp ...' directives.
This represents 'pragma omp atomic' directive.
Expr * getR()
Get 'r' part of the associated expression/statement.
Expr * getX()
Get 'x' part of the associated expression/statement.
bool isFailOnly() const
Return true if 'v' is updated only when the condition is evaluated false (compare capture only).
bool isPostfixUpdate() const
Return true if 'v' expression must be updated to original value of 'x', false if 'v' must be updated ...
Expr * getExpr()
Get 'expr' part of the associated expression/statement.
Expr * getV()
Get 'v' part of the associated expression/statement.
bool isXLHSInRHSPart() const
Return true if helper update expression has form 'OpaqueValueExpr(x) binop OpaqueValueExpr(expr)' and...
Expr * getD()
Get 'd' part of the associated expression/statement.
Expr * getUpdateExpr()
Get helper expression of the form 'OpaqueValueExpr(x) binop OpaqueValueExpr(expr)' or 'OpaqueValueExp...
Expr * getCondExpr()
Get the 'cond' part of the source atomic expression.
This represents 'pragma omp barrier' directive.
This represents 'bind' clause in the 'pragma omp ...' directives.
This represents 'pragma omp cancel' directive.
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
This represents 'pragma omp cancellation point' directive.
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
This represents clause 'copyin' in the 'pragma omp ...' directives.
This represents clause 'copyprivate' in the 'pragma omp ...' directives.
This represents 'pragma omp critical' directive.
DeclarationNameInfo getDirectiveName() const
Return name of the directive.
This represents implicit clause 'depend' for the 'pragma omp task' directive.
This represents implicit clause 'depobj' for the 'pragma omp depobj' directive. This clause does not ...
This represents 'pragma omp depobj' directive.
This represents 'destroy' clause in the 'pragma omp depobj' directive or the 'pragma omp interop' dir...
This represents 'device' clause in the 'pragma omp ...' directive.
This represents 'dist_schedule' clause in the 'pragma omp ...' directive.
This represents 'pragma omp distribute' directive.
This represents 'pragma omp distribute parallel for' composite directive.
This represents 'pragma omp distribute parallel for simd' composite directive.
This represents 'pragma omp distribute simd' composite directive.
This represents the 'doacross' clause for the 'pragma omp ordered' directive.
This represents 'pragma omp error' directive.
This represents 'filter' clause in the 'pragma omp ...' directive.
Represents the 'pragma omp flatten' loop transformation directive.
Stmt * getTransformedStmt() const
Gets the flattened loop after the transformation.
Stmt * getFinals() const
Return updates to the original loop variables after the flattened loop.
This represents implicit clause 'flush' for the 'pragma omp flush' directive. This clause does not ex...
This represents 'pragma omp flush' directive.
This represents 'pragma omp for' directive.
bool hasCancel() const
Return true if current directive has inner cancel directive.
This represents 'pragma omp for simd' directive.
Represents the 'pragma omp fuse' loop transformation directive.
Stmt * getFinals() const
Return finals statement.
Stmt * getTransformedStmt() const
Gets the associated loops after the transformation.
This represents 'pragma omp loop' directive.
This represents 'grainsize' clause in the 'pragma omp ...' directive.
This represents 'hint' clause in the 'pragma omp ...' directive.
This represents clause 'inclusive' in the 'pragma omp scan' directive.
This represents the 'init' clause in 'pragma omp ...' directives.
Represents the 'pragma omp interchange' loop transformation directive.
This represents 'pragma omp interop' directive.
This is a common base class for loop directives ('omp simd', 'omp for', 'omp for simd' etc....
Expr * getCombinedUpperBoundVariable() const
Expr * getPreCond() const
Expr * getPrevUpperBoundVariable() const
Expr * getIsLastIterVariable() const
Expr * getCombinedLowerBoundVariable() const
Expr * getCombinedCond() const
Expr * getCombinedInit() const
Expr * getLowerBoundVariable() const
Expr * getCombinedNextLowerBound() const
ArrayRef< Expr * > finals_conditions()
ArrayRef< Expr * > counters()
ArrayRef< Expr * > private_counters()
Expr * getUpperBoundVariable() const
Expr * getCombinedDistCond() const
Expr * getPrevLowerBoundVariable() const
ArrayRef< Expr * > dependent_inits()
Expr * getNextLowerBound() const
Expr * getDistInc() const
ArrayRef< Expr * > updates()
Expr * getNextUpperBound() const
Expr * getLastIteration() const
Expr * getEnsureUpperBound() const
Expr * getCalcLastIteration() const
Expr * getCombinedNextUpperBound() const
Expr * getIterationVariable() const
Expr * getStrideVariable() const
Expr * getNumIterations() const
ArrayRef< Expr * > finals()
Expr * getPrevEnsureUpperBound() const
Expr * getCombinedEnsureUpperBound() const
ArrayRef< Expr * > dependent_counters()
ArrayRef< Expr * > inits()
This represents 'pragma omp masked' directive.
This represents 'pragma omp masked taskloop' directive.
This represents 'pragma omp masked taskloop simd' directive.
This represents 'pragma omp master' directive.
This represents 'pragma omp master taskloop' directive.
This represents 'pragma omp master taskloop simd' directive.
This represents 'nogroup' clause in the 'pragma omp ...' directive.
This represents 'num_tasks' clause in the 'pragma omp ...' directive.
This represents 'num_teams' clause in the 'pragma omp ...' directive.
This represents 'order' clause in the 'pragma omp ...' directive.
This represents block-associated 'pragma omp ordered' directive.
This represents standalone 'pragma omp ordered' directive.
This represents 'pragma omp parallel for' directive.
bool hasCancel() const
Return true if current directive has inner cancel directive.
This represents 'pragma omp parallel for simd' directive.
This represents 'pragma omp parallel masked' directive.
This represents 'pragma omp parallel masked taskloop' directive.
This represents 'pragma omp parallel masked taskloop simd' directive.
This represents 'pragma omp parallel master' directive.
This represents 'pragma omp parallel master taskloop' directive.
This represents 'pragma omp parallel master taskloop simd' directive.
This represents 'pragma omp parallel sections' directive.
This represents 'priority' clause in the 'pragma omp ...' directive.
Represents the 'pragma omp reverse' loop transformation directive.
This represents 'simd' clause in the 'pragma omp ...' directive.
This represents 'pragma omp scan' directive.
This represents 'pragma omp scope' directive.
This represents 'pragma omp section' directive.
This represents 'pragma omp sections' directive.
bool hasCancel() const
Return true if current directive has inner cancel directive.
This represents 'pragma omp simd' directive.
This represents 'pragma omp single' directive.
Represents the 'pragma omp split' loop transformation directive.
This represents the 'pragma omp stripe' loop transformation directive.
This represents 'pragma omp target data' directive.
This represents 'pragma omp target' directive.
This represents 'pragma omp target enter data' directive.
This represents 'pragma omp target exit data' directive.
This represents 'pragma omp target parallel' directive.
This represents 'pragma omp target parallel for' directive.
bool hasCancel() const
Return true if current directive has inner cancel directive.
This represents 'pragma omp target parallel for simd' directive.
This represents 'pragma omp target parallel loop' directive.
This represents 'pragma omp target teams' directive.
This represents 'pragma omp target teams distribute' combined directive.
This represents 'pragma omp target teams distribute parallel for' combined directive.
This represents 'pragma omp target teams distribute parallel for simd' combined directive.
This represents 'pragma omp target teams distribute simd' combined directive.
This represents 'pragma omp target teams loop' directive.
bool canBeParallelFor() const
Return true if current loop directive's associated loop can be a parallel for.
This represents 'pragma omp target update' directive.
This represents 'pragma omp task' directive.
This represents 'pragma omp taskloop' directive.
This represents 'pragma omp taskloop simd' directive.
This represents 'pragma omp taskgroup' directive.
const Expr * getReductionRef() const
Returns reference to the task_reduction return variable.
This represents 'pragma omp taskwait' directive.
This represents 'pragma omp taskyield' directive.
This represents 'pragma omp teams' directive.
This represents 'pragma omp teams distribute' directive.
This represents 'pragma omp teams distribute parallel for' composite directive.
This represents 'pragma omp teams distribute parallel for simd' composite directive.
This represents 'pragma omp teams distribute simd' combined directive.
This represents 'pragma omp teams loop' directive.
This represents 'thread_limit' clause in the 'pragma omp ...' directive.
This represents the 'pragma omp tile' loop transformation directive.
This represents the 'pragma omp unroll' loop transformation directive.
This represents the 'use' clause in 'pragma omp ...' directives.
This represents clause 'use_device_addr' in the 'pragma omp ...' directives.
This represents clause 'use_device_ptr' in the 'pragma omp ...' directives.
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
SourceManager & getSourceManager()
TranslationUnitDecl * getTranslationUnitDecl() const
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
QualType getUIntPtrType() const
Return a type compatible with "uintptr_t" (C99 7.18.1.4), as defined by the target.
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth,...
TypeSourceInfo * getTrivialTypeSourceInfo(QualType T, SourceLocation Loc=SourceLocation()) const
Allocate a TypeSourceInfo where all locations have been initialized to a given location,...
unsigned getOpenMPDefaultSimdAlign(QualType T) const
Get default simd alignment of the specified complete type in bits.
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
QualType getFunctionType(QualType ResultTy, ArrayRef< QualType > Args, const FunctionProtoType::ExtProtoInfo &EPI) const
Return a normal function type with a typed argument list.
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
CanQualType getCanonicalTagType(const TagDecl *TD) const
ASTRecordLayout - This class contains layout information for one RecordDecl, which is a struct/union/...
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Represents an attribute applied to a statement.
ArrayRef< const Attr * > getAttrs() const
static BinaryOperator * Create(const ASTContext &C, Expr *lhs, Expr *rhs, Opcode opc, QualType ResTy, ExprValueKind VK, ExprObjectKind OK, SourceLocation opLoc, FPOptionsOverride FPFeatures)
A binding in a decomposition declaration.
DecompositionDecl * getDecomposedDecl() const
Get the decomposition declaration that this binding represents a decomposition of.
Represents the body of a CapturedStmt, and serves as its DeclContext.
unsigned getNumParams() const
ImplicitParamDecl * getContextParam() const
Retrieve the parameter containing captured variables.
unsigned getContextParamPosition() const
static CapturedDecl * Create(ASTContext &C, DeclContext *DC, unsigned NumParams)
param_iterator param_end() const
Retrieve an iterator one past the last parameter decl.
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
Stmt * getBody() const override
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
ImplicitParamDecl * getParam(unsigned i) const
This captures a statement into a function.
SourceLocation getEndLoc() const LLVM_READONLY
CapturedDecl * getCapturedDecl()
Retrieve the outlined function declaration.
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Stmt * getCapturedStmt()
Retrieve the statement being captured.
capture_init_iterator capture_init_begin()
Retrieve the first initialization argument.
SourceLocation getBeginLoc() const LLVM_READONLY
capture_init_iterator capture_init_end()
Retrieve the iterator pointing one past the last initialization argument.
Expr *const * const_capture_init_iterator
Const iterator that walks over the capture initialization arguments.
This is an opaque type for sizes expressed in character units.
CharUnits alignTo(CharUnits Align) const
Returns the next integer (mod 2**64) that is greater than or equal to this quantity and is a multiple...
QuantityType getQuantity() const
Get the raw integer representation of this quantity.
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
static CharUnits fromQuantity(QuantityType Quantity)
Construct a CharUnits quantity from a raw integer type.
std::string SampleProfileFile
Name of the profile file to use with -fprofile-sample-use.
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
CharUnits getAlignment() const
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Address withAlignment(CharUnits NewAlignment) const
Return address with different alignment, but same pointer and element type.
llvm::PointerType * getType() const
Return the type of the pointer value.
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored.
static ApplyDebugLocation CreateDefaultArtificial(CodeGenFunction &CGF, SourceLocation TemporaryLocation)
Apply TemporaryLocation if it is valid.
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
llvm::LoadInst * CreateAlignedLoad(llvm::Type *Ty, llvm::Value *Addr, CharUnits Align, const llvm::Twine &Name="")
CGFunctionInfo - Class to encapsulate the information about a function definition.
Manages list of lastprivate conditional decls for the specified directive.
static LastprivateConditionalRAII disable(CodeGenFunction &CGF, const OMPExecutableDirective &S)
Manages list of nontemporal decls for the specified directive.
Struct that keeps all the relevant information that should be kept throughout a 'target data' region.
llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap
Map between the a declaration of a capture and the corresponding new llvm address where the runtime r...
Manages list of nontemporal decls for the specified directive.
virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
Emit task region for the task directive.
virtual llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST)
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond, llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, const Expr *Message=nullptr)
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
virtual void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc, OpenMPNumThreadsClauseModifier Modifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, SourceLocation SeverityLoc=SourceLocation(), const Expr *Message=nullptr, SourceLocation MessageLoc=SourceLocation())
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32global_tid, kmp_int32 num_threads) ...
StringRef getTargetRegionParentName(const FunctionDecl *FD) const
Returns the parent name of the target regions in FD: its enclosing non-closure function if FD is a cl...
virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal)
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
virtual void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc)
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
virtual void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars)
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
virtual const VarDecl * translateParameter(const FieldDecl *FD, const VarDecl *NativeParam) const
Translates the native parameter of outlined function if this is required for target.
virtual llvm::Function * emitTeamsOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP teams directive D.
virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr * > NumIterations)
Emit initialization for doacross loop nesting support.
virtual void adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF, const OMPExecutableDirective &D) const
Adjust some parameters for the target-based directives, like addresses of the variables captured by r...
virtual Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const
Gets the address of the native argument basing on the address of the target-specific parameter.
virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32global_tid, kmp_int32 num_teams,...
virtual llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data)
Emit a code for initialization of task reduction clause.
virtual void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc, llvm::AtomicOrdering AO)
Emit flush of the variables specified in 'omp flush' directive.
virtual void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc)
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32global_tid, int proc_bind) to generate...
virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false)
Emit an implicit/explicit barrier for OpenMP threads.
virtual void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values)
virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind)
Call the appropriate runtime routine to notify that we finished all the work with current loop.
void emitIfClause(CodeGenFunction &CGF, const Expr *Cond, const RegionCodeGenTy &ThenGen, const RegionCodeGenTy &ElseGen)
Emits code for OpenMP 'if' clause using specified CodeGen function.
virtual llvm::Function * emitParallelOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP parallel directive D.
virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values)
Call the appropriate runtime routine to initialize it before start of loop.
virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static non-chunked.
virtual void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc)
Emits a master region.
virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Required to resolve existing problems in the runtime.
virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF, const Expr *LHS)
Checks if the provided LVal is lastprivate conditional and emits the code to update the value of the ...
llvm::OpenMPIRBuilder & getOMPBuilder()
virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Emit outilined function for 'target' directive.
virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned)
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
virtual void checkAndEmitSharedLastprivateConditional(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::DenseSet< CanonicalDeclPtr< const VarDecl > > &IgnoredDecls)
Checks if the lastprivate conditional was updated in inner region and writes the value.
virtual void emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool HasCancel=false)
Emit code for the directive that does not require outlining.
virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static chunked.
virtual void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Emit the target offloading code associated with D.
virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const
Check if the specified ScheduleKind is dynamic.
virtual void emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter=nullptr)
Emits a masked region.
virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues)
Call the appropriate runtime routine to initialize it before start of loop.
Address getAllocatedAddress() const
Returns the raw, allocated address, which is not necessarily the address of the object itself.
API for captured statement code generation.
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
RAII for correct setting/restoring of CapturedStmtInfo.
LValue getReferenceLValue(CodeGenFunction &CGF, const Expr *RefExpr) const
void ForceCleanup()
Force the emission of cleanups now, instead of waiting until this object is destroyed.
RAII for preserving necessary info during inlined region body codegen.
Cleanup action for allocate support.
RAII for preserving necessary info during Outlined region body codegen.
Controls insertion of cancellation exit blocks in worksharing constructs.
Save/restore original map of previously emitted local vars in case when we need to duplicate emission...
The class used to assign some variables some temporarily addresses.
bool setVarAddr(CodeGenFunction &CGF, const ValueDecl *LocalVD, Address TempAddr)
Sets the address of the variable LocalVD to be TempAddr in function CGF.
bool apply(CodeGenFunction &CGF)
Applies new addresses to the list of the variables.
void restore(CodeGenFunction &CGF)
Restores original addresses of the variables.
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
void restoreMap()
Restore all mapped variables w/o clean up.
bool Privatize()
Privatizes local variables previously registered as private.
bool addPrivate(const ValueDecl *LocalVD, Address Addr)
Registers LocalVD variable as a private with Addr as the address of the corresponding private variabl...
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
Manages parent directive for scan directives.
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited.
void ForceCleanup(std::initializer_list< llvm::Value ** > ValuesToReload={})
Force the emission of cleanups now, instead of waiting until this object is destroyed.
bool requiresCleanups() const
Determine whether this scope requires any cleanups.
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
void EmitOMPParallelMaskedTaskLoopDirective(const OMPParallelMaskedTaskLoopDirective &S)
void EmitOMPParallelMaskedDirective(const OMPParallelMaskedDirective &S)
void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S)
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount, Stmt::Likelihood LH=Stmt::LH_None, const Expr *ConditionalOp=nullptr, const VarDecl *ConditionalDecl=nullptr)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D, bool NoFinals, llvm::Value *IsLastIterCond=nullptr)
Emit final copying of lastprivate values to original variables at the end of the worksharing or simd ...
void processInReduction(const OMPExecutableDirective &S, OMPTaskDataTy &Data, CodeGenFunction &CGF, const CapturedStmt *CS, OMPPrivateScope &Scope)
llvm::Value * performAddrSpaceCast(llvm::Value *Src, llvm::Type *DestTy)
JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target)
The given basic block lies in the current EH scope, but may be a target of a potentially scope-crossi...
void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S)
void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy, SourceLocation Loc)
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
void EmitOMPCanonicalLoop(const OMPCanonicalLoop *S)
Emit an OMPCanonicalLoop using the OpenMPIRBuilder.
void EmitOMPGenericLoopDirective(const OMPGenericLoopDirective &S)
void EmitOMPScanDirective(const OMPScanDirective &S)
static bool hasScalarEvaluationKind(QualType T)
llvm::Type * ConvertType(QualType T)
llvm::function_ref< std::pair< llvm::Value *, llvm::Value * >(CodeGenFunction &, const OMPExecutableDirective &S, Address LB, Address UB)> CodeGenDispatchBoundsTy
LValue InitCapturedStruct(const CapturedStmt &S)
void EmitOMPFlattenDirective(const OMPFlattenDirective &S)
CGCapturedStmtInfo * CapturedStmtInfo
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
void EmitOMPParallelForDirective(const OMPParallelForDirective &S)
void EmitOMPMasterDirective(const OMPMasterDirective &S)
void EmitOMPParallelMasterTaskLoopSimdDirective(const OMPParallelMasterTaskLoopSimdDirective &S)
void EmitOMPSimdInit(const OMPLoopDirective &D)
Helpers for the OpenMP loop directives.
const OMPExecutableDirective * OMPParentLoopDirectiveForScan
Parent loop-based directive for scan directive.
void EmitOMPFlushDirective(const OMPFlushDirective &S)
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S)
void EmitOMPTargetTeamsDistributeParallelForSimdDirective(const OMPTargetTeamsDistributeParallelForSimdDirective &S)
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
void EmitOMPReductionClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope, bool ForInscan=false)
Emit initial code for reduction variables.
void eraseLocalDecl(const Decl *D)
void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S)
void EmitAutoVarDecl(const VarDecl &D)
EmitAutoVarDecl - Emit an auto variable declaration.
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
llvm::Function * GenerateOpenMPCapturedStmtFunctionAggregate(const CapturedStmt &S, const OMPExecutableDirective &D)
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
void EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S)
const LangOptions & getLangOpts() const
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
EmitAutoVarAlloca - Emit the alloca and debug information for a local variable.
void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO, const llvm::function_ref< RValue(RValue)> &UpdateOp, bool IsVolatile)
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPLoopDirective &S)
Emit device code for the target simd and target loop directives.
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Load a pointer with type PtrTy stored at address Ptr.
void EmitOMPSplitDirective(const OMPSplitDirective &S)
void EmitBranchThroughCleanup(JumpDest Dest)
EmitBranchThroughCleanup - Emit a branch from the current insert block through the normal cleanup han...
void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind)
Emit final update of reduction values to original variables at the end of the directive.
void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit)
Helper for the OpenMP loop directives.
void EmitOMPScopeDirective(const OMPScopeDirective &S)
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
llvm::AssertingVH< llvm::Instruction > AllocaInsertPt
AllocaInsertPoint - This is an instruction in the entry block before which we prefer to insert alloca...
void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy)
Emit an aggregate assignment.
CodeGenFunction * ParentCGF
JumpDest ReturnBlock
ReturnBlock - Unified return block.
void EmitOMPTargetTeamsDistributeSimdDirective(const OMPTargetTeamsDistributeSimdDirective &S)
const llvm::function_ref< void(CodeGenFunction &, llvm::Function *, const OMPTaskDataTy &)> TaskGenTy
llvm::DebugLoc SourceLocToDebugLoc(SourceLocation Location)
Converts Location to a DebugLoc, if debug information is enabled.
bool EmitOMPCopyinClause(const OMPExecutableDirective &D)
Emit code for copyin clause in D directive.
void EmitOMPLinearClause(const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope)
Emit initial code for linear clauses.
llvm::BasicBlock * OMPBeforeScanBlock
void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S)
void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S, OMPPrivateScope &LoopScope)
Emit initial code for loop counters of loop-based directives.
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value * > &CapturedVars)
void EmitOMPDepobjDirective(const OMPDepobjDirective &S)
void EmitOMPMetaDirective(const OMPMetaDirective &S)
void EmitOMPCriticalDirective(const OMPCriticalDirective &S)
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S)
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
void EmitOMPCancelDirective(const OMPCancelDirective &S)
void EmitOMPBarrierDirective(const OMPBarrierDirective &S)
llvm::Value * EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified complex type to the specified destination type,...
bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB, const CodeGenLoopBoundsTy &CodeGenLoopBounds, const CodeGenDispatchBoundsTy &CGDispatchBounds)
Emit code for the worksharing loop-based directive.
LValue EmitOMPSharedLValue(const Expr *E)
Emits the lvalue for the expression with possibly captured variable.
llvm::CanonicalLoopInfo * EmitOMPCollapsedCanonicalLoopNest(const Stmt *S, int Depth)
Emit the Stmt S and return its topmost canonical loop, if any.
void EmitOMPSectionsDirective(const OMPSectionsDirective &S)
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
void EmitOMPInteropDirective(const OMPInteropDirective &S)
void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S)
void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S)
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
void EmitOMPTargetParallelForSimdDirective(const OMPTargetParallelForSimdDirective &S)
void EmitOMPTargetParallelGenericLoopDirective(const OMPTargetParallelGenericLoopDirective &S)
Emit combined directive 'target parallel loop' as if its constituent constructs are 'target',...
void EmitOMPUseDeviceAddrClause(const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap)
void EmitOMPTeamsDistributeParallelForSimdDirective(const OMPTeamsDistributeParallelForSimdDirective &S)
void EmitOMPMaskedDirective(const OMPMaskedDirective &S)
llvm::Value * emitArrayLength(const ArrayType *arrayType, QualType &baseType, Address &addr)
emitArrayLength - Compute the length of an array, even if it's a VLA, and drill down to the base elem...
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
void EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S)
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
void EmitOMPOrderedBlockAssocDirective(const OMPOrderedBlockAssocDirective &S)
CGDebugInfo * getDebugInfo()
void EmitOMPDistributeLoop(const OMPLoopDirective &S, const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr)
Emit code for the distribute loop-based directive.
void EmitOMPMasterTaskLoopDirective(const OMPMasterTaskLoopDirective &S)
void EmitOMPReverseDirective(const OMPReverseDirective &S)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
void EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S)
void EmitOMPTargetTeamsDistributeParallelForDirective(const OMPTargetTeamsDistributeParallelForDirective &S)
void EmitOMPMaskedTaskLoopDirective(const OMPMaskedTaskLoopDirective &S)
llvm::BasicBlock * OMPScanDispatch
llvm::function_ref< std::pair< LValue, LValue >(CodeGenFunction &, const OMPExecutableDirective &S)> CodeGenLoopBoundsTy
void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S)
RawAddress CreateMemTempWithoutCast(QualType T, const Twine &Name="tmp")
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen without...
void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S)
void EmitOMPMaskedTaskLoopSimdDirective(const OMPMaskedTaskLoopSimdDirective &S)
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
VlaSizePair getVLASize(const VariableArrayType *vla)
Returns an LLVM value that corresponds to the size, in non-variably-sized elements,...
void EmitOMPParallelDirective(const OMPParallelDirective &S)
void EmitOMPTaskDirective(const OMPTaskDirective &S)
void EmitOMPMasterTaskLoopSimdDirective(const OMPMasterTaskLoopSimdDirective &S)
void EmitOMPDistributeParallelForDirective(const OMPDistributeParallelForDirective &S)
void EmitOMPAssumeDirective(const OMPAssumeDirective &S)
int ExpectedOMPLoopDepth
Number of nested loop to be consumed by the last surrounding loop-associated directive.
void EmitOMPPrivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S)
ASTContext & getContext() const
void EmitStopPoint(const Stmt *S)
EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S)
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
void EmitOMPTargetTeamsGenericLoopDirective(const OMPTargetTeamsGenericLoopDirective &S)
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
void EmitAutoVarCleanups(const AutoVarEmission &emission)
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
SmallVector< llvm::CanonicalLoopInfo *, 4 > OMPLoopNestStack
List of recently emitted OMPCanonicalLoops.
Address EmitOMPBindingOriginalAddr(const BindingDecl *BD, SourceLocation Loc)
Emits the original address for a structured binding.
void EmitOMPTeamsDistributeParallelForDirective(const OMPTeamsDistributeParallelForDirective &S)
llvm::AtomicRMWInst * emitAtomicRMWInst(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Order=llvm::AtomicOrdering::SequentiallyConsistent, llvm::SyncScope::ID SSID=llvm::SyncScope::System, const AtomicExpr *AE=nullptr)
Emit an atomicrmw instruction, and applying relevant metadata when applicable.
void EmitOMPFuseDirective(const OMPFuseDirective &S)
void EmitOMPTargetTeamsDistributeDirective(const OMPTargetTeamsDistributeDirective &S)
void EmitOMPUseDevicePtrClause(const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap)
LValue EmitOMPCapturedBindingLValue(const BindingDecl *BD)
Emit an LValue for a structured binding captured in an OpenMP region.
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
EmitAnyExpr - Emit code to compute the specified expression which can have any type.
void EmitStmt(const Stmt *S, ArrayRef< const Attr * > Attrs={})
EmitStmt - Emit the code for the statement.
llvm::DenseMap< const ValueDecl *, FieldDecl * > LambdaCaptureFields
void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S)
llvm::Type * ConvertTypeForMem(QualType T)
void EmitOMPInnerLoop(const OMPExecutableDirective &S, bool RequiresCleanup, const Expr *LoopCond, const Expr *IncExpr, const llvm::function_ref< void(CodeGenFunction &)> BodyGen, const llvm::function_ref< void(CodeGenFunction &)> PostIncGen)
Emit inner loop of the worksharing/simd construct.
void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S)
static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S)
void EmitOMPTargetDirective(const OMPTargetDirective &S)
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
llvm::BasicBlock * OMPScanExitBlock
void EmitOMPTeamsDirective(const OMPTeamsDirective &S)
void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D)
Emit simple code for OpenMP directives in Simd-only mode.
void EmitOMPErrorDirective(const OMPErrorDirective &S)
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
void EmitOMPParallelMaskedTaskLoopSimdDirective(const OMPParallelMaskedTaskLoopSimdDirective &S)
void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S)
void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S)
Address GenerateCapturedStmtArgument(const CapturedStmt &S)
bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for lastprivate variables.
static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Emit device code for the target teams distribute parallel for simd directive.
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block,...
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S, const OMPExecutableDirective &D)
void EmitOMPSimdDirective(const OMPSimdDirective &S)
RawAddress CreateMemTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
void EmitOMPParallelGenericLoopDirective(const OMPLoopDirective &S)
void EmitOMPTeamsGenericLoopDirective(const OMPTeamsGenericLoopDirective &S)
void EmitOMPTargetSimdDirective(const OMPLoopDirective &S)
Also emits 'target loop'.
void EmitOMPOrderedStandaloneDirective(const OMPOrderedStandaloneDirective &S)
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
bool EmitOMPLinearClauseInit(const OMPLoopDirective &D)
Emit initial code for linear variables.
static void EmitOMPTargetParallelGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelGenericLoopDirective &S)
Emit device code for the target parallel loop directive.
void EmitOMPUnrollDirective(const OMPUnrollDirective &S)
void EmitOMPStripeDirective(const OMPStripeDirective &S)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
void EmitOMPSingleDirective(const OMPSingleDirective &S)
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
llvm::function_ref< void(CodeGenFunction &, SourceLocation, const unsigned, const bool)> CodeGenOrderedTy
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation.
llvm::Function * GenerateCapturedStmtFunction(const CapturedStmt &S)
Creates the outlined function for a CapturedStmt.
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
uint64_t getProfileCount(const Stmt *S)
Get the profiler's count for the given statement.
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant,...
static void EmitOMPTargetTeamsGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsGenericLoopDirective &S)
Emit device code for the target teams loop directive.
void EmitOMPTileDirective(const OMPTileDirective &S)
void EmitDecl(const Decl &D, bool EvaluateConditionDecl=false)
EmitDecl - Emit a declaration.
void EmitOMPAtomicDirective(const OMPAtomicDirective &S)
llvm::BasicBlock * OMPAfterScanBlock
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
ConstantEmission tryEmitAsConstant(const DeclRefExpr *RefExpr)
Try to emit a reference to the given value without producing it as an l-value.
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
Store of global named registers are always calls to intrinsics.
void EmitOMPParallelMasterTaskLoopDirective(const OMPParallelMasterTaskLoopDirective &S)
void EmitOMPDistributeParallelForSimdDirective(const OMPDistributeParallelForSimdDirective &S)
void EmitOMPSectionDirective(const OMPSectionDirective &S)
void EnsureInsertPoint()
EnsureInsertPoint - Ensure that an insertion point is defined so that emitted IR has a place to go.
void EmitOMPForSimdDirective(const OMPForSimdDirective &S)
llvm::LLVMContext & getLLVMContext()
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler's counter for the given statement by StepV.
void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty, SourceLocation Loc, SourceLocation AssumptionLoc, llvm::Value *Alignment, llvm::Value *OffsetValue=nullptr)
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
llvm::function_ref< void(CodeGenFunction &, const OMPLoopDirective &, JumpDest)> CodeGenLoopTy
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
bool isTrivialInitializer(const Expr *Init)
Determine whether the given initializer is trivial in the sense that it requires no code to be genera...
void EmitOMPParallelMasterDirective(const OMPParallelMasterDirective &S)
void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion, const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen, OMPTaskDataTy &Data)
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
void EmitOMPForDirective(const OMPForDirective &S)
void EmitOMPLinearClauseFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
Emit final code for linear clauses.
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit)
EmitExprAsInit - Emits the code necessary to initialize a location in memory with the given initializ...
void EmitOMPSimdFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
This class organizes the cross-function state that is used while generating LLVM code.
void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
llvm::Module & getModule() const
DiagnosticsEngine & getDiags() const
const LangOptions & getLangOpts() const
CodeGenTypes & getTypes()
const llvm::DataLayout & getDataLayout() const
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
const llvm::Triple & getTriple() const
ASTContext & getContext() const
const CodeGenOptions & getCodeGenOpts() const
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
const CGFunctionInfo & arrangeDeviceKernelCallerDeclaration(QualType resultType, const FunctionArgList &args)
A device kernel caller function is an offload device entry point function with a target device depend...
FunctionArgList - Type for representing both the decl and type of parameters to a function.
LValue - This represents an lvalue references.
llvm::Value * getPointer(CodeGenFunction &CGF) const
Address getAddress() const
void setAddress(Address address)
A stack of loop information corresponding to loop nesting levels.
void setVectorizeWidth(unsigned W)
Set the vectorize width for the next loop pushed.
void setParallel(bool Enable=true)
Set the next pushed loop as parallel.
void push(llvm::BasicBlock *Header, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
Begin a new structured loop.
void setVectorizeEnable(bool Enable=true)
Set the next pushed loop 'vectorize.enable'.
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
virtual void Enter(CodeGenFunction &CGF)
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
static RValue get(llvm::Value *V)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
An abstract representation of an aligned address.
llvm::PointerType * getType() const
Return the type of the pointer value.
llvm::Value * getPointer() const
Class intended to support codegen of all kind of the reduction clauses.
LValue getSharedLValue(unsigned N) const
Returns LValue for the reduction item.
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
const VarDecl * getBaseDecl(unsigned N) const
Returns the base declaration of the reduction item.
void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N)
Emits lvalue for the shared and original reduction item.
void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, llvm::function_ref< bool(CodeGenFunction &)> DefaultInit)
Performs initialization of the private copy for the reduction item.
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations.
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
void setAction(PrePostActionTy &Action) const
Complex values, per C99 6.2.5p11.
CompoundStmt - This represents a group of statements like { stmt stmt }.
ConstStmtVisitor - This class implements a simple visitor for Stmt subclasses.
DeclContext * getParent()
getParent - Returns the containing DeclContext.
A reference to a declared variable, function, enum, etc.
static DeclRefExpr * Create(const ASTContext &Context, NestedNameSpecifierLoc QualifierLoc, SourceLocation TemplateKWLoc, ValueDecl *D, bool RefersToEnclosingVariableOrCapture, SourceLocation NameLoc, QualType T, ExprValueKind VK, NamedDecl *FoundD=nullptr, const TemplateArgumentListInfo *TemplateArgs=nullptr, NonOdrUseReason NOUR=NOUR_None)
DeclStmt - Adaptor class for mixing declarations with statements and expressions.
Decl - This represents one declaration (or definition), e.g.
static Decl * castFromDeclContext(const DeclContext *)
SourceLocation getBodyRBrace() const
getBodyRBrace - Gets the right brace of the body, if a body exists.
virtual bool hasBody() const
Returns true if this Decl represents a declaration for a body of code, such as a function or method d...
SourceLocation getLocation() const
The name of a declaration.
SourceLocation getBeginLoc() const LLVM_READONLY
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
This represents one expression.
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer,...
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
Expr * IgnoreImplicitAsWritten() LLVM_READONLY
Skip past any implicit AST nodes which might surround this expression until reaching a fixed point.
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
Expr * IgnoreImpCasts() LLVM_READONLY
Skip past any implicit casts which might surround this expression until reaching a fixed point.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Represents difference between two FPOptions values.
Represents a member of a struct/union/class.
Represents a function declaration or definition.
static FunctionDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation NLoc, DeclarationName N, QualType T, TypeSourceInfo *TInfo, StorageClass SC, bool UsesFPIntrin=false, bool isInlineSpecified=false, bool hasWrittenPrototype=true, ConstexprSpecKind ConstexprKind=ConstexprSpecKind::Unspecified, const AssociatedConstraint &TrailingRequiresClause={})
GlobalDecl - represents a global declaration.
One of these records is kept for each identifier that is lexed.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
static ImplicitCastExpr * Create(const ASTContext &Context, QualType T, CastKind Kind, Expr *Operand, const CXXCastPath *BasePath, ExprValueKind Cat, FPOptionsOverride FPO)
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, const IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
Represents a point when we exit a loop.
IdentifierInfo * getIdentifier() const
Get the identifier that names this declaration, if there is one.
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
A C++ nested-name-specifier augmented with source location information.
This is a basic class for representing single OpenMP clause.
This represents 'final' clause in the 'pragma omp ...' directive.
Representation of the 'full' clause of the 'pragma omp unroll' directive.
This represents 'if' clause in the 'pragma omp ...' directive.
This represents 'num_threads' clause in the 'pragma omp ...' directive.
Representation of the 'partial' clause of the 'pragma omp unroll' directive.
This represents 'safelen' clause in the 'pragma omp ...' directive.
This represents 'simdlen' clause in the 'pragma omp ...' directive.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class.
static ParmVarDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, const IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, StorageClass S, Expr *DefArg)
PointerType - C99 6.7.5.1 - Pointer Declarators.
Represents an unpacked "presumed" location which can be presented to the user.
const char * getFilename() const
Return the presumed filename of this location.
unsigned getLine() const
Return the presumed line number of this location.
If a crash happens while one of these objects are live, the message is printed out along with the spe...
A (possibly-)qualified type.
bool isConstant(const ASTContext &Ctx) const
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Represents a struct/union/class.
unsigned getNumFields() const
Returns the number of fields (non-static data members) in this record.
field_range fields() const
field_iterator field_begin() const
Base for LValueReferenceType and RValueReferenceType.
Scope - A scope is a transient data structure that is used while parsing the program.
Encodes a location in the source.
A trivial tuple used to represent a source range.
Stmt - This represents one statement.
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top,...
SourceLocation getBeginLoc() const LLVM_READONLY
bool isPointerType() const
const T * castAs() const
Member-template castAs<specific type>.
bool isReferenceType() const
bool isLValueReferenceType() const
bool isAnyComplexType() const
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g....
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type.
const T * getAs() const
Member-template getAs<specific type>'.
static UnaryOperator * Create(const ASTContext &C, Expr *input, Opcode opc, QualType type, ExprValueKind VK, ExprObjectKind OK, SourceLocation l, bool CanOverflow, FPOptionsOverride FPFeatures)
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Represents a variable declaration or definition.
TLSKind getTLSKind() const
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
@ CInit
C-style initialization with assignment.
bool hasGlobalStorage() const
Returns true for all variables that do not have local storage.
bool isStaticLocal() const
Returns true if a variable with function scope is a static local variable.
const Expr * getInit() const
bool hasLocalStorage() const
Returns true if a variable with function scope is a non-static local variable.
@ TLS_None
Not a TLS variable.
Represents a C array with a specified size that is not an integer-constant-expression.
Expr * getSizeExpr() const
@ Type
The l-value was considered opaque, so the alignment was determined from a type.
@ Decl
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
@ Address
A pointer to a ValueDecl.
bool Inc(InterpState &S, CodePtr OpPC, bool CanOverflow)
1) Pops a pointer from the stack 2) Load the value from the pointer 3) Writes the value increased by ...
Top level wrappers for InstallAPI frontend operations.
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
CanQual< Type > CanQualType
Represents a canonical, potentially-qualified type.
bool needsTaskBasedThreadLimit(OpenMPDirectiveKind DKind)
Checks if the specified target directive, combined or not, needs task based thread_limit.
Privates[]
This class represents the 'transparent' clause in the 'pragma omp task' directive.
bool isa(CodeGen::Address addr)
if(T->getSizeExpr()) TRY_TO(TraverseStmt(const_cast< Expr * >(T -> getSizeExpr())))
bool isOpenMPLoopTransformationDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a loop transformation directive.
@ OK_Ordinary
An ordinary object is located at an address in memory.
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
@ Tile
'tile' clause, allowed on 'loop' and Combined constructs.
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
@ OMPC_SCHEDULE_MODIFIER_unknown
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
OpenMPDistScheduleClauseKind
OpenMP attributes for 'dist_schedule' clause.
@ OMPC_DIST_SCHEDULE_unknown
bool isOpenMPTaskingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of tasking directives - task or any taskloop-based dire...
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
@ Result
The result type of a method or function.
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
bool isOpenMPGenericLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive constitutes a 'loop' directive in the outermost nest.
OpenMPBindClauseKind
OpenMP bindings for the 'bind' clause.
const FunctionProtoType * T
OpenMPDependClauseKind
OpenMP attributes for 'depend' clause.
OpenMPSeverityClauseKind
OpenMP attributes for 'severity' clause.
bool isOpenMPLoopBoundSharingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of the composite or combined directives that need loop ...
llvm::omp::Directive OpenMPDirectiveKind
OpenMP directives.
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
@ VK_PRValue
A pr-value expression (in the C++11 taxonomy) produces a temporary value.
@ VK_LValue
An l-value expression is a reference to an object with independent storage.
void getOpenMPCaptureRegions(llvm::SmallVectorImpl< OpenMPDirectiveKind > &CaptureRegions, OpenMPDirectiveKind DKind)
Return the captured regions of an OpenMP directive.
OpenMPNumThreadsClauseModifier
@ OMPC_NUMTHREADS_unknown
U cast(CodeGen::Address addr)
llvm::omp::Clause OpenMPClauseKind
OpenMP clauses.
@ ThreadPrivateVar
Parameter for Thread private variable.
@ Other
Other implicit parameter.
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
__builtin_elementwise_add_sat __builtin_elementwise_sub_sat uint32_t uint64_t __builtin_riscv_ssha_i32 __builtin_riscv_sshl_u32 uint64_t
unsigned ignore_denormal_mode
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
llvm::BasicBlock * getBlock() const
static Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc)
Returns address of the threadprivate variable for the current thread.
llvm::OpenMPIRBuilder::InsertPointTy InsertPointTy
static void EmitOMPOutlinedRegionBody(CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Twine RegionName)
Emit the body of an OMP region that will be outlined in OpenMPIRBuilder::finalize().
static Address getAddressOfLocalVariable(CodeGenFunction &CGF, const VarDecl *VD)
Gets the OpenMP-specific address of the local variable /p VD.
static std::string getNameWithSeparators(ArrayRef< StringRef > Parts, StringRef FirstSeparator=".", StringRef Separator=".")
Get the platform-specific name separator.
static void FinalizeOMPRegion(CodeGenFunction &CGF, InsertPointTy IP)
Emit the Finalization for an OMP region.
static void EmitOMPInlinedRegionBody(CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Twine RegionName)
Emit the body of an OMP region.
static void EmitCaptureStmt(CodeGenFunction &CGF, llvm::BasicBlock *CodeGenIPBB, llvm::BasicBlock &FiniBB, llvm::Function *Fn, ArrayRef< llvm::Value * > Args)
unsigned NumberOfTargetItems
Address BasePointersArray
llvm::PointerType * VoidPtrTy
llvm::IntegerType * Int64Ty
llvm::IntegerType * SizeTy
SmallVector< const Expr *, 4 > DepExprs
std::string getAsString() const
getAsString - Retrieve the human-readable string for this name.
EvalResult is a struct with detailed info about an evaluated expression.
Extra information about a function prototype.
Scheduling data for loop-based OpenMP directives.
bool UseFusedDistChunkSchedule
Request the fused distr_static_chunk + static_chunkone runtime schedule in for_static_init.
OpenMPScheduleClauseModifier M2
OpenMPScheduleClauseModifier M1
OpenMPScheduleClauseKind Schedule