30#include "llvm/ADT/ArrayRef.h"
31#include "llvm/ADT/SmallVector.h"
32#include "llvm/ADT/StringExtras.h"
33#include "llvm/Bitcode/BitcodeReader.h"
34#include "llvm/IR/Constants.h"
35#include "llvm/IR/DerivedTypes.h"
36#include "llvm/IR/GlobalValue.h"
37#include "llvm/IR/InstrTypes.h"
38#include "llvm/IR/Value.h"
39#include "llvm/Support/AtomicOrdering.h"
40#include "llvm/Support/raw_ostream.h"
48using namespace llvm::omp;
55 enum CGOpenMPRegionKind {
58 ParallelOutlinedRegion,
68 CGOpenMPRegionInfo(
const CapturedStmt &CS,
69 const CGOpenMPRegionKind RegionKind,
72 : CGCapturedStmtInfo(CS,
CR_OpenMP), RegionKind(RegionKind),
73 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
75 CGOpenMPRegionInfo(
const CGOpenMPRegionKind RegionKind,
78 : CGCapturedStmtInfo(
CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
79 Kind(Kind), HasCancel(HasCancel) {}
83 virtual const VarDecl *getThreadIDVariable()
const = 0;
86 void EmitBody(CodeGenFunction &CGF,
const Stmt *S)
override;
90 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
92 virtual void emitUntiedSwitch(CodeGenFunction & ) {}
94 CGOpenMPRegionKind getRegionKind()
const {
return RegionKind; }
98 bool hasCancel()
const {
return HasCancel; }
100 static bool classof(
const CGCapturedStmtInfo *Info) {
104 ~CGOpenMPRegionInfo()
override =
default;
107 CGOpenMPRegionKind RegionKind;
108 RegionCodeGenTy CodeGen;
114class CGOpenMPOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
116 CGOpenMPOutlinedRegionInfo(
const CapturedStmt &CS,
const VarDecl *ThreadIDVar,
117 const RegionCodeGenTy &CodeGen,
119 StringRef HelperName)
120 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen,
Kind,
122 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
123 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
128 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
131 StringRef getHelperName()
const override {
return HelperName; }
133 static bool classof(
const CGCapturedStmtInfo *Info) {
134 return CGOpenMPRegionInfo::classof(Info) &&
136 ParallelOutlinedRegion;
142 const VarDecl *ThreadIDVar;
143 StringRef HelperName;
147class CGOpenMPTaskOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
149 class UntiedTaskActionTy final :
public PrePostActionTy {
151 const VarDecl *PartIDVar;
152 const RegionCodeGenTy UntiedCodeGen;
153 llvm::SwitchInst *UntiedSwitch =
nullptr;
156 UntiedTaskActionTy(
bool Tied,
const VarDecl *PartIDVar,
157 const RegionCodeGenTy &UntiedCodeGen)
158 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
159 void Enter(CodeGenFunction &CGF)
override {
164 PartIDVar->
getType()->castAs<PointerType>());
168 UntiedSwitch = CGF.
Builder.CreateSwitch(Res, DoneBB);
172 UntiedSwitch->addCase(CGF.
Builder.getInt32(0),
174 emitUntiedSwitch(CGF);
177 void emitUntiedSwitch(CodeGenFunction &CGF)
const {
181 PartIDVar->
getType()->castAs<PointerType>());
185 CodeGenFunction::JumpDest CurPoint =
189 UntiedSwitch->addCase(CGF.
Builder.getInt32(UntiedSwitch->getNumCases()),
195 unsigned getNumberOfParts()
const {
return UntiedSwitch->getNumCases(); }
197 CGOpenMPTaskOutlinedRegionInfo(
const CapturedStmt &CS,
198 const VarDecl *ThreadIDVar,
199 const RegionCodeGenTy &CodeGen,
201 const UntiedTaskActionTy &Action)
202 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen,
Kind, HasCancel),
203 ThreadIDVar(ThreadIDVar), Action(Action) {
204 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
209 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
212 LValue getThreadIDVariableLValue(CodeGenFunction &CGF)
override;
215 StringRef getHelperName()
const override {
return ".omp_outlined."; }
217 void emitUntiedSwitch(CodeGenFunction &CGF)
override {
218 Action.emitUntiedSwitch(CGF);
221 static bool classof(
const CGCapturedStmtInfo *Info) {
222 return CGOpenMPRegionInfo::classof(Info) &&
230 const VarDecl *ThreadIDVar;
232 const UntiedTaskActionTy &Action;
237class CGOpenMPInlinedRegionInfo :
public CGOpenMPRegionInfo {
239 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
240 const RegionCodeGenTy &CodeGen,
242 : CGOpenMPRegionInfo(InlinedRegion, CodeGen,
Kind, HasCancel),
244 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
247 llvm::Value *getContextValue()
const override {
249 return OuterRegionInfo->getContextValue();
250 llvm_unreachable(
"No context value for inlined OpenMP region");
253 void setContextValue(llvm::Value *
V)
override {
254 if (OuterRegionInfo) {
255 OuterRegionInfo->setContextValue(
V);
258 llvm_unreachable(
"No context value for inlined OpenMP region");
262 const FieldDecl *lookup(
const VarDecl *VD)
const override {
264 return OuterRegionInfo->lookup(VD);
270 FieldDecl *getThisFieldDecl()
const override {
272 return OuterRegionInfo->getThisFieldDecl();
278 const VarDecl *getThreadIDVariable()
const override {
280 return OuterRegionInfo->getThreadIDVariable();
285 LValue getThreadIDVariableLValue(CodeGenFunction &CGF)
override {
287 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
288 llvm_unreachable(
"No LValue for inlined OpenMP construct");
292 StringRef getHelperName()
const override {
293 if (
auto *OuterRegionInfo = getOldCSI())
294 return OuterRegionInfo->getHelperName();
295 llvm_unreachable(
"No helper name for inlined OpenMP construct");
298 void emitUntiedSwitch(CodeGenFunction &CGF)
override {
300 OuterRegionInfo->emitUntiedSwitch(CGF);
303 CodeGenFunction::CGCapturedStmtInfo *getOldCSI()
const {
return OldCSI; }
305 static bool classof(
const CGCapturedStmtInfo *Info) {
306 return CGOpenMPRegionInfo::classof(Info) &&
310 ~CGOpenMPInlinedRegionInfo()
override =
default;
314 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
315 CGOpenMPRegionInfo *OuterRegionInfo;
323class CGOpenMPTargetRegionInfo final :
public CGOpenMPRegionInfo {
325 CGOpenMPTargetRegionInfo(
const CapturedStmt &CS,
326 const RegionCodeGenTy &CodeGen, StringRef HelperName)
327 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
329 HelperName(HelperName) {}
333 const VarDecl *getThreadIDVariable()
const override {
return nullptr; }
336 StringRef getHelperName()
const override {
return HelperName; }
338 static bool classof(
const CGCapturedStmtInfo *Info) {
339 return CGOpenMPRegionInfo::classof(Info) &&
344 StringRef HelperName;
348 llvm_unreachable(
"No codegen for expressions");
352class CGOpenMPInnerExprInfo final :
public CGOpenMPInlinedRegionInfo {
354 CGOpenMPInnerExprInfo(CodeGenFunction &CGF,
const CapturedStmt &CS)
355 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
363 if (!C.capturesVariable() && !C.capturesVariableByCopy())
366 const VarDecl *VD = C.getCapturedVar();
367 if (VD->isLocalVarDeclOrParm())
370 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
372 VD->getType().getNonReferenceType(), VK_LValue,
374 PrivScope.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress());
376 (
void)PrivScope.Privatize();
380 const FieldDecl *lookup(
const VarDecl *VD)
const override {
381 if (
const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
387 void EmitBody(CodeGenFunction &CGF,
const Stmt *S)
override {
388 llvm_unreachable(
"No body for expressions");
393 const VarDecl *getThreadIDVariable()
const override {
394 llvm_unreachable(
"No thread id for expressions");
398 StringRef getHelperName()
const override {
399 llvm_unreachable(
"No helper name for expressions");
402 static bool classof(
const CGCapturedStmtInfo *Info) {
return false; }
406 CodeGenFunction::OMPPrivateScope PrivScope;
410class InlinedOpenMPRegionRAII {
411 CodeGenFunction &CGF;
412 llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
413 FieldDecl *LambdaThisCaptureField =
nullptr;
414 const CodeGen::CGBlockInfo *BlockInfo =
nullptr;
415 bool NoInheritance =
false;
422 InlinedOpenMPRegionRAII(CodeGenFunction &CGF,
const RegionCodeGenTy &CodeGen,
424 bool NoInheritance =
true)
425 : CGF(CGF), NoInheritance(NoInheritance) {
427 CGF.CapturedStmtInfo =
new CGOpenMPInlinedRegionInfo(
428 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
430 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
431 LambdaThisCaptureField = CGF.LambdaThisCaptureField;
432 CGF.LambdaThisCaptureField =
nullptr;
433 BlockInfo = CGF.BlockInfo;
434 CGF.BlockInfo =
nullptr;
438 ~InlinedOpenMPRegionRAII() {
442 delete CGF.CapturedStmtInfo;
443 CGF.CapturedStmtInfo = OldCSI;
445 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
446 CGF.LambdaThisCaptureField = LambdaThisCaptureField;
447 CGF.BlockInfo = BlockInfo;
455enum OpenMPLocationFlags :
unsigned {
457 OMP_IDENT_IMD = 0x01,
459 OMP_IDENT_KMPC = 0x02,
461 OMP_ATOMIC_REDUCE = 0x10,
463 OMP_IDENT_BARRIER_EXPL = 0x20,
465 OMP_IDENT_BARRIER_IMPL = 0x40,
467 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
469 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
471 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
473 OMP_IDENT_WORK_LOOP = 0x200,
475 OMP_IDENT_WORK_SECTIONS = 0x400,
477 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
507enum IdentFieldIndex {
509 IdentField_Reserved_1,
513 IdentField_Reserved_2,
515 IdentField_Reserved_3,
524enum OpenMPSchedType {
527 OMP_sch_static_chunked = 33,
529 OMP_sch_dynamic_chunked = 35,
530 OMP_sch_guided_chunked = 36,
531 OMP_sch_runtime = 37,
534 OMP_sch_static_balanced_chunked = 45,
537 OMP_ord_static_chunked = 65,
539 OMP_ord_dynamic_chunked = 67,
540 OMP_ord_guided_chunked = 68,
541 OMP_ord_runtime = 69,
543 OMP_sch_default = OMP_sch_static,
545 OMP_dist_sch_static_chunked = 91,
546 OMP_dist_sch_static = 92,
549 OMP_sch_modifier_monotonic = (1 << 29),
551 OMP_sch_modifier_nonmonotonic = (1 << 30),
556class CleanupTy final :
public EHScopeStack::Cleanup {
557 PrePostActionTy *Action;
560 explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
561 void Emit(CodeGenFunction &CGF, Flags )
override {
574 Callback(CodeGen, CGF, *PrePostAction);
577 Callback(CodeGen, CGF, Action);
585 if (
const auto *CE = dyn_cast<CallExpr>(ReductionOp))
586 if (
const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
587 if (
const auto *DRE =
588 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
589 if (
const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
600 std::pair<llvm::Function *, llvm::Function *>
Reduction =
620 auto *GV =
new llvm::GlobalVariable(
622 llvm::GlobalValue::PrivateLinkage,
Init, Name);
663 llvm::Value *NumElements = CGF.
emitArrayLength(ArrayTy, ElementTy, DestAddr);
667 llvm::Value *SrcBegin =
nullptr;
669 SrcBegin = SrcAddr.emitRawPointer(CGF);
672 llvm::Value *DestEnd =
677 llvm::Value *IsEmpty =
678 CGF.
Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arrayinit.isempty");
679 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
682 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
687 llvm::PHINode *SrcElementPHI =
nullptr;
690 SrcElementPHI = CGF.
Builder.CreatePHI(SrcBegin->getType(), 2,
691 "omp.arraycpy.srcElementPast");
692 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
694 Address(SrcElementPHI, SrcAddr.getElementType(),
695 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
697 llvm::PHINode *DestElementPHI = CGF.
Builder.CreatePHI(
698 DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
699 DestElementPHI->addIncoming(DestBegin, EntryBB);
707 if (EmitDeclareReductionInit) {
709 SrcElementCurrent, ElementTy);
717 llvm::Value *SrcElementNext = CGF.
Builder.CreateConstGEP1_32(
718 SrcAddr.getElementType(), SrcElementPHI, 1,
719 "omp.arraycpy.dest.element");
720 SrcElementPHI->addIncoming(SrcElementNext, CGF.
Builder.GetInsertBlock());
724 llvm::Value *DestElementNext = CGF.
Builder.CreateConstGEP1_32(
726 "omp.arraycpy.dest.element");
729 CGF.
Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
730 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
731 DestElementPHI->addIncoming(DestElementNext, CGF.
Builder.GetInsertBlock());
743 if (
const auto *OASE = dyn_cast<ArraySectionExpr>(E))
748void ReductionCodeGen::emitAggregateInitialization(
750 const OMPDeclareReductionDecl *DRD) {
754 const auto *PrivateVD =
756 bool EmitDeclareReductionInit =
759 EmitDeclareReductionInit,
760 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
761 : PrivateVD->getInit(),
769 ClausesData.reserve(Shareds.size());
770 SharedAddresses.reserve(Shareds.size());
771 Sizes.reserve(Shareds.size());
772 BaseDecls.reserve(Shareds.size());
773 const auto *IOrig = Origs.begin();
774 const auto *IPriv = Privates.begin();
775 const auto *IRed = ReductionOps.begin();
776 for (
const Expr *Ref : Shareds) {
777 ClausesData.emplace_back(Ref, *IOrig, *IPriv, *IRed);
778 std::advance(IOrig, 1);
779 std::advance(IPriv, 1);
780 std::advance(IRed, 1);
785 assert(SharedAddresses.size() == N && OrigAddresses.size() == N &&
786 "Number of generated lvalues must be exactly N.");
787 LValue First = emitSharedLValue(CGF, ClausesData[N].Shared);
788 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Shared);
789 SharedAddresses.emplace_back(
First, Second);
790 if (ClausesData[N].Shared == ClausesData[N].Ref) {
791 OrigAddresses.emplace_back(
First, Second);
793 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
794 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
795 OrigAddresses.emplace_back(
First, Second);
804 CGF.
getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()),
809 llvm::Value *SizeInChars;
810 auto *ElemType = OrigAddresses[N].first.getAddress().getElementType();
811 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
812 if (AsArraySection) {
813 Size = CGF.
Builder.CreatePtrDiff(ElemType,
814 OrigAddresses[N].second.getPointer(CGF),
815 OrigAddresses[N].first.getPointer(CGF));
816 Size = CGF.
Builder.CreateNUWAdd(
817 Size, llvm::ConstantInt::get(Size->getType(), 1));
818 SizeInChars = CGF.
Builder.CreateNUWMul(Size, ElemSizeOf);
821 CGF.
getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType());
822 Size = CGF.
Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
824 Sizes.emplace_back(SizeInChars, Size);
837 assert(!Size && !Sizes[N].second &&
838 "Size should be nullptr for non-variably modified reduction "
853 assert(SharedAddresses.size() > N &&
"No variable was generated");
854 const auto *PrivateVD =
860 (void)DefaultInit(CGF);
861 emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD);
862 }
else if (DRD && (DRD->
getInitializer() || !PrivateVD->hasInit())) {
863 (void)DefaultInit(CGF);
864 QualType SharedType = SharedAddresses[N].first.getType();
866 PrivateAddr, SharedAddr, SharedType);
867 }
else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
870 PrivateVD->
getType().getQualifiers(),
888 CGF.
pushDestroy(DTorKind, PrivateAddr, PrivateType);
907 BaseLV.getType(), BaseLV.getBaseInfo(),
941 const VarDecl *OrigVD =
nullptr;
942 if (
const auto *OASE = dyn_cast<ArraySectionExpr>(Ref)) {
943 const Expr *
Base = OASE->getBase()->IgnoreParenImpCasts();
944 while (
const auto *TempOASE = dyn_cast<ArraySectionExpr>(
Base))
945 Base = TempOASE->getBase()->IgnoreParenImpCasts();
946 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
947 Base = TempASE->getBase()->IgnoreParenImpCasts();
950 }
else if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
951 const Expr *
Base = ASE->getBase()->IgnoreParenImpCasts();
952 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
953 Base = TempASE->getBase()->IgnoreParenImpCasts();
964 BaseDecls.emplace_back(OrigVD);
967 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
969 Address SharedAddr = SharedAddresses[N].first.getAddress();
970 llvm::Value *Adjustment = CGF.
Builder.CreatePtrDiff(
973 llvm::Value *PrivatePointer =
979 SharedAddresses[N].first.getType(),
982 BaseDecls.emplace_back(
996 getThreadIDVariable()->
getType()->castAs<PointerType>());
1014LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1017 getThreadIDVariable()->
getType(),
1035 llvm::OpenMPIRBuilderConfig Config(
1036 CGM.getLangOpts().OpenMPIsTargetDevice,
isGPU(),
1037 CGM.getLangOpts().OpenMPOffloadMandatory,
1040 Config.setDefaultTargetAS(
1046 CGM.getLangOpts().OpenMPIsTargetDevice
1047 ?
CGM.getLangOpts().OMPHostIRFile
1052 if (
CGM.getLangOpts().OpenMPForceUSM) {
1054 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(
true);
1062 if (!
Data.getValue().pointsToAliveValue())
1064 auto *GV = dyn_cast<llvm::GlobalVariable>(
Data.getValue());
1067 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1069 GV->eraseFromParent();
1074 return OMPBuilder.createPlatformSpecificName(Parts);
1077static llvm::Function *
1079 const Expr *CombinerInitializer,
const VarDecl *In,
1080 const VarDecl *Out,
bool IsCombiner) {
1083 QualType PtrTy =
C.getPointerType(Ty).withRestrict();
1089 Args.push_back(&OmpOutParm);
1090 Args.push_back(&OmpInParm);
1095 {IsCombiner ?
"omp_combiner" :
"omp_initializer",
""});
1096 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1100 Fn->removeFnAttr(llvm::Attribute::NoInline);
1101 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1102 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1108 Out->getLocation());
1118 (void)
Scope.Privatize();
1119 if (!IsCombiner && Out->hasInit() &&
1122 Out->getType().getQualifiers(),
1125 if (CombinerInitializer)
1127 Scope.ForceCleanup();
1156std::pair<llvm::Function *, llvm::Function *>
1168struct PushAndPopStackRAII {
1169 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder,
CodeGenFunction &CGF,
1170 bool HasCancel, llvm::omp::Directive Kind)
1171 : OMPBuilder(OMPBuilder) {
1187 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1188 assert(IP.getBlock()->end() == IP.getPoint() &&
1189 "Clang CG should cause non-terminated block!");
1190 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1195 return llvm::Error::success();
1200 llvm::OpenMPIRBuilder::FinalizationInfo FI({FiniCB, Kind, HasCancel});
1201 OMPBuilder->pushFinalizationCB(std::move(FI));
1203 ~PushAndPopStackRAII() {
1205 OMPBuilder->popFinalizationCB();
1207 llvm::OpenMPIRBuilder *OMPBuilder;
1216 "thread id variable must be of type kmp_int32 *");
1218 bool HasCancel =
false;
1219 if (
const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1220 HasCancel = OPD->hasCancel();
1221 else if (
const auto *OPD = dyn_cast<OMPTargetParallelDirective>(&D))
1222 HasCancel = OPD->hasCancel();
1223 else if (
const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1224 HasCancel = OPSD->hasCancel();
1225 else if (
const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1226 HasCancel = OPFD->hasCancel();
1227 else if (
const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1228 HasCancel = OPFD->hasCancel();
1229 else if (
const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1230 HasCancel = OPFD->hasCancel();
1231 else if (
const auto *OPFD =
1232 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1233 HasCancel = OPFD->hasCancel();
1234 else if (
const auto *OPFD =
1235 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1236 HasCancel = OPFD->hasCancel();
1241 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind);
1242 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar,
CodeGen, InnermostKind,
1243 HasCancel, OutlinedHelperName);
1249 std::string Suffix =
getName({
"omp_outlined"});
1250 return (Name + Suffix).str();
1258 std::string Suffix =
getName({
"omp",
"reduction",
"reduction_func"});
1259 return (Name + Suffix).str();
1266 const CapturedStmt *CS = D.getCapturedStmt(OMPD_parallel);
1276 const CapturedStmt *CS = D.getCapturedStmt(OMPD_teams);
1286 bool Tied,
unsigned &NumberOfParts) {
1289 llvm::Value *ThreadID =
getThreadID(CGF, D.getBeginLoc());
1291 llvm::Value *TaskArgs[] = {
1293 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1296 CGF.EmitRuntimeCall(
OMPBuilder.getOrCreateRuntimeFunction(
1297 CGM.getModule(), OMPRTL___kmpc_omp_task),
1300 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1304 "thread id variable must be of type kmp_int32 for tasks");
1309 bool HasCancel =
false;
1310 if (
const auto *TD = dyn_cast<OMPTaskDirective>(&D))
1311 HasCancel = TD->hasCancel();
1312 else if (
const auto *TD = dyn_cast<OMPTaskLoopDirective>(&D))
1313 HasCancel = TD->hasCancel();
1314 else if (
const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(&D))
1315 HasCancel = TD->hasCancel();
1316 else if (
const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(&D))
1317 HasCancel = TD->hasCancel();
1320 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar,
CodeGen,
1321 InnermostKind, HasCancel, Action);
1323 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1325 NumberOfParts = Action.getNumberOfParts();
1330 bool AtCurrentPoint) {
1332 assert(!Elem.ServiceInsertPt &&
"Insert point is set already.");
1334 llvm::Value *Undef = llvm::UndefValue::get(CGF.
Int32Ty);
1335 if (AtCurrentPoint) {
1336 Elem.ServiceInsertPt =
new llvm::BitCastInst(Undef, CGF.
Int32Ty,
"svcpt",
1337 CGF.
Builder.GetInsertBlock());
1339 Elem.ServiceInsertPt =
new llvm::BitCastInst(Undef, CGF.
Int32Ty,
"svcpt");
1340 Elem.ServiceInsertPt->insertAfter(CGF.
AllocaInsertPt->getIterator());
1346 if (Elem.ServiceInsertPt) {
1347 llvm::Instruction *Ptr = Elem.ServiceInsertPt;
1348 Elem.ServiceInsertPt =
nullptr;
1349 Ptr->eraseFromParent();
1356 llvm::raw_svector_ostream OS(Buffer);
1365 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.
CurFuncDecl))
1366 OS << FD->getQualifiedNameAsString();
1373 unsigned Flags,
bool EmitLoc) {
1374 uint32_t SrcLocStrSize;
1375 llvm::Constant *SrcLocStr;
1376 if ((!EmitLoc &&
CGM.getCodeGenOpts().getDebugInfo() ==
1377 llvm::codegenoptions::NoDebugInfo) ||
1379 SrcLocStr =
OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1381 std::string FunctionName;
1383 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.
CurFuncDecl))
1384 FunctionName = FD->getQualifiedNameAsString();
1397 SrcLocStr, SrcLocStrSize, llvm::omp::IdentFlag(Flags), Reserved2Flags);
1402 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1405 if (
CGM.getLangOpts().OpenMPIRBuilder) {
1408 uint32_t SrcLocStrSize;
1409 auto *SrcLocStr =
OMPBuilder.getOrCreateSrcLocStr(
1412 OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize));
1415 llvm::Value *ThreadID =
nullptr;
1420 ThreadID = I->second.ThreadID;
1421 if (ThreadID !=
nullptr)
1425 if (
auto *OMPRegionInfo =
1427 if (OMPRegionInfo->getThreadIDVariable()) {
1429 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1433 CGF.
Builder.GetInsertBlock() == TopBlock ||
1438 CGF.
Builder.GetInsertBlock()) {
1442 if (CGF.
Builder.GetInsertBlock() == TopBlock)
1454 if (!Elem.ServiceInsertPt)
1456 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1457 CGF.
Builder.SetInsertPoint(Elem.ServiceInsertPt);
1461 OMPRTL___kmpc_global_thread_num),
1464 Elem.ThreadID =
Call;
1469 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1475 for (
const auto *D : I->second)
1480 for (
const auto *D : I->second)
1492static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind
1494 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
1495 OMPDeclareTargetDeclAttr::getDeviceType(VD);
1497 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1499 switch ((
int)*DevTy) {
1500 case OMPDeclareTargetDeclAttr::DT_Host:
1501 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseHost;
1503 case OMPDeclareTargetDeclAttr::DT_NoHost:
1504 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNoHost;
1506 case OMPDeclareTargetDeclAttr::DT_Any:
1507 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny;
1510 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1515static llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind
1517 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> MapType =
1518 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1520 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1521 switch ((
int)*MapType) {
1522 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_To:
1523 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
1525 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Enter:
1526 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter;
1528 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Link:
1529 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
1532 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1541 auto FileInfoCallBack = [&]() {
1545 llvm::sys::fs::UniqueID ID;
1546 if (llvm::sys::fs::getUniqueID(PLoc.
getFilename(), ID)) {
1547 PLoc =
SM.getPresumedLoc(BeginLoc,
false);
1553 return OMPBuilder.getTargetEntryUniqueInfo(FileInfoCallBack, ParentName);
1557 auto AddrOfGlobal = [&VD,
this]() {
return CGM.GetAddrOfGlobal(VD); };
1559 auto LinkageForVariable = [&VD,
this]() {
1560 return CGM.getLLVMLinkageVarDefinition(VD);
1563 std::vector<llvm::GlobalVariable *> GeneratedRefs;
1565 llvm::Type *LlvmPtrTy =
CGM.getTypes().ConvertTypeForMem(
1566 CGM.getContext().getPointerType(VD->
getType()));
1567 llvm::Constant *addr =
OMPBuilder.getAddrOfDeclareTargetVar(
1573 CGM.getMangledName(VD), GeneratedRefs,
CGM.getLangOpts().OpenMPSimd,
1574 CGM.getLangOpts().OMPTargetTriples, LlvmPtrTy, AddrOfGlobal,
1575 LinkageForVariable);
1584 assert(!
CGM.getLangOpts().OpenMPUseTLS ||
1585 !
CGM.getContext().getTargetInfo().isTLSSupported());
1587 std::string Suffix =
getName({
"cache",
""});
1588 return OMPBuilder.getOrCreateInternalVariable(
1589 CGM.Int8PtrPtrTy, Twine(
CGM.getMangledName(VD)).concat(Suffix).str());
1596 if (
CGM.getLangOpts().OpenMPUseTLS &&
1597 CGM.getContext().getTargetInfo().isTLSSupported())
1601 llvm::Value *Args[] = {
1604 CGM.getSize(
CGM.GetTargetTypeStoreSize(VarTy)),
1609 CGM.getModule(), OMPRTL___kmpc_threadprivate_cached),
1621 CGM.getModule(), OMPRTL___kmpc_global_thread_num),
1625 llvm::Value *Args[] = {
1628 Ctor, CopyCtor, Dtor};
1631 CGM.getModule(), OMPRTL___kmpc_threadprivate_register),
1638 if (
CGM.getLangOpts().OpenMPUseTLS &&
1639 CGM.getContext().getTargetInfo().isTLSSupported())
1646 llvm::Value *Ctor =
nullptr, *CopyCtor =
nullptr, *Dtor =
nullptr;
1648 if (
CGM.getLangOpts().CPlusPlus && PerformInit) {
1654 nullptr,
CGM.getContext().VoidPtrTy,
1656 Args.push_back(&Dst);
1658 const auto &FI =
CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1659 CGM.getContext().VoidPtrTy, Args);
1660 llvm::FunctionType *FTy =
CGM.getTypes().GetFunctionType(FI);
1661 std::string Name =
getName({
"__kmpc_global_ctor_",
""});
1662 llvm::Function *Fn =
1663 CGM.CreateGlobalInitOrCleanUpFunction(FTy, Name, FI, Loc);
1686 nullptr,
CGM.getContext().VoidPtrTy,
1688 Args.push_back(&Dst);
1690 const auto &FI =
CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1691 CGM.getContext().VoidTy, Args);
1692 llvm::FunctionType *FTy =
CGM.getTypes().GetFunctionType(FI);
1693 std::string Name =
getName({
"__kmpc_global_dtor_",
""});
1694 llvm::Function *Fn =
1695 CGM.CreateGlobalInitOrCleanUpFunction(FTy, Name, FI, Loc);
1718 CopyCtor = llvm::Constant::getNullValue(
CGM.UnqualPtrTy);
1719 if (Ctor ==
nullptr) {
1720 Ctor = llvm::Constant::getNullValue(
CGM.UnqualPtrTy);
1722 if (Dtor ==
nullptr) {
1723 Dtor = llvm::Constant::getNullValue(
CGM.UnqualPtrTy);
1726 auto *InitFunctionTy =
1727 llvm::FunctionType::get(
CGM.VoidTy,
false);
1728 std::string Name =
getName({
"__omp_threadprivate_init_",
""});
1729 llvm::Function *InitFunction =
CGM.CreateGlobalInitOrCleanUpFunction(
1730 InitFunctionTy, Name,
CGM.getTypes().arrangeNullaryFunction());
1734 CGM.getTypes().arrangeNullaryFunction(), ArgList,
1738 return InitFunction;
1746 llvm::GlobalValue *GV) {
1747 std::optional<OMPDeclareTargetDeclAttr *> ActiveAttr =
1748 OMPDeclareTargetDeclAttr::getActiveAttr(FD);
1751 if (!ActiveAttr || !(*ActiveAttr)->getIndirect())
1758 OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(Name, EntryInfo);
1764 llvm::GlobalValue *
Addr = GV;
1765 if (
CGM.getLangOpts().OpenMPIsTargetDevice) {
1766 Addr =
new llvm::GlobalVariable(
1767 CGM.getModule(),
CGM.VoidPtrTy,
1768 true, llvm::GlobalValue::ExternalLinkage, GV, Name,
1769 nullptr, llvm::GlobalValue::NotThreadLocal,
1770 CGM.getModule().getDataLayout().getDefaultGlobalsAddressSpace());
1771 Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1774 OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(
1775 Name,
Addr,
CGM.GetTargetTypeStoreSize(
CGM.VoidPtrTy).getQuantity(),
1776 llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect,
1777 llvm::GlobalValue::WeakODRLinkage);
1783 std::string Suffix =
getName({
"artificial",
""});
1785 llvm::GlobalVariable *GAddr =
OMPBuilder.getOrCreateInternalVariable(
1786 VarLVType, Twine(Name).concat(Suffix).str());
1787 if (
CGM.getLangOpts().OpenMP &&
CGM.getLangOpts().OpenMPUseTLS &&
1788 CGM.getTarget().isTLSSupported()) {
1789 GAddr->setThreadLocal(
true);
1790 return Address(GAddr, GAddr->getValueType(),
1791 CGM.getContext().getTypeAlignInChars(VarType));
1793 std::string CacheSuffix =
getName({
"cache",
""});
1794 llvm::Value *Args[] = {
1802 Twine(Name).concat(Suffix).concat(CacheSuffix).str())};
1807 CGM.getModule(), OMPRTL___kmpc_threadprivate_cached),
1810 VarLVType,
CGM.getContext().getTypeAlignInChars(VarType));
1860 auto &M =
CGM.getModule();
1861 auto &&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc,
1864 llvm::Value *Args[] = {
1866 CGF.
Builder.getInt32(CapturedVars.size()),
1869 RealArgs.append(std::begin(Args), std::end(Args));
1870 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
1872 llvm::FunctionCallee RTLFn =
1873 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_fork_call);
1876 auto &&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc, Loc,
1882 llvm::Value *Args[] = {RTLoc, ThreadID};
1884 M, OMPRTL___kmpc_serialized_parallel),
1891 ".bound.zero.addr");
1896 OutlinedFnArgs.push_back(ZeroAddrBound.
getPointer());
1897 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
1905 OutlinedFn->removeFnAttr(llvm::Attribute::AlwaysInline);
1906 OutlinedFn->addFnAttr(llvm::Attribute::NoInline);
1912 M, OMPRTL___kmpc_end_serialized_parallel),
1931 if (
auto *OMPRegionInfo =
1933 if (OMPRegionInfo->getThreadIDVariable())
1934 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
1943 return ThreadIDTemp;
1947 std::string Prefix = Twine(
"gomp_critical_user_", CriticalName).str();
1948 std::string Name =
getName({Prefix,
"var"});
1955 llvm::FunctionCallee EnterCallee;
1957 llvm::FunctionCallee ExitCallee;
1960 llvm::BasicBlock *ContBlock =
nullptr;
1963 CommonActionTy(llvm::FunctionCallee EnterCallee,
1965 llvm::FunctionCallee ExitCallee,
1967 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
1972 llvm::Value *CallBool = CGF.
Builder.CreateIsNotNull(EnterRes);
1976 CGF.
Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
1980 void Done(CodeGenFunction &CGF) {
1985 void Exit(CodeGenFunction &CGF)
override {
1992 StringRef CriticalName,
2006 EnterArgs.push_back(CGF.
Builder.CreateIntCast(
2009 CommonActionTy Action(
2012 Hint ? OMPRTL___kmpc_critical_with_hint : OMPRTL___kmpc_critical),
2015 OMPRTL___kmpc_end_critical),
2032 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2033 CGM.getModule(), OMPRTL___kmpc_master),
2036 CGM.getModule(), OMPRTL___kmpc_end_master),
2054 llvm::Value *FilterVal = Filter
2056 : llvm::ConstantInt::get(
CGM.Int32Ty, 0);
2061 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2062 CGM.getModule(), OMPRTL___kmpc_masked),
2065 CGM.getModule(), OMPRTL___kmpc_end_masked),
2081 llvm::Value *Args[] = {
2083 llvm::ConstantInt::get(
CGM.IntTy, 0,
true)};
2085 CGM.getModule(), OMPRTL___kmpc_omp_taskyield),
2089 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
2090 Region->emitUntiedSwitch(CGF);
2103 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2104 CGM.getModule(), OMPRTL___kmpc_taskgroup),
2107 CGM.getModule(), OMPRTL___kmpc_end_taskgroup),
2116 unsigned Index,
const VarDecl *Var) {
2137 Args.push_back(&LHSArg);
2138 Args.push_back(&RHSArg);
2144 llvm::GlobalValue::InternalLinkage, Name,
2147 Fn->setDoesNotRecurse();
2164 for (
unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
2165 const auto *DestVar =
2169 const auto *SrcVar =
2175 CGF.
EmitOMPCopy(
Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
2190 assert(CopyprivateVars.size() == SrcExprs.size() &&
2191 CopyprivateVars.size() == DstExprs.size() &&
2192 CopyprivateVars.size() == AssignmentOps.size());
2204 if (!CopyprivateVars.empty()) {
2207 C.getIntTypeForBitwidth(32, 1);
2208 DidIt = CGF.
CreateMemTemp(KmpInt32Ty,
".omp.copyprivate.did_it");
2213 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2214 CGM.getModule(), OMPRTL___kmpc_single),
2217 CGM.getModule(), OMPRTL___kmpc_end_single),
2230 llvm::APInt ArraySize(32, CopyprivateVars.size());
2231 QualType CopyprivateArrayTy =
C.getConstantArrayType(
2236 CGF.
CreateMemTemp(CopyprivateArrayTy,
".omp.copyprivate.cpr_list");
2237 for (
unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
2249 SrcExprs, DstExprs, AssignmentOps, Loc);
2250 llvm::Value *BufSize = CGF.
getTypeSize(CopyprivateArrayTy);
2254 llvm::Value *Args[] = {
2258 CL.emitRawPointer(CGF),
2263 CGM.getModule(), OMPRTL___kmpc_copyprivate),
2279 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2280 CGM.getModule(), OMPRTL___kmpc_ordered),
2283 CGM.getModule(), OMPRTL___kmpc_end_ordered),
2294 if (Kind == OMPD_for)
2295 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
2296 else if (Kind == OMPD_sections)
2297 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
2298 else if (Kind == OMPD_single)
2299 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
2300 else if (Kind == OMPD_barrier)
2301 Flags = OMP_IDENT_BARRIER_EXPL;
2303 Flags = OMP_IDENT_BARRIER_IMPL;
2315 ScheduleKind = OMPC_SCHEDULE_static;
2317 llvm::APInt ChunkSize(32, 1);
2327 bool ForceSimpleCall) {
2329 auto *OMPRegionInfo =
2332 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
2335 CGF.
Builder.restoreIP(AfterIP);
2348 if (OMPRegionInfo) {
2349 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
2352 OMPRTL___kmpc_cancel_barrier),
2361 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
2373 CGM.getModule(), OMPRTL___kmpc_barrier),
2378 Expr *ME,
bool IsFatal) {
2380 : llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
2383 llvm::Value *Args[] = {
2385 llvm::ConstantInt::get(
CGM.Int32Ty, IsFatal ? 2 : 1),
2386 CGF.
Builder.CreatePointerCast(MVL,
CGM.Int8PtrTy)};
2388 CGM.getModule(), OMPRTL___kmpc_error),
2394 bool Chunked,
bool Ordered) {
2395 switch (ScheduleKind) {
2396 case OMPC_SCHEDULE_static:
2397 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
2398 : (Ordered ? OMP_ord_static : OMP_sch_static);
2399 case OMPC_SCHEDULE_dynamic:
2400 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
2401 case OMPC_SCHEDULE_guided:
2402 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
2403 case OMPC_SCHEDULE_runtime:
2404 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
2405 case OMPC_SCHEDULE_auto:
2406 return Ordered ? OMP_ord_auto : OMP_sch_auto;
2408 assert(!Chunked &&
"chunk was specified but schedule kind not known");
2409 return Ordered ? OMP_ord_static : OMP_sch_static;
2411 llvm_unreachable(
"Unexpected runtime schedule");
2415static OpenMPSchedType
2418 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
2422 bool Chunked)
const {
2423 OpenMPSchedType Schedule =
2425 return Schedule == OMP_sch_static;
2431 return Schedule == OMP_dist_sch_static;
2435 bool Chunked)
const {
2436 OpenMPSchedType Schedule =
2438 return Schedule == OMP_sch_static_chunked;
2444 return Schedule == OMP_dist_sch_static_chunked;
2448 OpenMPSchedType Schedule =
2450 assert(Schedule != OMP_sch_static_chunked &&
"cannot be chunked here");
2451 return Schedule != OMP_sch_static;
2459 case OMPC_SCHEDULE_MODIFIER_monotonic:
2460 Modifier = OMP_sch_modifier_monotonic;
2462 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2463 Modifier = OMP_sch_modifier_nonmonotonic;
2465 case OMPC_SCHEDULE_MODIFIER_simd:
2466 if (Schedule == OMP_sch_static_chunked)
2467 Schedule = OMP_sch_static_balanced_chunked;
2474 case OMPC_SCHEDULE_MODIFIER_monotonic:
2475 Modifier = OMP_sch_modifier_monotonic;
2477 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2478 Modifier = OMP_sch_modifier_nonmonotonic;
2480 case OMPC_SCHEDULE_MODIFIER_simd:
2481 if (Schedule == OMP_sch_static_chunked)
2482 Schedule = OMP_sch_static_balanced_chunked;
2494 if (CGM.
getLangOpts().OpenMP >= 50 && Modifier == 0) {
2495 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
2496 Schedule == OMP_sch_static_balanced_chunked ||
2497 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||
2498 Schedule == OMP_dist_sch_static_chunked ||
2499 Schedule == OMP_dist_sch_static))
2500 Modifier = OMP_sch_modifier_nonmonotonic;
2502 return Schedule | Modifier;
2512 ScheduleKind.
Schedule, DispatchValues.
Chunk !=
nullptr, Ordered);
2514 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
2515 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&
2516 Schedule != OMP_sch_static_balanced_chunked));
2523 llvm::Value *Chunk = DispatchValues.
Chunk ? DispatchValues.
Chunk
2524 : CGF.
Builder.getIntN(IVSize, 1);
2525 llvm::Value *Args[] = {
2529 CGM, Schedule, ScheduleKind.
M1, ScheduleKind.
M2)),
2532 CGF.
Builder.getIntN(IVSize, 1),
2549 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
2550 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
2557 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||
2558 Schedule == OMP_sch_static_balanced_chunked ||
2559 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||
2560 Schedule == OMP_dist_sch_static ||
2561 Schedule == OMP_dist_sch_static_chunked);
2568 llvm::Value *Chunk = Values.
Chunk;
2569 if (Chunk ==
nullptr) {
2570 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
2571 Schedule == OMP_dist_sch_static) &&
2572 "expected static non-chunked schedule");
2576 assert((Schedule == OMP_sch_static_chunked ||
2577 Schedule == OMP_sch_static_balanced_chunked ||
2578 Schedule == OMP_ord_static_chunked ||
2579 Schedule == OMP_dist_sch_static_chunked) &&
2580 "expected static chunked schedule");
2582 llvm::Value *Args[] = {
2605 "Expected loop-based or sections-based directive.");
2608 ? OMP_IDENT_WORK_LOOP
2609 : OMP_IDENT_WORK_SECTIONS);
2611 llvm::FunctionCallee StaticInitFunction =
2616 ScheduleNum, ScheduleKind.
M1, ScheduleKind.
M2, Values);
2623 OpenMPSchedType ScheduleNum =
2625 llvm::Value *UpdatedLocation =
2628 llvm::FunctionCallee StaticInitFunction;
2629 bool isGPUDistribute =
2630 CGM.getLangOpts().OpenMPIsTargetDevice &&
CGM.getTriple().isGPU();
2631 StaticInitFunction =
OMPBuilder.createForStaticInitFunction(
2642 assert((DKind == OMPD_distribute || DKind == OMPD_for ||
2643 DKind == OMPD_sections) &&
2644 "Expected distribute, for, or sections directive kind");
2648 llvm::Value *Args[] = {
2651 (DKind == OMPD_target_teams_loop)
2652 ? OMP_IDENT_WORK_DISTRIBUTE
2654 ? OMP_IDENT_WORK_LOOP
2655 : OMP_IDENT_WORK_SECTIONS),
2659 CGM.getLangOpts().OpenMPIsTargetDevice &&
CGM.getTriple().isGPU())
2662 CGM.getModule(), OMPRTL___kmpc_distribute_static_fini),
2666 CGM.getModule(), OMPRTL___kmpc_for_static_fini),
2691 llvm::Value *Args[] = {
2699 OMPBuilder.createDispatchNextFunction(IVSize, IVSigned), Args);
2706 const Expr *Message) {
2708 return llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
2723 return llvm::ConstantInt::get(
CGM.Int32Ty,
2724 Severity == OMPC_SEVERITY_warning ? 1 : 2);
2736 const Expr *Message) {
2745 RuntimeFunction FnID = OMPRTL___kmpc_push_num_threads;
2746 if (Modifier == OMPC_NUMTHREADS_strict) {
2747 FnID = OMPRTL___kmpc_push_num_threads_strict;
2752 OMPBuilder.getOrCreateRuntimeFunction(
CGM.getModule(), FnID), Args);
2756 ProcBindKind ProcBind,
2760 assert(ProcBind != OMP_PROC_BIND_unknown &&
"Unsupported proc_bind value.");
2762 llvm::Value *Args[] = {
2764 llvm::ConstantInt::get(
CGM.IntTy,
unsigned(ProcBind),
true)};
2766 CGM.getModule(), OMPRTL___kmpc_push_proc_bind),
2779 CGM.getModule(), OMPRTL___kmpc_flush),
2786enum KmpTaskTFields {
2813 if (
CGM.getLangOpts().OpenMPSimd ||
OMPBuilder.OffloadInfoManager.empty())
2816 llvm::OpenMPIRBuilder::EmitMetadataErrorReportFunctionTy &&ErrorReportFn =
2817 [
this](llvm::OpenMPIRBuilder::EmitMetadataErrorKind Kind,
2818 const llvm::TargetRegionEntryInfo &EntryInfo) ->
void {
2820 if (Kind != llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR) {
2821 for (
auto I =
CGM.getContext().getSourceManager().fileinfo_begin(),
2822 E =
CGM.getContext().getSourceManager().fileinfo_end();
2824 if (I->getFirst().getUniqueID().getDevice() == EntryInfo.DeviceID &&
2825 I->getFirst().getUniqueID().getFile() == EntryInfo.FileID) {
2826 Loc =
CGM.getContext().getSourceManager().translateFileLineCol(
2827 I->getFirst(), EntryInfo.Line, 1);
2833 case llvm::OpenMPIRBuilder::EMIT_MD_TARGET_REGION_ERROR: {
2834 unsigned DiagID =
CGM.getDiags().getCustomDiagID(
2836 "%0 is incorrect: either the "
2837 "address or the ID is invalid.");
2838 CGM.getDiags().Report(Loc, DiagID) << EntryInfo.ParentName;
2840 case llvm::OpenMPIRBuilder::EMIT_MD_DECLARE_TARGET_ERROR: {
2841 unsigned DiagID =
CGM.getDiags().getCustomDiagID(
2843 "variable %0 is incorrect: the "
2844 "address is invalid.");
2845 CGM.getDiags().Report(Loc, DiagID) << EntryInfo.ParentName;
2847 case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR: {
2848 unsigned DiagID =
CGM.getDiags().getCustomDiagID(
2850 "Offloading entry for declare target variable is incorrect: the "
2851 "address is invalid.");
2852 CGM.getDiags().Report(DiagID);
2857 OMPBuilder.createOffloadEntriesAndInfoMetadata(ErrorReportFn);
2864 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty,
C.VoidPtrTy};
2867 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
2873struct PrivateHelpersTy {
2874 PrivateHelpersTy(
const Expr *OriginalRef,
const VarDecl *Original,
2876 : OriginalRef(OriginalRef), Original(Original), PrivateCopy(PrivateCopy),
2877 PrivateElemInit(PrivateElemInit) {}
2878 PrivateHelpersTy(
const VarDecl *Original) : Original(Original) {}
2879 const Expr *OriginalRef =
nullptr;
2880 const VarDecl *Original =
nullptr;
2881 const VarDecl *PrivateCopy =
nullptr;
2882 const VarDecl *PrivateElemInit =
nullptr;
2883 bool isLocalPrivate()
const {
2884 return !OriginalRef && !PrivateCopy && !PrivateElemInit;
2887typedef std::pair<CharUnits , PrivateHelpersTy> PrivateDataTy;
2892 if (!CVD->
hasAttr<OMPAllocateDeclAttr>())
2894 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
2896 return !(AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc &&
2897 !AA->getAllocator());
2902 if (!Privates.empty()) {
2907 RecordDecl *RD =
C.buildImplicitRecord(
".kmp_privates.t");
2909 for (
const auto &Pair : Privates) {
2910 const VarDecl *VD = Pair.second.Original;
2914 if (Pair.second.isLocalPrivate()) {
2937 QualType KmpRoutineEntryPointerQTy) {
2957 CanQualType KmpCmplrdataTy =
C.getCanonicalTagType(UD);
2958 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_task_t");
2988 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_task_t_with_privates");
3008static llvm::Function *
3011 QualType KmpTaskTWithPrivatesPtrQTy,
3013 QualType SharedsPtrTy, llvm::Function *TaskFunction,
3014 llvm::Value *TaskPrivatesMap) {
3022 Args.push_back(&GtidArg);
3023 Args.push_back(&TaskTypeArg);
3024 const auto &TaskEntryFnInfo =
3026 llvm::FunctionType *TaskEntryTy =
3029 auto *TaskEntry = llvm::Function::Create(
3030 TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.
getModule());
3032 TaskEntry->setDoesNotRecurse();
3047 const auto *KmpTaskTWithPrivatesQTyRD =
3052 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
3054 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF);
3056 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
3062 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
3063 llvm::Value *PrivatesParam;
3064 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
3067 PrivatesLVal.getPointer(CGF), CGF.
VoidPtrTy);
3069 PrivatesParam = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
3072 llvm::Value *CommonArgs[] = {
3073 GtidParam, PartidParam, PrivatesParam, TaskPrivatesMap,
3079 std::end(CommonArgs));
3081 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
3084 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
3087 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
3090 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
3093 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
3096 CallArgs.push_back(LBParam);
3097 CallArgs.push_back(UBParam);
3098 CallArgs.push_back(StParam);
3099 CallArgs.push_back(LIParam);
3100 CallArgs.push_back(RParam);
3102 CallArgs.push_back(SharedsParam);
3115 QualType KmpTaskTWithPrivatesPtrQTy,
3116 QualType KmpTaskTWithPrivatesQTy) {
3124 Args.push_back(&GtidArg);
3125 Args.push_back(&TaskTypeArg);
3126 const auto &DestructorFnInfo =
3128 llvm::FunctionType *DestructorFnTy =
3132 auto *DestructorFn =
3133 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
3137 DestructorFn->setDoesNotRecurse();
3145 const auto *KmpTaskTWithPrivatesQTyRD =
3147 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
3149 for (
const auto *Field : FI->getType()->castAsRecordDecl()->fields()) {
3151 Field->getType().isDestructedType()) {
3153 CGF.
pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType());
3157 return DestructorFn;
3177 C,
nullptr, Loc,
nullptr,
3178 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
3180 Args.push_back(&TaskPrivatesArg);
3181 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>,
unsigned> PrivateVarsPos;
3182 unsigned Counter = 1;
3183 for (
const Expr *E :
Data.PrivateVars) {
3185 C,
nullptr, Loc,
nullptr,
3186 C.getPointerType(
C.getPointerType(E->
getType()))
3191 PrivateVarsPos[VD] = Counter;
3194 for (
const Expr *E :
Data.FirstprivateVars) {
3196 C,
nullptr, Loc,
nullptr,
3197 C.getPointerType(
C.getPointerType(E->
getType()))
3202 PrivateVarsPos[VD] = Counter;
3205 for (
const Expr *E :
Data.LastprivateVars) {
3207 C,
nullptr, Loc,
nullptr,
3208 C.getPointerType(
C.getPointerType(E->
getType()))
3213 PrivateVarsPos[VD] = Counter;
3219 Ty =
C.getPointerType(Ty);
3221 Ty =
C.getPointerType(Ty);
3223 C,
nullptr, Loc,
nullptr,
3224 C.getPointerType(
C.getPointerType(Ty)).withConst().withRestrict(),
3226 PrivateVarsPos[VD] = Counter;
3229 const auto &TaskPrivatesMapFnInfo =
3231 llvm::FunctionType *TaskPrivatesMapTy =
3235 auto *TaskPrivatesMap = llvm::Function::Create(
3236 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
3239 TaskPrivatesMapFnInfo);
3241 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
3242 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
3243 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
3247 TaskPrivatesMapFnInfo, Args, Loc, Loc);
3255 for (
const FieldDecl *Field : PrivatesQTyRD->fields()) {
3257 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
3261 RefLVal.getAddress(), RefLVal.getType()->castAs<
PointerType>());
3266 return TaskPrivatesMap;
3272 Address KmpTaskSharedsPtr, LValue TDBase,
3278 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
3292 if ((!IsTargetTask && !
Data.FirstprivateVars.empty() && ForDup) ||
3293 (IsTargetTask && KmpTaskSharedsPtr.
isValid())) {
3300 FI = FI->getType()->castAsRecordDecl()->field_begin();
3301 for (
const PrivateDataTy &Pair : Privates) {
3303 if (Pair.second.isLocalPrivate()) {
3307 const VarDecl *VD = Pair.second.PrivateCopy;
3312 if (
const VarDecl *Elem = Pair.second.PrivateElemInit) {
3313 const VarDecl *OriginalVD = Pair.second.Original;
3316 LValue SharedRefLValue;
3319 if (IsTargetTask && !SharedField) {
3323 ->getNumParams() == 0 &&
3326 ->getDeclContext()) &&
3327 "Expected artificial target data variable.");
3330 }
else if (ForDup) {
3333 SharedRefLValue.getAddress().withAlignment(
3334 C.getDeclAlign(OriginalVD)),
3336 SharedRefLValue.getTBAAInfo());
3338 Pair.second.Original->getCanonicalDecl()) > 0 ||
3340 SharedRefLValue = CGF.
EmitLValue(Pair.second.OriginalRef);
3343 InlinedOpenMPRegionRAII Region(
3346 SharedRefLValue = CGF.
EmitLValue(Pair.second.OriginalRef);
3357 PrivateLValue.getAddress(), SharedRefLValue.getAddress(),
Type,
3358 [&CGF, Elem,
Init, &CapturesInfo](
Address DestElement,
3361 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3362 InitScope.addPrivate(Elem, SrcElement);
3363 (void)InitScope.Privatize();
3365 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
3366 CGF, &CapturesInfo);
3367 CGF.EmitAnyExprToMem(Init, DestElement,
3368 Init->getType().getQualifiers(),
3374 InitScope.addPrivate(Elem, SharedRefLValue.getAddress());
3375 (void)InitScope.Privatize();
3391 bool InitRequired =
false;
3392 for (
const PrivateDataTy &Pair : Privates) {
3393 if (Pair.second.isLocalPrivate())
3395 const VarDecl *VD = Pair.second.PrivateCopy;
3397 InitRequired = InitRequired || (isa_and_nonnull<CXXConstructExpr>(
Init) &&
3402 return InitRequired;
3419 QualType KmpTaskTWithPrivatesPtrQTy,
3427 KmpTaskTWithPrivatesPtrQTy,
3430 KmpTaskTWithPrivatesPtrQTy,
3434 Args.push_back(&DstArg);
3435 Args.push_back(&SrcArg);
3436 Args.push_back(&LastprivArg);
3437 const auto &TaskDupFnInfo =
3441 auto *TaskDup = llvm::Function::Create(
3442 TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.
getModule());
3444 TaskDup->setDoesNotRecurse();
3454 auto LIFI = std::next(KmpTaskTQTyRD->
field_begin(), KmpTaskTLastIter);
3456 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
3464 assert(!Privates.empty());
3466 if (!
Data.FirstprivateVars.empty()) {
3471 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
3479 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
3480 SharedsTy, SharedsPtrTy,
Data, Privates,
true);
3490 for (
const PrivateDataTy &P : Privates) {
3491 if (P.second.isLocalPrivate())
3493 QualType Ty = P.second.Original->getType().getNonReferenceType();
3502class OMPIteratorGeneratorScope final
3504 CodeGenFunction &CGF;
3505 const OMPIteratorExpr *E =
nullptr;
3506 SmallVector<CodeGenFunction::JumpDest, 4> ContDests;
3507 SmallVector<CodeGenFunction::JumpDest, 4> ExitDests;
3508 OMPIteratorGeneratorScope() =
delete;
3509 OMPIteratorGeneratorScope(OMPIteratorGeneratorScope &) =
delete;
3512 OMPIteratorGeneratorScope(CodeGenFunction &CGF,
const OMPIteratorExpr *E)
3513 : CodeGenFunction::OMPPrivateScope(CGF), CGF(CGF), E(E) {
3516 SmallVector<llvm::Value *, 4> Uppers;
3518 Uppers.push_back(CGF.EmitScalarExpr(E->getHelper(I).Upper));
3519 const auto *VD = cast<VarDecl>(E->getIteratorDecl(I));
3520 addPrivate(VD, CGF.CreateMemTemp(VD->getType(), VD->getName()));
3521 const OMPIteratorHelperData &HelperData = E->getHelper(I);
3523 HelperData.CounterVD,
3524 CGF.CreateMemTemp(HelperData.CounterVD->getType(),
"counter.addr"));
3529 const OMPIteratorHelperData &HelperData = E->getHelper(I);
3531 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(HelperData.CounterVD),
3532 HelperData.CounterVD->getType());
3534 CGF.EmitStoreOfScalar(
3535 llvm::ConstantInt::get(CLVal.getAddress().getElementType(), 0),
3537 CodeGenFunction::JumpDest &ContDest =
3538 ContDests.emplace_back(CGF.getJumpDestInCurrentScope(
"iter.cont"));
3539 CodeGenFunction::JumpDest &ExitDest =
3540 ExitDests.emplace_back(CGF.getJumpDestInCurrentScope(
"iter.exit"));
3542 llvm::Value *N = Uppers[I];
3545 CGF.EmitBlock(ContDest.getBlock());
3547 CGF.EmitLoadOfScalar(CLVal, HelperData.CounterVD->getLocation());
3549 HelperData.CounterVD->getType()->isSignedIntegerOrEnumerationType()
3550 ? CGF.Builder.CreateICmpSLT(CVal, N)
3551 : CGF.Builder.CreateICmpULT(CVal, N);
3552 llvm::BasicBlock *BodyBB = CGF.createBasicBlock(
"iter.body");
3553 CGF.Builder.CreateCondBr(Cmp, BodyBB, ExitDest.getBlock());
3555 CGF.EmitBlock(BodyBB);
3557 CGF.EmitIgnoredExpr(HelperData.Update);
3560 ~OMPIteratorGeneratorScope() {
3565 const OMPIteratorHelperData &HelperData = E->
getHelper(I - 1);
3570 CGF.
EmitBlock(ExitDests[I - 1].getBlock(), I == 1);
3576static std::pair<llvm::Value *, llvm::Value *>
3578 const auto *OASE = dyn_cast<OMPArrayShapingExpr>(E);
3581 const Expr *
Base = OASE->getBase();
3586 llvm::Value *SizeVal;
3589 SizeVal = CGF.
getTypeSize(OASE->getBase()->getType()->getPointeeType());
3590 for (
const Expr *SE : OASE->getDimensions()) {
3594 SizeVal = CGF.
Builder.CreateNUWMul(SizeVal, Sz);
3596 }
else if (
const auto *ASE =
3599 Address UpAddrAddress = UpAddrLVal.getAddress();
3600 llvm::Value *UpAddr = CGF.
Builder.CreateConstGEP1_32(
3604 llvm::Value *UpIntPtr = CGF.
Builder.CreatePtrToInt(UpAddr, CGF.
SizeTy);
3605 SizeVal = CGF.
Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
3609 return std::make_pair(
Addr, SizeVal);
3614 QualType FlagsTy =
C.getIntTypeForBitwidth(32,
false);
3615 if (KmpTaskAffinityInfoTy.
isNull()) {
3617 C.buildImplicitRecord(
"kmp_task_affinity_info_t");
3623 KmpTaskAffinityInfoTy =
C.getCanonicalTagType(KmpAffinityInfoRD);
3630 llvm::Function *TaskFunction,
QualType SharedsTy,
3635 const auto *I =
Data.PrivateCopies.begin();
3636 for (
const Expr *E :
Data.PrivateVars) {
3638 Privates.emplace_back(
3644 I =
Data.FirstprivateCopies.begin();
3645 const auto *IElemInitRef =
Data.FirstprivateInits.begin();
3646 for (
const Expr *E :
Data.FirstprivateVars) {
3648 Privates.emplace_back(
3656 I =
Data.LastprivateCopies.begin();
3657 for (
const Expr *E :
Data.LastprivateVars) {
3659 Privates.emplace_back(
3667 Privates.emplace_back(
CGM.getPointerAlign(), PrivateHelpersTy(VD));
3669 Privates.emplace_back(
C.getDeclAlign(VD), PrivateHelpersTy(VD));
3671 llvm::stable_sort(Privates,
3672 [](
const PrivateDataTy &L,
const PrivateDataTy &R) {
3673 return L.first > R.first;
3675 QualType KmpInt32Ty =
C.getIntTypeForBitwidth(32, 1);
3686 assert((D.getDirectiveKind() == OMPD_task ||
3689 "Expected taskloop, task or target directive");
3696 const auto *KmpTaskTQTyRD =
KmpTaskTQTy->castAsRecordDecl();
3698 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
3701 C.getCanonicalTagType(KmpTaskTWithPrivatesQTyRD);
3702 QualType KmpTaskTWithPrivatesPtrQTy =
3703 C.getPointerType(KmpTaskTWithPrivatesQTy);
3704 llvm::Type *KmpTaskTWithPrivatesPtrTy = CGF.
Builder.getPtrTy(0);
3705 llvm::Value *KmpTaskTWithPrivatesTySize =
3707 QualType SharedsPtrTy =
C.getPointerType(SharedsTy);
3710 llvm::Value *TaskPrivatesMap =
nullptr;
3711 llvm::Type *TaskPrivatesMapTy =
3712 std::next(TaskFunction->arg_begin(), 3)->getType();
3713 if (!Privates.empty()) {
3714 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
3718 TaskPrivatesMap, TaskPrivatesMapTy);
3720 TaskPrivatesMap = llvm::ConstantPointerNull::get(
3726 CGM, Loc, D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
3727 KmpTaskTWithPrivatesQTy,
KmpTaskTQTy, SharedsPtrTy, TaskFunction,
3739 DestructorsFlag = 0x8,
3740 PriorityFlag = 0x20,
3741 DetachableFlag = 0x40,
3743 unsigned Flags =
Data.Tied ? TiedFlag : 0;
3744 bool NeedsCleanup =
false;
3745 if (!Privates.empty()) {
3749 Flags = Flags | DestructorsFlag;
3751 if (
Data.Priority.getInt())
3752 Flags = Flags | PriorityFlag;
3754 Flags = Flags | DetachableFlag;
3755 llvm::Value *TaskFlags =
3756 Data.Final.getPointer()
3757 ? CGF.
Builder.CreateSelect(
Data.Final.getPointer(),
3758 CGF.
Builder.getInt32(FinalFlag),
3760 : CGF.
Builder.getInt32(
Data.Final.getInt() ? FinalFlag : 0);
3761 TaskFlags = CGF.
Builder.CreateOr(TaskFlags, CGF.
Builder.getInt32(Flags));
3762 llvm::Value *SharedsSize =
CGM.getSize(
C.getTypeSizeInChars(SharedsTy));
3764 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
3767 llvm::Value *NewTask;
3774 llvm::Value *DeviceID;
3779 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
3780 AllocArgs.push_back(DeviceID);
3783 CGM.getModule(), OMPRTL___kmpc_omp_target_task_alloc),
3788 CGM.getModule(), OMPRTL___kmpc_omp_task_alloc),
3801 llvm::Value *Tid =
getThreadID(CGF, DC->getBeginLoc());
3802 Tid = CGF.
Builder.CreateIntCast(Tid, CGF.
IntTy,
false);
3805 CGM.getModule(), OMPRTL___kmpc_task_allow_completion_event),
3806 {Loc, Tid, NewTask});
3817 llvm::Value *NumOfElements =
nullptr;
3818 unsigned NumAffinities = 0;
3820 if (
const Expr *Modifier =
C->getModifier()) {
3822 for (
unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
3826 NumOfElements ? CGF.
Builder.CreateNUWMul(NumOfElements, Sz) : Sz;
3829 NumAffinities +=
C->varlist_size();
3834 enum RTLAffinityInfoFieldsTy { BaseAddr, Len, Flags };
3836 QualType KmpTaskAffinityInfoArrayTy;
3837 if (NumOfElements) {
3838 NumOfElements = CGF.
Builder.CreateNUWAdd(
3839 llvm::ConstantInt::get(CGF.
SizeTy, NumAffinities), NumOfElements);
3842 C.getIntTypeForBitwidth(
C.getTypeSize(
C.getSizeType()), 0),
3846 KmpTaskAffinityInfoArrayTy =
C.getVariableArrayType(
3854 NumOfElements = CGF.
Builder.CreateIntCast(NumOfElements, CGF.
Int32Ty,
3857 KmpTaskAffinityInfoArrayTy =
C.getConstantArrayType(
3859 llvm::APInt(
C.getTypeSize(
C.getSizeType()), NumAffinities),
nullptr,
3862 CGF.
CreateMemTemp(KmpTaskAffinityInfoArrayTy,
".affs.arr.addr");
3864 NumOfElements = llvm::ConstantInt::get(
CGM.Int32Ty, NumAffinities,
3871 bool HasIterator =
false;
3873 if (
C->getModifier()) {
3877 for (
const Expr *E :
C->varlist()) {
3886 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
3891 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
3905 const Expr *Modifier =
C->getModifier();
3908 OMPIteratorGeneratorScope IteratorScope(
3910 for (
const Expr *E :
C->varlist()) {
3920 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
3925 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
3927 Idx = CGF.
Builder.CreateNUWAdd(
3928 Idx, llvm::ConstantInt::get(Idx->getType(), 1));
3943 CGM.getModule(), OMPRTL___kmpc_omp_reg_task_with_affinity),
3944 {LocRef, GTid, NewTask, NumOfElements, AffinListPtr});
3946 llvm::Value *NewTaskNewTaskTTy =
3948 NewTask, KmpTaskTWithPrivatesPtrTy);
3950 KmpTaskTWithPrivatesQTy);
3961 *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds)),
3963 CGF.
Int8Ty,
CGM.getNaturalTypeAlignment(SharedsTy));
3970 if (!Privates.empty()) {
3972 SharedsTy, SharedsPtrTy,
Data, Privates,
3977 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
3978 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy,
Data, Privates,
3979 !
Data.LastprivateVars.empty());
3983 enum { Priority = 0, Destructors = 1 };
3985 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
3986 const auto *KmpCmplrdataUD = (*FI)->getType()->castAsRecordDecl();
3987 assert(KmpCmplrdataUD->isUnion());
3990 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
3991 KmpTaskTWithPrivatesQTy);
3994 Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
4000 if (
Data.Priority.getInt()) {
4002 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
4004 Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority));
4007 Result.NewTask = NewTask;
4008 Result.TaskEntry = TaskEntry;
4009 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
4011 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
4017 RTLDependenceKindTy DepKind;
4019 case OMPC_DEPEND_in:
4020 DepKind = RTLDependenceKindTy::DepIn;
4023 case OMPC_DEPEND_out:
4024 case OMPC_DEPEND_inout:
4025 DepKind = RTLDependenceKindTy::DepInOut;
4027 case OMPC_DEPEND_mutexinoutset:
4028 DepKind = RTLDependenceKindTy::DepMutexInOutSet;
4030 case OMPC_DEPEND_inoutset:
4031 DepKind = RTLDependenceKindTy::DepInOutSet;
4033 case OMPC_DEPEND_outallmemory:
4034 DepKind = RTLDependenceKindTy::DepOmpAllMem;
4036 case OMPC_DEPEND_source:
4037 case OMPC_DEPEND_sink:
4038 case OMPC_DEPEND_depobj:
4039 case OMPC_DEPEND_inoutallmemory:
4041 llvm_unreachable(
"Unknown task dependence type");
4049 FlagsTy =
C.getIntTypeForBitwidth(
C.getTypeSize(
C.BoolTy),
false);
4050 if (KmpDependInfoTy.
isNull()) {
4051 RecordDecl *KmpDependInfoRD =
C.buildImplicitRecord(
"kmp_depend_info");
4057 KmpDependInfoTy =
C.getCanonicalTagType(KmpDependInfoRD);
4061std::pair<llvm::Value *, LValue>
4074 CGF,
Base.getAddress(),
4075 llvm::ConstantInt::get(CGF.
IntPtrTy, -1,
true));
4081 *std::next(KmpDependInfoRD->field_begin(),
4082 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4084 return std::make_pair(NumDeps,
Base);
4088 llvm::PointerUnion<unsigned *, LValue *> Pos,
4098 OMPIteratorGeneratorScope IteratorScope(
4099 CGF, cast_or_null<OMPIteratorExpr>(
4100 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4102 for (
const Expr *E :
Data.DepExprs) {
4112 Size = llvm::ConstantInt::get(CGF.
SizeTy, 0);
4115 if (
unsigned *P = dyn_cast<unsigned *>(Pos)) {
4119 assert(E &&
"Expected a non-null expression");
4128 *std::next(KmpDependInfoRD->field_begin(),
4129 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4133 Base, *std::next(KmpDependInfoRD->field_begin(),
4134 static_cast<unsigned int>(RTLDependInfoFields::Len)));
4140 *std::next(KmpDependInfoRD->field_begin(),
4141 static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4143 llvm::ConstantInt::get(LLVMFlagsTy,
static_cast<unsigned int>(DepKind)),
4145 if (
unsigned *P = dyn_cast<unsigned *>(Pos)) {
4150 Idx = CGF.
Builder.CreateNUWAdd(Idx,
4151 llvm::ConstantInt::get(Idx->getType(), 1));
4160 assert(
Data.DepKind == OMPC_DEPEND_depobj &&
4161 "Expected depobj dependency kind.");
4166 OMPIteratorGeneratorScope IteratorScope(
4167 CGF, cast_or_null<OMPIteratorExpr>(
4168 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4170 for (
const Expr *E :
Data.DepExprs) {
4171 llvm::Value *NumDeps;
4174 std::tie(NumDeps,
Base) =
4178 C.getUIntPtrType());
4182 llvm::Value *Add = CGF.
Builder.CreateNUWAdd(PrevVal, NumDeps);
4184 SizeLVals.push_back(NumLVal);
4187 for (
unsigned I = 0, E = SizeLVals.size(); I < E; ++I) {
4190 Sizes.push_back(Size);
4200 assert(
Data.DepKind == OMPC_DEPEND_depobj &&
4201 "Expected depobj dependency kind.");
4204 OMPIteratorGeneratorScope IteratorScope(
4205 CGF, cast_or_null<OMPIteratorExpr>(
4206 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4208 for (
const Expr *E :
Data.DepExprs) {
4209 llvm::Value *NumDeps;
4212 std::tie(NumDeps,
Base) =
4216 llvm::Value *Size = CGF.
Builder.CreateNUWMul(
4225 llvm::Value *Add = CGF.
Builder.CreateNUWAdd(Pos, NumDeps);
4241 llvm::Value *NumOfElements =
nullptr;
4242 unsigned NumDependencies = std::accumulate(
4243 Dependencies.begin(), Dependencies.end(), 0,
4245 return D.DepKind == OMPC_DEPEND_depobj
4247 : (V + (D.IteratorExpr ? 0 : D.DepExprs.size()));
4251 bool HasDepobjDeps =
false;
4252 bool HasRegularWithIterators =
false;
4253 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4254 llvm::Value *NumOfRegularWithIterators =
4255 llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4259 if (D.
DepKind == OMPC_DEPEND_depobj) {
4262 for (llvm::Value *Size : Sizes) {
4263 NumOfDepobjElements =
4264 CGF.
Builder.CreateNUWAdd(NumOfDepobjElements, Size);
4266 HasDepobjDeps =
true;
4271 if (
const auto *IE = cast_or_null<OMPIteratorExpr>(D.
IteratorExpr)) {
4272 llvm::Value *ClauseIteratorSpace =
4273 llvm::ConstantInt::get(CGF.
IntPtrTy, 1);
4277 ClauseIteratorSpace = CGF.
Builder.CreateNUWMul(Sz, ClauseIteratorSpace);
4279 llvm::Value *NumClauseDeps = CGF.
Builder.CreateNUWMul(
4280 ClauseIteratorSpace,
4282 NumOfRegularWithIterators =
4283 CGF.
Builder.CreateNUWAdd(NumOfRegularWithIterators, NumClauseDeps);
4284 HasRegularWithIterators =
true;
4290 if (HasDepobjDeps || HasRegularWithIterators) {
4291 NumOfElements = llvm::ConstantInt::get(
CGM.IntPtrTy, NumDependencies,
4293 if (HasDepobjDeps) {
4295 CGF.
Builder.CreateNUWAdd(NumOfDepobjElements, NumOfElements);
4297 if (HasRegularWithIterators) {
4299 CGF.
Builder.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements);
4302 Loc,
C.getIntTypeForBitwidth(64, 0),
4306 KmpDependInfoArrayTy =
4315 NumOfElements = CGF.
Builder.CreateIntCast(NumOfElements, CGF.
Int32Ty,
4318 KmpDependInfoArrayTy =
C.getConstantArrayType(
4324 NumOfElements = llvm::ConstantInt::get(
CGM.Int32Ty, NumDependencies,
4329 if (Dep.DepKind == OMPC_DEPEND_depobj || Dep.IteratorExpr)
4335 CGF.
CreateMemTemp(
C.getSizeType(),
"dep.counter.addr"),
C.getSizeType());
4338 if (Dep.DepKind == OMPC_DEPEND_depobj || !Dep.IteratorExpr)
4343 if (HasDepobjDeps) {
4345 if (Dep.DepKind != OMPC_DEPEND_depobj)
4352 return std::make_pair(NumOfElements, DependenciesArray);
4363 unsigned NumDependencies = Dependencies.
DepExprs.size();
4373 llvm::Value *NumDepsVal;
4375 if (
const auto *IE =
4376 cast_or_null<OMPIteratorExpr>(Dependencies.
IteratorExpr)) {
4377 NumDepsVal = llvm::ConstantInt::get(CGF.
SizeTy, 1);
4381 NumDepsVal = CGF.
Builder.CreateNUWMul(NumDepsVal, Sz);
4383 Size = CGF.
Builder.CreateNUWAdd(llvm::ConstantInt::get(CGF.
SizeTy, 1),
4387 llvm::Value *RecSize =
CGM.getSize(SizeInBytes);
4388 Size = CGF.
Builder.CreateNUWMul(Size, RecSize);
4392 QualType KmpDependInfoArrayTy =
C.getConstantArrayType(
4395 CharUnits Sz =
C.getTypeSizeInChars(KmpDependInfoArrayTy);
4397 NumDepsVal = llvm::ConstantInt::get(CGF.
IntPtrTy, NumDependencies);
4402 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4403 llvm::Value *Args[] = {ThreadID, Size, Allocator};
4407 CGM.getModule(), OMPRTL___kmpc_alloc),
4408 Args,
".dep.arr.addr");
4412 DependenciesArray =
Address(
Addr, KmpDependInfoLlvmTy, Align);
4418 *std::next(KmpDependInfoRD->field_begin(),
4419 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4421 llvm::PointerUnion<unsigned *, LValue *> Pos;
4438 return DependenciesArray;
4453 Addr.getElementType(),
Addr.emitRawPointer(CGF),
4454 llvm::ConstantInt::get(CGF.
IntPtrTy, -1,
true));
4459 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4460 llvm::Value *Args[] = {ThreadID, DepObjAddr, Allocator};
4464 CGM.getModule(), OMPRTL___kmpc_free),
4476 llvm::Value *NumDeps;
4487 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
4489 llvm::PHINode *ElementPHI =
4494 Base.getTBAAInfo());
4498 Base, *std::next(KmpDependInfoRD->field_begin(),
4499 static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4501 llvm::ConstantInt::get(LLVMFlagsTy,
static_cast<unsigned int>(DepKind)),
4505 llvm::Value *ElementNext =
4508 ElementPHI->addIncoming(ElementNext, CGF.
Builder.GetInsertBlock());
4509 llvm::Value *IsEmpty =
4510 CGF.
Builder.CreateICmpEQ(ElementNext, End,
"omp.isempty");
4511 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
4518 llvm::Function *TaskFunction,
4527 llvm::Value *NewTask =
Result.NewTask;
4528 llvm::Function *TaskEntry =
Result.TaskEntry;
4529 llvm::Value *NewTaskNewTaskTTy =
Result.NewTaskNewTaskTTy;
4534 llvm::Value *NumOfElements;
4535 std::tie(NumOfElements, DependenciesArray) =
4546 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
4547 llvm::Value *DepTaskArgs[7];
4548 if (!
Data.Dependences.empty()) {
4549 DepTaskArgs[0] = UpLoc;
4550 DepTaskArgs[1] = ThreadID;
4551 DepTaskArgs[2] = NewTask;
4552 DepTaskArgs[3] = NumOfElements;
4554 DepTaskArgs[5] = CGF.
Builder.getInt32(0);
4555 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4557 auto &&ThenCodeGen = [
this, &
Data, TDBase, KmpTaskTQTyRD, &TaskArgs,
4560 auto PartIdFI = std::next(KmpTaskTQTyRD->
field_begin(), KmpTaskTPartId);
4564 if (!
Data.Dependences.empty()) {
4567 CGM.getModule(), OMPRTL___kmpc_omp_task_with_deps),
4571 CGM.getModule(), OMPRTL___kmpc_omp_task),
4577 Region->emitUntiedSwitch(CGF);
4580 llvm::Value *DepWaitTaskArgs[7];
4581 if (!
Data.Dependences.empty()) {
4582 DepWaitTaskArgs[0] = UpLoc;
4583 DepWaitTaskArgs[1] = ThreadID;
4584 DepWaitTaskArgs[2] = NumOfElements;
4586 DepWaitTaskArgs[4] = CGF.
Builder.getInt32(0);
4587 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4588 DepWaitTaskArgs[6] =
4589 llvm::ConstantInt::get(CGF.
Int32Ty,
Data.HasNowaitClause);
4591 auto &M =
CGM.getModule();
4592 auto &&ElseCodeGen = [
this, &M, &TaskArgs, ThreadID, NewTaskNewTaskTTy,
4593 TaskEntry, &
Data, &DepWaitTaskArgs,
4600 if (!
Data.Dependences.empty())
4602 M, OMPRTL___kmpc_omp_taskwait_deps_51),
4605 auto &&
CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
4608 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
4609 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
4618 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
4619 M, OMPRTL___kmpc_omp_task_begin_if0),
4622 M, OMPRTL___kmpc_omp_task_complete_if0),
4638 llvm::Function *TaskFunction,
4658 IfVal = llvm::ConstantInt::getSigned(CGF.
IntTy, 1);
4663 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
4670 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
4677 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
4685 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
4686 if (
Data.Reductions) {
4692 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
4701 llvm::ConstantInt::getSigned(
4703 llvm::ConstantInt::getSigned(
4705 ?
Data.Schedule.getInt() ? NumTasks : Grainsize
4707 Data.Schedule.getPointer()
4710 : llvm::ConstantInt::get(CGF.
Int64Ty, 0)};
4711 if (
Data.HasModifier)
4712 TaskArgs.push_back(llvm::ConstantInt::get(CGF.
Int32Ty, 1));
4714 TaskArgs.push_back(
Result.TaskDupFn
4717 : llvm::ConstantPointerNull::get(CGF.
VoidPtrTy));
4719 CGM.getModule(),
Data.HasModifier
4720 ? OMPRTL___kmpc_taskloop_5
4721 : OMPRTL___kmpc_taskloop),
4738 const Expr *,
const Expr *)> &RedOpGen,
4739 const Expr *XExpr =
nullptr,
const Expr *EExpr =
nullptr,
4740 const Expr *UpExpr =
nullptr) {
4748 llvm::Value *NumElements = CGF.
emitArrayLength(ArrayTy, ElementTy, LHSAddr);
4753 llvm::Value *LHSEnd =
4758 llvm::Value *IsEmpty =
4759 CGF.
Builder.CreateICmpEQ(LHSBegin, LHSEnd,
"omp.arraycpy.isempty");
4760 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
4763 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
4768 llvm::PHINode *RHSElementPHI = CGF.
Builder.CreatePHI(
4769 RHSBegin->getType(), 2,
"omp.arraycpy.srcElementPast");
4770 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
4775 llvm::PHINode *LHSElementPHI = CGF.
Builder.CreatePHI(
4776 LHSBegin->getType(), 2,
"omp.arraycpy.destElementPast");
4777 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
4784 Scope.addPrivate(LHSVar, LHSElementCurrent);
4785 Scope.addPrivate(RHSVar, RHSElementCurrent);
4787 RedOpGen(CGF, XExpr, EExpr, UpExpr);
4788 Scope.ForceCleanup();
4791 llvm::Value *LHSElementNext = CGF.
Builder.CreateConstGEP1_32(
4793 "omp.arraycpy.dest.element");
4794 llvm::Value *RHSElementNext = CGF.
Builder.CreateConstGEP1_32(
4796 "omp.arraycpy.src.element");
4799 CGF.
Builder.CreateICmpEQ(LHSElementNext, LHSEnd,
"omp.arraycpy.done");
4800 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
4801 LHSElementPHI->addIncoming(LHSElementNext, CGF.
Builder.GetInsertBlock());
4802 RHSElementPHI->addIncoming(RHSElementNext, CGF.
Builder.GetInsertBlock());
4812 const Expr *ReductionOp) {
4813 if (
const auto *CE = dyn_cast<CallExpr>(ReductionOp))
4814 if (
const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
4815 if (
const auto *DRE =
4816 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
4817 if (
const auto *DRD =
4818 dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
4819 std::pair<llvm::Function *, llvm::Function *>
Reduction =
4830 StringRef ReducerName,
SourceLocation Loc, llvm::Type *ArgsElemType,
4841 Args.push_back(&LHSArg);
4842 Args.push_back(&RHSArg);
4844 CGM.getTypes().arrangeBuiltinFunctionDeclaration(
C.VoidTy, Args);
4846 auto *Fn = llvm::Function::Create(
CGM.getTypes().GetFunctionType(CGFI),
4847 llvm::GlobalValue::InternalLinkage, Name,
4850 Fn->setDoesNotRecurse();
4869 const auto *IPriv = Privates.begin();
4871 for (
unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
4872 const auto *RHSVar =
4875 const auto *LHSVar =
4878 QualType PrivTy = (*IPriv)->getType();
4893 IPriv = Privates.begin();
4894 const auto *ILHS = LHSExprs.begin();
4895 const auto *IRHS = RHSExprs.begin();
4896 for (
const Expr *E : ReductionOps) {
4897 if ((*IPriv)->getType()->isArrayType()) {
4902 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
4904 emitReductionCombiner(CGF, E);
4914 Scope.ForceCleanup();
4920 const Expr *ReductionOp,
4921 const Expr *PrivateRef,
4929 CGF, PrivateRef->
getType(), LHSVar, RHSVar,
4931 emitReductionCombiner(CGF, ReductionOp);
4940 llvm::StringRef Prefix,
const Expr *Ref);
4944 const Expr *LHSExprs,
const Expr *RHSExprs,
const Expr *ReductionOps) {
4971 std::string ReductionVarNameStr;
4973 ReductionVarNameStr =
4976 ReductionVarNameStr =
"unnamed_priv_var";
4979 std::string SharedName =
4980 CGM.getOpenMPRuntime().getName({
"internal_pivate_", ReductionVarNameStr});
4981 llvm::GlobalVariable *SharedVar =
OMPBuilder.getOrCreateInternalVariable(
4982 LLVMType,
".omp.reduction." + SharedName);
4984 SharedVar->setAlignment(
4992 llvm::Value *BarrierArgs[] = {BarrierLoc, ThreadId};
4997 llvm::Value *IsWorker = CGF.
Builder.CreateICmpEQ(
4998 ThreadId, llvm::ConstantInt::get(ThreadId->getType(), 0));
4999 CGF.
Builder.CreateCondBr(IsWorker, InitBB, InitEndBB);
5003 auto EmitSharedInit = [&]() {
5006 std::pair<llvm::Function *, llvm::Function *> FnPair =
5008 llvm::Function *InitializerFn = FnPair.second;
5009 if (InitializerFn) {
5010 if (
const auto *CE =
5011 dyn_cast<CallExpr>(UDRInitExpr->IgnoreParenImpCasts())) {
5018 LocalScope.addPrivate(OutVD, SharedResult);
5020 (void)LocalScope.Privatize();
5021 if (
const auto *OVE = dyn_cast<OpaqueValueExpr>(
5022 CE->getCallee()->IgnoreParenImpCasts())) {
5048 if (
const auto *DRE = dyn_cast<DeclRefExpr>(Privates)) {
5049 if (
const auto *VD = dyn_cast<VarDecl>(DRE->getDecl())) {
5060 CGF.
Builder.CreateBr(InitEndBB);
5064 CGM.getModule(), OMPRTL___kmpc_barrier),
5067 const Expr *ReductionOp = ReductionOps;
5072 auto EmitCriticalReduction = [&](
auto ReductionGen) {
5073 std::string CriticalName =
getName({
"reduction_critical"});
5081 std::pair<llvm::Function *, llvm::Function *> FnPair =
5084 if (
const auto *CE = dyn_cast<CallExpr>(ReductionOp)) {
5096 (void)LocalScope.Privatize();
5101 EmitCriticalReduction(ReductionGen);
5106 if (
const auto *Cleanup = dyn_cast<ExprWithCleanups>(ReductionClauseExpr))
5109 const Expr *AssignRHS =
nullptr;
5110 if (
const auto *BinOp = dyn_cast<BinaryOperator>(ReductionClauseExpr)) {
5111 if (BinOp->getOpcode() == BO_Assign)
5112 AssignRHS = BinOp->getRHS();
5113 }
else if (
const auto *OpCall =
5114 dyn_cast<CXXOperatorCallExpr>(ReductionClauseExpr)) {
5115 if (OpCall->getOperator() == OO_Equal)
5116 AssignRHS = OpCall->getArg(1);
5120 "Private Variable Reduction : Invalid ReductionOp expression");
5125 const auto *OmpOutDRE =
5127 const auto *OmpInDRE =
5130 OmpOutDRE && OmpInDRE &&
5131 "Private Variable Reduction : LHSExpr/RHSExpr must be DeclRefExprs");
5135 LocalScope.addPrivate(OmpOutVD, SharedLV.
getAddress());
5136 LocalScope.addPrivate(OmpInVD, LHSLV.
getAddress());
5137 (void)LocalScope.Privatize();
5141 EmitCriticalReduction(ReductionGen);
5145 CGM.getModule(), OMPRTL___kmpc_barrier),
5151 llvm::Value *FinalResultVal =
nullptr;
5155 FinalResultAddr = SharedResult;
5169 CGM.getModule(), OMPRTL___kmpc_barrier),
5180 EmitCriticalReduction(OriginalListCombiner);
5232 if (SimpleReduction) {
5234 const auto *IPriv = OrgPrivates.begin();
5235 const auto *ILHS = OrgLHSExprs.begin();
5236 const auto *IRHS = OrgRHSExprs.begin();
5237 for (
const Expr *E : OrgReductionOps) {
5250 FilteredRHSExprs, FilteredReductionOps;
5251 for (
unsigned I : llvm::seq<unsigned>(
5252 std::min(OrgReductionOps.size(), OrgLHSExprs.size()))) {
5254 FilteredPrivates.emplace_back(OrgPrivates[I]);
5255 FilteredLHSExprs.emplace_back(OrgLHSExprs[I]);
5256 FilteredRHSExprs.emplace_back(OrgRHSExprs[I]);
5257 FilteredReductionOps.emplace_back(OrgReductionOps[I]);
5269 auto Size = RHSExprs.size();
5270 for (
const Expr *E : Privates) {
5275 llvm::APInt ArraySize(32, Size);
5276 QualType ReductionArrayTy =
C.getConstantArrayType(
5280 CGF.
CreateMemTemp(ReductionArrayTy,
".omp.reduction.red_list");
5281 const auto *IPriv = Privates.begin();
5283 for (
unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
5289 if ((*IPriv)->getType()->isVariablyModifiedType()) {
5293 llvm::Value *Size = CGF.
Builder.CreateIntCast(
5306 Privates, LHSExprs, RHSExprs, ReductionOps);
5309 std::string Name =
getName({
"reduction"});
5316 llvm::Value *ReductionArrayTySize = CGF.
getTypeSize(ReductionArrayTy);
5319 llvm::Value *Args[] = {
5322 CGF.
Builder.getInt32(RHSExprs.size()),
5323 ReductionArrayTySize,
5331 WithNowait ? OMPRTL___kmpc_reduce_nowait : OMPRTL___kmpc_reduce),
5335 llvm::BasicBlock *DefaultBB = CGF.
createBasicBlock(
".omp.reduction.default");
5336 llvm::SwitchInst *SwInst =
5337 CGF.
Builder.CreateSwitch(Res, DefaultBB, 2);
5346 SwInst->addCase(CGF.
Builder.getInt32(1), Case1BB);
5350 llvm::Value *EndArgs[] = {
5355 auto &&
CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps](
5358 const auto *IPriv = Privates.begin();
5359 const auto *ILHS = LHSExprs.begin();
5360 const auto *IRHS = RHSExprs.begin();
5361 for (
const Expr *E : ReductionOps) {
5370 CommonActionTy Action(
5373 CGM.getModule(), WithNowait ? OMPRTL___kmpc_end_reduce_nowait
5374 : OMPRTL___kmpc_end_reduce),
5387 SwInst->addCase(CGF.
Builder.getInt32(2), Case2BB);
5390 auto &&AtomicCodeGen = [Loc, Privates, LHSExprs, RHSExprs, ReductionOps](
5392 const auto *ILHS = LHSExprs.begin();
5393 const auto *IRHS = RHSExprs.begin();
5394 const auto *IPriv = Privates.begin();
5395 for (
const Expr *E : ReductionOps) {
5396 const Expr *XExpr =
nullptr;
5397 const Expr *EExpr =
nullptr;
5398 const Expr *UpExpr =
nullptr;
5400 if (
const auto *BO = dyn_cast<BinaryOperator>(E)) {
5401 if (BO->getOpcode() == BO_Assign) {
5402 XExpr = BO->getLHS();
5403 UpExpr = BO->getRHS();
5407 const Expr *RHSExpr = UpExpr;
5410 if (
const auto *ACO = dyn_cast<AbstractConditionalOperator>(
5414 RHSExpr = ACO->getCond();
5416 if (
const auto *BORHS =
5418 EExpr = BORHS->getRHS();
5419 BO = BORHS->getOpcode();
5424 auto &&AtomicRedGen = [BO, VD,
5426 const Expr *EExpr,
const Expr *UpExpr) {
5427 LValue X = CGF.EmitLValue(XExpr);
5430 E = CGF.EmitAnyExpr(EExpr);
5431 CGF.EmitOMPAtomicSimpleUpdateExpr(
5433 llvm::AtomicOrdering::Monotonic, Loc,
5434 [&CGF, UpExpr, VD, Loc](
RValue XRValue) {
5436 Address LHSTemp = CGF.CreateMemTemp(VD->getType());
5437 CGF.emitOMPSimpleStore(
5438 CGF.MakeAddrLValue(LHSTemp, VD->
getType()), XRValue,
5439 VD->getType().getNonReferenceType(), Loc);
5442 return CGF.EmitAnyExpr(UpExpr);
5445 if ((*IPriv)->getType()->isArrayType()) {
5447 const auto *RHSVar =
5450 AtomicRedGen, XExpr, EExpr, UpExpr);
5453 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5460 std::string Name = RT.
getName({
"atomic_reduction"});
5469 if ((*IPriv)->getType()->isArrayType()) {
5470 const auto *LHSVar =
5472 const auto *RHSVar =
5477 CritRedGen(CGF,
nullptr,
nullptr,
nullptr);
5488 llvm::Value *EndArgs[] = {
5493 CommonActionTy Action(
nullptr, {},
5495 CGM.getModule(), OMPRTL___kmpc_end_reduce),
5505 assert(OrgLHSExprs.size() == OrgPrivates.size() &&
5506 "PrivateVarReduction: Privates size mismatch");
5507 assert(OrgLHSExprs.size() == OrgReductionOps.size() &&
5508 "PrivateVarReduction: ReductionOps size mismatch");
5509 for (
unsigned I : llvm::seq<unsigned>(
5510 std::min(OrgReductionOps.size(), OrgLHSExprs.size()))) {
5513 OrgRHSExprs[I], OrgReductionOps[I]);
5522 llvm::raw_svector_ostream Out(Buffer);
5530 Out << Prefix << Name <<
"_"
5532 return std::string(Out.str());
5554 Args.emplace_back(&Param);
5555 Args.emplace_back(&ParamOrig);
5556 const auto &FnInfo =
5560 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5563 Fn->setDoesNotRecurse();
5570 llvm::Value *Size =
nullptr;
5613 const Expr *ReductionOp,
5615 const Expr *PrivateRef) {
5624 Args.emplace_back(&ParamInOut);
5625 Args.emplace_back(&ParamIn);
5626 const auto &FnInfo =
5630 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5633 Fn->setDoesNotRecurse();
5636 llvm::Value *Size =
nullptr;
5657 C.getPointerType(LHSVD->getType())->castAs<
PointerType>()));
5664 C.getPointerType(RHSVD->getType())->castAs<
PointerType>()));
5693 Args.emplace_back(&Param);
5694 const auto &FnInfo =
5698 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5701 Fn->setDoesNotRecurse();
5706 llvm::Value *Size =
nullptr;
5741 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_taskred_input_t");
5750 C, RD,
C.getIntTypeForBitwidth(32,
false));
5753 unsigned Size =
Data.ReductionVars.size();
5754 llvm::APInt ArraySize(64, Size);
5756 C.getConstantArrayType(RDType, ArraySize,
nullptr,
5761 Data.ReductionCopies,
Data.ReductionOps);
5762 for (
unsigned Cnt = 0; Cnt < Size; ++Cnt) {
5764 llvm::Value *Idxs[] = {llvm::ConstantInt::get(
CGM.SizeTy, 0),
5765 llvm::ConstantInt::get(
CGM.SizeTy, Cnt)};
5781 llvm::Value *SizeValInChars;
5782 llvm::Value *SizeVal;
5783 std::tie(SizeValInChars, SizeVal) = RCG.
getSizes(Cnt);
5789 bool DelayedCreation = !!SizeVal;
5790 SizeValInChars = CGF.
Builder.CreateIntCast(SizeValInChars,
CGM.SizeTy,
5801 llvm::Value *FiniAddr =
5802 Fini ? Fini : llvm::ConstantPointerNull::get(
CGM.VoidPtrTy);
5807 CGM, Loc, RCG, Cnt,
Data.ReductionOps[Cnt], LHSExprs[Cnt],
5808 RHSExprs[Cnt],
Data.ReductionCopies[Cnt]);
5812 if (DelayedCreation) {
5814 llvm::ConstantInt::get(
CGM.Int32Ty, 1,
true),
5819 if (
Data.IsReductionWithTaskMod) {
5825 llvm::Value *Args[] = {
5827 llvm::ConstantInt::get(
CGM.IntTy,
Data.IsWorksharingReduction ? 1 : 0,
5829 llvm::ConstantInt::get(
CGM.IntTy, Size,
true),
5834 CGM.getModule(), OMPRTL___kmpc_taskred_modifier_init),
5838 llvm::Value *Args[] = {
5841 llvm::ConstantInt::get(
CGM.IntTy, Size,
true),
5845 CGM.getModule(), OMPRTL___kmpc_taskred_init),
5851 bool IsWorksharingReduction) {
5857 llvm::Value *Args[] = {IdentTLoc, GTid,
5858 llvm::ConstantInt::get(
CGM.IntTy,
5859 IsWorksharingReduction ? 1 : 0,
5863 CGM.getModule(), OMPRTL___kmpc_task_reduction_modifier_fini),
5875 llvm::Value *SizeVal = CGF.
Builder.CreateIntCast(Sizes.second,
CGM.SizeTy,
5878 CGF,
CGM.getContext().getSizeType(),
5886 llvm::Value *ReductionsPtr,
5899 CGM.getModule(), OMPRTL___kmpc_task_reduction_get_th_data),
5915 auto &M =
CGM.getModule();
5917 llvm::Value *NumOfElements;
5918 std::tie(NumOfElements, DependenciesArray) =
5920 if (!
Data.Dependences.empty()) {
5921 llvm::Value *DepWaitTaskArgs[7];
5922 DepWaitTaskArgs[0] = UpLoc;
5923 DepWaitTaskArgs[1] = ThreadID;
5924 DepWaitTaskArgs[2] = NumOfElements;
5926 DepWaitTaskArgs[4] = CGF.
Builder.getInt32(0);
5927 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
5928 DepWaitTaskArgs[6] =
5929 llvm::ConstantInt::get(CGF.
Int32Ty,
Data.HasNowaitClause);
5938 M, OMPRTL___kmpc_omp_taskwait_deps_51),
5945 llvm::Value *Args[] = {UpLoc, ThreadID};
5948 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_omp_taskwait),
5953 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
5954 Region->emitUntiedSwitch(CGF);
5963 InlinedOpenMPRegionRAII Region(CGF,
CodeGen, InnerKind, HasCancel,
5964 InnerKind != OMPD_critical &&
5965 InnerKind != OMPD_master &&
5966 InnerKind != OMPD_masked);
5981 RTCancelKind CancelKind = CancelNoreq;
5982 if (CancelRegion == OMPD_parallel)
5983 CancelKind = CancelParallel;
5984 else if (CancelRegion == OMPD_for)
5985 CancelKind = CancelLoop;
5986 else if (CancelRegion == OMPD_sections)
5987 CancelKind = CancelSections;
5989 assert(CancelRegion == OMPD_taskgroup);
5990 CancelKind = CancelTaskgroup;
6002 if (
auto *OMPRegionInfo =
6006 if (CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) {
6007 llvm::Value *Args[] = {
6013 CGM.getModule(), OMPRTL___kmpc_cancellationpoint),
6022 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
6024 if (CancelRegion == OMPD_parallel)
6042 auto &M =
CGM.getModule();
6043 if (
auto *OMPRegionInfo =
6045 auto &&ThenGen = [
this, &M, Loc, CancelRegion,
6048 llvm::Value *Args[] = {
6052 llvm::Value *
Result = CGF.EmitRuntimeCall(
6053 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_cancel), Args);
6058 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
".cancel.exit");
6059 llvm::BasicBlock *ContBB = CGF.createBasicBlock(
".cancel.continue");
6060 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(
Result);
6061 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
6062 CGF.EmitBlock(ExitBB);
6063 if (CancelRegion == OMPD_parallel)
6067 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
6068 CGF.EmitBranchThroughCleanup(CancelDest);
6069 CGF.EmitBlock(ContBB,
true);
6087 OMPUsesAllocatorsActionTy(
6088 ArrayRef<std::pair<const Expr *, const Expr *>> Allocators)
6089 : Allocators(Allocators) {}
6093 for (
const auto &AllocatorData : Allocators) {
6095 CGF, AllocatorData.first, AllocatorData.second);
6098 void Exit(CodeGenFunction &CGF)
override {
6101 for (
const auto &AllocatorData : Allocators) {
6103 AllocatorData.first);
6111 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6113 assert(!ParentName.empty() &&
"Invalid target entry parent name!");
6117 for (
unsigned I = 0, E =
C->getNumberOfAllocators(); I < E; ++I) {
6124 OMPUsesAllocatorsActionTy UsesAllocatorAction(Allocators);
6125 CodeGen.setAction(UsesAllocatorAction);
6131 const Expr *Allocator,
6132 const Expr *AllocatorTraits) {
6134 ThreadId = CGF.
Builder.CreateIntCast(ThreadId, CGF.
IntTy,
true);
6136 llvm::Value *MemSpaceHandle = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
6137 llvm::Value *NumTraits = llvm::ConstantInt::get(
6141 .getLimitedValue());
6148 llvm::Value *Traits =
Addr.emitRawPointer(CGF);
6150 llvm::Value *AllocatorVal =
6152 CGM.getModule(), OMPRTL___kmpc_init_allocator),
6153 {ThreadId, MemSpaceHandle, NumTraits, Traits});
6165 const Expr *Allocator) {
6167 ThreadId = CGF.
Builder.CreateIntCast(ThreadId, CGF.
IntTy,
true);
6169 llvm::Value *AllocatorVal =
6176 OMPRTL___kmpc_destroy_allocator),
6177 {ThreadId, AllocatorVal});
6182 llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs) {
6183 assert(Attrs.MaxTeams.size() == 1 && Attrs.MaxThreads.size() == 1 &&
6184 "invalid default attrs structure");
6185 int32_t &MaxTeamsVal = Attrs.MaxTeams.front();
6186 int32_t &MaxThreadsVal = Attrs.MaxThreads.front();
6193 for (
auto *A :
C->getAttrs()) {
6194 int32_t AttrMinThreadsVal = 1, AttrMaxThreadsVal = -1;
6195 int32_t AttrMinBlocksVal = 1, AttrMaxBlocksVal = -1;
6196 if (
auto *
Attr = dyn_cast<CUDALaunchBoundsAttr>(A))
6197 CGM.handleCUDALaunchBoundsAttr(
nullptr,
Attr, &AttrMaxThreadsVal,
6198 &AttrMinBlocksVal, &AttrMaxBlocksVal);
6199 else if (
auto *
Attr = dyn_cast<AMDGPUFlatWorkGroupSizeAttr>(A))
6200 CGM.handleAMDGPUFlatWorkGroupSizeAttr(
6201 nullptr,
Attr,
nullptr, &AttrMinThreadsVal,
6202 &AttrMaxThreadsVal);
6206 Attrs.MinThreads = std::max(Attrs.MinThreads, AttrMinThreadsVal);
6207 if (AttrMaxThreadsVal > 0)
6208 MaxThreadsVal = MaxThreadsVal > 0
6209 ? std::min(MaxThreadsVal, AttrMaxThreadsVal)
6210 : AttrMaxThreadsVal;
6211 Attrs.MinTeams = std::max(Attrs.MinTeams, AttrMinBlocksVal);
6212 if (AttrMaxBlocksVal > 0)
6213 MaxTeamsVal = MaxTeamsVal > 0 ? std::min(MaxTeamsVal, AttrMaxBlocksVal)
6221 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6224 llvm::TargetRegionEntryInfo EntryInfo =
6228 llvm::OpenMPIRBuilder::FunctionGenCallback &&GenerateOutlinedFunction =
6229 [&CGF, &D, &
CodeGen](StringRef EntryFnName) {
6230 const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target);
6232 CGOpenMPTargetRegionInfo CGInfo(CS,
CodeGen, EntryFnName);
6237 cantFail(
OMPBuilder.emitTargetRegionFunction(
6238 EntryInfo, GenerateOutlinedFunction, IsOffloadEntry, OutlinedFn,
6244 CGM.getTargetCodeGenInfo().setTargetAttributes(
nullptr, OutlinedFn,
CGM);
6247 for (
auto *A :
C->getAttrs()) {
6248 if (
auto *
Attr = dyn_cast<AMDGPUWavesPerEUAttr>(A))
6249 CGM.handleAMDGPUWavesPerEUAttr(OutlinedFn,
Attr);
6267 while (
const auto *
C = dyn_cast_or_null<CompoundStmt>(Child)) {
6269 for (
const Stmt *S :
C->body()) {
6270 if (
const auto *E = dyn_cast<Expr>(S)) {
6279 if (
const auto *DS = dyn_cast<DeclStmt>(S)) {
6280 if (llvm::all_of(DS->decls(), [](
const Decl *D) {
6281 if (isa<EmptyDecl>(D) || isa<DeclContext>(D) ||
6282 isa<TypeDecl>(D) || isa<PragmaCommentDecl>(D) ||
6283 isa<PragmaDetectMismatchDecl>(D) || isa<UsingDecl>(D) ||
6284 isa<UsingDirectiveDecl>(D) ||
6285 isa<OMPDeclareReductionDecl>(D) ||
6286 isa<OMPThreadPrivateDecl>(D) || isa<OMPAllocateDecl>(D))
6288 const auto *VD = dyn_cast<VarDecl>(D);
6291 return VD->hasGlobalStorage() || !VD->isUsed();
6301 Child = Child->IgnoreContainers();
6308 int32_t &MaxTeamsVal) {
6312 "Expected target-based executable directive.");
6313 switch (DirectiveKind) {
6315 const auto *CS = D.getInnermostCapturedStmt();
6318 const Stmt *ChildStmt =
6320 if (
const auto *NestedDir =
6321 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
6330 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();
6333 MinTeamsVal = MaxTeamsVal = 0;
6336 MinTeamsVal = MaxTeamsVal = 1;
6340 MinTeamsVal = MaxTeamsVal = -1;
6343 case OMPD_target_teams_loop:
6344 case OMPD_target_teams:
6345 case OMPD_target_teams_distribute:
6346 case OMPD_target_teams_distribute_simd:
6347 case OMPD_target_teams_distribute_parallel_for:
6348 case OMPD_target_teams_distribute_parallel_for_simd: {
6350 const Expr *NumTeams =
6354 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();
6357 MinTeamsVal = MaxTeamsVal = 0;
6360 case OMPD_target_parallel:
6361 case OMPD_target_parallel_for:
6362 case OMPD_target_parallel_for_simd:
6363 case OMPD_target_parallel_loop:
6364 case OMPD_target_simd:
6365 MinTeamsVal = MaxTeamsVal = 1;
6369 case OMPD_parallel_for:
6370 case OMPD_parallel_loop:
6371 case OMPD_parallel_master:
6372 case OMPD_parallel_sections:
6374 case OMPD_parallel_for_simd:
6376 case OMPD_cancellation_point:
6378 case OMPD_threadprivate:
6389 case OMPD_taskyield:
6392 case OMPD_taskgroup:
6398 case OMPD_target_data:
6399 case OMPD_target_exit_data:
6400 case OMPD_target_enter_data:
6401 case OMPD_distribute:
6402 case OMPD_distribute_simd:
6403 case OMPD_distribute_parallel_for:
6404 case OMPD_distribute_parallel_for_simd:
6405 case OMPD_teams_distribute:
6406 case OMPD_teams_distribute_simd:
6407 case OMPD_teams_distribute_parallel_for:
6408 case OMPD_teams_distribute_parallel_for_simd:
6409 case OMPD_target_update:
6410 case OMPD_declare_simd:
6411 case OMPD_declare_variant:
6412 case OMPD_begin_declare_variant:
6413 case OMPD_end_declare_variant:
6414 case OMPD_declare_target:
6415 case OMPD_end_declare_target:
6416 case OMPD_declare_reduction:
6417 case OMPD_declare_mapper:
6419 case OMPD_taskloop_simd:
6420 case OMPD_master_taskloop:
6421 case OMPD_master_taskloop_simd:
6422 case OMPD_parallel_master_taskloop:
6423 case OMPD_parallel_master_taskloop_simd:
6425 case OMPD_metadirective:
6431 llvm_unreachable(
"Unexpected directive kind.");
6437 "Clauses associated with the teams directive expected to be emitted "
6438 "only for the host!");
6440 int32_t MinNT = -1, MaxNT = -1;
6441 const Expr *NumTeams =
6443 if (NumTeams !=
nullptr) {
6446 switch (DirectiveKind) {
6448 const auto *CS = D.getInnermostCapturedStmt();
6449 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6453 return Bld.CreateIntCast(NumTeamsVal, CGF.
Int32Ty,
6456 case OMPD_target_teams:
6457 case OMPD_target_teams_distribute:
6458 case OMPD_target_teams_distribute_simd:
6459 case OMPD_target_teams_distribute_parallel_for:
6460 case OMPD_target_teams_distribute_parallel_for_simd: {
6464 return Bld.CreateIntCast(NumTeamsVal, CGF.
Int32Ty,
6472 assert(MinNT == MaxNT &&
"Num threads ranges require handling here.");
6473 return llvm::ConstantInt::get(CGF.
Int32Ty, MinNT);
6481 const Expr **E, int32_t &UpperBound,
6482 bool UpperBoundOnly, llvm::Value **CondVal) {
6485 const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child);
6492 if (CondVal && Dir->hasClausesOfKind<
OMPIfClause>()) {
6493 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6496 for (
const auto *
C : Dir->getClausesOfKind<
OMPIfClause>()) {
6497 if (
C->getNameModifier() == OMPD_unknown ||
6498 C->getNameModifier() == OMPD_parallel) {
6513 if (
const auto *PreInit =
6515 for (
const auto *I : PreInit->decls()) {
6516 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6532 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6534 const auto *NumThreadsClause =
6536 const Expr *NTExpr = NumThreadsClause->getNumThreads();
6537 if (NTExpr->isIntegerConstantExpr(CGF.
getContext()))
6538 if (
auto Constant = NTExpr->getIntegerConstantExpr(CGF.
getContext()))
6541 ? Constant->getZExtValue()
6542 : std::min(UpperBound,
6543 static_cast<int32_t
>(Constant->getZExtValue()));
6546 if (UpperBound == -1)
6551 if (
const auto *PreInit =
6552 cast_or_null<DeclStmt>(NumThreadsClause->getPreInitStmt())) {
6553 for (
const auto *I : PreInit->decls()) {
6554 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6573 bool UpperBoundOnly, llvm::Value **CondVal,
const Expr **ThreadLimitExpr) {
6574 assert((!CGF.
getLangOpts().OpenMPIsTargetDevice || UpperBoundOnly) &&
6575 "Clauses associated with the teams directive expected to be emitted "
6576 "only for the host!");
6579 "Expected target-based executable directive.");
6581 const Expr *NT =
nullptr;
6582 const Expr **NTPtr = UpperBoundOnly ?
nullptr : &NT;
6584 auto CheckForConstExpr = [&](
const Expr *E,
const Expr **EPtr) {
6587 UpperBound = UpperBound ? Constant->getZExtValue()
6588 : std::min(UpperBound,
6589 int32_t(Constant->getZExtValue()));
6593 if (UpperBound == -1)
6599 auto ReturnSequential = [&]() {
6604 switch (DirectiveKind) {
6607 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6613 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6615 ThreadLimitClause = TLC;
6616 if (ThreadLimitExpr) {
6617 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6621 ThreadLimitClause->getThreadLimit().front()->getSourceRange());
6622 if (
const auto *PreInit =
6623 cast_or_null<DeclStmt>(ThreadLimitClause->getPreInitStmt())) {
6624 for (
const auto *I : PreInit->decls()) {
6625 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6637 if (ThreadLimitClause)
6638 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
6640 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6643 CS = Dir->getInnermostCapturedStmt();
6646 Dir = dyn_cast_or_null<OMPExecutableDirective>(Child);
6649 CS = Dir->getInnermostCapturedStmt();
6650 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6652 return ReturnSequential();
6656 case OMPD_target_teams: {
6660 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
6664 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6667 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6668 if (Dir->getDirectiveKind() == OMPD_distribute) {
6669 CS = Dir->getInnermostCapturedStmt();
6670 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6675 case OMPD_target_teams_distribute:
6679 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
6682 getNumThreads(CGF, D.getInnermostCapturedStmt(), NTPtr, UpperBound,
6683 UpperBoundOnly, CondVal);
6685 case OMPD_target_teams_loop:
6686 case OMPD_target_parallel_loop:
6687 case OMPD_target_parallel:
6688 case OMPD_target_parallel_for:
6689 case OMPD_target_parallel_for_simd:
6690 case OMPD_target_teams_distribute_parallel_for:
6691 case OMPD_target_teams_distribute_parallel_for_simd: {
6692 if (CondVal && D.hasClausesOfKind<
OMPIfClause>()) {
6694 for (
const auto *
C : D.getClausesOfKind<
OMPIfClause>()) {
6695 if (
C->getNameModifier() == OMPD_unknown ||
6696 C->getNameModifier() == OMPD_parallel) {
6706 return ReturnSequential();
6716 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
6722 CheckForConstExpr(NumThreadsClause->getNumThreads(),
nullptr);
6723 return NumThreadsClause->getNumThreads();
6727 case OMPD_target_teams_distribute_simd:
6728 case OMPD_target_simd:
6729 return ReturnSequential();
6733 llvm_unreachable(
"Unsupported directive kind.");
6738 llvm::Value *NumThreadsVal =
nullptr;
6739 llvm::Value *CondVal =
nullptr;
6740 llvm::Value *ThreadLimitVal =
nullptr;
6741 const Expr *ThreadLimitExpr =
nullptr;
6742 int32_t UpperBound = -1;
6745 CGF, D, UpperBound,
false, &CondVal,
6749 if (ThreadLimitExpr) {
6752 ThreadLimitVal = CGF.
Builder.CreateIntCast(ThreadLimitVal, CGF.
Int32Ty,
6757 if (UpperBound == 1) {
6758 NumThreadsVal = CGF.
Builder.getInt32(UpperBound);
6761 NumThreadsVal = CGF.
Builder.CreateIntCast(NumThreadsVal, CGF.
Int32Ty,
6763 }
else if (ThreadLimitVal) {
6766 NumThreadsVal = ThreadLimitVal;
6767 ThreadLimitVal =
nullptr;
6770 assert(!ThreadLimitVal &&
"Default not applicable with thread limit value");
6771 NumThreadsVal = CGF.
Builder.getInt32(0);
6778 NumThreadsVal = CGF.
Builder.CreateSelect(CondVal, NumThreadsVal,
6784 if (ThreadLimitVal) {
6785 NumThreadsVal = CGF.
Builder.CreateSelect(
6786 CGF.
Builder.CreateICmpULT(ThreadLimitVal, NumThreadsVal),
6787 ThreadLimitVal, NumThreadsVal);
6790 return NumThreadsVal;
6800class MappableExprsHandler {
6803 static unsigned getFlagMemberOffset() {
6804 unsigned Offset = 0;
6805 for (uint64_t Remain =
6806 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
6807 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
6808 !(Remain & 1); Remain = Remain >> 1)
6815 class MappingExprInfo {
6820 const Expr *MapExpr =
nullptr;
6823 MappingExprInfo(
const ValueDecl *MapDecl,
const Expr *MapExpr =
nullptr)
6824 : MapDecl(MapDecl), MapExpr(MapExpr) {}
6826 const ValueDecl *getMapDecl()
const {
return MapDecl; }
6827 const Expr *getMapExpr()
const {
return MapExpr; }
6830 using DeviceInfoTy = llvm::OpenMPIRBuilder::DeviceInfoTy;
6831 using MapBaseValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
6832 using MapValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
6833 using MapFlagsArrayTy = llvm::OpenMPIRBuilder::MapFlagsArrayTy;
6834 using MapDimArrayTy = llvm::OpenMPIRBuilder::MapDimArrayTy;
6835 using MapNonContiguousArrayTy =
6836 llvm::OpenMPIRBuilder::MapNonContiguousArrayTy;
6837 using MapExprsArrayTy = SmallVector<MappingExprInfo, 4>;
6838 using MapValueDeclsArrayTy = SmallVector<const ValueDecl *, 4>;
6842 bool ,
const ValueDecl *,
const Expr *>;
6843 using MapDataArrayTy = SmallVector<MapData, 4>;
6848 struct MapCombinedInfoTy : llvm::OpenMPIRBuilder::MapInfosTy {
6849 MapExprsArrayTy Exprs;
6850 MapValueDeclsArrayTy Mappers;
6851 MapValueDeclsArrayTy DevicePtrDecls;
6854 void append(MapCombinedInfoTy &CurInfo) {
6855 Exprs.append(CurInfo.Exprs.begin(), CurInfo.Exprs.end());
6856 DevicePtrDecls.append(CurInfo.DevicePtrDecls.begin(),
6857 CurInfo.DevicePtrDecls.end());
6858 Mappers.append(CurInfo.Mappers.begin(), CurInfo.Mappers.end());
6859 llvm::OpenMPIRBuilder::MapInfosTy::append(CurInfo);
6867 struct StructRangeInfoTy {
6868 MapCombinedInfoTy PreliminaryMapData;
6869 std::pair<
unsigned , Address > LowestElem = {
6871 std::pair<
unsigned , Address > HighestElem = {
6875 bool IsArraySection =
false;
6876 bool HasCompleteRecord =
false;
6884 ArrayRef<OpenMPMapModifierKind> MapModifiers;
6885 ArrayRef<OpenMPMotionModifierKind> MotionModifiers;
6886 bool ReturnDevicePointer =
false;
6887 bool IsImplicit =
false;
6888 const ValueDecl *Mapper =
nullptr;
6889 const Expr *VarRef =
nullptr;
6890 bool ForDeviceAddr =
false;
6892 MapInfo() =
default;
6896 ArrayRef<OpenMPMapModifierKind> MapModifiers,
6897 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
6898 bool ReturnDevicePointer,
bool IsImplicit,
6899 const ValueDecl *Mapper =
nullptr,
const Expr *VarRef =
nullptr,
6900 bool ForDeviceAddr =
false)
6901 : Components(Components), MapType(MapType), MapModifiers(MapModifiers),
6902 MotionModifiers(MotionModifiers),
6903 ReturnDevicePointer(ReturnDevicePointer), IsImplicit(IsImplicit),
6904 Mapper(Mapper), VarRef(VarRef), ForDeviceAddr(ForDeviceAddr) {}
6910 struct DeferredDevicePtrEntryTy {
6911 const Expr *IE =
nullptr;
6912 const ValueDecl *VD =
nullptr;
6913 bool ForDeviceAddr =
false;
6915 DeferredDevicePtrEntryTy(
const Expr *IE,
const ValueDecl *VD,
6917 : IE(IE), VD(VD), ForDeviceAddr(ForDeviceAddr) {}
6922 llvm::PointerUnion<
const OMPExecutableDirective *,
6923 const OMPDeclareMapperDecl *>
6927 CodeGenFunction &CGF;
6932 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>,
bool> FirstPrivateDecls;
6938 SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>>
6945 SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>>
6949 llvm::DenseMap<const ValueDecl *, const OMPMapClause *> LambdasMap;
6951 llvm::Value *getExprTypeSize(
const Expr *E)
const {
6955 if (
const auto *OAE = dyn_cast<OMPArrayShapingExpr>(E)) {
6957 CGF.
getTypeSize(OAE->getBase()->getType()->getPointeeType());
6958 for (
const Expr *SE : OAE->getDimensions()) {
6969 if (
const auto *RefTy = ExprTy->
getAs<ReferenceType>())
6975 if (
const auto *OAE = dyn_cast<ArraySectionExpr>(E)) {
6977 OAE->getBase()->IgnoreParenImpCasts())
6983 if (!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
6984 !OAE->getLowerBound())
6987 llvm::Value *ElemSize;
6988 if (
const auto *PTy = BaseTy->
getAs<PointerType>()) {
6989 ElemSize = CGF.
getTypeSize(PTy->getPointeeType().getCanonicalType());
6992 assert(ATy &&
"Expecting array type if not a pointer type.");
6993 ElemSize = CGF.
getTypeSize(ATy->getElementType().getCanonicalType());
6998 if (!OAE->getLength() && OAE->getColonLocFirst().isInvalid())
7001 if (
const Expr *LenExpr = OAE->getLength()) {
7005 LenExpr->getExprLoc());
7006 return CGF.
Builder.CreateNUWMul(LengthVal, ElemSize);
7008 assert(!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
7009 OAE->getLowerBound() &&
"expected array_section[lb:].");
7015 OAE->getLowerBound()->getExprLoc());
7016 LBVal = CGF.
Builder.CreateNUWMul(LBVal, ElemSize);
7017 llvm::Value *Cmp = CGF.
Builder.CreateICmpUGT(LengthVal, LBVal);
7018 llvm::Value *TrueVal = CGF.
Builder.CreateNUWSub(LengthVal, LBVal);
7019 LengthVal = CGF.
Builder.CreateSelect(
7020 Cmp, TrueVal, llvm::ConstantInt::get(CGF.
SizeTy, 0));
7030 OpenMPOffloadMappingFlags getMapTypeBits(
7032 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
bool IsImplicit,
7033 bool AddPtrFlag,
bool AddIsTargetParamFlag,
bool IsNonContiguous)
const {
7034 OpenMPOffloadMappingFlags Bits =
7035 IsImplicit ? OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT
7036 : OpenMPOffloadMappingFlags::OMP_MAP_NONE;
7038 case OMPC_MAP_alloc:
7039 case OMPC_MAP_release:
7046 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO;
7049 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_FROM;
7051 case OMPC_MAP_tofrom:
7052 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO |
7053 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
7055 case OMPC_MAP_delete:
7056 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_DELETE;
7059 llvm_unreachable(
"Unexpected map type!");
7062 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
7063 if (AddIsTargetParamFlag)
7064 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
7065 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_always))
7066 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS;
7067 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_close))
7068 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_CLOSE;
7069 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_present) ||
7070 llvm::is_contained(MotionModifiers, OMPC_MOTION_MODIFIER_present))
7071 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
7072 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_ompx_hold))
7073 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
7074 if (IsNonContiguous)
7075 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG;
7081 bool isFinalArraySectionExpression(
const Expr *E)
const {
7082 const auto *OASE = dyn_cast<ArraySectionExpr>(E);
7089 if (OASE->getColonLocFirst().isInvalid())
7092 const Expr *Length = OASE->getLength();
7099 OASE->getBase()->IgnoreParenImpCasts())
7101 if (
const auto *ATy = dyn_cast<ConstantArrayType>(BaseQTy.
getTypePtr()))
7102 return ATy->getSExtSize() != 1;
7114 llvm::APSInt ConstLength =
Result.Val.getInt();
7115 return ConstLength.getSExtValue() != 1;
7122 class CopyOverlappedEntryGaps {
7123 CodeGenFunction &CGF;
7124 MapCombinedInfoTy &CombinedInfo;
7125 OpenMPOffloadMappingFlags Flags = OpenMPOffloadMappingFlags::OMP_MAP_NONE;
7126 const ValueDecl *MapDecl =
nullptr;
7127 const Expr *MapExpr =
nullptr;
7129 bool IsNonContiguous =
false;
7133 const RecordDecl *LastParent =
nullptr;
7135 unsigned LastIndex = -1u;
7139 CopyOverlappedEntryGaps(CodeGenFunction &CGF,
7140 MapCombinedInfoTy &CombinedInfo,
7141 OpenMPOffloadMappingFlags Flags,
7142 const ValueDecl *MapDecl,
const Expr *MapExpr,
7143 Address BP, Address LB,
bool IsNonContiguous,
7145 : CGF(CGF), CombinedInfo(CombinedInfo), Flags(Flags), MapDecl(MapDecl),
7146 MapExpr(MapExpr), BP(BP), IsNonContiguous(IsNonContiguous),
7147 DimSize(DimSize), LB(LB) {}
7150 const OMPClauseMappableExprCommon::MappableComponent &MC,
7151 const FieldDecl *FD,
7152 llvm::function_ref<LValue(CodeGenFunction &,
const MemberExpr *)>
7153 EmitMemberExprBase) {
7163 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
7175 copyUntilField(FD, ComponentLB);
7178 if (((int64_t)FieldOffset - (int64_t)Cursor) > 0)
7179 copyUntilField(FD, ComponentLB);
7181 Cursor = FieldOffset + FieldSize;
7186 void copyUntilField(
const FieldDecl *FD, Address ComponentLB) {
7190 CGF.
Builder.CreatePtrDiff(CGF.
Int8Ty, ComponentLBPtr, LBPtr);
7191 copySizedChunk(LBPtr, Size);
7194 void copyUntilEnd(Address HB) {
7196 const ASTRecordLayout &RL =
7205 copySizedChunk(LBPtr, Size);
7208 void copySizedChunk(llvm::Value *Base, llvm::Value *Size) {
7209 CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
7211 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
7212 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
7213 CombinedInfo.Pointers.push_back(Base);
7214 CombinedInfo.Sizes.push_back(
7216 CombinedInfo.Types.push_back(Flags);
7217 CombinedInfo.Mappers.push_back(
nullptr);
7218 CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize : 1);
7227 void generateInfoForComponentList(
7229 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
7231 MapCombinedInfoTy &CombinedInfo,
7232 MapCombinedInfoTy &StructBaseCombinedInfo,
7233 StructRangeInfoTy &PartialStruct,
bool IsFirstComponentList,
7234 bool IsImplicit,
bool GenerateAllInfoForClauses,
7235 const ValueDecl *Mapper =
nullptr,
bool ForDeviceAddr =
false,
7236 const ValueDecl *BaseDecl =
nullptr,
const Expr *MapExpr =
nullptr,
7237 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
7238 OverlappedElements = {},
7239 bool AreBothBasePtrAndPteeMapped =
false)
const {
7421 bool IsCaptureFirstInfo = IsFirstComponentList;
7425 bool RequiresReference =
false;
7428 auto CI = Components.rbegin();
7429 auto CE = Components.rend();
7434 bool IsExpressionFirstInfo =
true;
7435 bool FirstPointerInComplexData =
false;
7437 const Expr *AssocExpr = I->getAssociatedExpression();
7438 const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr);
7439 const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr);
7440 const auto *OAShE = dyn_cast<OMPArrayShapingExpr>(AssocExpr);
7442 if (AreBothBasePtrAndPteeMapped && std::next(I) == CE)
7448 }
else if ((AE &&
isa<CXXThisExpr>(AE->getBase()->IgnoreParenImpCasts())) ||
7462 if (
const auto *VD =
7463 dyn_cast_or_null<VarDecl>(I->getAssociatedDeclaration())) {
7464 if (std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
7465 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) {
7466 if ((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
7467 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
7468 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
7470 RequiresReference =
true;
7480 I->getAssociatedDeclaration()->
getType().getNonReferenceType();
7485 const auto *VD = dyn_cast<VarDecl>(I->getAssociatedDeclaration());
7486 if (!AreBothBasePtrAndPteeMapped &&
7491 FirstPointerInComplexData =
true;
7510 bool ShouldBeMemberOf =
false;
7519 const MemberExpr *EncounteredME =
nullptr;
7531 bool IsNonContiguous =
7532 CombinedInfo.NonContigInfo.IsNonContiguous ||
7533 any_of(Components, [&](
const auto &Component) {
7535 dyn_cast<ArraySectionExpr>(Component.getAssociatedExpression());
7539 const Expr *StrideExpr = OASE->getStride();
7543 const auto Constant =
7548 return !Constant->isOne();
7551 bool IsPrevMemberReference =
false;
7553 bool IsPartialMapped =
7554 !PartialStruct.PreliminaryMapData.BasePointers.empty();
7561 bool IsMappingWholeStruct =
true;
7562 if (!GenerateAllInfoForClauses) {
7563 IsMappingWholeStruct =
false;
7565 for (
auto TempI = I; TempI != CE; ++TempI) {
7566 const MemberExpr *PossibleME =
7567 dyn_cast<MemberExpr>(TempI->getAssociatedExpression());
7569 IsMappingWholeStruct =
false;
7575 for (; I != CE; ++I) {
7577 if (!EncounteredME) {
7578 EncounteredME = dyn_cast<MemberExpr>(I->getAssociatedExpression());
7581 if (EncounteredME) {
7582 ShouldBeMemberOf =
true;
7585 if (FirstPointerInComplexData) {
7586 QualType Ty = std::prev(I)
7587 ->getAssociatedDeclaration()
7589 .getNonReferenceType();
7591 FirstPointerInComplexData =
false;
7596 auto Next = std::next(I);
7606 bool IsFinalArraySection =
7608 isFinalArraySectionExpression(I->getAssociatedExpression());
7612 const ValueDecl *MapDecl = (I->getAssociatedDeclaration())
7613 ? I->getAssociatedDeclaration()
7615 MapExpr = (I->getAssociatedExpression()) ? I->getAssociatedExpression()
7622 dyn_cast<ArraySectionExpr>(I->getAssociatedExpression());
7624 dyn_cast<OMPArrayShapingExpr>(I->getAssociatedExpression());
7625 const auto *UO = dyn_cast<UnaryOperator>(I->getAssociatedExpression());
7626 const auto *BO = dyn_cast<BinaryOperator>(I->getAssociatedExpression());
7632 I->getAssociatedExpression()->getType()->isAnyPointerType();
7633 bool IsMemberReference =
isa<MemberExpr>(I->getAssociatedExpression()) &&
7636 bool IsNonDerefPointer = IsPointer &&
7637 !(UO && UO->getOpcode() != UO_Deref) && !BO &&
7643 if (
Next == CE || IsMemberReference || IsNonDerefPointer ||
7644 IsFinalArraySection) {
7647 assert((
Next == CE ||
7654 "Unexpected expression");
7658 auto &&EmitMemberExprBase = [](CodeGenFunction &CGF,
7659 const MemberExpr *E) {
7660 const Expr *BaseExpr = E->getBase();
7665 LValueBaseInfo BaseInfo;
7666 TBAAAccessInfo TBAAInfo;
7680 OAShE->getBase()->getType()->getPointeeType()),
7682 OAShE->getBase()->getType()));
7683 }
else if (IsMemberReference) {
7685 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
7700 bool IsMemberPointerOrAddr =
7702 (((IsPointer || ForDeviceAddr) &&
7703 I->getAssociatedExpression() == EncounteredME) ||
7704 (IsPrevMemberReference && !IsPointer) ||
7705 (IsMemberReference &&
Next != CE &&
7706 !
Next->getAssociatedExpression()->getType()->isPointerType()));
7707 if (!OverlappedElements.empty() &&
Next == CE) {
7709 assert(!PartialStruct.Base.isValid() &&
"The base element is set.");
7710 assert(!IsPointer &&
7711 "Unexpected base element with the pointer type.");
7714 PartialStruct.LowestElem = {0, LowestElem};
7716 I->getAssociatedExpression()->getType());
7721 PartialStruct.HighestElem = {
7722 std::numeric_limits<
decltype(
7723 PartialStruct.HighestElem.first)>
::max(),
7725 PartialStruct.Base = BP;
7726 PartialStruct.LB = LB;
7728 PartialStruct.PreliminaryMapData.BasePointers.empty() &&
7729 "Overlapped elements must be used only once for the variable.");
7730 std::swap(PartialStruct.PreliminaryMapData, CombinedInfo);
7732 OpenMPOffloadMappingFlags Flags =
7733 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
7734 getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,
7736 false, IsNonContiguous);
7737 CopyOverlappedEntryGaps CopyGaps(CGF, CombinedInfo, Flags, MapDecl,
7738 MapExpr, BP, LB, IsNonContiguous,
7742 Component : OverlappedElements) {
7743 for (
const OMPClauseMappableExprCommon::MappableComponent &MC :
7746 if (
const auto *FD = dyn_cast<FieldDecl>(VD)) {
7747 CopyGaps.processField(MC, FD, EmitMemberExprBase);
7752 CopyGaps.copyUntilEnd(HB);
7755 llvm::Value *
Size = getExprTypeSize(I->getAssociatedExpression());
7762 if ((!IsMemberPointerOrAddr && !IsPartialMapped) ||
7764 if (!IsMappingWholeStruct) {
7765 CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
7767 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
7768 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
7770 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
7772 CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize
7775 StructBaseCombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
7776 StructBaseCombinedInfo.BasePointers.push_back(
7778 StructBaseCombinedInfo.DevicePtrDecls.push_back(
nullptr);
7779 StructBaseCombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
7780 StructBaseCombinedInfo.Pointers.push_back(LB.
emitRawPointer(CGF));
7781 StructBaseCombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
7783 StructBaseCombinedInfo.NonContigInfo.Dims.push_back(
7784 IsNonContiguous ? DimSize : 1);
7788 bool HasMapper = Mapper &&
Next == CE;
7789 if (!IsMappingWholeStruct)
7790 CombinedInfo.Mappers.push_back(HasMapper ? Mapper :
nullptr);
7792 StructBaseCombinedInfo.Mappers.push_back(HasMapper ? Mapper
7799 OpenMPOffloadMappingFlags Flags =
7800 getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,
7801 !IsExpressionFirstInfo || RequiresReference ||
7802 FirstPointerInComplexData || IsMemberReference,
7803 AreBothBasePtrAndPteeMapped ||
7804 (IsCaptureFirstInfo && !RequiresReference),
7807 if (!IsExpressionFirstInfo || IsMemberReference) {
7810 if (IsPointer || (IsMemberReference &&
Next != CE))
7811 Flags &= ~(OpenMPOffloadMappingFlags::OMP_MAP_TO |
7812 OpenMPOffloadMappingFlags::OMP_MAP_FROM |
7813 OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS |
7814 OpenMPOffloadMappingFlags::OMP_MAP_DELETE |
7815 OpenMPOffloadMappingFlags::OMP_MAP_CLOSE);
7817 if (ShouldBeMemberOf) {
7820 Flags |= OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;
7823 ShouldBeMemberOf =
false;
7827 if (!IsMappingWholeStruct)
7828 CombinedInfo.Types.push_back(Flags);
7830 StructBaseCombinedInfo.Types.push_back(Flags);
7836 if (EncounteredME) {
7841 if (!PartialStruct.Base.isValid()) {
7842 PartialStruct.LowestElem = {FieldIndex, LowestElem};
7843 if (IsFinalArraySection && OASE) {
7847 PartialStruct.HighestElem = {FieldIndex, HB};
7849 PartialStruct.HighestElem = {FieldIndex, LowestElem};
7851 PartialStruct.Base = BP;
7852 PartialStruct.LB = BP;
7853 }
else if (FieldIndex < PartialStruct.LowestElem.first) {
7854 PartialStruct.LowestElem = {FieldIndex, LowestElem};
7855 }
else if (FieldIndex > PartialStruct.HighestElem.first) {
7856 if (IsFinalArraySection && OASE) {
7860 PartialStruct.HighestElem = {FieldIndex, HB};
7862 PartialStruct.HighestElem = {FieldIndex, LowestElem};
7868 if (IsFinalArraySection || IsNonContiguous)
7869 PartialStruct.IsArraySection =
true;
7872 if (IsFinalArraySection)
7877 BP = IsMemberReference ? LowestElem : LB;
7878 if (!IsPartialMapped)
7879 IsExpressionFirstInfo =
false;
7880 IsCaptureFirstInfo =
false;
7881 FirstPointerInComplexData =
false;
7882 IsPrevMemberReference = IsMemberReference;
7883 }
else if (FirstPointerInComplexData) {
7884 QualType Ty = Components.rbegin()
7885 ->getAssociatedDeclaration()
7887 .getNonReferenceType();
7889 FirstPointerInComplexData =
false;
7895 PartialStruct.HasCompleteRecord =
true;
7897 if (!IsNonContiguous)
7900 const ASTContext &Context = CGF.
getContext();
7904 MapValuesArrayTy CurOffsets = {llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, 0)};
7905 MapValuesArrayTy CurCounts = {llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, 1)};
7906 MapValuesArrayTy CurStrides;
7907 MapValuesArrayTy DimSizes{llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, 1)};
7913 for (
const OMPClauseMappableExprCommon::MappableComponent &Component :
7915 const Expr *AssocExpr = Component.getAssociatedExpression();
7916 const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr);
7926 assert((VAT || CAT || &Component == &*Components.begin()) &&
7927 "Should be either ConstantArray or VariableArray if not the "
7931 if (CurStrides.empty()) {
7932 const Type *ElementType =
nullptr;
7934 ElementType = CAT->getElementType().getTypePtr();
7936 ElementType = VAT->getElementType().getTypePtr();
7938 assert(&Component == &*Components.begin() &&
7939 "Only expect pointer (non CAT or VAT) when this is the "
7947 if (&Component != &*Components.begin())
7951 CurStrides.push_back(
7952 llvm::ConstantInt::get(CGF.
Int64Ty, ElementTypeSize));
7957 if (DimSizes.size() < Components.size() - 1) {
7960 llvm::ConstantInt::get(CGF.
Int64Ty, CAT->getZExtSize()));
7962 DimSizes.push_back(CGF.
Builder.CreateIntCast(
7969 auto *DI = DimSizes.begin() + 1;
7971 llvm::Value *DimProd =
7972 llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, ElementTypeSize);
7981 for (
const OMPClauseMappableExprCommon::MappableComponent &Component :
7983 const Expr *AssocExpr = Component.getAssociatedExpression();
7985 if (
const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr)) {
7986 llvm::Value *Offset = CGF.
Builder.CreateIntCast(
7989 CurOffsets.push_back(Offset);
7990 CurCounts.push_back(llvm::ConstantInt::get(CGF.
Int64Ty, 1));
7991 CurStrides.push_back(CurStrides.back());
7995 const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr);
8001 const Expr *OffsetExpr = OASE->getLowerBound();
8002 llvm::Value *Offset =
nullptr;
8005 Offset = llvm::ConstantInt::get(CGF.
Int64Ty, 0);
8011 CurOffsets.push_back(Offset);
8014 const Expr *CountExpr = OASE->getLength();
8015 llvm::Value *Count =
nullptr;
8021 if (!OASE->getColonLocFirst().isValid() &&
8022 !OASE->getColonLocSecond().isValid()) {
8023 Count = llvm::ConstantInt::get(CGF.
Int64Ty, 1);
8029 const Expr *StrideExpr = OASE->getStride();
8030 llvm::Value *Stride =
8036 Count = CGF.
Builder.CreateUDiv(
8037 CGF.
Builder.CreateNUWSub(*DI, Offset), Stride);
8039 Count = CGF.
Builder.CreateNUWSub(*DI, Offset);
8045 CurCounts.push_back(Count);
8054 const Expr *StrideExpr = OASE->getStride();
8055 llvm::Value *Stride =
8060 DimProd = CGF.
Builder.CreateNUWMul(DimProd, *(DI - 1));
8062 CurStrides.push_back(CGF.
Builder.CreateNUWMul(DimProd, Stride));
8064 CurStrides.push_back(DimProd);
8065 if (DI != DimSizes.end())
8069 CombinedInfo.NonContigInfo.Offsets.push_back(CurOffsets);
8070 CombinedInfo.NonContigInfo.Counts.push_back(CurCounts);
8071 CombinedInfo.NonContigInfo.Strides.push_back(CurStrides);
8077 OpenMPOffloadMappingFlags
8078 getMapModifiersForPrivateClauses(
const CapturedStmt::Capture &Cap)
const {
8086 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
8087 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
8088 return OpenMPOffloadMappingFlags::OMP_MAP_PRIVATE |
8089 OpenMPOffloadMappingFlags::OMP_MAP_TO;
8092 if (I != LambdasMap.end())
8094 return getMapTypeBits(
8095 I->getSecond()->getMapType(), I->getSecond()->getMapTypeModifiers(),
8096 {}, I->getSecond()->isImplicit(),
8100 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
8101 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
8104 void getPlainLayout(
const CXXRecordDecl *RD,
8105 llvm::SmallVectorImpl<const FieldDecl *> &Layout,
8106 bool AsBase)
const {
8109 llvm::StructType *St =
8112 unsigned NumElements = St->getNumElements();
8114 llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>, 4>
8115 RecordLayout(NumElements);
8118 for (
const auto &I : RD->
bases()) {
8122 QualType BaseTy = I.getType();
8133 RecordLayout[FieldIndex] =
Base;
8136 for (
const auto &I : RD->
vbases()) {
8137 QualType BaseTy = I.getType();
8144 if (RecordLayout[FieldIndex])
8146 RecordLayout[FieldIndex] =
Base;
8149 assert(!RD->
isUnion() &&
"Unexpected union.");
8150 for (
const auto *Field : RD->
fields()) {
8153 if (!
Field->isBitField() &&
8156 RecordLayout[FieldIndex] =
Field;
8159 for (
const llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>
8160 &
Data : RecordLayout) {
8163 if (
const auto *Base = dyn_cast<const CXXRecordDecl *>(
Data))
8164 getPlainLayout(Base, Layout,
true);
8175 void generateAllInfoForClauses(
8176 ArrayRef<const OMPClause *> Clauses, MapCombinedInfoTy &CombinedInfo,
8177 llvm::OpenMPIRBuilder &OMPBuilder,
8178 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkipVarSet =
8179 llvm::DenseSet<CanonicalDeclPtr<const Decl>>())
const {
8184 llvm::MapVector<CanonicalDeclPtr<const Decl>,
8185 SmallVector<SmallVector<MapInfo, 8>, 4>>
8191 [&Info, &SkipVarSet](
8192 const ValueDecl *D, MapKind
Kind,
8195 ArrayRef<OpenMPMapModifierKind> MapModifiers,
8196 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
8197 bool ReturnDevicePointer,
bool IsImplicit,
const ValueDecl *Mapper,
8198 const Expr *VarRef =
nullptr,
bool ForDeviceAddr =
false) {
8199 if (SkipVarSet.contains(D))
8201 auto It = Info.try_emplace(D, Total).first;
8202 It->second[
Kind].emplace_back(
8203 L, MapType, MapModifiers, MotionModifiers, ReturnDevicePointer,
8204 IsImplicit, Mapper, VarRef, ForDeviceAddr);
8207 for (
const auto *
Cl : Clauses) {
8208 const auto *
C = dyn_cast<OMPMapClause>(
Cl);
8212 if (llvm::is_contained(
C->getMapTypeModifiers(),
8213 OMPC_MAP_MODIFIER_present))
8215 else if (
C->getMapType() == OMPC_MAP_alloc)
8217 const auto *EI =
C->getVarRefs().begin();
8218 for (
const auto L :
C->component_lists()) {
8219 const Expr *E = (
C->getMapLoc().isValid()) ? *EI :
nullptr;
8220 InfoGen(std::get<0>(L), Kind, std::get<1>(L),
C->getMapType(),
8221 C->getMapTypeModifiers(), {},
8222 false,
C->isImplicit(), std::get<2>(L),
8227 for (
const auto *
Cl : Clauses) {
8228 const auto *
C = dyn_cast<OMPToClause>(
Cl);
8232 if (llvm::is_contained(
C->getMotionModifiers(),
8233 OMPC_MOTION_MODIFIER_present))
8235 const auto *EI =
C->getVarRefs().begin();
8236 for (
const auto L :
C->component_lists()) {
8237 InfoGen(std::get<0>(L), Kind, std::get<1>(L), OMPC_MAP_to, {},
8238 C->getMotionModifiers(),
false,
8239 C->isImplicit(), std::get<2>(L), *EI);
8243 for (
const auto *
Cl : Clauses) {
8244 const auto *
C = dyn_cast<OMPFromClause>(
Cl);
8248 if (llvm::is_contained(
C->getMotionModifiers(),
8249 OMPC_MOTION_MODIFIER_present))
8251 const auto *EI =
C->getVarRefs().begin();
8252 for (
const auto L :
C->component_lists()) {
8253 InfoGen(std::get<0>(L), Kind, std::get<1>(L), OMPC_MAP_from, {},
8254 C->getMotionModifiers(),
8255 false,
C->isImplicit(), std::get<2>(L),
8268 llvm::MapVector<CanonicalDeclPtr<const Decl>,
8269 SmallVector<DeferredDevicePtrEntryTy, 4>>
8271 MapCombinedInfoTy UseDeviceDataCombinedInfo;
8273 auto &&UseDeviceDataCombinedInfoGen =
8274 [&UseDeviceDataCombinedInfo](
const ValueDecl *VD, llvm::Value *Ptr,
8275 CodeGenFunction &CGF,
bool IsDevAddr) {
8276 UseDeviceDataCombinedInfo.Exprs.push_back(VD);
8277 UseDeviceDataCombinedInfo.BasePointers.emplace_back(Ptr);
8278 UseDeviceDataCombinedInfo.DevicePtrDecls.emplace_back(VD);
8279 UseDeviceDataCombinedInfo.DevicePointers.emplace_back(
8280 IsDevAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer);
8281 UseDeviceDataCombinedInfo.Pointers.push_back(Ptr);
8282 UseDeviceDataCombinedInfo.Sizes.push_back(
8283 llvm::Constant::getNullValue(CGF.Int64Ty));
8284 UseDeviceDataCombinedInfo.Types.push_back(
8285 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM);
8286 UseDeviceDataCombinedInfo.Mappers.push_back(
nullptr);
8290 [&DeferredInfo, &UseDeviceDataCombinedInfoGen,
8291 &InfoGen](CodeGenFunction &CGF,
const Expr *IE,
const ValueDecl *VD,
8294 bool IsImplicit,
bool IsDevAddr) {
8307 false, IsImplicit,
nullptr,
nullptr,
8309 DeferredInfo[
nullptr].emplace_back(IE, VD, IsDevAddr);
8313 if (IE->isGLValue())
8314 Ptr = CGF.EmitLValue(IE).getPointer(CGF);
8316 Ptr = CGF.EmitScalarExpr(IE);
8318 Ptr = CGF.EmitLoadOfScalar(CGF.EmitLValue(IE), IE->getExprLoc());
8320 UseDeviceDataCombinedInfoGen(VD, Ptr, CGF, IsDevAddr);
8324 auto &&IsMapInfoExist = [&Info](CodeGenFunction &CGF,
const ValueDecl *VD,
8325 const Expr *IE,
bool IsDevAddr) ->
bool {
8333 if (It != Info.end()) {
8335 for (
auto &
Data : It->second) {
8336 auto *CI = llvm::find_if(
Data, [VD](
const MapInfo &MI) {
8337 return MI.Components.back().getAssociatedDeclaration() == VD;
8345 if (CI !=
Data.end()) {
8347 CI->ForDeviceAddr = IsDevAddr;
8348 CI->ReturnDevicePointer =
true;
8352 auto PrevCI = std::next(CI->Components.rbegin());
8353 const auto *VarD = dyn_cast<VarDecl>(VD);
8356 !VD->getType().getNonReferenceType()->isPointerType() ||
8357 PrevCI == CI->Components.rend() ||
8359 VarD->hasLocalStorage()) {
8360 CI->ForDeviceAddr = IsDevAddr;
8361 CI->ReturnDevicePointer =
true;
8379 for (
const auto *
Cl : Clauses) {
8380 const auto *
C = dyn_cast<OMPUseDevicePtrClause>(
Cl);
8383 for (
const auto L :
C->component_lists()) {
8386 assert(!Components.empty() &&
8387 "Not expecting empty list of components!");
8388 const ValueDecl *VD = Components.back().getAssociatedDeclaration();
8390 const Expr *IE = Components.back().getAssociatedExpression();
8391 if (IsMapInfoExist(CGF, VD, IE,
false))
8393 MapInfoGen(CGF, IE, VD, Components,
C->isImplicit(),
8398 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
8399 for (
const auto *
Cl : Clauses) {
8400 const auto *
C = dyn_cast<OMPUseDeviceAddrClause>(
Cl);
8403 for (
const auto L :
C->component_lists()) {
8406 assert(!std::get<1>(L).empty() &&
8407 "Not expecting empty list of components!");
8408 const ValueDecl *VD = std::get<1>(L).back().getAssociatedDeclaration();
8409 if (!Processed.insert(VD).second)
8412 const Expr *IE = std::get<1>(L).back().getAssociatedExpression();
8413 if (IsMapInfoExist(CGF, VD, IE,
true))
8415 MapInfoGen(CGF, IE, VD, Components,
C->isImplicit(),
8420 for (
const auto &
Data : Info) {
8421 StructRangeInfoTy PartialStruct;
8423 MapCombinedInfoTy CurInfo;
8425 MapCombinedInfoTy StructBaseCurInfo;
8427 const ValueDecl *VD = cast_or_null<ValueDecl>(D);
8428 bool HasMapBasePtr =
false;
8429 bool HasMapArraySec =
false;
8431 for (
const auto &M :
Data.second) {
8432 HasMapBasePtr = any_of(M, [](
const MapInfo &L) {
8433 return isa_and_present<DeclRefExpr>(L.VarRef);
8435 HasMapArraySec = any_of(M, [](
const MapInfo &L) {
8436 return isa_and_present<ArraySectionExpr, ArraySubscriptExpr>(
8439 if (HasMapBasePtr && HasMapArraySec)
8443 for (
const auto &M :
Data.second) {
8444 for (
const MapInfo &L : M) {
8445 assert(!L.Components.empty() &&
8446 "Not expecting declaration with no component lists.");
8449 unsigned CurrentBasePointersIdx = CurInfo.BasePointers.size();
8450 unsigned StructBasePointersIdx =
8451 StructBaseCurInfo.BasePointers.size();
8452 CurInfo.NonContigInfo.IsNonContiguous =
8453 L.Components.back().isNonContiguous();
8454 generateInfoForComponentList(
8455 L.MapType, L.MapModifiers, L.MotionModifiers, L.Components,
8456 CurInfo, StructBaseCurInfo, PartialStruct,
8457 false, L.IsImplicit,
8458 true, L.Mapper, L.ForDeviceAddr, VD,
8460 HasMapBasePtr && HasMapArraySec);
8464 if (L.ReturnDevicePointer) {
8468 assert((CurrentBasePointersIdx < CurInfo.BasePointers.size() ||
8469 StructBasePointersIdx <
8470 StructBaseCurInfo.BasePointers.size()) &&
8471 "Unexpected number of mapped base pointers.");
8474 const ValueDecl *RelevantVD =
8475 L.Components.back().getAssociatedDeclaration();
8476 assert(RelevantVD &&
8477 "No relevant declaration related with device pointer??");
8484 if (StructBasePointersIdx < StructBaseCurInfo.BasePointers.size()) {
8485 StructBaseCurInfo.DevicePtrDecls[StructBasePointersIdx] =
8487 StructBaseCurInfo.DevicePointers[StructBasePointersIdx] =
8488 L.ForDeviceAddr ? DeviceInfoTy::Address
8489 : DeviceInfoTy::Pointer;
8490 StructBaseCurInfo.Types[StructBasePointersIdx] |=
8491 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
8493 CurInfo.DevicePtrDecls[CurrentBasePointersIdx] = RelevantVD;
8494 CurInfo.DevicePointers[CurrentBasePointersIdx] =
8495 L.ForDeviceAddr ? DeviceInfoTy::Address
8496 : DeviceInfoTy::Pointer;
8497 CurInfo.Types[CurrentBasePointersIdx] |=
8498 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
8506 auto CI = DeferredInfo.find(
Data.first);
8507 if (CI != DeferredInfo.end()) {
8508 for (
const DeferredDevicePtrEntryTy &L : CI->second) {
8509 llvm::Value *BasePtr;
8511 if (L.ForDeviceAddr) {
8512 if (L.IE->isGLValue())
8520 CurInfo.Types.push_back(
8521 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM |
8522 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
8526 L.IE->getExprLoc());
8530 CurInfo.Types.push_back(
8531 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
8532 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM |
8533 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
8535 CurInfo.Exprs.push_back(L.VD);
8536 CurInfo.BasePointers.emplace_back(BasePtr);
8537 CurInfo.DevicePtrDecls.emplace_back(L.VD);
8538 CurInfo.DevicePointers.emplace_back(
8539 L.ForDeviceAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer);
8540 CurInfo.Pointers.push_back(Ptr);
8541 CurInfo.Sizes.push_back(
8542 llvm::Constant::getNullValue(this->CGF.
Int64Ty));
8543 CurInfo.Mappers.push_back(
nullptr);
8549 MapCombinedInfoTy UnionCurInfo;
8550 UnionCurInfo.append(StructBaseCurInfo);
8551 UnionCurInfo.append(CurInfo);
8555 if (PartialStruct.Base.isValid()) {
8556 UnionCurInfo.NonContigInfo.Dims.push_back(0);
8558 emitCombinedEntry(CombinedInfo, UnionCurInfo.Types, PartialStruct,
8559 !VD, OMPBuilder, VD);
8563 CombinedInfo.append(UnionCurInfo);
8566 CombinedInfo.append(UseDeviceDataCombinedInfo);
8570 MappableExprsHandler(
const OMPExecutableDirective &Dir, CodeGenFunction &CGF)
8571 : CurDir(&Dir), CGF(CGF) {
8573 for (
const auto *
C : Dir.getClausesOfKind<OMPFirstprivateClause>())
8574 for (
const auto *D :
C->varlist())
8575 FirstPrivateDecls.try_emplace(
8578 for (
const auto *
C : Dir.getClausesOfKind<OMPUsesAllocatorsClause>()) {
8579 for (unsigned I = 0, E = C->getNumberOfAllocators(); I < E; ++I) {
8580 OMPUsesAllocatorsClause::Data D = C->getAllocatorData(I);
8581 if (const auto *DRE = dyn_cast_or_null<DeclRefExpr>(D.AllocatorTraits))
8582 FirstPrivateDecls.try_emplace(cast<VarDecl>(DRE->getDecl()),
8584 else if (const auto *VD = dyn_cast<VarDecl>(
8585 cast<DeclRefExpr>(D.Allocator->IgnoreParenImpCasts())
8587 FirstPrivateDecls.try_emplace(VD, true);
8591 for (
const auto *
C : Dir.getClausesOfKind<OMPIsDevicePtrClause>())
8592 for (
auto L :
C->component_lists())
8593 DevPointersMap[std::get<0>(L)].push_back(std::get<1>(L));
8595 for (
const auto *
C : Dir.getClausesOfKind<OMPHasDeviceAddrClause>())
8596 for (
auto L :
C->component_lists())
8597 HasDevAddrsMap[std::get<0>(L)].push_back(std::get<1>(L));
8599 for (
const auto *
C : Dir.getClausesOfKind<OMPMapClause>()) {
8600 if (C->getMapType() != OMPC_MAP_to)
8602 for (auto L : C->component_lists()) {
8603 const ValueDecl *VD = std::get<0>(L);
8604 const auto *RD = VD ? VD->getType()
8606 .getNonReferenceType()
8607 ->getAsCXXRecordDecl()
8609 if (RD && RD->isLambda())
8610 LambdasMap.try_emplace(std::get<0>(L), C);
8616 MappableExprsHandler(
const OMPDeclareMapperDecl &Dir,
CodeGenFunction &CGF)
8617 : CurDir(&Dir), CGF(CGF) {}
8622 void emitCombinedEntry(MapCombinedInfoTy &CombinedInfo,
8623 MapFlagsArrayTy &CurTypes,
8624 const StructRangeInfoTy &PartialStruct,
bool IsMapThis,
8625 llvm::OpenMPIRBuilder &OMPBuilder,
8626 const ValueDecl *VD =
nullptr,
8627 unsigned OffsetForMemberOfFlag = 0,
8628 bool NotTargetParams =
true)
const {
8629 if (CurTypes.size() == 1 &&
8630 ((CurTypes.back() & OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) !=
8631 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) &&
8632 !PartialStruct.IsArraySection)
8634 Address LBAddr = PartialStruct.LowestElem.second;
8635 Address HBAddr = PartialStruct.HighestElem.second;
8636 if (PartialStruct.HasCompleteRecord) {
8637 LBAddr = PartialStruct.LB;
8638 HBAddr = PartialStruct.LB;
8640 CombinedInfo.Exprs.push_back(VD);
8642 CombinedInfo.BasePointers.push_back(PartialStruct.Base.emitRawPointer(CGF));
8643 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8644 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8647 const CXXMethodDecl *MD =
8649 const CXXRecordDecl *RD = MD ? MD->
getParent() :
nullptr;
8650 bool HasBaseClass = RD && IsMapThis ? RD->
getNumBases() > 0 :
false;
8660 CombinedInfo.Pointers.push_back(PartialStruct.Base.emitRawPointer(CGF));
8665 CombinedInfo.Sizes.push_back(Size);
8667 CombinedInfo.Pointers.push_back(LB);
8670 llvm::Value *HAddr = CGF.
Builder.CreateConstGEP1_32(
8674 llvm::Value *Diff = CGF.
Builder.CreatePtrDiff(CGF.
Int8Ty, CHAddr, CLAddr);
8677 CombinedInfo.Sizes.push_back(Size);
8679 CombinedInfo.Mappers.push_back(
nullptr);
8681 CombinedInfo.Types.push_back(
8682 NotTargetParams ? OpenMPOffloadMappingFlags::OMP_MAP_NONE
8683 : !PartialStruct.PreliminaryMapData.BasePointers.empty()
8684 ? OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ
8685 : OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
8688 if (CurTypes.end() !=
8689 llvm::find_if(CurTypes, [](OpenMPOffloadMappingFlags
Type) {
8690 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
8691 Type & OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
8693 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
8695 (*CurTypes.begin()) &= ~OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
8702 if (CurTypes.end() !=
8703 llvm::find_if(CurTypes, [](OpenMPOffloadMappingFlags
Type) {
8704 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
8705 Type & OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD);
8707 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
8708 for (
auto &M : CurTypes)
8709 M |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
8715 OpenMPOffloadMappingFlags MemberOfFlag = OMPBuilder.getMemberOfFlag(
8716 OffsetForMemberOfFlag + CombinedInfo.BasePointers.size() - 1);
8717 for (
auto &M : CurTypes)
8718 OMPBuilder.setCorrectMemberOfFlag(M, MemberOfFlag);
8726 void generateAllInfo(
8727 MapCombinedInfoTy &CombinedInfo, llvm::OpenMPIRBuilder &OMPBuilder,
8728 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkipVarSet =
8729 llvm::DenseSet<CanonicalDeclPtr<const Decl>>())
const {
8731 "Expect a executable directive");
8733 generateAllInfoForClauses(CurExecDir->clauses(), CombinedInfo, OMPBuilder,
8740 void generateAllInfoForMapper(MapCombinedInfoTy &CombinedInfo,
8741 llvm::OpenMPIRBuilder &OMPBuilder)
const {
8743 "Expect a declare mapper directive");
8745 generateAllInfoForClauses(CurMapperDir->clauses(), CombinedInfo,
8750 void generateInfoForLambdaCaptures(
8751 const ValueDecl *VD, llvm::Value *Arg, MapCombinedInfoTy &CombinedInfo,
8752 llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers)
const {
8760 llvm::DenseMap<const ValueDecl *, FieldDecl *> Captures;
8761 FieldDecl *ThisCapture =
nullptr;
8767 LambdaPointers.try_emplace(ThisLVal.getPointer(CGF),
8768 VDLVal.getPointer(CGF));
8769 CombinedInfo.Exprs.push_back(VD);
8770 CombinedInfo.BasePointers.push_back(ThisLVal.getPointer(CGF));
8771 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8772 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8773 CombinedInfo.Pointers.push_back(ThisLValVal.getPointer(CGF));
8774 CombinedInfo.Sizes.push_back(
8777 CombinedInfo.Types.push_back(
8778 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
8779 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
8780 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
8781 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
8782 CombinedInfo.Mappers.push_back(
nullptr);
8784 for (
const LambdaCapture &LC : RD->
captures()) {
8785 if (!LC.capturesVariable())
8790 auto It = Captures.find(VD);
8791 assert(It != Captures.end() &&
"Found lambda capture without field.");
8795 LambdaPointers.try_emplace(VarLVal.getPointer(CGF),
8796 VDLVal.getPointer(CGF));
8797 CombinedInfo.Exprs.push_back(VD);
8798 CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF));
8799 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8800 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8801 CombinedInfo.Pointers.push_back(VarLValVal.getPointer(CGF));
8802 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
8808 LambdaPointers.try_emplace(VarLVal.getPointer(CGF),
8809 VDLVal.getPointer(CGF));
8810 CombinedInfo.Exprs.push_back(VD);
8811 CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF));
8812 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8813 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8814 CombinedInfo.Pointers.push_back(VarRVal.
getScalarVal());
8815 CombinedInfo.Sizes.push_back(llvm::ConstantInt::get(CGF.
Int64Ty, 0));
8817 CombinedInfo.Types.push_back(
8818 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
8819 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
8820 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
8821 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
8822 CombinedInfo.Mappers.push_back(
nullptr);
8827 void adjustMemberOfForLambdaCaptures(
8828 llvm::OpenMPIRBuilder &OMPBuilder,
8829 const llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers,
8830 MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
8831 MapFlagsArrayTy &Types)
const {
8832 for (
unsigned I = 0, E = Types.size(); I < E; ++I) {
8834 if (Types[I] != (OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
8835 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
8836 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
8837 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT))
8839 llvm::Value *BasePtr = LambdaPointers.lookup(BasePointers[I]);
8840 assert(BasePtr &&
"Unable to find base lambda address.");
8842 for (
unsigned J = I; J > 0; --J) {
8843 unsigned Idx = J - 1;
8844 if (Pointers[Idx] != BasePtr)
8849 assert(TgtIdx != -1 &&
"Unable to find parent lambda.");
8853 OpenMPOffloadMappingFlags MemberOfFlag =
8854 OMPBuilder.getMemberOfFlag(TgtIdx);
8855 OMPBuilder.setCorrectMemberOfFlag(Types[I], MemberOfFlag);
8862 void generateInfoForCaptureFromClauseInfo(
8863 const CapturedStmt::Capture *Cap, llvm::Value *Arg,
8864 MapCombinedInfoTy &CurCaptureVarInfo, llvm::OpenMPIRBuilder &OMPBuilder,
8865 unsigned OffsetForMemberOfFlag)
const {
8867 "Not expecting to generate map info for a variable array type!");
8876 if (LambdasMap.count(VD))
8882 if (VD && (DevPointersMap.count(VD) || HasDevAddrsMap.count(VD))) {
8883 CurCaptureVarInfo.Exprs.push_back(VD);
8884 CurCaptureVarInfo.BasePointers.emplace_back(Arg);
8885 CurCaptureVarInfo.DevicePtrDecls.emplace_back(VD);
8886 CurCaptureVarInfo.DevicePointers.emplace_back(DeviceInfoTy::Pointer);
8887 CurCaptureVarInfo.Pointers.push_back(Arg);
8888 CurCaptureVarInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
8891 CurCaptureVarInfo.Types.push_back(
8892 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
8893 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
8894 CurCaptureVarInfo.Mappers.push_back(
nullptr);
8898 MapDataArrayTy DeclComponentLists;
8902 auto It = DevPointersMap.find(VD);
8903 if (It != DevPointersMap.end())
8904 for (
const auto &MCL : It->second)
8905 DeclComponentLists.emplace_back(MCL, OMPC_MAP_to,
Unknown,
8908 auto I = HasDevAddrsMap.find(VD);
8909 if (I != HasDevAddrsMap.end())
8910 for (
const auto &MCL : I->second)
8911 DeclComponentLists.emplace_back(MCL, OMPC_MAP_tofrom,
Unknown,
8915 "Expect a executable directive");
8917 bool HasMapBasePtr =
false;
8918 bool HasMapArraySec =
false;
8919 for (
const auto *
C : CurExecDir->getClausesOfKind<OMPMapClause>()) {
8920 const auto *EI =
C->getVarRefs().begin();
8921 for (
const auto L :
C->decl_component_lists(VD)) {
8922 const ValueDecl *VDecl, *Mapper;
8924 const Expr *E = (
C->getMapLoc().isValid()) ? *EI :
nullptr;
8926 std::tie(VDecl, Components, Mapper) = L;
8927 assert(VDecl == VD &&
"We got information for the wrong declaration??");
8928 assert(!Components.empty() &&
8929 "Not expecting declaration with no component lists.");
8931 HasMapBasePtr =
true;
8934 HasMapArraySec =
true;
8935 DeclComponentLists.emplace_back(Components,
C->getMapType(),
8936 C->getMapTypeModifiers(),
8937 C->isImplicit(), Mapper, E);
8941 llvm::stable_sort(DeclComponentLists, [](
const MapData &LHS,
8942 const MapData &RHS) {
8943 ArrayRef<OpenMPMapModifierKind> MapModifiers = std::get<2>(LHS);
8946 llvm::is_contained(MapModifiers, clang::OMPC_MAP_MODIFIER_present);
8947 bool HasAllocs = MapType == OMPC_MAP_alloc;
8948 MapModifiers = std::get<2>(RHS);
8949 MapType = std::get<1>(LHS);
8951 llvm::is_contained(MapModifiers, clang::OMPC_MAP_MODIFIER_present);
8952 bool HasAllocsR = MapType == OMPC_MAP_alloc;
8953 return (HasPresent && !HasPresentR) || (HasAllocs && !HasAllocsR);
8956 auto GenerateInfoForComponentLists =
8957 [&](ArrayRef<MapData> DeclComponentLists,
8958 bool IsEligibleForTargetParamFlag) {
8959 MapCombinedInfoTy CurInfoForComponentLists;
8960 StructRangeInfoTy PartialStruct;
8962 if (DeclComponentLists.empty())
8965 generateInfoForCaptureFromComponentLists(
8966 VD, DeclComponentLists, CurInfoForComponentLists, PartialStruct,
8967 IsEligibleForTargetParamFlag,
8968 HasMapBasePtr && HasMapArraySec);
8973 if (PartialStruct.Base.isValid()) {
8974 CurCaptureVarInfo.append(PartialStruct.PreliminaryMapData);
8976 CurCaptureVarInfo, CurInfoForComponentLists.Types,
8977 PartialStruct, Cap->
capturesThis(), OMPBuilder,
nullptr,
8978 OffsetForMemberOfFlag,
8979 !IsEligibleForTargetParamFlag);
8983 if (CurInfoForComponentLists.BasePointers.empty())
8986 CurCaptureVarInfo.append(CurInfoForComponentLists);
8989 GenerateInfoForComponentLists(DeclComponentLists,
8996 void generateInfoForCaptureFromComponentLists(
8997 const ValueDecl *VD, ArrayRef<MapData> DeclComponentLists,
8998 MapCombinedInfoTy &CurComponentListInfo, StructRangeInfoTy &PartialStruct,
8999 bool IsListEligibleForTargetParamFlag,
9000 bool AreBothBasePtrAndPteeMapped =
false)
const {
9002 llvm::SmallDenseMap<
9009 for (
const MapData &L : DeclComponentLists) {
9012 ArrayRef<OpenMPMapModifierKind> MapModifiers;
9014 const ValueDecl *Mapper;
9016 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =
9019 for (
const MapData &L1 : ArrayRef(DeclComponentLists).slice(Count)) {
9021 std::tie(Components1, MapType, MapModifiers, IsImplicit, Mapper,
9023 auto CI = Components.rbegin();
9024 auto CE = Components.rend();
9025 auto SI = Components1.rbegin();
9026 auto SE = Components1.rend();
9027 for (; CI != CE && SI != SE; ++CI, ++SI) {
9028 if (CI->getAssociatedExpression()->getStmtClass() !=
9029 SI->getAssociatedExpression()->getStmtClass())
9032 if (CI->getAssociatedDeclaration() != SI->getAssociatedDeclaration())
9037 if (CI == CE || SI == SE) {
9039 if (CI == CE && SI == SE)
9041 const auto It = (SI == SE) ? CI : SI;
9048 (std::prev(It)->getAssociatedDeclaration() &&
9050 ->getAssociatedDeclaration()
9052 ->isPointerType()) ||
9053 (It->getAssociatedDeclaration() &&
9054 It->getAssociatedDeclaration()->getType()->isPointerType() &&
9055 std::next(It) != CE && std::next(It) != SE))
9057 const MapData &BaseData = CI == CE ? L : L1;
9059 SI == SE ? Components : Components1;
9060 OverlappedData[&BaseData].push_back(SubData);
9065 llvm::SmallVector<const FieldDecl *, 4> Layout;
9066 if (!OverlappedData.empty()) {
9069 while (BaseType != OrigType) {
9075 getPlainLayout(CRD, Layout,
false);
9081 for (
auto &Pair : OverlappedData) {
9088 auto CI = First.rbegin();
9089 auto CE = First.rend();
9090 auto SI = Second.rbegin();
9091 auto SE = Second.rend();
9092 for (; CI != CE && SI != SE; ++CI, ++SI) {
9093 if (CI->getAssociatedExpression()->getStmtClass() !=
9094 SI->getAssociatedExpression()->getStmtClass())
9097 if (CI->getAssociatedDeclaration() !=
9098 SI->getAssociatedDeclaration())
9103 if (CI == CE && SI == SE)
9107 if (CI == CE || SI == SE)
9112 if (FD1->getParent() == FD2->getParent())
9113 return FD1->getFieldIndex() < FD2->getFieldIndex();
9115 llvm::find_if(Layout, [FD1, FD2](
const FieldDecl *FD) {
9116 return FD == FD1 || FD == FD2;
9124 bool AddTargetParamFlag = IsListEligibleForTargetParamFlag;
9125 MapCombinedInfoTy StructBaseCombinedInfo;
9126 for (
const auto &Pair : OverlappedData) {
9127 const MapData &L = *Pair.getFirst();
9130 ArrayRef<OpenMPMapModifierKind> MapModifiers;
9132 const ValueDecl *Mapper;
9134 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =
9136 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
9137 OverlappedComponents = Pair.getSecond();
9138 generateInfoForComponentList(
9139 MapType, MapModifiers, {}, Components, CurComponentListInfo,
9140 StructBaseCombinedInfo, PartialStruct, AddTargetParamFlag, IsImplicit,
9142 false, VD, VarRef, OverlappedComponents);
9143 AddTargetParamFlag =
false;
9146 for (
const MapData &L : DeclComponentLists) {
9149 ArrayRef<OpenMPMapModifierKind> MapModifiers;
9151 const ValueDecl *Mapper;
9153 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =
9155 auto It = OverlappedData.find(&L);
9156 if (It == OverlappedData.end())
9157 generateInfoForComponentList(
9158 MapType, MapModifiers, {}, Components, CurComponentListInfo,
9159 StructBaseCombinedInfo, PartialStruct, AddTargetParamFlag,
9160 IsImplicit,
false, Mapper,
9162 {}, AreBothBasePtrAndPteeMapped);
9163 AddTargetParamFlag =
false;
9169 void generateDefaultMapInfo(
const CapturedStmt::Capture &CI,
9170 const FieldDecl &RI, llvm::Value *CV,
9171 MapCombinedInfoTy &CombinedInfo)
const {
9172 bool IsImplicit =
true;
9175 CombinedInfo.Exprs.push_back(
nullptr);
9176 CombinedInfo.BasePointers.push_back(CV);
9177 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
9178 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
9179 CombinedInfo.Pointers.push_back(CV);
9181 CombinedInfo.Sizes.push_back(
9185 CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_TO |
9186 OpenMPOffloadMappingFlags::OMP_MAP_FROM);
9190 CombinedInfo.BasePointers.push_back(CV);
9191 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
9192 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
9193 CombinedInfo.Pointers.push_back(CV);
9197 CombinedInfo.Types.push_back(
9198 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
9199 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
9204 CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_NONE);
9205 CombinedInfo.Sizes.push_back(llvm::Constant::getNullValue(CGF.
Int64Ty));
9207 auto I = FirstPrivateDecls.find(VD);
9208 if (I != FirstPrivateDecls.end())
9209 IsImplicit = I->getSecond();
9214 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
9219 CombinedInfo.Types.push_back(getMapModifiersForPrivateClauses(CI));
9221 auto I = FirstPrivateDecls.find(VD);
9223 CombinedInfo.BasePointers.push_back(CV);
9224 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
9225 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
9232 CombinedInfo.Pointers.push_back(CV);
9234 if (I != FirstPrivateDecls.end())
9235 IsImplicit = I->getSecond();
9238 CombinedInfo.Types.back() |=
9239 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
9243 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
9246 CombinedInfo.Mappers.push_back(
nullptr);
9258 dyn_cast<MemberExpr>(OASE->getBase()->IgnoreParenImpCasts()))
9259 return ME->getMemberDecl();
9265static llvm::Constant *
9267 MappableExprsHandler::MappingExprInfo &MapExprs) {
9269 uint32_t SrcLocStrSize;
9270 if (!MapExprs.getMapDecl() && !MapExprs.getMapExpr())
9271 return OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
9274 if (!MapExprs.getMapDecl() && MapExprs.getMapExpr()) {
9278 Loc = MapExprs.getMapExpr()->getExprLoc();
9280 Loc = MapExprs.getMapDecl()->getLocation();
9283 std::string ExprName;
9284 if (MapExprs.getMapExpr()) {
9286 llvm::raw_string_ostream OS(ExprName);
9287 MapExprs.getMapExpr()->printPretty(OS,
nullptr, P);
9289 ExprName = MapExprs.getMapDecl()->getNameAsString();
9298 return OMPBuilder.getOrCreateSrcLocStr(
FileName, ExprName, PLoc.
getLine(),
9305 CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
9307 bool IsNonContiguous =
false,
bool ForEndCall =
false) {
9310 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
9313 InsertPointTy CodeGenIP(CGF.
Builder.GetInsertBlock(),
9314 CGF.
Builder.GetInsertPoint());
9316 auto DeviceAddrCB = [&](
unsigned int I, llvm::Value *NewDecl) {
9317 if (
const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
9322 auto CustomMapperCB = [&](
unsigned int I) {
9323 llvm::Function *MFunc =
nullptr;
9324 if (CombinedInfo.Mappers[I]) {
9325 Info.HasMapper =
true;
9331 cantFail(OMPBuilder.emitOffloadingArraysAndArgs(
9332 AllocaIP, CodeGenIP, Info, Info.RTArgs, CombinedInfo, CustomMapperCB,
9333 IsNonContiguous, ForEndCall, DeviceAddrCB));
9337static const OMPExecutableDirective *
9339 const auto *CS = D.getInnermostCapturedStmt();
9342 const Stmt *ChildStmt =
9345 if (
const auto *NestedDir =
9346 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
9348 switch (D.getDirectiveKind()) {
9354 if (DKind == OMPD_teams) {
9355 Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers(
9360 if (
const auto *NND =
9361 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
9362 DKind = NND->getDirectiveKind();
9368 case OMPD_target_teams:
9372 case OMPD_target_parallel:
9373 case OMPD_target_simd:
9374 case OMPD_target_parallel_for:
9375 case OMPD_target_parallel_for_simd:
9377 case OMPD_target_teams_distribute:
9378 case OMPD_target_teams_distribute_simd:
9379 case OMPD_target_teams_distribute_parallel_for:
9380 case OMPD_target_teams_distribute_parallel_for_simd:
9383 case OMPD_parallel_for:
9384 case OMPD_parallel_master:
9385 case OMPD_parallel_sections:
9387 case OMPD_parallel_for_simd:
9389 case OMPD_cancellation_point:
9391 case OMPD_threadprivate:
9402 case OMPD_taskyield:
9405 case OMPD_taskgroup:
9411 case OMPD_target_data:
9412 case OMPD_target_exit_data:
9413 case OMPD_target_enter_data:
9414 case OMPD_distribute:
9415 case OMPD_distribute_simd:
9416 case OMPD_distribute_parallel_for:
9417 case OMPD_distribute_parallel_for_simd:
9418 case OMPD_teams_distribute:
9419 case OMPD_teams_distribute_simd:
9420 case OMPD_teams_distribute_parallel_for:
9421 case OMPD_teams_distribute_parallel_for_simd:
9422 case OMPD_target_update:
9423 case OMPD_declare_simd:
9424 case OMPD_declare_variant:
9425 case OMPD_begin_declare_variant:
9426 case OMPD_end_declare_variant:
9427 case OMPD_declare_target:
9428 case OMPD_end_declare_target:
9429 case OMPD_declare_reduction:
9430 case OMPD_declare_mapper:
9432 case OMPD_taskloop_simd:
9433 case OMPD_master_taskloop:
9434 case OMPD_master_taskloop_simd:
9435 case OMPD_parallel_master_taskloop:
9436 case OMPD_parallel_master_taskloop_simd:
9438 case OMPD_metadirective:
9441 llvm_unreachable(
"Unexpected directive.");
9486 auto *MapperVarDecl =
9488 CharUnits ElementSize =
C.getTypeSizeInChars(Ty);
9489 llvm::Type *ElemTy =
CGM.getTypes().ConvertTypeForMem(Ty);
9492 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
9493 auto PrivatizeAndGenMapInfoCB =
9494 [&](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP, llvm::Value *PtrPHI,
9495 llvm::Value *BeginArg) -> llvm::OpenMPIRBuilder::MapInfosTy & {
9496 MapperCGF.
Builder.restoreIP(CodeGenIP);
9506 Scope.addPrivate(MapperVarDecl, PtrCurrent);
9507 (void)
Scope.Privatize();
9510 MappableExprsHandler MEHandler(*D, MapperCGF);
9511 MEHandler.generateAllInfoForMapper(CombinedInfo,
OMPBuilder);
9513 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
9516 if (
CGM.getCodeGenOpts().getDebugInfo() !=
9517 llvm::codegenoptions::NoDebugInfo) {
9518 CombinedInfo.Names.resize(CombinedInfo.Exprs.size());
9519 llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),
9523 return CombinedInfo;
9526 auto CustomMapperCB = [&](
unsigned I) {
9527 llvm::Function *MapperFunc =
nullptr;
9528 if (CombinedInfo.Mappers[I]) {
9532 assert(MapperFunc &&
"Expect a valid mapper function is available.");
9538 llvm::raw_svector_ostream Out(TyStr);
9539 CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(Ty, Out);
9542 llvm::Function *NewFn = cantFail(
OMPBuilder.emitUserDefinedMapper(
9543 PrivatizeAndGenMapInfoCB, ElemTy, Name, CustomMapperCB));
9544 UDMMap.try_emplace(D, NewFn);
9568 Kind != OMPD_target_teams_loop)
9571 return llvm::ConstantInt::get(CGF.
Int64Ty, 0);
9574 if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD))
9575 return NumIterations;
9576 return llvm::ConstantInt::get(CGF.
Int64Ty, 0);
9585 if (OffloadingMandatory) {
9586 CGF.
Builder.CreateUnreachable();
9588 if (RequiresOuterTask) {
9589 CapturedVars.clear();
9598 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
9601 llvm::Value *DeviceID;
9602 if (
Device.getPointer()) {
9604 Device.getInt() == OMPC_DEVICE_device_num) &&
9605 "Expected device_num modifier.");
9610 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
9617 llvm::Value *DynCGroupMem = CGF.
Builder.getInt32(0);
9622 DynMemClause->getSize(),
true);
9623 DynCGroupMem = CGF.
Builder.CreateIntCast(DynCGroupMemVal, CGF.
Int32Ty,
9626 return DynCGroupMem;
9631 llvm::OpenMPIRBuilder &OMPBuilder,
9633 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) {
9635 llvm::DenseMap<llvm::Value *, llvm::Value *> LambdaPointers;
9637 auto *CV = CapturedVars.begin();
9640 CI != CE; ++CI, ++RI, ++CV) {
9641 MappableExprsHandler::MapCombinedInfoTy CurInfo;
9646 CurInfo.Exprs.push_back(
nullptr);
9647 CurInfo.BasePointers.push_back(*CV);
9648 CurInfo.DevicePtrDecls.push_back(
nullptr);
9649 CurInfo.DevicePointers.push_back(
9650 MappableExprsHandler::DeviceInfoTy::None);
9651 CurInfo.Pointers.push_back(*CV);
9652 CurInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
9655 CurInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9656 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM |
9657 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
9658 CurInfo.Mappers.push_back(
nullptr);
9662 MEHandler.generateInfoForCaptureFromClauseInfo(
9663 CI, *CV, CurInfo, OMPBuilder,
9664 CombinedInfo.BasePointers.size());
9669 MappedVarSet.insert(
nullptr);
9671 if (CurInfo.BasePointers.empty())
9672 MEHandler.generateDefaultMapInfo(*CI, **RI, *CV, CurInfo);
9677 MEHandler.generateInfoForLambdaCaptures(CI->
getCapturedVar(), *CV,
9678 CurInfo, LambdaPointers);
9681 assert(!CurInfo.BasePointers.empty() &&
9682 "Non-existing map pointer for capture!");
9683 assert(CurInfo.BasePointers.size() == CurInfo.Pointers.size() &&
9684 CurInfo.BasePointers.size() == CurInfo.Sizes.size() &&
9685 CurInfo.BasePointers.size() == CurInfo.Types.size() &&
9686 CurInfo.BasePointers.size() == CurInfo.Mappers.size() &&
9687 "Inconsistent map information sizes!");
9690 CombinedInfo.append(CurInfo);
9693 MEHandler.adjustMemberOfForLambdaCaptures(
9694 OMPBuilder, LambdaPointers, CombinedInfo.BasePointers,
9695 CombinedInfo.Pointers, CombinedInfo.Types);
9699 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
9700 llvm::OpenMPIRBuilder &OMPBuilder,
9707 MEHandler.generateAllInfo(CombinedInfo, OMPBuilder, SkippedVarSet);
9709 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
9713 llvm::codegenoptions::NoDebugInfo) {
9714 CombinedInfo.Names.resize(CombinedInfo.Exprs.size());
9715 llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),
9723 llvm::OpenMPIRBuilder &OMPBuilder,
9724 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) {
9726 MappableExprsHandler MEHandler(D, CGF);
9727 llvm::DenseSet<CanonicalDeclPtr<const Decl>> MappedVarSet;
9730 MappedVarSet, CombinedInfo);
9731 genMapInfo(MEHandler, CGF, CombinedInfo, OMPBuilder, MappedVarSet);
9734template <
typename ClauseTy>
9739 const auto *
C = D.getSingleClause<ClauseTy>();
9740 assert(!
C->varlist_empty() &&
9741 "ompx_bare requires explicit num_teams and thread_limit");
9743 for (
auto *E :
C->varlist()) {
9755 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
9757 llvm::Value *&MapTypesArray, llvm::Value *&MapNamesArray,
9762 llvm::OpenMPIRBuilder &OMPBuilder = OMPRuntime->
getOMPBuilder();
9765 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
9767 genMapInfo(D, CGF, CS, CapturedVars, OMPBuilder, CombinedInfo);
9781 MapTypesArray = Info.RTArgs.MapTypesArray;
9782 MapNamesArray = Info.RTArgs.MapNamesArray;
9784 auto &&ThenGen = [&OMPRuntime, OutlinedFn, &D, &CapturedVars,
9785 RequiresOuterTask, &CS, OffloadingMandatory,
Device,
9786 OutlinedFnID, &InputInfo, &MapTypesArray, &MapNamesArray,
9788 bool IsReverseOffloading =
Device.getInt() == OMPC_DEVICE_ancestor;
9790 if (IsReverseOffloading) {
9796 RequiresOuterTask, CS, OffloadingMandatory, CGF);
9801 unsigned NumTargetItems = InputInfo.NumberOfTargetItems;
9803 llvm::Value *BasePointersArray =
9804 InputInfo.BasePointersArray.emitRawPointer(CGF);
9805 llvm::Value *PointersArray = InputInfo.PointersArray.emitRawPointer(CGF);
9806 llvm::Value *SizesArray = InputInfo.SizesArray.emitRawPointer(CGF);
9807 llvm::Value *MappersArray = InputInfo.MappersArray.emitRawPointer(CGF);
9809 auto &&EmitTargetCallFallbackCB =
9810 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
9811 OffloadingMandatory, &CGF](llvm::OpenMPIRBuilder::InsertPointTy IP)
9812 -> llvm::OpenMPIRBuilder::InsertPointTy {
9815 RequiresOuterTask, CS, OffloadingMandatory, CGF);
9828 NumThreads.push_back(
9834 llvm::Value *NumIterations =
9837 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
9840 llvm::OpenMPIRBuilder::TargetDataRTArgs RTArgs(
9841 BasePointersArray, PointersArray, SizesArray, MapTypesArray,
9842 nullptr , MappersArray, MapNamesArray);
9844 llvm::OpenMPIRBuilder::TargetKernelArgs Args(
9845 NumTargetItems, RTArgs, NumIterations, NumTeams, NumThreads,
9846 DynCGGroupMem, HasNoWait);
9848 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
9850 CGF.
Builder, OutlinedFnID, EmitTargetCallFallbackCB, Args, DeviceID,
9852 CGF.
Builder.restoreIP(AfterIP);
9855 if (RequiresOuterTask)
9870 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
9873 RequiresOuterTask, CS, OffloadingMandatory, CGF);
9876 if (RequiresOuterTask) {
9886 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID,
const Expr *IfCond,
9887 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
9894 const bool OffloadingMandatory = !
CGM.getLangOpts().OpenMPIsTargetDevice &&
9895 CGM.getLangOpts().OpenMPOffloadMandatory;
9897 assert((OffloadingMandatory || OutlinedFn) &&
"Invalid outlined function!");
9899 const bool RequiresOuterTask =
9903 (
CGM.getLangOpts().OpenMP >= 51 &&
9907 const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target);
9915 llvm::Value *MapTypesArray =
nullptr;
9916 llvm::Value *MapNamesArray =
nullptr;
9918 auto &&TargetThenGen = [
this, OutlinedFn, &D, &CapturedVars,
9919 RequiresOuterTask, &CS, OffloadingMandatory,
Device,
9920 OutlinedFnID, &InputInfo, &MapTypesArray,
9924 RequiresOuterTask, CS, OffloadingMandatory,
9925 Device, OutlinedFnID, InputInfo, MapTypesArray,
9926 MapNamesArray, SizeEmitter, CGF,
CGM);
9929 auto &&TargetElseGen =
9930 [
this, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
9933 CS, OffloadingMandatory, CGF);
9942 emitIfClause(CGF, IfCond, TargetThenGen, TargetElseGen);
9954 StringRef ParentName) {
9959 bool RequiresDeviceCodegen =
9964 if (RequiresDeviceCodegen) {
9972 if (!
OMPBuilder.OffloadInfoManager.hasTargetRegionEntryInfo(EntryInfo))
9975 switch (E.getDirectiveKind()) {
9980 case OMPD_target_parallel:
9984 case OMPD_target_teams:
9988 case OMPD_target_teams_distribute:
9992 case OMPD_target_teams_distribute_simd:
9996 case OMPD_target_parallel_for:
10000 case OMPD_target_parallel_for_simd:
10004 case OMPD_target_simd:
10008 case OMPD_target_teams_distribute_parallel_for:
10013 case OMPD_target_teams_distribute_parallel_for_simd:
10019 case OMPD_target_teams_loop:
10023 case OMPD_target_parallel_loop:
10027 case OMPD_parallel:
10029 case OMPD_parallel_for:
10030 case OMPD_parallel_master:
10031 case OMPD_parallel_sections:
10032 case OMPD_for_simd:
10033 case OMPD_parallel_for_simd:
10035 case OMPD_cancellation_point:
10037 case OMPD_threadprivate:
10038 case OMPD_allocate:
10043 case OMPD_sections:
10047 case OMPD_critical:
10048 case OMPD_taskyield:
10050 case OMPD_taskwait:
10051 case OMPD_taskgroup:
10057 case OMPD_target_data:
10058 case OMPD_target_exit_data:
10059 case OMPD_target_enter_data:
10060 case OMPD_distribute:
10061 case OMPD_distribute_simd:
10062 case OMPD_distribute_parallel_for:
10063 case OMPD_distribute_parallel_for_simd:
10064 case OMPD_teams_distribute:
10065 case OMPD_teams_distribute_simd:
10066 case OMPD_teams_distribute_parallel_for:
10067 case OMPD_teams_distribute_parallel_for_simd:
10068 case OMPD_target_update:
10069 case OMPD_declare_simd:
10070 case OMPD_declare_variant:
10071 case OMPD_begin_declare_variant:
10072 case OMPD_end_declare_variant:
10073 case OMPD_declare_target:
10074 case OMPD_end_declare_target:
10075 case OMPD_declare_reduction:
10076 case OMPD_declare_mapper:
10077 case OMPD_taskloop:
10078 case OMPD_taskloop_simd:
10079 case OMPD_master_taskloop:
10080 case OMPD_master_taskloop_simd:
10081 case OMPD_parallel_master_taskloop:
10082 case OMPD_parallel_master_taskloop_simd:
10083 case OMPD_requires:
10084 case OMPD_metadirective:
10087 llvm_unreachable(
"Unknown target directive for OpenMP device codegen.");
10092 if (
const auto *E = dyn_cast<OMPExecutableDirective>(S)) {
10093 if (!E->hasAssociatedStmt() || !E->getAssociatedStmt())
10101 if (
const auto *L = dyn_cast<LambdaExpr>(S))
10110 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
10111 OMPDeclareTargetDeclAttr::getDeviceType(VD);
10115 if (!IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_NoHost)
10118 if (IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_Host)
10126 if (!
CGM.getLangOpts().OpenMPIsTargetDevice) {
10127 if (
const auto *FD = dyn_cast<FunctionDecl>(GD.
getDecl()))
10129 CGM.getLangOpts().OpenMPIsTargetDevice))
10136 if (
const auto *FD = dyn_cast<FunctionDecl>(VD)) {
10137 StringRef Name =
CGM.getMangledName(GD);
10140 CGM.getLangOpts().OpenMPIsTargetDevice))
10145 return !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD) &&
10151 CGM.getLangOpts().OpenMPIsTargetDevice))
10154 if (!
CGM.getLangOpts().OpenMPIsTargetDevice)
10163 StringRef ParentName =
10168 StringRef ParentName =
10175 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
10176 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(
10178 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
10179 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
10180 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
10189 llvm::Constant *
Addr) {
10190 if (
CGM.getLangOpts().OMPTargetTriples.empty() &&
10191 !
CGM.getLangOpts().OpenMPIsTargetDevice)
10194 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
10195 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
10199 if (Res && *Res != OMPDeclareTargetDeclAttr::MT_Link &&
10204 if (
CGM.getLangOpts().OpenMPIsTargetDevice) {
10207 StringRef VarName =
CGM.getMangledName(VD);
10213 auto AddrOfGlobal = [&VD,
this]() {
return CGM.GetAddrOfGlobal(VD); };
10214 auto LinkageForVariable = [&VD,
this]() {
10215 return CGM.getLLVMLinkageVarDefinition(VD);
10218 std::vector<llvm::GlobalVariable *> GeneratedRefs;
10225 CGM.getMangledName(VD), GeneratedRefs,
CGM.getLangOpts().OpenMPSimd,
10226 CGM.getLangOpts().OMPTargetTriples, AddrOfGlobal, LinkageForVariable,
10227 CGM.getTypes().ConvertTypeForMem(
10228 CGM.getContext().getPointerType(VD->
getType())),
10231 for (
auto *ref : GeneratedRefs)
10232 CGM.addCompilerUsedGlobal(ref);
10245 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
10246 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
10249 if ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
10250 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
10252 CGM.EmitGlobal(VD);
10254 assert((*Res == OMPDeclareTargetDeclAttr::MT_Link ||
10255 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
10256 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
10258 "Expected link clause or to clause with unified memory.");
10259 (void)
CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
10267 " Expected target-based directive.");
10272 if (Clause->getClauseKind() == OMPC_unified_shared_memory) {
10274 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(
true);
10275 }
else if (
const auto *AC =
10276 dyn_cast<OMPAtomicDefaultMemOrderClause>(Clause)) {
10277 switch (AC->getAtomicDefaultMemOrderKind()) {
10278 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_acq_rel:
10281 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_seq_cst:
10284 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_relaxed:
10300 if (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())
10302 const auto *A = VD->
getAttr<OMPAllocateDeclAttr>();
10303 switch(A->getAllocatorType()) {
10304 case OMPAllocateDeclAttr::OMPNullMemAlloc:
10305 case OMPAllocateDeclAttr::OMPDefaultMemAlloc:
10307 case OMPAllocateDeclAttr::OMPLargeCapMemAlloc:
10308 case OMPAllocateDeclAttr::OMPCGroupMemAlloc:
10309 case OMPAllocateDeclAttr::OMPHighBWMemAlloc:
10310 case OMPAllocateDeclAttr::OMPLowLatMemAlloc:
10311 case OMPAllocateDeclAttr::OMPThreadMemAlloc:
10312 case OMPAllocateDeclAttr::OMPConstMemAlloc:
10313 case OMPAllocateDeclAttr::OMPPTeamMemAlloc:
10316 case OMPAllocateDeclAttr::OMPUserDefinedMemAlloc:
10317 llvm_unreachable(
"Expected predefined allocator for the variables with the "
10318 "static storage.");
10330 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
10331 SavedShouldMarkAsGlobal = CGM.getOpenMPRuntime().ShouldMarkAsGlobal;
10332 CGM.getOpenMPRuntime().ShouldMarkAsGlobal = false;
10337 if (CGM.getLangOpts().OpenMPIsTargetDevice)
10338 CGM.getOpenMPRuntime().ShouldMarkAsGlobal = SavedShouldMarkAsGlobal;
10348 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(D)) {
10350 if (
auto *F = dyn_cast_or_null<llvm::Function>(
10351 CGM.GetGlobalValue(
CGM.getMangledName(GD))))
10352 return !F->isDeclaration();
10364 llvm::Function *OutlinedFn,
10373 llvm::Value *Args[] = {
10375 CGF.
Builder.getInt32(CapturedVars.size()),
10378 RealArgs.append(std::begin(Args), std::end(Args));
10379 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
10381 llvm::FunctionCallee RTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
10382 CGM.getModule(), OMPRTL___kmpc_fork_teams);
10387 const Expr *NumTeams,
10388 const Expr *ThreadLimit,
10395 llvm::Value *NumTeamsVal =
10401 llvm::Value *ThreadLimitVal =
10408 llvm::Value *PushNumTeamsArgs[] = {RTLoc,
getThreadID(CGF, Loc), NumTeamsVal,
10411 CGM.getModule(), OMPRTL___kmpc_push_num_teams),
10416 const Expr *ThreadLimit,
10419 llvm::Value *ThreadLimitVal =
10426 llvm::Value *ThreadLimitArgs[] = {RTLoc,
getThreadID(CGF, Loc),
10429 CGM.getModule(), OMPRTL___kmpc_set_thread_limit),
10444 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
10446 llvm::Value *IfCondVal =
nullptr;
10451 llvm::Value *DeviceID =
nullptr;
10456 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
10460 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
10461 auto GenMapInfoCB =
10462 [&](InsertPointTy CodeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {
10463 CGF.
Builder.restoreIP(CodeGenIP);
10465 MappableExprsHandler MEHandler(D, CGF);
10466 MEHandler.generateAllInfo(CombinedInfo,
OMPBuilder);
10468 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
10471 if (
CGM.getCodeGenOpts().getDebugInfo() !=
10472 llvm::codegenoptions::NoDebugInfo) {
10473 CombinedInfo.Names.resize(CombinedInfo.Exprs.size());
10474 llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),
10478 return CombinedInfo;
10480 using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;
10481 auto BodyCB = [&](InsertPointTy CodeGenIP, BodyGenTy BodyGenType) {
10482 CGF.
Builder.restoreIP(CodeGenIP);
10483 switch (BodyGenType) {
10484 case BodyGenTy::Priv:
10488 case BodyGenTy::DupNoPriv:
10490 CodeGen.setAction(NoPrivAction);
10494 case BodyGenTy::NoPriv:
10496 CodeGen.setAction(NoPrivAction);
10501 return InsertPointTy(CGF.
Builder.GetInsertBlock(),
10502 CGF.
Builder.GetInsertPoint());
10505 auto DeviceAddrCB = [&](
unsigned int I, llvm::Value *NewDecl) {
10506 if (
const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
10511 auto CustomMapperCB = [&](
unsigned int I) {
10512 llvm::Function *MFunc =
nullptr;
10513 if (CombinedInfo.Mappers[I]) {
10514 Info.HasMapper =
true;
10526 InsertPointTy CodeGenIP(CGF.
Builder.GetInsertBlock(),
10527 CGF.
Builder.GetInsertPoint());
10528 llvm::OpenMPIRBuilder::LocationDescription OmpLoc(CodeGenIP);
10529 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
10531 OmpLoc, AllocaIP, CodeGenIP, DeviceID, IfCondVal, Info, GenMapInfoCB,
10533 nullptr, BodyCB, DeviceAddrCB, RTLoc));
10534 CGF.
Builder.restoreIP(AfterIP);
10546 "Expecting either target enter, exit data, or update directives.");
10549 llvm::Value *MapTypesArray =
nullptr;
10550 llvm::Value *MapNamesArray =
nullptr;
10552 auto &&ThenGen = [
this, &D,
Device, &InputInfo, &MapTypesArray,
10555 llvm::Value *DeviceID =
nullptr;
10560 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
10564 llvm::Constant *PointerNum =
10571 {RTLoc, DeviceID, PointerNum,
10580 RuntimeFunction RTLFn;
10581 switch (D.getDirectiveKind()) {
10582 case OMPD_target_enter_data:
10583 RTLFn = HasNowait ? OMPRTL___tgt_target_data_begin_nowait_mapper
10584 : OMPRTL___tgt_target_data_begin_mapper;
10586 case OMPD_target_exit_data:
10587 RTLFn = HasNowait ? OMPRTL___tgt_target_data_end_nowait_mapper
10588 : OMPRTL___tgt_target_data_end_mapper;
10590 case OMPD_target_update:
10591 RTLFn = HasNowait ? OMPRTL___tgt_target_data_update_nowait_mapper
10592 : OMPRTL___tgt_target_data_update_mapper;
10594 case OMPD_parallel:
10596 case OMPD_parallel_for:
10597 case OMPD_parallel_master:
10598 case OMPD_parallel_sections:
10599 case OMPD_for_simd:
10600 case OMPD_parallel_for_simd:
10602 case OMPD_cancellation_point:
10604 case OMPD_threadprivate:
10605 case OMPD_allocate:
10610 case OMPD_sections:
10614 case OMPD_critical:
10615 case OMPD_taskyield:
10617 case OMPD_taskwait:
10618 case OMPD_taskgroup:
10624 case OMPD_target_data:
10625 case OMPD_distribute:
10626 case OMPD_distribute_simd:
10627 case OMPD_distribute_parallel_for:
10628 case OMPD_distribute_parallel_for_simd:
10629 case OMPD_teams_distribute:
10630 case OMPD_teams_distribute_simd:
10631 case OMPD_teams_distribute_parallel_for:
10632 case OMPD_teams_distribute_parallel_for_simd:
10633 case OMPD_declare_simd:
10634 case OMPD_declare_variant:
10635 case OMPD_begin_declare_variant:
10636 case OMPD_end_declare_variant:
10637 case OMPD_declare_target:
10638 case OMPD_end_declare_target:
10639 case OMPD_declare_reduction:
10640 case OMPD_declare_mapper:
10641 case OMPD_taskloop:
10642 case OMPD_taskloop_simd:
10643 case OMPD_master_taskloop:
10644 case OMPD_master_taskloop_simd:
10645 case OMPD_parallel_master_taskloop:
10646 case OMPD_parallel_master_taskloop_simd:
10648 case OMPD_target_simd:
10649 case OMPD_target_teams_distribute:
10650 case OMPD_target_teams_distribute_simd:
10651 case OMPD_target_teams_distribute_parallel_for:
10652 case OMPD_target_teams_distribute_parallel_for_simd:
10653 case OMPD_target_teams:
10654 case OMPD_target_parallel:
10655 case OMPD_target_parallel_for:
10656 case OMPD_target_parallel_for_simd:
10657 case OMPD_requires:
10658 case OMPD_metadirective:
10661 llvm_unreachable(
"Unexpected standalone target data directive.");
10665 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.
Int32Ty));
10666 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.
VoidPtrTy));
10667 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.
Int32Ty));
10668 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.
VoidPtrTy));
10671 OMPBuilder.getOrCreateRuntimeFunction(
CGM.getModule(), RTLFn),
10675 auto &&TargetThenGen = [
this, &ThenGen, &D, &InputInfo, &MapTypesArray,
10679 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
10681 MappableExprsHandler MEHandler(D, CGF);
10693 CGM.getPointerAlign());
10698 MapTypesArray = Info.RTArgs.MapTypesArray;
10699 MapNamesArray = Info.RTArgs.MapNamesArray;
10700 if (RequiresOuterTask)
10726struct ParamAttrTy {
10727 ParamKindTy Kind =
Vector;
10728 llvm::APSInt StrideOrArg;
10729 llvm::APSInt Alignment;
10730 bool HasVarStride =
false;
10763 unsigned Offset = 0;
10764 if (
const auto *MD = dyn_cast<CXXMethodDecl>(FD)) {
10765 if (ParamAttrs[Offset].Kind ==
Vector)
10766 CDT =
C.getPointerType(
C.getCanonicalTagType(MD->
getParent()));
10770 for (
unsigned I = 0, E = FD->
getNumParams(); I < E; ++I) {
10771 if (ParamAttrs[I + Offset].Kind ==
Vector) {
10783 return C.getTypeSize(CDT);
10791 llvm::raw_svector_ostream Out(Buffer);
10792 for (
const auto &ParamAttr : ParamAttrs) {
10793 switch (ParamAttr.Kind) {
10813 if (ParamAttr.HasVarStride)
10814 Out <<
"s" << ParamAttr.StrideOrArg;
10815 else if (ParamAttr.Kind == Linear || ParamAttr.Kind == LinearRef ||
10816 ParamAttr.Kind == LinearUVal || ParamAttr.Kind == LinearVal) {
10819 if (ParamAttr.StrideOrArg < 0)
10820 Out <<
'n' << -ParamAttr.StrideOrArg;
10821 else if (ParamAttr.StrideOrArg != 1)
10822 Out << ParamAttr.StrideOrArg;
10825 if (!!ParamAttr.Alignment)
10826 Out <<
'a' << ParamAttr.Alignment;
10829 return std::string(Out.str());
10834 const llvm::APSInt &VLENVal,
10836 OMPDeclareSimdDeclAttr::BranchStateTy State) {
10839 unsigned VecRegSize;
10841 ISADataTy ISAData[] = {
10857 case OMPDeclareSimdDeclAttr::BS_Undefined:
10858 Masked.push_back(
'N');
10859 Masked.push_back(
'M');
10861 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
10862 Masked.push_back(
'N');
10864 case OMPDeclareSimdDeclAttr::BS_Inbranch:
10865 Masked.push_back(
'M');
10868 for (
char Mask : Masked) {
10869 for (
const ISADataTy &
Data : ISAData) {
10871 llvm::raw_svector_ostream Out(Buffer);
10872 Out <<
"_ZGV" <<
Data.ISA << Mask;
10875 assert(NumElts &&
"Non-zero simdlen/cdtsize expected");
10876 Out << llvm::APSInt::getUnsigned(
Data.VecRegSize / NumElts);
10881 Out <<
'_' << Fn->getName();
10882 Fn->addFnAttr(Out.str());
10900 if (Kind == ParamKindTy::Uniform)
10903 if (Kind == ParamKindTy::LinearUVal || Kind == ParamKindTy::LinearRef)
10906 if ((Kind == ParamKindTy::Linear || Kind == ParamKindTy::LinearVal) &&
10916 unsigned Size =
C.getTypeSize(QT);
10919 if (Size != 8 && Size != 16 && Size != 32 && Size != 64 && Size != 128)
10943 return C.getTypeSize(PTy);
10946 return C.getTypeSize(QT);
10948 return C.getTypeSize(
C.getUIntPtrType());
10954static std::tuple<unsigned, unsigned, bool>
10960 bool OutputBecomesInput =
false;
10964 Sizes.push_back(
getAArch64LS(RetType, ParamKindTy::Vector,
C));
10966 OutputBecomesInput =
true;
10968 for (
unsigned I = 0, E = FD->
getNumParams(); I < E; ++I) {
10973 assert(!Sizes.empty() &&
"Unable to determine NDS and WDS.");
10976 assert(llvm::all_of(Sizes,
10977 [](
unsigned Size) {
10978 return Size == 8 || Size == 16 || Size == 32 ||
10979 Size == 64 || Size == 128;
10983 return std::make_tuple(*llvm::min_element(Sizes), *llvm::max_element(Sizes),
10984 OutputBecomesInput);
10990template <
typename T>
10992 char ISA, StringRef ParSeq,
10993 StringRef MangledName,
bool OutputBecomesInput,
10994 llvm::Function *Fn) {
10996 llvm::raw_svector_ostream Out(Buffer);
10997 Out << Prefix << ISA << LMask << VLEN;
10998 if (OutputBecomesInput)
11000 Out << ParSeq <<
"_" << MangledName;
11001 Fn->addFnAttr(Out.str());
11007 StringRef Prefix,
char ISA,
11008 StringRef ParSeq, StringRef MangledName,
11009 bool OutputBecomesInput,
11010 llvm::Function *Fn) {
11014 OutputBecomesInput, Fn);
11016 OutputBecomesInput, Fn);
11020 OutputBecomesInput, Fn);
11022 OutputBecomesInput, Fn);
11026 OutputBecomesInput, Fn);
11028 OutputBecomesInput, Fn);
11033 OutputBecomesInput, Fn);
11036 llvm_unreachable(
"Scalar type is too wide.");
11044 OMPDeclareSimdDeclAttr::BranchStateTy State, StringRef MangledName,
11045 char ISA,
unsigned VecRegSize, llvm::Function *Fn,
SourceLocation SLoc) {
11049 const unsigned NDS = std::get<0>(
Data);
11050 const unsigned WDS = std::get<1>(
Data);
11051 const bool OutputBecomesInput = std::get<2>(
Data);
11055 if (UserVLEN == 1) {
11058 "The clause simdlen(1) has no effect when targeting aarch64.");
11065 if (ISA ==
'n' && UserVLEN && !llvm::isPowerOf2_32(UserVLEN)) {
11068 "power of 2 when targeting Advanced SIMD.");
11075 if (ISA ==
's' && UserVLEN != 0) {
11076 if ((UserVLEN * WDS > 2048) || (UserVLEN * WDS % 128 != 0)) {
11079 "lanes in the architectural constraints "
11080 "for SVE (min is 128-bit, max is "
11081 "2048-bit, by steps of 128-bit)");
11089 StringRef Prefix =
"_ZGV";
11095 OutputBecomesInput, Fn);
11097 assert(ISA ==
'n' &&
"Expected ISA either 's' or 'n'.");
11101 case OMPDeclareSimdDeclAttr::BS_Undefined:
11103 OutputBecomesInput, Fn);
11105 OutputBecomesInput, Fn);
11107 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
11109 OutputBecomesInput, Fn);
11111 case OMPDeclareSimdDeclAttr::BS_Inbranch:
11113 OutputBecomesInput, Fn);
11123 OutputBecomesInput, Fn);
11125 assert(ISA ==
'n' &&
"Expected ISA either 's' or 'n'.");
11130 case OMPDeclareSimdDeclAttr::BS_Undefined:
11132 OutputBecomesInput, Fn);
11134 OutputBecomesInput, Fn);
11136 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
11138 OutputBecomesInput, Fn);
11140 case OMPDeclareSimdDeclAttr::BS_Inbranch:
11142 OutputBecomesInput, Fn);
11150 llvm::Function *Fn) {
11155 llvm::DenseMap<const Decl *, unsigned> ParamPositions;
11157 ParamPositions.try_emplace(FD, 0);
11158 unsigned ParamPos = ParamPositions.size();
11160 ParamPositions.try_emplace(P->getCanonicalDecl(), ParamPos);
11166 for (
const Expr *E :
Attr->uniforms()) {
11170 Pos = ParamPositions[FD];
11173 ->getCanonicalDecl();
11174 auto It = ParamPositions.find(PVD);
11175 assert(It != ParamPositions.end() &&
"Function parameter not found");
11178 ParamAttrs[Pos].Kind = Uniform;
11181 auto *NI =
Attr->alignments_begin();
11182 for (
const Expr *E :
Attr->aligneds()) {
11187 Pos = ParamPositions[FD];
11191 ->getCanonicalDecl();
11192 auto It = ParamPositions.find(PVD);
11193 assert(It != ParamPositions.end() &&
"Function parameter not found");
11195 ParmTy = PVD->getType();
11197 ParamAttrs[Pos].Alignment =
11199 ? (*NI)->EvaluateKnownConstInt(
C)
11200 : llvm::APSInt::getUnsigned(
11201 C.toCharUnitsFromBits(
C.getOpenMPDefaultSimdAlign(ParmTy))
11206 auto *SI =
Attr->steps_begin();
11207 auto *MI =
Attr->modifiers_begin();
11208 for (
const Expr *E :
Attr->linears()) {
11211 bool IsReferenceType =
false;
11214 unsigned PtrRescalingFactor = 1;
11216 Pos = ParamPositions[FD];
11218 PtrRescalingFactor =
CGM.getContext()
11219 .getTypeSizeInChars(P->getPointeeType())
11223 ->getCanonicalDecl();
11224 auto It = ParamPositions.find(PVD);
11225 assert(It != ParamPositions.end() &&
"Function parameter not found");
11227 if (
auto *P = dyn_cast<PointerType>(PVD->getType()))
11228 PtrRescalingFactor =
CGM.getContext()
11229 .getTypeSizeInChars(P->getPointeeType())
11231 else if (PVD->getType()->isReferenceType()) {
11232 IsReferenceType =
true;
11233 PtrRescalingFactor =
11235 .getTypeSizeInChars(PVD->getType().getNonReferenceType())
11239 ParamAttrTy &ParamAttr = ParamAttrs[Pos];
11240 if (*MI == OMPC_LINEAR_ref)
11241 ParamAttr.Kind = LinearRef;
11242 else if (*MI == OMPC_LINEAR_uval)
11243 ParamAttr.Kind = LinearUVal;
11244 else if (IsReferenceType)
11245 ParamAttr.Kind = LinearVal;
11247 ParamAttr.Kind = Linear;
11249 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(1);
11253 if (
const auto *DRE =
11255 if (
const auto *StridePVD =
11256 dyn_cast<ParmVarDecl>(DRE->getDecl())) {
11257 ParamAttr.HasVarStride =
true;
11258 auto It = ParamPositions.find(StridePVD->getCanonicalDecl());
11259 assert(It != ParamPositions.end() &&
11260 "Function parameter not found");
11261 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(It->second);
11265 ParamAttr.StrideOrArg =
Result.Val.getInt();
11271 if (!ParamAttr.HasVarStride &&
11272 (ParamAttr.Kind == Linear || ParamAttr.Kind == LinearRef))
11273 ParamAttr.StrideOrArg = ParamAttr.StrideOrArg * PtrRescalingFactor;
11277 llvm::APSInt VLENVal;
11279 const Expr *VLENExpr =
Attr->getSimdlen();
11284 OMPDeclareSimdDeclAttr::BranchStateTy State =
Attr->getBranchState();
11285 if (
CGM.getTriple().isX86()) {
11287 }
else if (
CGM.getTriple().getArch() == llvm::Triple::aarch64) {
11288 unsigned VLEN = VLENVal.getExtValue();
11289 StringRef MangledName = Fn->getName();
11290 if (
CGM.getTarget().hasFeature(
"sve"))
11292 MangledName,
's', 128, Fn, ExprLoc);
11293 else if (
CGM.getTarget().hasFeature(
"neon"))
11295 MangledName,
'n', 128, Fn, ExprLoc);
11304class DoacrossCleanupTy final :
public EHScopeStack::Cleanup {
11306 static const int DoacrossFinArgs = 2;
11309 llvm::FunctionCallee RTLFn;
11310 llvm::Value *Args[DoacrossFinArgs];
11313 DoacrossCleanupTy(llvm::FunctionCallee RTLFn,
11316 assert(CallArgs.size() == DoacrossFinArgs);
11317 std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args));
11334 QualType Int64Ty =
C.getIntTypeForBitwidth(64,
true);
11342 RD =
C.buildImplicitRecord(
"kmp_dim");
11350 RD =
KmpDimTy->castAsRecordDecl();
11352 llvm::APInt Size(32, NumIterations.size());
11358 enum { LowerFD = 0, UpperFD, StrideFD };
11360 for (
unsigned I = 0, E = NumIterations.size(); I < E; ++I) {
11365 DimsLVal, *std::next(RD->
field_begin(), UpperFD));
11367 CGF.
EmitScalarExpr(NumIterations[I]), NumIterations[I]->getType(),
11368 Int64Ty, NumIterations[I]->getExprLoc());
11372 DimsLVal, *std::next(RD->
field_begin(), StrideFD));
11379 llvm::Value *Args[] = {
11382 llvm::ConstantInt::getSigned(
CGM.Int32Ty, NumIterations.size()),
11387 llvm::FunctionCallee RTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
11388 CGM.getModule(), OMPRTL___kmpc_doacross_init);
11390 llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = {
11392 llvm::FunctionCallee FiniRTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
11393 CGM.getModule(), OMPRTL___kmpc_doacross_fini);
11398template <
typename T>
11400 const T *
C, llvm::Value *ULoc,
11401 llvm::Value *ThreadID) {
11404 llvm::APInt Size(32,
C->getNumLoops());
11408 for (
unsigned I = 0, E =
C->getNumLoops(); I < E; ++I) {
11409 const Expr *CounterVal =
C->getLoopData(I);
11410 assert(CounterVal);
11417 llvm::Value *Args[] = {
11420 llvm::FunctionCallee RTLFn;
11422 OMPDoacrossKind<T> ODK;
11423 if (ODK.isSource(
C)) {
11425 OMPRTL___kmpc_doacross_post);
11427 assert(ODK.isSink(
C) &&
"Expect sink modifier.");
11429 OMPRTL___kmpc_doacross_wait);
11449 llvm::FunctionCallee Callee,
11451 assert(Loc.
isValid() &&
"Outlined function call location must be valid.");
11454 if (
auto *Fn = dyn_cast<llvm::Function>(Callee.getCallee())) {
11455 if (Fn->doesNotThrow()) {
11466 emitCall(CGF, Loc, OutlinedFn, Args);
11470 if (
const auto *FD = dyn_cast<FunctionDecl>(D))
11471 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(FD))
11477 const VarDecl *TargetParam)
const {
11484 const Expr *Allocator) {
11485 llvm::Value *AllocVal;
11495 AllocVal = llvm::Constant::getNullValue(
11505 if (!AllocateAlignment)
11508 return llvm::ConstantInt::get(
CGM.
SizeTy, AllocateAlignment->getQuantity());
11521 auto I = UntiedData.find(VD);
11522 if (I != UntiedData.end()) {
11523 UntiedAddr = I->second.first;
11524 UntiedRealAddr = I->second.second;
11528 if (CVD->
hasAttr<OMPAllocateDeclAttr>()) {
11537 Size = CGF.
Builder.CreateNUWAdd(
11539 Size = CGF.
Builder.CreateUDiv(Size,
CGM.getSize(Align));
11540 Size = CGF.
Builder.CreateNUWMul(Size,
CGM.getSize(Align));
11546 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
11547 const Expr *Allocator = AA->getAllocator();
11551 Args.push_back(ThreadID);
11553 Args.push_back(Alignment);
11554 Args.push_back(Size);
11555 Args.push_back(AllocVal);
11556 llvm::omp::RuntimeFunction FnID =
11557 Alignment ? OMPRTL___kmpc_aligned_alloc : OMPRTL___kmpc_alloc;
11559 OMPBuilder.getOrCreateRuntimeFunction(
CGM.getModule(), FnID), Args,
11561 llvm::FunctionCallee FiniRTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
11562 CGM.getModule(), OMPRTL___kmpc_free);
11570 class OMPAllocateCleanupTy final :
public EHScopeStack::Cleanup {
11571 llvm::FunctionCallee RTLFn;
11574 const Expr *AllocExpr;
11577 OMPAllocateCleanupTy(llvm::FunctionCallee RTLFn,
11579 const Expr *AllocExpr)
11580 : RTLFn(RTLFn), LocEncoding(LocEncoding),
Addr(
Addr),
11581 AllocExpr(AllocExpr) {}
11585 llvm::Value *Args[3];
11591 Args[2] = AllocVal;
11599 CGF.
EHStack.pushCleanup<OMPAllocateCleanupTy>(
11601 VDAddr, Allocator);
11602 if (UntiedRealAddr.
isValid())
11605 Region->emitUntiedSwitch(CGF);
11622 assert(CGM.getLangOpts().OpenMP &&
"Not in OpenMP mode.");
11626 CGM.getOpenMPRuntime().NontemporalDeclsStack.emplace_back();
11628 for (
const Stmt *Ref :
C->private_refs()) {
11629 const auto *SimpleRefExpr =
cast<Expr>(Ref)->IgnoreParenImpCasts();
11631 if (
const auto *DRE = dyn_cast<DeclRefExpr>(SimpleRefExpr)) {
11632 VD = DRE->getDecl();
11635 assert((ME->isImplicitCXXThis() ||
11637 "Expected member of current class.");
11638 VD = ME->getMemberDecl();
11648 CGM.getOpenMPRuntime().NontemporalDeclsStack.pop_back();
11654 std::pair<Address, Address>> &LocalVars)
11655 : CGM(CGF.CGM), NeedToPush(!LocalVars.empty()) {
11659 CGF.
CurFn, CGM.getOpenMPRuntime().UntiedLocalVarsStack.size());
11660 CGM.getOpenMPRuntime().UntiedLocalVarsStack.push_back(LocalVars);
11666 CGM.getOpenMPRuntime().UntiedLocalVarsStack.pop_back();
11670 assert(
CGM.getLangOpts().OpenMP &&
"Not in OpenMP mode.");
11672 return llvm::any_of(
11673 CGM.getOpenMPRuntime().NontemporalDeclsStack,
11677void CGOpenMPRuntime::LastprivateConditionalRAII::tryToDisableInnerAnalysis(
11681 llvm::DenseSet<CanonicalDeclPtr<const Decl>> NeedToCheckForLPCs;
11687 const CapturedStmt *CS = S.getCapturedStmt(CaptureRegions.front());
11695 for (
const Expr *Ref :
C->varlist()) {
11696 if (!Ref->getType()->isScalarType())
11698 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
11701 NeedToCheckForLPCs.insert(DRE->getDecl());
11704 for (
const auto *
C : S.getClausesOfKind<OMPFirstprivateClause>()) {
11705 for (
const Expr *Ref :
C->varlist()) {
11706 if (!Ref->getType()->isScalarType())
11708 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
11711 NeedToCheckForLPCs.insert(DRE->getDecl());
11714 for (
const auto *
C : S.getClausesOfKind<OMPLastprivateClause>()) {
11715 for (
const Expr *Ref :
C->varlist()) {
11716 if (!Ref->getType()->isScalarType())
11718 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
11721 NeedToCheckForLPCs.insert(DRE->getDecl());
11724 for (
const auto *
C : S.getClausesOfKind<OMPReductionClause>()) {
11725 for (
const Expr *Ref :
C->varlist()) {
11726 if (!Ref->getType()->isScalarType())
11728 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
11731 NeedToCheckForLPCs.insert(DRE->getDecl());
11734 for (
const auto *
C : S.getClausesOfKind<OMPLinearClause>()) {
11735 for (
const Expr *Ref :
C->varlist()) {
11736 if (!Ref->getType()->isScalarType())
11738 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
11741 NeedToCheckForLPCs.insert(DRE->getDecl());
11744 for (
const Decl *VD : NeedToCheckForLPCs) {
11746 llvm::reverse(
CGM.getOpenMPRuntime().LastprivateConditionalStack)) {
11747 if (
Data.DeclToUniqueName.count(VD) > 0) {
11748 if (!
Data.Disabled)
11749 NeedToAddForLPCsAsDisabled.insert(VD);
11756CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
11759 Action((CGM.getLangOpts().OpenMP >= 50 &&
11762 return C->getKind() ==
11763 OMPC_LASTPRIVATE_conditional;
11765 ? ActionToDo::PushAsLastprivateConditional
11766 : ActionToDo::DoNotPush) {
11767 assert(
CGM.getLangOpts().OpenMP &&
"Not in OpenMP mode.");
11768 if (
CGM.getLangOpts().OpenMP < 50 || Action == ActionToDo::DoNotPush)
11770 assert(Action == ActionToDo::PushAsLastprivateConditional &&
11771 "Expected a push action.");
11773 CGM.getOpenMPRuntime().LastprivateConditionalStack.emplace_back();
11775 if (
C->getKind() != OMPC_LASTPRIVATE_conditional)
11778 for (
const Expr *Ref :
C->varlist()) {
11779 Data.DeclToUniqueName.insert(std::make_pair(
11784 Data.IVLVal = IVLVal;
11788CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
11790 :
CGM(CGF.
CGM), Action(ActionToDo::DoNotPush) {
11794 llvm::DenseSet<CanonicalDeclPtr<const Decl>> NeedToAddForLPCsAsDisabled;
11795 tryToDisableInnerAnalysis(S, NeedToAddForLPCsAsDisabled);
11796 if (!NeedToAddForLPCsAsDisabled.empty()) {
11797 Action = ActionToDo::DisableLastprivateConditional;
11798 LastprivateConditionalData &
Data =
11800 for (
const Decl *VD : NeedToAddForLPCsAsDisabled)
11801 Data.DeclToUniqueName.try_emplace(VD);
11803 Data.Disabled =
true;
11807CGOpenMPRuntime::LastprivateConditionalRAII
11810 return LastprivateConditionalRAII(CGF, S);
11814 if (CGM.getLangOpts().OpenMP < 50)
11816 if (Action == ActionToDo::DisableLastprivateConditional) {
11817 assert(CGM.getOpenMPRuntime().LastprivateConditionalStack.back().Disabled &&
11818 "Expected list of disabled private vars.");
11819 CGM.getOpenMPRuntime().LastprivateConditionalStack.pop_back();
11821 if (Action == ActionToDo::PushAsLastprivateConditional) {
11823 !CGM.getOpenMPRuntime().LastprivateConditionalStack.back().Disabled &&
11824 "Expected list of lastprivate conditional vars.");
11825 CGM.getOpenMPRuntime().LastprivateConditionalStack.pop_back();
11837 auto VI = I->getSecond().find(VD);
11838 if (VI == I->getSecond().end()) {
11839 RecordDecl *RD =
C.buildImplicitRecord(
"lasprivate.conditional");
11844 NewType =
C.getCanonicalTagType(RD);
11847 I->getSecond().try_emplace(VD, NewType, VDField, FiredField, BaseLVal);
11849 NewType = std::get<0>(VI->getSecond());
11850 VDField = std::get<1>(VI->getSecond());
11851 FiredField = std::get<2>(VI->getSecond());
11852 BaseLVal = std::get<3>(VI->getSecond());
11864class LastprivateConditionalRefChecker final
11867 const Expr *FoundE =
nullptr;
11868 const Decl *FoundD =
nullptr;
11869 StringRef UniqueDeclName;
11871 llvm::Function *FoundFn =
nullptr;
11877 llvm::reverse(LPM)) {
11878 auto It = D.DeclToUniqueName.find(E->
getDecl());
11879 if (It == D.DeclToUniqueName.end())
11885 UniqueDeclName = It->second;
11890 return FoundE == E;
11896 llvm::reverse(LPM)) {
11898 if (It == D.DeclToUniqueName.end())
11904 UniqueDeclName = It->second;
11909 return FoundE == E;
11911 bool VisitStmt(
const Stmt *S) {
11912 for (
const Stmt *Child : S->
children()) {
11915 if (
const auto *E = dyn_cast<Expr>(Child))
11923 explicit LastprivateConditionalRefChecker(
11924 ArrayRef<CGOpenMPRuntime::LastprivateConditionalData> LPM)
11926 std::tuple<const Expr *, const Decl *, StringRef, LValue, llvm::Function *>
11927 getFoundData()
const {
11928 return std::make_tuple(FoundE, FoundD, UniqueDeclName, IVLVal, FoundFn);
11935 StringRef UniqueDeclName,
11941 llvm::Constant *LastIV =
OMPBuilder.getOrCreateInternalVariable(
11942 LLIVTy,
getName({UniqueDeclName,
"iv"}));
11950 llvm::GlobalVariable *
Last =
OMPBuilder.getOrCreateInternalVariable(
11966 auto &&
CodeGen = [&LastIVLVal, &IVLVal, IVVal, &LVal, &LastLVal,
11972 llvm::Value *CmpRes;
11974 CmpRes = CGF.
Builder.CreateICmpSLE(LastIVVal, IVVal);
11977 "Loop iteration variable must be integer.");
11978 CmpRes = CGF.
Builder.CreateICmpULE(LastIVVal, IVVal);
11982 CGF.
Builder.CreateCondBr(CmpRes, ThenBB, ExitBB);
12003 "Aggregates are not supported in lastprivate conditional.");
12012 if (
CGM.getLangOpts().OpenMPSimd) {
12026 if (!Checker.Visit(LHS))
12028 const Expr *FoundE;
12029 const Decl *FoundD;
12030 StringRef UniqueDeclName;
12032 llvm::Function *FoundFn;
12033 std::tie(FoundE, FoundD, UniqueDeclName, IVLVal, FoundFn) =
12034 Checker.getFoundData();
12035 if (FoundFn != CGF.
CurFn) {
12040 "Lastprivate conditional is not found in outer region.");
12041 QualType StructTy = std::get<0>(It->getSecond());
12042 const FieldDecl* FiredDecl = std::get<2>(It->getSecond());
12053 FiredLVal, llvm::AtomicOrdering::Unordered,
12071 auto It = llvm::find_if(
12073 if (It == Range.end() || It->Fn != CGF.
CurFn)
12077 "Lastprivates must be registered already.");
12080 const CapturedStmt *CS = D.getCapturedStmt(CaptureRegions.back());
12081 for (
const auto &Pair : It->DeclToUniqueName) {
12082 const auto *VD =
cast<VarDecl>(Pair.first->getCanonicalDecl());
12085 auto I = LPCI->getSecond().find(Pair.first);
12086 assert(I != LPCI->getSecond().end() &&
12087 "Lastprivate must be rehistered already.");
12089 LValue BaseLVal = std::get<3>(I->getSecond());
12093 llvm::Value *Cmp = CGF.
Builder.CreateIsNotNull(Res);
12097 CGF.
Builder.CreateCondBr(Cmp, ThenBB, DoneBB);
12122 "Unknown lastprivate conditional variable.");
12123 StringRef UniqueName = It->second;
12124 llvm::GlobalVariable *GV =
CGM.getModule().getNamedGlobal(UniqueName);
12138 llvm_unreachable(
"Not supported in SIMD-only mode");
12145 llvm_unreachable(
"Not supported in SIMD-only mode");
12152 bool Tied,
unsigned &NumberOfParts) {
12153 llvm_unreachable(
"Not supported in SIMD-only mode");
12161 llvm_unreachable(
"Not supported in SIMD-only mode");
12167 const Expr *Hint) {
12168 llvm_unreachable(
"Not supported in SIMD-only mode");
12174 llvm_unreachable(
"Not supported in SIMD-only mode");
12180 const Expr *Filter) {
12181 llvm_unreachable(
"Not supported in SIMD-only mode");
12186 llvm_unreachable(
"Not supported in SIMD-only mode");
12192 llvm_unreachable(
"Not supported in SIMD-only mode");
12200 llvm_unreachable(
"Not supported in SIMD-only mode");
12207 llvm_unreachable(
"Not supported in SIMD-only mode");
12214 bool ForceSimpleCall) {
12215 llvm_unreachable(
"Not supported in SIMD-only mode");
12222 llvm_unreachable(
"Not supported in SIMD-only mode");
12227 llvm_unreachable(
"Not supported in SIMD-only mode");
12233 llvm_unreachable(
"Not supported in SIMD-only mode");
12239 llvm_unreachable(
"Not supported in SIMD-only mode");
12246 llvm_unreachable(
"Not supported in SIMD-only mode");
12252 llvm_unreachable(
"Not supported in SIMD-only mode");
12257 unsigned IVSize,
bool IVSigned,
12260 llvm_unreachable(
"Not supported in SIMD-only mode");
12266 const Expr *Message) {
12267 llvm_unreachable(
"Not supported in SIMD-only mode");
12271 ProcBindKind ProcBind,
12273 llvm_unreachable(
"Not supported in SIMD-only mode");
12280 llvm_unreachable(
"Not supported in SIMD-only mode");
12286 llvm_unreachable(
"Not supported in SIMD-only mode");
12291 llvm_unreachable(
"Not supported in SIMD-only mode");
12297 llvm::AtomicOrdering AO) {
12298 llvm_unreachable(
"Not supported in SIMD-only mode");
12303 llvm::Function *TaskFunction,
12305 const Expr *IfCond,
12307 llvm_unreachable(
"Not supported in SIMD-only mode");
12314 llvm_unreachable(
"Not supported in SIMD-only mode");
12321 assert(Options.
SimpleReduction &&
"Only simple reduction is expected.");
12323 ReductionOps, Options);
12329 llvm_unreachable(
"Not supported in SIMD-only mode");
12334 bool IsWorksharingReduction) {
12335 llvm_unreachable(
"Not supported in SIMD-only mode");
12342 llvm_unreachable(
"Not supported in SIMD-only mode");
12347 llvm::Value *ReductionsPtr,
12349 llvm_unreachable(
"Not supported in SIMD-only mode");
12355 llvm_unreachable(
"Not supported in SIMD-only mode");
12361 llvm_unreachable(
"Not supported in SIMD-only mode");
12367 llvm_unreachable(
"Not supported in SIMD-only mode");
12372 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
12374 llvm_unreachable(
"Not supported in SIMD-only mode");
12379 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID,
const Expr *IfCond,
12380 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
12384 llvm_unreachable(
"Not supported in SIMD-only mode");
12388 llvm_unreachable(
"Not supported in SIMD-only mode");
12392 llvm_unreachable(
"Not supported in SIMD-only mode");
12402 llvm::Function *OutlinedFn,
12404 llvm_unreachable(
"Not supported in SIMD-only mode");
12408 const Expr *NumTeams,
12409 const Expr *ThreadLimit,
12411 llvm_unreachable(
"Not supported in SIMD-only mode");
12418 llvm_unreachable(
"Not supported in SIMD-only mode");
12424 llvm_unreachable(
"Not supported in SIMD-only mode");
12430 llvm_unreachable(
"Not supported in SIMD-only mode");
12435 llvm_unreachable(
"Not supported in SIMD-only mode");
12440 llvm_unreachable(
"Not supported in SIMD-only mode");
12445 const VarDecl *NativeParam)
const {
12446 llvm_unreachable(
"Not supported in SIMD-only mode");
12452 const VarDecl *TargetParam)
const {
12453 llvm_unreachable(
"Not supported in SIMD-only mode");
static llvm::Value * emitCopyprivateCopyFunction(CodeGenModule &CGM, llvm::Type *ArgsElemType, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps, SourceLocation Loc)
static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF, SourceLocation Loc, SmallString< 128 > &Buffer)
static void emitOffloadingArraysAndArgs(CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo, CGOpenMPRuntime::TargetDataInfo &Info, llvm::OpenMPIRBuilder &OMPBuilder, bool IsNonContiguous=false, bool ForEndCall=false)
Emit the arrays used to pass the captures and map information to the offloading runtime library.
static RecordDecl * createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy, ArrayRef< PrivateDataTy > Privates)
static void emitInitWithReductionInitializer(CodeGenFunction &CGF, const OMPDeclareReductionDecl *DRD, const Expr *InitOp, Address Private, Address Original, QualType Ty)
static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, Address OriginalBaseAddress, llvm::Value *Addr)
static void emitPrivatesInit(CodeGenFunction &CGF, const OMPExecutableDirective &D, Address KmpTaskSharedsPtr, LValue TDBase, const RecordDecl *KmpTaskTWithPrivatesQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef< PrivateDataTy > Privates, bool ForDup)
Emit initialization for private variables in task-based directives.
static void emitClauseForBareTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &Values)
static llvm::Value * emitDestructorsFunction(CodeGenModule &CGM, SourceLocation Loc, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy)
static unsigned evaluateCDTSize(const FunctionDecl *FD, ArrayRef< ParamAttrTy > ParamAttrs)
static void EmitOMPAggregateReduction(CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar, const VarDecl *RHSVar, const llvm::function_ref< void(CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *)> &RedOpGen, const Expr *XExpr=nullptr, const Expr *EExpr=nullptr, const Expr *UpExpr=nullptr)
Emit reduction operation for each element of array (required for array sections) LHS op = RHS.
static void emitTargetCallFallback(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, CodeGenFunction &CGF)
static llvm::Value * emitReduceInitFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Emits reduction initializer function:
static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion)
static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy, llvm::PointerUnion< unsigned *, LValue * > Pos, const OMPTaskDataTy::DependData &Data, Address DependenciesArray)
static llvm::Value * emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc, const OMPTaskDataTy &Data, QualType PrivatesQTy, ArrayRef< PrivateDataTy > Privates)
Emit a privates mapping function for correct handling of private and firstprivate variables.
static llvm::Value * emitReduceCombFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N, const Expr *ReductionOp, const Expr *LHS, const Expr *RHS, const Expr *PrivateRef)
Emits reduction combiner function:
static RecordDecl * createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef< PrivateDataTy > Privates)
static llvm::Value * getAllocatorVal(CodeGenFunction &CGF, const Expr *Allocator)
Return allocator value from expression, or return a null allocator (default when no allocator specifi...
static llvm::Function * emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy, QualType SharedsPtrTy, llvm::Function *TaskFunction, llvm::Value *TaskPrivatesMap)
Emit a proxy function which accepts kmp_task_t as the second argument.
static void addAArch64VectorName(T VLEN, StringRef LMask, StringRef Prefix, char ISA, StringRef ParSeq, StringRef MangledName, bool OutputBecomesInput, llvm::Function *Fn)
static bool isAllocatableDecl(const VarDecl *VD)
static llvm::Value * getAlignmentValue(CodeGenModule &CGM, const VarDecl *VD)
Return the alignment from an allocate directive if present.
static void emitTargetCallKernelLaunch(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::Value *OutlinedFnID, CodeGenFunction::OMPTargetDataInfo &InputInfo, llvm::Value *&MapTypesArray, llvm::Value *&MapNamesArray, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter, CodeGenFunction &CGF, CodeGenModule &CGM)
static llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind convertCaptureClause(const VarDecl *VD)
static std::tuple< unsigned, unsigned, bool > getNDSWDS(const FunctionDecl *FD, ArrayRef< ParamAttrTy > ParamAttrs)
static const OMPExecutableDirective * getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D)
Check for inner distribute directive.
static std::pair< llvm::Value *, llvm::Value * > getPointerAndSize(CodeGenFunction &CGF, const Expr *E)
static const VarDecl * getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE)
static bool isTrivial(ASTContext &Ctx, const Expr *E)
Checks if the expression is constant or does not have non-trivial function calls.
static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind, bool Chunked, bool Ordered)
Map the OpenMP loop schedule to the runtime enumeration.
static void getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS, const Expr **E, int32_t &UpperBound, bool UpperBoundOnly, llvm::Value **CondVal)
Check for a num threads constant value (stored in DefaultVal), or expression (stored in E).
static llvm::Value * emitDeviceID(llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, CodeGenFunction &CGF)
static const OMPDeclareReductionDecl * getReductionInit(const Expr *ReductionOp)
Check if the combiner is a call to UDR combiner and if it is so return the UDR decl used for reductio...
static bool checkInitIsRequired(CodeGenFunction &CGF, ArrayRef< PrivateDataTy > Privates)
Check if duplication function is required for taskloops.
static bool checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD, ArrayRef< PrivateDataTy > Privates)
Checks if destructor function is required to be generated.
static llvm::TargetRegionEntryInfo getEntryInfoFromPresumedLoc(CodeGenModule &CGM, llvm::OpenMPIRBuilder &OMPBuilder, SourceLocation BeginLoc, llvm::StringRef ParentName="")
static void genMapInfo(MappableExprsHandler &MEHandler, CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo, llvm::OpenMPIRBuilder &OMPBuilder, const llvm::DenseSet< CanonicalDeclPtr< const Decl > > &SkippedVarSet=llvm::DenseSet< CanonicalDeclPtr< const Decl > >())
static void emitForStaticInitCall(CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId, llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2, const CGOpenMPRuntime::StaticRTInput &Values)
static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, LValue BaseLV)
static void getKmpAffinityType(ASTContext &C, QualType &KmpTaskAffinityInfoTy)
Builds kmp_depend_info, if it is not built yet, and builds flags type.
static llvm::Constant * emitMappingInformation(CodeGenFunction &CGF, llvm::OpenMPIRBuilder &OMPBuilder, MappableExprsHandler::MappingExprInfo &MapExprs)
Emit a string constant containing the names of the values mapped to the offloading runtime library.
static void getDependTypes(ASTContext &C, QualType &KmpDependInfoTy, QualType &FlagsTy)
Builds kmp_depend_info, if it is not built yet, and builds flags type.
static llvm::Value * emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc, const OMPExecutableDirective &D, QualType KmpTaskTWithPrivatesPtrQTy, const RecordDecl *KmpTaskTWithPrivatesQTyRD, const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef< PrivateDataTy > Privates, bool WithLastIter)
Emit task_dup function (for initialization of private/firstprivate/lastprivate vars and last_iter fla...
static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind convertDeviceClause(const VarDecl *VD)
static llvm::Value * emitReduceFiniFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Emits reduction finalizer function:
static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, QualType Type, bool EmitDeclareReductionInit, const Expr *Init, const OMPDeclareReductionDecl *DRD, Address SrcAddr=Address::invalid())
Emit initialization of arrays of complex types.
static bool getAArch64PBV(QualType QT, ASTContext &C)
Pass By Value (PBV), as defined in 3.1.2 of the AAVFABI.
static unsigned getAArch64LS(QualType QT, ParamKindTy Kind, ASTContext &C)
Computes the lane size (LS) of a return type or of an input parameter, as defined by LS(P) in 3....
static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM, const T *C, llvm::Value *ULoc, llvm::Value *ThreadID)
static RTLDependenceKindTy translateDependencyKind(OpenMPDependClauseKind K)
Translates internal dependency kind into the runtime kind.
static void emitTargetCallElse(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, CodeGenFunction &CGF)
static llvm::Function * emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty, const Expr *CombinerInitializer, const VarDecl *In, const VarDecl *Out, bool IsCombiner)
static void emitX86DeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn, const llvm::APSInt &VLENVal, ArrayRef< ParamAttrTy > ParamAttrs, OMPDeclareSimdDeclAttr::BranchStateTy State)
static void emitReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp)
Emit reduction combiner.
static std::string mangleVectorParameters(ArrayRef< ParamAttrTy > ParamAttrs)
Mangle the parameter part of the vector function name according to their OpenMP classification.
static std::string generateUniqueName(CodeGenModule &CGM, llvm::StringRef Prefix, const Expr *Ref)
static llvm::Function * emitParallelOrTeamsOutlinedFunction(CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen)
static void emitAArch64DeclareSimdFunction(CodeGenModule &CGM, const FunctionDecl *FD, unsigned UserVLEN, ArrayRef< ParamAttrTy > ParamAttrs, OMPDeclareSimdDeclAttr::BranchStateTy State, StringRef MangledName, char ISA, unsigned VecRegSize, llvm::Function *Fn, SourceLocation SLoc)
Emit vector function attributes for AArch64, as defined in the AAVFABI.
static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array, unsigned Index, const VarDecl *Var)
Given an array of pointers to variables, project the address of a given variable.
static llvm::Value * emitDynCGGroupMem(const OMPExecutableDirective &D, CodeGenFunction &CGF)
static bool isAssumedToBeNotEmitted(const ValueDecl *VD, bool IsDevice)
static void addAArch64AdvSIMDNDSNames(unsigned NDS, StringRef Mask, StringRef Prefix, char ISA, StringRef ParSeq, StringRef MangledName, bool OutputBecomesInput, llvm::Function *Fn)
static FieldDecl * addFieldToRecordDecl(ASTContext &C, DeclContext *DC, QualType FieldTy)
static ValueDecl * getDeclFromThisExpr(const Expr *E)
static void genMapInfoForCaptures(MappableExprsHandler &MEHandler, CodeGenFunction &CGF, const CapturedStmt &CS, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, llvm::OpenMPIRBuilder &OMPBuilder, llvm::DenseSet< CanonicalDeclPtr< const Decl > > &MappedVarSet, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo)
static RecordDecl * createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpRoutineEntryPointerQTy)
static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2)
static bool getAArch64MTV(QualType QT, ParamKindTy Kind)
Maps To Vector (MTV), as defined in 4.1.1 of the AAVFABI (2021Q1).
@ LLVM_MARK_AS_BITMASK_ENUM
This file defines OpenMP AST classes for clauses.
Defines some OpenMP-specific enums and functions.
Defines the SourceManager interface.
This file defines OpenMP AST classes for executable directives and clauses.
__DEVICE__ int max(int __a, int __b)
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
SourceManager & getSourceManager()
const ConstantArrayType * getAsConstantArrayType(QualType T) const
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
const ASTRecordLayout & getASTRecordLayout(const RecordDecl *D) const
Get or compute information about the layout of the specified record (struct/union/class) D,...
bool hasSameType(QualType T1, QualType T2) const
Determine whether the given types T1 and T2 are equivalent.
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
const LangOptions & getLangOpts() const
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth,...
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
const VariableArrayType * getAsVariableArrayType(QualType T) const
QualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
unsigned getTypeAlign(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in bits.
CharUnits getSize() const
getSize - Get the record size in characters.
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
CharUnits getNonVirtualSize() const
getNonVirtualSize - Get the non-virtual size (in chars) of an object, which is the size of the object...
static QualType getBaseOriginalType(const Expr *Base)
Return original type of the base expression for array section.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Attr - This represents one attribute.
Represents a C++ constructor within a class.
Represents a C++ destructor within a class.
const CXXRecordDecl * getParent() const
Return the parent of this method declaration, which is the class in which this method is defined.
QualType getFunctionObjectParameterType() const
bool isLambda() const
Determine whether this class describes a lambda function object.
void getCaptureFields(llvm::DenseMap< const ValueDecl *, FieldDecl * > &Captures, FieldDecl *&ThisCapture) const
For a closure type, retrieve the mapping from captured variables and this to the non-static data memb...
unsigned getNumBases() const
Retrieves the number of base classes of this class.
base_class_range vbases()
capture_const_range captures() const
CXXDestructorDecl * getDestructor() const
Returns the destructor decl for this class.
CanProxy< U > castAs() const
A wrapper class around a pointer that always points to its canonical declaration.
Describes the capture of either a variable, or 'this', or variable-length array type.
bool capturesVariableByCopy() const
Determine whether this capture handles a variable by copy.
VarDecl * getCapturedVar() const
Retrieve the declaration of the variable being captured.
bool capturesVariableArrayType() const
Determine whether this capture handles a variable-length array type.
bool capturesThis() const
Determine whether this capture handles the C++ 'this' pointer.
bool capturesVariable() const
Determine whether this capture handles a variable (by reference).
This captures a statement into a function.
const Capture * const_capture_iterator
capture_iterator capture_end() const
Retrieve an iterator pointing past the end of the sequence of captures.
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Stmt * getCapturedStmt()
Retrieve the statement being captured.
bool capturesVariable(const VarDecl *Var) const
True if this variable has been captured.
capture_iterator capture_begin()
Retrieve an iterator pointing to the first capture.
CharUnits - This is an opaque type for sizes expressed in character units.
bool isZero() const
isZero - Test whether the quantity equals zero.
llvm::Align getAsAlign() const
getAsAlign - Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit b...
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
CharUnits alignTo(const CharUnits &Align) const
alignTo - Returns the next integer (mod 2**64) that is greater than or equal to this quantity and is ...
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
CharUnits getAlignment() const
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Address withPointer(llvm::Value *NewPointer, KnownNonNull_t IsKnownNonNull) const
Return address with different pointer, but same element type and alignment.
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
llvm::PointerType * getType() const
Return the type of the pointer value.
static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateDefaultArtificial(CodeGenFunction &CGF, SourceLocation TemporaryLocation)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF)
Set the IRBuilder to not attach debug locations.
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
Address CreateGEP(CodeGenFunction &CGF, Address Addr, llvm::Value *Index, const llvm::Twine &Name="")
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Address CreateConstArrayGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = [n x T]* ... produce name = getelementptr inbounds addr, i64 0, i64 index where i64 is a...
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
Address CreateConstGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = T* ... produce name = getelementptr inbounds addr, i64 index where i64 is actually the t...
CGFunctionInfo - Class to encapsulate the information about a function definition.
DisableAutoDeclareTargetRAII(CodeGenModule &CGM)
~DisableAutoDeclareTargetRAII()
~LastprivateConditionalRAII()
static LastprivateConditionalRAII disable(CodeGenFunction &CGF, const OMPExecutableDirective &S)
NontemporalDeclsRAII(CodeGenModule &CGM, const OMPLoopDirective &S)
Struct that keeps all the relevant information that should be kept throughout a 'target data' region.
llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap
Map between the a declaration of a capture and the corresponding new llvm address where the runtime r...
~UntiedTaskLocalDeclsRAII()
UntiedTaskLocalDeclsRAII(CodeGenFunction &CGF, const llvm::MapVector< CanonicalDeclPtr< const VarDecl >, std::pair< Address, Address > > &LocalVars)
virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc)
Emits address of the word in a memory where current thread id is stored.
llvm::StringSet ThreadPrivateWithDefinition
Set of threadprivate variables with the generated initializer.
CGOpenMPRuntime(CodeGenModule &CGM)
virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
Emit task region for the task directive.
void createOffloadEntriesAndInfoMetadata()
Creates all the offload entries in the current compilation unit along with the associated metadata.
const Expr * getNumTeamsExprForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &MinTeamsVal, int32_t &MaxTeamsVal)
Emit the number of teams for a target directive.
virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc)
Returns address of the threadprivate variable for the current thread.
void emitDeferredTargetDecls() const
Emit deferred declare target variables marked for deferred emission.
virtual llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST)
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
bool markAsGlobalTarget(GlobalDecl GD)
Marks the declaration as already emitted for the device code and returns true, if it was marked alrea...
virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond, llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, const Expr *Message=nullptr)
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
llvm::SmallDenseSet< CanonicalDeclPtr< const Decl > > NontemporalDeclsSet
virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device)
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
QualType SavedKmpTaskloopTQTy
Saved kmp_task_t for taskloop-based directive.
virtual void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps)
Emits a single region.
virtual bool emitTargetGlobal(GlobalDecl GD)
Emit the global GD if it is meaningful for the target.
void setLocThreadIdInsertPt(CodeGenFunction &CGF, bool AtCurrentPoint=false)
std::string getOutlinedHelperName(StringRef Name) const
Get the function name of an outlined region.
bool HasEmittedDeclareTargetRegion
Flag for keeping track of weather a device routine has been emitted.
llvm::Constant * getOrCreateThreadPrivateCache(const VarDecl *VD)
If the specified mangled name is not in the module, create and return threadprivate cache object.
virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal)
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
virtual void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc)
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
void emitCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee Callee, ArrayRef< llvm::Value * > Args={}) const
Emits Callee function call with arguments Args with location Loc.
virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF, const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const
Choose default schedule type and chunk value for the schedule clause.
virtual std::pair< llvm::Function *, llvm::Function * > getUserDefinedReduction(const OMPDeclareReductionDecl *D)
Get combiner/initializer for the specified user-defined reduction, if any.
virtual bool isGPU() const
Returns true if the current target is a GPU.
static const Stmt * getSingleCompoundChild(ASTContext &Ctx, const Stmt *Body)
Checks if the Body is the CompoundStmt and returns its child statement iff there is only one that is ...
virtual void emitDeclareTargetFunction(const FunctionDecl *FD, llvm::GlobalValue *GV)
Emit code for handling declare target functions in the runtime.
bool HasRequiresUnifiedSharedMemory
Flag for keeping track of weather a requires unified_shared_memory directive is present.
llvm::Value * emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc, unsigned Flags=0, bool EmitLoc=false)
Emits object of ident_t type with info for source location.
bool isLocalVarInUntiedTask(CodeGenFunction &CGF, const VarDecl *VD) const
Returns true if the variable is a local variable in untied task.
virtual void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars)
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
virtual void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancellation point' construct.
virtual llvm::Function * emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF=nullptr)
Emit a code for initialization of threadprivate variable.
FunctionUDMMapTy FunctionUDMMap
virtual ConstantAddress getAddrOfDeclareTargetVar(const VarDecl *VD)
Returns the address of the variable marked as declare target with link clause OR as declare target wi...
llvm::Function * getOrCreateUserDefinedMapperFunc(const OMPDeclareMapperDecl *D)
Get the function for the specified user-defined mapper.
OpenMPLocThreadIDMapTy OpenMPLocThreadIDMap
virtual void functionFinished(CodeGenFunction &CGF)
Cleans up references to the objects in finished function.
virtual llvm::Function * emitTeamsOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP teams directive D.
QualType KmpTaskTQTy
Type typedef struct kmp_task { void * shareds; /**< pointer to block of pointers to shared vars / k...
llvm::OpenMPIRBuilder OMPBuilder
An OpenMP-IR-Builder instance.
virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr * > NumIterations)
Emit initialization for doacross loop nesting support.
virtual void adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF, const OMPExecutableDirective &D) const
Adjust some parameters for the target-based directives, like addresses of the variables captured by r...
FunctionUDRMapTy FunctionUDRMap
virtual void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, CGOpenMPRuntime::TargetDataInfo &Info)
Emit the target data mapping code associated with D.
virtual unsigned getDefaultLocationReserved2Flags() const
Returns additional flags that can be stored in reserved_2 field of the default location.
virtual Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const
Gets the address of the native argument basing on the address of the target-specific parameter.
void emitUsesAllocatorsFini(CodeGenFunction &CGF, const Expr *Allocator)
Destroys user defined allocators specified in the uses_allocators clause.
QualType KmpTaskAffinityInfoTy
Type typedef struct kmp_task_affinity_info { kmp_intptr_t base_addr; size_t len; struct { bool flag1 ...
void emitPrivateReduction(CodeGenFunction &CGF, SourceLocation Loc, const Expr *Privates, const Expr *LHSExprs, const Expr *RHSExprs, const Expr *ReductionOps)
Emits code for private variable reduction.
llvm::Value * emitNumTeamsForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D)
virtual void emitTargetOutlinedFunctionHelper(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Helper to emit outlined function for 'target' directive.
void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName)
Start scanning from statement S and emit all target regions found along the way.
SmallVector< llvm::Value *, 4 > emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy, const OMPTaskDataTy::DependData &Data)
virtual void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc)
Emit a taskgroup region.
virtual llvm::Value * emitMessageClause(CodeGenFunction &CGF, const Expr *Message)
llvm::DenseMap< llvm::Function *, llvm::DenseMap< CanonicalDeclPtr< const Decl >, std::tuple< QualType, const FieldDecl *, const FieldDecl *, LValue > > > LastprivateConditionalToTypes
Maps local variables marked as lastprivate conditional to their internal types.
virtual bool emitTargetGlobalVariable(GlobalDecl GD)
Emit the global variable if it is a valid device global variable.
virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32global_tid, kmp_int32 num_teams,...
bool hasRequiresUnifiedSharedMemory() const
Return whether the unified_shared_memory has been specified.
virtual Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, QualType VarType, StringRef Name)
Creates artificial threadprivate variable with name Name and type VarType.
void emitUserDefinedMapper(const OMPDeclareMapperDecl *D, CodeGenFunction *CGF=nullptr)
Emit the function for the user defined mapper construct.
bool HasEmittedTargetRegion
Flag for keeping track of weather a target region has been emitted.
void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy, LValue PosLVal, const OMPTaskDataTy::DependData &Data, Address DependenciesArray)
std::string getReductionFuncName(StringRef Name) const
Get the function name of a reduction function.
virtual void processRequiresDirective(const OMPRequiresDecl *D)
Perform check on requires decl to ensure that target architecture supports unified addressing.
llvm::DenseSet< CanonicalDeclPtr< const Decl > > AlreadyEmittedTargetDecls
List of the emitted declarations.
virtual llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data)
Emit a code for initialization of task reduction clause.
llvm::Value * getThreadID(CodeGenFunction &CGF, SourceLocation Loc)
Gets thread id value for the current thread.
void emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, OpenMPDependClauseKind NewDepKind, SourceLocation Loc)
Updates the dependency kind in the specified depobj object.
virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD, SourceLocation Loc)
Gets the address of the global copy used for lastprivate conditional update, if any.
llvm::MapVector< CanonicalDeclPtr< const VarDecl >, std::pair< Address, Address > > UntiedLocalVarsAddressesMap
virtual void emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, Expr *ME, bool IsFatal)
Emit __kmpc_error call for error directive extern void __kmpc_error(ident_t *loc, int severity,...
void clearLocThreadIdInsertPt(CodeGenFunction &CGF)
virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc)
Emits code for a taskyield directive.
std::string getName(ArrayRef< StringRef > Parts) const
Get the platform-specific name separator.
QualType KmpRoutineEntryPtrQTy
void computeMinAndMaxThreadsAndTeams(const OMPExecutableDirective &D, CodeGenFunction &CGF, llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs)
Helper to determine the min/max number of threads/teams for D.
virtual void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc, llvm::AtomicOrdering AO)
Emit flush of the variables specified in 'omp flush' directive.
virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPTaskDataTy &Data)
Emit code for 'taskwait' directive.
virtual void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc)
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32global_tid, int proc_bind) to generate...
void emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LValue IVLVal, StringRef UniqueDeclName, LValue LVal, SourceLocation Loc)
Emit update for lastprivate conditional data.
virtual void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
Emit task region for the taskloop directive.
virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false)
Emit an implicit/explicit barrier for OpenMP threads.
static unsigned getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind)
Returns default flags for the barriers depending on the directive, for which this barier is going to ...
virtual bool emitTargetFunctions(GlobalDecl GD)
Emit the target regions enclosed in GD function definition or the function itself in case it is a val...
TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const OMPTaskDataTy &Data)
Emit task region for the task directive.
llvm::Value * emitTargetNumIterationsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Return the trip count of loops associated with constructs / 'target teams distribute' and 'teams dist...
llvm::StringMap< llvm::AssertingVH< llvm::GlobalVariable >, llvm::BumpPtrAllocator > InternalVars
An ordered map of auto-generated variables to their unique names.
virtual void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values)
llvm::SmallVector< UntiedLocalVarsAddressesMap, 4 > UntiedLocalVarsStack
virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind)
Call the appropriate runtime routine to notify that we finished all the work with current loop.
virtual void emitThreadLimitClause(CodeGenFunction &CGF, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_set_thread_limit(ident_t *loc, kmp_int32global_tid, kmp_int32 thread_limit)...
void emitIfClause(CodeGenFunction &CGF, const Expr *Cond, const RegionCodeGenTy &ThenGen, const RegionCodeGenTy &ElseGen)
Emits code for OpenMP 'if' clause using specified CodeGen function.
Address emitDepobjDependClause(CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies, SourceLocation Loc)
Emits list of dependecies based on the provided data (array of dependence/expression pairs) for depob...
virtual llvm::Value * emitSeverityClause(OpenMPSeverityClauseKind Severity)
bool isNontemporalDecl(const ValueDecl *VD) const
Checks if the VD variable is marked as nontemporal declaration in current context.
virtual llvm::Function * emitParallelOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP parallel directive D.
virtual void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc, OpenMPNumThreadsClauseModifier Modifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, const Expr *Message=nullptr)
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32global_tid, kmp_int32 num_threads) ...
const Expr * getNumThreadsExprForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &UpperBound, bool UpperBoundOnly, llvm::Value **CondExpr=nullptr, const Expr **ThreadLimitExpr=nullptr)
Check for a number of threads upper bound constant value (stored in UpperBound), or expression (retur...
llvm::SmallVector< LastprivateConditionalData, 4 > LastprivateConditionalStack
Stack for list of addresses of declarations in current context marked as lastprivate conditional.
virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values)
Call the appropriate runtime routine to initialize it before start of loop.
virtual void emitDeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn)
Marks function Fn with properly mangled versions of vector functions.
llvm::AtomicOrdering getDefaultMemoryOrdering() const
Gets default memory ordering as specified in requires directive.
virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static non-chunked.
llvm::Value * getCriticalRegionLock(StringRef CriticalName)
Returns corresponding lock object for the specified critical region name.
virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancel' construct.
QualType SavedKmpTaskTQTy
Saved kmp_task_t for task directive.
virtual void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc)
Emits a master region.
virtual llvm::Function * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts)
Emits outlined function for the OpenMP task directive D.
llvm::DenseMap< llvm::Function *, unsigned > FunctionToUntiedTaskStackMap
Maps function to the position of the untied task locals stack.
void emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal, SourceLocation Loc)
Emits the code to destroy the dependency object provided in depobj directive.
virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Required to resolve existing problems in the runtime.
llvm::ArrayType * KmpCriticalNameTy
Type kmp_critical_name, originally defined as typedef kmp_int32 kmp_critical_name[8];.
virtual void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C)
Emit code for doacross ordered directive with 'depend' clause.
llvm::DenseMap< const OMPDeclareMapperDecl *, llvm::Function * > UDMMap
Map from the user-defined mapper declaration to its corresponding functions.
virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF, const Expr *LHS)
Checks if the provided LVal is lastprivate conditional and emits the code to update the value of the ...
std::pair< llvm::Value *, LValue > getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal, SourceLocation Loc)
Returns the number of the elements and the address of the depobj dependency array.
llvm::SmallDenseSet< const VarDecl * > DeferredGlobalVariables
List of variables that can become declare target implicitly and, thus, must be emitted.
void emitUsesAllocatorsInit(CodeGenFunction &CGF, const Expr *Allocator, const Expr *AllocatorTraits)
Initializes user defined allocators specified in the uses_allocators clauses.
llvm::Type * KmpRoutineEntryPtrTy
Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *);.
llvm::Type * getIdentTyPointerTy()
Returns pointer to ident_t type.
void emitSingleReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp, const Expr *PrivateRef, const DeclRefExpr *LHS, const DeclRefExpr *RHS)
Emits single reduction combiner.
llvm::OpenMPIRBuilder & getOMPBuilder()
virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Emit outilined function for 'target' directive.
virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr)
Emits a critical region.
virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned)
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
virtual void emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, ArrayRef< llvm::Value * > Args={}) const
Emits call of the outlined function with the provided arguments, translating these arguments to corre...
llvm::Value * emitNumThreadsForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D)
Emit an expression that denotes the number of threads a target region shall use.
void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor, llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc)
Emits initialization code for the threadprivate variables.
virtual void emitUserDefinedReduction(CodeGenFunction *CGF, const OMPDeclareReductionDecl *D)
Emit code for the specified user defined reduction construct.
virtual void checkAndEmitSharedLastprivateConditional(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::DenseSet< CanonicalDeclPtr< const VarDecl > > &IgnoredDecls)
Checks if the lastprivate conditional was updated in inner region and writes the value.
QualType KmpDimTy
struct kmp_dim { // loop bounds info casted to kmp_int64 kmp_int64 lo; // lower kmp_int64 up; // uppe...
virtual void emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool HasCancel=false)
Emit code for the directive that does not require outlining.
virtual void registerTargetGlobalVariable(const VarDecl *VD, llvm::Constant *Addr)
Checks if the provided global decl GD is a declare target variable and registers it when emitting cod...
virtual void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D)
Emits OpenMP-specific function prolog.
void emitKmpRoutineEntryT(QualType KmpInt32Ty)
Build type kmp_routine_entry_t (if not built yet).
virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static chunked.
virtual void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Emit the target offloading code associated with D.
virtual bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS)
Checks if the variable has associated OMPAllocateDeclAttr attribute with the predefined allocator and...
llvm::AtomicOrdering RequiresAtomicOrdering
Atomic ordering from the omp requires directive.
virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps, ReductionOptionsTy Options)
Emit a code for reduction clause.
std::pair< llvm::Value *, Address > emitDependClause(CodeGenFunction &CGF, ArrayRef< OMPTaskDataTy::DependData > Dependencies, SourceLocation Loc)
Emits list of dependecies based on the provided data (array of dependence/expression pairs).
llvm::StringMap< llvm::WeakTrackingVH > EmittedNonTargetVariables
List of the global variables with their addresses that should not be emitted for the target.
virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const
Check if the specified ScheduleKind is dynamic.
Address emitLastprivateConditionalInit(CodeGenFunction &CGF, const VarDecl *VD)
Create specialized alloca to handle lastprivate conditionals.
virtual void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads)
Emit an ordered region.
virtual Address getAddressOfLocalVariable(CodeGenFunction &CGF, const VarDecl *VD)
Gets the OpenMP-specific address of the local variable.
virtual void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, bool IsWorksharingReduction)
Emits the following code for reduction clause with task modifier:
virtual void emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter=nullptr)
Emits a masked region.
QualType KmpDependInfoTy
Type typedef struct kmp_depend_info { kmp_intptr_t base_addr; size_t len; struct { bool in:1; bool ou...
llvm::Function * emitReductionFunction(StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps)
Emits reduction function.
virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues)
Call the appropriate runtime routine to initialize it before start of loop.
Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal) override
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr) override
Emits a critical region.
void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values) override
void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) override
Call the appropriate runtime routine to initialize it before start of loop.
bool emitTargetGlobalVariable(GlobalDecl GD) override
Emit the global variable if it is a valid device global variable.
llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST) override
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc, OpenMPNumThreadsClauseModifier Modifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, const Expr *Message=nullptr) override
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32global_tid, kmp_int32 num_threads) ...
llvm::Function * emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF=nullptr) override
Emit a code for initialization of threadprivate variable.
void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device) override
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
llvm::Function * emitTeamsOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) override
Emits outlined function for the specified OpenMP teams directive D.
void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond, llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, const Expr *Message=nullptr) override
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps, ReductionOptionsTy Options) override
Emit a code for reduction clause.
void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc, llvm::AtomicOrdering AO) override
Emit flush of the variables specified in 'omp flush' directive.
void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C) override
Emit code for doacross ordered directive with 'depend' clause.
void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc) override
Emits a masked region.
Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, QualType VarType, StringRef Name) override
Creates artificial threadprivate variable with name Name and type VarType.
Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc) override
Returns address of the threadprivate variable for the current thread.
void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps) override
Emits a single region.
void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N) override
Required to resolve existing problems in the runtime.
llvm::Function * emitParallelOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) override
Emits outlined function for the specified OpenMP parallel directive D.
void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion) override
Emit code for 'cancellation point' construct.
void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false) override
Emit an implicit/explicit barrier for OpenMP threads.
Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const override
Gets the address of the native argument basing on the address of the target-specific parameter.
void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars) override
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned) override
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
bool emitTargetGlobal(GlobalDecl GD) override
Emit the global GD if it is meaningful for the target.
void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, bool IsWorksharingReduction) override
Emits the following code for reduction clause with task modifier:
void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads) override
Emit an ordered region.
void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind) override
Call the appropriate runtime routine to notify that we finished all the work with current loop.
llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data) override
Emit a code for initialization of task reduction clause.
void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc) override
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32global_tid, int proc_bind) to generate...
void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) override
Emit outilined function for 'target' directive.
void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc) override
Emits a master region.
void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc) override
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32global_tid, kmp_int32 num_teams,...
void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc) override
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
const VarDecl * translateParameter(const FieldDecl *FD, const VarDecl *NativeParam) const override
Translates the native parameter of outlined function if this is required for target.
void emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter=nullptr) override
Emits a masked region.
void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data) override
Emit task region for the task directive.
void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter) override
Emit the target offloading code associated with D.
bool emitTargetFunctions(GlobalDecl GD) override
Emit the target regions enclosed in GD function definition or the function itself in case it is a val...
void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr * > NumIterations) override
Emit initialization for doacross loop nesting support.
void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion) override
Emit code for 'cancel' construct.
void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPTaskDataTy &Data) override
Emit code for 'taskwait' directive.
void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc) override
Emit a taskgroup region.
void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, CGOpenMPRuntime::TargetDataInfo &Info) override
Emit the target data mapping code associated with D.
void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues) override
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
llvm::Function * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts) override
Emits outlined function for the OpenMP task directive D.
void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data) override
Emit task region for the taskloop directive.
unsigned getNonVirtualBaseLLVMFieldNo(const CXXRecordDecl *RD) const
llvm::StructType * getLLVMType() const
Return the "complete object" LLVM type associated with this record.
llvm::StructType * getBaseSubobjectLLVMType() const
Return the "base subobject" LLVM type associated with this record.
unsigned getLLVMFieldNo(const FieldDecl *FD) const
Return llvm::StructType element number that corresponds to the field FD.
unsigned getVirtualBaseIndex(const CXXRecordDecl *base) const
Return the LLVM field index corresponding to the given virtual base.
API for captured statement code generation.
virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S)
Emit the captured statement body.
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
RAII for correct setting/restoring of CapturedStmtInfo.
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
bool Privatize()
Privatizes local variables previously registered as private.
bool addPrivate(const VarDecl *LocalVD, Address Addr)
Registers LocalVD variable as a private with Addr as the address of the corresponding private variabl...
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited.
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
LValue EmitLoadOfReferenceLValue(LValue RefLVal)
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount, Stmt::Likelihood LH=Stmt::LH_None, const Expr *ConditionalOp=nullptr, const VarDecl *ConditionalDecl=nullptr)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
void emitDestroy(Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray)
emitDestroy - Immediately perform the destruction of the given object.
JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target)
The given basic block lies in the current EH scope, but may be a target of a potentially scope-crossi...
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
void EmitNullInitialization(Address DestPtr, QualType Ty)
EmitNullInitialization - Generate code to set a value of the given type to null, If the type contains...
Address LoadCXXThisAddress()
CGCapturedStmtInfo * CapturedStmtInfo
ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc)
EmitLoadOfComplex - Load a complex number from the specified l-value.
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
const LangOptions & getLangOpts() const
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
EmitAutoVarAlloca - Emit the alloca and debug information for a local variable.
void pushDestroy(QualType::DestructionKind dtorKind, Address addr, QualType type)
pushDestroy - Push the standard destructor for the given type as at least a normal cleanup.
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Load a pointer with type PtrTy stored at address Ptr.
void EmitBranchThroughCleanup(JumpDest Dest)
EmitBranchThroughCleanup - Emit a branch from the current insert block through the normal cleanup han...
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
Destroyer * getDestroyer(QualType::DestructionKind destructionKind)
llvm::AssertingVH< llvm::Instruction > AllocaInsertPt
AllocaInsertPoint - This is an instruction in the entry block before which we prefer to insert alloca...
void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy)
Emit an aggregate assignment.
JumpDest ReturnBlock
ReturnBlock - Unified return block.
void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy, AggValueSlot::Overlap_t MayOverlap, bool isVolatile=false)
EmitAggregateCopy - Emit an aggregate copy.
LValue EmitLValueForField(LValue Base, const FieldDecl *Field, bool IsInBounds=true)
RawAddress CreateDefaultAlignTempAlloca(llvm::Type *Ty, const Twine &Name="tmp")
CreateDefaultAlignedTempAlloca - This creates an alloca with the default ABI alignment of the given L...
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value * > &CapturedVars)
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
LValue EmitArraySectionExpr(const ArraySectionExpr *E, bool IsLowerBound=true)
LValue EmitOMPSharedLValue(const Expr *E)
Emits the lvalue for the expression with possibly captured variable.
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
llvm::Value * emitArrayLength(const ArrayType *arrayType, QualType &baseType, Address &addr)
emitArrayLength - Compute the length of an array, even if it's a VLA, and drill down to the base elem...
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
CGDebugInfo * getDebugInfo()
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
LValue MakeRawAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment, AlignmentSource Source=AlignmentSource::Type)
Same as MakeAddrLValue above except that the pointer is known to be unsigned.
LValue EmitLValueForFieldInitialization(LValue Base, const FieldDecl *Field)
EmitLValueForFieldInitialization - Like EmitLValueForField, except that if the Field is a reference,...
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler's counter for the given statement by StepV.
VlaSizePair getVLASize(const VariableArrayType *vla)
Returns an LLVM value that corresponds to the size, in non-variably-sized elements,...
llvm::CallInst * EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
ASTContext & getContext() const
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
void EmitAutoVarCleanups(const AutoVarEmission &emission)
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy)
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
bool needsEHCleanup(QualType::DestructionKind kind)
Determines whether an EH cleanup is required to destroy a type with the given destruction kind.
llvm::DenseMap< const ValueDecl *, FieldDecl * > LambdaCaptureFields
llvm::CallInst * EmitRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
llvm::Type * ConvertTypeForMem(QualType T)
static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S)
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
CodeGenTypes & getTypes() const
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Emit device code for the target teams distribute parallel for simd directive.
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block,...
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S, const OMPExecutableDirective &D)
RawAddress CreateMemTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
llvm::Value * EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr, ArrayRef< llvm::Value * > IdxList, bool SignedIndices, bool IsSubtraction, SourceLocation Loc, const Twine &Name="")
Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to detect undefined behavior whe...
static void EmitOMPTargetParallelGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelGenericLoopDirective &S)
Emit device code for the target parallel loop directive.
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
static bool IsWrappedCXXThis(const Expr *E)
Check if E is a C++ "this" pointer wrapped in value-preserving casts.
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S)
Emit device code for the target simd directive.
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant,...
static void EmitOMPTargetTeamsGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsGenericLoopDirective &S)
Emit device code for the target teams loop directive.
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
void EmitVariablyModifiedType(QualType Ty)
EmitVLASize - Capture all the sizes for the VLA expressions in the given variably-modified type and s...
bool isTrivialInitializer(const Expr *Init)
Determine whether the given initializer is trivial in the sense that it requires no code to be genera...
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit)
EmitExprAsInit - Emits the code necessary to initialize a location in memory with the given initializ...
LValue MakeNaturalAlignRawAddrLValue(llvm::Value *V, QualType T)
This class organizes the cross-function state that is used while generating LLVM code.
void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
llvm::Module & getModule() const
DiagnosticsEngine & getDiags() const
const LangOptions & getLangOpts() const
CharUnits getNaturalTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, bool forPointeeType=false)
CodeGenTypes & getTypes()
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
TBAAAccessInfo getTBAAInfoForSubobject(LValue Base, QualType AccessType)
getTBAAInfoForSubobject - Get TBAA information for an access with a given base lvalue.
ASTContext & getContext() const
const CodeGenOptions & getCodeGenOpts() const
StringRef getMangledName(GlobalDecl GD)
std::optional< CharUnits > getOMPAllocateAlignment(const VarDecl *VD)
Return the alignment specified in an allocate directive, if present.
llvm::Constant * EmitNullConstant(QualType T)
Return the result of value-initializing the given type, i.e.
llvm::Type * ConvertType(QualType T)
ConvertType - Convert type T into a llvm::Type.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
const CGRecordLayout & getCGRecordLayout(const RecordDecl *)
getCGRecordLayout - Return record layout info for the given record decl.
A specialization of Address that requires the address to be an LLVM Constant.
static ConstantAddress invalid()
bool requiresLandingPad() const
void pushTerminate()
Push a terminate handler on the stack.
void popTerminate()
Pops a terminate handler off the stack.
FunctionArgList - Type for representing both the decl and type of parameters to a function.
LValue - This represents an lvalue references.
CharUnits getAlignment() const
llvm::Value * getPointer(CodeGenFunction &CGF) const
const Qualifiers & getQuals() const
Address getAddress() const
LValueBaseInfo getBaseInfo() const
TBAAAccessInfo getTBAAInfo() const
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
virtual void Enter(CodeGenFunction &CGF)
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
static RValue get(llvm::Value *V)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
An abstract representation of an aligned address.
llvm::Type * getElementType() const
Return the type of the values stored in this address.
llvm::Value * getPointer() const
static RawAddress invalid()
Class intended to support codegen of all kind of the reduction clauses.
LValue getSharedLValue(unsigned N) const
Returns LValue for the reduction item.
const Expr * getRefExpr(unsigned N) const
Returns the base declaration of the reduction item.
LValue getOrigLValue(unsigned N) const
Returns LValue for the original reduction item.
bool needCleanups(unsigned N)
Returns true if the private copy requires cleanups.
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
const VarDecl * getBaseDecl(unsigned N) const
Returns the base declaration of the reduction item.
QualType getPrivateType(unsigned N) const
Return the type of the private item.
bool usesReductionInitializer(unsigned N) const
Returns true if the initialization of the reduction item uses initializer from declare reduction cons...
void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N)
Emits lvalue for the shared and original reduction item.
void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, llvm::function_ref< bool(CodeGenFunction &)> DefaultInit)
Performs initialization of the private copy for the reduction item.
std::pair< llvm::Value *, llvm::Value * > getSizes(unsigned N) const
Returns the size of the reduction item (in chars and total number of elements in the item),...
ReductionCodeGen(ArrayRef< const Expr * > Shareds, ArrayRef< const Expr * > Origs, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > ReductionOps)
void emitCleanups(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Emits cleanup code for the reduction item.
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations.
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
void operator()(CodeGenFunction &CGF) const
void setAction(PrePostActionTy &Action) const
ConstStmtVisitor - This class implements a simple visitor for Stmt subclasses.
DeclContext - This is used only as base class of specific decl types that can act as declaration cont...
void addDecl(Decl *D)
Add the declaration D into this context.
A reference to a declared variable, function, enum, etc.
Decl - This represents one declaration (or definition), e.g.
ASTContext & getASTContext() const LLVM_READONLY
virtual Stmt * getBody() const
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
llvm::iterator_range< specific_attr_iterator< T > > specific_attrs() const
SourceLocation getLocation() const
DeclContext * getDeclContext()
virtual Decl * getCanonicalDecl()
Retrieves the "canonical" declaration of the given declaration.
SourceLocation getBeginLoc() const LLVM_READONLY
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
unsigned getCustomDiagID(Level L, const char(&FormatString)[N])
Return an ID for a diagnostic with the specified format string and level.
This represents one expression.
bool isIntegerConstantExpr(const ASTContext &Ctx) const
@ SE_AllowSideEffects
Allow any unmodeled side effect.
@ SE_AllowUndefinedBehavior
Allow UB that we can give a value, but not arbitrary unmodeled side effects.
Expr * IgnoreParenCasts() LLVM_READONLY
Skip past any parentheses and casts which might surround this expression until reaching a fixed point...
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
bool isEvaluatable(const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects) const
isEvaluatable - Call EvaluateAsRValue to see if this expression can be constant folded without side-e...
std::optional< llvm::APSInt > getIntegerConstantExpr(const ASTContext &Ctx) const
isIntegerConstantExpr - Return the value if this expression is a valid integer constant expression.
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
bool EvaluateAsBooleanCondition(bool &Result, const ASTContext &Ctx, bool InConstantContext=false) const
EvaluateAsBooleanCondition - Return true if this is a constant which we can fold and convert to a boo...
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
bool hasNonTrivialCall(const ASTContext &Ctx) const
Determine whether this expression involves a call to any function that is not trivial.
Represents a member of a struct/union/class.
unsigned getFieldIndex() const
Returns the index of this field within its record, as appropriate for passing to ASTRecordLayout::get...
const RecordDecl * getParent() const
Returns the parent of this field declaration, which is the struct in which this field is defined.
static FieldDecl * Create(const ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, const IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, Expr *BW, bool Mutable, InClassInitStyle InitStyle)
Represents a function declaration or definition.
const ParmVarDecl * getParamDecl(unsigned i) const
QualType getReturnType() const
ArrayRef< ParmVarDecl * > parameters() const
FunctionDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
FunctionDecl * getMostRecentDecl()
Returns the most recent (re)declaration of this declaration.
unsigned getNumParams() const
Return the number of parameters this function must have based on its FunctionType.
FunctionDecl * getPreviousDecl()
Return the previous declaration of this declaration or NULL if this is the first declaration.
GlobalDecl - represents a global declaration.
const Decl * getDecl() const
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
static IntegerLiteral * Create(const ASTContext &C, const llvm::APInt &V, QualType type, SourceLocation l)
Returns a new integer literal with value 'V' and type 'type'.
MemberExpr - [C99 6.5.2.3] Structure and Union Members.
ValueDecl * getMemberDecl() const
Retrieve the member declaration to which this expression refers.
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
bool isExternallyVisible() const
This represents clause 'affinity' in the 'pragma omp task'-based directives.
Expr * getAssociatedExpression() const
ValueDecl * getAssociatedDeclaration() const
ArrayRef< MappableComponent > MappableExprComponentListRef
const Stmt * getPreInitStmt() const
Get pre-initialization statement for the clause.
This is a basic class for representing single OpenMP clause.
This represents 'pragma omp declare mapper ...' directive.
Expr * getMapperVarRef()
Get the variable declared in the mapper.
This represents 'pragma omp declare reduction ...' directive.
Expr * getInitializer()
Get initializer expression (if specified) of the declare reduction construct.
Expr * getInitPriv()
Get Priv variable of the initializer.
Expr * getCombinerOut()
Get Out variable of the combiner.
Expr * getCombinerIn()
Get In variable of the combiner.
Expr * getCombiner()
Get combiner expression of the declare reduction construct.
Expr * getInitOrig()
Get Orig variable of the initializer.
OMPDeclareReductionInitKind getInitializerKind() const
Get initializer kind.
This represents implicit clause 'depend' for the 'pragma omp task' directive.
This represents 'detach' clause in the 'pragma omp task' directive.
This represents 'device' clause in the 'pragma omp ...' directive.
This represents the 'doacross' clause for the 'pragma omp ordered' directive.
This represents 'if' clause in the 'pragma omp ...' directive.
Expr * getCondition() const
Returns condition.
This represents clause 'in_reduction' in the 'pragma omp task' directives.
OMPIteratorHelperData & getHelper(unsigned I)
Fetches helper data for the specified iteration space.
unsigned numOfIterators() const
Returns number of iterator definitions.
This represents clause 'lastprivate' in the 'pragma omp ...' directives.
This represents the 'message' clause in the 'pragma omp error' and the 'pragma omp parallel' directiv...
Expr * getMessageString() const
Returns message string of the clause.
This represents clause 'nontemporal' in the 'pragma omp ...' directives.
This represents 'nowait' clause in the 'pragma omp ...' directive.
This represents 'num_teams' clause in the 'pragma omp ...' directive.
This represents 'num_threads' clause in the 'pragma omp ...' directive.
This represents 'ordered' clause in the 'pragma omp ...' directive.
This represents clause 'private' in the 'pragma omp ...' directives.
This represents 'pragma omp requires...' directive.
clauselist_range clauselists()
This represents the 'severity' clause in the 'pragma omp error' and the 'pragma omp parallel' directi...
OpenMPSeverityClauseKind getSeverityKind() const
Returns kind of the clause.
This represents 'thread_limit' clause in the 'pragma omp ...' directive.
This represents clause 'uses_allocators' in the 'pragma omp target'-based directives.
This represents 'ompx_attribute' clause in a directive that might generate an outlined function.
This represents 'ompx_bare' clause in the 'pragma omp target teams ...' directive.
This represents 'ompx_dyn_cgroup_mem' clause in the 'pragma omp target ...' directive.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class.
Represents a parameter to a function.
PointerType - C99 6.7.5.1 - Pointer Declarators.
Represents an unpacked "presumed" location which can be presented to the user.
unsigned getColumn() const
Return the presumed column number of this location.
const char * getFilename() const
Return the presumed filename of this location.
unsigned getLine() const
Return the presumed line number of this location.
A (possibly-)qualified type.
void addRestrict()
Add the restrict qualifier to this QualType.
QualType withRestrict() const
bool isNull() const
Return true if this QualType doesn't point to a type yet.
const Type * getTypePtr() const
Retrieves a pointer to the underlying (unqualified) type.
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
QualType getCanonicalType() const
DestructionKind isDestructedType() const
Returns a nonzero value if objects of this type require non-trivial work to clean up after.
Represents a struct/union/class.
field_iterator field_end() const
field_range fields() const
virtual void completeDefinition()
Note that the definition of this type is now complete.
field_iterator field_begin() const
Scope - A scope is a transient data structure that is used while parsing the program.
Encodes a location in the source.
static SourceLocation getFromRawEncoding(UIntTy Encoding)
Turn a raw encoding of a SourceLocation object into a real SourceLocation.
bool isValid() const
Return true if this is a valid SourceLocation object.
UIntTy getRawEncoding() const
When a SourceLocation itself cannot be used, this returns an (opaque) 32-bit integer encoding for it.
This class handles loading and caching of source files into memory.
PresumedLoc getPresumedLoc(SourceLocation Loc, bool UseLineDirectives=true) const
Returns the "presumed" location of a SourceLocation specifies.
Stmt - This represents one statement.
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top,...
SourceLocation getBeginLoc() const LLVM_READONLY
void startDefinition()
Starts the definition of this tag declaration.
The base class of the type hierarchy.
const Type * getPointeeOrArrayElementType() const
If this is a pointer type, return the pointee type.
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
RecordDecl * getAsRecordDecl() const
Retrieves the RecordDecl this type refers to.
bool isPointerType() const
CanQualType getCanonicalTypeUnqualified() const
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
const T * castAs() const
Member-template castAs<specific type>.
bool isReferenceType() const
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
bool isLValueReferenceType() const
bool isAggregateType() const
Determines whether the type is a C++ aggregate type or C aggregate or union type.
RecordDecl * castAsRecordDecl() const
QualType getCanonicalTypeInternal() const
const Type * getBaseElementTypeUnsafe() const
Get the base element type of this type, potentially discarding type qualifiers.
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type.
bool isFloatingType() const
bool isUnsignedIntegerType() const
Return true if this is an integer type that is unsigned, according to C99 6.2.5p6 [which returns true...
bool isAnyPointerType() const
const T * getAs() const
Member-template getAs<specific type>'.
bool isRecordType() const
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Represents a variable declaration or definition.
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
VarDecl * getDefinition(ASTContext &)
Get the real (not just tentative) definition for this declaration.
const Expr * getInit() const
bool hasExternalStorage() const
Returns true if a variable has extern or private_extern storage.
bool hasLocalStorage() const
Returns true if a variable with function scope is a non-static local variable.
@ DeclarationOnly
This declaration is only a declaration.
DefinitionKind hasDefinition(ASTContext &) const
Check whether this variable is defined in this translation unit.
bool isLocalVarDeclOrParm() const
Similar to isLocalVarDecl but also includes parameters.
const Expr * getAnyInitializer() const
Get the initializer for this variable, no matter which declaration it is attached to.
Represents a C array with a specified size that is not an integer-constant-expression.
Expr * getSizeExpr() const
specific_attr_iterator - Iterates over a subrange of an AttrVec, only providing attributes that are o...
bool isEmptyRecordForLayout(const ASTContext &Context, QualType T)
isEmptyRecordForLayout - Return true iff a structure contains only empty base classes (per isEmptyRec...
@ Type
The l-value was considered opaque, so the alignment was determined from a type.
@ Decl
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
bool isEmptyFieldForLayout(const ASTContext &Context, const FieldDecl *FD)
isEmptyFieldForLayout - Return true iff the field is "empty", that is, either a zero-width bit-field ...
The JSON file list parser is used to communicate input to InstallAPI.
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
CanQual< Type > CanQualType
Represents a canonical, potentially-qualified type.
bool needsTaskBasedThreadLimit(OpenMPDirectiveKind DKind)
Checks if the specified target directive, combined or not, needs task based thread_limit.
@ Ctor_Complete
Complete object ctor.
bool isa(CodeGen::Address addr)
if(T->getSizeExpr()) TRY_TO(TraverseStmt(const_cast< Expr * >(T -> getSizeExpr())))
bool isOpenMPTargetDataManagementDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target data offload directive.
@ Conditional
A conditional (?:) operator.
@ ICIS_NoInit
No in-class initializer.
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
@ LCK_ByRef
Capturing by reference.
LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE()
@ Private
'private' clause, allowed on 'parallel', 'serial', 'loop', 'parallel loop', and 'serial loop' constru...
@ Vector
'vector' clause, allowed on 'loop', Combined, and 'routine' directives.
@ Reduction
'reduction' clause, allowed on Parallel, Serial, Loop, and the combined constructs.
@ Present
'present' clause, allowed on Compute and Combined constructs, plus 'data' and 'declare'.
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
@ OMPC_SCHEDULE_MODIFIER_last
@ OMPC_SCHEDULE_MODIFIER_unknown
nullptr
This class represents a compute construct, representing a 'Kind' of ‘parallel’, 'serial',...
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
OpenMPDistScheduleClauseKind
OpenMP attributes for 'dist_schedule' clause.
static bool classof(const Stmt *T)
bool isOpenMPTaskingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of tasking directives - task, taskloop,...
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
@ Result
The result type of a method or function.
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
const FunctionProtoType * T
OpenMPDependClauseKind
OpenMP attributes for 'depend' clause.
@ Dtor_Complete
Complete object dtor.
@ Union
The "union" keyword.
@ Type
The name was classified as a type.
bool isOpenMPLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a directive with an associated loop construct.
OpenMPSeverityClauseKind
OpenMP attributes for 'severity' clause.
LangAS
Defines the address space values used by the address space qualifier of QualType.
llvm::omp::Directive OpenMPDirectiveKind
OpenMP directives.
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
@ VK_PRValue
A pr-value expression (in the C++11 taxonomy) produces a temporary value.
@ VK_LValue
An l-value expression is a reference to an object with independent storage.
for(const auto &A :T->param_types())
void getOpenMPCaptureRegions(llvm::SmallVectorImpl< OpenMPDirectiveKind > &CaptureRegions, OpenMPDirectiveKind DKind)
Return the captured regions of an OpenMP directive.
OpenMPNumThreadsClauseModifier
@ OMPC_ATOMIC_DEFAULT_MEM_ORDER_unknown
U cast(CodeGen::Address addr)
OpenMPMapModifierKind
OpenMP modifier kind for 'map' clause.
@ OMPC_MAP_MODIFIER_unknown
@ Other
Other implicit parameter.
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
OpenMPMapClauseKind
OpenMP mapping kind for 'map' clause.
Diagnostic wrappers for TextAPI types for error reporting.
Maps the expression for the lastprivate variable to the global copy used to store new value because o...
llvm::SmallVector< bool, 8 > IsPrivateVarReduction
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
llvm::BasicBlock * getBlock() const
unsigned NumberOfTargetItems
Address BasePointersArray
llvm::PointerType * VoidPtrTy
llvm::IntegerType * Int64Ty
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::CallingConv::ID getRuntimeCC() const
llvm::IntegerType * SizeTy
llvm::PointerType * VoidPtrPtrTy
llvm::IntegerType * Int32Ty
llvm::IntegerType * IntPtrTy
llvm::IntegerType * IntTy
int
CharUnits getPointerAlign() const
OpenMPDependClauseKind DepKind
const Expr * IteratorExpr
SmallVector< const Expr *, 4 > DepExprs
EvalResult is a struct with detailed info about an evaluated expression.
Extra information about a function prototype.
Expr * CounterUpdate
Updater for the internal counter: ++CounterVD;.
Data for list of allocators.
Expr * AllocatorTraits
Allocator traits.
Expr * Allocator
Allocator.
Scheduling data for loop-based OpenMP directives.
OpenMPScheduleClauseModifier M2
OpenMPScheduleClauseModifier M1
OpenMPScheduleClauseKind Schedule
Describes how types, statements, expressions, and declarations should be printed.