30#include "llvm/ADT/ArrayRef.h"
31#include "llvm/ADT/SetOperations.h"
32#include "llvm/ADT/SmallBitVector.h"
33#include "llvm/ADT/StringExtras.h"
34#include "llvm/Bitcode/BitcodeReader.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DerivedTypes.h"
37#include "llvm/IR/GlobalValue.h"
38#include "llvm/IR/InstrTypes.h"
39#include "llvm/IR/Value.h"
40#include "llvm/Support/AtomicOrdering.h"
41#include "llvm/Support/Format.h"
42#include "llvm/Support/raw_ostream.h"
49using namespace CodeGen;
50using namespace llvm::omp;
54class CGOpenMPRegionInfo :
public CodeGenFunction::CGCapturedStmtInfo {
57 enum CGOpenMPRegionKind {
60 ParallelOutlinedRegion,
71 const CGOpenMPRegionKind RegionKind,
74 : CGCapturedStmtInfo(CS,
CR_OpenMP), RegionKind(RegionKind),
75 CodeGen(CodeGen),
Kind(
Kind), HasCancel(HasCancel) {}
77 CGOpenMPRegionInfo(
const CGOpenMPRegionKind RegionKind,
80 : CGCapturedStmtInfo(
CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
85 virtual const VarDecl *getThreadIDVariable()
const = 0;
96 CGOpenMPRegionKind getRegionKind()
const {
return RegionKind; }
100 bool hasCancel()
const {
return HasCancel; }
102 static bool classof(
const CGCapturedStmtInfo *Info) {
106 ~CGOpenMPRegionInfo()
override =
default;
109 CGOpenMPRegionKind RegionKind;
116class CGOpenMPOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
121 StringRef HelperName)
122 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen,
Kind,
124 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
125 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
130 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
133 StringRef getHelperName()
const override {
return HelperName; }
135 static bool classof(
const CGCapturedStmtInfo *Info) {
136 return CGOpenMPRegionInfo::classof(Info) &&
137 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
138 ParallelOutlinedRegion;
145 StringRef HelperName;
149class CGOpenMPTaskOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
155 llvm::SwitchInst *UntiedSwitch =
nullptr;
158 UntiedTaskActionTy(
bool Tied,
const VarDecl *PartIDVar,
160 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
170 UntiedSwitch = CGF.
Builder.CreateSwitch(Res, DoneBB);
174 UntiedSwitch->addCase(CGF.
Builder.getInt32(0),
176 emitUntiedSwitch(CGF);
187 CodeGenFunction::JumpDest CurPoint =
191 UntiedSwitch->addCase(CGF.
Builder.getInt32(UntiedSwitch->getNumCases()),
197 unsigned getNumberOfParts()
const {
return UntiedSwitch->getNumCases(); }
203 const UntiedTaskActionTy &Action)
204 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen,
Kind, HasCancel),
205 ThreadIDVar(ThreadIDVar), Action(Action) {
206 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
211 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
217 StringRef getHelperName()
const override {
return ".omp_outlined."; }
220 Action.emitUntiedSwitch(CGF);
223 static bool classof(
const CGCapturedStmtInfo *Info) {
224 return CGOpenMPRegionInfo::classof(Info) &&
225 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
234 const UntiedTaskActionTy &Action;
239class CGOpenMPInlinedRegionInfo :
public CGOpenMPRegionInfo {
241 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
244 : CGOpenMPRegionInfo(InlinedRegion, CodeGen,
Kind, HasCancel),
246 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
249 llvm::Value *getContextValue()
const override {
251 return OuterRegionInfo->getContextValue();
252 llvm_unreachable(
"No context value for inlined OpenMP region");
255 void setContextValue(llvm::Value *
V)
override {
256 if (OuterRegionInfo) {
257 OuterRegionInfo->setContextValue(
V);
260 llvm_unreachable(
"No context value for inlined OpenMP region");
266 return OuterRegionInfo->lookup(VD);
272 FieldDecl *getThisFieldDecl()
const override {
274 return OuterRegionInfo->getThisFieldDecl();
280 const VarDecl *getThreadIDVariable()
const override {
282 return OuterRegionInfo->getThreadIDVariable();
289 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
290 llvm_unreachable(
"No LValue for inlined OpenMP construct");
294 StringRef getHelperName()
const override {
295 if (
auto *OuterRegionInfo = getOldCSI())
296 return OuterRegionInfo->getHelperName();
297 llvm_unreachable(
"No helper name for inlined OpenMP construct");
302 OuterRegionInfo->emitUntiedSwitch(CGF);
305 CodeGenFunction::CGCapturedStmtInfo *getOldCSI()
const {
return OldCSI; }
307 static bool classof(
const CGCapturedStmtInfo *Info) {
308 return CGOpenMPRegionInfo::classof(Info) &&
309 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
312 ~CGOpenMPInlinedRegionInfo()
override =
default;
316 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
317 CGOpenMPRegionInfo *OuterRegionInfo;
325class CGOpenMPTargetRegionInfo final :
public CGOpenMPRegionInfo {
329 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
331 HelperName(HelperName) {}
335 const VarDecl *getThreadIDVariable()
const override {
return nullptr; }
338 StringRef getHelperName()
const override {
return HelperName; }
340 static bool classof(
const CGCapturedStmtInfo *Info) {
341 return CGOpenMPRegionInfo::classof(Info) &&
342 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
346 StringRef HelperName;
350 llvm_unreachable(
"No codegen for expressions");
354class CGOpenMPInnerExprInfo final :
public CGOpenMPInlinedRegionInfo {
357 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
365 if (!
C.capturesVariable() && !
C.capturesVariableByCopy())
368 const VarDecl *VD =
C.getCapturedVar();
378 (void)PrivScope.Privatize();
383 if (
const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
390 llvm_unreachable(
"No body for expressions");
395 const VarDecl *getThreadIDVariable()
const override {
396 llvm_unreachable(
"No thread id for expressions");
400 StringRef getHelperName()
const override {
401 llvm_unreachable(
"No helper name for expressions");
404 static bool classof(
const CGCapturedStmtInfo *Info) {
return false; }
408 CodeGenFunction::OMPPrivateScope PrivScope;
412class InlinedOpenMPRegionRAII {
414 llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
415 FieldDecl *LambdaThisCaptureField =
nullptr;
417 bool NoInheritance =
false;
426 bool NoInheritance =
true)
427 : CGF(CGF), NoInheritance(NoInheritance) {
440 ~InlinedOpenMPRegionRAII() {
457enum OpenMPLocationFlags :
unsigned {
459 OMP_IDENT_IMD = 0x01,
461 OMP_IDENT_KMPC = 0x02,
463 OMP_ATOMIC_REDUCE = 0x10,
465 OMP_IDENT_BARRIER_EXPL = 0x20,
467 OMP_IDENT_BARRIER_IMPL = 0x40,
469 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
471 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
473 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
475 OMP_IDENT_WORK_LOOP = 0x200,
477 OMP_IDENT_WORK_SECTIONS = 0x400,
479 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
509enum IdentFieldIndex {
511 IdentField_Reserved_1,
515 IdentField_Reserved_2,
517 IdentField_Reserved_3,
526enum OpenMPSchedType {
529 OMP_sch_static_chunked = 33,
531 OMP_sch_dynamic_chunked = 35,
532 OMP_sch_guided_chunked = 36,
533 OMP_sch_runtime = 37,
536 OMP_sch_static_balanced_chunked = 45,
539 OMP_ord_static_chunked = 65,
541 OMP_ord_dynamic_chunked = 67,
542 OMP_ord_guided_chunked = 68,
543 OMP_ord_runtime = 69,
545 OMP_sch_default = OMP_sch_static,
547 OMP_dist_sch_static_chunked = 91,
548 OMP_dist_sch_static = 92,
551 OMP_sch_modifier_monotonic = (1 << 29),
553 OMP_sch_modifier_nonmonotonic = (1 << 30),
576 Callback(CodeGen, CGF, *PrePostAction);
579 Callback(CodeGen, CGF, Action);
587 if (
const auto *CE = dyn_cast<CallExpr>(ReductionOp))
588 if (
const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
589 if (
const auto *DRE =
590 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
591 if (
const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
602 std::pair<llvm::Function *, llvm::Function *>
Reduction =
604 const auto *CE = cast<CallExpr>(InitOp);
605 const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
609 cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
611 cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
612 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
613 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
Private);
614 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()), Original);
615 (void)PrivateScope.Privatize();
617 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE,
Func);
622 auto *GV =
new llvm::GlobalVariable(
624 llvm::GlobalValue::PrivateLinkage,
Init, Name);
637 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, LV);
644 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
665 llvm::Value *NumElements = CGF.
emitArrayLength(ArrayTy, ElementTy, DestAddr);
669 llvm::Value *SrcBegin =
nullptr;
671 SrcBegin = SrcAddr.getPointer();
672 llvm::Value *DestBegin = DestAddr.
getPointer();
674 llvm::Value *DestEnd =
679 llvm::Value *IsEmpty =
680 CGF.
Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arrayinit.isempty");
681 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
684 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
689 llvm::PHINode *SrcElementPHI =
nullptr;
692 SrcElementPHI = CGF.
Builder.CreatePHI(SrcBegin->getType(), 2,
693 "omp.arraycpy.srcElementPast");
694 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
696 Address(SrcElementPHI, SrcAddr.getElementType(),
697 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
699 llvm::PHINode *DestElementPHI = CGF.
Builder.CreatePHI(
700 DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
701 DestElementPHI->addIncoming(DestBegin, EntryBB);
708 CodeGenFunction::RunCleanupsScope InitScope(CGF);
709 if (EmitDeclareReductionInit) {
711 SrcElementCurrent, ElementTy);
719 llvm::Value *SrcElementNext = CGF.
Builder.CreateConstGEP1_32(
720 SrcAddr.getElementType(), SrcElementPHI, 1,
721 "omp.arraycpy.dest.element");
722 SrcElementPHI->addIncoming(SrcElementNext, CGF.
Builder.GetInsertBlock());
726 llvm::Value *DestElementNext = CGF.
Builder.CreateConstGEP1_32(
728 "omp.arraycpy.dest.element");
731 CGF.
Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
732 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
733 DestElementPHI->addIncoming(DestElementNext, CGF.
Builder.GetInsertBlock());
745 if (
const auto *OASE = dyn_cast<OMPArraySectionExpr>(E))
750void ReductionCodeGen::emitAggregateInitialization(
756 const auto *PrivateVD =
757 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].
Private)->getDecl());
758 bool EmitDeclareReductionInit =
761 EmitDeclareReductionInit,
762 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
763 : PrivateVD->getInit(),
771 ClausesData.reserve(Shareds.size());
772 SharedAddresses.reserve(Shareds.size());
773 Sizes.reserve(Shareds.size());
774 BaseDecls.reserve(Shareds.size());
775 const auto *IOrig = Origs.begin();
776 const auto *IPriv = Privates.begin();
777 const auto *IRed = ReductionOps.begin();
778 for (
const Expr *Ref : Shareds) {
779 ClausesData.emplace_back(Ref, *IOrig, *IPriv, *IRed);
780 std::advance(IOrig, 1);
781 std::advance(IPriv, 1);
782 std::advance(IRed, 1);
787 assert(SharedAddresses.size() == N && OrigAddresses.size() == N &&
788 "Number of generated lvalues must be exactly N.");
789 LValue First = emitSharedLValue(CGF, ClausesData[N].Shared);
790 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Shared);
791 SharedAddresses.emplace_back(
First, Second);
792 if (ClausesData[N].Shared == ClausesData[N].Ref) {
793 OrigAddresses.emplace_back(
First, Second);
795 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
796 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
797 OrigAddresses.emplace_back(
First, Second);
803 bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
806 CGF.
getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()),
811 llvm::Value *SizeInChars;
812 auto *ElemType = OrigAddresses[N].first.getAddress(CGF).getElementType();
813 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
814 if (AsArraySection) {
815 Size = CGF.
Builder.CreatePtrDiff(ElemType,
816 OrigAddresses[N].second.getPointer(CGF),
817 OrigAddresses[N].first.getPointer(CGF));
818 Size = CGF.
Builder.CreateNUWAdd(
819 Size, llvm::ConstantInt::get(Size->getType(), 1));
820 SizeInChars = CGF.
Builder.CreateNUWMul(Size, ElemSizeOf);
823 CGF.
getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType());
824 Size = CGF.
Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
826 Sizes.emplace_back(SizeInChars, Size);
829 cast<OpaqueValueExpr>(
839 assert(!Size && !Sizes[N].second &&
840 "Size should be nullptr for non-variably modified reduction "
846 cast<OpaqueValueExpr>(
855 assert(SharedAddresses.size() > N &&
"No variable was generated");
856 const auto *PrivateVD =
857 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].
Private)->getDecl());
862 (void)DefaultInit(CGF);
863 emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD);
864 }
else if (DRD && (DRD->
getInitializer() || !PrivateVD->hasInit())) {
865 (void)DefaultInit(CGF);
866 QualType SharedType = SharedAddresses[N].first.getType();
868 PrivateAddr, SharedAddr, SharedType);
869 }
else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
872 PrivateVD->
getType().getQualifiers(),
890 CGF.
pushDestroy(DTorKind, PrivateAddr, PrivateType);
914 Address OriginalBaseAddress, llvm::Value *Addr) {
938 Addr, OriginalBaseAddress.
getType());
943 const VarDecl *OrigVD =
nullptr;
944 if (
const auto *OASE = dyn_cast<OMPArraySectionExpr>(Ref)) {
945 const Expr *
Base = OASE->getBase()->IgnoreParenImpCasts();
946 while (
const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(
Base))
947 Base = TempOASE->getBase()->IgnoreParenImpCasts();
948 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
949 Base = TempASE->getBase()->IgnoreParenImpCasts();
950 DE = cast<DeclRefExpr>(
Base);
951 OrigVD = cast<VarDecl>(DE->
getDecl());
952 }
else if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
953 const Expr *
Base = ASE->getBase()->IgnoreParenImpCasts();
954 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
955 Base = TempASE->getBase()->IgnoreParenImpCasts();
956 DE = cast<DeclRefExpr>(
Base);
957 OrigVD = cast<VarDecl>(DE->
getDecl());
966 BaseDecls.emplace_back(OrigVD);
969 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
971 Address SharedAddr = SharedAddresses[N].first.getAddress(CGF);
972 llvm::Value *Adjustment = CGF.
Builder.CreatePtrDiff(
975 llvm::Value *PrivatePointer =
981 SharedAddresses[N].first.getType(),
984 BaseDecls.emplace_back(
985 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
998 getThreadIDVariable()->getType()->castAs<PointerType>());
1016LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1019 getThreadIDVariable()->getType(),
1035 : CGM(CGM), OMPBuilder(CGM.getModule()) {
1037 llvm::OpenMPIRBuilderConfig Config(
1052 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(
true);
1060 if (!
Data.getValue().pointsToAliveValue())
1062 auto *GV = dyn_cast<llvm::GlobalVariable>(
Data.getValue());
1065 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1067 GV->eraseFromParent();
1072 return OMPBuilder.createPlatformSpecificName(Parts);
1075static llvm::Function *
1077 const Expr *CombinerInitializer,
const VarDecl *In,
1078 const VarDecl *Out,
bool IsCombiner) {
1081 QualType PtrTy =
C.getPointerType(Ty).withRestrict();
1087 Args.push_back(&OmpOutParm);
1088 Args.push_back(&OmpInParm);
1093 {IsCombiner ?
"omp_combiner" :
"omp_initializer",
""});
1094 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1098 Fn->removeFnAttr(llvm::Attribute::NoInline);
1099 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1100 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1106 Out->getLocation());
1107 CodeGenFunction::OMPPrivateScope
Scope(CGF);
1116 (void)
Scope.Privatize();
1117 if (!IsCombiner && Out->hasInit() &&
1120 Out->getType().getQualifiers(),
1123 if (CombinerInitializer)
1125 Scope.ForceCleanup();
1136 cast<VarDecl>(cast<DeclRefExpr>(D->
getCombinerIn())->getDecl()),
1137 cast<VarDecl>(cast<DeclRefExpr>(D->
getCombinerOut())->getDecl()),
1145 cast<VarDecl>(cast<DeclRefExpr>(D->
getInitOrig())->getDecl()),
1146 cast<VarDecl>(cast<DeclRefExpr>(D->
getInitPriv())->getDecl()),
1152 Decls.second.push_back(D);
1156std::pair<llvm::Function *, llvm::Function *>
1168struct PushAndPopStackRAII {
1169 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder,
CodeGenFunction &CGF,
1170 bool HasCancel, llvm::omp::Directive Kind)
1171 : OMPBuilder(OMPBuilder) {
1187 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1188 assert(IP.getBlock()->end() == IP.getPoint() &&
1189 "Clang CG should cause non-terminated block!");
1190 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1199 llvm::OpenMPIRBuilder::FinalizationInfo FI({FiniCB, Kind, HasCancel});
1200 OMPBuilder->pushFinalizationCB(std::move(FI));
1202 ~PushAndPopStackRAII() {
1204 OMPBuilder->popFinalizationCB();
1206 llvm::OpenMPIRBuilder *OMPBuilder;
1215 "thread id variable must be of type kmp_int32 *");
1217 bool HasCancel =
false;
1218 if (
const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1219 HasCancel = OPD->hasCancel();
1220 else if (
const auto *OPD = dyn_cast<OMPTargetParallelDirective>(&D))
1221 HasCancel = OPD->hasCancel();
1222 else if (
const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1223 HasCancel = OPSD->hasCancel();
1224 else if (
const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1225 HasCancel = OPFD->hasCancel();
1226 else if (
const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1227 HasCancel = OPFD->hasCancel();
1228 else if (
const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1229 HasCancel = OPFD->hasCancel();
1230 else if (
const auto *OPFD =
1231 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1232 HasCancel = OPFD->hasCancel();
1233 else if (
const auto *OPFD =
1234 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1235 HasCancel = OPFD->hasCancel();
1240 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind);
1241 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1242 HasCancel, OutlinedHelperName);
1243 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1248 std::string Suffix =
getName({
"omp_outlined"});
1249 return (Name + Suffix).str();
1257 std::string Suffix =
getName({
"omp",
"reduction",
"reduction_func"});
1258 return (Name + Suffix).str();
1285 bool Tied,
unsigned &NumberOfParts) {
1290 llvm::Value *TaskArgs[] = {
1292 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1295 CGF.EmitRuntimeCall(
OMPBuilder.getOrCreateRuntimeFunction(
1299 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1303 "thread id variable must be of type kmp_int32 for tasks");
1308 bool HasCancel =
false;
1309 if (
const auto *TD = dyn_cast<OMPTaskDirective>(&D))
1310 HasCancel = TD->hasCancel();
1311 else if (
const auto *TD = dyn_cast<OMPTaskLoopDirective>(&D))
1312 HasCancel = TD->hasCancel();
1313 else if (
const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(&D))
1314 HasCancel = TD->hasCancel();
1315 else if (
const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(&D))
1316 HasCancel = TD->hasCancel();
1319 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1320 InnermostKind, HasCancel, Action);
1322 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1324 NumberOfParts = Action.getNumberOfParts();
1329 bool AtCurrentPoint) {
1331 assert(!Elem.second.ServiceInsertPt &&
"Insert point is set already.");
1333 llvm::Value *Undef = llvm::UndefValue::get(CGF.
Int32Ty);
1334 if (AtCurrentPoint) {
1335 Elem.second.ServiceInsertPt =
new llvm::BitCastInst(
1338 Elem.second.ServiceInsertPt =
1339 new llvm::BitCastInst(Undef, CGF.
Int32Ty,
"svcpt");
1346 if (Elem.second.ServiceInsertPt) {
1347 llvm::Instruction *Ptr = Elem.second.ServiceInsertPt;
1348 Elem.second.ServiceInsertPt =
nullptr;
1349 Ptr->eraseFromParent();
1356 llvm::raw_svector_ostream OS(Buffer);
1360 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.
CurFuncDecl))
1361 OS << FD->getQualifiedNameAsString();
1368 unsigned Flags,
bool EmitLoc) {
1369 uint32_t SrcLocStrSize;
1370 llvm::Constant *SrcLocStr;
1372 llvm::codegenoptions::NoDebugInfo) ||
1374 SrcLocStr =
OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1376 std::string FunctionName;
1377 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.
CurFuncDecl))
1378 FunctionName = FD->getQualifiedNameAsString();
1388 SrcLocStr, SrcLocStrSize, llvm::omp::IdentFlag(Flags), Reserved2Flags);
1393 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1399 uint32_t SrcLocStrSize;
1400 auto *SrcLocStr =
OMPBuilder.getOrCreateSrcLocStr(
1403 OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize));
1406 llvm::Value *ThreadID =
nullptr;
1411 ThreadID = I->second.ThreadID;
1412 if (ThreadID !=
nullptr)
1416 if (
auto *OMPRegionInfo =
1418 if (OMPRegionInfo->getThreadIDVariable()) {
1420 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1424 CGF.
Builder.GetInsertBlock() == TopBlock ||
1425 !isa<llvm::Instruction>(LVal.
getPointer(CGF)) ||
1426 cast<llvm::Instruction>(LVal.
getPointer(CGF))->getParent() ==
1428 cast<llvm::Instruction>(LVal.
getPointer(CGF))->getParent() ==
1429 CGF.
Builder.GetInsertBlock()) {
1433 if (CGF.
Builder.GetInsertBlock() == TopBlock) {
1435 Elem.second.ThreadID = ThreadID;
1447 if (!Elem.second.ServiceInsertPt)
1449 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1450 CGF.
Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1454 OMPRTL___kmpc_global_thread_num),
1457 Elem.second.ThreadID =
Call;
1462 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1474 for(
const auto *D : I->second)
1489 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(
CGM.
Int32Ty),
1496llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind
1498 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
1499 OMPDeclareTargetDeclAttr::getDeviceType(VD);
1501 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1503 switch ((
int)*DevTy) {
1504 case OMPDeclareTargetDeclAttr::DT_Host:
1505 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseHost;
1507 case OMPDeclareTargetDeclAttr::DT_NoHost:
1508 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNoHost;
1510 case OMPDeclareTargetDeclAttr::DT_Any:
1511 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny;
1514 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1519llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind
1521 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> MapType =
1522 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1524 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1525 switch ((
int)*MapType) {
1526 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_To:
1527 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
1529 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Enter:
1530 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter;
1532 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Link:
1533 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
1536 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1545 auto FileInfoCallBack = [&]() {
1549 llvm::sys::fs::UniqueID ID;
1550 if (llvm::sys::fs::getUniqueID(PLoc.
getFilename(), ID)) {
1551 PLoc =
SM.getPresumedLoc(BeginLoc,
false);
1557 return OMPBuilder.getTargetEntryUniqueInfo(FileInfoCallBack, ParentName);
1563 auto LinkageForVariable = [&VD,
this]() {
1567 std::vector<llvm::GlobalVariable *> GeneratedRefs;
1571 llvm::Constant *addr =
OMPBuilder.getAddrOfDeclareTargetVar(
1579 LinkageForVariable);
1591 std::string Suffix =
getName({
"cache",
""});
1592 return OMPBuilder.getOrCreateInternalVariable(
1605 llvm::Value *Args[] = {
1629 llvm::Value *Args[] = {
1631 Ctor, CopyCtor, Dtor};
1634 CGM.
getModule(), OMPRTL___kmpc_threadprivate_register),
1649 llvm::Value *Ctor =
nullptr, *CopyCtor =
nullptr, *Dtor =
nullptr;
1659 Args.push_back(&Dst);
1664 std::string Name =
getName({
"__kmpc_global_ctor_",
""});
1665 llvm::Function *Fn =
1691 Args.push_back(&Dst);
1696 std::string Name =
getName({
"__kmpc_global_dtor_",
""});
1697 llvm::Function *Fn =
1719 auto *CopyCtorTy = llvm::FunctionType::get(
CGM.
VoidPtrTy, CopyCtorTyArgs,
1725 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
1726 if (Ctor ==
nullptr) {
1730 Ctor = llvm::Constant::getNullValue(CtorTy);
1732 if (Dtor ==
nullptr) {
1736 Dtor = llvm::Constant::getNullValue(DtorTy);
1739 auto *InitFunctionTy =
1740 llvm::FunctionType::get(
CGM.
VoidTy,
false);
1741 std::string Name =
getName({
"__omp_threadprivate_init_",
""});
1751 return InitFunction;
1759 llvm::GlobalValue *GV) {
1760 std::optional<OMPDeclareTargetDeclAttr *> ActiveAttr =
1761 OMPDeclareTargetDeclAttr::getActiveAttr(FD);
1764 if (!ActiveAttr || !(*ActiveAttr)->getIndirect())
1771 OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(Name, EntryInfo);
1777 llvm::GlobalValue *Addr = GV;
1779 Addr =
new llvm::GlobalVariable(
1781 true, llvm::GlobalValue::ExternalLinkage, GV, Name,
1782 nullptr, llvm::GlobalValue::NotThreadLocal,
1783 CGM.
getModule().getDataLayout().getDefaultGlobalsAddressSpace());
1784 Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1787 OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(
1789 llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect,
1790 llvm::GlobalValue::WeakODRLinkage);
1796 std::string Suffix =
getName({
"artificial",
""});
1798 llvm::GlobalVariable *GAddr =
OMPBuilder.getOrCreateInternalVariable(
1799 VarLVType, Twine(Name).concat(Suffix).str());
1802 GAddr->setThreadLocal(
true);
1803 return Address(GAddr, GAddr->getValueType(),
1806 std::string CacheSuffix =
getName({
"cache",
""});
1807 llvm::Value *Args[] = {
1815 Twine(Name).concat(Suffix).concat(CacheSuffix).str())};
1822 VarLVType->getPointerTo(0)),
1866 llvm::Function *OutlinedFn,
1869 llvm::Value *NumThreads) {
1874 auto &&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc,
1878 llvm::Value *Args[] = {
1880 CGF.
Builder.getInt32(CapturedVars.size()),
1883 RealArgs.append(std::begin(Args), std::end(Args));
1884 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
1886 llvm::FunctionCallee RTLFn =
1887 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_fork_call);
1890 auto &&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc, Loc,
1896 llvm::Value *Args[] = {RTLoc, ThreadID};
1898 M, OMPRTL___kmpc_serialized_parallel),
1905 ".bound.zero.addr");
1909 OutlinedFnArgs.push_back(ThreadIDAddr.
getPointer());
1910 OutlinedFnArgs.push_back(ZeroAddrBound.
getPointer());
1911 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
1919 OutlinedFn->removeFnAttr(llvm::Attribute::AlwaysInline);
1920 OutlinedFn->addFnAttr(llvm::Attribute::NoInline);
1926 M, OMPRTL___kmpc_end_serialized_parallel),
1945 if (
auto *OMPRegionInfo =
1947 if (OMPRegionInfo->getThreadIDVariable())
1948 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(CGF);
1957 return ThreadIDTemp;
1961 std::string Prefix = Twine(
"gomp_critical_user_", CriticalName).str();
1962 std::string Name =
getName({Prefix,
"var"});
1969 llvm::FunctionCallee EnterCallee;
1971 llvm::FunctionCallee ExitCallee;
1974 llvm::BasicBlock *ContBlock =
nullptr;
1977 CommonActionTy(llvm::FunctionCallee EnterCallee,
1979 llvm::FunctionCallee ExitCallee,
1981 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
1982 ExitArgs(ExitArgs), Conditional(Conditional) {}
1986 llvm::Value *CallBool = CGF.
Builder.CreateIsNotNull(EnterRes);
1990 CGF.
Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
2006 StringRef CriticalName,
2020 EnterArgs.push_back(CGF.
Builder.CreateIntCast(
2023 CommonActionTy Action(
2026 Hint ? OMPRTL___kmpc_critical_with_hint : OMPRTL___kmpc_critical),
2029 OMPRTL___kmpc_end_critical),
2046 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2068 llvm::Value *FilterVal = Filter
2075 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2095 llvm::Value *Args[] = {
2097 llvm::ConstantInt::get(
CGM.
IntTy, 0,
true)};
2103 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
2104 Region->emitUntiedSwitch(CGF);
2117 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2130 unsigned Index,
const VarDecl *Var) {
2138 Ptr, ElemTy->getPointerTo(Ptr->getType()->getPointerAddressSpace())),
2154 Args.push_back(&LHSArg);
2155 Args.push_back(&RHSArg);
2161 llvm::GlobalValue::InternalLinkage, Name,
2164 Fn->setDoesNotRecurse();
2171 ArgsElemType->getPointerTo()),
2175 ArgsElemType->getPointerTo()),
2181 for (
unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
2182 const auto *DestVar =
2183 cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
2186 const auto *SrcVar =
2187 cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
2190 const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
2192 CGF.
EmitOMPCopy(
Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
2207 assert(CopyprivateVars.size() == SrcExprs.size() &&
2208 CopyprivateVars.size() == DstExprs.size() &&
2209 CopyprivateVars.size() == AssignmentOps.size());
2221 if (!CopyprivateVars.empty()) {
2224 C.getIntTypeForBitwidth(32, 1);
2225 DidIt = CGF.
CreateMemTemp(KmpInt32Ty,
".omp.copyprivate.did_it");
2230 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2247 llvm::APInt ArraySize(32, CopyprivateVars.size());
2248 QualType CopyprivateArrayTy =
C.getConstantArrayType(
2253 CGF.
CreateMemTemp(CopyprivateArrayTy,
".omp.copyprivate.cpr_list");
2254 for (
unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
2266 SrcExprs, DstExprs, AssignmentOps, Loc);
2267 llvm::Value *BufSize = CGF.
getTypeSize(CopyprivateArrayTy);
2271 llvm::Value *Args[] = {
2296 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2311 if (Kind == OMPD_for)
2312 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
2313 else if (Kind == OMPD_sections)
2314 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
2315 else if (Kind == OMPD_single)
2316 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
2317 else if (Kind == OMPD_barrier)
2318 Flags = OMP_IDENT_BARRIER_EXPL;
2320 Flags = OMP_IDENT_BARRIER_IMPL;
2332 ScheduleKind = OMPC_SCHEDULE_static;
2334 llvm::APInt ChunkSize(32, 1);
2344 bool ForceSimpleCall) {
2346 auto *OMPRegionInfo =
2350 CGF.
Builder, Kind, ForceSimpleCall, EmitChecks));
2363 if (OMPRegionInfo) {
2364 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
2367 OMPRTL___kmpc_cancel_barrier),
2376 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
2393 Expr *ME,
bool IsFatal) {
2396 : llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
2399 llvm::Value *Args[] = {
2401 llvm::ConstantInt::get(
CGM.
Int32Ty, IsFatal ? 2 : 1),
2410 bool Chunked,
bool Ordered) {
2411 switch (ScheduleKind) {
2412 case OMPC_SCHEDULE_static:
2413 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
2414 : (Ordered ? OMP_ord_static : OMP_sch_static);
2415 case OMPC_SCHEDULE_dynamic:
2416 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
2417 case OMPC_SCHEDULE_guided:
2418 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
2419 case OMPC_SCHEDULE_runtime:
2420 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
2421 case OMPC_SCHEDULE_auto:
2422 return Ordered ? OMP_ord_auto : OMP_sch_auto;
2424 assert(!Chunked &&
"chunk was specified but schedule kind not known");
2425 return Ordered ? OMP_ord_static : OMP_sch_static;
2427 llvm_unreachable(
"Unexpected runtime schedule");
2431static OpenMPSchedType
2434 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
2438 bool Chunked)
const {
2439 OpenMPSchedType Schedule =
2441 return Schedule == OMP_sch_static;
2447 return Schedule == OMP_dist_sch_static;
2451 bool Chunked)
const {
2452 OpenMPSchedType Schedule =
2454 return Schedule == OMP_sch_static_chunked;
2460 return Schedule == OMP_dist_sch_static_chunked;
2464 OpenMPSchedType Schedule =
2466 assert(Schedule != OMP_sch_static_chunked &&
"cannot be chunked here");
2467 return Schedule != OMP_sch_static;
2475 case OMPC_SCHEDULE_MODIFIER_monotonic:
2476 Modifier = OMP_sch_modifier_monotonic;
2478 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2479 Modifier = OMP_sch_modifier_nonmonotonic;
2481 case OMPC_SCHEDULE_MODIFIER_simd:
2482 if (Schedule == OMP_sch_static_chunked)
2483 Schedule = OMP_sch_static_balanced_chunked;
2490 case OMPC_SCHEDULE_MODIFIER_monotonic:
2491 Modifier = OMP_sch_modifier_monotonic;
2493 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2494 Modifier = OMP_sch_modifier_nonmonotonic;
2496 case OMPC_SCHEDULE_MODIFIER_simd:
2497 if (Schedule == OMP_sch_static_chunked)
2498 Schedule = OMP_sch_static_balanced_chunked;
2510 if (CGM.
getLangOpts().OpenMP >= 50 && Modifier == 0) {
2511 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
2512 Schedule == OMP_sch_static_balanced_chunked ||
2513 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||
2514 Schedule == OMP_dist_sch_static_chunked ||
2515 Schedule == OMP_dist_sch_static))
2516 Modifier = OMP_sch_modifier_nonmonotonic;
2518 return Schedule | Modifier;
2528 ScheduleKind.
Schedule, DispatchValues.
Chunk !=
nullptr, Ordered);
2530 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
2531 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&
2532 Schedule != OMP_sch_static_balanced_chunked));
2539 llvm::Value *Chunk = DispatchValues.
Chunk ? DispatchValues.
Chunk
2540 : CGF.
Builder.getIntN(IVSize, 1);
2541 llvm::Value *Args[] = {
2545 CGM, Schedule, ScheduleKind.
M1, ScheduleKind.
M2)),
2548 CGF.
Builder.getIntN(IVSize, 1),
2556 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
2557 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
2564 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||
2565 Schedule == OMP_sch_static_balanced_chunked ||
2566 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||
2567 Schedule == OMP_dist_sch_static ||
2568 Schedule == OMP_dist_sch_static_chunked);
2575 llvm::Value *Chunk = Values.
Chunk;
2576 if (Chunk ==
nullptr) {
2577 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
2578 Schedule == OMP_dist_sch_static) &&
2579 "expected static non-chunked schedule");
2583 assert((Schedule == OMP_sch_static_chunked ||
2584 Schedule == OMP_sch_static_balanced_chunked ||
2585 Schedule == OMP_ord_static_chunked ||
2586 Schedule == OMP_dist_sch_static_chunked) &&
2587 "expected static chunked schedule");
2589 llvm::Value *Args[] = {
2612 "Expected loop-based or sections-based directive.");
2615 ? OMP_IDENT_WORK_LOOP
2616 : OMP_IDENT_WORK_SECTIONS);
2618 llvm::FunctionCallee StaticInitFunction =
2623 ScheduleNum, ScheduleKind.
M1, ScheduleKind.
M2, Values);
2630 OpenMPSchedType ScheduleNum =
2632 llvm::Value *UpdatedLocation =
2635 llvm::FunctionCallee StaticInitFunction;
2636 bool isGPUDistribute =
2639 StaticInitFunction =
OMPBuilder.createForStaticInitFunction(
2650 assert(DKind == OMPD_distribute || DKind == OMPD_for ||
2651 DKind == OMPD_sections &&
2652 "Expected distribute, for, or sections directive kind");
2656 llvm::Value *Args[] = {
2659 ? OMP_IDENT_WORK_DISTRIBUTE
2661 ? OMP_IDENT_WORK_LOOP
2662 : OMP_IDENT_WORK_SECTIONS),
2670 CGM.
getModule(), OMPRTL___kmpc_distribute_static_fini),
2699 llvm::Value *Args[] = {
2708 OMPBuilder.createDispatchNextFunction(IVSize, IVSigned), Args);
2715 llvm::Value *NumThreads,
2720 llvm::Value *Args[] = {
2729 ProcBindKind ProcBind,
2733 assert(ProcBind != OMP_PROC_BIND_unknown &&
"Unsupported proc_bind value.");
2735 llvm::Value *Args[] = {
2737 llvm::ConstantInt::get(
CGM.
IntTy,
unsigned(ProcBind),
true)};
2759enum KmpTaskTFields {
2789 llvm::OpenMPIRBuilder::EmitMetadataErrorReportFunctionTy &&ErrorReportFn =
2790 [
this](llvm::OpenMPIRBuilder::EmitMetadataErrorKind Kind,
2791 const llvm::TargetRegionEntryInfo &EntryInfo) ->
void {
2793 if (Kind != llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR) {
2797 if (I->getFirst().getUniqueID().getDevice() == EntryInfo.DeviceID &&
2798 I->getFirst().getUniqueID().getFile() == EntryInfo.FileID) {
2800 I->getFirst(), EntryInfo.Line, 1);
2806 case llvm::OpenMPIRBuilder::EMIT_MD_TARGET_REGION_ERROR: {
2809 "%0 is incorrect: either the "
2810 "address or the ID is invalid.");
2813 case llvm::OpenMPIRBuilder::EMIT_MD_DECLARE_TARGET_ERROR: {
2816 "variable %0 is incorrect: the "
2817 "address is invalid.");
2820 case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR: {
2823 "Offloading entry for declare target variable is incorrect: the "
2824 "address is invalid.");
2830 OMPBuilder.createOffloadEntriesAndInfoMetadata(ErrorReportFn);
2837 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty,
C.VoidPtrTy};
2840 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
2846struct PrivateHelpersTy {
2847 PrivateHelpersTy(
const Expr *OriginalRef,
const VarDecl *Original,
2849 : OriginalRef(OriginalRef), Original(Original), PrivateCopy(PrivateCopy),
2850 PrivateElemInit(PrivateElemInit) {}
2851 PrivateHelpersTy(
const VarDecl *Original) : Original(Original) {}
2852 const Expr *OriginalRef =
nullptr;
2853 const VarDecl *Original =
nullptr;
2854 const VarDecl *PrivateCopy =
nullptr;
2855 const VarDecl *PrivateElemInit =
nullptr;
2856 bool isLocalPrivate()
const {
2857 return !OriginalRef && !PrivateCopy && !PrivateElemInit;
2860typedef std::pair<
CharUnits , PrivateHelpersTy> PrivateDataTy;
2865 if (!CVD->
hasAttr<OMPAllocateDeclAttr>())
2867 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
2869 return !(AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc &&
2870 !AA->getAllocator());
2875 if (!Privates.empty()) {
2880 RecordDecl *RD =
C.buildImplicitRecord(
".kmp_privates.t");
2882 for (
const auto &Pair : Privates) {
2883 const VarDecl *VD = Pair.second.Original;
2887 if (Pair.second.isLocalPrivate()) {
2910 QualType KmpRoutineEntryPointerQTy) {
2930 QualType KmpCmplrdataTy =
C.getRecordType(UD);
2931 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_task_t");
2961 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_task_t_with_privates");
2981static llvm::Function *
2984 QualType KmpTaskTWithPrivatesPtrQTy,
2986 QualType SharedsPtrTy, llvm::Function *TaskFunction,
2987 llvm::Value *TaskPrivatesMap) {
2995 Args.push_back(&GtidArg);
2996 Args.push_back(&TaskTypeArg);
2997 const auto &TaskEntryFnInfo =
2999 llvm::FunctionType *TaskEntryTy =
3002 auto *TaskEntry = llvm::Function::Create(
3003 TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.
getModule());
3005 TaskEntry->setDoesNotRecurse();
3020 const auto *KmpTaskTWithPrivatesQTyRD =
3021 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
3024 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->
getAsTagDecl());
3025 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
3027 llvm::Value *PartidParam = PartIdLVal.
getPointer(CGF);
3029 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
3035 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
3036 llvm::Value *PrivatesParam;
3037 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
3042 PrivatesParam = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
3045 llvm::Value *CommonArgs[] = {
3046 GtidParam, PartidParam, PrivatesParam, TaskPrivatesMap,
3052 std::end(CommonArgs));
3054 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
3057 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
3060 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
3063 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
3066 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
3069 CallArgs.push_back(LBParam);
3070 CallArgs.push_back(UBParam);
3071 CallArgs.push_back(StParam);
3072 CallArgs.push_back(LIParam);
3073 CallArgs.push_back(RParam);
3075 CallArgs.push_back(SharedsParam);
3088 QualType KmpTaskTWithPrivatesPtrQTy,
3089 QualType KmpTaskTWithPrivatesQTy) {
3097 Args.push_back(&GtidArg);
3098 Args.push_back(&TaskTypeArg);
3099 const auto &DestructorFnInfo =
3101 llvm::FunctionType *DestructorFnTy =
3105 auto *DestructorFn =
3106 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
3110 DestructorFn->setDoesNotRecurse();
3118 const auto *KmpTaskTWithPrivatesQTyRD =
3119 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
3120 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
3122 for (
const auto *Field :
3123 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
3125 Field->getType().isDestructedType()) {
3131 return DestructorFn;
3151 C,
nullptr, Loc,
nullptr,
3152 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
3154 Args.push_back(&TaskPrivatesArg);
3155 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>,
unsigned> PrivateVarsPos;
3156 unsigned Counter = 1;
3157 for (
const Expr *E :
Data.PrivateVars) {
3159 C,
nullptr, Loc,
nullptr,
3160 C.getPointerType(
C.getPointerType(E->
getType()))
3164 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3165 PrivateVarsPos[VD] = Counter;
3168 for (
const Expr *E :
Data.FirstprivateVars) {
3170 C,
nullptr, Loc,
nullptr,
3171 C.getPointerType(
C.getPointerType(E->
getType()))
3175 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3176 PrivateVarsPos[VD] = Counter;
3179 for (
const Expr *E :
Data.LastprivateVars) {
3181 C,
nullptr, Loc,
nullptr,
3182 C.getPointerType(
C.getPointerType(E->
getType()))
3186 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3187 PrivateVarsPos[VD] = Counter;
3193 Ty =
C.getPointerType(Ty);
3195 Ty =
C.getPointerType(Ty);
3197 C,
nullptr, Loc,
nullptr,
3198 C.getPointerType(
C.getPointerType(Ty)).withConst().withRestrict(),
3200 PrivateVarsPos[VD] = Counter;
3203 const auto &TaskPrivatesMapFnInfo =
3205 llvm::FunctionType *TaskPrivatesMapTy =
3209 auto *TaskPrivatesMap = llvm::Function::Create(
3210 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
3213 TaskPrivatesMapFnInfo);
3215 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
3216 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
3217 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
3221 TaskPrivatesMapFnInfo, Args, Loc, Loc);
3227 const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->
getAsTagDecl());
3229 for (
const FieldDecl *Field : PrivatesQTyRD->fields()) {
3231 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
3240 return TaskPrivatesMap;
3252 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
3258 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
3266 if ((!IsTargetTask && !
Data.FirstprivateVars.empty() && ForDup) ||
3267 (IsTargetTask && KmpTaskSharedsPtr.
isValid())) {
3274 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
3275 for (
const PrivateDataTy &Pair : Privates) {
3277 if (Pair.second.isLocalPrivate()) {
3281 const VarDecl *VD = Pair.second.PrivateCopy;
3283 if (
Init && (!ForDup || (isa<CXXConstructExpr>(
Init) &&
3286 if (
const VarDecl *Elem = Pair.second.PrivateElemInit) {
3287 const VarDecl *OriginalVD = Pair.second.Original;
3292 const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);
3293 if (IsTargetTask && !SharedField) {
3294 assert(isa<ImplicitParamDecl>(OriginalVD) &&
3297 ->getNumParams() == 0 &&
3298 isa<TranslationUnitDecl>(
3300 ->getDeclContext()) &&
3301 "Expected artificial target data variable.");
3304 }
else if (ForDup) {
3308 C.getDeclAlign(OriginalVD)),
3312 Pair.second.Original->getCanonicalDecl()) > 0 ||
3314 SharedRefLValue = CGF.
EmitLValue(Pair.second.OriginalRef);
3317 InlinedOpenMPRegionRAII Region(
3320 SharedRefLValue = CGF.
EmitLValue(Pair.second.OriginalRef);
3333 [&CGF, Elem,
Init, &CapturesInfo](
Address DestElement,
3336 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3337 InitScope.addPrivate(Elem, SrcElement);
3338 (void)InitScope.Privatize();
3340 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
3341 CGF, &CapturesInfo);
3342 CGF.EmitAnyExprToMem(Init, DestElement,
3343 Init->getType().getQualifiers(),
3348 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3349 InitScope.addPrivate(Elem, SharedRefLValue.
getAddress(CGF));
3350 (void)InitScope.Privatize();
3351 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
3366 bool InitRequired =
false;
3367 for (
const PrivateDataTy &Pair : Privates) {
3368 if (Pair.second.isLocalPrivate())
3370 const VarDecl *VD = Pair.second.PrivateCopy;
3372 InitRequired = InitRequired || (isa_and_nonnull<CXXConstructExpr>(
Init) &&
3377 return InitRequired;
3394 QualType KmpTaskTWithPrivatesPtrQTy,
3402 KmpTaskTWithPrivatesPtrQTy,
3405 KmpTaskTWithPrivatesPtrQTy,
3409 Args.push_back(&DstArg);
3410 Args.push_back(&SrcArg);
3411 Args.push_back(&LastprivArg);
3412 const auto &TaskDupFnInfo =
3416 auto *TaskDup = llvm::Function::Create(
3417 TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.
getModule());
3419 TaskDup->setDoesNotRecurse();
3429 auto LIFI = std::next(KmpTaskTQTyRD->
field_begin(), KmpTaskTLastIter);
3431 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
3439 assert(!Privates.empty());
3441 if (!
Data.FirstprivateVars.empty()) {
3446 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
3454 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
3455 SharedsTy, SharedsPtrTy,
Data, Privates,
true);
3465 for (
const PrivateDataTy &
P : Privates) {
3466 if (
P.second.isLocalPrivate())
3468 QualType Ty =
P.second.Original->getType().getNonReferenceType();
3477class OMPIteratorGeneratorScope final
3478 :
public CodeGenFunction::OMPPrivateScope {
3483 OMPIteratorGeneratorScope() =
delete;
3484 OMPIteratorGeneratorScope(OMPIteratorGeneratorScope &) =
delete;
3512 CodeGenFunction::JumpDest &ContDest =
3514 CodeGenFunction::JumpDest &ExitDest =
3517 llvm::Value *N = Uppers[I];
3525 ? CGF.
Builder.CreateICmpSLT(CVal, N)
3526 : CGF.
Builder.CreateICmpULT(CVal, N);
3528 CGF.
Builder.CreateCondBr(Cmp, BodyBB, ExitDest.getBlock());
3535 ~OMPIteratorGeneratorScope() {
3545 CGF.
EmitBlock(ExitDests[I - 1].getBlock(), I == 1);
3551static std::pair<llvm::Value *, llvm::Value *>
3553 const auto *OASE = dyn_cast<OMPArrayShapingExpr>(E);
3556 const Expr *
Base = OASE->getBase();
3561 llvm::Value *SizeVal;
3564 SizeVal = CGF.
getTypeSize(OASE->getBase()->getType()->getPointeeType());
3565 for (
const Expr *SE : OASE->getDimensions()) {
3569 SizeVal = CGF.
Builder.CreateNUWMul(SizeVal, Sz);
3571 }
else if (
const auto *ASE =
3576 llvm::Value *UpAddr = CGF.
Builder.CreateConstGEP1_32(
3578 llvm::Value *LowIntPtr = CGF.
Builder.CreatePtrToInt(Addr, CGF.
SizeTy);
3579 llvm::Value *UpIntPtr = CGF.
Builder.CreatePtrToInt(UpAddr, CGF.
SizeTy);
3580 SizeVal = CGF.
Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
3584 return std::make_pair(Addr, SizeVal);
3589 QualType FlagsTy =
C.getIntTypeForBitwidth(32,
false);
3590 if (KmpTaskAffinityInfoTy.
isNull()) {
3592 C.buildImplicitRecord(
"kmp_task_affinity_info_t");
3598 KmpTaskAffinityInfoTy =
C.getRecordType(KmpAffinityInfoRD);
3605 llvm::Function *TaskFunction,
QualType SharedsTy,
3610 const auto *I =
Data.PrivateCopies.begin();
3611 for (
const Expr *E :
Data.PrivateVars) {
3612 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3613 Privates.emplace_back(
3615 PrivateHelpersTy(E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
3619 I =
Data.FirstprivateCopies.begin();
3620 const auto *IElemInitRef =
Data.FirstprivateInits.begin();
3621 for (
const Expr *E :
Data.FirstprivateVars) {
3622 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3623 Privates.emplace_back(
3626 E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
3627 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
3631 I =
Data.LastprivateCopies.begin();
3632 for (
const Expr *E :
Data.LastprivateVars) {
3633 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3634 Privates.emplace_back(
3636 PrivateHelpersTy(E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
3644 Privates.emplace_back(
C.getDeclAlign(VD), PrivateHelpersTy(VD));
3646 llvm::stable_sort(Privates,
3647 [](
const PrivateDataTy &L,
const PrivateDataTy &R) {
3648 return L.first > R.first;
3650 QualType KmpInt32Ty =
C.getIntTypeForBitwidth(32, 1);
3664 "Expected taskloop, task or target directive");
3673 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
3675 QualType KmpTaskTWithPrivatesQTy =
C.getRecordType(KmpTaskTWithPrivatesQTyRD);
3676 QualType KmpTaskTWithPrivatesPtrQTy =
3677 C.getPointerType(KmpTaskTWithPrivatesQTy);
3678 llvm::Type *KmpTaskTWithPrivatesTy = CGF.
ConvertType(KmpTaskTWithPrivatesQTy);
3679 llvm::Type *KmpTaskTWithPrivatesPtrTy =
3680 KmpTaskTWithPrivatesTy->getPointerTo();
3681 llvm::Value *KmpTaskTWithPrivatesTySize =
3683 QualType SharedsPtrTy =
C.getPointerType(SharedsTy);
3686 llvm::Value *TaskPrivatesMap =
nullptr;
3687 llvm::Type *TaskPrivatesMapTy =
3688 std::next(TaskFunction->arg_begin(), 3)->getType();
3689 if (!Privates.empty()) {
3690 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
3694 TaskPrivatesMap, TaskPrivatesMapTy);
3696 TaskPrivatesMap = llvm::ConstantPointerNull::get(
3697 cast<llvm::PointerType>(TaskPrivatesMapTy));
3703 KmpTaskTWithPrivatesQTy,
KmpTaskTQTy, SharedsPtrTy, TaskFunction,
3715 DestructorsFlag = 0x8,
3716 PriorityFlag = 0x20,
3717 DetachableFlag = 0x40,
3719 unsigned Flags =
Data.Tied ? TiedFlag : 0;
3720 bool NeedsCleanup =
false;
3721 if (!Privates.empty()) {
3725 Flags = Flags | DestructorsFlag;
3727 if (
Data.Priority.getInt())
3728 Flags = Flags | PriorityFlag;
3730 Flags = Flags | DetachableFlag;
3731 llvm::Value *TaskFlags =
3732 Data.Final.getPointer()
3733 ? CGF.
Builder.CreateSelect(
Data.Final.getPointer(),
3734 CGF.
Builder.getInt32(FinalFlag),
3736 : CGF.
Builder.getInt32(
Data.Final.getInt() ? FinalFlag : 0);
3737 TaskFlags = CGF.
Builder.CreateOr(TaskFlags, CGF.
Builder.getInt32(Flags));
3738 llvm::Value *SharedsSize =
CGM.
getSize(
C.getTypeSizeInChars(SharedsTy));
3740 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
3743 llvm::Value *NewTask;
3750 llvm::Value *DeviceID;
3755 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
3756 AllocArgs.push_back(DeviceID);
3777 llvm::Value *Tid =
getThreadID(CGF, DC->getBeginLoc());
3778 Tid = CGF.
Builder.CreateIntCast(Tid, CGF.
IntTy,
false);
3781 CGM.
getModule(), OMPRTL___kmpc_task_allow_completion_event),
3782 {Loc, Tid, NewTask});
3793 llvm::Value *NumOfElements =
nullptr;
3794 unsigned NumAffinities = 0;
3796 if (
const Expr *Modifier =
C->getModifier()) {
3797 const auto *IE = cast<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts());
3798 for (
unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
3802 NumOfElements ? CGF.
Builder.CreateNUWMul(NumOfElements, Sz) : Sz;
3805 NumAffinities +=
C->varlist_size();
3810 enum RTLAffinityInfoFieldsTy { BaseAddr, Len, Flags };
3812 QualType KmpTaskAffinityInfoArrayTy;
3813 if (NumOfElements) {
3814 NumOfElements = CGF.
Builder.CreateNUWAdd(
3815 llvm::ConstantInt::get(CGF.
SizeTy, NumAffinities), NumOfElements);
3818 C.getIntTypeForBitwidth(
C.getTypeSize(
C.getSizeType()), 0),
3822 KmpTaskAffinityInfoArrayTy =
C.getVariableArrayType(
3830 NumOfElements = CGF.
Builder.CreateIntCast(NumOfElements, CGF.
Int32Ty,
3833 KmpTaskAffinityInfoArrayTy =
C.getConstantArrayType(
3835 llvm::APInt(
C.getTypeSize(
C.getSizeType()), NumAffinities),
nullptr,
3838 CGF.
CreateMemTemp(KmpTaskAffinityInfoArrayTy,
".affs.arr.addr");
3840 NumOfElements = llvm::ConstantInt::get(
CGM.
Int32Ty, NumAffinities,
3847 bool HasIterator =
false;
3849 if (
C->getModifier()) {
3853 for (
const Expr *E :
C->varlists()) {
3862 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
3867 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
3881 const Expr *Modifier =
C->getModifier();
3884 OMPIteratorGeneratorScope IteratorScope(
3886 for (
const Expr *E :
C->varlists()) {
3895 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
3900 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
3902 Idx = CGF.
Builder.CreateNUWAdd(
3903 Idx, llvm::ConstantInt::get(Idx->getType(), 1));
3918 CGM.
getModule(), OMPRTL___kmpc_omp_reg_task_with_affinity),
3919 {LocRef, GTid, NewTask, NumOfElements, AffinListPtr});
3921 llvm::Value *NewTaskNewTaskTTy =
3923 NewTask, KmpTaskTWithPrivatesPtrTy);
3925 KmpTaskTWithPrivatesQTy);
3936 *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds)),
3945 if (!Privates.empty()) {
3947 SharedsTy, SharedsPtrTy,
Data, Privates,
3952 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
3953 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy,
Data, Privates,
3954 !
Data.LastprivateVars.empty());
3958 enum {
Priority = 0, Destructors = 1 };
3960 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
3962 (*FI)->getType()->getAsUnionType()->getDecl();
3965 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
3966 KmpTaskTWithPrivatesQTy);
3969 Data1LV, *std::next(KmpCmplrdataUD->
field_begin(), Destructors));
3975 if (
Data.Priority.getInt()) {
3977 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
3982 Result.NewTask = NewTask;
3983 Result.TaskEntry = TaskEntry;
3984 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
3986 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
3992 RTLDependenceKindTy DepKind;
3994 case OMPC_DEPEND_in:
3995 DepKind = RTLDependenceKindTy::DepIn;
3998 case OMPC_DEPEND_out:
3999 case OMPC_DEPEND_inout:
4000 DepKind = RTLDependenceKindTy::DepInOut;
4002 case OMPC_DEPEND_mutexinoutset:
4003 DepKind = RTLDependenceKindTy::DepMutexInOutSet;
4005 case OMPC_DEPEND_inoutset:
4006 DepKind = RTLDependenceKindTy::DepInOutSet;
4008 case OMPC_DEPEND_outallmemory:
4009 DepKind = RTLDependenceKindTy::DepOmpAllMem;
4011 case OMPC_DEPEND_source:
4012 case OMPC_DEPEND_sink:
4013 case OMPC_DEPEND_depobj:
4014 case OMPC_DEPEND_inoutallmemory:
4016 llvm_unreachable(
"Unknown task dependence type");
4024 FlagsTy =
C.getIntTypeForBitwidth(
C.getTypeSize(
C.BoolTy),
false);
4025 if (KmpDependInfoTy.
isNull()) {
4026 RecordDecl *KmpDependInfoRD =
C.buildImplicitRecord(
"kmp_depend_info");
4032 KmpDependInfoTy =
C.getRecordType(KmpDependInfoRD);
4036std::pair<llvm::Value *, LValue>
4050 Base.getAddress(CGF),
4051 llvm::ConstantInt::get(CGF.
IntPtrTy, -1,
true));
4058 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4060 return std::make_pair(NumDeps,
Base);
4064 llvm::PointerUnion<unsigned *, LValue *> Pos,
4075 OMPIteratorGeneratorScope IteratorScope(
4076 CGF, cast_or_null<OMPIteratorExpr>(
4077 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4079 for (
const Expr *E :
Data.DepExprs) {
4088 Addr = llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4089 Size = llvm::ConstantInt::get(CGF.
SizeTy, 0);
4092 if (
unsigned *
P = Pos.dyn_cast<
unsigned *>()) {
4096 assert(E &&
"Expected a non-null expression");
4106 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4111 static_cast<unsigned int>(RTLDependInfoFields::Len)));
4118 static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4120 llvm::ConstantInt::get(LLVMFlagsTy,
static_cast<unsigned int>(DepKind)),
4122 if (
unsigned *
P = Pos.dyn_cast<
unsigned *>()) {
4127 Idx = CGF.
Builder.CreateNUWAdd(Idx,
4128 llvm::ConstantInt::get(Idx->getType(), 1));
4137 assert(
Data.DepKind == OMPC_DEPEND_depobj &&
4138 "Expected depobj dependency kind.");
4143 OMPIteratorGeneratorScope IteratorScope(
4144 CGF, cast_or_null<OMPIteratorExpr>(
4145 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4147 for (
const Expr *E :
Data.DepExprs) {
4148 llvm::Value *NumDeps;
4151 std::tie(NumDeps,
Base) =
4155 C.getUIntPtrType());
4159 llvm::Value *Add = CGF.
Builder.CreateNUWAdd(PrevVal, NumDeps);
4161 SizeLVals.push_back(NumLVal);
4164 for (
unsigned I = 0, E = SizeLVals.size(); I < E; ++I) {
4167 Sizes.push_back(Size);
4177 assert(
Data.DepKind == OMPC_DEPEND_depobj &&
4178 "Expected depobj dependency kind.");
4181 OMPIteratorGeneratorScope IteratorScope(
4182 CGF, cast_or_null<OMPIteratorExpr>(
4183 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4185 for (
unsigned I = 0, End =
Data.DepExprs.size(); I < End; ++I) {
4187 llvm::Value *NumDeps;
4190 std::tie(NumDeps,
Base) =
4194 llvm::Value *Size = CGF.
Builder.CreateNUWMul(
4203 llvm::Value *Add = CGF.
Builder.CreateNUWAdd(Pos, NumDeps);
4219 llvm::Value *NumOfElements =
nullptr;
4220 unsigned NumDependencies = std::accumulate(
4221 Dependencies.begin(), Dependencies.end(), 0,
4223 return D.DepKind == OMPC_DEPEND_depobj
4225 : (V + (D.IteratorExpr ? 0 : D.DepExprs.size()));
4229 bool HasDepobjDeps =
false;
4230 bool HasRegularWithIterators =
false;
4231 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4232 llvm::Value *NumOfRegularWithIterators =
4233 llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4237 if (D.
DepKind == OMPC_DEPEND_depobj) {
4240 for (llvm::Value *Size : Sizes) {
4241 NumOfDepobjElements =
4242 CGF.
Builder.CreateNUWAdd(NumOfDepobjElements, Size);
4244 HasDepobjDeps =
true;
4249 if (
const auto *IE = cast_or_null<OMPIteratorExpr>(D.
IteratorExpr)) {
4253 llvm::Value *NumClauseDeps = CGF.
Builder.CreateNUWMul(
4255 NumOfRegularWithIterators =
4256 CGF.
Builder.CreateNUWAdd(NumOfRegularWithIterators, NumClauseDeps);
4258 HasRegularWithIterators =
true;
4264 if (HasDepobjDeps || HasRegularWithIterators) {
4265 NumOfElements = llvm::ConstantInt::get(
CGM.
IntPtrTy, NumDependencies,
4267 if (HasDepobjDeps) {
4269 CGF.
Builder.CreateNUWAdd(NumOfDepobjElements, NumOfElements);
4271 if (HasRegularWithIterators) {
4273 CGF.
Builder.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements);
4276 Loc,
C.getIntTypeForBitwidth(64, 0),
4280 KmpDependInfoArrayTy =
4289 NumOfElements = CGF.
Builder.CreateIntCast(NumOfElements, CGF.
Int32Ty,
4292 KmpDependInfoArrayTy =
C.getConstantArrayType(
4298 NumOfElements = llvm::ConstantInt::get(
CGM.
Int32Ty, NumDependencies,
4302 for (
unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4303 if (Dependencies[I].DepKind == OMPC_DEPEND_depobj ||
4304 Dependencies[I].IteratorExpr)
4311 CGF.
CreateMemTemp(
C.getSizeType(),
"dep.counter.addr"),
C.getSizeType());
4313 for (
unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4314 if (Dependencies[I].DepKind == OMPC_DEPEND_depobj ||
4315 !Dependencies[I].IteratorExpr)
4321 if (HasDepobjDeps) {
4322 for (
unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4323 if (Dependencies[I].DepKind != OMPC_DEPEND_depobj)
4331 return std::make_pair(NumOfElements, DependenciesArray);
4342 unsigned NumDependencies = Dependencies.
DepExprs.size();
4353 llvm::Value *NumDepsVal;
4355 if (
const auto *IE =
4356 cast_or_null<OMPIteratorExpr>(Dependencies.
IteratorExpr)) {
4357 NumDepsVal = llvm::ConstantInt::get(CGF.
SizeTy, 1);
4361 NumDepsVal = CGF.
Builder.CreateNUWMul(NumDepsVal, Sz);
4363 Size = CGF.
Builder.CreateNUWAdd(llvm::ConstantInt::get(CGF.
SizeTy, 1),
4367 llvm::Value *RecSize =
CGM.
getSize(SizeInBytes);
4368 Size = CGF.
Builder.CreateNUWMul(Size, RecSize);
4372 QualType KmpDependInfoArrayTy =
C.getConstantArrayType(
4375 CharUnits Sz =
C.getTypeSizeInChars(KmpDependInfoArrayTy);
4377 NumDepsVal = llvm::ConstantInt::get(CGF.
IntPtrTy, NumDependencies);
4382 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4383 llvm::Value *Args[] = {ThreadID, Size, Allocator};
4388 Args,
".dep.arr.addr");
4391 Addr, KmpDependInfoLlvmTy->getPointerTo());
4392 DependenciesArray =
Address(Addr, KmpDependInfoLlvmTy, Align);
4399 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4401 llvm::PointerUnion<unsigned *, LValue *> Pos;
4418 return DependenciesArray;
4434 llvm::ConstantInt::get(CGF.
IntPtrTy, -1,
true));
4439 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4440 llvm::Value *Args[] = {ThreadID, DepObjAddr, Allocator};
4457 llvm::Value *NumDeps;
4464 Begin.getElementType(),
Begin.getPointer(), NumDeps);
4468 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
4470 llvm::PHINode *ElementPHI =
4471 CGF.
Builder.CreatePHI(
Begin.getType(), 2,
"omp.elementPast");
4472 ElementPHI->addIncoming(
Begin.getPointer(), EntryBB);
4475 Base.getTBAAInfo());
4480 static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4482 llvm::ConstantInt::get(LLVMFlagsTy,
static_cast<unsigned int>(DepKind)),
4488 ElementPHI->addIncoming(ElementNext.
getPointer(),
4489 CGF.
Builder.GetInsertBlock());
4490 llvm::Value *IsEmpty =
4492 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
4499 llvm::Function *TaskFunction,
4508 llvm::Value *NewTask =
Result.NewTask;
4509 llvm::Function *TaskEntry =
Result.TaskEntry;
4510 llvm::Value *NewTaskNewTaskTTy =
Result.NewTaskNewTaskTTy;
4515 llvm::Value *NumOfElements;
4516 std::tie(NumOfElements, DependenciesArray) =
4527 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
4528 llvm::Value *DepTaskArgs[7];
4529 if (!
Data.Dependences.empty()) {
4530 DepTaskArgs[0] = UpLoc;
4531 DepTaskArgs[1] = ThreadID;
4532 DepTaskArgs[2] = NewTask;
4533 DepTaskArgs[3] = NumOfElements;
4534 DepTaskArgs[4] = DependenciesArray.
getPointer();
4535 DepTaskArgs[5] = CGF.
Builder.getInt32(0);
4536 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4538 auto &&ThenCodeGen = [
this, &
Data, TDBase, KmpTaskTQTyRD, &TaskArgs,
4541 auto PartIdFI = std::next(KmpTaskTQTyRD->
field_begin(), KmpTaskTPartId);
4545 if (!
Data.Dependences.empty()) {
4558 Region->emitUntiedSwitch(CGF);
4561 llvm::Value *DepWaitTaskArgs[7];
4562 if (!
Data.Dependences.empty()) {
4563 DepWaitTaskArgs[0] = UpLoc;
4564 DepWaitTaskArgs[1] = ThreadID;
4565 DepWaitTaskArgs[2] = NumOfElements;
4566 DepWaitTaskArgs[3] = DependenciesArray.
getPointer();
4567 DepWaitTaskArgs[4] = CGF.
Builder.getInt32(0);
4568 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4569 DepWaitTaskArgs[6] =
4570 llvm::ConstantInt::get(CGF.
Int32Ty,
Data.HasNowaitClause);
4573 auto &&ElseCodeGen = [
this, &M, &TaskArgs, ThreadID, NewTaskNewTaskTTy,
4574 TaskEntry, &
Data, &DepWaitTaskArgs,
4581 if (!
Data.Dependences.empty())
4583 M, OMPRTL___kmpc_omp_taskwait_deps_51),
4586 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
4589 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
4590 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
4599 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
4600 M, OMPRTL___kmpc_omp_task_begin_if0),
4603 M, OMPRTL___kmpc_omp_task_complete_if0),
4619 llvm::Function *TaskFunction,
4639 IfVal = llvm::ConstantInt::getSigned(CGF.
IntTy, 1);
4644 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
4652 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
4660 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
4669 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
4670 if (
Data.Reductions) {
4676 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
4677 llvm::Value *TaskArgs[] = {
4685 llvm::ConstantInt::getSigned(
4687 llvm::ConstantInt::getSigned(
4689 ?
Data.Schedule.getInt() ? NumTasks : Grainsize
4691 Data.Schedule.getPointer()
4694 : llvm::ConstantInt::get(CGF.
Int64Ty, 0),
4697 : llvm::ConstantPointerNull::get(CGF.
VoidPtrTy)};
4716 const Expr *,
const Expr *)> &RedOpGen,
4717 const Expr *XExpr =
nullptr,
const Expr *EExpr =
nullptr,
4718 const Expr *UpExpr =
nullptr) {
4726 llvm::Value *NumElements = CGF.
emitArrayLength(ArrayTy, ElementTy, LHSAddr);
4728 llvm::Value *RHSBegin = RHSAddr.
getPointer();
4729 llvm::Value *LHSBegin = LHSAddr.
getPointer();
4731 llvm::Value *LHSEnd =
4736 llvm::Value *IsEmpty =
4737 CGF.
Builder.CreateICmpEQ(LHSBegin, LHSEnd,
"omp.arraycpy.isempty");
4738 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
4741 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
4746 llvm::PHINode *RHSElementPHI = CGF.
Builder.CreatePHI(
4747 RHSBegin->getType(), 2,
"omp.arraycpy.srcElementPast");
4748 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
4753 llvm::PHINode *LHSElementPHI = CGF.
Builder.CreatePHI(
4754 LHSBegin->getType(), 2,
"omp.arraycpy.destElementPast");
4755 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
4761 CodeGenFunction::OMPPrivateScope
Scope(CGF);
4762 Scope.addPrivate(LHSVar, LHSElementCurrent);
4763 Scope.addPrivate(RHSVar, RHSElementCurrent);
4765 RedOpGen(CGF, XExpr, EExpr, UpExpr);
4766 Scope.ForceCleanup();
4769 llvm::Value *LHSElementNext = CGF.
Builder.CreateConstGEP1_32(
4771 "omp.arraycpy.dest.element");
4772 llvm::Value *RHSElementNext = CGF.
Builder.CreateConstGEP1_32(
4774 "omp.arraycpy.src.element");
4777 CGF.
Builder.CreateICmpEQ(LHSElementNext, LHSEnd,
"omp.arraycpy.done");
4778 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
4779 LHSElementPHI->addIncoming(LHSElementNext, CGF.
Builder.GetInsertBlock());
4780 RHSElementPHI->addIncoming(RHSElementNext, CGF.
Builder.GetInsertBlock());
4790 const Expr *ReductionOp) {
4791 if (
const auto *CE = dyn_cast<CallExpr>(ReductionOp))
4792 if (
const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
4793 if (
const auto *DRE =
4794 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
4795 if (
const auto *DRD =
4796 dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
4797 std::pair<llvm::Function *, llvm::Function *>
Reduction =
4800 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE,
Func);
4808 StringRef ReducerName,
SourceLocation Loc, llvm::Type *ArgsElemType,
4819 Args.push_back(&LHSArg);
4820 Args.push_back(&RHSArg);
4825 llvm::GlobalValue::InternalLinkage, Name,
4828 Fn->setDoesNotRecurse();
4836 ArgsElemType->getPointerTo()),
4840 ArgsElemType->getPointerTo()),
4847 const auto *IPriv = Privates.begin();
4849 for (
unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
4850 const auto *RHSVar =
4851 cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl());
4853 const auto *LHSVar =
4854 cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl());
4856 QualType PrivTy = (*IPriv)->getType();
4864 const auto *OVE = cast<OpaqueValueExpr>(VLA->
getSizeExpr());
4871 IPriv = Privates.begin();
4872 const auto *ILHS = LHSExprs.begin();
4873 const auto *IRHS = RHSExprs.begin();
4874 for (
const Expr *E : ReductionOps) {
4875 if ((*IPriv)->getType()->isArrayType()) {
4877 const auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
4878 const auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
4880 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
4882 emitReductionCombiner(CGF, E);
4892 Scope.ForceCleanup();
4898 const Expr *ReductionOp,
4899 const Expr *PrivateRef,
4904 const auto *LHSVar = cast<VarDecl>(LHS->
getDecl());
4905 const auto *RHSVar = cast<VarDecl>(RHS->
getDecl());
4907 CGF, PrivateRef->
getType(), LHSVar, RHSVar,
4909 emitReductionCombiner(CGF, ReductionOp);
4926 bool WithNowait = Options.WithNowait;
4927 bool SimpleReduction = Options.SimpleReduction;
4966 if (SimpleReduction) {
4968 const auto *IPriv = Privates.begin();
4969 const auto *ILHS = LHSExprs.begin();
4970 const auto *IRHS = RHSExprs.begin();
4971 for (
const Expr *E : ReductionOps) {
4973 cast<DeclRefExpr>(*IRHS));
4983 auto Size = RHSExprs.size();
4984 for (
const Expr *E : Privates) {
4989 llvm::APInt ArraySize(32, Size);
4990 QualType ReductionArrayTy =
C.getConstantArrayType(
4994 CGF.
CreateMemTemp(ReductionArrayTy,
".omp.reduction.red_list");
4995 const auto *IPriv = Privates.begin();
4997 for (
unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
5003 if ((*IPriv)->getType()->isVariablyModifiedType()) {
5007 llvm::Value *Size = CGF.
Builder.CreateIntCast(
5020 Privates, LHSExprs, RHSExprs, ReductionOps);
5023 std::string Name =
getName({
"reduction"});
5030 llvm::Value *ReductionArrayTySize = CGF.
getTypeSize(ReductionArrayTy);
5033 llvm::Value *Args[] = {
5036 CGF.
Builder.getInt32(RHSExprs.size()),
5037 ReductionArrayTySize,
5045 WithNowait ? OMPRTL___kmpc_reduce_nowait : OMPRTL___kmpc_reduce),
5049 llvm::BasicBlock *DefaultBB = CGF.
createBasicBlock(
".omp.reduction.default");
5050 llvm::SwitchInst *SwInst =
5051 CGF.
Builder.CreateSwitch(Res, DefaultBB, 2);
5060 SwInst->addCase(CGF.
Builder.getInt32(1), Case1BB);
5064 llvm::Value *EndArgs[] = {
5069 auto &&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps](
5072 const auto *IPriv = Privates.begin();
5073 const auto *ILHS = LHSExprs.begin();
5074 const auto *IRHS = RHSExprs.begin();
5075 for (
const Expr *E : ReductionOps) {
5077 cast<DeclRefExpr>(*IRHS));
5084 CommonActionTy Action(
5085 nullptr, std::nullopt,
5087 CGM.
getModule(), WithNowait ? OMPRTL___kmpc_end_reduce_nowait
5088 : OMPRTL___kmpc_end_reduce),
5101 SwInst->addCase(CGF.
Builder.getInt32(2), Case2BB);
5104 auto &&AtomicCodeGen = [Loc, Privates, LHSExprs, RHSExprs, ReductionOps](
5106 const auto *ILHS = LHSExprs.begin();
5107 const auto *IRHS = RHSExprs.begin();
5108 const auto *IPriv = Privates.begin();
5109 for (
const Expr *E : ReductionOps) {
5110 const Expr *XExpr =
nullptr;
5111 const Expr *EExpr =
nullptr;
5112 const Expr *UpExpr =
nullptr;
5114 if (
const auto *BO = dyn_cast<BinaryOperator>(E)) {
5115 if (BO->getOpcode() == BO_Assign) {
5116 XExpr = BO->getLHS();
5117 UpExpr = BO->getRHS();
5121 const Expr *RHSExpr = UpExpr;
5124 if (
const auto *ACO = dyn_cast<AbstractConditionalOperator>(
5128 RHSExpr = ACO->getCond();
5130 if (
const auto *BORHS =
5132 EExpr = BORHS->getRHS();
5133 BO = BORHS->getOpcode();
5137 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5138 auto &&AtomicRedGen = [BO, VD,
5140 const Expr *EExpr,
const Expr *UpExpr) {
5141 LValue X = CGF.EmitLValue(XExpr);
5144 E = CGF.EmitAnyExpr(EExpr);
5145 CGF.EmitOMPAtomicSimpleUpdateExpr(
5147 llvm::AtomicOrdering::Monotonic, Loc,
5148 [&CGF, UpExpr, VD, Loc](
RValue XRValue) {
5150 Address LHSTemp = CGF.CreateMemTemp(VD->getType());
5151 CGF.emitOMPSimpleStore(
5152 CGF.MakeAddrLValue(LHSTemp, VD->
getType()), XRValue,
5153 VD->getType().getNonReferenceType(), Loc);
5156 return CGF.EmitAnyExpr(UpExpr);
5159 if ((*IPriv)->getType()->isArrayType()) {
5161 const auto *RHSVar =
5162 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5164 AtomicRedGen, XExpr, EExpr, UpExpr);
5167 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5174 std::string Name = RT.
getName({
"atomic_reduction"});
5183 if ((*IPriv)->getType()->isArrayType()) {
5184 const auto *LHSVar =
5185 cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5186 const auto *RHSVar =
5187 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5191 CritRedGen(CGF,
nullptr,
nullptr,
nullptr);
5202 llvm::Value *EndArgs[] = {
5207 CommonActionTy Action(
nullptr, std::nullopt,
5226 llvm::raw_svector_ostream Out(Buffer);
5230 D = cast<VarDecl>(cast<DeclRefExpr>(Ref)->getDecl());
5234 Out << Prefix << Name <<
"_"
5236 return std::string(Out.str());
5258 Args.emplace_back(&Param);
5259 Args.emplace_back(&ParamOrig);
5260 const auto &FnInfo =
5264 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5267 Fn->setDoesNotRecurse();
5275 llvm::Value *Size =
nullptr;
5318 const Expr *ReductionOp,
5320 const Expr *PrivateRef) {
5322 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(LHS)->getDecl());
5323 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(RHS)->getDecl());
5329 Args.emplace_back(&ParamInOut);
5330 Args.emplace_back(&ParamIn);
5331 const auto &FnInfo =
5335 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5338 Fn->setDoesNotRecurse();
5341 llvm::Value *Size =
nullptr;
5355 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5356 PrivateScope.addPrivate(
5363 C.getPointerType(LHSVD->getType())->castAs<
PointerType>()));
5364 PrivateScope.addPrivate(
5370 C.getPointerType(RHSVD->getType())->castAs<
PointerType>()));
5371 PrivateScope.Privatize();
5376 CGF, ReductionOp, PrivateRef, cast<DeclRefExpr>(LHS),
5377 cast<DeclRefExpr>(RHS));
5399 Args.emplace_back(&Param);
5400 const auto &FnInfo =
5404 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5407 Fn->setDoesNotRecurse();
5412 llvm::Value *Size =
nullptr;
5447 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_taskred_input_t");
5456 C, RD,
C.getIntTypeForBitwidth(32,
false));
5459 unsigned Size =
Data.ReductionVars.size();
5460 llvm::APInt ArraySize(64, Size);
5462 C.getConstantArrayType(RDType, ArraySize,
nullptr,
5467 Data.ReductionCopies,
Data.ReductionOps);
5468 for (
unsigned Cnt = 0; Cnt < Size; ++Cnt) {
5470 llvm::Value *Idxs[] = {llvm::ConstantInt::get(
CGM.
SizeTy, 0),
5471 llvm::ConstantInt::get(
CGM.
SizeTy, Cnt)};
5487 llvm::Value *SizeValInChars;
5488 llvm::Value *SizeVal;
5489 std::tie(SizeValInChars, SizeVal) = RCG.
getSizes(Cnt);
5495 bool DelayedCreation = !!SizeVal;
5507 llvm::Value *FiniAddr =
5508 Fini ? Fini : llvm::ConstantPointerNull::get(
CGM.
VoidPtrTy);
5513 CGM, Loc, RCG, Cnt,
Data.ReductionOps[Cnt], LHSExprs[Cnt],
5514 RHSExprs[Cnt],
Data.ReductionCopies[Cnt]);
5518 if (DelayedCreation) {
5520 llvm::ConstantInt::get(
CGM.
Int32Ty, 1,
true),
5526 if (
Data.IsReductionWithTaskMod) {
5532 llvm::Value *Args[] = {
5534 llvm::ConstantInt::get(
CGM.
IntTy,
Data.IsWorksharingReduction ? 1 : 0,
5536 llvm::ConstantInt::get(
CGM.
IntTy, Size,
true),
5545 llvm::Value *Args[] = {
5548 llvm::ConstantInt::get(
CGM.
IntTy, Size,
true),
5558 bool IsWorksharingReduction) {
5564 llvm::Value *Args[] = {IdentTLoc, GTid,
5566 IsWorksharingReduction ? 1 : 0,
5570 CGM.
getModule(), OMPRTL___kmpc_task_reduction_modifier_fini),
5593 llvm::Value *ReductionsPtr,
5606 CGM.
getModule(), OMPRTL___kmpc_task_reduction_get_th_data),
5624 llvm::Value *NumOfElements;
5625 std::tie(NumOfElements, DependenciesArray) =
5627 if (!
Data.Dependences.empty()) {
5628 llvm::Value *DepWaitTaskArgs[7];
5629 DepWaitTaskArgs[0] = UpLoc;
5630 DepWaitTaskArgs[1] = ThreadID;
5631 DepWaitTaskArgs[2] = NumOfElements;
5632 DepWaitTaskArgs[3] = DependenciesArray.
getPointer();
5633 DepWaitTaskArgs[4] = CGF.
Builder.getInt32(0);
5634 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
5635 DepWaitTaskArgs[6] =
5636 llvm::ConstantInt::get(CGF.
Int32Ty,
Data.HasNowaitClause);
5645 M, OMPRTL___kmpc_omp_taskwait_deps_51),
5652 llvm::Value *Args[] = {UpLoc, ThreadID};
5655 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_omp_taskwait),
5660 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
5661 Region->emitUntiedSwitch(CGF);
5670 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel,
5671 InnerKind != OMPD_critical &&
5672 InnerKind != OMPD_master &&
5673 InnerKind != OMPD_masked);
5688 RTCancelKind CancelKind = CancelNoreq;
5689 if (CancelRegion == OMPD_parallel)
5690 CancelKind = CancelParallel;
5691 else if (CancelRegion == OMPD_for)
5692 CancelKind = CancelLoop;
5693 else if (CancelRegion == OMPD_sections)
5694 CancelKind = CancelSections;
5696 assert(CancelRegion == OMPD_taskgroup);
5697 CancelKind = CancelTaskgroup;
5709 if (
auto *OMPRegionInfo =
5713 if (CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) {
5714 llvm::Value *Args[] = {
5729 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
5731 if (CancelRegion == OMPD_parallel)
5750 if (
auto *OMPRegionInfo =
5752 auto &&ThenGen = [
this, &M, Loc, CancelRegion,
5755 llvm::Value *Args[] = {
5759 llvm::Value *
Result = CGF.EmitRuntimeCall(
5760 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_cancel), Args);
5765 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
".cancel.exit");
5766 llvm::BasicBlock *ContBB = CGF.createBasicBlock(
".cancel.continue");
5767 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(
Result);
5768 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
5769 CGF.EmitBlock(ExitBB);
5770 if (CancelRegion == OMPD_parallel)
5774 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
5775 CGF.EmitBranchThroughCleanup(CancelDest);
5776 CGF.EmitBlock(ContBB,
true);
5794 OMPUsesAllocatorsActionTy(
5795 ArrayRef<std::pair<const Expr *, const Expr *>> Allocators)
5796 : Allocators(Allocators) {}
5800 for (
const auto &AllocatorData : Allocators) {
5802 CGF, AllocatorData.first, AllocatorData.second);
5808 for (
const auto &AllocatorData : Allocators) {
5810 AllocatorData.first);
5818 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
5820 assert(!ParentName.empty() &&
"Invalid target entry parent name!");
5824 for (
unsigned I = 0, E =
C->getNumberOfAllocators(); I < E; ++I) {
5831 OMPUsesAllocatorsActionTy UsesAllocatorAction(Allocators);
5834 IsOffloadEntry, CodeGen);
5838 const Expr *Allocator,
5839 const Expr *AllocatorTraits) {
5840 llvm::Value *ThreadId =
getThreadID(CGF, Allocator->getExprLoc());
5841 ThreadId = CGF.
Builder.CreateIntCast(ThreadId, CGF.
IntTy,
true);
5843 llvm::Value *MemSpaceHandle = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
5844 llvm::Value *NumTraits = llvm::ConstantInt::get(
5845 CGF.
IntTy, cast<ConstantArrayType>(
5848 .getLimitedValue());
5857 llvm::Value *AllocatorVal =
5860 {ThreadId, MemSpaceHandle, NumTraits, Traits});
5863 cast<DeclRefExpr>(Allocator->IgnoreParenImpCasts())->getDecl()));
5867 Allocator->getType(), Allocator->getExprLoc());
5872 const Expr *Allocator) {
5873 llvm::Value *ThreadId =
getThreadID(CGF, Allocator->getExprLoc());
5874 ThreadId = CGF.
Builder.CreateIntCast(ThreadId, CGF.
IntTy,
true);
5876 llvm::Value *AllocatorVal =
5880 Allocator->getExprLoc());
5883 OMPRTL___kmpc_destroy_allocator),
5884 {ThreadId, AllocatorVal});
5889 int32_t &MinThreadsVal, int32_t &MaxThreadsVal, int32_t &MinTeamsVal,
5890 int32_t &MaxTeamsVal) {
5897 for (
auto *A :
C->getAttrs()) {
5898 int32_t AttrMinThreadsVal = 1, AttrMaxThreadsVal = -1;
5899 int32_t AttrMinBlocksVal = 1, AttrMaxBlocksVal = -1;
5900 if (
auto *
Attr = dyn_cast<CUDALaunchBoundsAttr>(A))
5902 &AttrMinBlocksVal, &AttrMaxBlocksVal);
5903 else if (
auto *
Attr = dyn_cast<AMDGPUFlatWorkGroupSizeAttr>(A))
5905 nullptr,
Attr,
nullptr, &AttrMinThreadsVal,
5906 &AttrMaxThreadsVal);
5910 MinThreadsVal = std::max(MinThreadsVal, AttrMinThreadsVal);
5911 if (AttrMaxThreadsVal > 0)
5912 MaxThreadsVal = MaxThreadsVal > 0
5913 ? std::min(MaxThreadsVal, AttrMaxThreadsVal)
5914 : AttrMaxThreadsVal;
5915 MinTeamsVal = std::max(MinTeamsVal, AttrMinBlocksVal);
5916 if (AttrMaxBlocksVal > 0)
5917 MaxTeamsVal = MaxTeamsVal > 0 ? std::min(MaxTeamsVal, AttrMaxBlocksVal)
5925 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
5928 llvm::TargetRegionEntryInfo EntryInfo =
5932 llvm::OpenMPIRBuilder::FunctionGenCallback &&GenerateOutlinedFunction =
5933 [&CGF, &D, &CodeGen](StringRef EntryFnName) {
5936 CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName);
5941 OMPBuilder.emitTargetRegionFunction(EntryInfo, GenerateOutlinedFunction,
5942 IsOffloadEntry, OutlinedFn, OutlinedFnID);
5950 for (
auto *A :
C->getAttrs()) {
5951 if (
auto *
Attr = dyn_cast<AMDGPUWavesPerEUAttr>(A))
5970 while (
const auto *
C = dyn_cast_or_null<CompoundStmt>(Child)) {
5972 for (
const Stmt *S :
C->body()) {
5973 if (
const auto *E = dyn_cast<Expr>(S)) {
5978 if (isa<AsmStmt>(S) || isa<NullStmt>(S) || isa<OMPFlushDirective>(S) ||
5979 isa<OMPBarrierDirective>(S) || isa<OMPTaskyieldDirective>(S))
5982 if (
const auto *DS = dyn_cast<DeclStmt>(S)) {
5983 if (llvm::all_of(DS->decls(), [](
const Decl *D) {
5984 if (isa<EmptyDecl>(D) || isa<DeclContext>(D) ||
5985 isa<TypeDecl>(D) || isa<PragmaCommentDecl>(D) ||
5986 isa<PragmaDetectMismatchDecl>(D) || isa<UsingDecl>(D) ||
5987 isa<UsingDirectiveDecl>(D) ||
5988 isa<OMPDeclareReductionDecl>(D) ||
5989 isa<OMPThreadPrivateDecl>(D) || isa<OMPAllocateDecl>(D))
5991 const auto *VD = dyn_cast<VarDecl>(D);
5994 return VD->hasGlobalStorage() || !VD->isUsed();
6004 Child = Child->IgnoreContainers();
6011 int32_t &MaxTeamsVal) {
6015 "Expected target-based executable directive.");
6016 switch (DirectiveKind) {
6021 const Stmt *ChildStmt =
6023 if (
const auto *NestedDir =
6024 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
6027 const Expr *NumTeams =
6032 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();
6035 MinTeamsVal = MaxTeamsVal = 0;
6040 MinTeamsVal = MaxTeamsVal = 1;
6043 MinTeamsVal = MaxTeamsVal = 1;
6047 MinTeamsVal = MaxTeamsVal = -1;
6050 case OMPD_target_teams_loop:
6051 case OMPD_target_teams:
6052 case OMPD_target_teams_distribute:
6053 case OMPD_target_teams_distribute_simd:
6054 case OMPD_target_teams_distribute_parallel_for:
6055 case OMPD_target_teams_distribute_parallel_for_simd: {
6057 const Expr *NumTeams =
6061 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();
6064 MinTeamsVal = MaxTeamsVal = 0;
6067 case OMPD_target_parallel:
6068 case OMPD_target_parallel_for:
6069 case OMPD_target_parallel_for_simd:
6070 case OMPD_target_parallel_loop:
6071 case OMPD_target_simd:
6072 MinTeamsVal = MaxTeamsVal = 1;
6076 case OMPD_parallel_for:
6077 case OMPD_parallel_loop:
6078 case OMPD_parallel_master:
6079 case OMPD_parallel_sections:
6081 case OMPD_parallel_for_simd:
6083 case OMPD_cancellation_point:
6085 case OMPD_threadprivate:
6096 case OMPD_taskyield:
6099 case OMPD_taskgroup:
6105 case OMPD_target_data:
6106 case OMPD_target_exit_data:
6107 case OMPD_target_enter_data:
6108 case OMPD_distribute:
6109 case OMPD_distribute_simd:
6110 case OMPD_distribute_parallel_for:
6111 case OMPD_distribute_parallel_for_simd:
6112 case OMPD_teams_distribute:
6113 case OMPD_teams_distribute_simd:
6114 case OMPD_teams_distribute_parallel_for:
6115 case OMPD_teams_distribute_parallel_for_simd:
6116 case OMPD_target_update:
6117 case OMPD_declare_simd:
6118 case OMPD_declare_variant:
6119 case OMPD_begin_declare_variant:
6120 case OMPD_end_declare_variant:
6121 case OMPD_declare_target:
6122 case OMPD_end_declare_target:
6123 case OMPD_declare_reduction:
6124 case OMPD_declare_mapper:
6126 case OMPD_taskloop_simd:
6127 case OMPD_master_taskloop:
6128 case OMPD_master_taskloop_simd:
6129 case OMPD_parallel_master_taskloop:
6130 case OMPD_parallel_master_taskloop_simd:
6132 case OMPD_metadirective:
6138 llvm_unreachable(
"Unexpected directive kind.");
6144 "Clauses associated with the teams directive expected to be emitted "
6145 "only for the host!");
6147 int32_t MinNT = -1, MaxNT = -1;
6148 const Expr *NumTeams =
6150 if (NumTeams !=
nullptr) {
6153 switch (DirectiveKind) {
6156 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6160 return Bld.CreateIntCast(NumTeamsVal, CGF.
Int32Ty,
6163 case OMPD_target_teams:
6164 case OMPD_target_teams_distribute:
6165 case OMPD_target_teams_distribute_simd:
6166 case OMPD_target_teams_distribute_parallel_for:
6167 case OMPD_target_teams_distribute_parallel_for_simd: {
6171 return Bld.CreateIntCast(NumTeamsVal, CGF.
Int32Ty,
6179 assert(MinNT == MaxNT &&
"Num threads ranges require handling here.");
6180 return llvm::ConstantInt::get(CGF.
Int32Ty, MinNT);
6188 const Expr **E, int32_t &UpperBound,
6189 bool UpperBoundOnly, llvm::Value **CondVal) {
6192 const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child);
6199 if (CondVal && Dir->hasClausesOfKind<
OMPIfClause>()) {
6200 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6201 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6203 for (
const auto *
C : Dir->getClausesOfKind<
OMPIfClause>()) {
6204 if (
C->getNameModifier() == OMPD_unknown ||
6205 C->getNameModifier() == OMPD_parallel) {
6220 if (
const auto *PreInit =
6222 for (
const auto *I : PreInit->decls()) {
6223 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6226 CodeGenFunction::AutoVarEmission Emission =
6239 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6240 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6241 const auto *NumThreadsClause =
6243 const Expr *NTExpr = NumThreadsClause->getNumThreads();
6244 if (NTExpr->isIntegerConstantExpr(CGF.
getContext()))
6245 if (
auto Constant = NTExpr->getIntegerConstantExpr(CGF.
getContext()))
6248 ? Constant->getZExtValue()
6249 : std::min(UpperBound,
6250 static_cast<int32_t
>(Constant->getZExtValue()));
6253 if (UpperBound == -1)
6257 CodeGenFunction::LexicalScope
Scope(CGF, NTExpr->getSourceRange());
6258 if (
const auto *PreInit =
6259 cast_or_null<DeclStmt>(NumThreadsClause->getPreInitStmt())) {
6260 for (
const auto *I : PreInit->decls()) {
6261 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6264 CodeGenFunction::AutoVarEmission Emission =
6280 bool UpperBoundOnly, llvm::Value **CondVal,
const Expr **ThreadLimitExpr) {
6281 assert((!CGF.
getLangOpts().OpenMPIsTargetDevice || UpperBoundOnly) &&
6282 "Clauses associated with the teams directive expected to be emitted "
6283 "only for the host!");
6286 "Expected target-based executable directive.");
6288 const Expr *NT =
nullptr;
6289 const Expr **NTPtr = UpperBoundOnly ? nullptr : &NT;
6291 auto CheckForConstExpr = [&](
const Expr *E,
const Expr **EPtr) {
6294 UpperBound = UpperBound ? Constant->getZExtValue()
6295 : std::min(UpperBound,
6296 int32_t(Constant->getZExtValue()));
6300 if (UpperBound == -1)
6306 auto ReturnSequential = [&]() {
6311 switch (DirectiveKind) {
6314 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6320 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6322 ThreadLimitClause = TLC;
6323 if (ThreadLimitExpr) {
6324 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6327 CGF, ThreadLimitClause->getThreadLimit()->getSourceRange());
6328 if (
const auto *PreInit =
6329 cast_or_null<DeclStmt>(ThreadLimitClause->getPreInitStmt())) {
6330 for (
const auto *I : PreInit->decls()) {
6331 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6343 if (ThreadLimitClause)
6344 CheckForConstExpr(ThreadLimitClause->getThreadLimit(), ThreadLimitExpr);
6345 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6348 CS = Dir->getInnermostCapturedStmt();
6351 Dir = dyn_cast_or_null<OMPExecutableDirective>(Child);
6354 CS = Dir->getInnermostCapturedStmt();
6355 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6357 return ReturnSequential();
6361 case OMPD_target_teams: {
6365 CheckForConstExpr(ThreadLimitClause->getThreadLimit(), ThreadLimitExpr);
6368 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6371 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6372 if (Dir->getDirectiveKind() == OMPD_distribute) {
6373 CS = Dir->getInnermostCapturedStmt();
6374 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6379 case OMPD_target_teams_distribute:
6383 CheckForConstExpr(ThreadLimitClause->getThreadLimit(), ThreadLimitExpr);
6386 UpperBoundOnly, CondVal);
6388 case OMPD_target_teams_loop:
6389 case OMPD_target_parallel_loop:
6390 case OMPD_target_parallel:
6391 case OMPD_target_parallel_for:
6392 case OMPD_target_parallel_for_simd:
6393 case OMPD_target_teams_distribute_parallel_for:
6394 case OMPD_target_teams_distribute_parallel_for_simd: {
6398 if (
C->getNameModifier() == OMPD_unknown ||
6399 C->getNameModifier() == OMPD_parallel) {
6409 return ReturnSequential();
6419 CheckForConstExpr(ThreadLimitClause->getThreadLimit(), ThreadLimitExpr);
6424 CheckForConstExpr(NumThreadsClause->getNumThreads(),
nullptr);
6425 return NumThreadsClause->getNumThreads();
6429 case OMPD_target_teams_distribute_simd:
6430 case OMPD_target_simd:
6431 return ReturnSequential();
6435 llvm_unreachable(
"Unsupported directive kind.");
6440 llvm::Value *NumThreadsVal =
nullptr;
6441 llvm::Value *CondVal =
nullptr;
6442 llvm::Value *ThreadLimitVal =
nullptr;
6443 const Expr *ThreadLimitExpr =
nullptr;
6444 int32_t UpperBound = -1;
6447 CGF, D, UpperBound,
false, &CondVal,
6451 if (ThreadLimitExpr) {
6454 ThreadLimitVal = CGF.
Builder.CreateIntCast(ThreadLimitVal, CGF.
Int32Ty,
6459 if (UpperBound == 1) {
6460 NumThreadsVal = CGF.
Builder.getInt32(UpperBound);
6463 NumThreadsVal = CGF.
Builder.CreateIntCast(NumThreadsVal, CGF.
Int32Ty,
6465 }
else if (ThreadLimitVal) {
6468 NumThreadsVal = ThreadLimitVal;
6469 ThreadLimitVal =
nullptr;
6472 assert(!ThreadLimitVal &&
"Default not applicable with thread limit value");
6473 NumThreadsVal = CGF.
Builder.getInt32(0);
6480 NumThreadsVal = CGF.
Builder.CreateSelect(CondVal, NumThreadsVal,
6486 if (ThreadLimitVal) {
6487 NumThreadsVal = CGF.
Builder.CreateSelect(
6488 CGF.
Builder.CreateICmpULT(ThreadLimitVal, NumThreadsVal),
6489 ThreadLimitVal, NumThreadsVal);
6492 return NumThreadsVal;
6502class MappableExprsHandler {
6505 static unsigned getFlagMemberOffset() {
6506 unsigned Offset = 0;
6507 for (uint64_t Remain =
6508 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
6509 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
6510 !(Remain & 1); Remain = Remain >> 1)
6517 class MappingExprInfo {
6522 const Expr *MapExpr =
nullptr;
6525 MappingExprInfo(
const ValueDecl *MapDecl,
const Expr *MapExpr =
nullptr)
6526 : MapDecl(MapDecl), MapExpr(MapExpr) {}
6528 const ValueDecl *getMapDecl()
const {
return MapDecl; }
6529 const Expr *getMapExpr()
const {
return MapExpr; }
6532 using DeviceInfoTy = llvm::OpenMPIRBuilder::DeviceInfoTy;
6533 using MapBaseValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
6534 using MapValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
6535 using MapFlagsArrayTy = llvm::OpenMPIRBuilder::MapFlagsArrayTy;
6536 using MapDimArrayTy = llvm::OpenMPIRBuilder::MapDimArrayTy;
6537 using MapNonContiguousArrayTy =
6538 llvm::OpenMPIRBuilder::MapNonContiguousArrayTy;
6545 struct MapCombinedInfoTy : llvm::OpenMPIRBuilder::MapInfosTy {
6546 MapExprsArrayTy Exprs;
6547 MapValueDeclsArrayTy Mappers;
6548 MapValueDeclsArrayTy DevicePtrDecls;
6551 void append(MapCombinedInfoTy &CurInfo) {
6552 Exprs.append(CurInfo.Exprs.begin(), CurInfo.Exprs.end());
6553 DevicePtrDecls.append(CurInfo.DevicePtrDecls.begin(),
6554 CurInfo.DevicePtrDecls.end());
6555 Mappers.append(CurInfo.Mappers.begin(), CurInfo.Mappers.end());
6556 llvm::OpenMPIRBuilder::MapInfosTy::append(CurInfo);
6564 struct StructRangeInfoTy {
6565 MapCombinedInfoTy PreliminaryMapData;
6566 std::pair<
unsigned ,
Address > LowestElem = {
6568 std::pair<
unsigned ,
Address > HighestElem = {
6572 bool IsArraySection =
false;
6573 bool HasCompleteRecord =
false;
6583 bool ReturnDevicePointer =
false;
6584 bool IsImplicit =
false;
6586 const Expr *VarRef =
nullptr;
6587 bool ForDeviceAddr =
false;
6589 MapInfo() =
default;
6595 bool ReturnDevicePointer,
bool IsImplicit,
6596 const ValueDecl *Mapper =
nullptr,
const Expr *VarRef =
nullptr,
6597 bool ForDeviceAddr =
false)
6598 : Components(Components), MapType(MapType), MapModifiers(MapModifiers),
6599 MotionModifiers(MotionModifiers),
6600 ReturnDevicePointer(ReturnDevicePointer), IsImplicit(IsImplicit),
6601 Mapper(Mapper), VarRef(VarRef), ForDeviceAddr(ForDeviceAddr) {}
6607 struct DeferredDevicePtrEntryTy {
6608 const Expr *IE =
nullptr;
6610 bool ForDeviceAddr =
false;
6612 DeferredDevicePtrEntryTy(
const Expr *IE,
const ValueDecl *VD,
6614 : IE(IE), VD(VD), ForDeviceAddr(ForDeviceAddr) {}
6629 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>,
bool> FirstPrivateDecls;
6646 llvm::DenseMap<const ValueDecl *, const OMPMapClause *> LambdasMap;
6648 llvm::Value *getExprTypeSize(
const Expr *E)
const {
6652 if (
const auto *OAE = dyn_cast<OMPArrayShapingExpr>(E)) {
6654 CGF.
getTypeSize(OAE->getBase()->getType()->getPointeeType());
6655 for (
const Expr *SE : OAE->getDimensions()) {
6672 if (
const auto *OAE = dyn_cast<OMPArraySectionExpr>(E)) {
6674 OAE->getBase()->IgnoreParenImpCasts())
6680 if (!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
6681 !OAE->getLowerBound())
6684 llvm::Value *ElemSize;
6686 ElemSize = CGF.
getTypeSize(PTy->getPointeeType().getCanonicalType());
6688 const auto *ATy = cast<ArrayType>(BaseTy.
getTypePtr());
6689 assert(ATy &&
"Expecting array type if not a pointer type.");
6690 ElemSize = CGF.
getTypeSize(ATy->getElementType().getCanonicalType());
6695 if (!OAE->getLength() && OAE->getColonLocFirst().isInvalid())
6698 if (
const Expr *LenExpr = OAE->getLength()) {
6702 LenExpr->getExprLoc());
6703 return CGF.
Builder.CreateNUWMul(LengthVal, ElemSize);
6705 assert(!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
6706 OAE->getLowerBound() &&
"expected array_section[lb:].");
6712 OAE->getLowerBound()->getExprLoc());
6713 LBVal = CGF.
Builder.CreateNUWMul(LBVal, ElemSize);
6714 llvm::Value *Cmp = CGF.
Builder.CreateICmpUGT(LengthVal, LBVal);
6715 llvm::Value *TrueVal = CGF.
Builder.CreateNUWSub(LengthVal, LBVal);
6716 LengthVal = CGF.
Builder.CreateSelect(
6717 Cmp, TrueVal, llvm::ConstantInt::get(CGF.
SizeTy, 0));
6727 OpenMPOffloadMappingFlags getMapTypeBits(
6730 bool AddPtrFlag,
bool AddIsTargetParamFlag,
bool IsNonContiguous)
const {
6731 OpenMPOffloadMappingFlags Bits =
6732 IsImplicit ? OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT
6733 : OpenMPOffloadMappingFlags::OMP_MAP_NONE;
6735 case OMPC_MAP_alloc:
6736 case OMPC_MAP_release:
6743 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO;
6746 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_FROM;
6748 case OMPC_MAP_tofrom:
6749 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO |
6750 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
6752 case OMPC_MAP_delete:
6753 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_DELETE;
6756 llvm_unreachable(
"Unexpected map type!");
6759 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
6760 if (AddIsTargetParamFlag)
6761 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
6762 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_always))
6763 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS;
6764 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_close))
6765 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_CLOSE;
6766 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_present) ||
6767 llvm::is_contained(MotionModifiers, OMPC_MOTION_MODIFIER_present))
6768 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
6769 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_ompx_hold))
6770 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
6771 if (IsNonContiguous)
6772 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG;
6778 bool isFinalArraySectionExpression(
const Expr *E)
const {
6779 const auto *OASE = dyn_cast<OMPArraySectionExpr>(E);
6786 if (OASE->getColonLocFirst().isInvalid())
6789 const Expr *Length = OASE->getLength();
6796 OASE->getBase()->IgnoreParenImpCasts())
6798 if (
const auto *ATy = dyn_cast<ConstantArrayType>(BaseQTy.
getTypePtr()))
6799 return ATy->getSExtSize() != 1;
6808 if (!Length->EvaluateAsInt(Result, CGF.
getContext()))
6811 llvm::APSInt ConstLength = Result.Val.getInt();
6812 return ConstLength.getSExtValue() != 1;
6820 void generateInfoForComponentList(
6824 MapCombinedInfoTy &CombinedInfo,
6825 MapCombinedInfoTy &StructBaseCombinedInfo,
6826 StructRangeInfoTy &PartialStruct,
bool IsFirstComponentList,
6827 bool IsImplicit,
bool GenerateAllInfoForClauses,
6828 const ValueDecl *Mapper =
nullptr,
bool ForDeviceAddr =
false,
6829 const ValueDecl *BaseDecl =
nullptr,
const Expr *MapExpr =
nullptr,
6831 OverlappedElements = std::nullopt)
const {
7009 bool IsCaptureFirstInfo = IsFirstComponentList;
7013 bool RequiresReference =
false;
7016 auto CI = Components.rbegin();
7017 auto CE = Components.rend();
7022 bool IsExpressionFirstInfo =
true;
7023 bool FirstPointerInComplexData =
false;
7025 const Expr *AssocExpr = I->getAssociatedExpression();
7026 const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr);
7027 const auto *OASE = dyn_cast<OMPArraySectionExpr>(AssocExpr);
7028 const auto *OAShE = dyn_cast<OMPArrayShapingExpr>(AssocExpr);
7030 if (isa<MemberExpr>(AssocExpr)) {
7034 }
else if ((AE && isa<CXXThisExpr>(AE->getBase()->IgnoreParenImpCasts())) ||
7036 isa<CXXThisExpr>(OASE->getBase()->IgnoreParenImpCasts()))) {
7039 isa<CXXThisExpr>(OAShE->getBase()->IgnoreParenCasts())) {
7048 if (
const auto *VD =
7049 dyn_cast_or_null<VarDecl>(I->getAssociatedDeclaration())) {
7050 if (std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
7051 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) {
7052 if ((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
7053 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
7054 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
7056 RequiresReference =
true;
7066 I->getAssociatedDeclaration()->getType().getNonReferenceType();
7071 const auto *VD = dyn_cast<VarDecl>(I->getAssociatedDeclaration());
7076 FirstPointerInComplexData =
true;
7095 bool ShouldBeMemberOf =
false;
7110 bool IsNonContiguous = CombinedInfo.NonContigInfo.IsNonContiguous;
7111 bool IsPrevMemberReference =
false;
7118 bool IsMappingWholeStruct =
true;
7119 if (!GenerateAllInfoForClauses) {
7120 IsMappingWholeStruct =
false;
7122 for (
auto TempI = I; TempI != CE; ++TempI) {
7124 dyn_cast<MemberExpr>(TempI->getAssociatedExpression());
7126 IsMappingWholeStruct =
false;
7132 for (; I != CE; ++I) {
7134 if (!EncounteredME) {
7135 EncounteredME = dyn_cast<MemberExpr>(I->getAssociatedExpression());
7138 if (EncounteredME) {
7139 ShouldBeMemberOf =
true;
7142 if (FirstPointerInComplexData) {
7144 ->getAssociatedDeclaration()
7146 .getNonReferenceType();
7148 FirstPointerInComplexData =
false;
7153 auto Next = std::next(I);
7163 bool IsFinalArraySection =
7165 isFinalArraySectionExpression(I->getAssociatedExpression());
7169 const ValueDecl *MapDecl = (I->getAssociatedDeclaration())
7170 ? I->getAssociatedDeclaration()
7172 MapExpr = (I->getAssociatedExpression()) ? I->getAssociatedExpression()
7179 dyn_cast<OMPArraySectionExpr>(I->getAssociatedExpression());
7181 dyn_cast<OMPArrayShapingExpr>(I->getAssociatedExpression());
7182 const auto *UO = dyn_cast<UnaryOperator>(I->getAssociatedExpression());
7183 const auto *BO = dyn_cast<BinaryOperator>(I->getAssociatedExpression());
7189 I->getAssociatedExpression()->getType()->isAnyPointerType();
7190 bool IsMemberReference = isa<MemberExpr>(I->getAssociatedExpression()) &&
7193 bool IsNonDerefPointer = IsPointer &&
7194 !(UO && UO->getOpcode() != UO_Deref) && !BO &&
7200 if (Next == CE || IsMemberReference || IsNonDerefPointer ||
7201 IsFinalArraySection) {
7204 assert((Next == CE ||
7205 isa<MemberExpr>(Next->getAssociatedExpression()) ||
7206 isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) ||
7207 isa<OMPArraySectionExpr>(Next->getAssociatedExpression()) ||
7208 isa<OMPArrayShapingExpr>(Next->getAssociatedExpression()) ||
7209 isa<UnaryOperator>(Next->getAssociatedExpression()) ||
7210 isa<BinaryOperator>(Next->getAssociatedExpression())) &&
7211 "Unexpected expression");
7217 const Expr *BaseExpr = E->getBase();
7237 OAShE->getBase()->getType()->getPointeeType()),
7239 OAShE->getBase()->getType()));
7240 }
else if (IsMemberReference) {
7241 const auto *ME = cast<MemberExpr>(I->getAssociatedExpression());
7242 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
7244 BaseLVal, cast<FieldDecl>(MapDecl))
7257 bool IsMemberPointerOrAddr =
7259 (((IsPointer || ForDeviceAddr) &&
7260 I->getAssociatedExpression() == EncounteredME) ||
7261 (IsPrevMemberReference && !IsPointer) ||
7262 (IsMemberReference && Next != CE &&
7263 !Next->getAssociatedExpression()->getType()->isPointerType()));
7264 if (!OverlappedElements.empty() && Next == CE) {
7266 assert(!PartialStruct.Base.isValid() &&
"The base element is set.");
7267 assert(!IsPointer &&
7268 "Unexpected base element with the pointer type.");
7271 PartialStruct.LowestElem = {0, LowestElem};
7273 I->getAssociatedExpression()->getType());
7278 PartialStruct.HighestElem = {
7279 std::numeric_limits<
decltype(
7280 PartialStruct.HighestElem.first)>
::max(),
7282 PartialStruct.Base = BP;
7283 PartialStruct.LB = LB;
7285 PartialStruct.PreliminaryMapData.BasePointers.empty() &&
7286 "Overlapped elements must be used only once for the variable.");
7287 std::swap(PartialStruct.PreliminaryMapData, CombinedInfo);
7289 OpenMPOffloadMappingFlags Flags =
7290 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
7291 getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,
7293 false, IsNonContiguous);
7294 llvm::Value *
Size =
nullptr;
7297 Component : OverlappedElements) {
7301 if (
const ValueDecl *VD = MC.getAssociatedDeclaration()) {
7302 const auto *FD = dyn_cast<FieldDecl>(VD);
7303 if (FD && FD->getType()->isLValueReferenceType()) {
7305 cast<MemberExpr>(MC.getAssociatedExpression());
7306 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
7320 assert(Size &&
"Failed to determine structure size");
7321 CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
7322 CombinedInfo.BasePointers.push_back(BP.
getPointer());
7323 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
7324 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
7325 CombinedInfo.Pointers.push_back(LB.
getPointer());
7326 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
7328 CombinedInfo.Types.push_back(Flags);
7329 CombinedInfo.Mappers.push_back(
nullptr);
7330 CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize
7334 CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
7335 CombinedInfo.BasePointers.push_back(BP.
getPointer());
7336 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
7337 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
7338 CombinedInfo.Pointers.push_back(LB.
getPointer());
7342 CombinedInfo.Sizes.push_back(
7344 CombinedInfo.Types.push_back(Flags);
7345 CombinedInfo.Mappers.push_back(
nullptr);
7346 CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize
7350 llvm::Value *
Size = getExprTypeSize(I->getAssociatedExpression());
7355 if (!IsMemberPointerOrAddr ||
7357 if (!IsMappingWholeStruct) {
7358 CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
7359 CombinedInfo.BasePointers.push_back(BP.
getPointer());
7360 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
7361 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
7362 CombinedInfo.Pointers.push_back(LB.
getPointer());
7363 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
7365 CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize
7368 StructBaseCombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
7369 StructBaseCombinedInfo.BasePointers.push_back(BP.
getPointer());
7370 StructBaseCombinedInfo.DevicePtrDecls.push_back(
nullptr);
7371 StructBaseCombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
7372 StructBaseCombinedInfo.Pointers.push_back(LB.
getPointer());
7373 StructBaseCombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
7375 StructBaseCombinedInfo.NonContigInfo.Dims.push_back(
7376 IsNonContiguous ? DimSize : 1);
7380 bool HasMapper = Mapper && Next == CE;
7381 if (!IsMappingWholeStruct)
7382 CombinedInfo.Mappers.push_back(HasMapper ? Mapper :
nullptr);
7384 StructBaseCombinedInfo.Mappers.push_back(HasMapper ? Mapper
7391 OpenMPOffloadMappingFlags Flags = getMapTypeBits(
7392 MapType, MapModifiers, MotionModifiers, IsImplicit,
7393 !IsExpressionFirstInfo || RequiresReference ||
7394 FirstPointerInComplexData || IsMemberReference,
7395 IsCaptureFirstInfo && !RequiresReference, IsNonContiguous);
7397 if (!IsExpressionFirstInfo || IsMemberReference) {
7400 if (IsPointer || (IsMemberReference && Next != CE))
7401 Flags &= ~(OpenMPOffloadMappingFlags::OMP_MAP_TO |
7402 OpenMPOffloadMappingFlags::OMP_MAP_FROM |
7403 OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS |
7404 OpenMPOffloadMappingFlags::OMP_MAP_DELETE |
7405 OpenMPOffloadMappingFlags::OMP_MAP_CLOSE);
7407 if (ShouldBeMemberOf) {
7410 Flags |= OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;
7413 ShouldBeMemberOf =
false;
7417 if (!IsMappingWholeStruct)
7418 CombinedInfo.Types.push_back(Flags);
7420 StructBaseCombinedInfo.Types.push_back(Flags);
7426 if (EncounteredME) {
7427 const auto *FD = cast<FieldDecl>(EncounteredME->
getMemberDecl());
7428 unsigned FieldIndex = FD->getFieldIndex();
7431 if (!PartialStruct.Base.isValid()) {
7432 PartialStruct.LowestElem = {FieldIndex, LowestElem};
7433 if (IsFinalArraySection) {
7437 PartialStruct.HighestElem = {FieldIndex, HB};
7439 PartialStruct.HighestElem = {FieldIndex, LowestElem};
7441 PartialStruct.Base = BP;
7442 PartialStruct.LB = BP;
7443 }
else if (FieldIndex < PartialStruct.LowestElem.first) {
7444 PartialStruct.LowestElem = {FieldIndex, LowestElem};
7445 }
else if (FieldIndex > PartialStruct.HighestElem.first) {
7446 if (IsFinalArraySection) {
7450 PartialStruct.HighestElem = {FieldIndex, HB};
7452 PartialStruct.HighestElem = {FieldIndex, LowestElem};
7458 if (IsFinalArraySection || IsNonContiguous)
7459 PartialStruct.IsArraySection =
true;
7462 if (IsFinalArraySection)
7467 BP = IsMemberReference ? LowestElem : LB;
7469 IsExpressionFirstInfo =
false;
7470 IsCaptureFirstInfo =
false;
7471 FirstPointerInComplexData =
false;
7472 IsPrevMemberReference = IsMemberReference;
7473 }
else if (FirstPointerInComplexData) {
7475 ->getAssociatedDeclaration()
7477 .getNonReferenceType();
7479 FirstPointerInComplexData =
false;
7485 PartialStruct.HasCompleteRecord =
true;
7487 if (!IsNonContiguous)
7494 MapValuesArrayTy CurOffsets = {llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, 0)};
7495 MapValuesArrayTy CurCounts = {llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, 1)};
7496 MapValuesArrayTy CurStrides;
7497 MapValuesArrayTy DimSizes{llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, 1)};
7505 const Expr *AssocExpr = Component.getAssociatedExpression();
7506 const auto *OASE = dyn_cast<OMPArraySectionExpr>(AssocExpr);
7516 assert((VAT || CAT || &Component == &*Components.begin()) &&
7517 "Should be either ConstantArray or VariableArray if not the "
7521 if (CurStrides.empty()) {
7522 const Type *ElementType =
nullptr;
7524 ElementType = CAT->getElementType().getTypePtr();
7526 ElementType = VAT->getElementType().getTypePtr();
7528 assert(&Component == &*Components.begin() &&
7529 "Only expect pointer (non CAT or VAT) when this is the "
7537 if (&Component != &*Components.begin())
7541 CurStrides.push_back(
7542 llvm::ConstantInt::get(CGF.
Int64Ty, ElementTypeSize));
7547 if (DimSizes.size() < Components.size() - 1) {
7550 llvm::ConstantInt::get(CGF.
Int64Ty, CAT->getZExtSize()));
7552 DimSizes.push_back(CGF.
Builder.CreateIntCast(
7559 auto *DI = DimSizes.begin() + 1;
7561 llvm::Value *DimProd =
7562 llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, ElementTypeSize);
7573 const Expr *AssocExpr = Component.getAssociatedExpression();
7575 if (
const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr)) {
7576 llvm::Value *Offset = CGF.
Builder.CreateIntCast(
7579 CurOffsets.push_back(Offset);
7580 CurCounts.push_back(llvm::ConstantInt::get(CGF.
Int64Ty, 1));
7581 CurStrides.push_back(CurStrides.back());
7585 const auto *OASE = dyn_cast<OMPArraySectionExpr>(AssocExpr);
7591 const Expr *OffsetExpr = OASE->getLowerBound();
7592 llvm::Value *Offset =
nullptr;
7595 Offset = llvm::ConstantInt::get(CGF.
Int64Ty, 0);
7601 CurOffsets.push_back(Offset);
7604 const Expr *CountExpr = OASE->getLength();
7605 llvm::Value *Count =
nullptr;
7611 if (!OASE->getColonLocFirst().isValid() &&
7612 !OASE->getColonLocSecond().isValid()) {
7613 Count = llvm::ConstantInt::get(CGF.
Int64Ty, 1);
7619 const Expr *StrideExpr = OASE->getStride();
7620 llvm::Value *Stride =
7626 Count = CGF.
Builder.CreateUDiv(
7627 CGF.
Builder.CreateNUWSub(*DI, Offset), Stride);
7629 Count = CGF.
Builder.CreateNUWSub(*DI, Offset);
7635 CurCounts.push_back(Count);
7644 const Expr *StrideExpr = OASE->getStride();
7645 llvm::Value *Stride =
7650 DimProd = CGF.
Builder.CreateNUWMul(DimProd, *(DI - 1));
7652 CurStrides.push_back(CGF.
Builder.CreateNUWMul(DimProd, Stride));
7654 CurStrides.push_back(DimProd);
7655 if (DI != DimSizes.end())
7659 CombinedInfo.NonContigInfo.Offsets.push_back(CurOffsets);
7660 CombinedInfo.NonContigInfo.Counts.push_back(CurCounts);
7661 CombinedInfo.NonContigInfo.Strides.push_back(CurStrides);
7667 OpenMPOffloadMappingFlags
7676 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
7677 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
7678 return OpenMPOffloadMappingFlags::OMP_MAP_PRIVATE |
7679 OpenMPOffloadMappingFlags::OMP_MAP_TO;
7682 if (I != LambdasMap.end())
7684 return getMapTypeBits(
7685 I->getSecond()->getMapType(), I->getSecond()->getMapTypeModifiers(),
7686 std::nullopt, I->getSecond()->isImplicit(),
7690 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
7691 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
7696 bool AsBase)
const {
7699 llvm::StructType *St =
7702 unsigned NumElements = St->getNumElements();
7704 llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>, 4>
7705 RecordLayout(NumElements);
7708 for (
const auto &I : RD->
bases()) {
7711 const auto *
Base = I.getType()->getAsCXXRecordDecl();
7720 RecordLayout[FieldIndex] =
Base;
7723 for (
const auto &I : RD->
vbases()) {
7724 const auto *
Base = I.getType()->getAsCXXRecordDecl();
7726 if (
Base->isEmpty())
7729 if (RecordLayout[FieldIndex])
7731 RecordLayout[FieldIndex] =
Base;
7734 assert(!RD->
isUnion() &&
"Unexpected union.");
7735 for (
const auto *Field : RD->
fields()) {
7740 RecordLayout[FieldIndex] =
Field;
7743 for (
const llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>
7744 &
Data : RecordLayout) {
7748 getPlainLayout(
Base, Layout,
true);
7759 void generateAllInfoForClauses(
7761 llvm::OpenMPIRBuilder &OMPBuilder,
7768 llvm::MapVector<CanonicalDeclPtr<const Decl>,
7775 [&Info, &SkipVarSet](
7781 bool ReturnDevicePointer,
bool IsImplicit,
const ValueDecl *Mapper,
7782 const Expr *VarRef =
nullptr,
bool ForDeviceAddr =
false) {
7783 if (SkipVarSet.contains(D))
7785 auto It = Info.find(D);
7786 if (It == Info.end())
7788 .insert(std::make_pair(
7791 It->second[
Kind].emplace_back(
7792 L, MapType, MapModifiers, MotionModifiers, ReturnDevicePointer,
7793 IsImplicit, Mapper, VarRef, ForDeviceAddr);
7796 for (
const auto *
Cl : Clauses) {
7797 const auto *
C = dyn_cast<OMPMapClause>(
Cl);
7801 if (llvm::is_contained(
C->getMapTypeModifiers(),
7802 OMPC_MAP_MODIFIER_present))
7804 else if (
C->getMapType() == OMPC_MAP_alloc)
7806 const auto *EI =
C->getVarRefs().begin();
7807 for (
const auto L :
C->component_lists()) {
7808 const Expr *E = (
C->getMapLoc().isValid()) ? *EI :
nullptr;
7809 InfoGen(std::get<0>(L), Kind, std::get<1>(L),
C->getMapType(),
7810 C->getMapTypeModifiers(), std::nullopt,
7811 false,
C->isImplicit(), std::get<2>(L),
7816 for (
const auto *
Cl : Clauses) {
7817 const auto *
C = dyn_cast<OMPToClause>(
Cl);
7821 if (llvm::is_contained(
C->getMotionModifiers(),
7822 OMPC_MOTION_MODIFIER_present))
7824 const auto *EI =
C->getVarRefs().begin();
7825 for (
const auto L :
C->component_lists()) {
7826 InfoGen(std::get<0>(L), Kind, std::get<1>(L), OMPC_MAP_to, std::nullopt,
7827 C->getMotionModifiers(),
false,
7828 C->isImplicit(), std::get<2>(L), *EI);
7832 for (
const auto *
Cl : Clauses) {
7833 const auto *
C = dyn_cast<OMPFromClause>(
Cl);
7837 if (llvm::is_contained(
C->getMotionModifiers(),
7838 OMPC_MOTION_MODIFIER_present))
7840 const auto *EI =
C->getVarRefs().begin();
7841 for (
const auto L :
C->component_lists()) {
7842 InfoGen(std::get<0>(L), Kind, std::get<1>(L), OMPC_MAP_from,
7843 std::nullopt,
C->getMotionModifiers(),
7844 false,
C->isImplicit(), std::get<2>(L),
7857 llvm::MapVector<CanonicalDeclPtr<const Decl>,
7860 MapCombinedInfoTy UseDeviceDataCombinedInfo;
7862 auto &&UseDeviceDataCombinedInfoGen =
7863 [&UseDeviceDataCombinedInfo](
const ValueDecl *VD, llvm::Value *Ptr,
7865 UseDeviceDataCombinedInfo.Exprs.push_back(VD);
7866 UseDeviceDataCombinedInfo.BasePointers.emplace_back(Ptr);
7867 UseDeviceDataCombinedInfo.DevicePtrDecls.emplace_back(VD);
7868 UseDeviceDataCombinedInfo.DevicePointers.emplace_back(
7869 IsDevAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer);
7870 UseDeviceDataCombinedInfo.Pointers.push_back(Ptr);
7871 UseDeviceDataCombinedInfo.Sizes.push_back(
7872 llvm::Constant::getNullValue(CGF.Int64Ty));
7873 UseDeviceDataCombinedInfo.Types.push_back(
7874 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM);
7875 UseDeviceDataCombinedInfo.Mappers.push_back(
nullptr);
7879 [&DeferredInfo, &UseDeviceDataCombinedInfoGen,
7883 bool IsImplicit,
bool IsDevAddr) {
7887 if (isa<MemberExpr>(IE)) {
7896 std::nullopt,
false, IsImplicit,
7897 nullptr,
nullptr, IsDevAddr);
7898 DeferredInfo[
nullptr].emplace_back(IE, VD, IsDevAddr);
7902 if (IE->isGLValue())
7903 Ptr = CGF.EmitLValue(IE).getPointer(CGF);
7905 Ptr = CGF.EmitScalarExpr(IE);
7907 Ptr = CGF.EmitLoadOfScalar(CGF.EmitLValue(IE), IE->getExprLoc());
7909 UseDeviceDataCombinedInfoGen(VD, Ptr, CGF, IsDevAddr);
7914 const Expr *IE,
bool IsDevAddr) ->
bool {
7921 auto It = Info.find(isa<MemberExpr>(IE) ? nullptr : VD);
7922 if (It != Info.end()) {
7924 for (
auto &
Data : It->second) {
7925 auto *CI = llvm::find_if(
Data, [VD](
const MapInfo &MI) {
7926 return MI.Components.back().getAssociatedDeclaration() == VD;
7934 if (CI !=
Data.end()) {
7936 CI->ForDeviceAddr = IsDevAddr;
7937 CI->ReturnDevicePointer =
true;
7941 auto PrevCI = std::next(CI->Components.rbegin());
7942 const auto *VarD = dyn_cast<VarDecl>(VD);
7944 isa<MemberExpr>(IE) ||
7945 !VD->getType().getNonReferenceType()->isPointerType() ||
7946 PrevCI == CI->Components.rend() ||
7947 isa<MemberExpr>(PrevCI->getAssociatedExpression()) || !VarD ||
7948 VarD->hasLocalStorage()) {
7949 CI->ForDeviceAddr = IsDevAddr;
7950 CI->ReturnDevicePointer =
true;
7968 for (
const auto *
Cl : Clauses) {
7969 const auto *
C = dyn_cast<OMPUseDevicePtrClause>(
Cl);
7972 for (
const auto L :
C->component_lists()) {
7975 assert(!Components.empty() &&
7976 "Not expecting empty list of components!");
7977 const ValueDecl *VD = Components.back().getAssociatedDeclaration();
7979 const Expr *IE = Components.back().getAssociatedExpression();
7980 if (IsMapInfoExist(CGF, VD, IE,
false))
7982 MapInfoGen(CGF, IE, VD, Components,
C->isImplicit(),
7987 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
7988 for (
const auto *
Cl : Clauses) {
7989 const auto *
C = dyn_cast<OMPUseDeviceAddrClause>(
Cl);
7992 for (
const auto L :
C->component_lists()) {
7995 assert(!std::get<1>(L).empty() &&
7996 "Not expecting empty list of components!");
7997 const ValueDecl *VD = std::get<1>(L).back().getAssociatedDeclaration();
7998 if (!Processed.insert(VD).second)
8001 const Expr *IE = std::get<1>(L).back().getAssociatedExpression();
8002 if (IsMapInfoExist(CGF, VD, IE,
true))
8004 MapInfoGen(CGF, IE, VD, Components,
C->isImplicit(),
8009 for (
const auto &
Data : Info) {
8010 StructRangeInfoTy PartialStruct;
8012 MapCombinedInfoTy CurInfo;
8014 MapCombinedInfoTy StructBaseCurInfo;
8016 const ValueDecl *VD = cast_or_null<ValueDecl>(D);
8017 for (
const auto &M :
Data.second) {
8018 for (
const MapInfo &L : M) {
8019 assert(!L.Components.empty() &&
8020 "Not expecting declaration with no component lists.");
8023 unsigned CurrentBasePointersIdx = CurInfo.BasePointers.size();
8024 unsigned StructBasePointersIdx =
8025 StructBaseCurInfo.BasePointers.size();
8026 CurInfo.NonContigInfo.IsNonContiguous =
8027 L.Components.back().isNonContiguous();
8028 generateInfoForComponentList(
8029 L.MapType, L.MapModifiers, L.MotionModifiers, L.Components,
8030 CurInfo, StructBaseCurInfo, PartialStruct,
8031 false, L.IsImplicit,
8032 true, L.Mapper, L.ForDeviceAddr, VD,
8037 if (L.ReturnDevicePointer) {
8041 assert((CurrentBasePointersIdx < CurInfo.BasePointers.size() ||
8042 StructBasePointersIdx <
8043 StructBaseCurInfo.BasePointers.size()) &&
8044 "Unexpected number of mapped base pointers.");
8048 L.Components.back().getAssociatedDeclaration();
8049 assert(RelevantVD &&
8050 "No relevant declaration related with device pointer??");
8057 if (StructBasePointersIdx < StructBaseCurInfo.BasePointers.size()) {
8058 StructBaseCurInfo.DevicePtrDecls[StructBasePointersIdx] =
8060 StructBaseCurInfo.DevicePointers[StructBasePointersIdx] =
8061 L.ForDeviceAddr ? DeviceInfoTy::Address
8062 : DeviceInfoTy::Pointer;
8063 StructBaseCurInfo.Types[StructBasePointersIdx] |=
8064 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
8066 CurInfo.DevicePtrDecls[CurrentBasePointersIdx] = RelevantVD;
8067 CurInfo.DevicePointers[CurrentBasePointersIdx] =
8068 L.ForDeviceAddr ? DeviceInfoTy::Address
8069 : DeviceInfoTy::Pointer;
8070 CurInfo.Types[CurrentBasePointersIdx] |=
8071 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
8079 auto CI = DeferredInfo.find(
Data.first);
8080 if (CI != DeferredInfo.end()) {
8081 for (
const DeferredDevicePtrEntryTy &L : CI->second) {
8082 llvm::Value *BasePtr;
8084 if (L.ForDeviceAddr) {
8085 if (L.IE->isGLValue())
8093 CurInfo.Types.push_back(
8094 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM |
8095 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
8099 L.IE->getExprLoc());
8103 CurInfo.Types.push_back(
8104 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
8105 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM |
8106 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
8108 CurInfo.Exprs.push_back(L.VD);
8109 CurInfo.BasePointers.emplace_back(BasePtr);
8110 CurInfo.DevicePtrDecls.emplace_back(L.VD);
8111 CurInfo.DevicePointers.emplace_back(
8112 L.ForDeviceAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer);
8113 CurInfo.Pointers.push_back(Ptr);
8114 CurInfo.Sizes.push_back(
8115 llvm::Constant::getNullValue(this->CGF.
Int64Ty));
8116 CurInfo.Mappers.push_back(
nullptr);
8122 MapCombinedInfoTy UnionCurInfo;
8123 UnionCurInfo.append(StructBaseCurInfo);
8124 UnionCurInfo.append(CurInfo);
8128 if (PartialStruct.Base.isValid()) {
8129 UnionCurInfo.NonContigInfo.Dims.push_back(0);
8131 emitCombinedEntry(CombinedInfo, UnionCurInfo.Types, PartialStruct,
8132 !VD, OMPBuilder, VD);
8136 CombinedInfo.append(UnionCurInfo);
8139 CombinedInfo.append(UseDeviceDataCombinedInfo);
8144 : CurDir(&Dir), CGF(CGF) {
8147 for (
const auto *D :
C->varlists())
8148 FirstPrivateDecls.try_emplace(
8149 cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl()),
C->isImplicit());
8152 for (
unsigned I = 0, E =
C->getNumberOfAllocators(); I < E; ++I) {
8154 if (
const auto *DRE = dyn_cast_or_null<DeclRefExpr>(D.
AllocatorTraits))
8155 FirstPrivateDecls.try_emplace(cast<VarDecl>(DRE->getDecl()),
8157 else if (
const auto *VD = dyn_cast<VarDecl>(
8160 FirstPrivateDecls.try_emplace(VD,
true);
8165 for (
auto L :
C->component_lists())
8166 DevPointersMap[std::get<0>(L)].push_back(std::get<1>(L));
8169 for (
auto L :
C->component_lists())
8170 HasDevAddrsMap[std::get<0>(L)].push_back(std::get<1>(L));
8173 if (
C->getMapType() != OMPC_MAP_to)
8175 for (
auto L :
C->component_lists()) {
8177 const auto *RD = VD ? VD->
getType()
8183 LambdasMap.try_emplace(std::get<0>(L),
C);
8190 : CurDir(&Dir), CGF(CGF) {}
8195 void emitCombinedEntry(MapCombinedInfoTy &CombinedInfo,
8196 MapFlagsArrayTy &CurTypes,
8197 const StructRangeInfoTy &PartialStruct,
bool IsMapThis,
8198 llvm::OpenMPIRBuilder &OMPBuilder,
8200 bool NotTargetParams =
true)
const {
8201 if (CurTypes.size() == 1 &&
8202 ((CurTypes.back() & OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) !=
8203 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) &&
8204 !PartialStruct.IsArraySection)
8206 Address LBAddr = PartialStruct.LowestElem.second;
8207 Address HBAddr = PartialStruct.HighestElem.second;
8208 if (PartialStruct.HasCompleteRecord) {
8209 LBAddr = PartialStruct.LB;
8210 HBAddr = PartialStruct.LB;
8212 CombinedInfo.Exprs.push_back(VD);
8214 CombinedInfo.BasePointers.push_back(PartialStruct.Base.getPointer());
8215 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8216 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8222 bool HasBaseClass = RD && IsMapThis ? RD->
getNumBases() > 0 :
false;
8232 CombinedInfo.Pointers.push_back(PartialStruct.Base.getPointer());
8237 CombinedInfo.Sizes.push_back(Size);
8239 CombinedInfo.Pointers.push_back(LB);
8242 llvm::Value *HAddr = CGF.
Builder.CreateConstGEP1_32(
8246 llvm::Value *Diff = CGF.
Builder.CreatePtrDiff(CGF.
Int8Ty, CHAddr, CLAddr);
8249 CombinedInfo.Sizes.push_back(Size);
8251 CombinedInfo.Mappers.push_back(
nullptr);
8253 CombinedInfo.Types.push_back(
8254 NotTargetParams ? OpenMPOffloadMappingFlags::OMP_MAP_NONE
8255 : OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
8258 if (CurTypes.end() !=
8259 llvm::find_if(CurTypes, [](OpenMPOffloadMappingFlags
Type) {
8260 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
8261 Type & OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
8263 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
8265 (*CurTypes.begin()) &= ~OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
8272 if (CurTypes.end() !=
8273 llvm::find_if(CurTypes, [](OpenMPOffloadMappingFlags
Type) {
8274 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
8275 Type & OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD);
8277 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
8278 for (
auto &M : CurTypes)
8279 M |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
8285 OpenMPOffloadMappingFlags MemberOfFlag =
8286 OMPBuilder.getMemberOfFlag(CombinedInfo.BasePointers.size() - 1);
8287 for (
auto &M : CurTypes)
8288 OMPBuilder.setCorrectMemberOfFlag(M, MemberOfFlag);
8296 void generateAllInfo(
8297 MapCombinedInfoTy &CombinedInfo, llvm::OpenMPIRBuilder &OMPBuilder,
8301 "Expect a executable directive");
8303 generateAllInfoForClauses(CurExecDir->clauses(), CombinedInfo, OMPBuilder,
8310 void generateAllInfoForMapper(MapCombinedInfoTy &CombinedInfo,
8311 llvm::OpenMPIRBuilder &OMPBuilder)
const {
8313 "Expect a declare mapper directive");
8315 generateAllInfoForClauses(CurMapperDir->clauses(), CombinedInfo,
8320 void generateInfoForLambdaCaptures(
8321 const ValueDecl *VD, llvm::Value *Arg, MapCombinedInfoTy &CombinedInfo,
8322 llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers)
const {
8330 llvm::DenseMap<const ValueDecl *, FieldDecl *> Captures;
8337 LambdaPointers.try_emplace(ThisLVal.
getPointer(CGF),
8339 CombinedInfo.Exprs.push_back(VD);
8340 CombinedInfo.BasePointers.push_back(ThisLVal.
getPointer(CGF));
8341 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8342 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8343 CombinedInfo.Pointers.push_back(ThisLValVal.
getPointer(CGF));
8344 CombinedInfo.Sizes.push_back(
8347 CombinedInfo.Types.push_back(
8348 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
8349 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
8350 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
8351 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
8352 CombinedInfo.Mappers.push_back(
nullptr);
8355 if (!LC.capturesVariable())
8357 const VarDecl *VD = cast<VarDecl>(LC.getCapturedVar());
8360 auto It = Captures.find(VD);
8361 assert(It != Captures.end() &&
"Found lambda capture without field.");
8365 LambdaPointers.try_emplace(VarLVal.
getPointer(CGF),
8367 CombinedInfo.Exprs.push_back(VD);
8368 CombinedInfo.BasePointers.push_back(VarLVal.
getPointer(CGF));
8369 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8370 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8371 CombinedInfo.Pointers.push_back(VarLValVal.
getPointer(CGF));
8372 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
8378 LambdaPointers.try_emplace(VarLVal.
getPointer(CGF),
8380 CombinedInfo.Exprs.push_back(VD);
8381 CombinedInfo.BasePointers.push_back(VarLVal.
getPointer(CGF));
8382 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8383 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8384 CombinedInfo.Pointers.push_back(VarRVal.
getScalarVal());
8385 CombinedInfo.Sizes.push_back(llvm::ConstantInt::get(CGF.
Int64Ty, 0));
8387 CombinedInfo.Types.push_back(
8388 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
8389 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
8390 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
8391 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
8392 CombinedInfo.Mappers.push_back(
nullptr);
8397 void adjustMemberOfForLambdaCaptures(
8398 llvm::OpenMPIRBuilder &OMPBuilder,
8399 const llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers,
8400 MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
8401 MapFlagsArrayTy &Types)
const {
8402 for (
unsigned I = 0, E = Types.size(); I < E; ++I) {
8404 if (Types[I] != (OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
8405 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
8406 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
8407 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT))
8409 llvm::Value *BasePtr = LambdaPointers.lookup(BasePointers[I]);
8410 assert(BasePtr &&
"Unable to find base lambda address.");
8412 for (
unsigned J = I; J > 0; --J) {
8413 unsigned Idx = J - 1;
8414 if (Pointers[Idx] != BasePtr)
8419 assert(TgtIdx != -1 &&
"Unable to find parent lambda.");
8423 OpenMPOffloadMappingFlags MemberOfFlag =
8424 OMPBuilder.getMemberOfFlag(TgtIdx);
8425 OMPBuilder.setCorrectMemberOfFlag(Types[I], MemberOfFlag);
8432 llvm::Value *Arg, MapCombinedInfoTy &CombinedInfo,
8433 StructRangeInfoTy &PartialStruct)
const {
8435 "Not expecting to generate map info for a variable array type!");
8444 if (LambdasMap.count(VD))
8450 if (VD && (DevPointersMap.count(VD) || HasDevAddrsMap.count(VD))) {
8451 CombinedInfo.Exprs.push_back(VD);
8452 CombinedInfo.BasePointers.emplace_back(Arg);
8453 CombinedInfo.DevicePtrDecls.emplace_back(VD);
8454 CombinedInfo.DevicePointers.emplace_back(DeviceInfoTy::Pointer);
8455 CombinedInfo.Pointers.push_back(Arg);
8456 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
8459 CombinedInfo.Types.push_back(
8460 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
8461 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
8462 CombinedInfo.Mappers.push_back(
nullptr);
8474 auto It = DevPointersMap.find(VD);
8475 if (It != DevPointersMap.end())
8476 for (
const auto &MCL : It->second)
8477 DeclComponentLists.emplace_back(MCL, OMPC_MAP_to,
Unknown,
8480 auto I = HasDevAddrsMap.find(VD);
8481 if (I != HasDevAddrsMap.end())
8482 for (
const auto &MCL : I->second)
8483 DeclComponentLists.emplace_back(MCL, OMPC_MAP_tofrom,
Unknown,
8487 "Expect a executable directive");
8489 for (
const auto *
C : CurExecDir->getClausesOfKind<
OMPMapClause>()) {
8490 const auto *EI =
C->getVarRefs().begin();
8491 for (
const auto L :
C->decl_component_lists(VD)) {
8494 const Expr *E = (
C->getMapLoc().isValid()) ? *EI :
nullptr;
8496 std::tie(VDecl, Components, Mapper) = L;
8497 assert(VDecl == VD &&
"We got information for the wrong declaration??");
8498 assert(!Components.empty() &&
8499 "Not expecting declaration with no component lists.");
8500 DeclComponentLists.emplace_back(Components,
C->getMapType(),
8501 C->getMapTypeModifiers(),
8502 C->isImplicit(), Mapper, E);
8506 llvm::stable_sort(DeclComponentLists, [](
const MapData &LHS,
8507 const MapData &RHS) {
8511 llvm::is_contained(MapModifiers, clang::OMPC_MAP_MODIFIER_present);
8512 bool HasAllocs = MapType == OMPC_MAP_alloc;
8513 MapModifiers = std::get<2>(RHS);
8514 MapType = std::get<1>(LHS);
8516 llvm::is_contained(MapModifiers, clang::OMPC_MAP_MODIFIER_present);
8517 bool HasAllocsR = MapType == OMPC_MAP_alloc;
8518 return (HasPresent && !HasPresentR) || (HasAllocs && !HasAllocsR);
8522 llvm::SmallDenseMap<
8529 for (
const MapData &L : DeclComponentLists) {
8536 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =
8539 for (
const MapData &L1 :
ArrayRef(DeclComponentLists).slice(Count)) {
8541 std::tie(Components1, MapType, MapModifiers, IsImplicit, Mapper,
8543 auto CI = Components.rbegin();
8544 auto CE = Components.rend();
8545 auto SI = Components1.rbegin();
8546 auto SE = Components1.rend();
8547 for (; CI != CE && SI != SE; ++CI, ++SI) {
8548 if (CI->getAssociatedExpression()->getStmtClass() !=
8549 SI->getAssociatedExpression()->getStmtClass())
8552 if (CI->getAssociatedDeclaration() != SI->getAssociatedDeclaration())
8557 if (CI == CE || SI == SE) {
8559 if (CI == CE && SI == SE)
8561 const auto It = (SI == SE) ? CI : SI;
8567 if (!isa<MemberExpr>(It->getAssociatedExpression()) ||
8568 (std::prev(It)->getAssociatedDeclaration() &&
8570 ->getAssociatedDeclaration()
8572 ->isPointerType()) ||
8573 (It->getAssociatedDeclaration() &&
8574 It->getAssociatedDeclaration()->getType()->isPointerType() &&
8575 std::next(It) != CE && std::next(It) != SE))
8577 const MapData &BaseData = CI == CE ? L : L1;
8579 SI == SE ? Components : Components1;
8580 auto &OverlappedElements = OverlappedData.FindAndConstruct(&BaseData);
8581 OverlappedElements.getSecond().push_back(SubData);
8587 if (!OverlappedData.empty()) {
8590 while (BaseType != OrigType) {
8596 getPlainLayout(CRD, Layout,
false);
8602 for (
auto &Pair : OverlappedData) {
8609 auto CI = First.rbegin();
8610 auto CE = First.rend();
8611 auto SI = Second.rbegin();
8612 auto SE = Second.rend();
8613 for (; CI != CE && SI != SE; ++CI, ++SI) {
8614 if (CI->getAssociatedExpression()->getStmtClass() !=
8615 SI->getAssociatedExpression()->getStmtClass())
8618 if (CI->getAssociatedDeclaration() !=
8619 SI->getAssociatedDeclaration())
8624 if (CI == CE && SI == SE)
8628 if (CI == CE || SI == SE)
8631 const auto *FD1 = cast<FieldDecl>(CI->getAssociatedDeclaration());
8632 const auto *FD2 = cast<FieldDecl>(SI->getAssociatedDeclaration());
8633 if (FD1->getParent() == FD2->getParent())
8634 return FD1->getFieldIndex() < FD2->getFieldIndex();
8636 llvm::find_if(Layout, [FD1, FD2](
const FieldDecl *FD) {
8637 return FD == FD1 || FD == FD2;
8645 bool IsFirstComponentList =
true;
8646 MapCombinedInfoTy StructBaseCombinedInfo;
8647 for (
const auto &Pair : OverlappedData) {
8648 const MapData &L = *Pair.getFirst();
8655 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =
8658 OverlappedComponents = Pair.getSecond();
8659 generateInfoForComponentList(
8660 MapType, MapModifiers, std::nullopt, Components, CombinedInfo,
8661 StructBaseCombinedInfo, PartialStruct, IsFirstComponentList,
8662 IsImplicit,
false, Mapper,
8663 false, VD, VarRef, OverlappedComponents);
8664 IsFirstComponentList =
false;
8667 for (
const MapData &L : DeclComponentLists) {
8674 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =
8676 auto It = OverlappedData.find(&L);
8677 if (It == OverlappedData.end())
8678 generateInfoForComponentList(
8679 MapType, MapModifiers, std::nullopt, Components, CombinedInfo,
8680 StructBaseCombinedInfo, PartialStruct, IsFirstComponentList,
8681 IsImplicit,
false, Mapper,
8683 IsFirstComponentList =
false;
8691 MapCombinedInfoTy &CombinedInfo)
const {
8692 bool IsImplicit =
true;
8695 CombinedInfo.Exprs.push_back(
nullptr);
8696 CombinedInfo.BasePointers.push_back(CV);
8697 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8698 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8699 CombinedInfo.Pointers.push_back(CV);
8701 CombinedInfo.Sizes.push_back(
8705 CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_TO |
8706 OpenMPOffloadMappingFlags::OMP_MAP_FROM);
8710 CombinedInfo.BasePointers.push_back(CV);
8711 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8712 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8713 CombinedInfo.Pointers.push_back(CV);
8717 CombinedInfo.Types.push_back(
8718 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
8719 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
8724 CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_NONE);
8725 CombinedInfo.Sizes.push_back(llvm::Constant::getNullValue(CGF.
Int64Ty));
8727 auto I = FirstPrivateDecls.find(VD);
8728 if (I != FirstPrivateDecls.end())
8729 IsImplicit = I->getSecond();
8734 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
8739 CombinedInfo.Types.push_back(getMapModifiersForPrivateClauses(CI));
8741 auto I = FirstPrivateDecls.find(VD);
8743 CombinedInfo.BasePointers.push_back(CV);
8744 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8745 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8750 CombinedInfo.Pointers.push_back(PtrAddr.
getPointer());
8752 CombinedInfo.Pointers.push_back(CV);
8754 if (I != FirstPrivateDecls.end())
8755 IsImplicit = I->getSecond();
8758 CombinedInfo.Types.back() |=
8759 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
8763 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
8766 CombinedInfo.Mappers.push_back(
nullptr);
8776 if (
const auto *OASE = dyn_cast<OMPArraySectionExpr>(E->
IgnoreParenCasts()))
8778 dyn_cast<MemberExpr>(OASE->getBase()->IgnoreParenImpCasts()))
8779 return ME->getMemberDecl();
8787 MappableExprsHandler::MappingExprInfo &MapExprs) {
8789 uint32_t SrcLocStrSize;
8790 if (!MapExprs.getMapDecl() && !MapExprs.getMapExpr())
8791 return OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
8794 if (!MapExprs.getMapDecl() && MapExprs.getMapExpr()) {
8798 Loc = MapExprs.getMapExpr()->getExprLoc();
8800 Loc = MapExprs.getMapDecl()->getLocation();
8803 std::string ExprName;
8804 if (MapExprs.getMapExpr()) {
8806 llvm::raw_string_ostream OS(ExprName);
8807 MapExprs.getMapExpr()->printPretty(OS,
nullptr,
P);
8810 ExprName = MapExprs.getMapDecl()->getNameAsString();
8814 return OMPBuilder.getOrCreateSrcLocStr(PLoc.
getFilename(), ExprName,
8823 CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
8825 bool IsNonContiguous =
false) {
8829 Info.clearArrayInfo();
8830 Info.NumberOfPtrs = CombinedInfo.BasePointers.size();
8832 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
8835 InsertPointTy CodeGenIP(CGF.
Builder.GetInsertBlock(),
8836 CGF.
Builder.GetInsertPoint());
8838 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
8842 llvm::codegenoptions::NoDebugInfo) {
8843 CombinedInfo.Names.resize(CombinedInfo.Exprs.size());
8844 llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),
8848 auto DeviceAddrCB = [&](
unsigned int I, llvm::Value *NewDecl) {
8849 if (
const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
8854 auto CustomMapperCB = [&](
unsigned int I) {
8855 llvm::Value *MFunc =
nullptr;
8856 if (CombinedInfo.Mappers[I]) {
8857 Info.HasMapper =
true;
8859 cast<OMPDeclareMapperDecl>(CombinedInfo.Mappers[I]));
8863 OMPBuilder.emitOffloadingArrays(AllocaIP, CodeGenIP, CombinedInfo, Info,
8874 const Stmt *ChildStmt =
8877 if (
const auto *NestedDir =
8878 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
8885 if (DKind == OMPD_teams) {
8886 Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers(
8891 if (
const auto *NND =
8892 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
8893 DKind = NND->getDirectiveKind();
8899 case OMPD_target_teams:
8903 case OMPD_target_parallel:
8904 case OMPD_target_simd:
8905 case OMPD_target_parallel_for:
8906 case OMPD_target_parallel_for_simd:
8908 case OMPD_target_teams_distribute:
8909 case OMPD_target_teams_distribute_simd:
8910 case OMPD_target_teams_distribute_parallel_for:
8911 case OMPD_target_teams_distribute_parallel_for_simd:
8914 case OMPD_parallel_for:
8915 case OMPD_parallel_master:
8916 case OMPD_parallel_sections:
8918 case OMPD_parallel_for_simd:
8920 case OMPD_cancellation_point:
8922 case OMPD_threadprivate:
8933 case OMPD_taskyield:
8936 case OMPD_taskgroup:
8942 case OMPD_target_data:
8943 case OMPD_target_exit_data:
8944 case OMPD_target_enter_data:
8945 case OMPD_distribute:
8946 case OMPD_distribute_simd:
8947 case OMPD_distribute_parallel_for:
8948 case OMPD_distribute_parallel_for_simd:
8949 case OMPD_teams_distribute:
8950 case OMPD_teams_distribute_simd:
8951 case OMPD_teams_distribute_parallel_for:
8952 case OMPD_teams_distribute_parallel_for_simd:
8953 case OMPD_target_update:
8954 case OMPD_declare_simd:
8955 case OMPD_declare_variant:
8956 case OMPD_begin_declare_variant:
8957 case OMPD_end_declare_variant:
8958 case OMPD_declare_target:
8959 case OMPD_end_declare_target:
8960 case OMPD_declare_reduction:
8961 case OMPD_declare_mapper:
8963 case OMPD_taskloop_simd:
8964 case OMPD_master_taskloop:
8965 case OMPD_master_taskloop_simd:
8966 case OMPD_parallel_master_taskloop:
8967 case OMPD_parallel_master_taskloop_simd:
8969 case OMPD_metadirective:
8972 llvm_unreachable(
"Unexpected directive.");
9017 QualType PtrTy =
C.getPointerType(Ty).withRestrict();
9018 QualType Int64Ty =
C.getIntTypeForBitwidth(64,
true);
9019 auto *MapperVarDecl =
9022 CharUnits ElementSize =
C.getTypeSizeInChars(Ty);
9039 Args.push_back(&HandleArg);
9040 Args.push_back(&BaseArg);
9041 Args.push_back(&BeginArg);
9042 Args.push_back(&SizeArg);
9043 Args.push_back(&TypeArg);
9044 Args.push_back(&NameArg);
9049 llvm::raw_svector_ostream Out(TyStr);
9052 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
9055 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
9062 C.getPointerType(Int64Ty), Loc);
9066 false,
C.getPointerType(
C.VoidPtrTy), Loc);
9069 false,
C.getPointerType(
C.VoidPtrTy), Loc);
9072 false,
C.getPointerType(
C.VoidPtrTy), Loc);
9074 Size = MapperCGF.
Builder.CreateExactUDiv(
9076 llvm::Value *PtrBegin = MapperCGF.
Builder.CreateBitCast(
9081 C.getPointerType(Int64Ty), Loc);
9084 false,
C.getPointerType(
C.VoidPtrTy), Loc);
9088 llvm::BasicBlock *HeadBB = MapperCGF.
createBasicBlock(
"omp.arraymap.head");
9090 MapName, ElementSize, HeadBB,
true);
9096 llvm::BasicBlock *BodyBB = MapperCGF.
createBasicBlock(
"omp.arraymap.body");
9099 llvm::Value *IsEmpty =
9100 MapperCGF.
Builder.CreateICmpEQ(PtrBegin, PtrEnd,
"omp.arraymap.isempty");
9101 MapperCGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
9102 llvm::BasicBlock *EntryBB = MapperCGF.
Builder.GetInsertBlock();
9106 llvm::BasicBlock *LastBB = BodyBB;
9107 llvm::PHINode *PtrPHI = MapperCGF.
Builder.CreatePHI(
9108 PtrBegin->getType(), 2,
"omp.arraymap.ptrcurrent");
9109 PtrPHI->addIncoming(PtrBegin, EntryBB);
9110 Address PtrCurrent(PtrPHI, ElemTy,
9116 Scope.addPrivate(MapperVarDecl, PtrCurrent);
9117 (void)
Scope.Privatize();
9120 MappableExprsHandler::MapCombinedInfoTy Info;
9121 MappableExprsHandler MEHandler(*D, MapperCGF);
9122 MEHandler.generateAllInfoForMapper(Info,
OMPBuilder);
9126 llvm::Value *OffloadingArgs[] = {Handle};
9129 OMPRTL___tgt_mapper_num_components),
9131 llvm::Value *ShiftedPreviousSize = MapperCGF.
Builder.CreateShl(
9133 MapperCGF.
Builder.getInt64(MappableExprsHandler::getFlagMemberOffset()));
9136 for (
unsigned I = 0; I < Info.BasePointers.size(); ++I) {
9137 llvm::Value *CurBaseArg = MapperCGF.
Builder.CreateBitCast(
9139 llvm::Value *CurBeginArg = MapperCGF.
Builder.CreateBitCast(
9141 llvm::Value *CurSizeArg = Info.Sizes[I];
9142 llvm::Value *CurNameArg =
9144 llvm::codegenoptions::NoDebugInfo)
9149 llvm::Value *OriMapType = MapperCGF.
Builder.getInt64(
9150 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9152 llvm::Value *MemberMapType =
9153 MapperCGF.
Builder.CreateNUWAdd(OriMapType, ShiftedPreviousSize);
9167 llvm::Value *LeftToFrom = MapperCGF.
Builder.CreateAnd(
9170 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9171 OpenMPOffloadMappingFlags::OMP_MAP_TO |
9172 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
9174 llvm::BasicBlock *AllocElseBB =
9177 llvm::BasicBlock *ToElseBB = MapperCGF.
createBasicBlock(
"omp.type.to.else");
9180 llvm::Value *IsAlloc = MapperCGF.
Builder.CreateIsNull(LeftToFrom);
9181 MapperCGF.
Builder.CreateCondBr(IsAlloc, AllocBB, AllocElseBB);
9184 llvm::Value *AllocMapType = MapperCGF.
Builder.CreateAnd(
9187 ~
static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9188 OpenMPOffloadMappingFlags::OMP_MAP_TO |
9189 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
9190 MapperCGF.
Builder.CreateBr(EndBB);
9192 llvm::Value *IsTo = MapperCGF.
Builder.CreateICmpEQ(
9195 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9196 OpenMPOffloadMappingFlags::OMP_MAP_TO)));
9197 MapperCGF.
Builder.CreateCondBr(IsTo, ToBB, ToElseBB);
9200 llvm::Value *ToMapType = MapperCGF.
Builder.CreateAnd(
9203 ~
static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9204 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
9205 MapperCGF.
Builder.CreateBr(EndBB);
9207 llvm::Value *IsFrom = MapperCGF.
Builder.CreateICmpEQ(
9210 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9211 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
9212 MapperCGF.
Builder.CreateCondBr(IsFrom, FromBB, EndBB);
9215 llvm::Value *FromMapType = MapperCGF.
Builder.CreateAnd(
9218 ~
static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9219 OpenMPOffloadMappingFlags::OMP_MAP_TO)));
9223 llvm::PHINode *CurMapType =
9225 CurMapType->addIncoming(AllocMapType, AllocBB);
9226 CurMapType->addIncoming(ToMapType, ToBB);
9227 CurMapType->addIncoming(FromMapType, FromBB);
9228 CurMapType->addIncoming(MemberMapType, ToElseBB);
9230 llvm::Value *OffloadingArgs[] = {Handle, CurBaseArg, CurBeginArg,
9231 CurSizeArg, CurMapType, CurNameArg};
9232 if (Info.Mappers[I]) {
9235 cast<OMPDeclareMapperDecl>(Info.Mappers[I]));
9236 assert(MapperFunc &&
"Expect a valid mapper function is available.");
9250 llvm::Value *PtrNext = MapperCGF.
Builder.CreateConstGEP1_32(
9251 ElemTy, PtrPHI, 1,
"omp.arraymap.next");
9252 PtrPHI->addIncoming(PtrNext, LastBB);
9253 llvm::Value *IsDone =
9254 MapperCGF.
Builder.CreateICmpEQ(PtrNext, PtrEnd,
"omp.arraymap.isdone");
9255 llvm::BasicBlock *ExitBB = MapperCGF.
createBasicBlock(
"omp.arraymap.exit");
9256 MapperCGF.
Builder.CreateCondBr(IsDone, ExitBB, BodyBB);
9262 MapName, ElementSize, DoneBB,
false);
9267 UDMMap.try_emplace(D, Fn);
9270 Decls.second.push_back(D);
9282 llvm::Value *
Begin, llvm::Value *Size, llvm::Value *MapType,
9283 llvm::Value *MapName,
CharUnits ElementSize, llvm::BasicBlock *ExitBB,
9285 StringRef Prefix = IsInit ?
".init" :
".del";
9288 llvm::BasicBlock *BodyBB =
9290 llvm::Value *IsArray = MapperCGF.
Builder.CreateICmpSGT(
9291 Size, MapperCGF.
Builder.getInt64(1),
"omp.arrayinit.isarray");
9292 llvm::Value *DeleteBit = MapperCGF.
Builder.CreateAnd(
9295 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9296 OpenMPOffloadMappingFlags::OMP_MAP_DELETE)));
9297 llvm::Value *DeleteCond;
9303 llvm::Value *PtrAndObjBit = MapperCGF.
Builder.CreateAnd(
9306 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9307 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ)));
9308 PtrAndObjBit = MapperCGF.
Builder.CreateIsNotNull(PtrAndObjBit);
9309 BaseIsBegin = MapperCGF.
Builder.CreateAnd(BaseIsBegin, PtrAndObjBit);
9310 Cond = MapperCGF.
Builder.CreateOr(IsArray, BaseIsBegin);
9311 DeleteCond = MapperCGF.
Builder.CreateIsNull(
9312 DeleteBit,
getName({
"omp.array", Prefix,
".delete"}));
9315 DeleteCond = MapperCGF.
Builder.CreateIsNotNull(
9316 DeleteBit,
getName({
"omp.array", Prefix,
".delete"}));
9318 Cond = MapperCGF.
Builder.CreateAnd(Cond, DeleteCond);
9319 MapperCGF.
Builder.CreateCondBr(Cond, BodyBB, ExitBB);
9324 llvm::Value *ArraySize = MapperCGF.
Builder.CreateNUWMul(
9328 llvm::Value *MapTypeArg = MapperCGF.
Builder.CreateAnd(
9331 ~
static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9332 OpenMPOffloadMappingFlags::OMP_MAP_TO |
9333 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
9334 MapTypeArg = MapperCGF.
Builder.CreateOr(
9337 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9338 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT)));
9342 llvm::Value *OffloadingArgs[] = {Handle,
Base,
Begin,
9343 ArraySize, MapTypeArg, MapName};
9346 OMPRTL___tgt_push_mapper_component),
9368 Kind != OMPD_target_teams_loop)
9371 return llvm::ConstantInt::get(CGF.
Int64Ty, 0);
9373 const auto *LD = cast<OMPLoopDirective>(TD);
9374 if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD))
9375 return NumIterations;
9376 return llvm::ConstantInt::get(CGF.
Int64Ty, 0);
9385 if (OffloadingMandatory) {
9386 CGF.
Builder.CreateUnreachable();
9388 if (RequiresOuterTask) {
9389 CapturedVars.clear();
9398 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
9401 llvm::Value *DeviceID;
9402 if (
Device.getPointer()) {
9404 Device.getInt() == OMPC_DEVICE_device_num) &&
9405 "Expected device_num modifier.");
9410 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
9417 llvm::Value *DynCGroupMem = CGF.
Builder.getInt32(0);
9420 CodeGenFunction::RunCleanupsScope DynCGroupMemScope(CGF);
9422 DynMemClause->getSize(),
true);
9423 DynCGroupMem = CGF.
Builder.CreateIntCast(DynCGroupMemVal, CGF.
Int32Ty,
9426 return DynCGroupMem;
9434 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
9435 llvm::Value *OutlinedFnID, CodeGenFunction::OMPTargetDataInfo &InputInfo,
9436 llvm::Value *&MapTypesArray, llvm::Value *&MapNamesArray,
9441 llvm::OpenMPIRBuilder &OMPBuilder = OMPRuntime->
getOMPBuilder();
9444 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
9447 MappableExprsHandler MEHandler(D, CGF);
9448 llvm::DenseMap<llvm::Value *, llvm::Value *> LambdaPointers;
9452 auto *CV = CapturedVars.begin();
9455 CI != CE; ++CI, ++RI, ++CV) {
9456 MappableExprsHandler::MapCombinedInfoTy CurInfo;
9457 MappableExprsHandler::StructRangeInfoTy PartialStruct;
9462 CurInfo.Exprs.push_back(
nullptr);
9463 CurInfo.BasePointers.push_back(*CV);
9464 CurInfo.DevicePtrDecls.push_back(
nullptr);
9465 CurInfo.DevicePointers.push_back(
9466 MappableExprsHandler::DeviceInfoTy::None);
9467 CurInfo.Pointers.push_back(*CV);
9468 CurInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
9471 CurInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9472 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM |
9473 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
9474 CurInfo.Mappers.push_back(
nullptr);
9478 MEHandler.generateInfoForCapture(CI, *CV, CurInfo, PartialStruct);
9482 MappedVarSet.insert(
nullptr);
9483 if (CurInfo.BasePointers.empty() && !PartialStruct.Base.isValid())
9484 MEHandler.generateDefaultMapInfo(*CI, **RI, *CV, CurInfo);
9488 MEHandler.generateInfoForLambdaCaptures(CI->
getCapturedVar(), *CV,
9489 CurInfo, LambdaPointers);
9492 assert((!CurInfo.BasePointers.empty() || PartialStruct.Base.isValid()) &&
9493 "Non-existing map pointer for capture!");
9494 assert(CurInfo.BasePointers.size() == CurInfo.Pointers.size() &&
9495 CurInfo.BasePointers.size() == CurInfo.Sizes.size() &&
9496 CurInfo.BasePointers.size() == CurInfo.Types.size() &&
9497 CurInfo.BasePointers.size() == CurInfo.Mappers.size() &&
9498 "Inconsistent map information sizes!");
9502 if (PartialStruct.Base.isValid()) {
9503 CombinedInfo.append(PartialStruct.PreliminaryMapData);
9504 MEHandler.emitCombinedEntry(
9505 CombinedInfo, CurInfo.Types, PartialStruct, CI->
capturesThis(),
9506 OMPBuilder,
nullptr,
9507 !PartialStruct.PreliminaryMapData.BasePointers.empty());
9511 CombinedInfo.append(CurInfo);
9514 MEHandler.adjustMemberOfForLambdaCaptures(
9515 OMPBuilder, LambdaPointers, CombinedInfo.BasePointers,
9516 CombinedInfo.Pointers, CombinedInfo.Types);
9519 MEHandler.generateAllInfo(CombinedInfo, OMPBuilder, MappedVarSet);
9525 llvm::codegenoptions::NoDebugInfo;
9526 OMPBuilder.emitOffloadingArraysArgument(CGF.
Builder, Info.RTArgs, Info,
9530 InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
9531 InputInfo.BasePointersArray =
Address(Info.RTArgs.BasePointersArray,
9533 InputInfo.PointersArray =
9535 InputInfo.SizesArray =
9537 InputInfo.MappersArray =
9539 MapTypesArray = Info.RTArgs.MapTypesArray;
9540 MapNamesArray = Info.RTArgs.MapNamesArray;
9542 auto &&ThenGen = [&OMPRuntime, OutlinedFn, &D, &CapturedVars,
9543 RequiresOuterTask, &CS, OffloadingMandatory,
Device,
9544 OutlinedFnID, &InputInfo, &MapTypesArray, &MapNamesArray,
9546 bool IsReverseOffloading =
Device.getInt() == OMPC_DEVICE_ancestor;
9548 if (IsReverseOffloading) {
9554 RequiresOuterTask, CS, OffloadingMandatory, CGF);
9559 unsigned NumTargetItems = InputInfo.NumberOfTargetItems;
9561 llvm::Value *BasePointersArray = InputInfo.BasePointersArray.getPointer();
9562 llvm::Value *PointersArray = InputInfo.PointersArray.getPointer();
9563 llvm::Value *SizesArray = InputInfo.SizesArray.getPointer();
9564 llvm::Value *MappersArray = InputInfo.MappersArray.getPointer();
9566 auto &&EmitTargetCallFallbackCB =
9567 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
9568 OffloadingMandatory, &CGF](llvm::OpenMPIRBuilder::InsertPointTy IP)
9569 -> llvm::OpenMPIRBuilder::InsertPointTy {
9572 RequiresOuterTask, CS, OffloadingMandatory, CGF);
9578 llvm::Value *NumThreads =
9581 llvm::Value *NumIterations =
9584 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
9587 llvm::OpenMPIRBuilder::TargetDataRTArgs RTArgs(
9588 BasePointersArray, PointersArray, SizesArray, MapTypesArray,
9589 nullptr , MappersArray, MapNamesArray);
9591 llvm::OpenMPIRBuilder::TargetKernelArgs Args(
9592 NumTargetItems, RTArgs, NumIterations, NumTeams, NumThreads,
9593 DynCGGroupMem, HasNoWait);
9596 CGF.
Builder, OutlinedFn, OutlinedFnID, EmitTargetCallFallbackCB, Args,
9597 DeviceID, RTLoc, AllocaIP));
9600 if (RequiresOuterTask)
9615 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
9618 RequiresOuterTask, CS, OffloadingMandatory, CGF);
9621 if (RequiresOuterTask) {
9622 CodeGenFunction::OMPTargetDataInfo InputInfo;
9631 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID,
const Expr *IfCond,
9632 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
9639 const bool OffloadingMandatory = !
CGM.
getLangOpts().OpenMPIsTargetDevice &&
9642 assert((OffloadingMandatory || OutlinedFn) &&
"Invalid outlined function!");
9644 const bool RequiresOuterTask =
9660 llvm::Value *MapTypesArray =
nullptr;
9661 llvm::Value *MapNamesArray =
nullptr;
9663 auto &&TargetThenGen = [
this, OutlinedFn, &D, &CapturedVars,
9664 RequiresOuterTask, &CS, OffloadingMandatory,
Device,
9665 OutlinedFnID, &InputInfo, &MapTypesArray,
9669 RequiresOuterTask, CS, OffloadingMandatory,
9670 Device, OutlinedFnID, InputInfo, MapTypesArray,
9671 MapNamesArray, SizeEmitter, CGF,
CGM);
9674 auto &&TargetElseGen =
9675 [
this, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
9678 CS, OffloadingMandatory, CGF);
9687 emitIfClause(CGF, IfCond, TargetThenGen, TargetElseGen);
9699 StringRef ParentName) {
9704 bool RequiresDeviceCodegen =
9705 isa<OMPExecutableDirective>(S) &&
9707 cast<OMPExecutableDirective>(S)->getDirectiveKind());
9709 if (RequiresDeviceCodegen) {
9710 const auto &E = *cast<OMPExecutableDirective>(S);
9717 if (!
OMPBuilder.OffloadInfoManager.hasTargetRegionEntryInfo(EntryInfo))
9720 switch (E.getDirectiveKind()) {
9723 cast<OMPTargetDirective>(E));
9725 case OMPD_target_parallel:
9727 CGM, ParentName, cast<OMPTargetParallelDirective>(E));
9729 case OMPD_target_teams:
9731 CGM, ParentName, cast<OMPTargetTeamsDirective>(E));
9733 case OMPD_target_teams_distribute:
9735 CGM, ParentName, cast<OMPTargetTeamsDistributeDirective>(E));
9737 case OMPD_target_teams_distribute_simd:
9739 CGM, ParentName, cast<OMPTargetTeamsDistributeSimdDirective>(E));
9741 case OMPD_target_parallel_for:
9743 CGM, ParentName, cast<OMPTargetParallelForDirective>(E));
9745 case OMPD_target_parallel_for_simd:
9747 CGM, ParentName, cast<OMPTargetParallelForSimdDirective>(E));
9749 case OMPD_target_simd:
9751 CGM, ParentName, cast<OMPTargetSimdDirective>(E));
9753 case OMPD_target_teams_distribute_parallel_for:
9756 cast<OMPTargetTeamsDistributeParallelForDirective>(E));
9758 case OMPD_target_teams_distribute_parallel_for_simd:
9762 cast<OMPTargetTeamsDistributeParallelForSimdDirective>(E));
9764 case OMPD_target_teams_loop:
9766 CGM, ParentName, cast<OMPTargetTeamsGenericLoopDirective>(E));
9768 case OMPD_target_parallel_loop:
9770 CGM, ParentName, cast<OMPTargetParallelGenericLoopDirective>(E));
9774 case OMPD_parallel_for:
9775 case OMPD_parallel_master:
9776 case OMPD_parallel_sections:
9778 case OMPD_parallel_for_simd:
9780 case OMPD_cancellation_point:
9782 case OMPD_threadprivate:
9793 case OMPD_taskyield:
9796 case OMPD_taskgroup:
9802 case OMPD_target_data:
9803 case OMPD_target_exit_data:
9804 case OMPD_target_enter_data:
9805 case OMPD_distribute:
9806 case OMPD_distribute_simd:
9807 case OMPD_distribute_parallel_for:
9808 case OMPD_distribute_parallel_for_simd:
9809 case OMPD_teams_distribute:
9810 case OMPD_teams_distribute_simd:
9811 case OMPD_teams_distribute_parallel_for:
9812 case OMPD_teams_distribute_parallel_for_simd:
9813 case OMPD_target_update:
9814 case OMPD_declare_simd:
9815 case OMPD_declare_variant:
9816 case OMPD_begin_declare_variant:
9817 case OMPD_end_declare_variant:
9818 case OMPD_declare_target:
9819 case OMPD_end_declare_target:
9820 case OMPD_declare_reduction:
9821 case OMPD_declare_mapper:
9823 case OMPD_taskloop_simd:
9824 case OMPD_master_taskloop:
9825 case OMPD_master_taskloop_simd:
9826 case OMPD_parallel_master_taskloop:
9827 case OMPD_parallel_master_taskloop_simd:
9829 case OMPD_metadirective:
9832 llvm_unreachable(
"Unknown target directive for OpenMP device codegen.");
9837 if (
const auto *E = dyn_cast<OMPExecutableDirective>(S)) {
9838 if (!E->hasAssociatedStmt() || !E->getAssociatedStmt())
9846 if (
const auto *L = dyn_cast<LambdaExpr>(S))
9850 for (
const Stmt *II : S->children())
9855 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
9856 OMPDeclareTargetDeclAttr::getDeviceType(VD);
9860 if (!IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_NoHost)
9863 if (IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_Host)
9872 if (
const auto *FD = dyn_cast<FunctionDecl>(GD.
getDecl()))
9881 if (
const auto *FD = dyn_cast<FunctionDecl>(VD)) {
9890 return !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD) &&
9908 StringRef ParentName =
9913 StringRef ParentName =
9920 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
9921 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(
9923 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
9924 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
9925 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
9934 llvm::Constant *Addr) {
9939 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
9940 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
9944 if (Res && *Res != OMPDeclareTargetDeclAttr::MT_Link &&
9959 auto LinkageForVariable = [&VD,
this]() {
9963 std::vector<llvm::GlobalVariable *> GeneratedRefs;
9976 for (
auto *ref : GeneratedRefs)
9981 if (isa<FunctionDecl>(GD.
getDecl()) ||
9982 isa<OMPDeclareReductionDecl>(GD.
getDecl()))
9990 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
9991 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
9994 if ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
9995 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
9999 assert((*Res == OMPDeclareTargetDeclAttr::MT_Link ||
10000 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
10001 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
10003 "Expected link clause or to clause with unified memory.");
10012 " Expected target-based directive.");
10017 if (Clause->getClauseKind() == OMPC_unified_shared_memory) {
10019 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(
true);
10020 }
else if (
const auto *AC =
10021 dyn_cast<OMPAtomicDefaultMemOrderClause>(Clause)) {
10022 switch (AC->getAtomicDefaultMemOrderKind()) {
10023 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_acq_rel:
10026 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_seq_cst:
10029 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_relaxed:
10045 if (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())
10047 const auto *A = VD->
getAttr<OMPAllocateDeclAttr>();
10048 switch(A->getAllocatorType()) {
10049 case OMPAllocateDeclAttr::OMPNullMemAlloc:
10050 case OMPAllocateDeclAttr::OMPDefaultMemAlloc:
10052 case OMPAllocateDeclAttr::OMPLargeCapMemAlloc:
10053 case OMPAllocateDeclAttr::OMPCGroupMemAlloc:
10054 case OMPAllocateDeclAttr::OMPHighBWMemAlloc:
10055 case OMPAllocateDeclAttr::OMPLowLatMemAlloc:
10056 case OMPAllocateDeclAttr::OMPThreadMemAlloc:
10057 case OMPAllocateDeclAttr::OMPConstMemAlloc:
10058 case OMPAllocateDeclAttr::OMPPTeamMemAlloc:
10061 case OMPAllocateDeclAttr::OMPUserDefinedMemAlloc:
10062 llvm_unreachable(
"Expected predefined allocator for the variables with the "
10063 "static storage.");
10090 const auto *D = cast<FunctionDecl>(GD.
getDecl());
10093 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(D)) {
10095 if (
auto *F = dyn_cast_or_null<llvm::Function>(
10097 return !F->isDeclaration();
10109 llvm::Function *OutlinedFn,
10118 llvm::Value *Args[] = {
10120 CGF.
Builder.getInt32(CapturedVars.size()),
10123 RealArgs.append(std::begin(Args), std::end(Args));
10124 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
10126 llvm::FunctionCallee RTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
10132 const Expr *NumTeams,
10133 const Expr *ThreadLimit,
10140 llvm::Value *NumTeamsVal =
10146 llvm::Value *ThreadLimitVal =
10153 llvm::Value *PushNumTeamsArgs[] = {RTLoc,
getThreadID(CGF, Loc), NumTeamsVal,
10161 const Expr *ThreadLimit,
10164 llvm::Value *ThreadLimitVal =
10171 llvm::Value *ThreadLimitArgs[] = {RTLoc,
getThreadID(CGF, Loc),
10189 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
10191 llvm::Value *IfCondVal =
nullptr;
10196 llvm::Value *DeviceID =
nullptr;
10201 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
10205 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
10206 auto GenMapInfoCB =
10207 [&](InsertPointTy CodeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {
10208 CGF.
Builder.restoreIP(CodeGenIP);
10210 MappableExprsHandler MEHandler(D, CGF);
10211 MEHandler.generateAllInfo(CombinedInfo,
OMPBuilder);
10213 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
10217 llvm::codegenoptions::NoDebugInfo) {
10218 CombinedInfo.Names.resize(CombinedInfo.Exprs.size());
10219 llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),
10223 return CombinedInfo;
10225 using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;
10226 auto BodyCB = [&](InsertPointTy CodeGenIP, BodyGenTy BodyGenType) {
10227 CGF.
Builder.restoreIP(CodeGenIP);
10228 switch (BodyGenType) {
10229 case BodyGenTy::Priv:
10233 case BodyGenTy::DupNoPriv:
10239 case BodyGenTy::NoPriv:
10246 return InsertPointTy(CGF.
Builder.GetInsertBlock(),
10247 CGF.
Builder.GetInsertPoint());
10250 auto DeviceAddrCB = [&](
unsigned int I, llvm::Value *NewDecl) {
10251 if (
const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
10256 auto CustomMapperCB = [&](
unsigned int I) {
10257 llvm::Value *MFunc =
nullptr;
10258 if (CombinedInfo.Mappers[I]) {
10259 Info.HasMapper =
true;
10261 cast<OMPDeclareMapperDecl>(CombinedInfo.Mappers[I]));
10271 InsertPointTy CodeGenIP(CGF.
Builder.GetInsertBlock(),
10272 CGF.
Builder.GetInsertPoint());
10273 llvm::OpenMPIRBuilder::LocationDescription OmpLoc(CodeGenIP);
10275 OmpLoc, AllocaIP, CodeGenIP, DeviceID, IfCondVal, Info, GenMapInfoCB,
10276 nullptr, BodyCB, DeviceAddrCB, CustomMapperCB, RTLoc));
10285 assert((isa<OMPTargetEnterDataDirective>(D) ||
10286 isa<OMPTargetExitDataDirective>(D) ||
10287 isa<OMPTargetUpdateDirective>(D)) &&
10288 "Expecting either target enter, exit data, or update directives.");
10291 llvm::Value *MapTypesArray =
nullptr;
10292 llvm::Value *MapNamesArray =
nullptr;
10294 auto &&ThenGen = [
this, &D,
Device, &InputInfo, &MapTypesArray,
10297 llvm::Value *DeviceID =
nullptr;
10302 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
10306 llvm::Constant *PointerNum =
10312 llvm::Value *OffloadingArgs[] = {RTLoc,
10325 RuntimeFunction RTLFn;
10327 case OMPD_target_enter_data:
10328 RTLFn = HasNowait ? OMPRTL___tgt_target_data_begin_nowait_mapper
10329 : OMPRTL___tgt_target_data_begin_mapper;
10331 case OMPD_target_exit_data:
10332 RTLFn = HasNowait ? OMPRTL___tgt_target_data_end_nowait_mapper
10333 : OMPRTL___tgt_target_data_end_mapper;
10335 case OMPD_target_update:
10336 RTLFn = HasNowait ? OMPRTL___tgt_target_data_update_nowait_mapper
10337 : OMPRTL___tgt_target_data_update_mapper;
10339 case OMPD_parallel:
10341 case OMPD_parallel_for:
10342 case OMPD_parallel_master:
10343 case OMPD_parallel_sections:
10344 case OMPD_for_simd:
10345 case OMPD_parallel_for_simd:
10347 case OMPD_cancellation_point:
10349 case OMPD_threadprivate:
10350 case OMPD_allocate:
10355 case OMPD_sections:
10359 case OMPD_critical:
10360 case OMPD_taskyield:
10362 case OMPD_taskwait:
10363 case OMPD_taskgroup:
10369 case OMPD_target_data:
10370 case OMPD_distribute:
10371 case OMPD_distribute_simd:
10372 case OMPD_distribute_parallel_for:
10373 case OMPD_distribute_parallel_for_simd:
10374 case OMPD_teams_distribute:
10375 case OMPD_teams_distribute_simd:
10376 case OMPD_teams_distribute_parallel_for:
10377 case OMPD_teams_distribute_parallel_for_simd:
10378 case OMPD_declare_simd:
10379 case OMPD_declare_variant:
10380 case OMPD_begin_declare_variant:
10381 case OMPD_end_declare_variant:
10382 case OMPD_declare_target:
10383 case OMPD_end_declare_target:
10384 case OMPD_declare_reduction:
10385 case OMPD_declare_mapper:
10386 case OMPD_taskloop:
10387 case OMPD_taskloop_simd:
10388 case OMPD_master_taskloop:
10389 case OMPD_master_taskloop_simd:
10390 case OMPD_parallel_master_taskloop:
10391 case OMPD_parallel_master_taskloop_simd:
10393 case OMPD_target_simd:
10394 case OMPD_target_teams_distribute:
10395 case OMPD_target_teams_distribute_simd:
10396 case OMPD_target_teams_distribute_parallel_for:
10397 case OMPD_target_teams_distribute_parallel_for_simd:
10398 case OMPD_target_teams:
10399 case OMPD_target_parallel:
10400 case OMPD_target_parallel_for:
10401 case OMPD_target_parallel_for_simd:
10402 case OMPD_requires:
10403 case OMPD_metadirective:
10406 llvm_unreachable(
"Unexpected standalone target data directive.");
10414 auto &&TargetThenGen = [
this, &ThenGen, &D, &InputInfo, &MapTypesArray,
10418 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
10421 MappableExprsHandler MEHandler(D, CGF);
10422 MEHandler.generateAllInfo(CombinedInfo,
OMPBuilder);
10431 llvm::codegenoptions::NoDebugInfo;
10444 MapTypesArray = Info.RTArgs.MapTypesArray;
10445 MapNamesArray = Info.RTArgs.MapNamesArray;
10446 if (RequiresOuterTask)
10472struct ParamAttrTy {
10473 ParamKindTy Kind =
Vector;
10474 llvm::APSInt StrideOrArg;
10475 llvm::APSInt Alignment;
10476 bool HasVarStride =
false;
10509 unsigned Offset = 0;
10510 if (
const auto *MD = dyn_cast<CXXMethodDecl>(FD)) {
10511 if (ParamAttrs[Offset].Kind ==
Vector)
10512 CDT =
C.getPointerType(
C.getRecordType(MD->
getParent()));
10516 for (
unsigned I = 0, E = FD->
getNumParams(); I < E; ++I) {
10517 if (ParamAttrs[I + Offset].Kind ==
Vector) {
10529 return C.getTypeSize(CDT);
10537 llvm::raw_svector_ostream Out(Buffer);
10538 for (
const auto &ParamAttr : ParamAttrs) {
10539 switch (ParamAttr.Kind) {
10559 if (ParamAttr.HasVarStride)
10560 Out <<
"s" << ParamAttr.StrideOrArg;
10561 else if (ParamAttr.Kind == Linear || ParamAttr.Kind == LinearRef ||
10562 ParamAttr.Kind == LinearUVal || ParamAttr.Kind == LinearVal) {
10565 if (ParamAttr.StrideOrArg < 0)
10566 Out <<
'n' << -ParamAttr.StrideOrArg;
10567 else if (ParamAttr.StrideOrArg != 1)
10568 Out << ParamAttr.StrideOrArg;
10571 if (!!ParamAttr.Alignment)
10572 Out <<
'a' << ParamAttr.Alignment;
10575 return std::string(Out.str());
10580 const llvm::APSInt &VLENVal,
10582 OMPDeclareSimdDeclAttr::BranchStateTy State) {
10585 unsigned VecRegSize;
10587 ISADataTy ISAData[] = {
10603 case OMPDeclareSimdDeclAttr::BS_Undefined:
10604 Masked.push_back(
'N');
10605 Masked.push_back(
'M');
10607 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
10608 Masked.push_back(
'N');
10610 case OMPDeclareSimdDeclAttr::BS_Inbranch:
10611 Masked.push_back(
'M');
10614 for (
char Mask : Masked) {
10615 for (
const ISADataTy &
Data : ISAData) {
10617 llvm::raw_svector_ostream Out(Buffer);
10618 Out <<
"_ZGV" <<
Data.ISA << Mask;
10621 assert(NumElts &&
"Non-zero simdlen/cdtsize expected");
10622 Out << llvm::APSInt::getUnsigned(
Data.VecRegSize / NumElts);
10627 Out <<
'_' << Fn->getName();
10628 Fn->addFnAttr(Out.str());
10646 if (Kind == ParamKindTy::Uniform)
10649 if (Kind == ParamKindTy::LinearUVal || Kind == ParamKindTy::LinearRef)
10652 if ((Kind == ParamKindTy::Linear || Kind == ParamKindTy::LinearVal) &&
10662 unsigned Size =
C.getTypeSize(QT);
10665 if (Size != 8 && Size != 16 && Size != 32 && Size != 64 && Size != 128)
10689 return C.getTypeSize(PTy);
10692 return C.getTypeSize(QT);
10694 return C.getTypeSize(
C.getUIntPtrType());
10700static std::tuple<unsigned, unsigned, bool>
10706 bool OutputBecomesInput =
false;
10710 Sizes.push_back(
getAArch64LS(RetType, ParamKindTy::Vector,
C));
10712 OutputBecomesInput =
true;
10714 for (
unsigned I = 0, E = FD->
getNumParams(); I < E; ++I) {
10719 assert(!Sizes.empty() &&
"Unable to determine NDS and WDS.");
10722 assert(llvm::all_of(Sizes,
10723 [](
unsigned Size) {
10724 return Size == 8 || Size == 16 || Size == 32 ||
10725 Size == 64 || Size == 128;
10729 return std::make_tuple(*std::min_element(std::begin(Sizes), std::end(Sizes)),
10730 *std::max_element(std::begin(Sizes), std::end(Sizes)),
10731 OutputBecomesInput);
10737template <
typename T>
10739 char ISA, StringRef ParSeq,
10740 StringRef MangledName,
bool OutputBecomesInput,
10741 llvm::Function *Fn) {
10743 llvm::raw_svector_ostream Out(Buffer);
10744 Out << Prefix << ISA << LMask << VLEN;
10745 if (OutputBecomesInput)
10747 Out << ParSeq <<
"_" << MangledName;
10748 Fn->addFnAttr(Out.str());
10754 StringRef Prefix,
char ISA,
10755 StringRef ParSeq, StringRef MangledName,
10756 bool OutputBecomesInput,
10757 llvm::Function *Fn) {
10761 OutputBecomesInput, Fn);
10763 OutputBecomesInput, Fn);
10767 OutputBecomesInput, Fn);
10769 OutputBecomesInput, Fn);
10773 OutputBecomesInput, Fn);
10775 OutputBecomesInput, Fn);
10780 OutputBecomesInput, Fn);
10783 llvm_unreachable(
"Scalar type is too wide.");
10791 OMPDeclareSimdDeclAttr::BranchStateTy State, StringRef MangledName,
10792 char ISA,
unsigned VecRegSize, llvm::Function *Fn,
SourceLocation SLoc) {
10796 const unsigned NDS = std::get<0>(
Data);
10797 const unsigned WDS = std::get<1>(
Data);
10798 const bool OutputBecomesInput = std::get<2>(
Data);
10802 if (UserVLEN == 1) {
10805 "The clause simdlen(1) has no effect when targeting aarch64.");
10812 if (ISA ==
'n' && UserVLEN && !llvm::isPowerOf2_32(UserVLEN)) {
10815 "power of 2 when targeting Advanced SIMD.");
10822 if (ISA ==
's' && UserVLEN != 0) {
10823 if ((UserVLEN * WDS > 2048) || (UserVLEN * WDS % 128 != 0)) {
10826 "lanes in the architectural constraints "
10827 "for SVE (min is 128-bit, max is "
10828 "2048-bit, by steps of 128-bit)");
10836 StringRef Prefix =
"_ZGV";
10842 OutputBecomesInput, Fn);
10844 assert(ISA ==
'n' &&
"Expected ISA either 's' or 'n'.");
10848 case OMPDeclareSimdDeclAttr::BS_Undefined:
10850 OutputBecomesInput, Fn);
10852 OutputBecomesInput, Fn);
10854 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
10856 OutputBecomesInput, Fn);
10858 case OMPDeclareSimdDeclAttr::BS_Inbranch:
10860 OutputBecomesInput, Fn);
10870 OutputBecomesInput, Fn);
10872 assert(ISA ==
'n' &&
"Expected ISA either 's' or 'n'.");
10877 case OMPDeclareSimdDeclAttr::BS_Undefined:
10879 OutputBecomesInput, Fn);
10881 OutputBecomesInput, Fn);
10883 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
10885 OutputBecomesInput, Fn);
10887 case OMPDeclareSimdDeclAttr::BS_Inbranch:
10889 OutputBecomesInput, Fn);
10897 llvm::Function *Fn) {
10902 llvm::DenseMap<const Decl *, unsigned> ParamPositions;
10903 if (isa<CXXMethodDecl>(FD))
10904 ParamPositions.try_emplace(FD, 0);
10905 unsigned ParamPos = ParamPositions.size();
10907 ParamPositions.try_emplace(
P->getCanonicalDecl(), ParamPos);
10913 for (
const Expr *E :
Attr->uniforms()) {
10916 if (isa<CXXThisExpr>(E)) {
10917 Pos = ParamPositions[FD];
10919 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
10920 ->getCanonicalDecl();
10921 auto It = ParamPositions.find(PVD);
10922 assert(It != ParamPositions.end() &&
"Function parameter not found");
10925 ParamAttrs[Pos].Kind = Uniform;
10928 auto *NI =
Attr->alignments_begin();
10929 for (
const Expr *E :
Attr->aligneds()) {
10933 if (isa<CXXThisExpr>(E)) {
10934 Pos = ParamPositions[FD];
10937 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
10938 ->getCanonicalDecl();
10939 auto It = ParamPositions.find(PVD);
10940 assert(It != ParamPositions.end() &&
"Function parameter not found");
10942 ParmTy = PVD->getType();
10944 ParamAttrs[Pos].Alignment =
10946 ? (*NI)->EvaluateKnownConstInt(
C)
10947 : llvm::APSInt::getUnsigned(
10948 C.toCharUnitsFromBits(
C.getOpenMPDefaultSimdAlign(ParmTy))
10953 auto *SI =
Attr->steps_begin();
10954 auto *MI =
Attr->modifiers_begin();
10955 for (
const Expr *E :
Attr->linears()) {
10958 bool IsReferenceType =
false;
10961 unsigned PtrRescalingFactor = 1;
10962 if (isa<CXXThisExpr>(E)) {
10963 Pos = ParamPositions[FD];
10964 auto *
P = cast<PointerType>(E->
getType());
10969 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
10970 ->getCanonicalDecl();
10971 auto It = ParamPositions.find(PVD);
10972 assert(It != ParamPositions.end() &&
"Function parameter not found");
10974 if (
auto *
P = dyn_cast<PointerType>(PVD->getType()))
10978 else if (PVD->getType()->isReferenceType()) {
10979 IsReferenceType =
true;
10980 PtrRescalingFactor =
10986 ParamAttrTy &ParamAttr = ParamAttrs[Pos];
10987 if (*MI == OMPC_LINEAR_ref)
10988 ParamAttr.Kind = LinearRef;
10989 else if (*MI == OMPC_LINEAR_uval)
10990 ParamAttr.Kind = LinearUVal;
10991 else if (IsReferenceType)
10992 ParamAttr.Kind = LinearVal;
10994 ParamAttr.Kind = Linear;
10996 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(1);
11000 if (
const auto *DRE =
11001 cast<DeclRefExpr>((*SI)->IgnoreParenImpCasts())) {
11002 if (
const auto *StridePVD =
11003 dyn_cast<ParmVarDecl>(DRE->getDecl())) {
11004 ParamAttr.HasVarStride =
true;
11005 auto It = ParamPositions.find(StridePVD->getCanonicalDecl());
11006 assert(It != ParamPositions.end() &&
11007 "Function parameter not found");
11008 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(It->second);
11012 ParamAttr.StrideOrArg =
Result.Val.getInt();
11018 if (!ParamAttr.HasVarStride &&
11019 (ParamAttr.Kind == Linear || ParamAttr.Kind == LinearRef))
11020 ParamAttr.StrideOrArg = ParamAttr.StrideOrArg * PtrRescalingFactor;
11024 llvm::APSInt VLENVal;
11026 const Expr *VLENExpr =
Attr->getSimdlen();
11031 OMPDeclareSimdDeclAttr::BranchStateTy State =
Attr->getBranchState();
11034 }
else if (
CGM.
getTriple().getArch() == llvm::Triple::aarch64) {
11035 unsigned VLEN = VLENVal.getExtValue();
11036 StringRef MangledName = Fn->getName();
11039 MangledName,
's', 128, Fn, ExprLoc);
11042 MangledName,
'n', 128, Fn, ExprLoc);
11053 static const int DoacrossFinArgs = 2;
11056 llvm::FunctionCallee RTLFn;
11057 llvm::Value *Args[DoacrossFinArgs];
11060 DoacrossCleanupTy(llvm::FunctionCallee RTLFn,
11063 assert(CallArgs.size() == DoacrossFinArgs);
11064 std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args));
11081 QualType Int64Ty =
C.getIntTypeForBitwidth(64,
true);
11089 RD =
C.buildImplicitRecord(
"kmp_dim");
11099 llvm::APInt Size(32, NumIterations.size());
11105 enum { LowerFD = 0, UpperFD, StrideFD };
11107 for (
unsigned I = 0, E = NumIterations.size(); I < E; ++I) {
11112 DimsLVal, *std::next(RD->
field_begin(), UpperFD));
11114 CGF.
EmitScalarExpr(NumIterations[I]), NumIterations[I]->getType(),
11115 Int64Ty, NumIterations[I]->getExprLoc());
11119 DimsLVal, *std::next(RD->
field_begin(), StrideFD));
11126 llvm::Value *Args[] = {
11129 llvm::ConstantInt::getSigned(
CGM.
Int32Ty, NumIterations.size()),
11134 llvm::FunctionCallee RTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
11137 llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = {
11139 llvm::FunctionCallee FiniRTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
11145template <
typename T>
11147 const T *
C, llvm::Value *ULoc,
11148 llvm::Value *ThreadID) {
11151 llvm::APInt Size(32,
C->getNumLoops());
11155 for (
unsigned I = 0, E =
C->getNumLoops(); I < E; ++I) {
11156 const Expr *CounterVal =
C->getLoopData(I);
11157 assert(CounterVal);
11164 llvm::Value *Args[] = {
11166 llvm::FunctionCallee RTLFn;
11168 OMPDoacrossKind<T> ODK;
11169 if (ODK.isSource(
C)) {
11171 OMPRTL___kmpc_doacross_post);
11173 assert(ODK.isSink(
C) &&
"Expect sink modifier.");
11175 OMPRTL___kmpc_doacross_wait);
11182 return EmitDoacrossOrdered<OMPDependClause>(
11189 return EmitDoacrossOrdered<OMPDoacrossClause>(
11195 llvm::FunctionCallee Callee,
11197 assert(Loc.
isValid() &&
"Outlined function call location must be valid.");
11200 if (
auto *Fn = dyn_cast<llvm::Function>(Callee.getCallee())) {
11201 if (Fn->doesNotThrow()) {
11212 emitCall(CGF, Loc, OutlinedFn, Args);
11216 if (
const auto *FD = dyn_cast<FunctionDecl>(D))
11217 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(FD))
11223 const VarDecl *TargetParam)
const {
11230 const Expr *Allocator) {
11231 llvm::Value *AllocVal;
11238 Allocator->getExprLoc());
11241 AllocVal = llvm::Constant::getNullValue(
11251 if (!AllocateAlignment)
11254 return llvm::ConstantInt::get(
CGM.
SizeTy, AllocateAlignment->getQuantity());
11267 auto I = UntiedData.find(VD);
11268 if (I != UntiedData.end()) {
11269 UntiedAddr = I->second.first;
11270 UntiedRealAddr = I->second.second;
11274 if (CVD->
hasAttr<OMPAllocateDeclAttr>()) {
11283 Size = CGF.
Builder.CreateNUWAdd(
11292 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
11293 const Expr *Allocator = AA->getAllocator();
11297 Args.push_back(ThreadID);
11299 Args.push_back(Alignment);
11300 Args.push_back(Size);
11301 Args.push_back(AllocVal);
11302 llvm::omp::RuntimeFunction FnID =
11303 Alignment ? OMPRTL___kmpc_aligned_alloc : OMPRTL___kmpc_alloc;
11307 llvm::FunctionCallee FiniRTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
11317 llvm::FunctionCallee RTLFn;
11320 const Expr *AllocExpr;
11323 OMPAllocateCleanupTy(llvm::FunctionCallee RTLFn,
11325 const Expr *AllocExpr)
11326 : RTLFn(RTLFn), LocEncoding(LocEncoding), Addr(Addr),
11327 AllocExpr(AllocExpr) {}
11331 llvm::Value *Args[3];
11337 Args[2] = AllocVal;
11345 CGF.
EHStack.pushCleanup<OMPAllocateCleanupTy>(
11347 VDAddr, Allocator);
11348 if (UntiedRealAddr.
isValid())
11351 Region->emitUntiedSwitch(CGF);
11368 assert(CGM.
getLangOpts().OpenMP &&
"Not in OpenMP mode.");
11374 for (
const Stmt *Ref :
C->private_refs()) {
11375 const auto *SimpleRefExpr = cast<Expr>(Ref)->IgnoreParenImpCasts();
11377 if (
const auto *DRE = dyn_cast<DeclRefExpr>(SimpleRefExpr)) {
11378 VD = DRE->getDecl();
11380 const auto *ME = cast<MemberExpr>(SimpleRefExpr);
11381 assert((ME->isImplicitCXXThis() ||
11382 isa<CXXThisExpr>(ME->getBase()->IgnoreParenImpCasts())) &&
11383 "Expected member of current class.");
11384 VD = ME->getMemberDecl();
11400 std::pair<Address, Address>> &LocalVars)
11401 :
CGM(CGF.
CGM), NeedToPush(!LocalVars.empty()) {
11418 return llvm::any_of(
11423void CGOpenMPRuntime::LastprivateConditionalRAII::tryToDisableInnerAnalysis(
11433 const CapturedStmt *CS = S.getCapturedStmt(CaptureRegions.front());
11441 for (
const Expr *Ref :
C->varlists()) {
11442 if (!Ref->getType()->isScalarType())
11444 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
11447 NeedToCheckForLPCs.insert(DRE->getDecl());
11451 for (
const Expr *Ref :
C->varlists()) {
11452 if (!Ref->getType()->isScalarType())
11454 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
11457 NeedToCheckForLPCs.insert(DRE->getDecl());
11461 for (
const Expr *Ref :
C->varlists()) {
11462 if (!Ref->getType()->isScalarType())
11464 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
11467 NeedToCheckForLPCs.insert(DRE->getDecl());
11471 for (
const Expr *Ref :
C->varlists()) {
11472 if (!Ref->getType()->isScalarType())
11474 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
11477 NeedToCheckForLPCs.insert(DRE->getDecl());
11481 for (
const Expr *Ref :
C->varlists()) {
11482 if (!Ref->getType()->isScalarType())
11484 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
11487 NeedToCheckForLPCs.insert(DRE->getDecl());
11490 for (
const Decl *VD : NeedToCheckForLPCs) {
11491 for (
const LastprivateConditionalData &
Data :
11493 if (
Data.DeclToUniqueName.count(VD) > 0) {
11494 if (!
Data.Disabled)
11495 NeedToAddForLPCsAsDisabled.insert(VD);
11502CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
11505 Action((
CGM.getLangOpts().OpenMP >= 50 &&
11508 return C->getKind() ==
11509 OMPC_LASTPRIVATE_conditional;
11511 ? ActionToDo::PushAsLastprivateConditional
11512 : ActionToDo::DoNotPush) {
11514 if (
CGM.
getLangOpts().OpenMP < 50 || Action == ActionToDo::DoNotPush)
11516 assert(Action == ActionToDo::PushAsLastprivateConditional &&
11517 "Expected a push action.");
11521 if (
C->getKind() != OMPC_LASTPRIVATE_conditional)
11524 for (
const Expr *Ref :
C->varlists()) {
11525 Data.DeclToUniqueName.insert(std::make_pair(
11526 cast<DeclRefExpr>(Ref->IgnoreParenImpCasts())->getDecl(),
11530 Data.IVLVal = IVLVal;
11534CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
11536 :
CGM(CGF.
CGM), Action(ActionToDo::DoNotPush) {
11537 assert(CGM.
getLangOpts().OpenMP &&
"Not in OpenMP mode.");
11541 tryToDisableInnerAnalysis(S, NeedToAddForLPCsAsDisabled);
11542 if (!NeedToAddForLPCsAsDisabled.empty()) {
11543 Action = ActionToDo::DisableLastprivateConditional;
11544 LastprivateConditionalData &
Data =
11546 for (
const Decl *VD : NeedToAddForLPCsAsDisabled)
11549 Data.Disabled =
true;
11562 if (Action == ActionToDo::DisableLastprivateConditional) {
11564 "Expected list of disabled private vars.");
11567 if (Action == ActionToDo::PushAsLastprivateConditional) {
11570 "Expected list of lastprivate conditional vars.");
11585 auto VI = I->getSecond().find(VD);
11586 if (VI == I->getSecond().end()) {
11587 RecordDecl *RD =
C.buildImplicitRecord(
"lasprivate.conditional");
11592 NewType =
C.getRecordType(RD);
11595 I->getSecond().try_emplace(VD, NewType, VDField, FiredField, BaseLVal);
11597 NewType = std::get<0>(VI->getSecond());
11598 VDField = std::get<1>(VI->getSecond());
11599 FiredField = std::get<2>(VI->getSecond());
11600 BaseLVal = std::get<3>(VI->getSecond());
11612class LastprivateConditionalRefChecker final
11615 const Expr *FoundE =
nullptr;
11616 const Decl *FoundD =
nullptr;
11617 StringRef UniqueDeclName;
11619 llvm::Function *FoundFn =
nullptr;
11625 llvm::reverse(LPM)) {
11626 auto It = D.DeclToUniqueName.find(E->
getDecl());
11627 if (It == D.DeclToUniqueName.end())
11633 UniqueDeclName = It->second;
11638 return FoundE == E;
11641 if (!CodeGenFunction::IsWrappedCXXThis(E->
getBase()))
11644 llvm::reverse(LPM)) {
11646 if (It == D.DeclToUniqueName.end())
11652 UniqueDeclName = It->second;
11657 return FoundE == E;
11659 bool VisitStmt(
const Stmt *S) {
11660 for (
const Stmt *Child : S->children()) {
11663 if (
const auto *E = dyn_cast<Expr>(Child))
11671 explicit LastprivateConditionalRefChecker(
11674 std::tuple<const Expr *, const Decl *, StringRef, LValue, llvm::Function *>
11675 getFoundData()
const {
11676 return std::make_tuple(FoundE, FoundD, UniqueDeclName, IVLVal, FoundFn);
11683 StringRef UniqueDeclName,
11689 llvm::Constant *LastIV =
OMPBuilder.getOrCreateInternalVariable(
11690 LLIVTy,
getName({UniqueDeclName,
"iv"}));
11691 cast<llvm::GlobalVariable>(LastIV)->setAlignment(
11697 llvm::GlobalVariable *
Last =
OMPBuilder.getOrCreateInternalVariable(
11712 auto &&CodeGen = [&LastIVLVal, &IVLVal, IVVal, &LVal, &LastLVal,
11718 llvm::Value *CmpRes;
11720 CmpRes = CGF.
Builder.CreateICmpSLE(LastIVVal, IVVal);
11723 "Loop iteration variable must be integer.");
11724 CmpRes = CGF.
Builder.CreateICmpULE(LastIVVal, IVVal);
11728 CGF.
Builder.CreateCondBr(CmpRes, ThenBB, ExitBB);
11749 "Aggregates are not supported in lastprivate conditional.");
11772 if (!Checker.Visit(LHS))
11774 const Expr *FoundE;
11775 const Decl *FoundD;
11776 StringRef UniqueDeclName;
11778 llvm::Function *FoundFn;
11779 std::tie(FoundE, FoundD, UniqueDeclName, IVLVal, FoundFn) =
11780 Checker.getFoundData();
11781 if (FoundFn != CGF.
CurFn) {
11786 "Lastprivate conditional is not found in outer region.");
11787 QualType StructTy = std::get<0>(It->getSecond());
11788 const FieldDecl* FiredDecl = std::get<2>(It->getSecond());
11799 FiredLVal, llvm::AtomicOrdering::Unordered,
11817 auto It = llvm::find_if(
11819 if (It == Range.end() || It->Fn != CGF.
CurFn)
11823 "Lastprivates must be registered already.");
11827 for (
const auto &Pair : It->DeclToUniqueName) {
11828 const auto *VD = cast<VarDecl>(Pair.first->getCanonicalDecl());
11831 auto I = LPCI->getSecond().find(Pair.first);
11832 assert(I != LPCI->getSecond().end() &&
11833 "Lastprivate must be rehistered already.");
11835 LValue BaseLVal = std::get<3>(I->getSecond());
11839 llvm::Value *Cmp = CGF.
Builder.CreateIsNotNull(Res);
11843 CGF.
Builder.CreateCondBr(Cmp, ThenBB, DoneBB);
11868 "Unknown lastprivate conditional variable.");
11869 StringRef UniqueName = It->second;
11870 llvm::GlobalVariable *GV =
CGM.
getModule().getNamedGlobal(UniqueName);
11885 llvm_unreachable(
"Not supported in SIMD-only mode");
11892 llvm_unreachable(
"Not supported in SIMD-only mode");
11899 bool Tied,
unsigned &NumberOfParts) {
11900 llvm_unreachable(
"Not supported in SIMD-only mode");
11905 llvm::Function *OutlinedFn,
11907 const Expr *IfCond,
11908 llvm::Value *NumThreads) {
11909 llvm_unreachable(
"Not supported in SIMD-only mode");
11915 const Expr *Hint) {
11916 llvm_unreachable(
"Not supported in SIMD-only mode");
11922 llvm_unreachable(
"Not supported in SIMD-only mode");
11928 const Expr *Filter) {
11929 llvm_unreachable(
"Not supported in SIMD-only mode");
11934 llvm_unreachable(
"Not supported in SIMD-only mode");
11940 llvm_unreachable(
"Not supported in SIMD-only mode");
11948 llvm_unreachable(
"Not supported in SIMD-only mode");
11955 llvm_unreachable(
"Not supported in SIMD-only mode");
11962 bool ForceSimpleCall) {
11963 llvm_unreachable(
"Not supported in SIMD-only mode");
11970 llvm_unreachable(
"Not supported in SIMD-only mode");
11976 llvm_unreachable(
"Not supported in SIMD-only mode");
11982 llvm_unreachable(
"Not supported in SIMD-only mode");
11989 llvm_unreachable(
"Not supported in SIMD-only mode");
11995 llvm_unreachable(
"Not supported in SIMD-only mode");
12000 unsigned IVSize,
bool IVSigned,
12003 llvm_unreachable(
"Not supported in SIMD-only mode");
12007 llvm::Value *NumThreads,
12009 llvm_unreachable(
"Not supported in SIMD-only mode");
12013 ProcBindKind ProcBind,
12015 llvm_unreachable(
"Not supported in SIMD-only mode");
12022 llvm_unreachable(
"Not supported in SIMD-only mode");
12028 llvm_unreachable(
"Not supported in SIMD-only mode");
12033 llvm_unreachable(
"Not supported in SIMD-only mode");
12039 llvm::AtomicOrdering AO) {
12040 llvm_unreachable(
"Not supported in SIMD-only mode");
12045 llvm::Function *TaskFunction,
12047 const Expr *IfCond,
12049 llvm_unreachable(
"Not supported in SIMD-only mode");
12056 llvm_unreachable(
"Not supported in SIMD-only mode");
12063 assert(Options.SimpleReduction &&
"Only simple reduction is expected.");
12065 ReductionOps, Options);
12071 llvm_unreachable(
"Not supported in SIMD-only mode");
12076 bool IsWorksharingReduction) {
12077 llvm_unreachable(
"Not supported in SIMD-only mode");
12084 llvm_unreachable(
"Not supported in SIMD-only mode");
12089 llvm::Value *ReductionsPtr,
12091 llvm_unreachable(
"Not supported in SIMD-only mode");
12097 llvm_unreachable(
"Not supported in SIMD-only mode");
12103 llvm_unreachable(
"Not supported in SIMD-only mode");
12109 llvm_unreachable(
"Not supported in SIMD-only mode");
12114 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
12116 llvm_unreachable(
"Not supported in SIMD-only mode");
12121 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID,
const Expr *IfCond,
12122 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
12126 llvm_unreachable(
"Not supported in SIMD-only mode");
12130 llvm_unreachable(
"Not supported in SIMD-only mode");
12134 llvm_unreachable(
"Not supported in SIMD-only mode");
12144 llvm::Function *OutlinedFn,
12146 llvm_unreachable(
"Not supported in SIMD-only mode");
12150 const Expr *NumTeams,
12151 const Expr *ThreadLimit,
12153 llvm_unreachable(
"Not supported in SIMD-only mode");
12160 llvm_unreachable(
"Not supported in SIMD-only mode");
12166 llvm_unreachable(
"Not supported in SIMD-only mode");
12172 llvm_unreachable(
"Not supported in SIMD-only mode");
12177 llvm_unreachable(
"Not supported in SIMD-only mode");
12182 llvm_unreachable(
"Not supported in SIMD-only mode");
12187 const VarDecl *NativeParam)
const {
12188 llvm_unreachable(
"Not supported in SIMD-only mode");
12194 const VarDecl *TargetParam)
const {
12195 llvm_unreachable(
"Not supported in SIMD-only mode");
Provides LLVM's BitmaskEnum facility to enumeration types declared in namespace clang.
static llvm::Value * emitCopyprivateCopyFunction(CodeGenModule &CGM, llvm::Type *ArgsElemType, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps, SourceLocation Loc)
static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF, SourceLocation Loc, SmallString< 128 > &Buffer)
llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind convertCaptureClause(const VarDecl *VD)
static void emitOffloadingArrays(CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo, CGOpenMPRuntime::TargetDataInfo &Info, llvm::OpenMPIRBuilder &OMPBuilder, bool IsNonContiguous=false)
Emit the arrays used to pass the captures and map information to the offloading runtime library.
static RecordDecl * createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy, ArrayRef< PrivateDataTy > Privates)
llvm::Value * emitDynCGGroupMem(const OMPExecutableDirective &D, CodeGenFunction &CGF)
static void emitInitWithReductionInitializer(CodeGenFunction &CGF, const OMPDeclareReductionDecl *DRD, const Expr *InitOp, Address Private, Address Original, QualType Ty)
static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, Address OriginalBaseAddress, llvm::Value *Addr)
static void emitPrivatesInit(CodeGenFunction &CGF, const OMPExecutableDirective &D, Address KmpTaskSharedsPtr, LValue TDBase, const RecordDecl *KmpTaskTWithPrivatesQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef< PrivateDataTy > Privates, bool ForDup)
Emit initialization for private variables in task-based directives.
static llvm::Value * emitDestructorsFunction(CodeGenModule &CGM, SourceLocation Loc, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy)
static unsigned evaluateCDTSize(const FunctionDecl *FD, ArrayRef< ParamAttrTy > ParamAttrs)
static void EmitOMPAggregateReduction(CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar, const VarDecl *RHSVar, const llvm::function_ref< void(CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *)> &RedOpGen, const Expr *XExpr=nullptr, const Expr *EExpr=nullptr, const Expr *UpExpr=nullptr)
Emit reduction operation for each element of array (required for array sections) LHS op = RHS.
static void emitTargetCallFallback(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, CodeGenFunction &CGF)
static llvm::Value * emitReduceInitFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Emits reduction initializer function:
static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion)
static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy, llvm::PointerUnion< unsigned *, LValue * > Pos, const OMPTaskDataTy::DependData &Data, Address DependenciesArray)
static llvm::Value * emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc, const OMPTaskDataTy &Data, QualType PrivatesQTy, ArrayRef< PrivateDataTy > Privates)
Emit a privates mapping function for correct handling of private and firstprivate variables.
static llvm::Value * emitReduceCombFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N, const Expr *ReductionOp, const Expr *LHS, const Expr *RHS, const Expr *PrivateRef)
Emits reduction combiner function:
static RecordDecl * createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef< PrivateDataTy > Privates)
static llvm::Value * getAllocatorVal(CodeGenFunction &CGF, const Expr *Allocator)
Return allocator value from expression, or return a null allocator (default when no allocator specifi...
static llvm::Function * emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy, QualType SharedsPtrTy, llvm::Function *TaskFunction, llvm::Value *TaskPrivatesMap)
Emit a proxy function which accepts kmp_task_t as the second argument.
static void addAArch64VectorName(T VLEN, StringRef LMask, StringRef Prefix, char ISA, StringRef ParSeq, StringRef MangledName, bool OutputBecomesInput, llvm::Function *Fn)
static bool isAllocatableDecl(const VarDecl *VD)
static llvm::Value * getAlignmentValue(CodeGenModule &CGM, const VarDecl *VD)
Return the alignment from an allocate directive if present.
static void emitTargetCallKernelLaunch(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::Value *OutlinedFnID, CodeGenFunction::OMPTargetDataInfo &InputInfo, llvm::Value *&MapTypesArray, llvm::Value *&MapNamesArray, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter, CodeGenFunction &CGF, CodeGenModule &CGM)
static std::tuple< unsigned, unsigned, bool > getNDSWDS(const FunctionDecl *FD, ArrayRef< ParamAttrTy > ParamAttrs)
static const OMPExecutableDirective * getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D)
Check for inner distribute directive.
static std::pair< llvm::Value *, llvm::Value * > getPointerAndSize(CodeGenFunction &CGF, const Expr *E)
static const VarDecl * getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE)
static bool isTrivial(ASTContext &Ctx, const Expr *E)
Checks if the expression is constant or does not have non-trivial function calls.
static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind, bool Chunked, bool Ordered)
Map the OpenMP loop schedule to the runtime enumeration.
static void getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS, const Expr **E, int32_t &UpperBound, bool UpperBoundOnly, llvm::Value **CondVal)
Check for a num threads constant value (stored in DefaultVal), or expression (stored in E).
static llvm::Value * emitDeviceID(llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, CodeGenFunction &CGF)
static const OMPDeclareReductionDecl * getReductionInit(const Expr *ReductionOp)
Check if the combiner is a call to UDR combiner and if it is so return the UDR decl used for reductio...
static bool checkInitIsRequired(CodeGenFunction &CGF, ArrayRef< PrivateDataTy > Privates)
Check if duplication function is required for taskloops.
static bool checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD, ArrayRef< PrivateDataTy > Privates)
Checks if destructor function is required to be generated.
static llvm::TargetRegionEntryInfo getEntryInfoFromPresumedLoc(CodeGenModule &CGM, llvm::OpenMPIRBuilder &OMPBuilder, SourceLocation BeginLoc, llvm::StringRef ParentName="")
static std::string generateUniqueName(CodeGenModule &CGM, StringRef Prefix, const Expr *Ref)
Generates unique name for artificial threadprivate variables.
static void emitForStaticInitCall(CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId, llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2, const CGOpenMPRuntime::StaticRTInput &Values)
static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, LValue BaseLV)
static void getKmpAffinityType(ASTContext &C, QualType &KmpTaskAffinityInfoTy)
Builds kmp_depend_info, if it is not built yet, and builds flags type.
static void getDependTypes(ASTContext &C, QualType &KmpDependInfoTy, QualType &FlagsTy)
Builds kmp_depend_info, if it is not built yet, and builds flags type.
static llvm::Value * emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc, const OMPExecutableDirective &D, QualType KmpTaskTWithPrivatesPtrQTy, const RecordDecl *KmpTaskTWithPrivatesQTyRD, const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef< PrivateDataTy > Privates, bool WithLastIter)
Emit task_dup function (for initialization of private/firstprivate/lastprivate vars and last_iter fla...
static llvm::Value * emitReduceFiniFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Emits reduction finalizer function:
static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, QualType Type, bool EmitDeclareReductionInit, const Expr *Init, const OMPDeclareReductionDecl *DRD, Address SrcAddr=Address::invalid())
Emit initialization of arrays of complex types.
static bool getAArch64PBV(QualType QT, ASTContext &C)
Pass By Value (PBV), as defined in 3.1.2 of the AAVFABI.
static unsigned getAArch64LS(QualType QT, ParamKindTy Kind, ASTContext &C)
Computes the lane size (LS) of a return type or of an input parameter, as defined by LS(P) in 3....
static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM, const T *C, llvm::Value *ULoc, llvm::Value *ThreadID)
static RTLDependenceKindTy translateDependencyKind(OpenMPDependClauseKind K)
Translates internal dependency kind into the runtime kind.
static void emitTargetCallElse(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, CodeGenFunction &CGF)
static llvm::Function * emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty, const Expr *CombinerInitializer, const VarDecl *In, const VarDecl *Out, bool IsCombiner)
static void emitX86DeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn, const llvm::APSInt &VLENVal, ArrayRef< ParamAttrTy > ParamAttrs, OMPDeclareSimdDeclAttr::BranchStateTy State)
static void emitReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp)
Emit reduction combiner.
static std::string mangleVectorParameters(ArrayRef< ParamAttrTy > ParamAttrs)
Mangle the parameter part of the vector function name according to their OpenMP classification.
static llvm::Function * emitParallelOrTeamsOutlinedFunction(CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen)
static void emitAArch64DeclareSimdFunction(CodeGenModule &CGM, const FunctionDecl *FD, unsigned UserVLEN, ArrayRef< ParamAttrTy > ParamAttrs, OMPDeclareSimdDeclAttr::BranchStateTy State, StringRef MangledName, char ISA, unsigned VecRegSize, llvm::Function *Fn, SourceLocation SLoc)
Emit vector function attributes for AArch64, as defined in the AAVFABI.
static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array, unsigned Index, const VarDecl *Var)
Given an array of pointers to variables, project the address of a given variable.
llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind convertDeviceClause(const VarDecl *VD)
static bool isAssumedToBeNotEmitted(const ValueDecl *VD, bool IsDevice)
static void addAArch64AdvSIMDNDSNames(unsigned NDS, StringRef Mask, StringRef Prefix, char ISA, StringRef ParSeq, StringRef MangledName, bool OutputBecomesInput, llvm::Function *Fn)
static FieldDecl * addFieldToRecordDecl(ASTContext &C, DeclContext *DC, QualType FieldTy)
static ValueDecl * getDeclFromThisExpr(const Expr *E)
llvm::Constant * emitMappingInformation(CodeGenFunction &CGF, llvm::OpenMPIRBuilder &OMPBuilder, MappableExprsHandler::MappingExprInfo &MapExprs)
Emit a string constant containing the names of the values mapped to the offloading runtime library.
static RecordDecl * createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpRoutineEntryPointerQTy)
static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2)
static bool getAArch64MTV(QualType QT, ParamKindTy Kind)
Maps To Vector (MTV), as defined in 4.1.1 of the AAVFABI (2021Q1).
Defines the clang::FileManager interface and associated types.
This file defines OpenMP AST classes for clauses.
Defines some OpenMP-specific enums and functions.
Defines the SourceManager interface.
This file defines OpenMP AST classes for executable directives and clauses.
__DEVICE__ int max(int __a, int __b)
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
SourceManager & getSourceManager()
const ConstantArrayType * getAsConstantArrayType(QualType T) const
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
const ASTRecordLayout & getASTRecordLayout(const RecordDecl *D) const
Get or compute information about the layout of the specified record (struct/union/class) D,...
bool hasSameType(QualType T1, QualType T2) const
Determine whether the given types T1 and T2 are equivalent.
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
const LangOptions & getLangOpts() const
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth,...
CanQualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
const VariableArrayType * getAsVariableArrayType(QualType T) const
const TargetInfo & getTargetInfo() const
CharUnits getNonVirtualSize() const
getNonVirtualSize - Get the non-virtual size (in chars) of an object, which is the size of the object...
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Attr - This represents one attribute.
Represents a C++ constructor within a class.
Represents a C++ destructor within a class.
Represents a static or instance method of a struct/union/class.
const CXXRecordDecl * getParent() const
Return the parent of this method declaration, which is the class in which this method is defined.
QualType getFunctionObjectParameterType() const
Represents a C++ struct/union/class.
bool isLambda() const
Determine whether this class describes a lambda function object.
void getCaptureFields(llvm::DenseMap< const ValueDecl *, FieldDecl * > &Captures, FieldDecl *&ThisCapture) const
For a closure type, retrieve the mapping from captured variables and this to the non-static data memb...
unsigned getNumBases() const
Retrieves the number of base classes of this class.
base_class_range vbases()
capture_const_range captures() const
CXXDestructorDecl * getDestructor() const
Returns the destructor decl for this class.
CanProxy< U > castAs() const
A wrapper class around a pointer that always points to its canonical declaration.
Describes the capture of either a variable, or 'this', or variable-length array type.
bool capturesVariableByCopy() const
Determine whether this capture handles a variable by copy.
VarDecl * getCapturedVar() const
Retrieve the declaration of the variable being captured.
bool capturesVariableArrayType() const
Determine whether this capture handles a variable-length array type.
bool capturesThis() const
Determine whether this capture handles the C++ 'this' pointer.
bool capturesVariable() const
Determine whether this capture handles a variable (by reference).
This captures a statement into a function.
capture_iterator capture_end() const
Retrieve an iterator pointing past the end of the sequence of captures.
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Stmt * getCapturedStmt()
Retrieve the statement being captured.
bool capturesVariable(const VarDecl *Var) const
True if this variable has been captured.
capture_iterator capture_begin()
Retrieve an iterator pointing to the first capture.
CharUnits - This is an opaque type for sizes expressed in character units.
bool isZero() const
isZero - Test whether the quantity equals zero.
llvm::Align getAsAlign() const
getAsAlign - Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit b...
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
CharUnits alignTo(const CharUnits &Align) const
alignTo - Returns the next integer (mod 2**64) that is greater than or equal to this quantity and is ...
CharUnits getAlignment() const
Return the alignment of this pointer.
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Address withPointer(llvm::Value *NewPointer, KnownNonNull_t IsKnownNonNull) const
Return address with different pointer, but same element type and alignment.
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Address withAlignment(CharUnits NewAlignment) const
Return address with different alignment, but same pointer and element type.
llvm::Value * getPointer() const
llvm::PointerType * getType() const
Return the type of the pointer value.
static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateDefaultArtificial(CodeGenFunction &CGF, SourceLocation TemporaryLocation)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF)
Set the IRBuilder to not attach debug locations.
CGBlockInfo - Information to generate a block literal.
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Address CreateConstArrayGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = [n x T]* ... produce name = getelementptr inbounds addr, i64 0, i64 index where i64 is a...
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
Address CreateConstGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = T* ... produce name = getelementptr inbounds addr, i64 index where i64 is actually the t...
Address CreateGEP(Address Addr, llvm::Value *Index, const llvm::Twine &Name="")
MangleContext & getMangleContext()
Gets the mangle context.
CGFunctionInfo - Class to encapsulate the information about a function definition.
DisableAutoDeclareTargetRAII(CodeGenModule &CGM)
~DisableAutoDeclareTargetRAII()
Manages list of lastprivate conditional decls for the specified directive.
~LastprivateConditionalRAII()
static LastprivateConditionalRAII disable(CodeGenFunction &CGF, const OMPExecutableDirective &S)
NontemporalDeclsRAII(CodeGenModule &CGM, const OMPLoopDirective &S)
Struct that keeps all the relevant information that should be kept throughout a 'target data' region.
llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap
Map between the a declaration of a capture and the corresponding new llvm address where the runtime r...
~UntiedTaskLocalDeclsRAII()
UntiedTaskLocalDeclsRAII(CodeGenFunction &CGF, const llvm::MapVector< CanonicalDeclPtr< const VarDecl >, std::pair< Address, Address > > &LocalVars)
virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc)
Emits address of the word in a memory where current thread id is stored.
llvm::FunctionType * Kmpc_MicroTy
The type for a microtask which gets passed to __kmpc_fork_call().
llvm::StringSet ThreadPrivateWithDefinition
Set of threadprivate variables with the generated initializer.
CGOpenMPRuntime(CodeGenModule &CGM)
virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
Emit task region for the task directive.
void createOffloadEntriesAndInfoMetadata()
Creates all the offload entries in the current compilation unit along with the associated metadata.
const Expr * getNumTeamsExprForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &MinTeamsVal, int32_t &MaxTeamsVal)
Emit the number of teams for a target directive.
virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc)
Returns address of the threadprivate variable for the current thread.
void emitDeferredTargetDecls() const
Emit deferred declare target variables marked for deferred emission.
virtual llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST)
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
bool markAsGlobalTarget(GlobalDecl GD)
Marks the declaration as already emitted for the device code and returns true, if it was marked alrea...
virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device)
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
virtual void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc)
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)...
QualType SavedKmpTaskloopTQTy
Saved kmp_task_t for taskloop-based directive.
virtual void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps)
Emits a single region.
virtual bool emitTargetGlobal(GlobalDecl GD)
Emit the global GD if it is meaningful for the target.
void setLocThreadIdInsertPt(CodeGenFunction &CGF, bool AtCurrentPoint=false)
std::string getOutlinedHelperName(StringRef Name) const
Get the function name of an outlined region.
bool HasEmittedDeclareTargetRegion
Flag for keeping track of weather a device routine has been emitted.
llvm::Constant * getOrCreateThreadPrivateCache(const VarDecl *VD)
If the specified mangled name is not in the module, create and return threadprivate cache object.
virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal)
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF, const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const
Choose default schedule type and chunk value for the schedule clause.
virtual std::pair< llvm::Function *, llvm::Function * > getUserDefinedReduction(const OMPDeclareReductionDecl *D)
Get combiner/initializer for the specified user-defined reduction, if any.
virtual bool isGPU() const
Returns true if the current target is a GPU.
static const Stmt * getSingleCompoundChild(ASTContext &Ctx, const Stmt *Body)
Checks if the Body is the CompoundStmt and returns its child statement iff there is only one that is ...
virtual void emitDeclareTargetFunction(const FunctionDecl *FD, llvm::GlobalValue *GV)
Emit code for handling declare target functions in the runtime.
llvm::Type * getKmpc_MicroPointerTy()
Returns pointer to kmpc_micro type.
bool HasRequiresUnifiedSharedMemory
Flag for keeping track of weather a requires unified_shared_memory directive is present.
llvm::Value * emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc, unsigned Flags=0, bool EmitLoc=false)
Emits object of ident_t type with info for source location.
bool isLocalVarInUntiedTask(CodeGenFunction &CGF, const VarDecl *VD) const
Returns true if the variable is a local variable in untied task.
virtual void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars)
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
virtual void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancellation point' construct.
void emitUDMapperArrayInitOrDel(CodeGenFunction &MapperCGF, llvm::Value *Handle, llvm::Value *BasePtr, llvm::Value *Ptr, llvm::Value *Size, llvm::Value *MapType, llvm::Value *MapName, CharUnits ElementSize, llvm::BasicBlock *ExitBB, bool IsInit)
Emit the array initialization or deletion portion for user-defined mapper code generation.
virtual Address getAddrOfDeclareTargetVar(const VarDecl *VD)
Returns the address of the variable marked as declare target with link clause OR as declare target wi...
virtual llvm::Function * emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF=nullptr)
Emit a code for initialization of threadprivate variable.
FunctionUDMMapTy FunctionUDMMap
llvm::MapVector< CanonicalDeclPtr< const VarDecl >, std::pair< Address, Address > > UntiedLocalVarsAddressesMap
llvm::Function * getOrCreateUserDefinedMapperFunc(const OMPDeclareMapperDecl *D)
Get the function for the specified user-defined mapper.
OpenMPLocThreadIDMapTy OpenMPLocThreadIDMap
virtual void functionFinished(CodeGenFunction &CGF)
Cleans up references to the objects in finished function.
void emitCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee Callee, ArrayRef< llvm::Value * > Args=std::nullopt) const
Emits Callee function call with arguments Args with location Loc.
virtual llvm::Function * emitTeamsOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP teams directive D.
QualType KmpTaskTQTy
Type typedef struct kmp_task { void * shareds; /**< pointer to block of pointers to shared vars / k...
llvm::OpenMPIRBuilder OMPBuilder
An OpenMP-IR-Builder instance.
virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr * > NumIterations)
Emit initialization for doacross loop nesting support.
virtual void adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF, const OMPExecutableDirective &D) const
Adjust some parameters for the target-based directives, like addresses of the variables captured by r...
FunctionUDRMapTy FunctionUDRMap
virtual void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, CGOpenMPRuntime::TargetDataInfo &Info)
Emit the target data mapping code associated with D.
virtual unsigned getDefaultLocationReserved2Flags() const
Returns additional flags that can be stored in reserved_2 field of the default location.
void computeMinAndMaxThreadsAndTeams(const OMPExecutableDirective &D, CodeGenFunction &CGF, int32_t &MinThreadsVal, int32_t &MaxThreadsVal, int32_t &MinTeamsVal, int32_t &MaxTeamsVal)
Helper to determine the min/max number of threads/teams for D.
virtual Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const
Gets the address of the native argument basing on the address of the target-specific parameter.
void emitUsesAllocatorsFini(CodeGenFunction &CGF, const Expr *Allocator)
Destroys user defined allocators specified in the uses_allocators clause.
QualType KmpTaskAffinityInfoTy
Type typedef struct kmp_task_affinity_info { kmp_intptr_t base_addr; size_t len; struct { bool flag1 ...
llvm::SmallVector< NontemporalDeclsSet, 4 > NontemporalDeclsStack
Stack for list of declarations in current context marked as nontemporal.
llvm::Value * emitNumTeamsForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D)
virtual void emitTargetOutlinedFunctionHelper(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Helper to emit outlined function for 'target' directive.
void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName)
Start scanning from statement S and emit all target regions found along the way.
SmallVector< llvm::Value *, 4 > emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy, const OMPTaskDataTy::DependData &Data)
virtual void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc)
Emit a taskgroup region.
llvm::DenseMap< llvm::Function *, llvm::DenseMap< CanonicalDeclPtr< const Decl >, std::tuple< QualType, const FieldDecl *, const FieldDecl *, LValue > > > LastprivateConditionalToTypes
Maps local variables marked as lastprivate conditional to their internal types.
virtual bool emitTargetGlobalVariable(GlobalDecl GD)
Emit the global variable if it is a valid device global variable.
virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams,...
bool hasRequiresUnifiedSharedMemory() const
Return whether the unified_shared_memory has been specified.
virtual Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, QualType VarType, StringRef Name)
Creates artificial threadprivate variable with name Name and type VarType.
void emitUserDefinedMapper(const OMPDeclareMapperDecl *D, CodeGenFunction *CGF=nullptr)
Emit the function for the user defined mapper construct.
bool HasEmittedTargetRegion
Flag for keeping track of weather a target region has been emitted.
void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy, LValue PosLVal, const OMPTaskDataTy::DependData &Data, Address DependenciesArray)
std::string getReductionFuncName(StringRef Name) const
Get the function name of a reduction function.
virtual void processRequiresDirective(const OMPRequiresDecl *D)
Perform check on requires decl to ensure that target architecture supports unified addressing.
llvm::DenseSet< CanonicalDeclPtr< const Decl > > AlreadyEmittedTargetDecls
List of the emitted declarations.
virtual llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data)
Emit a code for initialization of task reduction clause.
llvm::Value * getThreadID(CodeGenFunction &CGF, SourceLocation Loc)
Gets thread id value for the current thread.
void emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, OpenMPDependClauseKind NewDepKind, SourceLocation Loc)
Updates the dependency kind in the specified depobj object.
virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD, SourceLocation Loc)
Gets the address of the global copy used for lastprivate conditional update, if any.
virtual void emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, Expr *ME, bool IsFatal)
Emit __kmpc_error call for error directive extern void __kmpc_error(ident_t *loc, int severity,...
void clearLocThreadIdInsertPt(CodeGenFunction &CGF)
virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc)
Emits code for a taskyield directive.
std::string getName(ArrayRef< StringRef > Parts) const
Get the platform-specific name separator.
QualType KmpRoutineEntryPtrQTy
virtual void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc, llvm::AtomicOrdering AO)
Emit flush of the variables specified in 'omp flush' directive.
virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPTaskDataTy &Data)
Emit code for 'taskwait' directive.
virtual void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc)
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, int proc_bind) to generat...
void emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LValue IVLVal, StringRef UniqueDeclName, LValue LVal, SourceLocation Loc)
Emit update for lastprivate conditional data.
virtual void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
Emit task region for the taskloop directive.
virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false)
Emit an implicit/explicit barrier for OpenMP threads.
static unsigned getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind)
Returns default flags for the barriers depending on the directive, for which this barier is going to ...
virtual bool emitTargetFunctions(GlobalDecl GD)
Emit the target regions enclosed in GD function definition or the function itself in case it is a val...
TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const OMPTaskDataTy &Data)
Emit task region for the task directive.
llvm::Value * emitTargetNumIterationsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Return the trip count of loops associated with constructs / 'target teams distribute' and 'teams dist...
llvm::StringMap< llvm::AssertingVH< llvm::GlobalVariable >, llvm::BumpPtrAllocator > InternalVars
An ordered map of auto-generated variables to their unique names.
virtual void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values)
llvm::SmallVector< UntiedLocalVarsAddressesMap, 4 > UntiedLocalVarsStack
virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind)
Call the appropriate runtime routine to notify that we finished all the work with current loop.
virtual void emitThreadLimitClause(CodeGenFunction &CGF, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_set_thread_limit(ident_t *loc, kmp_int32 global_tid, kmp_int32 thread_limit...
void emitIfClause(CodeGenFunction &CGF, const Expr *Cond, const RegionCodeGenTy &ThenGen, const RegionCodeGenTy &ElseGen)
Emits code for OpenMP 'if' clause using specified CodeGen function.
Address emitDepobjDependClause(CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies, SourceLocation Loc)
Emits list of dependecies based on the provided data (array of dependence/expression pairs) for depob...
bool isNontemporalDecl(const ValueDecl *VD) const
Checks if the VD variable is marked as nontemporal declaration in current context.
virtual llvm::Function * emitParallelOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP parallel directive D.
const Expr * getNumThreadsExprForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &UpperBound, bool UpperBoundOnly, llvm::Value **CondExpr=nullptr, const Expr **ThreadLimitExpr=nullptr)
Check for a number of threads upper bound constant value (stored in UpperBound), or expression (retur...
llvm::SmallVector< LastprivateConditionalData, 4 > LastprivateConditionalStack
Stack for list of addresses of declarations in current context marked as lastprivate conditional.
virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values)
Call the appropriate runtime routine to initialize it before start of loop.
virtual void emitDeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn)
Marks function Fn with properly mangled versions of vector functions.
llvm::AtomicOrdering getDefaultMemoryOrdering() const
Gets default memory ordering as specified in requires directive.
llvm::SmallDenseSet< CanonicalDeclPtr< const Decl > > NontemporalDeclsSet
virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static non-chunked.
llvm::Value * getCriticalRegionLock(StringRef CriticalName)
Returns corresponding lock object for the specified critical region name.
virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancel' construct.
QualType SavedKmpTaskTQTy
Saved kmp_task_t for task directive.
virtual void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc)
Emits a master region.
virtual llvm::Function * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts)
Emits outlined function for the OpenMP task directive D.
llvm::DenseMap< llvm::Function *, unsigned > FunctionToUntiedTaskStackMap
Maps function to the position of the untied task locals stack.
void emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal, SourceLocation Loc)
Emits the code to destroy the dependency object provided in depobj directive.
virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Required to resolve existing problems in the runtime.
llvm::ArrayType * KmpCriticalNameTy
Type kmp_critical_name, originally defined as typedef kmp_int32 kmp_critical_name[8];.
virtual void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C)
Emit code for doacross ordered directive with 'depend' clause.
llvm::DenseMap< const OMPDeclareMapperDecl *, llvm::Function * > UDMMap
Map from the user-defined mapper declaration to its corresponding functions.
virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF, const Expr *LHS)
Checks if the provided LVal is lastprivate conditional and emits the code to update the value of the ...
std::pair< llvm::Value *, LValue > getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal, SourceLocation Loc)
Returns the number of the elements and the address of the depobj dependency array.
llvm::SmallDenseSet< const VarDecl * > DeferredGlobalVariables
List of variables that can become declare target implicitly and, thus, must be emitted.
void emitUsesAllocatorsInit(CodeGenFunction &CGF, const Expr *Allocator, const Expr *AllocatorTraits)
Initializes user defined allocators specified in the uses_allocators clauses.
llvm::Type * KmpRoutineEntryPtrTy
Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *);.
llvm::Type * getIdentTyPointerTy()
Returns pointer to ident_t type.
void emitSingleReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp, const Expr *PrivateRef, const DeclRefExpr *LHS, const DeclRefExpr *RHS)
Emits single reduction combiner.
llvm::OpenMPIRBuilder & getOMPBuilder()
virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Emit outilined function for 'target' directive.
virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr)
Emits a critical region.
virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned)
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
llvm::Value * emitNumThreadsForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D)
Emit an expression that denotes the number of threads a target region shall use.
void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor, llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc)
Emits initialization code for the threadprivate variables.
virtual void emitUserDefinedReduction(CodeGenFunction *CGF, const OMPDeclareReductionDecl *D)
Emit code for the specified user defined reduction construct.
virtual void checkAndEmitSharedLastprivateConditional(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::DenseSet< CanonicalDeclPtr< const VarDecl > > &IgnoredDecls)
Checks if the lastprivate conditional was updated in inner region and writes the value.
QualType KmpDimTy
struct kmp_dim { // loop bounds info casted to kmp_int64 kmp_int64 lo; // lower kmp_int64 up; // uppe...
virtual void emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool HasCancel=false)
Emit code for the directive that does not require outlining.
virtual void registerTargetGlobalVariable(const VarDecl *VD, llvm::Constant *Addr)
Checks if the provided global decl GD is a declare target variable and registers it when emitting cod...
virtual void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D)
Emits OpenMP-specific function prolog.
virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond, llvm::Value *NumThreads)
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
void emitKmpRoutineEntryT(QualType KmpInt32Ty)
Build type kmp_routine_entry_t (if not built yet).
virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static chunked.
virtual void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Emit the target offloading code associated with D.
virtual bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS)
Checks if the variable has associated OMPAllocateDeclAttr attribute with the predefined allocator and...
llvm::AtomicOrdering RequiresAtomicOrdering
Atomic ordering from the omp requires directive.
virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps, ReductionOptionsTy Options)
Emit a code for reduction clause.
std::pair< llvm::Value *, Address > emitDependClause(CodeGenFunction &CGF, ArrayRef< OMPTaskDataTy::DependData > Dependencies, SourceLocation Loc)
Emits list of dependecies based on the provided data (array of dependence/expression pairs).
llvm::StringMap< llvm::WeakTrackingVH > EmittedNonTargetVariables
List of the global variables with their addresses that should not be emitted for the target.
virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const
Check if the specified ScheduleKind is dynamic.
Address emitLastprivateConditionalInit(CodeGenFunction &CGF, const VarDecl *VD)
Create specialized alloca to handle lastprivate conditionals.
virtual void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads)
Emit an ordered region.
virtual void emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, ArrayRef< llvm::Value * > Args=std::nullopt) const
Emits call of the outlined function with the provided arguments, translating these arguments to corre...
virtual Address getAddressOfLocalVariable(CodeGenFunction &CGF, const VarDecl *VD)
Gets the OpenMP-specific address of the local variable.
virtual void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, bool IsWorksharingReduction)
Emits the following code for reduction clause with task modifier:
virtual void emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter=nullptr)
Emits a masked region.
QualType KmpDependInfoTy
Type typedef struct kmp_depend_info { kmp_intptr_t base_addr; size_t len; struct { bool in:1; bool ou...
llvm::Function * emitReductionFunction(StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps)
Emits reduction function.
virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues)
Call the appropriate runtime routine to initialize it before start of loop.
Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal) override
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr) override
Emits a critical region.
void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values) override
void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) override
Call the appropriate runtime routine to initialize it before start of loop.
bool emitTargetGlobalVariable(GlobalDecl GD) override
Emit the global variable if it is a valid device global variable.
llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST) override
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
llvm::Function * emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF=nullptr) override
Emit a code for initialization of threadprivate variable.
void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device) override
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
llvm::Function * emitTeamsOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) override
Emits outlined function for the specified OpenMP teams directive D.
void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps, ReductionOptionsTy Options) override
Emit a code for reduction clause.
void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc, llvm::AtomicOrdering AO) override
Emit flush of the variables specified in 'omp flush' directive.
void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C) override
Emit code for doacross ordered directive with 'depend' clause.
void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc) override
Emits a masked region.
Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, QualType VarType, StringRef Name) override
Creates artificial threadprivate variable with name Name and type VarType.
Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc) override
Returns address of the threadprivate variable for the current thread.
void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps) override
Emits a single region.
void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N) override
Required to resolve existing problems in the runtime.
llvm::Function * emitParallelOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) override
Emits outlined function for the specified OpenMP parallel directive D.
void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion) override
Emit code for 'cancellation point' construct.
void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false) override
Emit an implicit/explicit barrier for OpenMP threads.
Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const override
Gets the address of the native argument basing on the address of the target-specific parameter.
void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars) override
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned) override
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
bool emitTargetGlobal(GlobalDecl GD) override
Emit the global GD if it is meaningful for the target.
void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, bool IsWorksharingReduction) override
Emits the following code for reduction clause with task modifier:
void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads) override
Emit an ordered region.
void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind) override
Call the appropriate runtime routine to notify that we finished all the work with current loop.
llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data) override
Emit a code for initialization of task reduction clause.
void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond, llvm::Value *NumThreads) override
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc) override
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, int proc_bind) to generat...
void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) override
Emit outilined function for 'target' directive.
void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc) override
Emits a master region.
void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc) override
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams,...
const VarDecl * translateParameter(const FieldDecl *FD, const VarDecl *NativeParam) const override
Translates the native parameter of outlined function if this is required for target.
void emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter=nullptr) override
Emits a masked region.
void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data) override
Emit task region for the task directive.
void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter) override
Emit the target offloading code associated with D.
bool emitTargetFunctions(GlobalDecl GD) override
Emit the target regions enclosed in GD function definition or the function itself in case it is a val...
void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc) override
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)...
void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr * > NumIterations) override
Emit initialization for doacross loop nesting support.
void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion) override
Emit code for 'cancel' construct.
void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPTaskDataTy &Data) override
Emit code for 'taskwait' directive.
void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc) override
Emit a taskgroup region.
void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, CGOpenMPRuntime::TargetDataInfo &Info) override
Emit the target data mapping code associated with D.
void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues) override
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
llvm::Function * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts) override
Emits outlined function for the OpenMP task directive D.
void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data) override
Emit task region for the taskloop directive.
CGRecordLayout - This class handles struct and union layout info while lowering AST types to LLVM typ...
unsigned getNonVirtualBaseLLVMFieldNo(const CXXRecordDecl *RD) const
llvm::StructType * getLLVMType() const
Return the "complete object" LLVM type associated with this record.
llvm::StructType * getBaseSubobjectLLVMType() const
Return the "base subobject" LLVM type associated with this record.
unsigned getLLVMFieldNo(const FieldDecl *FD) const
Return llvm::StructType element number that corresponds to the field FD.
unsigned getVirtualBaseIndex(const CXXRecordDecl *base) const
Return the LLVM field index corresponding to the given virtual base.
virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S)
Emit the captured statement body.
RAII for correct setting/restoring of CapturedStmtInfo.
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
bool Privatize()
Privatizes local variables previously registered as private.
bool addPrivate(const VarDecl *LocalVD, Address Addr)
Registers LocalVD variable as a private with Addr as the address of the corresponding private variabl...
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited.
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
void EmitNullInitialization(Address DestPtr, QualType Ty)
EmitNullInitialization - Generate code to set a value of the given type to null, If the type contains...
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
LValue EmitOMPArraySectionExpr(const OMPArraySectionExpr *E, bool IsLowerBound=true)
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount, Stmt::Likelihood LH=Stmt::LH_None, const Expr *ConditionalOp=nullptr)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target)
The given basic block lies in the current EH scope, but may be a target of a potentially scope-crossi...
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
FieldDecl * LambdaThisCaptureField
CGCapturedStmtInfo * CapturedStmtInfo
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Load a pointer with type PtrTy stored at address Ptr.
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
llvm::Value * emitArrayLength(const ArrayType *arrayType, QualType &baseType, Address &addr)
emitArrayLength - Compute the length of an array, even if it's a VLA, and drill down to the base elem...
VlaSizePair getVLASize(const VariableArrayType *vla)
Returns an LLVM value that corresponds to the size, in non-variably-sized elements,...
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
void EmitVariablyModifiedType(QualType Ty)
EmitVLASize - Capture all the sizes for the VLA expressions in the given variably-modified type and s...
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
const LangOptions & getLangOpts() const
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S)
Emit device code for the target simd directive.
LValue EmitLValueForFieldInitialization(LValue Base, const FieldDecl *Field)
EmitLValueForFieldInitialization - Like EmitLValueForField, except that if the Field is a reference,...
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
const CodeGen::CGBlockInfo * BlockInfo
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit)
EmitExprAsInit - Emits the code necessary to initialize a location in memory with the given initializ...
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
llvm::Type * ConvertTypeForMem(QualType T)
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
llvm::AssertingVH< llvm::Instruction > AllocaInsertPt
AllocaInsertPoint - This is an instruction in the entry block before which we prefer to insert alloca...
void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy)
Emit an aggregate assignment.
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value * > &CapturedVars)
JumpDest ReturnBlock
ReturnBlock - Unified return block.
static void EmitOMPTargetTeamsGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsGenericLoopDirective &S)
Emit device code for the target teams loop directive.
LValue EmitLValueForField(LValue Base, const FieldDecl *Field)
Destroyer * getDestroyer(QualType::DestructionKind destructionKind)
static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
void emitDestroy(Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray)
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc)
EmitLoadOfComplex - Load a complex number from the specified l-value.
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
bool isTrivialInitializer(const Expr *Init)
Determine whether the given initializer is trivial in the sense that it requires no code to be genera...
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block,...
void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy, AggValueSlot::Overlap_t MayOverlap, bool isVolatile=false)
EmitAggregateCopy - Emit an aggregate copy.
Address CreateDefaultAlignTempAlloca(llvm::Type *Ty, const Twine &Name="tmp")
CreateDefaultAlignedTempAlloca - This creates an alloca with the default ABI alignment of the given L...
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
llvm::CallInst * EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
ASTContext & getContext() const
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy)
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
void EmitBranchThroughCleanup(JumpDest Dest)
EmitBranchThroughCleanup - Emit a branch from the current insert block through the normal cleanup han...
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
void pushDestroy(QualType::DestructionKind dtorKind, Address addr, QualType type)
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant,...
void EmitAutoVarCleanups(const AutoVarEmission &emission)
bool needsEHCleanup(QualType::DestructionKind kind)
Determines whether an EH cleanup is required to destroy a type with the given destruction kind.
llvm::DenseMap< const ValueDecl *, FieldDecl * > LambdaCaptureFields
llvm::CallInst * EmitRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
llvm::Type * ConvertType(QualType T)
CodeGenTypes & getTypes() const
static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Emit device code for the target teams distribute parallel for simd directive.
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
LValue EmitStringLiteralLValue(const StringLiteral *E)
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
Address LoadCXXThisAddress()
LValue EmitOMPSharedLValue(const Expr *E)
Emits the lvalue for the expression with possibly captured variable.
llvm::Value * EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr, ArrayRef< llvm::Value * > IdxList, bool SignedIndices, bool IsSubtraction, SourceLocation Loc, const Twine &Name="")
Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to detect undefined behavior whe...
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S, SourceLocation Loc)
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
static void EmitOMPTargetParallelGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelGenericLoopDirective &S)
Emit device code for the target parallel loop directive.
LValue EmitLoadOfReferenceLValue(LValue RefLVal)
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler's counter for the given statement by StepV.
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
This class organizes the cross-function state that is used while generating LLVM code.
void handleCUDALaunchBoundsAttr(llvm::Function *F, const CUDALaunchBoundsAttr *A, int32_t *MaxThreadsVal=nullptr, int32_t *MinBlocksVal=nullptr, int32_t *MaxClusterRankVal=nullptr)
Emit the IR encoding to attach the CUDA launch bounds attribute to F.
void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
llvm::Module & getModule() const
void addCompilerUsedGlobal(llvm::GlobalValue *GV)
Add a global to a list to be added to the llvm.compiler.used metadata.
CharUnits GetTargetTypeStoreSize(llvm::Type *Ty) const
Return the store size, in character units, of the given LLVM type.
void handleAMDGPUWavesPerEUAttr(llvm::Function *F, const AMDGPUWavesPerEUAttr *A)
Emit the IR encoding to attach the AMD GPU waves-per-eu attribute to F.
DiagnosticsEngine & getDiags() const
const LangOptions & getLangOpts() const
CharUnits getNaturalTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, bool forPointeeType=false)
CodeGenTypes & getTypes()
const TargetInfo & getTarget() const
void EmitGlobal(GlobalDecl D)
Emit code for a single global function or var decl.
void handleAMDGPUFlatWorkGroupSizeAttr(llvm::Function *F, const AMDGPUFlatWorkGroupSizeAttr *A, const ReqdWorkGroupSizeAttr *ReqdWGS=nullptr, int32_t *MinThreadsVal=nullptr, int32_t *MaxThreadsVal=nullptr)
Emit the IR encoding to attach the AMD GPU flat-work-group-size attribute to F.
llvm::GlobalValue::LinkageTypes getLLVMLinkageVarDefinition(const VarDecl *VD)
Returns LLVM linkage for a declarator.
CGCXXABI & getCXXABI() const
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
const llvm::Triple & getTriple() const
TBAAAccessInfo getTBAAInfoForSubobject(LValue Base, QualType AccessType)
getTBAAInfoForSubobject - Get TBAA information for an access with a given base lvalue.
llvm::Constant * GetAddrOfGlobal(GlobalDecl GD, ForDefinition_t IsForDefinition=NotForDefinition)
ASTContext & getContext() const
const TargetCodeGenInfo & getTargetCodeGenInfo()
const CodeGenOptions & getCodeGenOpts() const
StringRef getMangledName(GlobalDecl GD)
std::optional< CharUnits > getOMPAllocateAlignment(const VarDecl *VD)
Return the alignment specified in an allocate directive, if present.
llvm::GlobalValue * GetGlobalValue(StringRef Ref)
llvm::Constant * EmitNullConstant(QualType T)
Return the result of value-initializing the given type, i.e.
llvm::Function * CreateGlobalInitOrCleanUpFunction(llvm::FunctionType *ty, const Twine &name, const CGFunctionInfo &FI, SourceLocation Loc=SourceLocation(), bool TLS=false, llvm::GlobalVariable::LinkageTypes Linkage=llvm::GlobalVariable::InternalLinkage)
llvm::ConstantInt * getSize(CharUnits numChars)
Emit the given number of characters as a value of type size_t.
llvm::Type * ConvertType(QualType T)
ConvertType - Convert type T into a llvm::Type.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
const CGRecordLayout & getCGRecordLayout(const RecordDecl *)
getCGRecordLayout - Return record layout info for the given record decl.
llvm::Type * ConvertTypeForMem(QualType T, bool ForBitField=false)
ConvertTypeForMem - Convert type T into a llvm::Type.
const CGFunctionInfo & arrangeNullaryFunction()
A nullary function is a freestanding function of type 'void ()'.
Information for lazily generating a cleanup.
bool requiresLandingPad() const
void popTerminate()
Pops a terminate handler off the stack.
void pushTerminate()
Push a terminate handler on the stack.
FunctionArgList - Type for representing both the decl and type of parameters to a function.
LValue - This represents an lvalue references.
CharUnits getAlignment() const
Address getAddress(CodeGenFunction &CGF) const
const Qualifiers & getQuals() const
LValueBaseInfo getBaseInfo() const
llvm::Value * getPointer(CodeGenFunction &CGF) const
TBAAAccessInfo getTBAAInfo() const
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
virtual void Enter(CodeGenFunction &CGF)
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
static RValue get(llvm::Value *V)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
Class intended to support codegen of all kind of the reduction clauses.
LValue getSharedLValue(unsigned N) const
Returns LValue for the reduction item.
const Expr * getRefExpr(unsigned N) const
Returns the base declaration of the reduction item.
LValue getOrigLValue(unsigned N) const
Returns LValue for the original reduction item.
bool needCleanups(unsigned N)
Returns true if the private copy requires cleanups.
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
const VarDecl * getBaseDecl(unsigned N) const
Returns the base declaration of the reduction item.
QualType getPrivateType(unsigned N) const
Return the type of the private item.
bool usesReductionInitializer(unsigned N) const
Returns true if the initialization of the reduction item uses initializer from declare reduction cons...
void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N)
Emits lvalue for the shared and original reduction item.
void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, llvm::function_ref< bool(CodeGenFunction &)> DefaultInit)
Performs initialization of the private copy for the reduction item.
std::pair< llvm::Value *, llvm::Value * > getSizes(unsigned N) const
Returns the size of the reduction item (in chars and total number of elements in the item),...
ReductionCodeGen(ArrayRef< const Expr * > Shareds, ArrayRef< const Expr * > Origs, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > ReductionOps)
void emitCleanups(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Emits cleanup code for the reduction item.
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations.
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
void operator()(CodeGenFunction &CGF) const
void setAction(PrePostActionTy &Action) const
virtual void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const
setTargetAttributes - Provides a convenient hook to handle extra target-specific attributes for the g...
ConstStmtVisitor - This class implements a simple visitor for Stmt subclasses.
DeclContext - This is used only as base class of specific decl types that can act as declaration cont...
void addDecl(Decl *D)
Add the declaration D into this context.
A reference to a declared variable, function, enum, etc.
Decl - This represents one declaration (or definition), e.g.
ASTContext & getASTContext() const LLVM_READONLY
llvm::iterator_range< specific_attr_iterator< T > > specific_attrs() const
SourceLocation getLocation() const
DeclContext * getDeclContext()
virtual Decl * getCanonicalDecl()
Retrieves the "canonical" declaration of the given declaration.
SourceLocation getBeginLoc() const LLVM_READONLY
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
unsigned getCustomDiagID(Level L, const char(&FormatString)[N])
Return an ID for a diagnostic with the specified format string and level.
The return type of classify().
This represents one expression.
@ SE_AllowSideEffects
Allow any unmodeled side effect.
@ SE_AllowUndefinedBehavior
Allow UB that we can give a value, but not arbitrary unmodeled side effects.
Expr * IgnoreParenCasts() LLVM_READONLY
Skip past any parentheses and casts which might surround this expression until reaching a fixed point...
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx, SmallVectorImpl< PartialDiagnosticAt > *Diag=nullptr) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
bool isEvaluatable(const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects) const
isEvaluatable - Call EvaluateAsRValue to see if this expression can be constant folded without side-e...
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
bool isIntegerConstantExpr(const ASTContext &Ctx, SourceLocation *Loc=nullptr) const
bool EvaluateAsBooleanCondition(bool &Result, const ASTContext &Ctx, bool InConstantContext=false) const
EvaluateAsBooleanCondition - Return true if this is a constant which we can fold and convert to a boo...
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
std::optional< llvm::APSInt > getIntegerConstantExpr(const ASTContext &Ctx, SourceLocation *Loc=nullptr) const
isIntegerConstantExpr - Return the value if this expression is a valid integer constant expression.
bool hasNonTrivialCall(const ASTContext &Ctx) const
Determine whether this expression involves a call to any function that is not trivial.
Represents a member of a struct/union/class.
static FieldDecl * Create(const ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, Expr *BW, bool Mutable, InClassInitStyle InitStyle)
Represents a function declaration or definition.
const ParmVarDecl * getParamDecl(unsigned i) const
QualType getReturnType() const
ArrayRef< ParmVarDecl * > parameters() const
FunctionDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
unsigned getNumParams() const
Return the number of parameters this function must have based on its FunctionType.
GlobalDecl - represents a global declaration.
const Decl * getDecl() const
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
static IntegerLiteral * Create(const ASTContext &C, const llvm::APInt &V, QualType type, SourceLocation l)
Returns a new integer literal with value 'V' and type 'type'.
Describes the capture of a variable or of this, or of a C++1y init-capture.
std::string OMPHostIRFile
Name of the IR file that contains the result of the OpenMP target host code generation.
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
virtual void mangleCanonicalTypeName(QualType T, raw_ostream &, bool NormalizeIntegers=false)=0
Generates a unique string for an externally visible type for use with TBAA or type uniquing.
MemberExpr - [C99 6.5.2.3] Structure and Union Members.
ValueDecl * getMemberDecl() const
Retrieve the member declaration to which this expression refers.
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
bool isExternallyVisible() const
This represents clause 'affinity' in the '#pragma omp task'-based directives.
static QualType getBaseOriginalType(const Expr *Base)
Return original type of the base expression for array section.
Class that represents a component of a mappable expression.
ArrayRef< MappableComponent > MappableExprComponentListRef
const Stmt * getPreInitStmt() const
Get pre-initialization statement for the clause.
This is a basic class for representing single OpenMP clause.
This represents '#pragma omp declare mapper ...' directive.
Expr * getMapperVarRef()
Get the variable declared in the mapper.
This represents '#pragma omp declare reduction ...' directive.
Expr * getInitializer()
Get initializer expression (if specified) of the declare reduction construct.
Expr * getInitPriv()
Get Priv variable of the initializer.
Expr * getCombinerOut()
Get Out variable of the combiner.
Expr * getCombinerIn()
Get In variable of the combiner.
Expr * getCombiner()
Get combiner expression of the declare reduction construct.
Expr * getInitOrig()
Get Orig variable of the initializer.
OMPDeclareReductionInitKind getInitializerKind() const
Get initializer kind.
This represents implicit clause 'depend' for the '#pragma omp task' directive.
This represents 'detach' clause in the '#pragma omp task' directive.
This represents 'device' clause in the '#pragma omp ...' directive.
This represents the 'doacross' clause for the '#pragma omp ordered' directive.
This is a basic class for representing single OpenMP executable directive.
CapturedStmt * getInnermostCapturedStmt()
Get innermost captured statement for the construct.
const CapturedStmt * getCapturedStmt(OpenMPDirectiveKind RegionKind) const
Returns the captured statement associated with the component region within the (combined) directive.
OpenMPDirectiveKind getDirectiveKind() const
SourceLocation getBeginLoc() const
Returns starting location of directive kind.
bool hasClausesOfKind() const
Returns true if the current directive has one or more clauses of a specific kind.
SourceLocation getEndLoc() const
Returns ending location of directive.
static const SpecificClause * getSingleClause(ArrayRef< OMPClause * > Clauses)
Gets a single clause of the specified kind associated with the current directive iff there is only on...
static llvm::iterator_range< specific_clause_iterator< SpecificClause > > getClausesOfKind(ArrayRef< OMPClause * > Clauses)
This represents clause 'firstprivate' in the '#pragma omp ...' directives.
This represents clause 'has_device_ptr' in the '#pragma omp ...' directives.
This represents 'if' clause in the '#pragma omp ...' directive.
Expr * getCondition() const
Returns condition.
This represents clause 'in_reduction' in the '#pragma omp task' directives.
This represents clause 'is_device_ptr' in the '#pragma omp ...' directives.
OpenMP 5.0 [2.1.6 Iterators] Iterators are identifiers that expand to multiple values in the clause o...
OMPIteratorHelperData & getHelper(unsigned I)
Fetches helper data for the specified iteration space.
unsigned numOfIterators() const
Returns number of iterator definitions.
Decl * getIteratorDecl(unsigned I)
Gets the iterator declaration for the given iterator.
This represents clause 'lastprivate' in the '#pragma omp ...' directives.
This represents clause 'linear' in the '#pragma omp ...' directives.
This is a common base class for loop directives ('omp simd', 'omp for', 'omp for simd' etc....
Expr * getStrideVariable() const
Expr * getUpperBoundVariable() const
Expr * getLowerBoundVariable() const
This represents clause 'map' in the '#pragma omp ...' directives.
This represents clause 'nontemporal' in the '#pragma omp ...' directives.
This represents 'nowait' clause in the '#pragma omp ...' directive.
This represents 'num_teams' clause in the '#pragma omp ...' directive.
This represents 'num_threads' clause in the '#pragma omp ...' directive.
This represents 'ordered' clause in the '#pragma omp ...' directive.
This represents clause 'private' in the '#pragma omp ...' directives.
This represents clause 'reduction' in the '#pragma omp ...' directives.
This represents '#pragma omp requires...' directive.
clauselist_range clauselists()
This represents 'thread_limit' clause in the '#pragma omp ...' directive.
This represents clause 'uses_allocators' in the '#pragma omp target'-based directives.
This represents 'ompx_attribute' clause in a directive that might generate an outlined function.
This represents 'ompx_dyn_cgroup_mem' clause in the '#pragma omp target ...' directive.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class.
Represents a parameter to a function.
PointerType - C99 6.7.5.1 - Pointer Declarators.
Represents an unpacked "presumed" location which can be presented to the user.
unsigned getColumn() const
Return the presumed column number of this location.
const char * getFilename() const
Return the presumed filename of this location.
unsigned getLine() const
Return the presumed line number of this location.
A (possibly-)qualified type.
void addRestrict()
Add the restrict qualifier to this QualType.
QualType withRestrict() const
bool isNull() const
Return true if this QualType doesn't point to a type yet.
const Type * getTypePtr() const
Retrieves a pointer to the underlying (unqualified) type.
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
QualType getCanonicalType() const
DestructionKind isDestructedType() const
Returns a nonzero value if objects of this type require non-trivial work to clean up after.
Represents a struct/union/class.
field_iterator field_end() const
field_range fields() const
virtual void completeDefinition()
Note that the definition of this type is now complete.
field_iterator field_begin() const
RecordDecl * getDecl() const
decl_type * getPreviousDecl()
Return the previous declaration of this declaration or NULL if this is the first declaration.
decl_type * getMostRecentDecl()
Returns the most recent (re)declaration of this declaration.
Base for LValueReferenceType and RValueReferenceType.
Scope - A scope is a transient data structure that is used while parsing the program.
Encodes a location in the source.
static SourceLocation getFromRawEncoding(UIntTy Encoding)
Turn a raw encoding of a SourceLocation object into a real SourceLocation.
bool isValid() const
Return true if this is a valid SourceLocation object.
UIntTy getRawEncoding() const
When a SourceLocation itself cannot be used, this returns an (opaque) 32-bit integer encoding for it.
This class handles loading and caching of source files into memory.
PresumedLoc getPresumedLoc(SourceLocation Loc, bool UseLineDirectives=true) const
Returns the "presumed" location of a SourceLocation specifies.
fileinfo_iterator fileinfo_end() const
SourceLocation translateFileLineCol(const FileEntry *SourceFile, unsigned Line, unsigned Col) const
Get the source location for the given file:line:col triplet.
fileinfo_iterator fileinfo_begin() const
A trivial tuple used to represent a source range.
Stmt - This represents one statement.
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top,...
SourceLocation getBeginLoc() const LLVM_READONLY
void startDefinition()
Starts the definition of this tag declaration.
bool isTLSSupported() const
Whether the target supports thread-local storage.
virtual bool hasFeature(StringRef Feature) const
Determine whether the given target has the given feature.
The base class of the type hierarchy.
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
bool isSignedIntegerOrEnumerationType() const
Determines whether this is an integer type that is signed or an enumeration types whose underlying ty...
const Type * getPointeeOrArrayElementType() const
If this is a pointer type, return the pointee type.
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
bool isPointerType() const
CanQualType getCanonicalTypeUnqualified() const
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
const T * castAs() const
Member-template castAs<specific type>.
bool isReferenceType() const
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
bool isLValueReferenceType() const
QualType getCanonicalTypeInternal() const
const RecordType * getAsStructureType() const
const Type * getBaseElementTypeUnsafe() const
Get the base element type of this type, potentially discarding type qualifiers.
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type.
bool isFloatingType() const
bool isUnsignedIntegerType() const
Return true if this is an integer type that is unsigned, according to C99 6.2.5p6 [which returns true...
bool isAnyPointerType() const
const T * getAs() const
Member-template getAs<specific type>'.
bool isRecordType() const
TagDecl * getAsTagDecl() const
Retrieves the TagDecl that this type refers to, either because the type is a TagType or because it is...
RecordDecl * getAsRecordDecl() const
Retrieves the RecordDecl this type refers to.
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Represents a variable declaration or definition.
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
VarDecl * getDefinition(ASTContext &)
Get the real (not just tentative) definition for this declaration.
bool hasExternalStorage() const
Returns true if a variable has extern or private_extern storage.
bool hasLocalStorage() const
Returns true if a variable with function scope is a non-static local variable.
@ DeclarationOnly
This declaration is only a declaration.
DefinitionKind hasDefinition(ASTContext &) const
Check whether this variable is defined in this translation unit.
bool isLocalVarDeclOrParm() const
Similar to isLocalVarDecl but also includes parameters.
const Expr * getAnyInitializer() const
Get the initializer for this variable, no matter which declaration it is attached to.
Represents a C array with a specified size that is not an integer-constant-expression.
Expr * getSizeExpr() const
specific_attr_iterator - Iterates over a subrange of an AttrVec, only providing attributes that are o...
@ Decl
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
@ LLVM_MARK_AS_BITMASK_ENUM
The JSON file list parser is used to communicate input to InstallAPI.
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
@ Private
'private' clause, allowed on 'parallel', 'serial', 'loop', 'parallel loop', and 'serial loop' constru...
@ Vector
'vector' clause, allowed on 'loop', Combined, and 'routine' directives.
@ Reduction
'reduction' clause, allowed on Parallel, Serial, Loop, and the combined constructs.
@ Present
'present' clause, allowed on Compute and Combined constructs, plus 'data' and 'declare'.
@ Device
'device' clause, allowed on the 'update' construct.
bool needsTaskBasedThreadLimit(OpenMPDirectiveKind DKind)
Checks if the specified target directive, combined or not, needs task based thread_limit.
@ Ctor_Complete
Complete object ctor.
bool isOpenMPTargetDataManagementDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target data offload directive.
llvm::omp::Directive OpenMPDirectiveKind
OpenMP directives.
@ ICIS_NoInit
No in-class initializer.
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
@ LCK_ByRef
Capturing by reference.
LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE()
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
@ OMPC_SCHEDULE_MODIFIER_last
@ OMPC_SCHEDULE_MODIFIER_unknown
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
OpenMPDistScheduleClauseKind
OpenMP attributes for 'dist_schedule' clause.
bool isOpenMPTaskingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of tasking directives - task, taskloop,...
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
@ Result
The result type of a method or function.
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
OpenMPDependClauseKind
OpenMP attributes for 'depend' clause.
@ Dtor_Complete
Complete object dtor.
@ Union
The "union" keyword.
bool isOpenMPLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a directive with an associated loop construct.
LangAS
Defines the address space values used by the address space qualifier of QualType.
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
@ VK_PRValue
A pr-value expression (in the C++11 taxonomy) produces a temporary value.
@ VK_LValue
An l-value expression is a reference to an object with independent storage.
void getOpenMPCaptureRegions(llvm::SmallVectorImpl< OpenMPDirectiveKind > &CaptureRegions, OpenMPDirectiveKind DKind)
Return the captured regions of an OpenMP directive.
@ OMPC_ATOMIC_DEFAULT_MEM_ORDER_unknown
if(T->getSizeExpr()) TRY_TO(TraverseStmt(T -> getSizeExpr()))
OpenMPMapModifierKind
OpenMP modifier kind for 'map' clause.
@ OMPC_MAP_MODIFIER_unknown
@ Other
Other implicit parameter.
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
OpenMPMapClauseKind
OpenMP mapping kind for 'map' clause.
YAML serialization mapping.
Maps the expression for the lastprivate variable to the global copy used to store new value because o...
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
llvm::BasicBlock * getBlock() const
unsigned NumberOfTargetItems
Address BasePointersArray
llvm::PointerType * VoidPtrTy
llvm::IntegerType * Int64Ty
llvm::PointerType * Int8PtrPtrTy
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::CallingConv::ID getRuntimeCC() const
llvm::IntegerType * SizeTy
llvm::PointerType * VoidPtrPtrTy
llvm::IntegerType * Int32Ty
llvm::IntegerType * IntPtrTy
llvm::IntegerType * IntTy
int
llvm::PointerType * Int8PtrTy
CharUnits getPointerAlign() const
OpenMPDependClauseKind DepKind
const Expr * IteratorExpr
SmallVector< const Expr *, 4 > DepExprs
EvalResult is a struct with detailed info about an evaluated expression.
Extra information about a function prototype.
Helper expressions and declaration for OMPIteratorExpr class for each iteration space.
Expr * CounterUpdate
Updater for the internal counter: ++CounterVD;.
Expr * Upper
Normalized upper bound.
Expr * Update
Update expression for the originally specified iteration variable, calculated as VD = Begin + Counter...
VarDecl * CounterVD
Internal normalized counter.
Data for list of allocators.
Expr * AllocatorTraits
Allocator traits.
Expr * Allocator
Allocator.
Scheduling data for loop-based OpenMP directives.
OpenMPScheduleClauseModifier M2
OpenMPScheduleClauseModifier M1
OpenMPScheduleClauseKind Schedule
Describes how types, statements, expressions, and declarations should be printed.