30#include "llvm/ADT/ArrayRef.h"
31#include "llvm/ADT/SetOperations.h"
32#include "llvm/ADT/SmallBitVector.h"
33#include "llvm/ADT/StringExtras.h"
34#include "llvm/Bitcode/BitcodeReader.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DerivedTypes.h"
37#include "llvm/IR/GlobalValue.h"
38#include "llvm/IR/InstrTypes.h"
39#include "llvm/IR/Value.h"
40#include "llvm/Support/AtomicOrdering.h"
41#include "llvm/Support/Format.h"
42#include "llvm/Support/raw_ostream.h"
49using namespace CodeGen;
50using namespace llvm::omp;
54class CGOpenMPRegionInfo :
public CodeGenFunction::CGCapturedStmtInfo {
57 enum CGOpenMPRegionKind {
60 ParallelOutlinedRegion,
71 const CGOpenMPRegionKind RegionKind,
74 : CGCapturedStmtInfo(CS,
CR_OpenMP), RegionKind(RegionKind),
75 CodeGen(CodeGen),
Kind(
Kind), HasCancel(HasCancel) {}
77 CGOpenMPRegionInfo(
const CGOpenMPRegionKind RegionKind,
80 : CGCapturedStmtInfo(
CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
85 virtual const VarDecl *getThreadIDVariable()
const = 0;
96 CGOpenMPRegionKind getRegionKind()
const {
return RegionKind; }
100 bool hasCancel()
const {
return HasCancel; }
102 static bool classof(
const CGCapturedStmtInfo *Info) {
106 ~CGOpenMPRegionInfo()
override =
default;
109 CGOpenMPRegionKind RegionKind;
116class CGOpenMPOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
121 StringRef HelperName)
122 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen,
Kind,
124 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
125 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
130 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
133 StringRef getHelperName()
const override {
return HelperName; }
135 static bool classof(
const CGCapturedStmtInfo *Info) {
136 return CGOpenMPRegionInfo::classof(Info) &&
137 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
138 ParallelOutlinedRegion;
145 StringRef HelperName;
149class CGOpenMPTaskOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
155 llvm::SwitchInst *UntiedSwitch =
nullptr;
158 UntiedTaskActionTy(
bool Tied,
const VarDecl *PartIDVar,
160 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
170 UntiedSwitch = CGF.
Builder.CreateSwitch(Res, DoneBB);
174 UntiedSwitch->addCase(CGF.
Builder.getInt32(0),
176 emitUntiedSwitch(CGF);
187 CodeGenFunction::JumpDest CurPoint =
191 UntiedSwitch->addCase(CGF.
Builder.getInt32(UntiedSwitch->getNumCases()),
197 unsigned getNumberOfParts()
const {
return UntiedSwitch->getNumCases(); }
203 const UntiedTaskActionTy &Action)
204 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen,
Kind, HasCancel),
205 ThreadIDVar(ThreadIDVar), Action(Action) {
206 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
211 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
217 StringRef getHelperName()
const override {
return ".omp_outlined."; }
220 Action.emitUntiedSwitch(CGF);
223 static bool classof(
const CGCapturedStmtInfo *Info) {
224 return CGOpenMPRegionInfo::classof(Info) &&
225 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
234 const UntiedTaskActionTy &Action;
239class CGOpenMPInlinedRegionInfo :
public CGOpenMPRegionInfo {
241 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
244 : CGOpenMPRegionInfo(InlinedRegion, CodeGen,
Kind, HasCancel),
246 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
249 llvm::Value *getContextValue()
const override {
251 return OuterRegionInfo->getContextValue();
252 llvm_unreachable(
"No context value for inlined OpenMP region");
255 void setContextValue(llvm::Value *
V)
override {
256 if (OuterRegionInfo) {
257 OuterRegionInfo->setContextValue(
V);
260 llvm_unreachable(
"No context value for inlined OpenMP region");
266 return OuterRegionInfo->lookup(VD);
272 FieldDecl *getThisFieldDecl()
const override {
274 return OuterRegionInfo->getThisFieldDecl();
280 const VarDecl *getThreadIDVariable()
const override {
282 return OuterRegionInfo->getThreadIDVariable();
289 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
290 llvm_unreachable(
"No LValue for inlined OpenMP construct");
294 StringRef getHelperName()
const override {
295 if (
auto *OuterRegionInfo = getOldCSI())
296 return OuterRegionInfo->getHelperName();
297 llvm_unreachable(
"No helper name for inlined OpenMP construct");
302 OuterRegionInfo->emitUntiedSwitch(CGF);
305 CodeGenFunction::CGCapturedStmtInfo *getOldCSI()
const {
return OldCSI; }
307 static bool classof(
const CGCapturedStmtInfo *Info) {
308 return CGOpenMPRegionInfo::classof(Info) &&
309 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
312 ~CGOpenMPInlinedRegionInfo()
override =
default;
316 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
317 CGOpenMPRegionInfo *OuterRegionInfo;
325class CGOpenMPTargetRegionInfo final :
public CGOpenMPRegionInfo {
329 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
331 HelperName(HelperName) {}
335 const VarDecl *getThreadIDVariable()
const override {
return nullptr; }
338 StringRef getHelperName()
const override {
return HelperName; }
340 static bool classof(
const CGCapturedStmtInfo *Info) {
341 return CGOpenMPRegionInfo::classof(Info) &&
342 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
346 StringRef HelperName;
350 llvm_unreachable(
"No codegen for expressions");
354class CGOpenMPInnerExprInfo final :
public CGOpenMPInlinedRegionInfo {
357 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
365 if (!
C.capturesVariable() && !
C.capturesVariableByCopy())
368 const VarDecl *VD =
C.getCapturedVar();
378 (void)PrivScope.Privatize();
383 if (
const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
390 llvm_unreachable(
"No body for expressions");
395 const VarDecl *getThreadIDVariable()
const override {
396 llvm_unreachable(
"No thread id for expressions");
400 StringRef getHelperName()
const override {
401 llvm_unreachable(
"No helper name for expressions");
404 static bool classof(
const CGCapturedStmtInfo *Info) {
return false; }
408 CodeGenFunction::OMPPrivateScope PrivScope;
412class InlinedOpenMPRegionRAII {
414 llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
415 FieldDecl *LambdaThisCaptureField =
nullptr;
417 bool NoInheritance =
false;
426 bool NoInheritance =
true)
427 : CGF(CGF), NoInheritance(NoInheritance) {
440 ~InlinedOpenMPRegionRAII() {
457enum OpenMPLocationFlags :
unsigned {
459 OMP_IDENT_IMD = 0x01,
461 OMP_IDENT_KMPC = 0x02,
463 OMP_ATOMIC_REDUCE = 0x10,
465 OMP_IDENT_BARRIER_EXPL = 0x20,
467 OMP_IDENT_BARRIER_IMPL = 0x40,
469 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
471 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
473 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
475 OMP_IDENT_WORK_LOOP = 0x200,
477 OMP_IDENT_WORK_SECTIONS = 0x400,
479 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
509enum IdentFieldIndex {
511 IdentField_Reserved_1,
515 IdentField_Reserved_2,
517 IdentField_Reserved_3,
526enum OpenMPSchedType {
529 OMP_sch_static_chunked = 33,
531 OMP_sch_dynamic_chunked = 35,
532 OMP_sch_guided_chunked = 36,
533 OMP_sch_runtime = 37,
536 OMP_sch_static_balanced_chunked = 45,
539 OMP_ord_static_chunked = 65,
541 OMP_ord_dynamic_chunked = 67,
542 OMP_ord_guided_chunked = 68,
543 OMP_ord_runtime = 69,
545 OMP_sch_default = OMP_sch_static,
547 OMP_dist_sch_static_chunked = 91,
548 OMP_dist_sch_static = 92,
551 OMP_sch_modifier_monotonic = (1 << 29),
553 OMP_sch_modifier_nonmonotonic = (1 << 30),
576 Callback(CodeGen, CGF, *PrePostAction);
579 Callback(CodeGen, CGF, Action);
587 if (
const auto *CE = dyn_cast<CallExpr>(ReductionOp))
588 if (
const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
589 if (
const auto *DRE =
590 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
591 if (
const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
602 std::pair<llvm::Function *, llvm::Function *>
Reduction =
604 const auto *CE = cast<CallExpr>(InitOp);
605 const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
609 cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
611 cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
612 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
613 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
Private);
614 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()), Original);
615 (void)PrivateScope.Privatize();
617 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE,
Func);
622 auto *GV =
new llvm::GlobalVariable(
624 llvm::GlobalValue::PrivateLinkage,
Init, Name);
637 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, LV);
644 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
665 llvm::Value *NumElements = CGF.
emitArrayLength(ArrayTy, ElementTy, DestAddr);
669 llvm::Value *SrcBegin =
nullptr;
671 SrcBegin = SrcAddr.emitRawPointer(CGF);
674 llvm::Value *DestEnd =
679 llvm::Value *IsEmpty =
680 CGF.
Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arrayinit.isempty");
681 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
684 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
689 llvm::PHINode *SrcElementPHI =
nullptr;
692 SrcElementPHI = CGF.
Builder.CreatePHI(SrcBegin->getType(), 2,
693 "omp.arraycpy.srcElementPast");
694 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
696 Address(SrcElementPHI, SrcAddr.getElementType(),
697 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
699 llvm::PHINode *DestElementPHI = CGF.
Builder.CreatePHI(
700 DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
701 DestElementPHI->addIncoming(DestBegin, EntryBB);
708 CodeGenFunction::RunCleanupsScope InitScope(CGF);
709 if (EmitDeclareReductionInit) {
711 SrcElementCurrent, ElementTy);
719 llvm::Value *SrcElementNext = CGF.
Builder.CreateConstGEP1_32(
720 SrcAddr.getElementType(), SrcElementPHI, 1,
721 "omp.arraycpy.dest.element");
722 SrcElementPHI->addIncoming(SrcElementNext, CGF.
Builder.GetInsertBlock());
726 llvm::Value *DestElementNext = CGF.
Builder.CreateConstGEP1_32(
728 "omp.arraycpy.dest.element");
731 CGF.
Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
732 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
733 DestElementPHI->addIncoming(DestElementNext, CGF.
Builder.GetInsertBlock());
745 if (
const auto *OASE = dyn_cast<ArraySectionExpr>(E))
750void ReductionCodeGen::emitAggregateInitialization(
756 const auto *PrivateVD =
757 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].
Private)->getDecl());
758 bool EmitDeclareReductionInit =
761 EmitDeclareReductionInit,
762 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
763 : PrivateVD->getInit(),
771 ClausesData.reserve(Shareds.size());
772 SharedAddresses.reserve(Shareds.size());
773 Sizes.reserve(Shareds.size());
774 BaseDecls.reserve(Shareds.size());
775 const auto *IOrig = Origs.begin();
776 const auto *IPriv = Privates.begin();
777 const auto *IRed = ReductionOps.begin();
778 for (
const Expr *Ref : Shareds) {
779 ClausesData.emplace_back(Ref, *IOrig, *IPriv, *IRed);
780 std::advance(IOrig, 1);
781 std::advance(IPriv, 1);
782 std::advance(IRed, 1);
787 assert(SharedAddresses.size() == N && OrigAddresses.size() == N &&
788 "Number of generated lvalues must be exactly N.");
789 LValue First = emitSharedLValue(CGF, ClausesData[N].Shared);
790 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Shared);
791 SharedAddresses.emplace_back(
First, Second);
792 if (ClausesData[N].Shared == ClausesData[N].Ref) {
793 OrigAddresses.emplace_back(
First, Second);
795 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
796 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
797 OrigAddresses.emplace_back(
First, Second);
803 bool AsArraySection = isa<ArraySectionExpr>(ClausesData[N].Ref);
806 CGF.
getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()),
811 llvm::Value *SizeInChars;
812 auto *ElemType = OrigAddresses[N].first.getAddress(CGF).getElementType();
813 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
814 if (AsArraySection) {
815 Size = CGF.
Builder.CreatePtrDiff(ElemType,
816 OrigAddresses[N].second.getPointer(CGF),
817 OrigAddresses[N].first.getPointer(CGF));
818 Size = CGF.
Builder.CreateNUWAdd(
819 Size, llvm::ConstantInt::get(Size->getType(), 1));
820 SizeInChars = CGF.
Builder.CreateNUWMul(Size, ElemSizeOf);
823 CGF.
getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType());
824 Size = CGF.
Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
826 Sizes.emplace_back(SizeInChars, Size);
829 cast<OpaqueValueExpr>(
839 assert(!Size && !Sizes[N].second &&
840 "Size should be nullptr for non-variably modified reduction "
846 cast<OpaqueValueExpr>(
855 assert(SharedAddresses.size() > N &&
"No variable was generated");
856 const auto *PrivateVD =
857 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].
Private)->getDecl());
862 (void)DefaultInit(CGF);
863 emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD);
864 }
else if (DRD && (DRD->
getInitializer() || !PrivateVD->hasInit())) {
865 (void)DefaultInit(CGF);
866 QualType SharedType = SharedAddresses[N].first.getType();
868 PrivateAddr, SharedAddr, SharedType);
869 }
else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
872 PrivateVD->
getType().getQualifiers(),
890 CGF.
pushDestroy(DTorKind, PrivateAddr, PrivateType);
914 Address OriginalBaseAddress, llvm::Value *Addr) {
938 Addr, OriginalBaseAddress.
getType());
943 const VarDecl *OrigVD =
nullptr;
944 if (
const auto *OASE = dyn_cast<ArraySectionExpr>(Ref)) {
945 const Expr *
Base = OASE->getBase()->IgnoreParenImpCasts();
946 while (
const auto *TempOASE = dyn_cast<ArraySectionExpr>(
Base))
947 Base = TempOASE->getBase()->IgnoreParenImpCasts();
948 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
949 Base = TempASE->getBase()->IgnoreParenImpCasts();
950 DE = cast<DeclRefExpr>(
Base);
951 OrigVD = cast<VarDecl>(DE->
getDecl());
952 }
else if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
953 const Expr *
Base = ASE->getBase()->IgnoreParenImpCasts();
954 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
955 Base = TempASE->getBase()->IgnoreParenImpCasts();
956 DE = cast<DeclRefExpr>(
Base);
957 OrigVD = cast<VarDecl>(DE->
getDecl());
966 BaseDecls.emplace_back(OrigVD);
969 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
971 Address SharedAddr = SharedAddresses[N].first.getAddress(CGF);
972 llvm::Value *Adjustment = CGF.
Builder.CreatePtrDiff(
975 llvm::Value *PrivatePointer =
981 SharedAddresses[N].first.getType(),
984 BaseDecls.emplace_back(
985 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
998 getThreadIDVariable()->getType()->castAs<PointerType>());
1016LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1019 getThreadIDVariable()->getType(),
1035 : CGM(CGM), OMPBuilder(CGM.getModule()) {
1037 llvm::OpenMPIRBuilderConfig Config(
1052 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(
true);
1060 if (!
Data.getValue().pointsToAliveValue())
1062 auto *GV = dyn_cast<llvm::GlobalVariable>(
Data.getValue());
1065 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1067 GV->eraseFromParent();
1072 return OMPBuilder.createPlatformSpecificName(Parts);
1075static llvm::Function *
1077 const Expr *CombinerInitializer,
const VarDecl *In,
1078 const VarDecl *Out,
bool IsCombiner) {
1081 QualType PtrTy =
C.getPointerType(Ty).withRestrict();
1087 Args.push_back(&OmpOutParm);
1088 Args.push_back(&OmpInParm);
1093 {IsCombiner ?
"omp_combiner" :
"omp_initializer",
""});
1094 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1098 Fn->removeFnAttr(llvm::Attribute::NoInline);
1099 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1100 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1106 Out->getLocation());
1107 CodeGenFunction::OMPPrivateScope
Scope(CGF);
1116 (void)
Scope.Privatize();
1117 if (!IsCombiner && Out->hasInit() &&
1120 Out->getType().getQualifiers(),
1123 if (CombinerInitializer)
1125 Scope.ForceCleanup();
1136 cast<VarDecl>(cast<DeclRefExpr>(D->
getCombinerIn())->getDecl()),
1137 cast<VarDecl>(cast<DeclRefExpr>(D->
getCombinerOut())->getDecl()),
1145 cast<VarDecl>(cast<DeclRefExpr>(D->
getInitOrig())->getDecl()),
1146 cast<VarDecl>(cast<DeclRefExpr>(D->
getInitPriv())->getDecl()),
1152 Decls.second.push_back(D);
1156std::pair<llvm::Function *, llvm::Function *>
1168struct PushAndPopStackRAII {
1169 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder,
CodeGenFunction &CGF,
1170 bool HasCancel, llvm::omp::Directive Kind)
1171 : OMPBuilder(OMPBuilder) {
1187 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1188 assert(IP.getBlock()->end() == IP.getPoint() &&
1189 "Clang CG should cause non-terminated block!");
1190 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1199 llvm::OpenMPIRBuilder::FinalizationInfo FI({FiniCB, Kind, HasCancel});
1200 OMPBuilder->pushFinalizationCB(std::move(FI));
1202 ~PushAndPopStackRAII() {
1204 OMPBuilder->popFinalizationCB();
1206 llvm::OpenMPIRBuilder *OMPBuilder;
1215 "thread id variable must be of type kmp_int32 *");
1217 bool HasCancel =
false;
1218 if (
const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1219 HasCancel = OPD->hasCancel();
1220 else if (
const auto *OPD = dyn_cast<OMPTargetParallelDirective>(&D))
1221 HasCancel = OPD->hasCancel();
1222 else if (
const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1223 HasCancel = OPSD->hasCancel();
1224 else if (
const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1225 HasCancel = OPFD->hasCancel();
1226 else if (
const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1227 HasCancel = OPFD->hasCancel();
1228 else if (
const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1229 HasCancel = OPFD->hasCancel();
1230 else if (
const auto *OPFD =
1231 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1232 HasCancel = OPFD->hasCancel();
1233 else if (
const auto *OPFD =
1234 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1235 HasCancel = OPFD->hasCancel();
1240 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind);
1241 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1242 HasCancel, OutlinedHelperName);
1243 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1248 std::string Suffix =
getName({
"omp_outlined"});
1249 return (Name + Suffix).str();
1257 std::string Suffix =
getName({
"omp",
"reduction",
"reduction_func"});
1258 return (Name + Suffix).str();
1285 bool Tied,
unsigned &NumberOfParts) {
1290 llvm::Value *TaskArgs[] = {
1292 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1295 CGF.EmitRuntimeCall(
OMPBuilder.getOrCreateRuntimeFunction(
1299 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1303 "thread id variable must be of type kmp_int32 for tasks");
1308 bool HasCancel =
false;
1309 if (
const auto *TD = dyn_cast<OMPTaskDirective>(&D))
1310 HasCancel = TD->hasCancel();
1311 else if (
const auto *TD = dyn_cast<OMPTaskLoopDirective>(&D))
1312 HasCancel = TD->hasCancel();
1313 else if (
const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(&D))
1314 HasCancel = TD->hasCancel();
1315 else if (
const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(&D))
1316 HasCancel = TD->hasCancel();
1319 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1320 InnermostKind, HasCancel, Action);
1322 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1324 NumberOfParts = Action.getNumberOfParts();
1329 bool AtCurrentPoint) {
1331 assert(!Elem.second.ServiceInsertPt &&
"Insert point is set already.");
1333 llvm::Value *Undef = llvm::UndefValue::get(CGF.
Int32Ty);
1334 if (AtCurrentPoint) {
1335 Elem.second.ServiceInsertPt =
new llvm::BitCastInst(
1338 Elem.second.ServiceInsertPt =
1339 new llvm::BitCastInst(Undef, CGF.
Int32Ty,
"svcpt");
1346 if (Elem.second.ServiceInsertPt) {
1347 llvm::Instruction *Ptr = Elem.second.ServiceInsertPt;
1348 Elem.second.ServiceInsertPt =
nullptr;
1349 Ptr->eraseFromParent();
1356 llvm::raw_svector_ostream OS(Buffer);
1360 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.
CurFuncDecl))
1361 OS << FD->getQualifiedNameAsString();
1368 unsigned Flags,
bool EmitLoc) {
1369 uint32_t SrcLocStrSize;
1370 llvm::Constant *SrcLocStr;
1372 llvm::codegenoptions::NoDebugInfo) ||
1374 SrcLocStr =
OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1376 std::string FunctionName;
1377 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.
CurFuncDecl))
1378 FunctionName = FD->getQualifiedNameAsString();
1388 SrcLocStr, SrcLocStrSize, llvm::omp::IdentFlag(Flags), Reserved2Flags);
1393 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1399 uint32_t SrcLocStrSize;
1400 auto *SrcLocStr =
OMPBuilder.getOrCreateSrcLocStr(
1403 OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize));
1406 llvm::Value *ThreadID =
nullptr;
1411 ThreadID = I->second.ThreadID;
1412 if (ThreadID !=
nullptr)
1416 if (
auto *OMPRegionInfo =
1418 if (OMPRegionInfo->getThreadIDVariable()) {
1420 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1424 CGF.
Builder.GetInsertBlock() == TopBlock ||
1425 !isa<llvm::Instruction>(LVal.
getPointer(CGF)) ||
1426 cast<llvm::Instruction>(LVal.
getPointer(CGF))->getParent() ==
1428 cast<llvm::Instruction>(LVal.
getPointer(CGF))->getParent() ==
1429 CGF.
Builder.GetInsertBlock()) {
1433 if (CGF.
Builder.GetInsertBlock() == TopBlock) {
1435 Elem.second.ThreadID = ThreadID;
1447 if (!Elem.second.ServiceInsertPt)
1449 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1450 CGF.
Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1454 OMPRTL___kmpc_global_thread_num),
1457 Elem.second.ThreadID =
Call;
1462 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1474 for(
const auto *D : I->second)
1489 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(
CGM.
Int32Ty),
1496llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind
1498 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
1499 OMPDeclareTargetDeclAttr::getDeviceType(VD);
1501 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1503 switch ((
int)*DevTy) {
1504 case OMPDeclareTargetDeclAttr::DT_Host:
1505 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseHost;
1507 case OMPDeclareTargetDeclAttr::DT_NoHost:
1508 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNoHost;
1510 case OMPDeclareTargetDeclAttr::DT_Any:
1511 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny;
1514 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1519llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind
1521 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> MapType =
1522 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1524 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1525 switch ((
int)*MapType) {
1526 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_To:
1527 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
1529 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Enter:
1530 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter;
1532 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Link:
1533 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
1536 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1545 auto FileInfoCallBack = [&]() {
1549 llvm::sys::fs::UniqueID ID;
1550 if (llvm::sys::fs::getUniqueID(PLoc.
getFilename(), ID)) {
1551 PLoc =
SM.getPresumedLoc(BeginLoc,
false);
1557 return OMPBuilder.getTargetEntryUniqueInfo(FileInfoCallBack, ParentName);
1563 auto LinkageForVariable = [&VD,
this]() {
1567 std::vector<llvm::GlobalVariable *> GeneratedRefs;
1571 llvm::Constant *addr =
OMPBuilder.getAddrOfDeclareTargetVar(
1579 LinkageForVariable);
1591 std::string Suffix =
getName({
"cache",
""});
1592 return OMPBuilder.getOrCreateInternalVariable(
1605 llvm::Value *Args[] = {
1629 llvm::Value *Args[] = {
1632 Ctor, CopyCtor, Dtor};
1635 CGM.
getModule(), OMPRTL___kmpc_threadprivate_register),
1650 llvm::Value *Ctor =
nullptr, *CopyCtor =
nullptr, *Dtor =
nullptr;
1660 Args.push_back(&Dst);
1665 std::string Name =
getName({
"__kmpc_global_ctor_",
""});
1666 llvm::Function *Fn =
1692 Args.push_back(&Dst);
1697 std::string Name =
getName({
"__kmpc_global_dtor_",
""});
1698 llvm::Function *Fn =
1720 auto *CopyCtorTy = llvm::FunctionType::get(
CGM.
VoidPtrTy, CopyCtorTyArgs,
1726 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
1727 if (Ctor ==
nullptr) {
1731 Ctor = llvm::Constant::getNullValue(CtorTy);
1733 if (Dtor ==
nullptr) {
1737 Dtor = llvm::Constant::getNullValue(DtorTy);
1740 auto *InitFunctionTy =
1741 llvm::FunctionType::get(
CGM.
VoidTy,
false);
1742 std::string Name =
getName({
"__omp_threadprivate_init_",
""});
1752 return InitFunction;
1760 llvm::GlobalValue *GV) {
1761 std::optional<OMPDeclareTargetDeclAttr *> ActiveAttr =
1762 OMPDeclareTargetDeclAttr::getActiveAttr(FD);
1765 if (!ActiveAttr || !(*ActiveAttr)->getIndirect())
1772 OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(Name, EntryInfo);
1778 llvm::GlobalValue *Addr = GV;
1780 Addr =
new llvm::GlobalVariable(
1782 true, llvm::GlobalValue::ExternalLinkage, GV, Name,
1783 nullptr, llvm::GlobalValue::NotThreadLocal,
1784 CGM.
getModule().getDataLayout().getDefaultGlobalsAddressSpace());
1785 Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1788 OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(
1790 llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect,
1791 llvm::GlobalValue::WeakODRLinkage);
1797 std::string Suffix =
getName({
"artificial",
""});
1799 llvm::GlobalVariable *GAddr =
OMPBuilder.getOrCreateInternalVariable(
1800 VarLVType, Twine(Name).concat(Suffix).str());
1803 GAddr->setThreadLocal(
true);
1804 return Address(GAddr, GAddr->getValueType(),
1807 std::string CacheSuffix =
getName({
"cache",
""});
1808 llvm::Value *Args[] = {
1816 Twine(Name).concat(Suffix).concat(CacheSuffix).str())};
1823 VarLVType->getPointerTo(0)),
1867 llvm::Function *OutlinedFn,
1870 llvm::Value *NumThreads) {
1875 auto &&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc,
1879 llvm::Value *Args[] = {
1881 CGF.
Builder.getInt32(CapturedVars.size()),
1884 RealArgs.append(std::begin(Args), std::end(Args));
1885 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
1887 llvm::FunctionCallee RTLFn =
1888 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_fork_call);
1891 auto &&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc, Loc,
1897 llvm::Value *Args[] = {RTLoc, ThreadID};
1899 M, OMPRTL___kmpc_serialized_parallel),
1906 ".bound.zero.addr");
1911 OutlinedFnArgs.push_back(ZeroAddrBound.
getPointer());
1912 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
1920 OutlinedFn->removeFnAttr(llvm::Attribute::AlwaysInline);
1921 OutlinedFn->addFnAttr(llvm::Attribute::NoInline);
1927 M, OMPRTL___kmpc_end_serialized_parallel),
1946 if (
auto *OMPRegionInfo =
1948 if (OMPRegionInfo->getThreadIDVariable())
1949 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(CGF);
1958 return ThreadIDTemp;
1962 std::string Prefix = Twine(
"gomp_critical_user_", CriticalName).str();
1963 std::string Name =
getName({Prefix,
"var"});
1970 llvm::FunctionCallee EnterCallee;
1972 llvm::FunctionCallee ExitCallee;
1975 llvm::BasicBlock *ContBlock =
nullptr;
1978 CommonActionTy(llvm::FunctionCallee EnterCallee,
1980 llvm::FunctionCallee ExitCallee,
1982 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
1983 ExitArgs(ExitArgs), Conditional(Conditional) {}
1987 llvm::Value *CallBool = CGF.
Builder.CreateIsNotNull(EnterRes);
1991 CGF.
Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
2007 StringRef CriticalName,
2021 EnterArgs.push_back(CGF.
Builder.CreateIntCast(
2024 CommonActionTy Action(
2027 Hint ? OMPRTL___kmpc_critical_with_hint : OMPRTL___kmpc_critical),
2030 OMPRTL___kmpc_end_critical),
2047 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2069 llvm::Value *FilterVal = Filter
2076 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2096 llvm::Value *Args[] = {
2098 llvm::ConstantInt::get(
CGM.
IntTy, 0,
true)};
2104 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
2105 Region->emitUntiedSwitch(CGF);
2118 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2131 unsigned Index,
const VarDecl *Var) {
2139 Ptr, ElemTy->getPointerTo(Ptr->getType()->getPointerAddressSpace())),
2155 Args.push_back(&LHSArg);
2156 Args.push_back(&RHSArg);
2162 llvm::GlobalValue::InternalLinkage, Name,
2165 Fn->setDoesNotRecurse();
2172 ArgsElemType->getPointerTo()),
2176 ArgsElemType->getPointerTo()),
2182 for (
unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
2183 const auto *DestVar =
2184 cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
2187 const auto *SrcVar =
2188 cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
2191 const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
2193 CGF.
EmitOMPCopy(
Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
2208 assert(CopyprivateVars.size() == SrcExprs.size() &&
2209 CopyprivateVars.size() == DstExprs.size() &&
2210 CopyprivateVars.size() == AssignmentOps.size());
2222 if (!CopyprivateVars.empty()) {
2225 C.getIntTypeForBitwidth(32, 1);
2226 DidIt = CGF.
CreateMemTemp(KmpInt32Ty,
".omp.copyprivate.did_it");
2231 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2248 llvm::APInt ArraySize(32, CopyprivateVars.size());
2249 QualType CopyprivateArrayTy =
C.getConstantArrayType(
2254 CGF.
CreateMemTemp(CopyprivateArrayTy,
".omp.copyprivate.cpr_list");
2255 for (
unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
2267 SrcExprs, DstExprs, AssignmentOps, Loc);
2268 llvm::Value *BufSize = CGF.
getTypeSize(CopyprivateArrayTy);
2272 llvm::Value *Args[] = {
2297 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2312 if (Kind == OMPD_for)
2313 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
2314 else if (Kind == OMPD_sections)
2315 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
2316 else if (Kind == OMPD_single)
2317 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
2318 else if (Kind == OMPD_barrier)
2319 Flags = OMP_IDENT_BARRIER_EXPL;
2321 Flags = OMP_IDENT_BARRIER_IMPL;
2333 ScheduleKind = OMPC_SCHEDULE_static;
2335 llvm::APInt ChunkSize(32, 1);
2345 bool ForceSimpleCall) {
2347 auto *OMPRegionInfo =
2351 CGF.
Builder, Kind, ForceSimpleCall, EmitChecks));
2364 if (OMPRegionInfo) {
2365 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
2368 OMPRTL___kmpc_cancel_barrier),
2377 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
2394 Expr *ME,
bool IsFatal) {
2397 : llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
2400 llvm::Value *Args[] = {
2402 llvm::ConstantInt::get(
CGM.
Int32Ty, IsFatal ? 2 : 1),
2411 bool Chunked,
bool Ordered) {
2412 switch (ScheduleKind) {
2413 case OMPC_SCHEDULE_static:
2414 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
2415 : (Ordered ? OMP_ord_static : OMP_sch_static);
2416 case OMPC_SCHEDULE_dynamic:
2417 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
2418 case OMPC_SCHEDULE_guided:
2419 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
2420 case OMPC_SCHEDULE_runtime:
2421 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
2422 case OMPC_SCHEDULE_auto:
2423 return Ordered ? OMP_ord_auto : OMP_sch_auto;
2425 assert(!Chunked &&
"chunk was specified but schedule kind not known");
2426 return Ordered ? OMP_ord_static : OMP_sch_static;
2428 llvm_unreachable(
"Unexpected runtime schedule");
2432static OpenMPSchedType
2435 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
2439 bool Chunked)
const {
2440 OpenMPSchedType Schedule =
2442 return Schedule == OMP_sch_static;
2448 return Schedule == OMP_dist_sch_static;
2452 bool Chunked)
const {
2453 OpenMPSchedType Schedule =
2455 return Schedule == OMP_sch_static_chunked;
2461 return Schedule == OMP_dist_sch_static_chunked;
2465 OpenMPSchedType Schedule =
2467 assert(Schedule != OMP_sch_static_chunked &&
"cannot be chunked here");
2468 return Schedule != OMP_sch_static;
2476 case OMPC_SCHEDULE_MODIFIER_monotonic:
2477 Modifier = OMP_sch_modifier_monotonic;
2479 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2480 Modifier = OMP_sch_modifier_nonmonotonic;
2482 case OMPC_SCHEDULE_MODIFIER_simd:
2483 if (Schedule == OMP_sch_static_chunked)
2484 Schedule = OMP_sch_static_balanced_chunked;
2491 case OMPC_SCHEDULE_MODIFIER_monotonic:
2492 Modifier = OMP_sch_modifier_monotonic;
2494 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2495 Modifier = OMP_sch_modifier_nonmonotonic;
2497 case OMPC_SCHEDULE_MODIFIER_simd:
2498 if (Schedule == OMP_sch_static_chunked)
2499 Schedule = OMP_sch_static_balanced_chunked;
2511 if (CGM.
getLangOpts().OpenMP >= 50 && Modifier == 0) {
2512 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
2513 Schedule == OMP_sch_static_balanced_chunked ||
2514 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||
2515 Schedule == OMP_dist_sch_static_chunked ||
2516 Schedule == OMP_dist_sch_static))
2517 Modifier = OMP_sch_modifier_nonmonotonic;
2519 return Schedule | Modifier;
2529 ScheduleKind.
Schedule, DispatchValues.
Chunk !=
nullptr, Ordered);
2531 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
2532 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&
2533 Schedule != OMP_sch_static_balanced_chunked));
2540 llvm::Value *Chunk = DispatchValues.
Chunk ? DispatchValues.
Chunk
2541 : CGF.
Builder.getIntN(IVSize, 1);
2542 llvm::Value *Args[] = {
2546 CGM, Schedule, ScheduleKind.
M1, ScheduleKind.
M2)),
2549 CGF.
Builder.getIntN(IVSize, 1),
2557 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
2558 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
2565 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||
2566 Schedule == OMP_sch_static_balanced_chunked ||
2567 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||
2568 Schedule == OMP_dist_sch_static ||
2569 Schedule == OMP_dist_sch_static_chunked);
2576 llvm::Value *Chunk = Values.
Chunk;
2577 if (Chunk ==
nullptr) {
2578 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
2579 Schedule == OMP_dist_sch_static) &&
2580 "expected static non-chunked schedule");
2584 assert((Schedule == OMP_sch_static_chunked ||
2585 Schedule == OMP_sch_static_balanced_chunked ||
2586 Schedule == OMP_ord_static_chunked ||
2587 Schedule == OMP_dist_sch_static_chunked) &&
2588 "expected static chunked schedule");
2590 llvm::Value *Args[] = {
2613 "Expected loop-based or sections-based directive.");
2616 ? OMP_IDENT_WORK_LOOP
2617 : OMP_IDENT_WORK_SECTIONS);
2619 llvm::FunctionCallee StaticInitFunction =
2624 ScheduleNum, ScheduleKind.
M1, ScheduleKind.
M2, Values);
2631 OpenMPSchedType ScheduleNum =
2633 llvm::Value *UpdatedLocation =
2636 llvm::FunctionCallee StaticInitFunction;
2637 bool isGPUDistribute =
2640 StaticInitFunction =
OMPBuilder.createForStaticInitFunction(
2651 assert((DKind == OMPD_distribute || DKind == OMPD_for ||
2652 DKind == OMPD_sections) &&
2653 "Expected distribute, for, or sections directive kind");
2657 llvm::Value *Args[] = {
2660 (DKind == OMPD_target_teams_loop)
2661 ? OMP_IDENT_WORK_DISTRIBUTE
2663 ? OMP_IDENT_WORK_LOOP
2664 : OMP_IDENT_WORK_SECTIONS),
2672 CGM.
getModule(), OMPRTL___kmpc_distribute_static_fini),
2701 llvm::Value *Args[] = {
2709 OMPBuilder.createDispatchNextFunction(IVSize, IVSigned), Args);
2716 llvm::Value *NumThreads,
2721 llvm::Value *Args[] = {
2730 ProcBindKind ProcBind,
2734 assert(ProcBind != OMP_PROC_BIND_unknown &&
"Unsupported proc_bind value.");
2736 llvm::Value *Args[] = {
2738 llvm::ConstantInt::get(
CGM.
IntTy,
unsigned(ProcBind),
true)};
2760enum KmpTaskTFields {
2790 llvm::OpenMPIRBuilder::EmitMetadataErrorReportFunctionTy &&ErrorReportFn =
2791 [
this](llvm::OpenMPIRBuilder::EmitMetadataErrorKind Kind,
2792 const llvm::TargetRegionEntryInfo &EntryInfo) ->
void {
2794 if (Kind != llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR) {
2798 if (I->getFirst().getUniqueID().getDevice() == EntryInfo.DeviceID &&
2799 I->getFirst().getUniqueID().getFile() == EntryInfo.FileID) {
2801 I->getFirst(), EntryInfo.Line, 1);
2807 case llvm::OpenMPIRBuilder::EMIT_MD_TARGET_REGION_ERROR: {
2810 "%0 is incorrect: either the "
2811 "address or the ID is invalid.");
2814 case llvm::OpenMPIRBuilder::EMIT_MD_DECLARE_TARGET_ERROR: {
2817 "variable %0 is incorrect: the "
2818 "address is invalid.");
2821 case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR: {
2824 "Offloading entry for declare target variable is incorrect: the "
2825 "address is invalid.");
2831 OMPBuilder.createOffloadEntriesAndInfoMetadata(ErrorReportFn);
2838 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty,
C.VoidPtrTy};
2841 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
2847struct PrivateHelpersTy {
2848 PrivateHelpersTy(
const Expr *OriginalRef,
const VarDecl *Original,
2850 : OriginalRef(OriginalRef), Original(Original), PrivateCopy(PrivateCopy),
2851 PrivateElemInit(PrivateElemInit) {}
2852 PrivateHelpersTy(
const VarDecl *Original) : Original(Original) {}
2853 const Expr *OriginalRef =
nullptr;
2854 const VarDecl *Original =
nullptr;
2855 const VarDecl *PrivateCopy =
nullptr;
2856 const VarDecl *PrivateElemInit =
nullptr;
2857 bool isLocalPrivate()
const {
2858 return !OriginalRef && !PrivateCopy && !PrivateElemInit;
2861typedef std::pair<
CharUnits , PrivateHelpersTy> PrivateDataTy;
2866 if (!CVD->
hasAttr<OMPAllocateDeclAttr>())
2868 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
2870 return !(AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc &&
2871 !AA->getAllocator());
2876 if (!Privates.empty()) {
2881 RecordDecl *RD =
C.buildImplicitRecord(
".kmp_privates.t");
2883 for (
const auto &Pair : Privates) {
2884 const VarDecl *VD = Pair.second.Original;
2888 if (Pair.second.isLocalPrivate()) {
2911 QualType KmpRoutineEntryPointerQTy) {
2931 QualType KmpCmplrdataTy =
C.getRecordType(UD);
2932 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_task_t");
2962 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_task_t_with_privates");
2982static llvm::Function *
2985 QualType KmpTaskTWithPrivatesPtrQTy,
2987 QualType SharedsPtrTy, llvm::Function *TaskFunction,
2988 llvm::Value *TaskPrivatesMap) {
2996 Args.push_back(&GtidArg);
2997 Args.push_back(&TaskTypeArg);
2998 const auto &TaskEntryFnInfo =
3000 llvm::FunctionType *TaskEntryTy =
3003 auto *TaskEntry = llvm::Function::Create(
3004 TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.
getModule());
3006 TaskEntry->setDoesNotRecurse();
3021 const auto *KmpTaskTWithPrivatesQTyRD =
3022 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
3025 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->
getAsTagDecl());
3026 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
3028 llvm::Value *PartidParam = PartIdLVal.
getPointer(CGF);
3030 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
3036 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
3037 llvm::Value *PrivatesParam;
3038 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
3043 PrivatesParam = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
3046 llvm::Value *CommonArgs[] = {
3047 GtidParam, PartidParam, PrivatesParam, TaskPrivatesMap,
3053 std::end(CommonArgs));
3055 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
3058 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
3061 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
3064 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
3067 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
3070 CallArgs.push_back(LBParam);
3071 CallArgs.push_back(UBParam);
3072 CallArgs.push_back(StParam);
3073 CallArgs.push_back(LIParam);
3074 CallArgs.push_back(RParam);
3076 CallArgs.push_back(SharedsParam);
3089 QualType KmpTaskTWithPrivatesPtrQTy,
3090 QualType KmpTaskTWithPrivatesQTy) {
3098 Args.push_back(&GtidArg);
3099 Args.push_back(&TaskTypeArg);
3100 const auto &DestructorFnInfo =
3102 llvm::FunctionType *DestructorFnTy =
3106 auto *DestructorFn =
3107 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
3111 DestructorFn->setDoesNotRecurse();
3119 const auto *KmpTaskTWithPrivatesQTyRD =
3120 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
3121 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
3123 for (
const auto *Field :
3124 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
3126 Field->getType().isDestructedType()) {
3132 return DestructorFn;
3152 C,
nullptr, Loc,
nullptr,
3153 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
3155 Args.push_back(&TaskPrivatesArg);
3156 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>,
unsigned> PrivateVarsPos;
3157 unsigned Counter = 1;
3158 for (
const Expr *E :
Data.PrivateVars) {
3160 C,
nullptr, Loc,
nullptr,
3161 C.getPointerType(
C.getPointerType(E->
getType()))
3165 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3166 PrivateVarsPos[VD] = Counter;
3169 for (
const Expr *E :
Data.FirstprivateVars) {
3171 C,
nullptr, Loc,
nullptr,
3172 C.getPointerType(
C.getPointerType(E->
getType()))
3176 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3177 PrivateVarsPos[VD] = Counter;
3180 for (
const Expr *E :
Data.LastprivateVars) {
3182 C,
nullptr, Loc,
nullptr,
3183 C.getPointerType(
C.getPointerType(E->
getType()))
3187 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3188 PrivateVarsPos[VD] = Counter;
3194 Ty =
C.getPointerType(Ty);
3196 Ty =
C.getPointerType(Ty);
3198 C,
nullptr, Loc,
nullptr,
3199 C.getPointerType(
C.getPointerType(Ty)).withConst().withRestrict(),
3201 PrivateVarsPos[VD] = Counter;
3204 const auto &TaskPrivatesMapFnInfo =
3206 llvm::FunctionType *TaskPrivatesMapTy =
3210 auto *TaskPrivatesMap = llvm::Function::Create(
3211 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
3214 TaskPrivatesMapFnInfo);
3216 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
3217 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
3218 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
3222 TaskPrivatesMapFnInfo, Args, Loc, Loc);
3228 const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->
getAsTagDecl());
3230 for (
const FieldDecl *Field : PrivatesQTyRD->fields()) {
3232 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
3241 return TaskPrivatesMap;
3253 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
3259 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
3267 if ((!IsTargetTask && !
Data.FirstprivateVars.empty() && ForDup) ||
3268 (IsTargetTask && KmpTaskSharedsPtr.
isValid())) {
3275 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
3276 for (
const PrivateDataTy &Pair : Privates) {
3278 if (Pair.second.isLocalPrivate()) {
3282 const VarDecl *VD = Pair.second.PrivateCopy;
3284 if (
Init && (!ForDup || (isa<CXXConstructExpr>(
Init) &&
3287 if (
const VarDecl *Elem = Pair.second.PrivateElemInit) {
3288 const VarDecl *OriginalVD = Pair.second.Original;
3293 const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);
3294 if (IsTargetTask && !SharedField) {
3295 assert(isa<ImplicitParamDecl>(OriginalVD) &&
3298 ->getNumParams() == 0 &&
3299 isa<TranslationUnitDecl>(
3301 ->getDeclContext()) &&
3302 "Expected artificial target data variable.");
3305 }
else if (ForDup) {
3309 C.getDeclAlign(OriginalVD)),
3313 Pair.second.Original->getCanonicalDecl()) > 0 ||
3315 SharedRefLValue = CGF.
EmitLValue(Pair.second.OriginalRef);
3318 InlinedOpenMPRegionRAII Region(
3321 SharedRefLValue = CGF.
EmitLValue(Pair.second.OriginalRef);
3334 [&CGF, Elem,
Init, &CapturesInfo](
Address DestElement,
3337 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3338 InitScope.addPrivate(Elem, SrcElement);
3339 (void)InitScope.Privatize();
3341 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
3342 CGF, &CapturesInfo);
3343 CGF.EmitAnyExprToMem(Init, DestElement,
3344 Init->getType().getQualifiers(),
3349 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3350 InitScope.addPrivate(Elem, SharedRefLValue.
getAddress(CGF));
3351 (void)InitScope.Privatize();
3352 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
3367 bool InitRequired =
false;
3368 for (
const PrivateDataTy &Pair : Privates) {
3369 if (Pair.second.isLocalPrivate())
3371 const VarDecl *VD = Pair.second.PrivateCopy;
3373 InitRequired = InitRequired || (isa_and_nonnull<CXXConstructExpr>(
Init) &&
3378 return InitRequired;
3395 QualType KmpTaskTWithPrivatesPtrQTy,
3403 KmpTaskTWithPrivatesPtrQTy,
3406 KmpTaskTWithPrivatesPtrQTy,
3410 Args.push_back(&DstArg);
3411 Args.push_back(&SrcArg);
3412 Args.push_back(&LastprivArg);
3413 const auto &TaskDupFnInfo =
3417 auto *TaskDup = llvm::Function::Create(
3418 TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.
getModule());
3420 TaskDup->setDoesNotRecurse();
3430 auto LIFI = std::next(KmpTaskTQTyRD->
field_begin(), KmpTaskTLastIter);
3432 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
3440 assert(!Privates.empty());
3442 if (!
Data.FirstprivateVars.empty()) {
3447 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
3455 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
3456 SharedsTy, SharedsPtrTy,
Data, Privates,
true);
3466 for (
const PrivateDataTy &
P : Privates) {
3467 if (
P.second.isLocalPrivate())
3469 QualType Ty =
P.second.Original->getType().getNonReferenceType();
3478class OMPIteratorGeneratorScope final
3479 :
public CodeGenFunction::OMPPrivateScope {
3484 OMPIteratorGeneratorScope() =
delete;
3485 OMPIteratorGeneratorScope(OMPIteratorGeneratorScope &) =
delete;
3513 CodeGenFunction::JumpDest &ContDest =
3515 CodeGenFunction::JumpDest &ExitDest =
3518 llvm::Value *N = Uppers[I];
3526 ? CGF.
Builder.CreateICmpSLT(CVal, N)
3527 : CGF.
Builder.CreateICmpULT(CVal, N);
3529 CGF.
Builder.CreateCondBr(Cmp, BodyBB, ExitDest.getBlock());
3536 ~OMPIteratorGeneratorScope() {
3546 CGF.
EmitBlock(ExitDests[I - 1].getBlock(), I == 1);
3552static std::pair<llvm::Value *, llvm::Value *>
3554 const auto *OASE = dyn_cast<OMPArrayShapingExpr>(E);
3557 const Expr *
Base = OASE->getBase();
3562 llvm::Value *SizeVal;
3565 SizeVal = CGF.
getTypeSize(OASE->getBase()->getType()->getPointeeType());
3566 for (
const Expr *SE : OASE->getDimensions()) {
3570 SizeVal = CGF.
Builder.CreateNUWMul(SizeVal, Sz);
3572 }
else if (
const auto *ASE =
3576 llvm::Value *UpAddr = CGF.
Builder.CreateConstGEP1_32(
3579 llvm::Value *LowIntPtr = CGF.
Builder.CreatePtrToInt(Addr, CGF.
SizeTy);
3580 llvm::Value *UpIntPtr = CGF.
Builder.CreatePtrToInt(UpAddr, CGF.
SizeTy);
3581 SizeVal = CGF.
Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
3585 return std::make_pair(Addr, SizeVal);
3590 QualType FlagsTy =
C.getIntTypeForBitwidth(32,
false);
3591 if (KmpTaskAffinityInfoTy.
isNull()) {
3593 C.buildImplicitRecord(
"kmp_task_affinity_info_t");
3599 KmpTaskAffinityInfoTy =
C.getRecordType(KmpAffinityInfoRD);
3606 llvm::Function *TaskFunction,
QualType SharedsTy,
3611 const auto *I =
Data.PrivateCopies.begin();
3612 for (
const Expr *E :
Data.PrivateVars) {
3613 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3614 Privates.emplace_back(
3616 PrivateHelpersTy(E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
3620 I =
Data.FirstprivateCopies.begin();
3621 const auto *IElemInitRef =
Data.FirstprivateInits.begin();
3622 for (
const Expr *E :
Data.FirstprivateVars) {
3623 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3624 Privates.emplace_back(
3627 E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
3628 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
3632 I =
Data.LastprivateCopies.begin();
3633 for (
const Expr *E :
Data.LastprivateVars) {
3634 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3635 Privates.emplace_back(
3637 PrivateHelpersTy(E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
3645 Privates.emplace_back(
C.getDeclAlign(VD), PrivateHelpersTy(VD));
3647 llvm::stable_sort(Privates,
3648 [](
const PrivateDataTy &L,
const PrivateDataTy &R) {
3649 return L.first > R.first;
3651 QualType KmpInt32Ty =
C.getIntTypeForBitwidth(32, 1);
3665 "Expected taskloop, task or target directive");
3674 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
3676 QualType KmpTaskTWithPrivatesQTy =
C.getRecordType(KmpTaskTWithPrivatesQTyRD);
3677 QualType KmpTaskTWithPrivatesPtrQTy =
3678 C.getPointerType(KmpTaskTWithPrivatesQTy);
3679 llvm::Type *KmpTaskTWithPrivatesTy = CGF.
ConvertType(KmpTaskTWithPrivatesQTy);
3680 llvm::Type *KmpTaskTWithPrivatesPtrTy =
3681 KmpTaskTWithPrivatesTy->getPointerTo();
3682 llvm::Value *KmpTaskTWithPrivatesTySize =
3684 QualType SharedsPtrTy =
C.getPointerType(SharedsTy);
3687 llvm::Value *TaskPrivatesMap =
nullptr;
3688 llvm::Type *TaskPrivatesMapTy =
3689 std::next(TaskFunction->arg_begin(), 3)->getType();
3690 if (!Privates.empty()) {
3691 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
3695 TaskPrivatesMap, TaskPrivatesMapTy);
3697 TaskPrivatesMap = llvm::ConstantPointerNull::get(
3698 cast<llvm::PointerType>(TaskPrivatesMapTy));
3704 KmpTaskTWithPrivatesQTy,
KmpTaskTQTy, SharedsPtrTy, TaskFunction,
3716 DestructorsFlag = 0x8,
3717 PriorityFlag = 0x20,
3718 DetachableFlag = 0x40,
3720 unsigned Flags =
Data.Tied ? TiedFlag : 0;
3721 bool NeedsCleanup =
false;
3722 if (!Privates.empty()) {
3726 Flags = Flags | DestructorsFlag;
3728 if (
Data.Priority.getInt())
3729 Flags = Flags | PriorityFlag;
3731 Flags = Flags | DetachableFlag;
3732 llvm::Value *TaskFlags =
3733 Data.Final.getPointer()
3734 ? CGF.
Builder.CreateSelect(
Data.Final.getPointer(),
3735 CGF.
Builder.getInt32(FinalFlag),
3737 : CGF.
Builder.getInt32(
Data.Final.getInt() ? FinalFlag : 0);
3738 TaskFlags = CGF.
Builder.CreateOr(TaskFlags, CGF.
Builder.getInt32(Flags));
3739 llvm::Value *SharedsSize =
CGM.
getSize(
C.getTypeSizeInChars(SharedsTy));
3741 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
3744 llvm::Value *NewTask;
3751 llvm::Value *DeviceID;
3756 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
3757 AllocArgs.push_back(DeviceID);
3778 llvm::Value *Tid =
getThreadID(CGF, DC->getBeginLoc());
3779 Tid = CGF.
Builder.CreateIntCast(Tid, CGF.
IntTy,
false);
3782 CGM.
getModule(), OMPRTL___kmpc_task_allow_completion_event),
3783 {Loc, Tid, NewTask});
3794 llvm::Value *NumOfElements =
nullptr;
3795 unsigned NumAffinities = 0;
3797 if (
const Expr *Modifier =
C->getModifier()) {
3798 const auto *IE = cast<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts());
3799 for (
unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
3803 NumOfElements ? CGF.
Builder.CreateNUWMul(NumOfElements, Sz) : Sz;
3806 NumAffinities +=
C->varlist_size();
3811 enum RTLAffinityInfoFieldsTy { BaseAddr, Len, Flags };
3813 QualType KmpTaskAffinityInfoArrayTy;
3814 if (NumOfElements) {
3815 NumOfElements = CGF.
Builder.CreateNUWAdd(
3816 llvm::ConstantInt::get(CGF.
SizeTy, NumAffinities), NumOfElements);
3819 C.getIntTypeForBitwidth(
C.getTypeSize(
C.getSizeType()), 0),
3823 KmpTaskAffinityInfoArrayTy =
C.getVariableArrayType(
3831 NumOfElements = CGF.
Builder.CreateIntCast(NumOfElements, CGF.
Int32Ty,
3834 KmpTaskAffinityInfoArrayTy =
C.getConstantArrayType(
3836 llvm::APInt(
C.getTypeSize(
C.getSizeType()), NumAffinities),
nullptr,
3839 CGF.
CreateMemTemp(KmpTaskAffinityInfoArrayTy,
".affs.arr.addr");
3841 NumOfElements = llvm::ConstantInt::get(
CGM.
Int32Ty, NumAffinities,
3848 bool HasIterator =
false;
3850 if (
C->getModifier()) {
3854 for (
const Expr *E :
C->varlists()) {
3863 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
3868 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
3882 const Expr *Modifier =
C->getModifier();
3885 OMPIteratorGeneratorScope IteratorScope(
3887 for (
const Expr *E :
C->varlists()) {
3897 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
3902 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
3904 Idx = CGF.
Builder.CreateNUWAdd(
3905 Idx, llvm::ConstantInt::get(Idx->getType(), 1));
3920 CGM.
getModule(), OMPRTL___kmpc_omp_reg_task_with_affinity),
3921 {LocRef, GTid, NewTask, NumOfElements, AffinListPtr});
3923 llvm::Value *NewTaskNewTaskTTy =
3925 NewTask, KmpTaskTWithPrivatesPtrTy);
3927 KmpTaskTWithPrivatesQTy);
3938 *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds)),
3947 if (!Privates.empty()) {
3949 SharedsTy, SharedsPtrTy,
Data, Privates,
3954 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
3955 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy,
Data, Privates,
3956 !
Data.LastprivateVars.empty());
3960 enum {
Priority = 0, Destructors = 1 };
3962 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
3964 (*FI)->getType()->getAsUnionType()->getDecl();
3967 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
3968 KmpTaskTWithPrivatesQTy);
3971 Data1LV, *std::next(KmpCmplrdataUD->
field_begin(), Destructors));
3977 if (
Data.Priority.getInt()) {
3979 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
3984 Result.NewTask = NewTask;
3985 Result.TaskEntry = TaskEntry;
3986 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
3988 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
3994 RTLDependenceKindTy DepKind;
3996 case OMPC_DEPEND_in:
3997 DepKind = RTLDependenceKindTy::DepIn;
4000 case OMPC_DEPEND_out:
4001 case OMPC_DEPEND_inout:
4002 DepKind = RTLDependenceKindTy::DepInOut;
4004 case OMPC_DEPEND_mutexinoutset:
4005 DepKind = RTLDependenceKindTy::DepMutexInOutSet;
4007 case OMPC_DEPEND_inoutset:
4008 DepKind = RTLDependenceKindTy::DepInOutSet;
4010 case OMPC_DEPEND_outallmemory:
4011 DepKind = RTLDependenceKindTy::DepOmpAllMem;
4013 case OMPC_DEPEND_source:
4014 case OMPC_DEPEND_sink:
4015 case OMPC_DEPEND_depobj:
4016 case OMPC_DEPEND_inoutallmemory:
4018 llvm_unreachable(
"Unknown task dependence type");
4026 FlagsTy =
C.getIntTypeForBitwidth(
C.getTypeSize(
C.BoolTy),
false);
4027 if (KmpDependInfoTy.
isNull()) {
4028 RecordDecl *KmpDependInfoRD =
C.buildImplicitRecord(
"kmp_depend_info");
4034 KmpDependInfoTy =
C.getRecordType(KmpDependInfoRD);
4038std::pair<llvm::Value *, LValue>
4052 CGF,
Base.getAddress(CGF),
4053 llvm::ConstantInt::get(CGF.
IntPtrTy, -1,
true));
4060 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4062 return std::make_pair(NumDeps,
Base);
4066 llvm::PointerUnion<unsigned *, LValue *> Pos,
4077 OMPIteratorGeneratorScope IteratorScope(
4078 CGF, cast_or_null<OMPIteratorExpr>(
4079 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4081 for (
const Expr *E :
Data.DepExprs) {
4090 Addr = llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4091 Size = llvm::ConstantInt::get(CGF.
SizeTy, 0);
4094 if (
unsigned *
P = Pos.dyn_cast<
unsigned *>()) {
4098 assert(E &&
"Expected a non-null expression");
4108 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4113 static_cast<unsigned int>(RTLDependInfoFields::Len)));
4120 static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4122 llvm::ConstantInt::get(LLVMFlagsTy,
static_cast<unsigned int>(DepKind)),
4124 if (
unsigned *
P = Pos.dyn_cast<
unsigned *>()) {
4129 Idx = CGF.
Builder.CreateNUWAdd(Idx,
4130 llvm::ConstantInt::get(Idx->getType(), 1));
4139 assert(
Data.DepKind == OMPC_DEPEND_depobj &&
4140 "Expected depobj dependency kind.");
4145 OMPIteratorGeneratorScope IteratorScope(
4146 CGF, cast_or_null<OMPIteratorExpr>(
4147 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4149 for (
const Expr *E :
Data.DepExprs) {
4150 llvm::Value *NumDeps;
4153 std::tie(NumDeps,
Base) =
4157 C.getUIntPtrType());
4161 llvm::Value *Add = CGF.
Builder.CreateNUWAdd(PrevVal, NumDeps);
4163 SizeLVals.push_back(NumLVal);
4166 for (
unsigned I = 0, E = SizeLVals.size(); I < E; ++I) {
4169 Sizes.push_back(Size);
4179 assert(
Data.DepKind == OMPC_DEPEND_depobj &&
4180 "Expected depobj dependency kind.");
4183 OMPIteratorGeneratorScope IteratorScope(
4184 CGF, cast_or_null<OMPIteratorExpr>(
4185 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4187 for (
unsigned I = 0, End =
Data.DepExprs.size(); I < End; ++I) {
4189 llvm::Value *NumDeps;
4192 std::tie(NumDeps,
Base) =
4196 llvm::Value *Size = CGF.
Builder.CreateNUWMul(
4205 llvm::Value *Add = CGF.
Builder.CreateNUWAdd(Pos, NumDeps);
4221 llvm::Value *NumOfElements =
nullptr;
4222 unsigned NumDependencies = std::accumulate(
4223 Dependencies.begin(), Dependencies.end(), 0,
4225 return D.DepKind == OMPC_DEPEND_depobj
4227 : (V + (D.IteratorExpr ? 0 : D.DepExprs.size()));
4231 bool HasDepobjDeps =
false;
4232 bool HasRegularWithIterators =
false;
4233 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4234 llvm::Value *NumOfRegularWithIterators =
4235 llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4239 if (D.
DepKind == OMPC_DEPEND_depobj) {
4242 for (llvm::Value *Size : Sizes) {
4243 NumOfDepobjElements =
4244 CGF.
Builder.CreateNUWAdd(NumOfDepobjElements, Size);
4246 HasDepobjDeps =
true;
4251 if (
const auto *IE = cast_or_null<OMPIteratorExpr>(D.
IteratorExpr)) {
4255 llvm::Value *NumClauseDeps = CGF.
Builder.CreateNUWMul(
4257 NumOfRegularWithIterators =
4258 CGF.
Builder.CreateNUWAdd(NumOfRegularWithIterators, NumClauseDeps);
4260 HasRegularWithIterators =
true;
4266 if (HasDepobjDeps || HasRegularWithIterators) {
4267 NumOfElements = llvm::ConstantInt::get(
CGM.
IntPtrTy, NumDependencies,
4269 if (HasDepobjDeps) {
4271 CGF.
Builder.CreateNUWAdd(NumOfDepobjElements, NumOfElements);
4273 if (HasRegularWithIterators) {
4275 CGF.
Builder.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements);
4278 Loc,
C.getIntTypeForBitwidth(64, 0),
4282 KmpDependInfoArrayTy =
4291 NumOfElements = CGF.
Builder.CreateIntCast(NumOfElements, CGF.
Int32Ty,
4294 KmpDependInfoArrayTy =
C.getConstantArrayType(
4300 NumOfElements = llvm::ConstantInt::get(
CGM.
Int32Ty, NumDependencies,
4304 for (
unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4305 if (Dependencies[I].DepKind == OMPC_DEPEND_depobj ||
4306 Dependencies[I].IteratorExpr)
4313 CGF.
CreateMemTemp(
C.getSizeType(),
"dep.counter.addr"),
C.getSizeType());
4315 for (
unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4316 if (Dependencies[I].DepKind == OMPC_DEPEND_depobj ||
4317 !Dependencies[I].IteratorExpr)
4323 if (HasDepobjDeps) {
4324 for (
unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4325 if (Dependencies[I].DepKind != OMPC_DEPEND_depobj)
4333 return std::make_pair(NumOfElements, DependenciesArray);
4344 unsigned NumDependencies = Dependencies.
DepExprs.size();
4355 llvm::Value *NumDepsVal;
4357 if (
const auto *IE =
4358 cast_or_null<OMPIteratorExpr>(Dependencies.
IteratorExpr)) {
4359 NumDepsVal = llvm::ConstantInt::get(CGF.
SizeTy, 1);
4363 NumDepsVal = CGF.
Builder.CreateNUWMul(NumDepsVal, Sz);
4365 Size = CGF.
Builder.CreateNUWAdd(llvm::ConstantInt::get(CGF.
SizeTy, 1),
4369 llvm::Value *RecSize =
CGM.
getSize(SizeInBytes);
4370 Size = CGF.
Builder.CreateNUWMul(Size, RecSize);
4374 QualType KmpDependInfoArrayTy =
C.getConstantArrayType(
4377 CharUnits Sz =
C.getTypeSizeInChars(KmpDependInfoArrayTy);
4379 NumDepsVal = llvm::ConstantInt::get(CGF.
IntPtrTy, NumDependencies);
4384 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4385 llvm::Value *Args[] = {ThreadID, Size, Allocator};
4390 Args,
".dep.arr.addr");
4393 Addr, KmpDependInfoLlvmTy->getPointerTo());
4394 DependenciesArray =
Address(Addr, KmpDependInfoLlvmTy, Align);
4401 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4403 llvm::PointerUnion<unsigned *, LValue *> Pos;
4420 return DependenciesArray;
4436 llvm::ConstantInt::get(CGF.
IntPtrTy, -1,
true));
4441 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4442 llvm::Value *Args[] = {ThreadID, DepObjAddr, Allocator};
4459 llvm::Value *NumDeps;
4466 Begin.emitRawPointer(CGF), NumDeps);
4470 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
4472 llvm::PHINode *ElementPHI =
4473 CGF.
Builder.CreatePHI(
Begin.getType(), 2,
"omp.elementPast");
4474 ElementPHI->addIncoming(
Begin.emitRawPointer(CGF), EntryBB);
4477 Base.getTBAAInfo());
4482 static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4484 llvm::ConstantInt::get(LLVMFlagsTy,
static_cast<unsigned int>(DepKind)),
4488 llvm::Value *ElementNext =
4491 ElementPHI->addIncoming(ElementNext, CGF.
Builder.GetInsertBlock());
4492 llvm::Value *IsEmpty =
4493 CGF.
Builder.CreateICmpEQ(ElementNext, End,
"omp.isempty");
4494 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
4501 llvm::Function *TaskFunction,
4510 llvm::Value *NewTask =
Result.NewTask;
4511 llvm::Function *TaskEntry =
Result.TaskEntry;
4512 llvm::Value *NewTaskNewTaskTTy =
Result.NewTaskNewTaskTTy;
4517 llvm::Value *NumOfElements;
4518 std::tie(NumOfElements, DependenciesArray) =
4529 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
4530 llvm::Value *DepTaskArgs[7];
4531 if (!
Data.Dependences.empty()) {
4532 DepTaskArgs[0] = UpLoc;
4533 DepTaskArgs[1] = ThreadID;
4534 DepTaskArgs[2] = NewTask;
4535 DepTaskArgs[3] = NumOfElements;
4537 DepTaskArgs[5] = CGF.
Builder.getInt32(0);
4538 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4540 auto &&ThenCodeGen = [
this, &
Data, TDBase, KmpTaskTQTyRD, &TaskArgs,
4543 auto PartIdFI = std::next(KmpTaskTQTyRD->
field_begin(), KmpTaskTPartId);
4547 if (!
Data.Dependences.empty()) {
4560 Region->emitUntiedSwitch(CGF);
4563 llvm::Value *DepWaitTaskArgs[7];
4564 if (!
Data.Dependences.empty()) {
4565 DepWaitTaskArgs[0] = UpLoc;
4566 DepWaitTaskArgs[1] = ThreadID;
4567 DepWaitTaskArgs[2] = NumOfElements;
4569 DepWaitTaskArgs[4] = CGF.
Builder.getInt32(0);
4570 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4571 DepWaitTaskArgs[6] =
4572 llvm::ConstantInt::get(CGF.
Int32Ty,
Data.HasNowaitClause);
4575 auto &&ElseCodeGen = [
this, &M, &TaskArgs, ThreadID, NewTaskNewTaskTTy,
4576 TaskEntry, &
Data, &DepWaitTaskArgs,
4583 if (!
Data.Dependences.empty())
4585 M, OMPRTL___kmpc_omp_taskwait_deps_51),
4588 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
4591 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
4592 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
4601 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
4602 M, OMPRTL___kmpc_omp_task_begin_if0),
4605 M, OMPRTL___kmpc_omp_task_complete_if0),
4621 llvm::Function *TaskFunction,
4641 IfVal = llvm::ConstantInt::getSigned(CGF.
IntTy, 1);
4646 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
4654 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
4662 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
4671 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
4672 if (
Data.Reductions) {
4678 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
4679 llvm::Value *TaskArgs[] = {
4687 llvm::ConstantInt::getSigned(
4689 llvm::ConstantInt::getSigned(
4691 ?
Data.Schedule.getInt() ? NumTasks : Grainsize
4693 Data.Schedule.getPointer()
4696 : llvm::ConstantInt::get(CGF.
Int64Ty, 0),
4699 : llvm::ConstantPointerNull::get(CGF.
VoidPtrTy)};
4718 const Expr *,
const Expr *)> &RedOpGen,
4719 const Expr *XExpr =
nullptr,
const Expr *EExpr =
nullptr,
4720 const Expr *UpExpr =
nullptr) {
4728 llvm::Value *NumElements = CGF.
emitArrayLength(ArrayTy, ElementTy, LHSAddr);
4733 llvm::Value *LHSEnd =
4738 llvm::Value *IsEmpty =
4739 CGF.
Builder.CreateICmpEQ(LHSBegin, LHSEnd,
"omp.arraycpy.isempty");
4740 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
4743 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
4748 llvm::PHINode *RHSElementPHI = CGF.
Builder.CreatePHI(
4749 RHSBegin->getType(), 2,
"omp.arraycpy.srcElementPast");
4750 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
4755 llvm::PHINode *LHSElementPHI = CGF.
Builder.CreatePHI(
4756 LHSBegin->getType(), 2,
"omp.arraycpy.destElementPast");
4757 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
4763 CodeGenFunction::OMPPrivateScope
Scope(CGF);
4764 Scope.addPrivate(LHSVar, LHSElementCurrent);
4765 Scope.addPrivate(RHSVar, RHSElementCurrent);
4767 RedOpGen(CGF, XExpr, EExpr, UpExpr);
4768 Scope.ForceCleanup();
4771 llvm::Value *LHSElementNext = CGF.
Builder.CreateConstGEP1_32(
4773 "omp.arraycpy.dest.element");
4774 llvm::Value *RHSElementNext = CGF.
Builder.CreateConstGEP1_32(
4776 "omp.arraycpy.src.element");
4779 CGF.
Builder.CreateICmpEQ(LHSElementNext, LHSEnd,
"omp.arraycpy.done");
4780 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
4781 LHSElementPHI->addIncoming(LHSElementNext, CGF.
Builder.GetInsertBlock());
4782 RHSElementPHI->addIncoming(RHSElementNext, CGF.
Builder.GetInsertBlock());
4792 const Expr *ReductionOp) {
4793 if (
const auto *CE = dyn_cast<CallExpr>(ReductionOp))
4794 if (
const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
4795 if (
const auto *DRE =
4796 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
4797 if (
const auto *DRD =
4798 dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
4799 std::pair<llvm::Function *, llvm::Function *>
Reduction =
4802 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE,
Func);
4810 StringRef ReducerName,
SourceLocation Loc, llvm::Type *ArgsElemType,
4821 Args.push_back(&LHSArg);
4822 Args.push_back(&RHSArg);
4827 llvm::GlobalValue::InternalLinkage, Name,
4830 Fn->setDoesNotRecurse();
4838 ArgsElemType->getPointerTo()),
4842 ArgsElemType->getPointerTo()),
4849 const auto *IPriv = Privates.begin();
4851 for (
unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
4852 const auto *RHSVar =
4853 cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl());
4855 const auto *LHSVar =
4856 cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl());
4858 QualType PrivTy = (*IPriv)->getType();
4866 const auto *OVE = cast<OpaqueValueExpr>(VLA->
getSizeExpr());
4873 IPriv = Privates.begin();
4874 const auto *ILHS = LHSExprs.begin();
4875 const auto *IRHS = RHSExprs.begin();
4876 for (
const Expr *E : ReductionOps) {
4877 if ((*IPriv)->getType()->isArrayType()) {
4879 const auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
4880 const auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
4882 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
4884 emitReductionCombiner(CGF, E);
4894 Scope.ForceCleanup();
4900 const Expr *ReductionOp,
4901 const Expr *PrivateRef,
4906 const auto *LHSVar = cast<VarDecl>(LHS->
getDecl());
4907 const auto *RHSVar = cast<VarDecl>(RHS->
getDecl());
4909 CGF, PrivateRef->
getType(), LHSVar, RHSVar,
4911 emitReductionCombiner(CGF, ReductionOp);
4928 bool WithNowait = Options.WithNowait;
4929 bool SimpleReduction = Options.SimpleReduction;
4968 if (SimpleReduction) {
4970 const auto *IPriv = Privates.begin();
4971 const auto *ILHS = LHSExprs.begin();
4972 const auto *IRHS = RHSExprs.begin();
4973 for (
const Expr *E : ReductionOps) {
4975 cast<DeclRefExpr>(*IRHS));
4985 auto Size = RHSExprs.size();
4986 for (
const Expr *E : Privates) {
4991 llvm::APInt ArraySize(32, Size);
4992 QualType ReductionArrayTy =
C.getConstantArrayType(
4996 CGF.
CreateMemTemp(ReductionArrayTy,
".omp.reduction.red_list");
4997 const auto *IPriv = Privates.begin();
4999 for (
unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
5005 if ((*IPriv)->getType()->isVariablyModifiedType()) {
5009 llvm::Value *Size = CGF.
Builder.CreateIntCast(
5022 Privates, LHSExprs, RHSExprs, ReductionOps);
5025 std::string Name =
getName({
"reduction"});
5032 llvm::Value *ReductionArrayTySize = CGF.
getTypeSize(ReductionArrayTy);
5035 llvm::Value *Args[] = {
5038 CGF.
Builder.getInt32(RHSExprs.size()),
5039 ReductionArrayTySize,
5047 WithNowait ? OMPRTL___kmpc_reduce_nowait : OMPRTL___kmpc_reduce),
5051 llvm::BasicBlock *DefaultBB = CGF.
createBasicBlock(
".omp.reduction.default");
5052 llvm::SwitchInst *SwInst =
5053 CGF.
Builder.CreateSwitch(Res, DefaultBB, 2);
5062 SwInst->addCase(CGF.
Builder.getInt32(1), Case1BB);
5066 llvm::Value *EndArgs[] = {
5071 auto &&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps](
5074 const auto *IPriv = Privates.begin();
5075 const auto *ILHS = LHSExprs.begin();
5076 const auto *IRHS = RHSExprs.begin();
5077 for (
const Expr *E : ReductionOps) {
5079 cast<DeclRefExpr>(*IRHS));
5086 CommonActionTy Action(
5087 nullptr, std::nullopt,
5089 CGM.
getModule(), WithNowait ? OMPRTL___kmpc_end_reduce_nowait
5090 : OMPRTL___kmpc_end_reduce),
5103 SwInst->addCase(CGF.
Builder.getInt32(2), Case2BB);
5106 auto &&AtomicCodeGen = [Loc, Privates, LHSExprs, RHSExprs, ReductionOps](
5108 const auto *ILHS = LHSExprs.begin();
5109 const auto *IRHS = RHSExprs.begin();
5110 const auto *IPriv = Privates.begin();
5111 for (
const Expr *E : ReductionOps) {
5112 const Expr *XExpr =
nullptr;
5113 const Expr *EExpr =
nullptr;
5114 const Expr *UpExpr =
nullptr;
5116 if (
const auto *BO = dyn_cast<BinaryOperator>(E)) {
5117 if (BO->getOpcode() == BO_Assign) {
5118 XExpr = BO->getLHS();
5119 UpExpr = BO->getRHS();
5123 const Expr *RHSExpr = UpExpr;
5126 if (
const auto *ACO = dyn_cast<AbstractConditionalOperator>(
5130 RHSExpr = ACO->getCond();
5132 if (
const auto *BORHS =
5134 EExpr = BORHS->getRHS();
5135 BO = BORHS->getOpcode();
5139 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5140 auto &&AtomicRedGen = [BO, VD,
5142 const Expr *EExpr,
const Expr *UpExpr) {
5143 LValue X = CGF.EmitLValue(XExpr);
5146 E = CGF.EmitAnyExpr(EExpr);
5147 CGF.EmitOMPAtomicSimpleUpdateExpr(
5149 llvm::AtomicOrdering::Monotonic, Loc,
5150 [&CGF, UpExpr, VD, Loc](
RValue XRValue) {
5152 Address LHSTemp = CGF.CreateMemTemp(VD->getType());
5153 CGF.emitOMPSimpleStore(
5154 CGF.MakeAddrLValue(LHSTemp, VD->
getType()), XRValue,
5155 VD->getType().getNonReferenceType(), Loc);
5158 return CGF.EmitAnyExpr(UpExpr);
5161 if ((*IPriv)->getType()->isArrayType()) {
5163 const auto *RHSVar =
5164 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5166 AtomicRedGen, XExpr, EExpr, UpExpr);
5169 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5176 std::string Name = RT.
getName({
"atomic_reduction"});
5185 if ((*IPriv)->getType()->isArrayType()) {
5186 const auto *LHSVar =
5187 cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5188 const auto *RHSVar =
5189 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5193 CritRedGen(CGF,
nullptr,
nullptr,
nullptr);
5204 llvm::Value *EndArgs[] = {
5209 CommonActionTy Action(
nullptr, std::nullopt,
5228 llvm::raw_svector_ostream Out(Buffer);
5232 D = cast<VarDecl>(cast<DeclRefExpr>(Ref)->getDecl());
5236 Out << Prefix << Name <<
"_"
5238 return std::string(Out.str());
5260 Args.emplace_back(&Param);
5261 Args.emplace_back(&ParamOrig);
5262 const auto &FnInfo =
5266 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5269 Fn->setDoesNotRecurse();
5277 llvm::Value *Size =
nullptr;
5320 const Expr *ReductionOp,
5322 const Expr *PrivateRef) {
5324 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(LHS)->getDecl());
5325 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(RHS)->getDecl());
5331 Args.emplace_back(&ParamInOut);
5332 Args.emplace_back(&ParamIn);
5333 const auto &FnInfo =
5337 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5340 Fn->setDoesNotRecurse();
5343 llvm::Value *Size =
nullptr;
5357 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5358 PrivateScope.addPrivate(
5365 C.getPointerType(LHSVD->getType())->castAs<
PointerType>()));
5366 PrivateScope.addPrivate(
5372 C.getPointerType(RHSVD->getType())->castAs<
PointerType>()));
5373 PrivateScope.Privatize();
5378 CGF, ReductionOp, PrivateRef, cast<DeclRefExpr>(LHS),
5379 cast<DeclRefExpr>(RHS));
5401 Args.emplace_back(&Param);
5402 const auto &FnInfo =
5406 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5409 Fn->setDoesNotRecurse();
5414 llvm::Value *Size =
nullptr;
5449 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_taskred_input_t");
5458 C, RD,
C.getIntTypeForBitwidth(32,
false));
5461 unsigned Size =
Data.ReductionVars.size();
5462 llvm::APInt ArraySize(64, Size);
5464 C.getConstantArrayType(RDType, ArraySize,
nullptr,
5469 Data.ReductionCopies,
Data.ReductionOps);
5470 for (
unsigned Cnt = 0; Cnt < Size; ++Cnt) {
5472 llvm::Value *Idxs[] = {llvm::ConstantInt::get(
CGM.
SizeTy, 0),
5473 llvm::ConstantInt::get(
CGM.
SizeTy, Cnt)};
5489 llvm::Value *SizeValInChars;
5490 llvm::Value *SizeVal;
5491 std::tie(SizeValInChars, SizeVal) = RCG.
getSizes(Cnt);
5497 bool DelayedCreation = !!SizeVal;
5509 llvm::Value *FiniAddr =
5510 Fini ? Fini : llvm::ConstantPointerNull::get(
CGM.
VoidPtrTy);
5515 CGM, Loc, RCG, Cnt,
Data.ReductionOps[Cnt], LHSExprs[Cnt],
5516 RHSExprs[Cnt],
Data.ReductionCopies[Cnt]);
5520 if (DelayedCreation) {
5522 llvm::ConstantInt::get(
CGM.
Int32Ty, 1,
true),
5528 if (
Data.IsReductionWithTaskMod) {
5534 llvm::Value *Args[] = {
5536 llvm::ConstantInt::get(
CGM.
IntTy,
Data.IsWorksharingReduction ? 1 : 0,
5538 llvm::ConstantInt::get(
CGM.
IntTy, Size,
true),
5547 llvm::Value *Args[] = {
5550 llvm::ConstantInt::get(
CGM.
IntTy, Size,
true),
5560 bool IsWorksharingReduction) {
5566 llvm::Value *Args[] = {IdentTLoc, GTid,
5568 IsWorksharingReduction ? 1 : 0,
5572 CGM.
getModule(), OMPRTL___kmpc_task_reduction_modifier_fini),
5595 llvm::Value *ReductionsPtr,
5608 CGM.
getModule(), OMPRTL___kmpc_task_reduction_get_th_data),
5626 llvm::Value *NumOfElements;
5627 std::tie(NumOfElements, DependenciesArray) =
5629 if (!
Data.Dependences.empty()) {
5630 llvm::Value *DepWaitTaskArgs[7];
5631 DepWaitTaskArgs[0] = UpLoc;
5632 DepWaitTaskArgs[1] = ThreadID;
5633 DepWaitTaskArgs[2] = NumOfElements;
5635 DepWaitTaskArgs[4] = CGF.
Builder.getInt32(0);
5636 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
5637 DepWaitTaskArgs[6] =
5638 llvm::ConstantInt::get(CGF.
Int32Ty,
Data.HasNowaitClause);
5647 M, OMPRTL___kmpc_omp_taskwait_deps_51),
5654 llvm::Value *Args[] = {UpLoc, ThreadID};
5657 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_omp_taskwait),
5662 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
5663 Region->emitUntiedSwitch(CGF);
5672 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel,
5673 InnerKind != OMPD_critical &&
5674 InnerKind != OMPD_master &&
5675 InnerKind != OMPD_masked);
5690 RTCancelKind CancelKind = CancelNoreq;
5691 if (CancelRegion == OMPD_parallel)
5692 CancelKind = CancelParallel;
5693 else if (CancelRegion == OMPD_for)
5694 CancelKind = CancelLoop;
5695 else if (CancelRegion == OMPD_sections)
5696 CancelKind = CancelSections;
5698 assert(CancelRegion == OMPD_taskgroup);
5699 CancelKind = CancelTaskgroup;
5711 if (
auto *OMPRegionInfo =
5715 if (CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) {
5716 llvm::Value *Args[] = {
5731 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
5733 if (CancelRegion == OMPD_parallel)
5752 if (
auto *OMPRegionInfo =
5754 auto &&ThenGen = [
this, &M, Loc, CancelRegion,
5757 llvm::Value *Args[] = {
5761 llvm::Value *
Result = CGF.EmitRuntimeCall(
5762 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_cancel), Args);
5767 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
".cancel.exit");
5768 llvm::BasicBlock *ContBB = CGF.createBasicBlock(
".cancel.continue");
5769 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(
Result);
5770 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
5771 CGF.EmitBlock(ExitBB);
5772 if (CancelRegion == OMPD_parallel)
5776 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
5777 CGF.EmitBranchThroughCleanup(CancelDest);
5778 CGF.EmitBlock(ContBB,
true);
5796 OMPUsesAllocatorsActionTy(
5797 ArrayRef<std::pair<const Expr *, const Expr *>> Allocators)
5798 : Allocators(Allocators) {}
5802 for (
const auto &AllocatorData : Allocators) {
5804 CGF, AllocatorData.first, AllocatorData.second);
5810 for (
const auto &AllocatorData : Allocators) {
5812 AllocatorData.first);
5820 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
5822 assert(!ParentName.empty() &&
"Invalid target entry parent name!");
5826 for (
unsigned I = 0, E =
C->getNumberOfAllocators(); I < E; ++I) {
5833 OMPUsesAllocatorsActionTy UsesAllocatorAction(Allocators);
5836 IsOffloadEntry, CodeGen);
5840 const Expr *Allocator,
5841 const Expr *AllocatorTraits) {
5842 llvm::Value *ThreadId =
getThreadID(CGF, Allocator->getExprLoc());
5843 ThreadId = CGF.
Builder.CreateIntCast(ThreadId, CGF.
IntTy,
true);
5845 llvm::Value *MemSpaceHandle = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
5846 llvm::Value *NumTraits = llvm::ConstantInt::get(
5847 CGF.
IntTy, cast<ConstantArrayType>(
5850 .getLimitedValue());
5859 llvm::Value *AllocatorVal =
5862 {ThreadId, MemSpaceHandle, NumTraits, Traits});
5865 cast<DeclRefExpr>(Allocator->IgnoreParenImpCasts())->getDecl()));
5869 Allocator->getType(), Allocator->getExprLoc());
5874 const Expr *Allocator) {
5875 llvm::Value *ThreadId =
getThreadID(CGF, Allocator->getExprLoc());
5876 ThreadId = CGF.
Builder.CreateIntCast(ThreadId, CGF.
IntTy,
true);
5878 llvm::Value *AllocatorVal =
5882 Allocator->getExprLoc());
5885 OMPRTL___kmpc_destroy_allocator),
5886 {ThreadId, AllocatorVal});
5891 int32_t &MinThreadsVal, int32_t &MaxThreadsVal, int32_t &MinTeamsVal,
5892 int32_t &MaxTeamsVal) {
5899 for (
auto *A :
C->getAttrs()) {
5900 int32_t AttrMinThreadsVal = 1, AttrMaxThreadsVal = -1;
5901 int32_t AttrMinBlocksVal = 1, AttrMaxBlocksVal = -1;
5902 if (
auto *
Attr = dyn_cast<CUDALaunchBoundsAttr>(A))
5904 &AttrMinBlocksVal, &AttrMaxBlocksVal);
5905 else if (
auto *
Attr = dyn_cast<AMDGPUFlatWorkGroupSizeAttr>(A))
5907 nullptr,
Attr,
nullptr, &AttrMinThreadsVal,
5908 &AttrMaxThreadsVal);
5912 MinThreadsVal = std::max(MinThreadsVal, AttrMinThreadsVal);
5913 if (AttrMaxThreadsVal > 0)
5914 MaxThreadsVal = MaxThreadsVal > 0
5915 ? std::min(MaxThreadsVal, AttrMaxThreadsVal)
5916 : AttrMaxThreadsVal;
5917 MinTeamsVal = std::max(MinTeamsVal, AttrMinBlocksVal);
5918 if (AttrMaxBlocksVal > 0)
5919 MaxTeamsVal = MaxTeamsVal > 0 ? std::min(MaxTeamsVal, AttrMaxBlocksVal)
5927 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
5930 llvm::TargetRegionEntryInfo EntryInfo =
5934 llvm::OpenMPIRBuilder::FunctionGenCallback &&GenerateOutlinedFunction =
5935 [&CGF, &D, &CodeGen](StringRef EntryFnName) {
5938 CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName);
5943 OMPBuilder.emitTargetRegionFunction(EntryInfo, GenerateOutlinedFunction,
5944 IsOffloadEntry, OutlinedFn, OutlinedFnID);
5952 for (
auto *A :
C->getAttrs()) {
5953 if (
auto *
Attr = dyn_cast<AMDGPUWavesPerEUAttr>(A))
5972 while (
const auto *
C = dyn_cast_or_null<CompoundStmt>(Child)) {
5974 for (
const Stmt *S :
C->body()) {
5975 if (
const auto *E = dyn_cast<Expr>(S)) {
5980 if (isa<AsmStmt>(S) || isa<NullStmt>(S) || isa<OMPFlushDirective>(S) ||
5981 isa<OMPBarrierDirective>(S) || isa<OMPTaskyieldDirective>(S))
5984 if (
const auto *DS = dyn_cast<DeclStmt>(S)) {
5985 if (llvm::all_of(DS->decls(), [](
const Decl *D) {
5986 if (isa<EmptyDecl>(D) || isa<DeclContext>(D) ||
5987 isa<TypeDecl>(D) || isa<PragmaCommentDecl>(D) ||
5988 isa<PragmaDetectMismatchDecl>(D) || isa<UsingDecl>(D) ||
5989 isa<UsingDirectiveDecl>(D) ||
5990 isa<OMPDeclareReductionDecl>(D) ||
5991 isa<OMPThreadPrivateDecl>(D) || isa<OMPAllocateDecl>(D))
5993 const auto *VD = dyn_cast<VarDecl>(D);
5996 return VD->hasGlobalStorage() || !VD->isUsed();
6006 Child = Child->IgnoreContainers();
6013 int32_t &MaxTeamsVal) {
6017 "Expected target-based executable directive.");
6018 switch (DirectiveKind) {
6023 const Stmt *ChildStmt =
6025 if (
const auto *NestedDir =
6026 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
6029 const Expr *NumTeams =
6034 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();
6037 MinTeamsVal = MaxTeamsVal = 0;
6042 MinTeamsVal = MaxTeamsVal = 1;
6045 MinTeamsVal = MaxTeamsVal = 1;
6049 MinTeamsVal = MaxTeamsVal = -1;
6052 case OMPD_target_teams_loop:
6053 case OMPD_target_teams:
6054 case OMPD_target_teams_distribute:
6055 case OMPD_target_teams_distribute_simd:
6056 case OMPD_target_teams_distribute_parallel_for:
6057 case OMPD_target_teams_distribute_parallel_for_simd: {
6059 const Expr *NumTeams =
6063 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();
6066 MinTeamsVal = MaxTeamsVal = 0;
6069 case OMPD_target_parallel:
6070 case OMPD_target_parallel_for:
6071 case OMPD_target_parallel_for_simd:
6072 case OMPD_target_parallel_loop:
6073 case OMPD_target_simd:
6074 MinTeamsVal = MaxTeamsVal = 1;
6078 case OMPD_parallel_for:
6079 case OMPD_parallel_loop:
6080 case OMPD_parallel_master:
6081 case OMPD_parallel_sections:
6083 case OMPD_parallel_for_simd:
6085 case OMPD_cancellation_point:
6087 case OMPD_threadprivate:
6098 case OMPD_taskyield:
6101 case OMPD_taskgroup:
6107 case OMPD_target_data:
6108 case OMPD_target_exit_data:
6109 case OMPD_target_enter_data:
6110 case OMPD_distribute:
6111 case OMPD_distribute_simd:
6112 case OMPD_distribute_parallel_for:
6113 case OMPD_distribute_parallel_for_simd:
6114 case OMPD_teams_distribute:
6115 case OMPD_teams_distribute_simd:
6116 case OMPD_teams_distribute_parallel_for:
6117 case OMPD_teams_distribute_parallel_for_simd:
6118 case OMPD_target_update:
6119 case OMPD_declare_simd:
6120 case OMPD_declare_variant:
6121 case OMPD_begin_declare_variant:
6122 case OMPD_end_declare_variant:
6123 case OMPD_declare_target:
6124 case OMPD_end_declare_target:
6125 case OMPD_declare_reduction:
6126 case OMPD_declare_mapper:
6128 case OMPD_taskloop_simd:
6129 case OMPD_master_taskloop:
6130 case OMPD_master_taskloop_simd:
6131 case OMPD_parallel_master_taskloop:
6132 case OMPD_parallel_master_taskloop_simd:
6134 case OMPD_metadirective:
6140 llvm_unreachable(
"Unexpected directive kind.");
6146 "Clauses associated with the teams directive expected to be emitted "
6147 "only for the host!");
6149 int32_t MinNT = -1, MaxNT = -1;
6150 const Expr *NumTeams =
6152 if (NumTeams !=
nullptr) {
6155 switch (DirectiveKind) {
6158 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6162 return Bld.CreateIntCast(NumTeamsVal, CGF.
Int32Ty,
6165 case OMPD_target_teams:
6166 case OMPD_target_teams_distribute:
6167 case OMPD_target_teams_distribute_simd:
6168 case OMPD_target_teams_distribute_parallel_for:
6169 case OMPD_target_teams_distribute_parallel_for_simd: {
6173 return Bld.CreateIntCast(NumTeamsVal, CGF.
Int32Ty,
6181 assert(MinNT == MaxNT &&
"Num threads ranges require handling here.");
6182 return llvm::ConstantInt::get(CGF.
Int32Ty, MinNT);
6190 const Expr **E, int32_t &UpperBound,
6191 bool UpperBoundOnly, llvm::Value **CondVal) {
6194 const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child);
6201 if (CondVal && Dir->hasClausesOfKind<
OMPIfClause>()) {
6202 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6203 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6205 for (
const auto *
C : Dir->getClausesOfKind<
OMPIfClause>()) {
6206 if (
C->getNameModifier() == OMPD_unknown ||
6207 C->getNameModifier() == OMPD_parallel) {
6222 if (
const auto *PreInit =
6224 for (
const auto *I : PreInit->decls()) {
6225 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6228 CodeGenFunction::AutoVarEmission Emission =
6241 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6242 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6243 const auto *NumThreadsClause =
6245 const Expr *NTExpr = NumThreadsClause->getNumThreads();
6246 if (NTExpr->isIntegerConstantExpr(CGF.
getContext()))
6247 if (
auto Constant = NTExpr->getIntegerConstantExpr(CGF.
getContext()))
6250 ? Constant->getZExtValue()
6251 : std::min(UpperBound,
6252 static_cast<int32_t
>(Constant->getZExtValue()));
6255 if (UpperBound == -1)
6259 CodeGenFunction::LexicalScope
Scope(CGF, NTExpr->getSourceRange());
6260 if (
const auto *PreInit =
6261 cast_or_null<DeclStmt>(NumThreadsClause->getPreInitStmt())) {
6262 for (
const auto *I : PreInit->decls()) {
6263 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6266 CodeGenFunction::AutoVarEmission Emission =
6282 bool UpperBoundOnly, llvm::Value **CondVal,
const Expr **ThreadLimitExpr) {
6283 assert((!CGF.
getLangOpts().OpenMPIsTargetDevice || UpperBoundOnly) &&
6284 "Clauses associated with the teams directive expected to be emitted "
6285 "only for the host!");
6288 "Expected target-based executable directive.");
6290 const Expr *NT =
nullptr;
6291 const Expr **NTPtr = UpperBoundOnly ? nullptr : &NT;
6293 auto CheckForConstExpr = [&](
const Expr *E,
const Expr **EPtr) {
6296 UpperBound = UpperBound ? Constant->getZExtValue()
6297 : std::min(UpperBound,
6298 int32_t(Constant->getZExtValue()));
6302 if (UpperBound == -1)
6308 auto ReturnSequential = [&]() {
6313 switch (DirectiveKind) {
6316 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6322 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6324 ThreadLimitClause = TLC;
6325 if (ThreadLimitExpr) {
6326 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6329 CGF, ThreadLimitClause->getThreadLimit()->getSourceRange());
6330 if (
const auto *PreInit =
6331 cast_or_null<DeclStmt>(ThreadLimitClause->getPreInitStmt())) {
6332 for (
const auto *I : PreInit->decls()) {
6333 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6345 if (ThreadLimitClause)
6346 CheckForConstExpr(ThreadLimitClause->getThreadLimit(), ThreadLimitExpr);
6347 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6350 CS = Dir->getInnermostCapturedStmt();
6353 Dir = dyn_cast_or_null<OMPExecutableDirective>(Child);
6356 CS = Dir->getInnermostCapturedStmt();
6357 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6359 return ReturnSequential();
6363 case OMPD_target_teams: {
6367 CheckForConstExpr(ThreadLimitClause->getThreadLimit(), ThreadLimitExpr);
6370 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6373 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6374 if (Dir->getDirectiveKind() == OMPD_distribute) {
6375 CS = Dir->getInnermostCapturedStmt();
6376 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6381 case OMPD_target_teams_distribute:
6385 CheckForConstExpr(ThreadLimitClause->getThreadLimit(), ThreadLimitExpr);
6388 UpperBoundOnly, CondVal);
6390 case OMPD_target_teams_loop:
6391 case OMPD_target_parallel_loop:
6392 case OMPD_target_parallel:
6393 case OMPD_target_parallel_for:
6394 case OMPD_target_parallel_for_simd:
6395 case OMPD_target_teams_distribute_parallel_for:
6396 case OMPD_target_teams_distribute_parallel_for_simd: {
6400 if (
C->getNameModifier() == OMPD_unknown ||
6401 C->getNameModifier() == OMPD_parallel) {
6411 return ReturnSequential();
6421 CheckForConstExpr(ThreadLimitClause->getThreadLimit(), ThreadLimitExpr);
6426 CheckForConstExpr(NumThreadsClause->getNumThreads(),
nullptr);
6427 return NumThreadsClause->getNumThreads();
6431 case OMPD_target_teams_distribute_simd:
6432 case OMPD_target_simd:
6433 return ReturnSequential();
6437 llvm_unreachable(
"Unsupported directive kind.");
6442 llvm::Value *NumThreadsVal =
nullptr;
6443 llvm::Value *CondVal =
nullptr;
6444 llvm::Value *ThreadLimitVal =
nullptr;
6445 const Expr *ThreadLimitExpr =
nullptr;
6446 int32_t UpperBound = -1;
6449 CGF, D, UpperBound,
false, &CondVal,
6453 if (ThreadLimitExpr) {
6456 ThreadLimitVal = CGF.
Builder.CreateIntCast(ThreadLimitVal, CGF.
Int32Ty,
6461 if (UpperBound == 1) {
6462 NumThreadsVal = CGF.
Builder.getInt32(UpperBound);
6465 NumThreadsVal = CGF.
Builder.CreateIntCast(NumThreadsVal, CGF.
Int32Ty,
6467 }
else if (ThreadLimitVal) {
6470 NumThreadsVal = ThreadLimitVal;
6471 ThreadLimitVal =
nullptr;
6474 assert(!ThreadLimitVal &&
"Default not applicable with thread limit value");
6475 NumThreadsVal = CGF.
Builder.getInt32(0);
6482 NumThreadsVal = CGF.
Builder.CreateSelect(CondVal, NumThreadsVal,
6488 if (ThreadLimitVal) {
6489 NumThreadsVal = CGF.
Builder.CreateSelect(
6490 CGF.
Builder.CreateICmpULT(ThreadLimitVal, NumThreadsVal),
6491 ThreadLimitVal, NumThreadsVal);
6494 return NumThreadsVal;
6504class MappableExprsHandler {
6507 static unsigned getFlagMemberOffset() {
6508 unsigned Offset = 0;
6509 for (uint64_t Remain =
6510 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
6511 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
6512 !(Remain & 1); Remain = Remain >> 1)
6519 class MappingExprInfo {
6524 const Expr *MapExpr =
nullptr;
6527 MappingExprInfo(
const ValueDecl *MapDecl,
const Expr *MapExpr =
nullptr)
6528 : MapDecl(MapDecl), MapExpr(MapExpr) {}
6530 const ValueDecl *getMapDecl()
const {
return MapDecl; }
6531 const Expr *getMapExpr()
const {
return MapExpr; }
6534 using DeviceInfoTy = llvm::OpenMPIRBuilder::DeviceInfoTy;
6535 using MapBaseValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
6536 using MapValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
6537 using MapFlagsArrayTy = llvm::OpenMPIRBuilder::MapFlagsArrayTy;
6538 using MapDimArrayTy = llvm::OpenMPIRBuilder::MapDimArrayTy;
6539 using MapNonContiguousArrayTy =
6540 llvm::OpenMPIRBuilder::MapNonContiguousArrayTy;
6547 struct MapCombinedInfoTy : llvm::OpenMPIRBuilder::MapInfosTy {
6548 MapExprsArrayTy Exprs;
6549 MapValueDeclsArrayTy Mappers;
6550 MapValueDeclsArrayTy DevicePtrDecls;
6553 void append(MapCombinedInfoTy &CurInfo) {
6554 Exprs.append(CurInfo.Exprs.begin(), CurInfo.Exprs.end());
6555 DevicePtrDecls.append(CurInfo.DevicePtrDecls.begin(),
6556 CurInfo.DevicePtrDecls.end());
6557 Mappers.append(CurInfo.Mappers.begin(), CurInfo.Mappers.end());
6558 llvm::OpenMPIRBuilder::MapInfosTy::append(CurInfo);
6566 struct StructRangeInfoTy {
6567 MapCombinedInfoTy PreliminaryMapData;
6568 std::pair<
unsigned ,
Address > LowestElem = {
6570 std::pair<
unsigned ,
Address > HighestElem = {
6574 bool IsArraySection =
false;
6575 bool HasCompleteRecord =
false;
6585 bool ReturnDevicePointer =
false;
6586 bool IsImplicit =
false;
6588 const Expr *VarRef =
nullptr;
6589 bool ForDeviceAddr =
false;
6591 MapInfo() =
default;
6597 bool ReturnDevicePointer,
bool IsImplicit,
6598 const ValueDecl *Mapper =
nullptr,
const Expr *VarRef =
nullptr,
6599 bool ForDeviceAddr =
false)
6600 : Components(Components), MapType(MapType), MapModifiers(MapModifiers),
6601 MotionModifiers(MotionModifiers),
6602 ReturnDevicePointer(ReturnDevicePointer), IsImplicit(IsImplicit),
6603 Mapper(Mapper), VarRef(VarRef), ForDeviceAddr(ForDeviceAddr) {}
6609 struct DeferredDevicePtrEntryTy {
6610 const Expr *IE =
nullptr;
6612 bool ForDeviceAddr =
false;
6614 DeferredDevicePtrEntryTy(
const Expr *IE,
const ValueDecl *VD,
6616 : IE(IE), VD(VD), ForDeviceAddr(ForDeviceAddr) {}
6631 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>,
bool> FirstPrivateDecls;
6648 llvm::DenseMap<const ValueDecl *, const OMPMapClause *> LambdasMap;
6650 llvm::Value *getExprTypeSize(
const Expr *E)
const {
6654 if (
const auto *OAE = dyn_cast<OMPArrayShapingExpr>(E)) {
6656 CGF.
getTypeSize(OAE->getBase()->getType()->getPointeeType());
6657 for (
const Expr *SE : OAE->getDimensions()) {
6674 if (
const auto *OAE = dyn_cast<ArraySectionExpr>(E)) {
6676 OAE->getBase()->IgnoreParenImpCasts())
6682 if (!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
6683 !OAE->getLowerBound())
6686 llvm::Value *ElemSize;
6688 ElemSize = CGF.
getTypeSize(PTy->getPointeeType().getCanonicalType());
6690 const auto *ATy = cast<ArrayType>(BaseTy.
getTypePtr());
6691 assert(ATy &&
"Expecting array type if not a pointer type.");
6692 ElemSize = CGF.
getTypeSize(ATy->getElementType().getCanonicalType());
6697 if (!OAE->getLength() && OAE->getColonLocFirst().isInvalid())
6700 if (
const Expr *LenExpr = OAE->getLength()) {
6704 LenExpr->getExprLoc());
6705 return CGF.
Builder.CreateNUWMul(LengthVal, ElemSize);
6707 assert(!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
6708 OAE->getLowerBound() &&
"expected array_section[lb:].");
6714 OAE->getLowerBound()->getExprLoc());
6715 LBVal = CGF.
Builder.CreateNUWMul(LBVal, ElemSize);
6716 llvm::Value *Cmp = CGF.
Builder.CreateICmpUGT(LengthVal, LBVal);
6717 llvm::Value *TrueVal = CGF.
Builder.CreateNUWSub(LengthVal, LBVal);
6718 LengthVal = CGF.
Builder.CreateSelect(
6719 Cmp, TrueVal, llvm::ConstantInt::get(CGF.
SizeTy, 0));
6729 OpenMPOffloadMappingFlags getMapTypeBits(
6732 bool AddPtrFlag,
bool AddIsTargetParamFlag,
bool IsNonContiguous)
const {
6733 OpenMPOffloadMappingFlags Bits =
6734 IsImplicit ? OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT
6735 : OpenMPOffloadMappingFlags::OMP_MAP_NONE;
6737 case OMPC_MAP_alloc:
6738 case OMPC_MAP_release:
6745 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO;
6748 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_FROM;
6750 case OMPC_MAP_tofrom:
6751 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO |
6752 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
6754 case OMPC_MAP_delete:
6755 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_DELETE;
6758 llvm_unreachable(
"Unexpected map type!");
6761 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
6762 if (AddIsTargetParamFlag)
6763 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
6764 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_always))
6765 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS;
6766 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_close))
6767 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_CLOSE;
6768 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_present) ||
6769 llvm::is_contained(MotionModifiers, OMPC_MOTION_MODIFIER_present))
6770 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
6771 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_ompx_hold))
6772 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
6773 if (IsNonContiguous)
6774 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG;
6780 bool isFinalArraySectionExpression(
const Expr *E)
const {
6781 const auto *OASE = dyn_cast<ArraySectionExpr>(E);
6788 if (OASE->getColonLocFirst().isInvalid())
6791 const Expr *Length = OASE->getLength();
6798 OASE->getBase()->IgnoreParenImpCasts())
6800 if (
const auto *ATy = dyn_cast<ConstantArrayType>(BaseQTy.
getTypePtr()))
6801 return ATy->getSExtSize() != 1;
6810 if (!Length->EvaluateAsInt(Result, CGF.
getContext()))
6813 llvm::APSInt ConstLength = Result.Val.getInt();
6814 return ConstLength.getSExtValue() != 1;
6822 void generateInfoForComponentList(
6826 MapCombinedInfoTy &CombinedInfo,
6827 MapCombinedInfoTy &StructBaseCombinedInfo,
6828 StructRangeInfoTy &PartialStruct,
bool IsFirstComponentList,
6829 bool IsImplicit,
bool GenerateAllInfoForClauses,
6830 const ValueDecl *Mapper =
nullptr,
bool ForDeviceAddr =
false,
6831 const ValueDecl *BaseDecl =
nullptr,
const Expr *MapExpr =
nullptr,
6833 OverlappedElements = std::nullopt)
const {
7011 bool IsCaptureFirstInfo = IsFirstComponentList;
7015 bool RequiresReference =
false;
7018 auto CI = Components.rbegin();
7019 auto CE = Components.rend();
7024 bool IsExpressionFirstInfo =
true;
7025 bool FirstPointerInComplexData =
false;
7027 const Expr *AssocExpr = I->getAssociatedExpression();
7028 const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr);
7029 const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr);
7030 const auto *OAShE = dyn_cast<OMPArrayShapingExpr>(AssocExpr);
7032 if (isa<MemberExpr>(AssocExpr)) {
7036 }
else if ((AE && isa<CXXThisExpr>(AE->getBase()->IgnoreParenImpCasts())) ||
7038 isa<CXXThisExpr>(OASE->getBase()->IgnoreParenImpCasts()))) {
7041 isa<CXXThisExpr>(OAShE->getBase()->IgnoreParenCasts())) {
7050 if (
const auto *VD =
7051 dyn_cast_or_null<VarDecl>(I->getAssociatedDeclaration())) {
7052 if (std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
7053 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) {
7054 if ((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
7055 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
7056 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
7058 RequiresReference =
true;
7068 I->getAssociatedDeclaration()->getType().getNonReferenceType();
7073 const auto *VD = dyn_cast<VarDecl>(I->getAssociatedDeclaration());
7078 FirstPointerInComplexData =
true;
7097 bool ShouldBeMemberOf =
false;
7112 bool IsNonContiguous = CombinedInfo.NonContigInfo.IsNonContiguous;
7113 bool IsPrevMemberReference =
false;
7120 bool IsMappingWholeStruct =
true;
7121 if (!GenerateAllInfoForClauses) {
7122 IsMappingWholeStruct =
false;
7124 for (
auto TempI = I; TempI != CE; ++TempI) {
7126 dyn_cast<MemberExpr>(TempI->getAssociatedExpression());
7128 IsMappingWholeStruct =
false;
7134 for (; I != CE; ++I) {
7136 if (!EncounteredME) {
7137 EncounteredME = dyn_cast<MemberExpr>(I->getAssociatedExpression());
7140 if (EncounteredME) {
7141 ShouldBeMemberOf =
true;
7144 if (FirstPointerInComplexData) {
7146 ->getAssociatedDeclaration()
7148 .getNonReferenceType();
7150 FirstPointerInComplexData =
false;
7155 auto Next = std::next(I);
7165 bool IsFinalArraySection =
7167 isFinalArraySectionExpression(I->getAssociatedExpression());
7171 const ValueDecl *MapDecl = (I->getAssociatedDeclaration())
7172 ? I->getAssociatedDeclaration()
7174 MapExpr = (I->getAssociatedExpression()) ? I->getAssociatedExpression()
7181 dyn_cast<ArraySectionExpr>(I->getAssociatedExpression());
7183 dyn_cast<OMPArrayShapingExpr>(I->getAssociatedExpression());
7184 const auto *UO = dyn_cast<UnaryOperator>(I->getAssociatedExpression());
7185 const auto *BO = dyn_cast<BinaryOperator>(I->getAssociatedExpression());
7191 I->getAssociatedExpression()->getType()->isAnyPointerType();
7192 bool IsMemberReference = isa<MemberExpr>(I->getAssociatedExpression()) &&
7195 bool IsNonDerefPointer = IsPointer &&
7196 !(UO && UO->getOpcode() != UO_Deref) && !BO &&
7202 if (Next == CE || IsMemberReference || IsNonDerefPointer ||
7203 IsFinalArraySection) {
7206 assert((Next == CE ||
7207 isa<MemberExpr>(Next->getAssociatedExpression()) ||
7208 isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) ||
7209 isa<ArraySectionExpr>(Next->getAssociatedExpression()) ||
7210 isa<OMPArrayShapingExpr>(Next->getAssociatedExpression()) ||
7211 isa<UnaryOperator>(Next->getAssociatedExpression()) ||
7212 isa<BinaryOperator>(Next->getAssociatedExpression())) &&
7213 "Unexpected expression");
7219 const Expr *BaseExpr = E->getBase();
7239 OAShE->getBase()->getType()->getPointeeType()),
7241 OAShE->getBase()->getType()));
7242 }
else if (IsMemberReference) {
7243 const auto *ME = cast<MemberExpr>(I->getAssociatedExpression());
7244 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
7246 BaseLVal, cast<FieldDecl>(MapDecl))
7259 bool IsMemberPointerOrAddr =
7261 (((IsPointer || ForDeviceAddr) &&
7262 I->getAssociatedExpression() == EncounteredME) ||
7263 (IsPrevMemberReference && !IsPointer) ||
7264 (IsMemberReference && Next != CE &&
7265 !Next->getAssociatedExpression()->getType()->isPointerType()));
7266 if (!OverlappedElements.empty() && Next == CE) {
7268 assert(!PartialStruct.Base.isValid() &&
"The base element is set.");
7269 assert(!IsPointer &&
7270 "Unexpected base element with the pointer type.");
7273 PartialStruct.LowestElem = {0, LowestElem};
7275 I->getAssociatedExpression()->getType());
7280 PartialStruct.HighestElem = {
7281 std::numeric_limits<
decltype(
7282 PartialStruct.HighestElem.first)>
::max(),
7284 PartialStruct.Base = BP;
7285 PartialStruct.LB = LB;
7287 PartialStruct.PreliminaryMapData.BasePointers.empty() &&
7288 "Overlapped elements must be used only once for the variable.");
7289 std::swap(PartialStruct.PreliminaryMapData, CombinedInfo);
7291 OpenMPOffloadMappingFlags Flags =
7292 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
7293 getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,
7295 false, IsNonContiguous);
7296 llvm::Value *
Size =
nullptr;
7299 Component : OverlappedElements) {
7303 if (
const ValueDecl *VD = MC.getAssociatedDeclaration()) {
7304 const auto *FD = dyn_cast<FieldDecl>(VD);
7305 if (FD && FD->getType()->isLValueReferenceType()) {
7307 cast<MemberExpr>(MC.getAssociatedExpression());
7308 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
7324 assert(Size &&
"Failed to determine structure size");
7325 CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
7327 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
7328 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
7330 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
7332 CombinedInfo.Types.push_back(Flags);
7333 CombinedInfo.Mappers.push_back(
nullptr);
7334 CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize
7338 CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
7340 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
7341 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
7347 CombinedInfo.Sizes.push_back(
7349 CombinedInfo.Types.push_back(Flags);
7350 CombinedInfo.Mappers.push_back(
nullptr);
7351 CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize
7355 llvm::Value *
Size = getExprTypeSize(I->getAssociatedExpression());
7360 if (!IsMemberPointerOrAddr ||
7362 if (!IsMappingWholeStruct) {
7363 CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
7365 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
7366 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
7368 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
7370 CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize
7373 StructBaseCombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
7374 StructBaseCombinedInfo.BasePointers.push_back(
7376 StructBaseCombinedInfo.DevicePtrDecls.push_back(
nullptr);
7377 StructBaseCombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
7378 StructBaseCombinedInfo.Pointers.push_back(LB.
emitRawPointer(CGF));
7379 StructBaseCombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
7381 StructBaseCombinedInfo.NonContigInfo.Dims.push_back(
7382 IsNonContiguous ? DimSize : 1);
7386 bool HasMapper = Mapper && Next == CE;
7387 if (!IsMappingWholeStruct)
7388 CombinedInfo.Mappers.push_back(HasMapper ? Mapper :
nullptr);
7390 StructBaseCombinedInfo.Mappers.push_back(HasMapper ? Mapper
7397 OpenMPOffloadMappingFlags Flags = getMapTypeBits(
7398 MapType, MapModifiers, MotionModifiers, IsImplicit,
7399 !IsExpressionFirstInfo || RequiresReference ||
7400 FirstPointerInComplexData || IsMemberReference,
7401 IsCaptureFirstInfo && !RequiresReference, IsNonContiguous);
7403 if (!IsExpressionFirstInfo || IsMemberReference) {
7406 if (IsPointer || (IsMemberReference && Next != CE))
7407 Flags &= ~(OpenMPOffloadMappingFlags::OMP_MAP_TO |
7408 OpenMPOffloadMappingFlags::OMP_MAP_FROM |
7409 OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS |
7410 OpenMPOffloadMappingFlags::OMP_MAP_DELETE |
7411 OpenMPOffloadMappingFlags::OMP_MAP_CLOSE);
7413 if (ShouldBeMemberOf) {
7416 Flags |= OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;
7419 ShouldBeMemberOf =
false;
7423 if (!IsMappingWholeStruct)
7424 CombinedInfo.Types.push_back(Flags);
7426 StructBaseCombinedInfo.Types.push_back(Flags);
7432 if (EncounteredME) {
7433 const auto *FD = cast<FieldDecl>(EncounteredME->
getMemberDecl());
7434 unsigned FieldIndex = FD->getFieldIndex();
7437 if (!PartialStruct.Base.isValid()) {
7438 PartialStruct.LowestElem = {FieldIndex, LowestElem};
7439 if (IsFinalArraySection) {
7443 PartialStruct.HighestElem = {FieldIndex, HB};
7445 PartialStruct.HighestElem = {FieldIndex, LowestElem};
7447 PartialStruct.Base = BP;
7448 PartialStruct.LB = BP;
7449 }
else if (FieldIndex < PartialStruct.LowestElem.first) {
7450 PartialStruct.LowestElem = {FieldIndex, LowestElem};
7451 }
else if (FieldIndex > PartialStruct.HighestElem.first) {
7452 if (IsFinalArraySection) {
7456 PartialStruct.HighestElem = {FieldIndex, HB};
7458 PartialStruct.HighestElem = {FieldIndex, LowestElem};
7464 if (IsFinalArraySection || IsNonContiguous)
7465 PartialStruct.IsArraySection =
true;
7468 if (IsFinalArraySection)
7473 BP = IsMemberReference ? LowestElem : LB;
7475 IsExpressionFirstInfo =
false;
7476 IsCaptureFirstInfo =
false;
7477 FirstPointerInComplexData =
false;
7478 IsPrevMemberReference = IsMemberReference;
7479 }
else if (FirstPointerInComplexData) {
7481 ->getAssociatedDeclaration()
7483 .getNonReferenceType();
7485 FirstPointerInComplexData =
false;
7491 PartialStruct.HasCompleteRecord =
true;
7493 if (!IsNonContiguous)
7500 MapValuesArrayTy CurOffsets = {llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, 0)};
7501 MapValuesArrayTy CurCounts = {llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, 1)};
7502 MapValuesArrayTy CurStrides;
7503 MapValuesArrayTy DimSizes{llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, 1)};
7511 const Expr *AssocExpr = Component.getAssociatedExpression();
7512 const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr);
7522 assert((VAT || CAT || &Component == &*Components.begin()) &&
7523 "Should be either ConstantArray or VariableArray if not the "
7527 if (CurStrides.empty()) {
7528 const Type *ElementType =
nullptr;
7530 ElementType = CAT->getElementType().getTypePtr();
7532 ElementType = VAT->getElementType().getTypePtr();
7534 assert(&Component == &*Components.begin() &&
7535 "Only expect pointer (non CAT or VAT) when this is the "
7543 if (&Component != &*Components.begin())
7547 CurStrides.push_back(
7548 llvm::ConstantInt::get(CGF.
Int64Ty, ElementTypeSize));
7553 if (DimSizes.size() < Components.size() - 1) {
7556 llvm::ConstantInt::get(CGF.
Int64Ty, CAT->getZExtSize()));
7558 DimSizes.push_back(CGF.
Builder.CreateIntCast(
7565 auto *DI = DimSizes.begin() + 1;
7567 llvm::Value *DimProd =
7568 llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, ElementTypeSize);
7579 const Expr *AssocExpr = Component.getAssociatedExpression();
7581 if (
const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr)) {
7582 llvm::Value *Offset = CGF.
Builder.CreateIntCast(
7585 CurOffsets.push_back(Offset);
7586 CurCounts.push_back(llvm::ConstantInt::get(CGF.
Int64Ty, 1));
7587 CurStrides.push_back(CurStrides.back());
7591 const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr);
7597 const Expr *OffsetExpr = OASE->getLowerBound();
7598 llvm::Value *Offset =
nullptr;
7601 Offset = llvm::ConstantInt::get(CGF.
Int64Ty, 0);
7607 CurOffsets.push_back(Offset);
7610 const Expr *CountExpr = OASE->getLength();
7611 llvm::Value *Count =
nullptr;
7617 if (!OASE->getColonLocFirst().isValid() &&
7618 !OASE->getColonLocSecond().isValid()) {
7619 Count = llvm::ConstantInt::get(CGF.
Int64Ty, 1);
7625 const Expr *StrideExpr = OASE->getStride();
7626 llvm::Value *Stride =
7632 Count = CGF.
Builder.CreateUDiv(
7633 CGF.
Builder.CreateNUWSub(*DI, Offset), Stride);
7635 Count = CGF.
Builder.CreateNUWSub(*DI, Offset);
7641 CurCounts.push_back(Count);
7650 const Expr *StrideExpr = OASE->getStride();
7651 llvm::Value *Stride =
7656 DimProd = CGF.
Builder.CreateNUWMul(DimProd, *(DI - 1));
7658 CurStrides.push_back(CGF.
Builder.CreateNUWMul(DimProd, Stride));
7660 CurStrides.push_back(DimProd);
7661 if (DI != DimSizes.end())
7665 CombinedInfo.NonContigInfo.Offsets.push_back(CurOffsets);
7666 CombinedInfo.NonContigInfo.Counts.push_back(CurCounts);
7667 CombinedInfo.NonContigInfo.Strides.push_back(CurStrides);
7673 OpenMPOffloadMappingFlags
7682 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
7683 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
7684 return OpenMPOffloadMappingFlags::OMP_MAP_PRIVATE |
7685 OpenMPOffloadMappingFlags::OMP_MAP_TO;
7688 if (I != LambdasMap.end())
7690 return getMapTypeBits(
7691 I->getSecond()->getMapType(), I->getSecond()->getMapTypeModifiers(),
7692 std::nullopt, I->getSecond()->isImplicit(),
7696 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
7697 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
7702 bool AsBase)
const {
7705 llvm::StructType *St =
7708 unsigned NumElements = St->getNumElements();
7710 llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>, 4>
7711 RecordLayout(NumElements);
7714 for (
const auto &I : RD->
bases()) {
7717 const auto *
Base = I.getType()->getAsCXXRecordDecl();
7726 RecordLayout[FieldIndex] =
Base;
7729 for (
const auto &I : RD->
vbases()) {
7730 const auto *
Base = I.getType()->getAsCXXRecordDecl();
7732 if (
Base->isEmpty())
7735 if (RecordLayout[FieldIndex])
7737 RecordLayout[FieldIndex] =
Base;
7740 assert(!RD->
isUnion() &&
"Unexpected union.");
7741 for (
const auto *Field : RD->
fields()) {
7746 RecordLayout[FieldIndex] =
Field;
7749 for (
const llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>
7750 &
Data : RecordLayout) {
7754 getPlainLayout(
Base, Layout,
true);
7765 void generateAllInfoForClauses(
7767 llvm::OpenMPIRBuilder &OMPBuilder,
7774 llvm::MapVector<CanonicalDeclPtr<const Decl>,
7781 [&Info, &SkipVarSet](
7787 bool ReturnDevicePointer,
bool IsImplicit,
const ValueDecl *Mapper,
7788 const Expr *VarRef =
nullptr,
bool ForDeviceAddr =
false) {
7789 if (SkipVarSet.contains(D))
7791 auto It = Info.find(D);
7792 if (It == Info.end())
7794 .insert(std::make_pair(
7797 It->second[
Kind].emplace_back(
7798 L, MapType, MapModifiers, MotionModifiers, ReturnDevicePointer,
7799 IsImplicit, Mapper, VarRef, ForDeviceAddr);
7802 for (
const auto *
Cl : Clauses) {
7803 const auto *
C = dyn_cast<OMPMapClause>(
Cl);
7807 if (llvm::is_contained(
C->getMapTypeModifiers(),
7808 OMPC_MAP_MODIFIER_present))
7810 else if (
C->getMapType() == OMPC_MAP_alloc)
7812 const auto *EI =
C->getVarRefs().begin();
7813 for (
const auto L :
C->component_lists()) {
7814 const Expr *E = (
C->getMapLoc().isValid()) ? *EI :
nullptr;
7815 InfoGen(std::get<0>(L), Kind, std::get<1>(L),
C->getMapType(),
7816 C->getMapTypeModifiers(), std::nullopt,
7817 false,
C->isImplicit(), std::get<2>(L),
7822 for (
const auto *
Cl : Clauses) {
7823 const auto *
C = dyn_cast<OMPToClause>(
Cl);
7827 if (llvm::is_contained(
C->getMotionModifiers(),
7828 OMPC_MOTION_MODIFIER_present))
7830 const auto *EI =
C->getVarRefs().begin();
7831 for (
const auto L :
C->component_lists()) {
7832 InfoGen(std::get<0>(L), Kind, std::get<1>(L), OMPC_MAP_to, std::nullopt,
7833 C->getMotionModifiers(),
false,
7834 C->isImplicit(), std::get<2>(L), *EI);
7838 for (
const auto *
Cl : Clauses) {
7839 const auto *
C = dyn_cast<OMPFromClause>(
Cl);
7843 if (llvm::is_contained(
C->getMotionModifiers(),
7844 OMPC_MOTION_MODIFIER_present))
7846 const auto *EI =
C->getVarRefs().begin();
7847 for (
const auto L :
C->component_lists()) {
7848 InfoGen(std::get<0>(L), Kind, std::get<1>(L), OMPC_MAP_from,
7849 std::nullopt,
C->getMotionModifiers(),
7850 false,
C->isImplicit(), std::get<2>(L),
7863 llvm::MapVector<CanonicalDeclPtr<const Decl>,
7866 MapCombinedInfoTy UseDeviceDataCombinedInfo;
7868 auto &&UseDeviceDataCombinedInfoGen =
7869 [&UseDeviceDataCombinedInfo](
const ValueDecl *VD, llvm::Value *Ptr,
7871 UseDeviceDataCombinedInfo.Exprs.push_back(VD);
7872 UseDeviceDataCombinedInfo.BasePointers.emplace_back(Ptr);
7873 UseDeviceDataCombinedInfo.DevicePtrDecls.emplace_back(VD);
7874 UseDeviceDataCombinedInfo.DevicePointers.emplace_back(
7875 IsDevAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer);
7876 UseDeviceDataCombinedInfo.Pointers.push_back(Ptr);
7877 UseDeviceDataCombinedInfo.Sizes.push_back(
7878 llvm::Constant::getNullValue(CGF.Int64Ty));
7879 UseDeviceDataCombinedInfo.Types.push_back(
7880 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM);
7881 UseDeviceDataCombinedInfo.Mappers.push_back(
nullptr);
7885 [&DeferredInfo, &UseDeviceDataCombinedInfoGen,
7889 bool IsImplicit,
bool IsDevAddr) {
7893 if (isa<MemberExpr>(IE)) {
7902 std::nullopt,
false, IsImplicit,
7903 nullptr,
nullptr, IsDevAddr);
7904 DeferredInfo[
nullptr].emplace_back(IE, VD, IsDevAddr);
7908 if (IE->isGLValue())
7909 Ptr = CGF.EmitLValue(IE).getPointer(CGF);
7911 Ptr = CGF.EmitScalarExpr(IE);
7913 Ptr = CGF.EmitLoadOfScalar(CGF.EmitLValue(IE), IE->getExprLoc());
7915 UseDeviceDataCombinedInfoGen(VD, Ptr, CGF, IsDevAddr);
7920 const Expr *IE,
bool IsDevAddr) ->
bool {
7927 auto It = Info.find(isa<MemberExpr>(IE) ? nullptr : VD);
7928 if (It != Info.end()) {
7930 for (
auto &
Data : It->second) {
7931 auto *CI = llvm::find_if(
Data, [VD](
const MapInfo &MI) {
7932 return MI.Components.back().getAssociatedDeclaration() == VD;
7940 if (CI !=
Data.end()) {
7942 CI->ForDeviceAddr = IsDevAddr;
7943 CI->ReturnDevicePointer =
true;
7947 auto PrevCI = std::next(CI->Components.rbegin());
7948 const auto *VarD = dyn_cast<VarDecl>(VD);
7950 isa<MemberExpr>(IE) ||
7951 !VD->getType().getNonReferenceType()->isPointerType() ||
7952 PrevCI == CI->Components.rend() ||
7953 isa<MemberExpr>(PrevCI->getAssociatedExpression()) || !VarD ||
7954 VarD->hasLocalStorage()) {
7955 CI->ForDeviceAddr = IsDevAddr;
7956 CI->ReturnDevicePointer =
true;
7974 for (
const auto *
Cl : Clauses) {
7975 const auto *
C = dyn_cast<OMPUseDevicePtrClause>(
Cl);
7978 for (
const auto L :
C->component_lists()) {
7981 assert(!Components.empty() &&
7982 "Not expecting empty list of components!");
7983 const ValueDecl *VD = Components.back().getAssociatedDeclaration();
7985 const Expr *IE = Components.back().getAssociatedExpression();
7986 if (IsMapInfoExist(CGF, VD, IE,
false))
7988 MapInfoGen(CGF, IE, VD, Components,
C->isImplicit(),
7993 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
7994 for (
const auto *
Cl : Clauses) {
7995 const auto *
C = dyn_cast<OMPUseDeviceAddrClause>(
Cl);
7998 for (
const auto L :
C->component_lists()) {
8001 assert(!std::get<1>(L).empty() &&
8002 "Not expecting empty list of components!");
8003 const ValueDecl *VD = std::get<1>(L).back().getAssociatedDeclaration();
8004 if (!Processed.insert(VD).second)
8007 const Expr *IE = std::get<1>(L).back().getAssociatedExpression();
8008 if (IsMapInfoExist(CGF, VD, IE,
true))
8010 MapInfoGen(CGF, IE, VD, Components,
C->isImplicit(),
8015 for (
const auto &
Data : Info) {
8016 StructRangeInfoTy PartialStruct;
8018 MapCombinedInfoTy CurInfo;
8020 MapCombinedInfoTy StructBaseCurInfo;
8022 const ValueDecl *VD = cast_or_null<ValueDecl>(D);
8023 for (
const auto &M :
Data.second) {
8024 for (
const MapInfo &L : M) {
8025 assert(!L.Components.empty() &&
8026 "Not expecting declaration with no component lists.");
8029 unsigned CurrentBasePointersIdx = CurInfo.BasePointers.size();
8030 unsigned StructBasePointersIdx =
8031 StructBaseCurInfo.BasePointers.size();
8032 CurInfo.NonContigInfo.IsNonContiguous =
8033 L.Components.back().isNonContiguous();
8034 generateInfoForComponentList(
8035 L.MapType, L.MapModifiers, L.MotionModifiers, L.Components,
8036 CurInfo, StructBaseCurInfo, PartialStruct,
8037 false, L.IsImplicit,
8038 true, L.Mapper, L.ForDeviceAddr, VD,
8043 if (L.ReturnDevicePointer) {
8047 assert((CurrentBasePointersIdx < CurInfo.BasePointers.size() ||
8048 StructBasePointersIdx <
8049 StructBaseCurInfo.BasePointers.size()) &&
8050 "Unexpected number of mapped base pointers.");
8054 L.Components.back().getAssociatedDeclaration();
8055 assert(RelevantVD &&
8056 "No relevant declaration related with device pointer??");
8063 if (StructBasePointersIdx < StructBaseCurInfo.BasePointers.size()) {
8064 StructBaseCurInfo.DevicePtrDecls[StructBasePointersIdx] =
8066 StructBaseCurInfo.DevicePointers[StructBasePointersIdx] =
8067 L.ForDeviceAddr ? DeviceInfoTy::Address
8068 : DeviceInfoTy::Pointer;
8069 StructBaseCurInfo.Types[StructBasePointersIdx] |=
8070 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
8072 CurInfo.DevicePtrDecls[CurrentBasePointersIdx] = RelevantVD;
8073 CurInfo.DevicePointers[CurrentBasePointersIdx] =
8074 L.ForDeviceAddr ? DeviceInfoTy::Address
8075 : DeviceInfoTy::Pointer;
8076 CurInfo.Types[CurrentBasePointersIdx] |=
8077 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
8085 auto CI = DeferredInfo.find(
Data.first);
8086 if (CI != DeferredInfo.end()) {
8087 for (
const DeferredDevicePtrEntryTy &L : CI->second) {
8088 llvm::Value *BasePtr;
8090 if (L.ForDeviceAddr) {
8091 if (L.IE->isGLValue())
8099 CurInfo.Types.push_back(
8100 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM |
8101 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
8105 L.IE->getExprLoc());
8109 CurInfo.Types.push_back(
8110 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
8111 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM |
8112 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
8114 CurInfo.Exprs.push_back(L.VD);
8115 CurInfo.BasePointers.emplace_back(BasePtr);
8116 CurInfo.DevicePtrDecls.emplace_back(L.VD);
8117 CurInfo.DevicePointers.emplace_back(
8118 L.ForDeviceAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer);
8119 CurInfo.Pointers.push_back(Ptr);
8120 CurInfo.Sizes.push_back(
8121 llvm::Constant::getNullValue(this->CGF.
Int64Ty));
8122 CurInfo.Mappers.push_back(
nullptr);
8128 MapCombinedInfoTy UnionCurInfo;
8129 UnionCurInfo.append(StructBaseCurInfo);
8130 UnionCurInfo.append(CurInfo);
8134 if (PartialStruct.Base.isValid()) {
8135 UnionCurInfo.NonContigInfo.Dims.push_back(0);
8137 emitCombinedEntry(CombinedInfo, UnionCurInfo.Types, PartialStruct,
8138 !VD, OMPBuilder, VD);
8142 CombinedInfo.append(UnionCurInfo);
8145 CombinedInfo.append(UseDeviceDataCombinedInfo);
8150 : CurDir(&Dir), CGF(CGF) {
8153 for (
const auto *D :
C->varlists())
8154 FirstPrivateDecls.try_emplace(
8155 cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl()),
C->isImplicit());
8158 for (
unsigned I = 0, E =
C->getNumberOfAllocators(); I < E; ++I) {
8160 if (
const auto *DRE = dyn_cast_or_null<DeclRefExpr>(D.
AllocatorTraits))
8161 FirstPrivateDecls.try_emplace(cast<VarDecl>(DRE->getDecl()),
8163 else if (
const auto *VD = dyn_cast<VarDecl>(
8166 FirstPrivateDecls.try_emplace(VD,
true);
8171 for (
auto L :
C->component_lists())
8172 DevPointersMap[std::get<0>(L)].push_back(std::get<1>(L));
8175 for (
auto L :
C->component_lists())
8176 HasDevAddrsMap[std::get<0>(L)].push_back(std::get<1>(L));
8179 if (
C->getMapType() != OMPC_MAP_to)
8181 for (
auto L :
C->component_lists()) {
8183 const auto *RD = VD ? VD->
getType()
8189 LambdasMap.try_emplace(std::get<0>(L),
C);
8196 : CurDir(&Dir), CGF(CGF) {}
8201 void emitCombinedEntry(MapCombinedInfoTy &CombinedInfo,
8202 MapFlagsArrayTy &CurTypes,
8203 const StructRangeInfoTy &PartialStruct,
bool IsMapThis,
8204 llvm::OpenMPIRBuilder &OMPBuilder,
8206 bool NotTargetParams =
true)
const {
8207 if (CurTypes.size() == 1 &&
8208 ((CurTypes.back() & OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) !=
8209 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) &&
8210 !PartialStruct.IsArraySection)
8212 Address LBAddr = PartialStruct.LowestElem.second;
8213 Address HBAddr = PartialStruct.HighestElem.second;
8214 if (PartialStruct.HasCompleteRecord) {
8215 LBAddr = PartialStruct.LB;
8216 HBAddr = PartialStruct.LB;
8218 CombinedInfo.Exprs.push_back(VD);
8220 CombinedInfo.BasePointers.push_back(PartialStruct.Base.emitRawPointer(CGF));
8221 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8222 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8228 bool HasBaseClass = RD && IsMapThis ? RD->
getNumBases() > 0 :
false;
8238 CombinedInfo.Pointers.push_back(PartialStruct.Base.emitRawPointer(CGF));
8243 CombinedInfo.Sizes.push_back(Size);
8245 CombinedInfo.Pointers.push_back(LB);
8248 llvm::Value *HAddr = CGF.
Builder.CreateConstGEP1_32(
8252 llvm::Value *Diff = CGF.
Builder.CreatePtrDiff(CGF.
Int8Ty, CHAddr, CLAddr);
8255 CombinedInfo.Sizes.push_back(Size);
8257 CombinedInfo.Mappers.push_back(
nullptr);
8259 CombinedInfo.Types.push_back(
8260 NotTargetParams ? OpenMPOffloadMappingFlags::OMP_MAP_NONE
8261 : OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
8264 if (CurTypes.end() !=
8265 llvm::find_if(CurTypes, [](OpenMPOffloadMappingFlags
Type) {
8266 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
8267 Type & OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
8269 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
8271 (*CurTypes.begin()) &= ~OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
8278 if (CurTypes.end() !=
8279 llvm::find_if(CurTypes, [](OpenMPOffloadMappingFlags
Type) {
8280 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
8281 Type & OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD);
8283 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
8284 for (
auto &M : CurTypes)
8285 M |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
8291 OpenMPOffloadMappingFlags MemberOfFlag =
8292 OMPBuilder.getMemberOfFlag(CombinedInfo.BasePointers.size() - 1);
8293 for (
auto &M : CurTypes)
8294 OMPBuilder.setCorrectMemberOfFlag(M, MemberOfFlag);
8302 void generateAllInfo(
8303 MapCombinedInfoTy &CombinedInfo, llvm::OpenMPIRBuilder &OMPBuilder,
8307 "Expect a executable directive");
8309 generateAllInfoForClauses(CurExecDir->clauses(), CombinedInfo, OMPBuilder,
8316 void generateAllInfoForMapper(MapCombinedInfoTy &CombinedInfo,
8317 llvm::OpenMPIRBuilder &OMPBuilder)
const {
8319 "Expect a declare mapper directive");
8321 generateAllInfoForClauses(CurMapperDir->clauses(), CombinedInfo,
8326 void generateInfoForLambdaCaptures(
8327 const ValueDecl *VD, llvm::Value *Arg, MapCombinedInfoTy &CombinedInfo,
8328 llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers)
const {
8336 llvm::DenseMap<const ValueDecl *, FieldDecl *> Captures;
8343 LambdaPointers.try_emplace(ThisLVal.
getPointer(CGF),
8345 CombinedInfo.Exprs.push_back(VD);
8346 CombinedInfo.BasePointers.push_back(ThisLVal.
getPointer(CGF));
8347 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8348 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8349 CombinedInfo.Pointers.push_back(ThisLValVal.
getPointer(CGF));
8350 CombinedInfo.Sizes.push_back(
8353 CombinedInfo.Types.push_back(
8354 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
8355 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
8356 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
8357 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
8358 CombinedInfo.Mappers.push_back(
nullptr);
8361 if (!LC.capturesVariable())
8363 const VarDecl *VD = cast<VarDecl>(LC.getCapturedVar());
8366 auto It = Captures.find(VD);
8367 assert(It != Captures.end() &&
"Found lambda capture without field.");
8371 LambdaPointers.try_emplace(VarLVal.
getPointer(CGF),
8373 CombinedInfo.Exprs.push_back(VD);
8374 CombinedInfo.BasePointers.push_back(VarLVal.
getPointer(CGF));
8375 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8376 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8377 CombinedInfo.Pointers.push_back(VarLValVal.
getPointer(CGF));
8378 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
8384 LambdaPointers.try_emplace(VarLVal.
getPointer(CGF),
8386 CombinedInfo.Exprs.push_back(VD);
8387 CombinedInfo.BasePointers.push_back(VarLVal.
getPointer(CGF));
8388 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8389 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8390 CombinedInfo.Pointers.push_back(VarRVal.
getScalarVal());
8391 CombinedInfo.Sizes.push_back(llvm::ConstantInt::get(CGF.
Int64Ty, 0));
8393 CombinedInfo.Types.push_back(
8394 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
8395 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
8396 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
8397 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
8398 CombinedInfo.Mappers.push_back(
nullptr);
8403 void adjustMemberOfForLambdaCaptures(
8404 llvm::OpenMPIRBuilder &OMPBuilder,
8405 const llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers,
8406 MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
8407 MapFlagsArrayTy &Types)
const {
8408 for (
unsigned I = 0, E = Types.size(); I < E; ++I) {
8410 if (Types[I] != (OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
8411 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
8412 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
8413 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT))
8415 llvm::Value *BasePtr = LambdaPointers.lookup(BasePointers[I]);
8416 assert(BasePtr &&
"Unable to find base lambda address.");
8418 for (
unsigned J = I; J > 0; --J) {
8419 unsigned Idx = J - 1;
8420 if (Pointers[Idx] != BasePtr)
8425 assert(TgtIdx != -1 &&
"Unable to find parent lambda.");
8429 OpenMPOffloadMappingFlags MemberOfFlag =
8430 OMPBuilder.getMemberOfFlag(TgtIdx);
8431 OMPBuilder.setCorrectMemberOfFlag(Types[I], MemberOfFlag);
8438 llvm::Value *Arg, MapCombinedInfoTy &CombinedInfo,
8439 StructRangeInfoTy &PartialStruct)
const {
8441 "Not expecting to generate map info for a variable array type!");
8450 if (LambdasMap.count(VD))
8456 if (VD && (DevPointersMap.count(VD) || HasDevAddrsMap.count(VD))) {
8457 CombinedInfo.Exprs.push_back(VD);
8458 CombinedInfo.BasePointers.emplace_back(Arg);
8459 CombinedInfo.DevicePtrDecls.emplace_back(VD);
8460 CombinedInfo.DevicePointers.emplace_back(DeviceInfoTy::Pointer);
8461 CombinedInfo.Pointers.push_back(Arg);
8462 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
8465 CombinedInfo.Types.push_back(
8466 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
8467 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
8468 CombinedInfo.Mappers.push_back(
nullptr);
8480 auto It = DevPointersMap.find(VD);
8481 if (It != DevPointersMap.end())
8482 for (
const auto &MCL : It->second)
8483 DeclComponentLists.emplace_back(MCL, OMPC_MAP_to,
Unknown,
8486 auto I = HasDevAddrsMap.find(VD);
8487 if (I != HasDevAddrsMap.end())
8488 for (
const auto &MCL : I->second)
8489 DeclComponentLists.emplace_back(MCL, OMPC_MAP_tofrom,
Unknown,
8493 "Expect a executable directive");
8495 for (
const auto *
C : CurExecDir->getClausesOfKind<
OMPMapClause>()) {
8496 const auto *EI =
C->getVarRefs().begin();
8497 for (
const auto L :
C->decl_component_lists(VD)) {
8500 const Expr *E = (
C->getMapLoc().isValid()) ? *EI :
nullptr;
8502 std::tie(VDecl, Components, Mapper) = L;
8503 assert(VDecl == VD &&
"We got information for the wrong declaration??");
8504 assert(!Components.empty() &&
8505 "Not expecting declaration with no component lists.");
8506 DeclComponentLists.emplace_back(Components,
C->getMapType(),
8507 C->getMapTypeModifiers(),
8508 C->isImplicit(), Mapper, E);
8512 llvm::stable_sort(DeclComponentLists, [](
const MapData &LHS,
8513 const MapData &RHS) {
8517 llvm::is_contained(MapModifiers, clang::OMPC_MAP_MODIFIER_present);
8518 bool HasAllocs = MapType == OMPC_MAP_alloc;
8519 MapModifiers = std::get<2>(RHS);
8520 MapType = std::get<1>(LHS);
8522 llvm::is_contained(MapModifiers, clang::OMPC_MAP_MODIFIER_present);
8523 bool HasAllocsR = MapType == OMPC_MAP_alloc;
8524 return (HasPresent && !HasPresentR) || (HasAllocs && !HasAllocsR);
8528 llvm::SmallDenseMap<
8535 for (
const MapData &L : DeclComponentLists) {
8542 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =
8545 for (
const MapData &L1 :
ArrayRef(DeclComponentLists).slice(Count)) {
8547 std::tie(Components1, MapType, MapModifiers, IsImplicit, Mapper,
8549 auto CI = Components.rbegin();
8550 auto CE = Components.rend();
8551 auto SI = Components1.rbegin();
8552 auto SE = Components1.rend();
8553 for (; CI != CE && SI != SE; ++CI, ++SI) {
8554 if (CI->getAssociatedExpression()->getStmtClass() !=
8555 SI->getAssociatedExpression()->getStmtClass())
8558 if (CI->getAssociatedDeclaration() != SI->getAssociatedDeclaration())
8563 if (CI == CE || SI == SE) {
8565 if (CI == CE && SI == SE)
8567 const auto It = (SI == SE) ? CI : SI;
8573 if (!isa<MemberExpr>(It->getAssociatedExpression()) ||
8574 (std::prev(It)->getAssociatedDeclaration() &&
8576 ->getAssociatedDeclaration()
8578 ->isPointerType()) ||
8579 (It->getAssociatedDeclaration() &&
8580 It->getAssociatedDeclaration()->getType()->isPointerType() &&
8581 std::next(It) != CE && std::next(It) != SE))
8583 const MapData &BaseData = CI == CE ? L : L1;
8585 SI == SE ? Components : Components1;
8586 auto &OverlappedElements = OverlappedData.FindAndConstruct(&BaseData);
8587 OverlappedElements.getSecond().push_back(SubData);
8593 if (!OverlappedData.empty()) {
8596 while (BaseType != OrigType) {
8602 getPlainLayout(CRD, Layout,
false);
8608 for (
auto &Pair : OverlappedData) {
8615 auto CI = First.rbegin();
8616 auto CE = First.rend();
8617 auto SI = Second.rbegin();
8618 auto SE = Second.rend();
8619 for (; CI != CE && SI != SE; ++CI, ++SI) {
8620 if (CI->getAssociatedExpression()->getStmtClass() !=
8621 SI->getAssociatedExpression()->getStmtClass())
8624 if (CI->getAssociatedDeclaration() !=
8625 SI->getAssociatedDeclaration())
8630 if (CI == CE && SI == SE)
8634 if (CI == CE || SI == SE)
8637 const auto *FD1 = cast<FieldDecl>(CI->getAssociatedDeclaration());
8638 const auto *FD2 = cast<FieldDecl>(SI->getAssociatedDeclaration());
8639 if (FD1->getParent() == FD2->getParent())
8640 return FD1->getFieldIndex() < FD2->getFieldIndex();
8642 llvm::find_if(Layout, [FD1, FD2](
const FieldDecl *FD) {
8643 return FD == FD1 || FD == FD2;
8651 bool IsFirstComponentList =
true;
8652 MapCombinedInfoTy StructBaseCombinedInfo;
8653 for (
const auto &Pair : OverlappedData) {
8654 const MapData &L = *Pair.getFirst();
8661 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =
8664 OverlappedComponents = Pair.getSecond();
8665 generateInfoForComponentList(
8666 MapType, MapModifiers, std::nullopt, Components, CombinedInfo,
8667 StructBaseCombinedInfo, PartialStruct, IsFirstComponentList,
8668 IsImplicit,
false, Mapper,
8669 false, VD, VarRef, OverlappedComponents);
8670 IsFirstComponentList =
false;
8673 for (
const MapData &L : DeclComponentLists) {
8680 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =
8682 auto It = OverlappedData.find(&L);
8683 if (It == OverlappedData.end())
8684 generateInfoForComponentList(
8685 MapType, MapModifiers, std::nullopt, Components, CombinedInfo,
8686 StructBaseCombinedInfo, PartialStruct, IsFirstComponentList,
8687 IsImplicit,
false, Mapper,
8689 IsFirstComponentList =
false;
8697 MapCombinedInfoTy &CombinedInfo)
const {
8698 bool IsImplicit =
true;
8701 CombinedInfo.Exprs.push_back(
nullptr);
8702 CombinedInfo.BasePointers.push_back(CV);
8703 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8704 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8705 CombinedInfo.Pointers.push_back(CV);
8707 CombinedInfo.Sizes.push_back(
8711 CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_TO |
8712 OpenMPOffloadMappingFlags::OMP_MAP_FROM);
8716 CombinedInfo.BasePointers.push_back(CV);
8717 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8718 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8719 CombinedInfo.Pointers.push_back(CV);
8723 CombinedInfo.Types.push_back(
8724 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
8725 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
8730 CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_NONE);
8731 CombinedInfo.Sizes.push_back(llvm::Constant::getNullValue(CGF.
Int64Ty));
8733 auto I = FirstPrivateDecls.find(VD);
8734 if (I != FirstPrivateDecls.end())
8735 IsImplicit = I->getSecond();
8740 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
8745 CombinedInfo.Types.push_back(getMapModifiersForPrivateClauses(CI));
8747 auto I = FirstPrivateDecls.find(VD);
8749 CombinedInfo.BasePointers.push_back(CV);
8750 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8751 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8758 CombinedInfo.Pointers.push_back(CV);
8760 if (I != FirstPrivateDecls.end())
8761 IsImplicit = I->getSecond();
8764 CombinedInfo.Types.back() |=
8765 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
8769 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
8772 CombinedInfo.Mappers.push_back(
nullptr);
8784 dyn_cast<MemberExpr>(OASE->getBase()->IgnoreParenImpCasts()))
8785 return ME->getMemberDecl();
8793 MappableExprsHandler::MappingExprInfo &MapExprs) {
8795 uint32_t SrcLocStrSize;
8796 if (!MapExprs.getMapDecl() && !MapExprs.getMapExpr())
8797 return OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
8800 if (!MapExprs.getMapDecl() && MapExprs.getMapExpr()) {
8804 Loc = MapExprs.getMapExpr()->getExprLoc();
8806 Loc = MapExprs.getMapDecl()->getLocation();
8809 std::string ExprName;
8810 if (MapExprs.getMapExpr()) {
8812 llvm::raw_string_ostream OS(ExprName);
8813 MapExprs.getMapExpr()->printPretty(OS,
nullptr,
P);
8816 ExprName = MapExprs.getMapDecl()->getNameAsString();
8820 return OMPBuilder.getOrCreateSrcLocStr(PLoc.
getFilename(), ExprName,
8829 CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
8831 bool IsNonContiguous =
false) {
8835 Info.clearArrayInfo();
8836 Info.NumberOfPtrs = CombinedInfo.BasePointers.size();
8838 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
8841 InsertPointTy CodeGenIP(CGF.
Builder.GetInsertBlock(),
8842 CGF.
Builder.GetInsertPoint());
8844 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
8848 llvm::codegenoptions::NoDebugInfo) {
8849 CombinedInfo.Names.resize(CombinedInfo.Exprs.size());
8850 llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),
8854 auto DeviceAddrCB = [&](
unsigned int I, llvm::Value *NewDecl) {
8855 if (
const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
8860 auto CustomMapperCB = [&](
unsigned int I) {
8861 llvm::Value *MFunc =
nullptr;
8862 if (CombinedInfo.Mappers[I]) {
8863 Info.HasMapper =
true;
8865 cast<OMPDeclareMapperDecl>(CombinedInfo.Mappers[I]));
8869 OMPBuilder.emitOffloadingArrays(AllocaIP, CodeGenIP, CombinedInfo, Info,
8880 const Stmt *ChildStmt =
8883 if (
const auto *NestedDir =
8884 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
8892 if (DKind == OMPD_teams) {
8893 Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers(
8898 if (
const auto *NND =
8899 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
8900 DKind = NND->getDirectiveKind();
8906 case OMPD_target_teams:
8910 case OMPD_target_parallel:
8911 case OMPD_target_simd:
8912 case OMPD_target_parallel_for:
8913 case OMPD_target_parallel_for_simd:
8915 case OMPD_target_teams_distribute:
8916 case OMPD_target_teams_distribute_simd:
8917 case OMPD_target_teams_distribute_parallel_for:
8918 case OMPD_target_teams_distribute_parallel_for_simd:
8921 case OMPD_parallel_for:
8922 case OMPD_parallel_master:
8923 case OMPD_parallel_sections:
8925 case OMPD_parallel_for_simd:
8927 case OMPD_cancellation_point:
8929 case OMPD_threadprivate:
8940 case OMPD_taskyield:
8943 case OMPD_taskgroup:
8949 case OMPD_target_data:
8950 case OMPD_target_exit_data:
8951 case OMPD_target_enter_data:
8952 case OMPD_distribute:
8953 case OMPD_distribute_simd:
8954 case OMPD_distribute_parallel_for:
8955 case OMPD_distribute_parallel_for_simd:
8956 case OMPD_teams_distribute:
8957 case OMPD_teams_distribute_simd:
8958 case OMPD_teams_distribute_parallel_for:
8959 case OMPD_teams_distribute_parallel_for_simd:
8960 case OMPD_target_update:
8961 case OMPD_declare_simd:
8962 case OMPD_declare_variant:
8963 case OMPD_begin_declare_variant:
8964 case OMPD_end_declare_variant:
8965 case OMPD_declare_target:
8966 case OMPD_end_declare_target:
8967 case OMPD_declare_reduction:
8968 case OMPD_declare_mapper:
8970 case OMPD_taskloop_simd:
8971 case OMPD_master_taskloop:
8972 case OMPD_master_taskloop_simd:
8973 case OMPD_parallel_master_taskloop:
8974 case OMPD_parallel_master_taskloop_simd:
8976 case OMPD_metadirective:
8979 llvm_unreachable(
"Unexpected directive.");
9024 QualType PtrTy =
C.getPointerType(Ty).withRestrict();
9025 QualType Int64Ty =
C.getIntTypeForBitwidth(64,
true);
9026 auto *MapperVarDecl =
9029 CharUnits ElementSize =
C.getTypeSizeInChars(Ty);
9046 Args.push_back(&HandleArg);
9047 Args.push_back(&BaseArg);
9048 Args.push_back(&BeginArg);
9049 Args.push_back(&SizeArg);
9050 Args.push_back(&TypeArg);
9051 Args.push_back(&NameArg);
9056 llvm::raw_svector_ostream Out(TyStr);
9059 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
9062 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
9069 C.getPointerType(Int64Ty), Loc);
9073 false,
C.getPointerType(
C.VoidPtrTy), Loc);
9076 false,
C.getPointerType(
C.VoidPtrTy), Loc);
9079 false,
C.getPointerType(
C.VoidPtrTy), Loc);
9081 Size = MapperCGF.
Builder.CreateExactUDiv(
9083 llvm::Value *PtrBegin = MapperCGF.
Builder.CreateBitCast(
9088 C.getPointerType(Int64Ty), Loc);
9091 false,
C.getPointerType(
C.VoidPtrTy), Loc);
9095 llvm::BasicBlock *HeadBB = MapperCGF.
createBasicBlock(
"omp.arraymap.head");
9097 MapName, ElementSize, HeadBB,
true);
9103 llvm::BasicBlock *BodyBB = MapperCGF.
createBasicBlock(
"omp.arraymap.body");
9106 llvm::Value *IsEmpty =
9107 MapperCGF.
Builder.CreateICmpEQ(PtrBegin, PtrEnd,
"omp.arraymap.isempty");
9108 MapperCGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
9109 llvm::BasicBlock *EntryBB = MapperCGF.
Builder.GetInsertBlock();
9113 llvm::BasicBlock *LastBB = BodyBB;
9114 llvm::PHINode *PtrPHI = MapperCGF.
Builder.CreatePHI(
9115 PtrBegin->getType(), 2,
"omp.arraymap.ptrcurrent");
9116 PtrPHI->addIncoming(PtrBegin, EntryBB);
9117 Address PtrCurrent(PtrPHI, ElemTy,
9123 Scope.addPrivate(MapperVarDecl, PtrCurrent);
9124 (void)
Scope.Privatize();
9127 MappableExprsHandler::MapCombinedInfoTy Info;
9128 MappableExprsHandler MEHandler(*D, MapperCGF);
9129 MEHandler.generateAllInfoForMapper(Info,
OMPBuilder);
9133 llvm::Value *OffloadingArgs[] = {Handle};
9136 OMPRTL___tgt_mapper_num_components),
9138 llvm::Value *ShiftedPreviousSize = MapperCGF.
Builder.CreateShl(
9140 MapperCGF.
Builder.getInt64(MappableExprsHandler::getFlagMemberOffset()));
9143 for (
unsigned I = 0; I < Info.BasePointers.size(); ++I) {
9144 llvm::Value *CurBaseArg = MapperCGF.
Builder.CreateBitCast(
9146 llvm::Value *CurBeginArg = MapperCGF.
Builder.CreateBitCast(
9148 llvm::Value *CurSizeArg = Info.Sizes[I];
9149 llvm::Value *CurNameArg =
9151 llvm::codegenoptions::NoDebugInfo)
9156 llvm::Value *OriMapType = MapperCGF.
Builder.getInt64(
9157 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9159 llvm::Value *MemberMapType =
9160 MapperCGF.
Builder.CreateNUWAdd(OriMapType, ShiftedPreviousSize);
9174 llvm::Value *LeftToFrom = MapperCGF.
Builder.CreateAnd(
9177 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9178 OpenMPOffloadMappingFlags::OMP_MAP_TO |
9179 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
9181 llvm::BasicBlock *AllocElseBB =
9184 llvm::BasicBlock *ToElseBB = MapperCGF.
createBasicBlock(
"omp.type.to.else");
9188 MapperCGF.
Builder.CreateCondBr(IsAlloc, AllocBB, AllocElseBB);
9191 llvm::Value *AllocMapType = MapperCGF.
Builder.CreateAnd(
9194 ~
static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9195 OpenMPOffloadMappingFlags::OMP_MAP_TO |
9196 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
9197 MapperCGF.
Builder.CreateBr(EndBB);
9199 llvm::Value *IsTo = MapperCGF.
Builder.CreateICmpEQ(
9202 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9203 OpenMPOffloadMappingFlags::OMP_MAP_TO)));
9204 MapperCGF.
Builder.CreateCondBr(IsTo, ToBB, ToElseBB);
9207 llvm::Value *ToMapType = MapperCGF.
Builder.CreateAnd(
9210 ~
static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9211 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
9212 MapperCGF.
Builder.CreateBr(EndBB);
9214 llvm::Value *IsFrom = MapperCGF.
Builder.CreateICmpEQ(
9217 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9218 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
9219 MapperCGF.
Builder.CreateCondBr(IsFrom, FromBB, EndBB);
9222 llvm::Value *FromMapType = MapperCGF.
Builder.CreateAnd(
9225 ~
static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9226 OpenMPOffloadMappingFlags::OMP_MAP_TO)));
9230 llvm::PHINode *CurMapType =
9232 CurMapType->addIncoming(AllocMapType, AllocBB);
9233 CurMapType->addIncoming(ToMapType, ToBB);
9234 CurMapType->addIncoming(FromMapType, FromBB);
9235 CurMapType->addIncoming(MemberMapType, ToElseBB);
9237 llvm::Value *OffloadingArgs[] = {Handle, CurBaseArg, CurBeginArg,
9238 CurSizeArg, CurMapType, CurNameArg};
9239 if (Info.Mappers[I]) {
9242 cast<OMPDeclareMapperDecl>(Info.Mappers[I]));
9243 assert(MapperFunc &&
"Expect a valid mapper function is available.");
9257 llvm::Value *PtrNext = MapperCGF.
Builder.CreateConstGEP1_32(
9258 ElemTy, PtrPHI, 1,
"omp.arraymap.next");
9259 PtrPHI->addIncoming(PtrNext, LastBB);
9260 llvm::Value *IsDone =
9261 MapperCGF.
Builder.CreateICmpEQ(PtrNext, PtrEnd,
"omp.arraymap.isdone");
9262 llvm::BasicBlock *ExitBB = MapperCGF.
createBasicBlock(
"omp.arraymap.exit");
9263 MapperCGF.
Builder.CreateCondBr(IsDone, ExitBB, BodyBB);
9269 MapName, ElementSize, DoneBB,
false);
9274 UDMMap.try_emplace(D, Fn);
9277 Decls.second.push_back(D);
9289 llvm::Value *
Begin, llvm::Value *Size, llvm::Value *MapType,
9290 llvm::Value *MapName,
CharUnits ElementSize, llvm::BasicBlock *ExitBB,
9292 StringRef Prefix = IsInit ?
".init" :
".del";
9295 llvm::BasicBlock *BodyBB =
9297 llvm::Value *IsArray = MapperCGF.
Builder.CreateICmpSGT(
9298 Size, MapperCGF.
Builder.getInt64(1),
"omp.arrayinit.isarray");
9299 llvm::Value *DeleteBit = MapperCGF.
Builder.CreateAnd(
9302 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9303 OpenMPOffloadMappingFlags::OMP_MAP_DELETE)));
9304 llvm::Value *DeleteCond;
9310 llvm::Value *PtrAndObjBit = MapperCGF.
Builder.CreateAnd(
9313 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9314 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ)));
9315 PtrAndObjBit = MapperCGF.
Builder.CreateIsNotNull(PtrAndObjBit);
9316 BaseIsBegin = MapperCGF.
Builder.CreateAnd(BaseIsBegin, PtrAndObjBit);
9317 Cond = MapperCGF.
Builder.CreateOr(IsArray, BaseIsBegin);
9319 DeleteBit,
getName({
"omp.array", Prefix,
".delete"}));
9322 DeleteCond = MapperCGF.
Builder.CreateIsNotNull(
9323 DeleteBit,
getName({
"omp.array", Prefix,
".delete"}));
9325 Cond = MapperCGF.
Builder.CreateAnd(Cond, DeleteCond);
9326 MapperCGF.
Builder.CreateCondBr(Cond, BodyBB, ExitBB);
9331 llvm::Value *ArraySize = MapperCGF.
Builder.CreateNUWMul(
9335 llvm::Value *MapTypeArg = MapperCGF.
Builder.CreateAnd(
9338 ~
static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9339 OpenMPOffloadMappingFlags::OMP_MAP_TO |
9340 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
9341 MapTypeArg = MapperCGF.
Builder.CreateOr(
9344 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9345 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT)));
9349 llvm::Value *OffloadingArgs[] = {Handle,
Base,
Begin,
9350 ArraySize, MapTypeArg, MapName};
9353 OMPRTL___tgt_push_mapper_component),
9376 Kind != OMPD_target_teams_loop)
9379 return llvm::ConstantInt::get(CGF.
Int64Ty, 0);
9381 const auto *LD = cast<OMPLoopDirective>(TD);
9382 if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD))
9383 return NumIterations;
9384 return llvm::ConstantInt::get(CGF.
Int64Ty, 0);
9393 if (OffloadingMandatory) {
9394 CGF.
Builder.CreateUnreachable();
9396 if (RequiresOuterTask) {
9397 CapturedVars.clear();
9406 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
9409 llvm::Value *DeviceID;
9410 if (
Device.getPointer()) {
9412 Device.getInt() == OMPC_DEVICE_device_num) &&
9413 "Expected device_num modifier.");
9418 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
9425 llvm::Value *DynCGroupMem = CGF.
Builder.getInt32(0);
9428 CodeGenFunction::RunCleanupsScope DynCGroupMemScope(CGF);
9430 DynMemClause->getSize(),
true);
9431 DynCGroupMem = CGF.
Builder.CreateIntCast(DynCGroupMemVal, CGF.
Int32Ty,
9434 return DynCGroupMem;
9442 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
9443 llvm::Value *OutlinedFnID, CodeGenFunction::OMPTargetDataInfo &InputInfo,
9444 llvm::Value *&MapTypesArray, llvm::Value *&MapNamesArray,
9449 llvm::OpenMPIRBuilder &OMPBuilder = OMPRuntime->
getOMPBuilder();
9452 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
9455 MappableExprsHandler MEHandler(D, CGF);
9456 llvm::DenseMap<llvm::Value *, llvm::Value *> LambdaPointers;
9460 auto *CV = CapturedVars.begin();
9463 CI != CE; ++CI, ++RI, ++CV) {
9464 MappableExprsHandler::MapCombinedInfoTy CurInfo;
9465 MappableExprsHandler::StructRangeInfoTy PartialStruct;
9470 CurInfo.Exprs.push_back(
nullptr);
9471 CurInfo.BasePointers.push_back(*CV);
9472 CurInfo.DevicePtrDecls.push_back(
nullptr);
9473 CurInfo.DevicePointers.push_back(
9474 MappableExprsHandler::DeviceInfoTy::None);
9475 CurInfo.Pointers.push_back(*CV);
9476 CurInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
9479 CurInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9480 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM |
9481 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
9482 CurInfo.Mappers.push_back(
nullptr);
9486 MEHandler.generateInfoForCapture(CI, *CV, CurInfo, PartialStruct);
9490 MappedVarSet.insert(
nullptr);
9491 if (CurInfo.BasePointers.empty() && !PartialStruct.Base.isValid())
9492 MEHandler.generateDefaultMapInfo(*CI, **RI, *CV, CurInfo);
9496 MEHandler.generateInfoForLambdaCaptures(CI->
getCapturedVar(), *CV,
9497 CurInfo, LambdaPointers);
9500 assert((!CurInfo.BasePointers.empty() || PartialStruct.Base.isValid()) &&
9501 "Non-existing map pointer for capture!");
9502 assert(CurInfo.BasePointers.size() == CurInfo.Pointers.size() &&
9503 CurInfo.BasePointers.size() == CurInfo.Sizes.size() &&
9504 CurInfo.BasePointers.size() == CurInfo.Types.size() &&
9505 CurInfo.BasePointers.size() == CurInfo.Mappers.size() &&
9506 "Inconsistent map information sizes!");
9510 if (PartialStruct.Base.isValid()) {
9511 CombinedInfo.append(PartialStruct.PreliminaryMapData);
9512 MEHandler.emitCombinedEntry(
9513 CombinedInfo, CurInfo.Types, PartialStruct, CI->
capturesThis(),
9514 OMPBuilder,
nullptr,
9515 !PartialStruct.PreliminaryMapData.BasePointers.empty());
9519 CombinedInfo.append(CurInfo);
9522 MEHandler.adjustMemberOfForLambdaCaptures(
9523 OMPBuilder, LambdaPointers, CombinedInfo.BasePointers,
9524 CombinedInfo.Pointers, CombinedInfo.Types);
9527 MEHandler.generateAllInfo(CombinedInfo, OMPBuilder, MappedVarSet);
9533 llvm::codegenoptions::NoDebugInfo;
9534 OMPBuilder.emitOffloadingArraysArgument(CGF.
Builder, Info.RTArgs, Info,
9538 InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
9539 InputInfo.BasePointersArray =
Address(Info.RTArgs.BasePointersArray,
9541 InputInfo.PointersArray =
9543 InputInfo.SizesArray =
9545 InputInfo.MappersArray =
9547 MapTypesArray = Info.RTArgs.MapTypesArray;
9548 MapNamesArray = Info.RTArgs.MapNamesArray;
9550 auto &&ThenGen = [&OMPRuntime, OutlinedFn, &D, &CapturedVars,
9551 RequiresOuterTask, &CS, OffloadingMandatory,
Device,
9552 OutlinedFnID, &InputInfo, &MapTypesArray, &MapNamesArray,
9554 bool IsReverseOffloading =
Device.getInt() == OMPC_DEVICE_ancestor;
9556 if (IsReverseOffloading) {
9562 RequiresOuterTask, CS, OffloadingMandatory, CGF);
9567 unsigned NumTargetItems = InputInfo.NumberOfTargetItems;
9569 llvm::Value *BasePointersArray =
9570 InputInfo.BasePointersArray.emitRawPointer(CGF);
9571 llvm::Value *PointersArray = InputInfo.PointersArray.emitRawPointer(CGF);
9572 llvm::Value *SizesArray = InputInfo.SizesArray.emitRawPointer(CGF);
9573 llvm::Value *MappersArray = InputInfo.MappersArray.emitRawPointer(CGF);
9575 auto &&EmitTargetCallFallbackCB =
9576 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
9577 OffloadingMandatory, &CGF](llvm::OpenMPIRBuilder::InsertPointTy IP)
9578 -> llvm::OpenMPIRBuilder::InsertPointTy {
9581 RequiresOuterTask, CS, OffloadingMandatory, CGF);
9587 llvm::Value *NumThreads =
9590 llvm::Value *NumIterations =
9593 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
9596 llvm::OpenMPIRBuilder::TargetDataRTArgs RTArgs(
9597 BasePointersArray, PointersArray, SizesArray, MapTypesArray,
9598 nullptr , MappersArray, MapNamesArray);
9600 llvm::OpenMPIRBuilder::TargetKernelArgs Args(
9601 NumTargetItems, RTArgs, NumIterations, NumTeams, NumThreads,
9602 DynCGGroupMem, HasNoWait);
9605 CGF.
Builder, OutlinedFn, OutlinedFnID, EmitTargetCallFallbackCB, Args,
9606 DeviceID, RTLoc, AllocaIP));
9609 if (RequiresOuterTask)
9624 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
9627 RequiresOuterTask, CS, OffloadingMandatory, CGF);
9630 if (RequiresOuterTask) {
9631 CodeGenFunction::OMPTargetDataInfo InputInfo;
9640 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID,
const Expr *IfCond,
9641 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
9648 const bool OffloadingMandatory = !
CGM.
getLangOpts().OpenMPIsTargetDevice &&
9651 assert((OffloadingMandatory || OutlinedFn) &&
"Invalid outlined function!");
9653 const bool RequiresOuterTask =
9669 llvm::Value *MapTypesArray =
nullptr;
9670 llvm::Value *MapNamesArray =
nullptr;
9672 auto &&TargetThenGen = [
this, OutlinedFn, &D, &CapturedVars,
9673 RequiresOuterTask, &CS, OffloadingMandatory,
Device,
9674 OutlinedFnID, &InputInfo, &MapTypesArray,
9678 RequiresOuterTask, CS, OffloadingMandatory,
9679 Device, OutlinedFnID, InputInfo, MapTypesArray,
9680 MapNamesArray, SizeEmitter, CGF,
CGM);
9683 auto &&TargetElseGen =
9684 [
this, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
9687 CS, OffloadingMandatory, CGF);
9696 emitIfClause(CGF, IfCond, TargetThenGen, TargetElseGen);
9708 StringRef ParentName) {
9713 bool RequiresDeviceCodegen =
9714 isa<OMPExecutableDirective>(S) &&
9716 cast<OMPExecutableDirective>(S)->getDirectiveKind());
9718 if (RequiresDeviceCodegen) {
9719 const auto &E = *cast<OMPExecutableDirective>(S);
9726 if (!
OMPBuilder.OffloadInfoManager.hasTargetRegionEntryInfo(EntryInfo))
9729 switch (E.getDirectiveKind()) {
9732 cast<OMPTargetDirective>(E));
9734 case OMPD_target_parallel:
9736 CGM, ParentName, cast<OMPTargetParallelDirective>(E));
9738 case OMPD_target_teams:
9740 CGM, ParentName, cast<OMPTargetTeamsDirective>(E));
9742 case OMPD_target_teams_distribute:
9744 CGM, ParentName, cast<OMPTargetTeamsDistributeDirective>(E));
9746 case OMPD_target_teams_distribute_simd:
9748 CGM, ParentName, cast<OMPTargetTeamsDistributeSimdDirective>(E));
9750 case OMPD_target_parallel_for:
9752 CGM, ParentName, cast<OMPTargetParallelForDirective>(E));
9754 case OMPD_target_parallel_for_simd:
9756 CGM, ParentName, cast<OMPTargetParallelForSimdDirective>(E));
9758 case OMPD_target_simd:
9760 CGM, ParentName, cast<OMPTargetSimdDirective>(E));
9762 case OMPD_target_teams_distribute_parallel_for:
9765 cast<OMPTargetTeamsDistributeParallelForDirective>(E));
9767 case OMPD_target_teams_distribute_parallel_for_simd:
9771 cast<OMPTargetTeamsDistributeParallelForSimdDirective>(E));
9773 case OMPD_target_teams_loop:
9775 CGM, ParentName, cast<OMPTargetTeamsGenericLoopDirective>(E));
9777 case OMPD_target_parallel_loop:
9779 CGM, ParentName, cast<OMPTargetParallelGenericLoopDirective>(E));
9783 case OMPD_parallel_for:
9784 case OMPD_parallel_master:
9785 case OMPD_parallel_sections:
9787 case OMPD_parallel_for_simd:
9789 case OMPD_cancellation_point:
9791 case OMPD_threadprivate:
9802 case OMPD_taskyield:
9805 case OMPD_taskgroup:
9811 case OMPD_target_data:
9812 case OMPD_target_exit_data:
9813 case OMPD_target_enter_data:
9814 case OMPD_distribute:
9815 case OMPD_distribute_simd:
9816 case OMPD_distribute_parallel_for:
9817 case OMPD_distribute_parallel_for_simd:
9818 case OMPD_teams_distribute:
9819 case OMPD_teams_distribute_simd:
9820 case OMPD_teams_distribute_parallel_for:
9821 case OMPD_teams_distribute_parallel_for_simd:
9822 case OMPD_target_update:
9823 case OMPD_declare_simd:
9824 case OMPD_declare_variant:
9825 case OMPD_begin_declare_variant:
9826 case OMPD_end_declare_variant:
9827 case OMPD_declare_target:
9828 case OMPD_end_declare_target:
9829 case OMPD_declare_reduction:
9830 case OMPD_declare_mapper:
9832 case OMPD_taskloop_simd:
9833 case OMPD_master_taskloop:
9834 case OMPD_master_taskloop_simd:
9835 case OMPD_parallel_master_taskloop:
9836 case OMPD_parallel_master_taskloop_simd:
9838 case OMPD_metadirective:
9841 llvm_unreachable(
"Unknown target directive for OpenMP device codegen.");
9846 if (
const auto *E = dyn_cast<OMPExecutableDirective>(S)) {
9847 if (!E->hasAssociatedStmt() || !E->getAssociatedStmt())
9855 if (
const auto *L = dyn_cast<LambdaExpr>(S))
9859 for (
const Stmt *II : S->children())
9864 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
9865 OMPDeclareTargetDeclAttr::getDeviceType(VD);
9869 if (!IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_NoHost)
9872 if (IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_Host)
9881 if (
const auto *FD = dyn_cast<FunctionDecl>(GD.
getDecl()))
9890 if (
const auto *FD = dyn_cast<FunctionDecl>(VD)) {
9899 return !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD) &&
9917 StringRef ParentName =
9922 StringRef ParentName =
9929 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
9930 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(
9932 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
9933 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
9934 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
9943 llvm::Constant *Addr) {
9948 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
9949 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
9953 if (Res && *Res != OMPDeclareTargetDeclAttr::MT_Link &&
9968 auto LinkageForVariable = [&VD,
this]() {
9972 std::vector<llvm::GlobalVariable *> GeneratedRefs;
9985 for (
auto *ref : GeneratedRefs)
9990 if (isa<FunctionDecl>(GD.
getDecl()) ||
9991 isa<OMPDeclareReductionDecl>(GD.
getDecl()))
9999 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
10000 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
10003 if ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
10004 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
10008 assert((*Res == OMPDeclareTargetDeclAttr::MT_Link ||
10009 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
10010 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
10012 "Expected link clause or to clause with unified memory.");
10021 " Expected target-based directive.");
10026 if (Clause->getClauseKind() == OMPC_unified_shared_memory) {
10028 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(
true);
10029 }
else if (
const auto *AC =
10030 dyn_cast<OMPAtomicDefaultMemOrderClause>(Clause)) {
10031 switch (AC->getAtomicDefaultMemOrderKind()) {
10032 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_acq_rel:
10035 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_seq_cst:
10038 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_relaxed:
10054 if (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())
10056 const auto *A = VD->
getAttr<OMPAllocateDeclAttr>();
10057 switch(A->getAllocatorType()) {
10058 case OMPAllocateDeclAttr::OMPNullMemAlloc:
10059 case OMPAllocateDeclAttr::OMPDefaultMemAlloc:
10061 case OMPAllocateDeclAttr::OMPLargeCapMemAlloc:
10062 case OMPAllocateDeclAttr::OMPCGroupMemAlloc:
10063 case OMPAllocateDeclAttr::OMPHighBWMemAlloc:
10064 case OMPAllocateDeclAttr::OMPLowLatMemAlloc:
10065 case OMPAllocateDeclAttr::OMPThreadMemAlloc:
10066 case OMPAllocateDeclAttr::OMPConstMemAlloc:
10067 case OMPAllocateDeclAttr::OMPPTeamMemAlloc:
10070 case OMPAllocateDeclAttr::OMPUserDefinedMemAlloc:
10071 llvm_unreachable(
"Expected predefined allocator for the variables with the "
10072 "static storage.");
10099 const auto *D = cast<FunctionDecl>(GD.
getDecl());
10102 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(D)) {
10104 if (
auto *F = dyn_cast_or_null<llvm::Function>(
10106 return !F->isDeclaration();
10118 llvm::Function *OutlinedFn,
10127 llvm::Value *Args[] = {
10129 CGF.
Builder.getInt32(CapturedVars.size()),
10132 RealArgs.append(std::begin(Args), std::end(Args));
10133 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
10135 llvm::FunctionCallee RTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
10141 const Expr *NumTeams,
10142 const Expr *ThreadLimit,
10149 llvm::Value *NumTeamsVal =
10155 llvm::Value *ThreadLimitVal =
10162 llvm::Value *PushNumTeamsArgs[] = {RTLoc,
getThreadID(CGF, Loc), NumTeamsVal,
10170 const Expr *ThreadLimit,
10173 llvm::Value *ThreadLimitVal =
10180 llvm::Value *ThreadLimitArgs[] = {RTLoc,
getThreadID(CGF, Loc),
10198 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
10200 llvm::Value *IfCondVal =
nullptr;
10205 llvm::Value *DeviceID =
nullptr;
10210 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
10214 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
10215 auto GenMapInfoCB =
10216 [&](InsertPointTy CodeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {
10217 CGF.
Builder.restoreIP(CodeGenIP);
10219 MappableExprsHandler MEHandler(D, CGF);
10220 MEHandler.generateAllInfo(CombinedInfo,
OMPBuilder);
10222 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
10226 llvm::codegenoptions::NoDebugInfo) {
10227 CombinedInfo.Names.resize(CombinedInfo.Exprs.size());
10228 llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),
10232 return CombinedInfo;
10234 using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;
10235 auto BodyCB = [&](InsertPointTy CodeGenIP, BodyGenTy BodyGenType) {
10236 CGF.
Builder.restoreIP(CodeGenIP);
10237 switch (BodyGenType) {
10238 case BodyGenTy::Priv:
10242 case BodyGenTy::DupNoPriv:
10248 case BodyGenTy::NoPriv:
10255 return InsertPointTy(CGF.
Builder.GetInsertBlock(),
10256 CGF.
Builder.GetInsertPoint());
10259 auto DeviceAddrCB = [&](
unsigned int I, llvm::Value *NewDecl) {
10260 if (
const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
10265 auto CustomMapperCB = [&](
unsigned int I) {
10266 llvm::Value *MFunc =
nullptr;
10267 if (CombinedInfo.Mappers[I]) {
10268 Info.HasMapper =
true;
10270 cast<OMPDeclareMapperDecl>(CombinedInfo.Mappers[I]));
10280 InsertPointTy CodeGenIP(CGF.
Builder.GetInsertBlock(),
10281 CGF.
Builder.GetInsertPoint());
10282 llvm::OpenMPIRBuilder::LocationDescription OmpLoc(CodeGenIP);
10284 OmpLoc, AllocaIP, CodeGenIP, DeviceID, IfCondVal, Info, GenMapInfoCB,
10285 nullptr, BodyCB, DeviceAddrCB, CustomMapperCB, RTLoc));
10294 assert((isa<OMPTargetEnterDataDirective>(D) ||
10295 isa<OMPTargetExitDataDirective>(D) ||
10296 isa<OMPTargetUpdateDirective>(D)) &&
10297 "Expecting either target enter, exit data, or update directives.");
10300 llvm::Value *MapTypesArray =
nullptr;
10301 llvm::Value *MapNamesArray =
nullptr;
10303 auto &&ThenGen = [
this, &D,
Device, &InputInfo, &MapTypesArray,
10306 llvm::Value *DeviceID =
nullptr;
10311 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
10315 llvm::Constant *PointerNum =
10321 llvm::Value *OffloadingArgs[] = {
10335 RuntimeFunction RTLFn;
10337 case OMPD_target_enter_data:
10338 RTLFn = HasNowait ? OMPRTL___tgt_target_data_begin_nowait_mapper
10339 : OMPRTL___tgt_target_data_begin_mapper;
10341 case OMPD_target_exit_data:
10342 RTLFn = HasNowait ? OMPRTL___tgt_target_data_end_nowait_mapper
10343 : OMPRTL___tgt_target_data_end_mapper;
10345 case OMPD_target_update:
10346 RTLFn = HasNowait ? OMPRTL___tgt_target_data_update_nowait_mapper
10347 : OMPRTL___tgt_target_data_update_mapper;
10349 case OMPD_parallel:
10351 case OMPD_parallel_for:
10352 case OMPD_parallel_master:
10353 case OMPD_parallel_sections:
10354 case OMPD_for_simd:
10355 case OMPD_parallel_for_simd:
10357 case OMPD_cancellation_point:
10359 case OMPD_threadprivate:
10360 case OMPD_allocate:
10365 case OMPD_sections:
10369 case OMPD_critical:
10370 case OMPD_taskyield:
10372 case OMPD_taskwait:
10373 case OMPD_taskgroup:
10379 case OMPD_target_data:
10380 case OMPD_distribute:
10381 case OMPD_distribute_simd:
10382 case OMPD_distribute_parallel_for:
10383 case OMPD_distribute_parallel_for_simd:
10384 case OMPD_teams_distribute:
10385 case OMPD_teams_distribute_simd:
10386 case OMPD_teams_distribute_parallel_for:
10387 case OMPD_teams_distribute_parallel_for_simd:
10388 case OMPD_declare_simd:
10389 case OMPD_declare_variant:
10390 case OMPD_begin_declare_variant:
10391 case OMPD_end_declare_variant:
10392 case OMPD_declare_target:
10393 case OMPD_end_declare_target:
10394 case OMPD_declare_reduction:
10395 case OMPD_declare_mapper:
10396 case OMPD_taskloop:
10397 case OMPD_taskloop_simd:
10398 case OMPD_master_taskloop:
10399 case OMPD_master_taskloop_simd:
10400 case OMPD_parallel_master_taskloop:
10401 case OMPD_parallel_master_taskloop_simd:
10403 case OMPD_target_simd:
10404 case OMPD_target_teams_distribute:
10405 case OMPD_target_teams_distribute_simd:
10406 case OMPD_target_teams_distribute_parallel_for:
10407 case OMPD_target_teams_distribute_parallel_for_simd:
10408 case OMPD_target_teams:
10409 case OMPD_target_parallel:
10410 case OMPD_target_parallel_for:
10411 case OMPD_target_parallel_for_simd:
10412 case OMPD_requires:
10413 case OMPD_metadirective:
10416 llvm_unreachable(
"Unexpected standalone target data directive.");
10424 auto &&TargetThenGen = [
this, &ThenGen, &D, &InputInfo, &MapTypesArray,
10428 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
10431 MappableExprsHandler MEHandler(D, CGF);
10432 MEHandler.generateAllInfo(CombinedInfo,
OMPBuilder);
10441 llvm::codegenoptions::NoDebugInfo;
10454 MapTypesArray = Info.RTArgs.MapTypesArray;
10455 MapNamesArray = Info.RTArgs.MapNamesArray;
10456 if (RequiresOuterTask)
10482struct ParamAttrTy {
10483 ParamKindTy Kind =
Vector;
10484 llvm::APSInt StrideOrArg;
10485 llvm::APSInt Alignment;
10486 bool HasVarStride =
false;
10519 unsigned Offset = 0;
10520 if (
const auto *MD = dyn_cast<CXXMethodDecl>(FD)) {
10521 if (ParamAttrs[Offset].Kind ==
Vector)
10522 CDT =
C.getPointerType(
C.getRecordType(MD->
getParent()));
10526 for (
unsigned I = 0, E = FD->
getNumParams(); I < E; ++I) {
10527 if (ParamAttrs[I + Offset].Kind ==
Vector) {
10539 return C.getTypeSize(CDT);
10547 llvm::raw_svector_ostream Out(Buffer);
10548 for (
const auto &ParamAttr : ParamAttrs) {
10549 switch (ParamAttr.Kind) {
10569 if (ParamAttr.HasVarStride)
10570 Out <<
"s" << ParamAttr.StrideOrArg;
10571 else if (ParamAttr.Kind == Linear || ParamAttr.Kind == LinearRef ||
10572 ParamAttr.Kind == LinearUVal || ParamAttr.Kind == LinearVal) {
10575 if (ParamAttr.StrideOrArg < 0)
10576 Out <<
'n' << -ParamAttr.StrideOrArg;
10577 else if (ParamAttr.StrideOrArg != 1)
10578 Out << ParamAttr.StrideOrArg;
10581 if (!!ParamAttr.Alignment)
10582 Out <<
'a' << ParamAttr.Alignment;
10585 return std::string(Out.str());
10590 const llvm::APSInt &VLENVal,
10592 OMPDeclareSimdDeclAttr::BranchStateTy State) {
10595 unsigned VecRegSize;
10597 ISADataTy ISAData[] = {
10613 case OMPDeclareSimdDeclAttr::BS_Undefined:
10614 Masked.push_back(
'N');
10615 Masked.push_back(
'M');
10617 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
10618 Masked.push_back(
'N');
10620 case OMPDeclareSimdDeclAttr::BS_Inbranch:
10621 Masked.push_back(
'M');
10624 for (
char Mask : Masked) {
10625 for (
const ISADataTy &
Data : ISAData) {
10627 llvm::raw_svector_ostream Out(Buffer);
10628 Out <<
"_ZGV" <<
Data.ISA << Mask;
10631 assert(NumElts &&
"Non-zero simdlen/cdtsize expected");
10632 Out << llvm::APSInt::getUnsigned(
Data.VecRegSize / NumElts);
10637 Out <<
'_' << Fn->getName();
10638 Fn->addFnAttr(Out.str());
10656 if (Kind == ParamKindTy::Uniform)
10659 if (Kind == ParamKindTy::LinearUVal || Kind == ParamKindTy::LinearRef)
10662 if ((Kind == ParamKindTy::Linear || Kind == ParamKindTy::LinearVal) &&
10672 unsigned Size =
C.getTypeSize(QT);
10675 if (Size != 8 && Size != 16 && Size != 32 && Size != 64 && Size != 128)
10699 return C.getTypeSize(PTy);
10702 return C.getTypeSize(QT);
10704 return C.getTypeSize(
C.getUIntPtrType());
10710static std::tuple<unsigned, unsigned, bool>
10716 bool OutputBecomesInput =
false;
10720 Sizes.push_back(
getAArch64LS(RetType, ParamKindTy::Vector,
C));
10722 OutputBecomesInput =
true;
10724 for (
unsigned I = 0, E = FD->
getNumParams(); I < E; ++I) {
10729 assert(!Sizes.empty() &&
"Unable to determine NDS and WDS.");
10732 assert(llvm::all_of(Sizes,
10733 [](
unsigned Size) {
10734 return Size == 8 || Size == 16 || Size == 32 ||
10735 Size == 64 || Size == 128;
10739 return std::make_tuple(*std::min_element(std::begin(Sizes), std::end(Sizes)),
10740 *std::max_element(std::begin(Sizes), std::end(Sizes)),
10741 OutputBecomesInput);
10747template <
typename T>
10749 char ISA, StringRef ParSeq,
10750 StringRef MangledName,
bool OutputBecomesInput,
10751 llvm::Function *Fn) {
10753 llvm::raw_svector_ostream Out(Buffer);
10754 Out << Prefix << ISA << LMask << VLEN;
10755 if (OutputBecomesInput)
10757 Out << ParSeq <<
"_" << MangledName;
10758 Fn->addFnAttr(Out.str());
10764 StringRef Prefix,
char ISA,
10765 StringRef ParSeq, StringRef MangledName,
10766 bool OutputBecomesInput,
10767 llvm::Function *Fn) {
10771 OutputBecomesInput, Fn);
10773 OutputBecomesInput, Fn);
10777 OutputBecomesInput, Fn);
10779 OutputBecomesInput, Fn);
10783 OutputBecomesInput, Fn);
10785 OutputBecomesInput, Fn);
10790 OutputBecomesInput, Fn);
10793 llvm_unreachable(
"Scalar type is too wide.");
10801 OMPDeclareSimdDeclAttr::BranchStateTy State, StringRef MangledName,
10802 char ISA,
unsigned VecRegSize, llvm::Function *Fn,
SourceLocation SLoc) {
10806 const unsigned NDS = std::get<0>(
Data);
10807 const unsigned WDS = std::get<1>(
Data);
10808 const bool OutputBecomesInput = std::get<2>(
Data);
10812 if (UserVLEN == 1) {
10815 "The clause simdlen(1) has no effect when targeting aarch64.");
10822 if (ISA ==
'n' && UserVLEN && !llvm::isPowerOf2_32(UserVLEN)) {
10825 "power of 2 when targeting Advanced SIMD.");
10832 if (ISA ==
's' && UserVLEN != 0) {
10833 if ((UserVLEN * WDS > 2048) || (UserVLEN * WDS % 128 != 0)) {
10836 "lanes in the architectural constraints "
10837 "for SVE (min is 128-bit, max is "
10838 "2048-bit, by steps of 128-bit)");
10846 StringRef Prefix =
"_ZGV";
10852 OutputBecomesInput, Fn);
10854 assert(ISA ==
'n' &&
"Expected ISA either 's' or 'n'.");
10858 case OMPDeclareSimdDeclAttr::BS_Undefined:
10860 OutputBecomesInput, Fn);
10862 OutputBecomesInput, Fn);
10864 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
10866 OutputBecomesInput, Fn);
10868 case OMPDeclareSimdDeclAttr::BS_Inbranch:
10870 OutputBecomesInput, Fn);
10880 OutputBecomesInput, Fn);
10882 assert(ISA ==
'n' &&
"Expected ISA either 's' or 'n'.");
10887 case OMPDeclareSimdDeclAttr::BS_Undefined:
10889 OutputBecomesInput, Fn);
10891 OutputBecomesInput, Fn);
10893 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
10895 OutputBecomesInput, Fn);
10897 case OMPDeclareSimdDeclAttr::BS_Inbranch:
10899 OutputBecomesInput, Fn);
10907 llvm::Function *Fn) {
10912 llvm::DenseMap<const Decl *, unsigned> ParamPositions;
10913 if (isa<CXXMethodDecl>(FD))
10914 ParamPositions.try_emplace(FD, 0);
10915 unsigned ParamPos = ParamPositions.size();
10917 ParamPositions.try_emplace(
P->getCanonicalDecl(), ParamPos);
10923 for (
const Expr *E :
Attr->uniforms()) {
10926 if (isa<CXXThisExpr>(E)) {
10927 Pos = ParamPositions[FD];
10929 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
10930 ->getCanonicalDecl();
10931 auto It = ParamPositions.find(PVD);
10932 assert(It != ParamPositions.end() &&
"Function parameter not found");
10935 ParamAttrs[Pos].Kind = Uniform;
10938 auto *NI =
Attr->alignments_begin();
10939 for (
const Expr *E :
Attr->aligneds()) {
10943 if (isa<CXXThisExpr>(E)) {
10944 Pos = ParamPositions[FD];
10947 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
10948 ->getCanonicalDecl();
10949 auto It = ParamPositions.find(PVD);
10950 assert(It != ParamPositions.end() &&
"Function parameter not found");
10952 ParmTy = PVD->getType();
10954 ParamAttrs[Pos].Alignment =
10956 ? (*NI)->EvaluateKnownConstInt(
C)
10957 : llvm::APSInt::getUnsigned(
10958 C.toCharUnitsFromBits(
C.getOpenMPDefaultSimdAlign(ParmTy))
10963 auto *SI =
Attr->steps_begin();
10964 auto *MI =
Attr->modifiers_begin();
10965 for (
const Expr *E :
Attr->linears()) {
10968 bool IsReferenceType =
false;
10971 unsigned PtrRescalingFactor = 1;
10972 if (isa<CXXThisExpr>(E)) {
10973 Pos = ParamPositions[FD];
10974 auto *
P = cast<PointerType>(E->
getType());
10979 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
10980 ->getCanonicalDecl();
10981 auto It = ParamPositions.find(PVD);
10982 assert(It != ParamPositions.end() &&
"Function parameter not found");
10984 if (
auto *
P = dyn_cast<PointerType>(PVD->getType()))
10988 else if (PVD->getType()->isReferenceType()) {
10989 IsReferenceType =
true;
10990 PtrRescalingFactor =
10996 ParamAttrTy &ParamAttr = ParamAttrs[Pos];
10997 if (*MI == OMPC_LINEAR_ref)
10998 ParamAttr.Kind = LinearRef;
10999 else if (*MI == OMPC_LINEAR_uval)
11000 ParamAttr.Kind = LinearUVal;
11001 else if (IsReferenceType)
11002 ParamAttr.Kind = LinearVal;
11004 ParamAttr.Kind = Linear;
11006 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(1);
11010 if (
const auto *DRE =
11011 cast<DeclRefExpr>((*SI)->IgnoreParenImpCasts())) {
11012 if (
const auto *StridePVD =
11013 dyn_cast<ParmVarDecl>(DRE->getDecl())) {
11014 ParamAttr.HasVarStride =
true;
11015 auto It = ParamPositions.find(StridePVD->getCanonicalDecl());
11016 assert(It != ParamPositions.end() &&
11017 "Function parameter not found");
11018 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(It->second);
11022 ParamAttr.StrideOrArg =
Result.Val.getInt();
11028 if (!ParamAttr.HasVarStride &&
11029 (ParamAttr.Kind == Linear || ParamAttr.Kind == LinearRef))
11030 ParamAttr.StrideOrArg = ParamAttr.StrideOrArg * PtrRescalingFactor;
11034 llvm::APSInt VLENVal;
11036 const Expr *VLENExpr =
Attr->getSimdlen();
11041 OMPDeclareSimdDeclAttr::BranchStateTy State =
Attr->getBranchState();
11044 }
else if (
CGM.
getTriple().getArch() == llvm::Triple::aarch64) {
11045 unsigned VLEN = VLENVal.getExtValue();
11046 StringRef MangledName = Fn->getName();
11049 MangledName,
's', 128, Fn, ExprLoc);
11052 MangledName,
'n', 128, Fn, ExprLoc);
11063 static const int DoacrossFinArgs = 2;
11066 llvm::FunctionCallee RTLFn;
11067 llvm::Value *Args[DoacrossFinArgs];
11070 DoacrossCleanupTy(llvm::FunctionCallee RTLFn,
11073 assert(CallArgs.size() == DoacrossFinArgs);
11074 std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args));
11091 QualType Int64Ty =
C.getIntTypeForBitwidth(64,
true);
11099 RD =
C.buildImplicitRecord(
"kmp_dim");
11109 llvm::APInt Size(32, NumIterations.size());
11115 enum { LowerFD = 0, UpperFD, StrideFD };
11117 for (
unsigned I = 0, E = NumIterations.size(); I < E; ++I) {
11122 DimsLVal, *std::next(RD->
field_begin(), UpperFD));
11124 CGF.
EmitScalarExpr(NumIterations[I]), NumIterations[I]->getType(),
11125 Int64Ty, NumIterations[I]->getExprLoc());
11129 DimsLVal, *std::next(RD->
field_begin(), StrideFD));
11136 llvm::Value *Args[] = {
11139 llvm::ConstantInt::getSigned(
CGM.
Int32Ty, NumIterations.size()),
11144 llvm::FunctionCallee RTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
11147 llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = {
11149 llvm::FunctionCallee FiniRTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
11155template <
typename T>
11157 const T *
C, llvm::Value *ULoc,
11158 llvm::Value *ThreadID) {
11161 llvm::APInt Size(32,
C->getNumLoops());
11165 for (
unsigned I = 0, E =
C->getNumLoops(); I < E; ++I) {
11166 const Expr *CounterVal =
C->getLoopData(I);
11167 assert(CounterVal);
11174 llvm::Value *Args[] = {
11177 llvm::FunctionCallee RTLFn;
11179 OMPDoacrossKind<T> ODK;
11180 if (ODK.isSource(
C)) {
11182 OMPRTL___kmpc_doacross_post);
11184 assert(ODK.isSink(
C) &&
"Expect sink modifier.");
11186 OMPRTL___kmpc_doacross_wait);
11193 return EmitDoacrossOrdered<OMPDependClause>(
11200 return EmitDoacrossOrdered<OMPDoacrossClause>(
11206 llvm::FunctionCallee Callee,
11208 assert(Loc.
isValid() &&
"Outlined function call location must be valid.");
11211 if (
auto *Fn = dyn_cast<llvm::Function>(Callee.getCallee())) {
11212 if (Fn->doesNotThrow()) {
11223 emitCall(CGF, Loc, OutlinedFn, Args);
11227 if (
const auto *FD = dyn_cast<FunctionDecl>(D))
11228 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(FD))
11234 const VarDecl *TargetParam)
const {
11241 const Expr *Allocator) {
11242 llvm::Value *AllocVal;
11249 Allocator->getExprLoc());
11252 AllocVal = llvm::Constant::getNullValue(
11262 if (!AllocateAlignment)
11265 return llvm::ConstantInt::get(
CGM.
SizeTy, AllocateAlignment->getQuantity());
11278 auto I = UntiedData.find(VD);
11279 if (I != UntiedData.end()) {
11280 UntiedAddr = I->second.first;
11281 UntiedRealAddr = I->second.second;
11285 if (CVD->
hasAttr<OMPAllocateDeclAttr>()) {
11294 Size = CGF.
Builder.CreateNUWAdd(
11303 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
11304 const Expr *Allocator = AA->getAllocator();
11308 Args.push_back(ThreadID);
11310 Args.push_back(Alignment);
11311 Args.push_back(Size);
11312 Args.push_back(AllocVal);
11313 llvm::omp::RuntimeFunction FnID =
11314 Alignment ? OMPRTL___kmpc_aligned_alloc : OMPRTL___kmpc_alloc;
11318 llvm::FunctionCallee FiniRTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
11328 llvm::FunctionCallee RTLFn;
11331 const Expr *AllocExpr;
11334 OMPAllocateCleanupTy(llvm::FunctionCallee RTLFn,
11336 const Expr *AllocExpr)
11337 : RTLFn(RTLFn), LocEncoding(LocEncoding), Addr(Addr),
11338 AllocExpr(AllocExpr) {}
11342 llvm::Value *Args[3];
11348 Args[2] = AllocVal;
11356 CGF.
EHStack.pushCleanup<OMPAllocateCleanupTy>(
11358 VDAddr, Allocator);
11359 if (UntiedRealAddr.
isValid())
11362 Region->emitUntiedSwitch(CGF);
11379 assert(CGM.
getLangOpts().OpenMP &&
"Not in OpenMP mode.");
11385 for (
const Stmt *Ref :
C->private_refs()) {
11386 const auto *SimpleRefExpr = cast<Expr>(Ref)->IgnoreParenImpCasts();
11388 if (
const auto *DRE = dyn_cast<DeclRefExpr>(SimpleRefExpr)) {
11389 VD = DRE->getDecl();
11391 const auto *ME = cast<MemberExpr>(SimpleRefExpr);
11392 assert((ME->isImplicitCXXThis() ||
11393 isa<CXXThisExpr>(ME->getBase()->IgnoreParenImpCasts())) &&
11394 "Expected member of current class.");
11395 VD = ME->getMemberDecl();
11411 std::pair<Address, Address>> &LocalVars)
11412 :
CGM(CGF.
CGM), NeedToPush(!LocalVars.empty()) {
11429 return llvm::any_of(
11434void CGOpenMPRuntime::LastprivateConditionalRAII::tryToDisableInnerAnalysis(
11444 const CapturedStmt *CS = S.getCapturedStmt(CaptureRegions.front());
11452 for (
const Expr *Ref :
C->varlists()) {
11453 if (!Ref->getType()->isScalarType())
11455 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
11458 NeedToCheckForLPCs.insert(DRE->getDecl());
11462 for (
const Expr *Ref :
C->varlists()) {
11463 if (!Ref->getType()->isScalarType())
11465 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
11468 NeedToCheckForLPCs.insert(DRE->getDecl());
11472 for (
const Expr *Ref :
C->varlists()) {
11473 if (!Ref->getType()->isScalarType())
11475 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
11478 NeedToCheckForLPCs.insert(DRE->getDecl());
11482 for (
const Expr *Ref :
C->varlists()) {
11483 if (!Ref->getType()->isScalarType())
11485 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
11488 NeedToCheckForLPCs.insert(DRE->getDecl());
11492 for (
const Expr *Ref :
C->varlists()) {
11493 if (!Ref->getType()->isScalarType())
11495 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
11498 NeedToCheckForLPCs.insert(DRE->getDecl());
11501 for (
const Decl *VD : NeedToCheckForLPCs) {
11502 for (
const LastprivateConditionalData &
Data :
11504 if (
Data.DeclToUniqueName.count(VD) > 0) {
11505 if (!
Data.Disabled)
11506 NeedToAddForLPCsAsDisabled.insert(VD);
11513CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
11516 Action((
CGM.getLangOpts().OpenMP >= 50 &&
11519 return C->getKind() ==
11520 OMPC_LASTPRIVATE_conditional;
11522 ? ActionToDo::PushAsLastprivateConditional
11523 : ActionToDo::DoNotPush) {
11525 if (
CGM.
getLangOpts().OpenMP < 50 || Action == ActionToDo::DoNotPush)
11527 assert(Action == ActionToDo::PushAsLastprivateConditional &&
11528 "Expected a push action.");
11532 if (
C->getKind() != OMPC_LASTPRIVATE_conditional)
11535 for (
const Expr *Ref :
C->varlists()) {
11536 Data.DeclToUniqueName.insert(std::make_pair(
11537 cast<DeclRefExpr>(Ref->IgnoreParenImpCasts())->getDecl(),
11541 Data.IVLVal = IVLVal;
11545CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
11547 :
CGM(CGF.
CGM), Action(ActionToDo::DoNotPush) {
11548 assert(CGM.
getLangOpts().OpenMP &&
"Not in OpenMP mode.");
11552 tryToDisableInnerAnalysis(S, NeedToAddForLPCsAsDisabled);
11553 if (!NeedToAddForLPCsAsDisabled.empty()) {
11554 Action = ActionToDo::DisableLastprivateConditional;
11555 LastprivateConditionalData &
Data =
11557 for (
const Decl *VD : NeedToAddForLPCsAsDisabled)
11560 Data.Disabled =
true;
11573 if (Action == ActionToDo::DisableLastprivateConditional) {
11575 "Expected list of disabled private vars.");
11578 if (Action == ActionToDo::PushAsLastprivateConditional) {
11581 "Expected list of lastprivate conditional vars.");
11596 auto VI = I->getSecond().find(VD);
11597 if (VI == I->getSecond().end()) {
11598 RecordDecl *RD =
C.buildImplicitRecord(
"lasprivate.conditional");
11603 NewType =
C.getRecordType(RD);
11606 I->getSecond().try_emplace(VD, NewType, VDField, FiredField, BaseLVal);
11608 NewType = std::get<0>(VI->getSecond());
11609 VDField = std::get<1>(VI->getSecond());
11610 FiredField = std::get<2>(VI->getSecond());
11611 BaseLVal = std::get<3>(VI->getSecond());
11623class LastprivateConditionalRefChecker final
11626 const Expr *FoundE =
nullptr;
11627 const Decl *FoundD =
nullptr;
11628 StringRef UniqueDeclName;
11630 llvm::Function *FoundFn =
nullptr;
11636 llvm::reverse(LPM)) {
11637 auto It = D.DeclToUniqueName.find(E->
getDecl());
11638 if (It == D.DeclToUniqueName.end())
11644 UniqueDeclName = It->second;
11649 return FoundE == E;
11652 if (!CodeGenFunction::IsWrappedCXXThis(E->
getBase()))
11655 llvm::reverse(LPM)) {
11657 if (It == D.DeclToUniqueName.end())
11663 UniqueDeclName = It->second;
11668 return FoundE == E;
11670 bool VisitStmt(
const Stmt *S) {
11671 for (
const Stmt *Child : S->children()) {
11674 if (
const auto *E = dyn_cast<Expr>(Child))
11682 explicit LastprivateConditionalRefChecker(
11685 std::tuple<const Expr *, const Decl *, StringRef, LValue, llvm::Function *>
11686 getFoundData()
const {
11687 return std::make_tuple(FoundE, FoundD, UniqueDeclName, IVLVal, FoundFn);
11694 StringRef UniqueDeclName,
11700 llvm::Constant *LastIV =
OMPBuilder.getOrCreateInternalVariable(
11701 LLIVTy,
getName({UniqueDeclName,
"iv"}));
11702 cast<llvm::GlobalVariable>(LastIV)->setAlignment(
11709 llvm::GlobalVariable *
Last =
OMPBuilder.getOrCreateInternalVariable(
11711 cast<llvm::GlobalVariable>(
Last)->setAlignment(
11725 auto &&CodeGen = [&LastIVLVal, &IVLVal, IVVal, &LVal, &LastLVal,
11731 llvm::Value *CmpRes;
11733 CmpRes = CGF.
Builder.CreateICmpSLE(LastIVVal, IVVal);
11736 "Loop iteration variable must be integer.");
11737 CmpRes = CGF.
Builder.CreateICmpULE(LastIVVal, IVVal);
11741 CGF.
Builder.CreateCondBr(CmpRes, ThenBB, ExitBB);
11762 "Aggregates are not supported in lastprivate conditional.");
11785 if (!Checker.Visit(LHS))
11787 const Expr *FoundE;
11788 const Decl *FoundD;
11789 StringRef UniqueDeclName;
11791 llvm::Function *FoundFn;
11792 std::tie(FoundE, FoundD, UniqueDeclName, IVLVal, FoundFn) =
11793 Checker.getFoundData();
11794 if (FoundFn != CGF.
CurFn) {
11799 "Lastprivate conditional is not found in outer region.");
11800 QualType StructTy = std::get<0>(It->getSecond());
11801 const FieldDecl* FiredDecl = std::get<2>(It->getSecond());
11812 FiredLVal, llvm::AtomicOrdering::Unordered,
11830 auto It = llvm::find_if(
11832 if (It == Range.end() || It->Fn != CGF.
CurFn)
11836 "Lastprivates must be registered already.");
11840 for (
const auto &Pair : It->DeclToUniqueName) {
11841 const auto *VD = cast<VarDecl>(Pair.first->getCanonicalDecl());
11844 auto I = LPCI->getSecond().find(Pair.first);
11845 assert(I != LPCI->getSecond().end() &&
11846 "Lastprivate must be rehistered already.");
11848 LValue BaseLVal = std::get<3>(I->getSecond());
11852 llvm::Value *Cmp = CGF.
Builder.CreateIsNotNull(Res);
11856 CGF.
Builder.CreateCondBr(Cmp, ThenBB, DoneBB);
11881 "Unknown lastprivate conditional variable.");
11882 StringRef UniqueName = It->second;
11883 llvm::GlobalVariable *GV =
CGM.
getModule().getNamedGlobal(UniqueName);
11897 llvm_unreachable(
"Not supported in SIMD-only mode");
11904 llvm_unreachable(
"Not supported in SIMD-only mode");
11911 bool Tied,
unsigned &NumberOfParts) {
11912 llvm_unreachable(
"Not supported in SIMD-only mode");
11917 llvm::Function *OutlinedFn,
11919 const Expr *IfCond,
11920 llvm::Value *NumThreads) {
11921 llvm_unreachable(
"Not supported in SIMD-only mode");
11927 const Expr *Hint) {
11928 llvm_unreachable(
"Not supported in SIMD-only mode");
11934 llvm_unreachable(
"Not supported in SIMD-only mode");
11940 const Expr *Filter) {
11941 llvm_unreachable(
"Not supported in SIMD-only mode");
11946 llvm_unreachable(
"Not supported in SIMD-only mode");
11952 llvm_unreachable(
"Not supported in SIMD-only mode");
11960 llvm_unreachable(
"Not supported in SIMD-only mode");
11967 llvm_unreachable(
"Not supported in SIMD-only mode");
11974 bool ForceSimpleCall) {
11975 llvm_unreachable(
"Not supported in SIMD-only mode");
11982 llvm_unreachable(
"Not supported in SIMD-only mode");
11988 llvm_unreachable(
"Not supported in SIMD-only mode");
11994 llvm_unreachable(
"Not supported in SIMD-only mode");
12001 llvm_unreachable(
"Not supported in SIMD-only mode");
12007 llvm_unreachable(
"Not supported in SIMD-only mode");
12012 unsigned IVSize,
bool IVSigned,
12015 llvm_unreachable(
"Not supported in SIMD-only mode");
12019 llvm::Value *NumThreads,
12021 llvm_unreachable(
"Not supported in SIMD-only mode");
12025 ProcBindKind ProcBind,
12027 llvm_unreachable(
"Not supported in SIMD-only mode");
12034 llvm_unreachable(
"Not supported in SIMD-only mode");
12040 llvm_unreachable(
"Not supported in SIMD-only mode");
12045 llvm_unreachable(
"Not supported in SIMD-only mode");
12051 llvm::AtomicOrdering AO) {
12052 llvm_unreachable(
"Not supported in SIMD-only mode");
12057 llvm::Function *TaskFunction,
12059 const Expr *IfCond,
12061 llvm_unreachable(
"Not supported in SIMD-only mode");
12068 llvm_unreachable(
"Not supported in SIMD-only mode");
12075 assert(Options.SimpleReduction &&
"Only simple reduction is expected.");
12077 ReductionOps, Options);
12083 llvm_unreachable(
"Not supported in SIMD-only mode");
12088 bool IsWorksharingReduction) {
12089 llvm_unreachable(
"Not supported in SIMD-only mode");
12096 llvm_unreachable(
"Not supported in SIMD-only mode");
12101 llvm::Value *ReductionsPtr,
12103 llvm_unreachable(
"Not supported in SIMD-only mode");
12109 llvm_unreachable(
"Not supported in SIMD-only mode");
12115 llvm_unreachable(
"Not supported in SIMD-only mode");
12121 llvm_unreachable(
"Not supported in SIMD-only mode");
12126 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
12128 llvm_unreachable(
"Not supported in SIMD-only mode");
12133 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID,
const Expr *IfCond,
12134 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
12138 llvm_unreachable(
"Not supported in SIMD-only mode");
12142 llvm_unreachable(
"Not supported in SIMD-only mode");
12146 llvm_unreachable(
"Not supported in SIMD-only mode");
12156 llvm::Function *OutlinedFn,
12158 llvm_unreachable(
"Not supported in SIMD-only mode");
12162 const Expr *NumTeams,
12163 const Expr *ThreadLimit,
12165 llvm_unreachable(
"Not supported in SIMD-only mode");
12172 llvm_unreachable(
"Not supported in SIMD-only mode");
12178 llvm_unreachable(
"Not supported in SIMD-only mode");
12184 llvm_unreachable(
"Not supported in SIMD-only mode");
12189 llvm_unreachable(
"Not supported in SIMD-only mode");
12194 llvm_unreachable(
"Not supported in SIMD-only mode");
12199 const VarDecl *NativeParam)
const {
12200 llvm_unreachable(
"Not supported in SIMD-only mode");
12206 const VarDecl *TargetParam)
const {
12207 llvm_unreachable(
"Not supported in SIMD-only mode");
Provides LLVM's BitmaskEnum facility to enumeration types declared in namespace clang.
static llvm::Value * emitCopyprivateCopyFunction(CodeGenModule &CGM, llvm::Type *ArgsElemType, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps, SourceLocation Loc)
static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF, SourceLocation Loc, SmallString< 128 > &Buffer)
llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind convertCaptureClause(const VarDecl *VD)
static void emitOffloadingArrays(CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo, CGOpenMPRuntime::TargetDataInfo &Info, llvm::OpenMPIRBuilder &OMPBuilder, bool IsNonContiguous=false)
Emit the arrays used to pass the captures and map information to the offloading runtime library.
static RecordDecl * createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy, ArrayRef< PrivateDataTy > Privates)
llvm::Value * emitDynCGGroupMem(const OMPExecutableDirective &D, CodeGenFunction &CGF)
static void emitInitWithReductionInitializer(CodeGenFunction &CGF, const OMPDeclareReductionDecl *DRD, const Expr *InitOp, Address Private, Address Original, QualType Ty)
static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, Address OriginalBaseAddress, llvm::Value *Addr)
static void emitPrivatesInit(CodeGenFunction &CGF, const OMPExecutableDirective &D, Address KmpTaskSharedsPtr, LValue TDBase, const RecordDecl *KmpTaskTWithPrivatesQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef< PrivateDataTy > Privates, bool ForDup)
Emit initialization for private variables in task-based directives.
static llvm::Value * emitDestructorsFunction(CodeGenModule &CGM, SourceLocation Loc, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy)
static unsigned evaluateCDTSize(const FunctionDecl *FD, ArrayRef< ParamAttrTy > ParamAttrs)
static void EmitOMPAggregateReduction(CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar, const VarDecl *RHSVar, const llvm::function_ref< void(CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *)> &RedOpGen, const Expr *XExpr=nullptr, const Expr *EExpr=nullptr, const Expr *UpExpr=nullptr)
Emit reduction operation for each element of array (required for array sections) LHS op = RHS.
static void emitTargetCallFallback(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, CodeGenFunction &CGF)
static llvm::Value * emitReduceInitFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Emits reduction initializer function:
static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion)
static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy, llvm::PointerUnion< unsigned *, LValue * > Pos, const OMPTaskDataTy::DependData &Data, Address DependenciesArray)
static llvm::Value * emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc, const OMPTaskDataTy &Data, QualType PrivatesQTy, ArrayRef< PrivateDataTy > Privates)
Emit a privates mapping function for correct handling of private and firstprivate variables.
static llvm::Value * emitReduceCombFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N, const Expr *ReductionOp, const Expr *LHS, const Expr *RHS, const Expr *PrivateRef)
Emits reduction combiner function:
static RecordDecl * createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef< PrivateDataTy > Privates)
static llvm::Value * getAllocatorVal(CodeGenFunction &CGF, const Expr *Allocator)
Return allocator value from expression, or return a null allocator (default when no allocator specifi...
static llvm::Function * emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy, QualType SharedsPtrTy, llvm::Function *TaskFunction, llvm::Value *TaskPrivatesMap)
Emit a proxy function which accepts kmp_task_t as the second argument.
static void addAArch64VectorName(T VLEN, StringRef LMask, StringRef Prefix, char ISA, StringRef ParSeq, StringRef MangledName, bool OutputBecomesInput, llvm::Function *Fn)
static bool isAllocatableDecl(const VarDecl *VD)
static llvm::Value * getAlignmentValue(CodeGenModule &CGM, const VarDecl *VD)
Return the alignment from an allocate directive if present.
static void emitTargetCallKernelLaunch(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::Value *OutlinedFnID, CodeGenFunction::OMPTargetDataInfo &InputInfo, llvm::Value *&MapTypesArray, llvm::Value *&MapNamesArray, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter, CodeGenFunction &CGF, CodeGenModule &CGM)
static std::tuple< unsigned, unsigned, bool > getNDSWDS(const FunctionDecl *FD, ArrayRef< ParamAttrTy > ParamAttrs)
static const OMPExecutableDirective * getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D)
Check for inner distribute directive.
static std::pair< llvm::Value *, llvm::Value * > getPointerAndSize(CodeGenFunction &CGF, const Expr *E)
static const VarDecl * getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE)
static bool isTrivial(ASTContext &Ctx, const Expr *E)
Checks if the expression is constant or does not have non-trivial function calls.
static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind, bool Chunked, bool Ordered)
Map the OpenMP loop schedule to the runtime enumeration.
static void getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS, const Expr **E, int32_t &UpperBound, bool UpperBoundOnly, llvm::Value **CondVal)
Check for a num threads constant value (stored in DefaultVal), or expression (stored in E).
static llvm::Value * emitDeviceID(llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, CodeGenFunction &CGF)
static const OMPDeclareReductionDecl * getReductionInit(const Expr *ReductionOp)
Check if the combiner is a call to UDR combiner and if it is so return the UDR decl used for reductio...
static bool checkInitIsRequired(CodeGenFunction &CGF, ArrayRef< PrivateDataTy > Privates)
Check if duplication function is required for taskloops.
static bool checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD, ArrayRef< PrivateDataTy > Privates)
Checks if destructor function is required to be generated.
static llvm::TargetRegionEntryInfo getEntryInfoFromPresumedLoc(CodeGenModule &CGM, llvm::OpenMPIRBuilder &OMPBuilder, SourceLocation BeginLoc, llvm::StringRef ParentName="")
static std::string generateUniqueName(CodeGenModule &CGM, StringRef Prefix, const Expr *Ref)
Generates unique name for artificial threadprivate variables.
static void emitForStaticInitCall(CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId, llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2, const CGOpenMPRuntime::StaticRTInput &Values)
static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, LValue BaseLV)
static void getKmpAffinityType(ASTContext &C, QualType &KmpTaskAffinityInfoTy)
Builds kmp_depend_info, if it is not built yet, and builds flags type.
static void getDependTypes(ASTContext &C, QualType &KmpDependInfoTy, QualType &FlagsTy)
Builds kmp_depend_info, if it is not built yet, and builds flags type.
static llvm::Value * emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc, const OMPExecutableDirective &D, QualType KmpTaskTWithPrivatesPtrQTy, const RecordDecl *KmpTaskTWithPrivatesQTyRD, const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef< PrivateDataTy > Privates, bool WithLastIter)
Emit task_dup function (for initialization of private/firstprivate/lastprivate vars and last_iter fla...
static llvm::Value * emitReduceFiniFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Emits reduction finalizer function:
static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, QualType Type, bool EmitDeclareReductionInit, const Expr *Init, const OMPDeclareReductionDecl *DRD, Address SrcAddr=Address::invalid())
Emit initialization of arrays of complex types.
static bool getAArch64PBV(QualType QT, ASTContext &C)
Pass By Value (PBV), as defined in 3.1.2 of the AAVFABI.
static unsigned getAArch64LS(QualType QT, ParamKindTy Kind, ASTContext &C)
Computes the lane size (LS) of a return type or of an input parameter, as defined by LS(P) in 3....
static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM, const T *C, llvm::Value *ULoc, llvm::Value *ThreadID)
static RTLDependenceKindTy translateDependencyKind(OpenMPDependClauseKind K)
Translates internal dependency kind into the runtime kind.
static void emitTargetCallElse(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, CodeGenFunction &CGF)
static llvm::Function * emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty, const Expr *CombinerInitializer, const VarDecl *In, const VarDecl *Out, bool IsCombiner)
static void emitX86DeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn, const llvm::APSInt &VLENVal, ArrayRef< ParamAttrTy > ParamAttrs, OMPDeclareSimdDeclAttr::BranchStateTy State)
static void emitReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp)
Emit reduction combiner.
static std::string mangleVectorParameters(ArrayRef< ParamAttrTy > ParamAttrs)
Mangle the parameter part of the vector function name according to their OpenMP classification.
static llvm::Function * emitParallelOrTeamsOutlinedFunction(CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen)
static void emitAArch64DeclareSimdFunction(CodeGenModule &CGM, const FunctionDecl *FD, unsigned UserVLEN, ArrayRef< ParamAttrTy > ParamAttrs, OMPDeclareSimdDeclAttr::BranchStateTy State, StringRef MangledName, char ISA, unsigned VecRegSize, llvm::Function *Fn, SourceLocation SLoc)
Emit vector function attributes for AArch64, as defined in the AAVFABI.
static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array, unsigned Index, const VarDecl *Var)
Given an array of pointers to variables, project the address of a given variable.
llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind convertDeviceClause(const VarDecl *VD)
static bool isAssumedToBeNotEmitted(const ValueDecl *VD, bool IsDevice)
static void addAArch64AdvSIMDNDSNames(unsigned NDS, StringRef Mask, StringRef Prefix, char ISA, StringRef ParSeq, StringRef MangledName, bool OutputBecomesInput, llvm::Function *Fn)
static FieldDecl * addFieldToRecordDecl(ASTContext &C, DeclContext *DC, QualType FieldTy)
static ValueDecl * getDeclFromThisExpr(const Expr *E)
llvm::Constant * emitMappingInformation(CodeGenFunction &CGF, llvm::OpenMPIRBuilder &OMPBuilder, MappableExprsHandler::MappingExprInfo &MapExprs)
Emit a string constant containing the names of the values mapped to the offloading runtime library.
static RecordDecl * createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpRoutineEntryPointerQTy)
static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2)
static bool getAArch64MTV(QualType QT, ParamKindTy Kind)
Maps To Vector (MTV), as defined in 4.1.1 of the AAVFABI (2021Q1).
Defines the clang::FileManager interface and associated types.
This file defines OpenMP AST classes for clauses.
Defines some OpenMP-specific enums and functions.
Defines the SourceManager interface.
This file defines OpenMP AST classes for executable directives and clauses.
__DEVICE__ int max(int __a, int __b)
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
SourceManager & getSourceManager()
const ConstantArrayType * getAsConstantArrayType(QualType T) const
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
const ASTRecordLayout & getASTRecordLayout(const RecordDecl *D) const
Get or compute information about the layout of the specified record (struct/union/class) D,...
bool hasSameType(QualType T1, QualType T2) const
Determine whether the given types T1 and T2 are equivalent.
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
const LangOptions & getLangOpts() const
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth,...
CanQualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
const VariableArrayType * getAsVariableArrayType(QualType T) const
const TargetInfo & getTargetInfo() const
CharUnits getNonVirtualSize() const
getNonVirtualSize - Get the non-virtual size (in chars) of an object, which is the size of the object...
static QualType getBaseOriginalType(const Expr *Base)
Return original type of the base expression for array section.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Attr - This represents one attribute.
Represents a C++ constructor within a class.
Represents a C++ destructor within a class.
Represents a static or instance method of a struct/union/class.
const CXXRecordDecl * getParent() const
Return the parent of this method declaration, which is the class in which this method is defined.
QualType getFunctionObjectParameterType() const
Represents a C++ struct/union/class.
bool isLambda() const
Determine whether this class describes a lambda function object.
void getCaptureFields(llvm::DenseMap< const ValueDecl *, FieldDecl * > &Captures, FieldDecl *&ThisCapture) const
For a closure type, retrieve the mapping from captured variables and this to the non-static data memb...
unsigned getNumBases() const
Retrieves the number of base classes of this class.
base_class_range vbases()
capture_const_range captures() const
CXXDestructorDecl * getDestructor() const
Returns the destructor decl for this class.
CanProxy< U > castAs() const
A wrapper class around a pointer that always points to its canonical declaration.
Describes the capture of either a variable, or 'this', or variable-length array type.
bool capturesVariableByCopy() const
Determine whether this capture handles a variable by copy.
VarDecl * getCapturedVar() const
Retrieve the declaration of the variable being captured.
bool capturesVariableArrayType() const
Determine whether this capture handles a variable-length array type.
bool capturesThis() const
Determine whether this capture handles the C++ 'this' pointer.
bool capturesVariable() const
Determine whether this capture handles a variable (by reference).
This captures a statement into a function.
capture_iterator capture_end() const
Retrieve an iterator pointing past the end of the sequence of captures.
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Stmt * getCapturedStmt()
Retrieve the statement being captured.
bool capturesVariable(const VarDecl *Var) const
True if this variable has been captured.
capture_iterator capture_begin()
Retrieve an iterator pointing to the first capture.
CharUnits - This is an opaque type for sizes expressed in character units.
bool isZero() const
isZero - Test whether the quantity equals zero.
llvm::Align getAsAlign() const
getAsAlign - Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit b...
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
CharUnits alignTo(const CharUnits &Align) const
alignTo - Returns the next integer (mod 2**64) that is greater than or equal to this quantity and is ...
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
CharUnits getAlignment() const
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Address withPointer(llvm::Value *NewPointer, KnownNonNull_t IsKnownNonNull) const
Return address with different pointer, but same element type and alignment.
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Address withAlignment(CharUnits NewAlignment) const
Return address with different alignment, but same pointer and element type.
llvm::PointerType * getType() const
Return the type of the pointer value.
static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateDefaultArtificial(CodeGenFunction &CGF, SourceLocation TemporaryLocation)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF)
Set the IRBuilder to not attach debug locations.
CGBlockInfo - Information to generate a block literal.
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
llvm::Value * CreateIsNull(Address Addr, const Twine &Name="")
Address CreateGEP(CodeGenFunction &CGF, Address Addr, llvm::Value *Index, const llvm::Twine &Name="")
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Address CreateConstArrayGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = [n x T]* ... produce name = getelementptr inbounds addr, i64 0, i64 index where i64 is a...
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
Address CreateConstGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = T* ... produce name = getelementptr inbounds addr, i64 index where i64 is actually the t...
MangleContext & getMangleContext()
Gets the mangle context.
CGFunctionInfo - Class to encapsulate the information about a function definition.
DisableAutoDeclareTargetRAII(CodeGenModule &CGM)
~DisableAutoDeclareTargetRAII()
Manages list of lastprivate conditional decls for the specified directive.
~LastprivateConditionalRAII()
static LastprivateConditionalRAII disable(CodeGenFunction &CGF, const OMPExecutableDirective &S)
NontemporalDeclsRAII(CodeGenModule &CGM, const OMPLoopDirective &S)
Struct that keeps all the relevant information that should be kept throughout a 'target data' region.
llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap
Map between the a declaration of a capture and the corresponding new llvm address where the runtime r...
~UntiedTaskLocalDeclsRAII()
UntiedTaskLocalDeclsRAII(CodeGenFunction &CGF, const llvm::MapVector< CanonicalDeclPtr< const VarDecl >, std::pair< Address, Address > > &LocalVars)
virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc)
Emits address of the word in a memory where current thread id is stored.
llvm::FunctionType * Kmpc_MicroTy
The type for a microtask which gets passed to __kmpc_fork_call().
llvm::StringSet ThreadPrivateWithDefinition
Set of threadprivate variables with the generated initializer.
CGOpenMPRuntime(CodeGenModule &CGM)
virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
Emit task region for the task directive.
void createOffloadEntriesAndInfoMetadata()
Creates all the offload entries in the current compilation unit along with the associated metadata.
const Expr * getNumTeamsExprForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &MinTeamsVal, int32_t &MaxTeamsVal)
Emit the number of teams for a target directive.
virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc)
Returns address of the threadprivate variable for the current thread.
void emitDeferredTargetDecls() const
Emit deferred declare target variables marked for deferred emission.
virtual llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST)
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
bool markAsGlobalTarget(GlobalDecl GD)
Marks the declaration as already emitted for the device code and returns true, if it was marked alrea...
virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device)
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
virtual void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc)
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)...
QualType SavedKmpTaskloopTQTy
Saved kmp_task_t for taskloop-based directive.
virtual void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps)
Emits a single region.
virtual bool emitTargetGlobal(GlobalDecl GD)
Emit the global GD if it is meaningful for the target.
void setLocThreadIdInsertPt(CodeGenFunction &CGF, bool AtCurrentPoint=false)
std::string getOutlinedHelperName(StringRef Name) const
Get the function name of an outlined region.
bool HasEmittedDeclareTargetRegion
Flag for keeping track of weather a device routine has been emitted.
llvm::Constant * getOrCreateThreadPrivateCache(const VarDecl *VD)
If the specified mangled name is not in the module, create and return threadprivate cache object.
virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal)
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF, const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const
Choose default schedule type and chunk value for the schedule clause.
virtual std::pair< llvm::Function *, llvm::Function * > getUserDefinedReduction(const OMPDeclareReductionDecl *D)
Get combiner/initializer for the specified user-defined reduction, if any.
virtual bool isGPU() const
Returns true if the current target is a GPU.
static const Stmt * getSingleCompoundChild(ASTContext &Ctx, const Stmt *Body)
Checks if the Body is the CompoundStmt and returns its child statement iff there is only one that is ...
virtual void emitDeclareTargetFunction(const FunctionDecl *FD, llvm::GlobalValue *GV)
Emit code for handling declare target functions in the runtime.
llvm::Type * getKmpc_MicroPointerTy()
Returns pointer to kmpc_micro type.
bool HasRequiresUnifiedSharedMemory
Flag for keeping track of weather a requires unified_shared_memory directive is present.
llvm::Value * emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc, unsigned Flags=0, bool EmitLoc=false)
Emits object of ident_t type with info for source location.
bool isLocalVarInUntiedTask(CodeGenFunction &CGF, const VarDecl *VD) const
Returns true if the variable is a local variable in untied task.
virtual void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars)
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
virtual void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancellation point' construct.
void emitUDMapperArrayInitOrDel(CodeGenFunction &MapperCGF, llvm::Value *Handle, llvm::Value *BasePtr, llvm::Value *Ptr, llvm::Value *Size, llvm::Value *MapType, llvm::Value *MapName, CharUnits ElementSize, llvm::BasicBlock *ExitBB, bool IsInit)
Emit the array initialization or deletion portion for user-defined mapper code generation.
virtual llvm::Function * emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF=nullptr)
Emit a code for initialization of threadprivate variable.
FunctionUDMMapTy FunctionUDMMap
virtual ConstantAddress getAddrOfDeclareTargetVar(const VarDecl *VD)
Returns the address of the variable marked as declare target with link clause OR as declare target wi...
llvm::MapVector< CanonicalDeclPtr< const VarDecl >, std::pair< Address, Address > > UntiedLocalVarsAddressesMap
llvm::Function * getOrCreateUserDefinedMapperFunc(const OMPDeclareMapperDecl *D)
Get the function for the specified user-defined mapper.
OpenMPLocThreadIDMapTy OpenMPLocThreadIDMap
virtual void functionFinished(CodeGenFunction &CGF)
Cleans up references to the objects in finished function.
void emitCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee Callee, ArrayRef< llvm::Value * > Args=std::nullopt) const
Emits Callee function call with arguments Args with location Loc.
virtual llvm::Function * emitTeamsOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP teams directive D.
QualType KmpTaskTQTy
Type typedef struct kmp_task { void * shareds; /**< pointer to block of pointers to shared vars / k...
llvm::OpenMPIRBuilder OMPBuilder
An OpenMP-IR-Builder instance.
virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr * > NumIterations)
Emit initialization for doacross loop nesting support.
virtual void adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF, const OMPExecutableDirective &D) const
Adjust some parameters for the target-based directives, like addresses of the variables captured by r...
FunctionUDRMapTy FunctionUDRMap
virtual void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, CGOpenMPRuntime::TargetDataInfo &Info)
Emit the target data mapping code associated with D.
virtual unsigned getDefaultLocationReserved2Flags() const
Returns additional flags that can be stored in reserved_2 field of the default location.
void computeMinAndMaxThreadsAndTeams(const OMPExecutableDirective &D, CodeGenFunction &CGF, int32_t &MinThreadsVal, int32_t &MaxThreadsVal, int32_t &MinTeamsVal, int32_t &MaxTeamsVal)
Helper to determine the min/max number of threads/teams for D.
virtual Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const
Gets the address of the native argument basing on the address of the target-specific parameter.
void emitUsesAllocatorsFini(CodeGenFunction &CGF, const Expr *Allocator)
Destroys user defined allocators specified in the uses_allocators clause.
QualType KmpTaskAffinityInfoTy
Type typedef struct kmp_task_affinity_info { kmp_intptr_t base_addr; size_t len; struct { bool flag1 ...
llvm::SmallVector< NontemporalDeclsSet, 4 > NontemporalDeclsStack
Stack for list of declarations in current context marked as nontemporal.
llvm::Value * emitNumTeamsForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D)
virtual void emitTargetOutlinedFunctionHelper(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Helper to emit outlined function for 'target' directive.
void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName)
Start scanning from statement S and emit all target regions found along the way.
SmallVector< llvm::Value *, 4 > emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy, const OMPTaskDataTy::DependData &Data)
virtual void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc)
Emit a taskgroup region.
llvm::DenseMap< llvm::Function *, llvm::DenseMap< CanonicalDeclPtr< const Decl >, std::tuple< QualType, const FieldDecl *, const FieldDecl *, LValue > > > LastprivateConditionalToTypes
Maps local variables marked as lastprivate conditional to their internal types.
virtual bool emitTargetGlobalVariable(GlobalDecl GD)
Emit the global variable if it is a valid device global variable.
virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams,...
bool hasRequiresUnifiedSharedMemory() const
Return whether the unified_shared_memory has been specified.
virtual Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, QualType VarType, StringRef Name)
Creates artificial threadprivate variable with name Name and type VarType.
void emitUserDefinedMapper(const OMPDeclareMapperDecl *D, CodeGenFunction *CGF=nullptr)
Emit the function for the user defined mapper construct.
bool HasEmittedTargetRegion
Flag for keeping track of weather a target region has been emitted.
void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy, LValue PosLVal, const OMPTaskDataTy::DependData &Data, Address DependenciesArray)
std::string getReductionFuncName(StringRef Name) const
Get the function name of a reduction function.
virtual void processRequiresDirective(const OMPRequiresDecl *D)
Perform check on requires decl to ensure that target architecture supports unified addressing.
llvm::DenseSet< CanonicalDeclPtr< const Decl > > AlreadyEmittedTargetDecls
List of the emitted declarations.
virtual llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data)
Emit a code for initialization of task reduction clause.
llvm::Value * getThreadID(CodeGenFunction &CGF, SourceLocation Loc)
Gets thread id value for the current thread.
void emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, OpenMPDependClauseKind NewDepKind, SourceLocation Loc)
Updates the dependency kind in the specified depobj object.
virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD, SourceLocation Loc)
Gets the address of the global copy used for lastprivate conditional update, if any.
virtual void emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, Expr *ME, bool IsFatal)
Emit __kmpc_error call for error directive extern void __kmpc_error(ident_t *loc, int severity,...
void clearLocThreadIdInsertPt(CodeGenFunction &CGF)
virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc)
Emits code for a taskyield directive.
std::string getName(ArrayRef< StringRef > Parts) const
Get the platform-specific name separator.
QualType KmpRoutineEntryPtrQTy
virtual void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc, llvm::AtomicOrdering AO)
Emit flush of the variables specified in 'omp flush' directive.
virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPTaskDataTy &Data)
Emit code for 'taskwait' directive.
virtual void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc)
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, int proc_bind) to generat...
void emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LValue IVLVal, StringRef UniqueDeclName, LValue LVal, SourceLocation Loc)
Emit update for lastprivate conditional data.
virtual void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
Emit task region for the taskloop directive.
virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false)
Emit an implicit/explicit barrier for OpenMP threads.
static unsigned getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind)
Returns default flags for the barriers depending on the directive, for which this barier is going to ...
virtual bool emitTargetFunctions(GlobalDecl GD)
Emit the target regions enclosed in GD function definition or the function itself in case it is a val...
TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const OMPTaskDataTy &Data)
Emit task region for the task directive.
llvm::Value * emitTargetNumIterationsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Return the trip count of loops associated with constructs / 'target teams distribute' and 'teams dist...
llvm::StringMap< llvm::AssertingVH< llvm::GlobalVariable >, llvm::BumpPtrAllocator > InternalVars
An ordered map of auto-generated variables to their unique names.
virtual void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values)
llvm::SmallVector< UntiedLocalVarsAddressesMap, 4 > UntiedLocalVarsStack
virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind)
Call the appropriate runtime routine to notify that we finished all the work with current loop.
virtual void emitThreadLimitClause(CodeGenFunction &CGF, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_set_thread_limit(ident_t *loc, kmp_int32 global_tid, kmp_int32 thread_limit...
void emitIfClause(CodeGenFunction &CGF, const Expr *Cond, const RegionCodeGenTy &ThenGen, const RegionCodeGenTy &ElseGen)
Emits code for OpenMP 'if' clause using specified CodeGen function.
Address emitDepobjDependClause(CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies, SourceLocation Loc)
Emits list of dependecies based on the provided data (array of dependence/expression pairs) for depob...
bool isNontemporalDecl(const ValueDecl *VD) const
Checks if the VD variable is marked as nontemporal declaration in current context.
virtual llvm::Function * emitParallelOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP parallel directive D.
const Expr * getNumThreadsExprForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &UpperBound, bool UpperBoundOnly, llvm::Value **CondExpr=nullptr, const Expr **ThreadLimitExpr=nullptr)
Check for a number of threads upper bound constant value (stored in UpperBound), or expression (retur...
llvm::SmallVector< LastprivateConditionalData, 4 > LastprivateConditionalStack
Stack for list of addresses of declarations in current context marked as lastprivate conditional.
virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values)
Call the appropriate runtime routine to initialize it before start of loop.
virtual void emitDeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn)
Marks function Fn with properly mangled versions of vector functions.
llvm::AtomicOrdering getDefaultMemoryOrdering() const
Gets default memory ordering as specified in requires directive.
llvm::SmallDenseSet< CanonicalDeclPtr< const Decl > > NontemporalDeclsSet
virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static non-chunked.
llvm::Value * getCriticalRegionLock(StringRef CriticalName)
Returns corresponding lock object for the specified critical region name.
virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancel' construct.
QualType SavedKmpTaskTQTy
Saved kmp_task_t for task directive.
virtual void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc)
Emits a master region.
virtual llvm::Function * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts)
Emits outlined function for the OpenMP task directive D.
llvm::DenseMap< llvm::Function *, unsigned > FunctionToUntiedTaskStackMap
Maps function to the position of the untied task locals stack.
void emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal, SourceLocation Loc)
Emits the code to destroy the dependency object provided in depobj directive.
virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Required to resolve existing problems in the runtime.
llvm::ArrayType * KmpCriticalNameTy
Type kmp_critical_name, originally defined as typedef kmp_int32 kmp_critical_name[8];.
virtual void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C)
Emit code for doacross ordered directive with 'depend' clause.
llvm::DenseMap< const OMPDeclareMapperDecl *, llvm::Function * > UDMMap
Map from the user-defined mapper declaration to its corresponding functions.
virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF, const Expr *LHS)
Checks if the provided LVal is lastprivate conditional and emits the code to update the value of the ...
std::pair< llvm::Value *, LValue > getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal, SourceLocation Loc)
Returns the number of the elements and the address of the depobj dependency array.
llvm::SmallDenseSet< const VarDecl * > DeferredGlobalVariables
List of variables that can become declare target implicitly and, thus, must be emitted.
void emitUsesAllocatorsInit(CodeGenFunction &CGF, const Expr *Allocator, const Expr *AllocatorTraits)
Initializes user defined allocators specified in the uses_allocators clauses.
llvm::Type * KmpRoutineEntryPtrTy
Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *);.
llvm::Type * getIdentTyPointerTy()
Returns pointer to ident_t type.
void emitSingleReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp, const Expr *PrivateRef, const DeclRefExpr *LHS, const DeclRefExpr *RHS)
Emits single reduction combiner.
llvm::OpenMPIRBuilder & getOMPBuilder()
virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Emit outilined function for 'target' directive.
virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr)
Emits a critical region.
virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned)
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
llvm::Value * emitNumThreadsForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D)
Emit an expression that denotes the number of threads a target region shall use.
void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor, llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc)
Emits initialization code for the threadprivate variables.
virtual void emitUserDefinedReduction(CodeGenFunction *CGF, const OMPDeclareReductionDecl *D)
Emit code for the specified user defined reduction construct.
virtual void checkAndEmitSharedLastprivateConditional(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::DenseSet< CanonicalDeclPtr< const VarDecl > > &IgnoredDecls)
Checks if the lastprivate conditional was updated in inner region and writes the value.
QualType KmpDimTy
struct kmp_dim { // loop bounds info casted to kmp_int64 kmp_int64 lo; // lower kmp_int64 up; // uppe...
virtual void emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool HasCancel=false)
Emit code for the directive that does not require outlining.
virtual void registerTargetGlobalVariable(const VarDecl *VD, llvm::Constant *Addr)
Checks if the provided global decl GD is a declare target variable and registers it when emitting cod...
virtual void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D)
Emits OpenMP-specific function prolog.
virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond, llvm::Value *NumThreads)
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
void emitKmpRoutineEntryT(QualType KmpInt32Ty)
Build type kmp_routine_entry_t (if not built yet).
virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static chunked.
virtual void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Emit the target offloading code associated with D.
virtual bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS)
Checks if the variable has associated OMPAllocateDeclAttr attribute with the predefined allocator and...
llvm::AtomicOrdering RequiresAtomicOrdering
Atomic ordering from the omp requires directive.
virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps, ReductionOptionsTy Options)
Emit a code for reduction clause.
std::pair< llvm::Value *, Address > emitDependClause(CodeGenFunction &CGF, ArrayRef< OMPTaskDataTy::DependData > Dependencies, SourceLocation Loc)
Emits list of dependecies based on the provided data (array of dependence/expression pairs).
llvm::StringMap< llvm::WeakTrackingVH > EmittedNonTargetVariables
List of the global variables with their addresses that should not be emitted for the target.
virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const
Check if the specified ScheduleKind is dynamic.
Address emitLastprivateConditionalInit(CodeGenFunction &CGF, const VarDecl *VD)
Create specialized alloca to handle lastprivate conditionals.
virtual void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads)
Emit an ordered region.
virtual void emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, ArrayRef< llvm::Value * > Args=std::nullopt) const
Emits call of the outlined function with the provided arguments, translating these arguments to corre...
virtual Address getAddressOfLocalVariable(CodeGenFunction &CGF, const VarDecl *VD)
Gets the OpenMP-specific address of the local variable.
virtual void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, bool IsWorksharingReduction)
Emits the following code for reduction clause with task modifier:
virtual void emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter=nullptr)
Emits a masked region.
QualType KmpDependInfoTy
Type typedef struct kmp_depend_info { kmp_intptr_t base_addr; size_t len; struct { bool in:1; bool ou...
llvm::Function * emitReductionFunction(StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps)
Emits reduction function.
virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues)
Call the appropriate runtime routine to initialize it before start of loop.
Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal) override
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr) override
Emits a critical region.
void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values) override
void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) override
Call the appropriate runtime routine to initialize it before start of loop.
bool emitTargetGlobalVariable(GlobalDecl GD) override
Emit the global variable if it is a valid device global variable.
llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST) override
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
llvm::Function * emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF=nullptr) override
Emit a code for initialization of threadprivate variable.
void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device) override
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
llvm::Function * emitTeamsOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) override
Emits outlined function for the specified OpenMP teams directive D.
void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps, ReductionOptionsTy Options) override
Emit a code for reduction clause.
void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc, llvm::AtomicOrdering AO) override
Emit flush of the variables specified in 'omp flush' directive.
void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C) override
Emit code for doacross ordered directive with 'depend' clause.
void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc) override
Emits a masked region.
Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, QualType VarType, StringRef Name) override
Creates artificial threadprivate variable with name Name and type VarType.
Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc) override
Returns address of the threadprivate variable for the current thread.
void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps) override
Emits a single region.
void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N) override
Required to resolve existing problems in the runtime.
llvm::Function * emitParallelOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) override
Emits outlined function for the specified OpenMP parallel directive D.
void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion) override
Emit code for 'cancellation point' construct.
void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false) override
Emit an implicit/explicit barrier for OpenMP threads.
Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const override
Gets the address of the native argument basing on the address of the target-specific parameter.
void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars) override
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned) override
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
bool emitTargetGlobal(GlobalDecl GD) override
Emit the global GD if it is meaningful for the target.
void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, bool IsWorksharingReduction) override
Emits the following code for reduction clause with task modifier:
void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads) override
Emit an ordered region.
void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind) override
Call the appropriate runtime routine to notify that we finished all the work with current loop.
llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data) override
Emit a code for initialization of task reduction clause.
void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond, llvm::Value *NumThreads) override
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc) override
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, int proc_bind) to generat...
void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) override
Emit outilined function for 'target' directive.
void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc) override
Emits a master region.
void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc) override
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams,...
const VarDecl * translateParameter(const FieldDecl *FD, const VarDecl *NativeParam) const override
Translates the native parameter of outlined function if this is required for target.
void emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter=nullptr) override
Emits a masked region.
void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data) override
Emit task region for the task directive.
void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter) override
Emit the target offloading code associated with D.
bool emitTargetFunctions(GlobalDecl GD) override
Emit the target regions enclosed in GD function definition or the function itself in case it is a val...
void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc) override
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)...
void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr * > NumIterations) override
Emit initialization for doacross loop nesting support.
void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion) override
Emit code for 'cancel' construct.
void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPTaskDataTy &Data) override
Emit code for 'taskwait' directive.
void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc) override
Emit a taskgroup region.
void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, CGOpenMPRuntime::TargetDataInfo &Info) override
Emit the target data mapping code associated with D.
void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues) override
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
llvm::Function * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts) override
Emits outlined function for the OpenMP task directive D.
void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data) override
Emit task region for the taskloop directive.
CGRecordLayout - This class handles struct and union layout info while lowering AST types to LLVM typ...
unsigned getNonVirtualBaseLLVMFieldNo(const CXXRecordDecl *RD) const
llvm::StructType * getLLVMType() const
Return the "complete object" LLVM type associated with this record.
llvm::StructType * getBaseSubobjectLLVMType() const
Return the "base subobject" LLVM type associated with this record.
unsigned getLLVMFieldNo(const FieldDecl *FD) const
Return llvm::StructType element number that corresponds to the field FD.
unsigned getVirtualBaseIndex(const CXXRecordDecl *base) const
Return the LLVM field index corresponding to the given virtual base.
virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S)
Emit the captured statement body.
RAII for correct setting/restoring of CapturedStmtInfo.
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
bool Privatize()
Privatizes local variables previously registered as private.
bool addPrivate(const VarDecl *LocalVD, Address Addr)
Registers LocalVD variable as a private with Addr as the address of the corresponding private variabl...
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited.
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
void EmitNullInitialization(Address DestPtr, QualType Ty)
EmitNullInitialization - Generate code to set a value of the given type to null, If the type contains...
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount, Stmt::Likelihood LH=Stmt::LH_None, const Expr *ConditionalOp=nullptr)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target)
The given basic block lies in the current EH scope, but may be a target of a potentially scope-crossi...
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
FieldDecl * LambdaThisCaptureField
CGCapturedStmtInfo * CapturedStmtInfo
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Load a pointer with type PtrTy stored at address Ptr.
RawAddress CreateDefaultAlignTempAlloca(llvm::Type *Ty, const Twine &Name="tmp")
CreateDefaultAlignedTempAlloca - This creates an alloca with the default ABI alignment of the given L...
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
llvm::Value * emitArrayLength(const ArrayType *arrayType, QualType &baseType, Address &addr)
emitArrayLength - Compute the length of an array, even if it's a VLA, and drill down to the base elem...
VlaSizePair getVLASize(const VariableArrayType *vla)
Returns an LLVM value that corresponds to the size, in non-variably-sized elements,...
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
void EmitVariablyModifiedType(QualType Ty)
EmitVLASize - Capture all the sizes for the VLA expressions in the given variably-modified type and s...
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
const LangOptions & getLangOpts() const
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S)
Emit device code for the target simd directive.
LValue EmitLValueForFieldInitialization(LValue Base, const FieldDecl *Field)
EmitLValueForFieldInitialization - Like EmitLValueForField, except that if the Field is a reference,...
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
const CodeGen::CGBlockInfo * BlockInfo
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit)
EmitExprAsInit - Emits the code necessary to initialize a location in memory with the given initializ...
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
llvm::Type * ConvertTypeForMem(QualType T)
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
llvm::AssertingVH< llvm::Instruction > AllocaInsertPt
AllocaInsertPoint - This is an instruction in the entry block before which we prefer to insert alloca...
void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy)
Emit an aggregate assignment.
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value * > &CapturedVars)
JumpDest ReturnBlock
ReturnBlock - Unified return block.
static void EmitOMPTargetTeamsGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsGenericLoopDirective &S)
Emit device code for the target teams loop directive.
LValue EmitLValueForField(LValue Base, const FieldDecl *Field)
RawAddress CreateMemTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Destroyer * getDestroyer(QualType::DestructionKind destructionKind)
static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
void emitDestroy(Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray)
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc)
EmitLoadOfComplex - Load a complex number from the specified l-value.
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
bool isTrivialInitializer(const Expr *Init)
Determine whether the given initializer is trivial in the sense that it requires no code to be genera...
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block,...
LValue MakeRawAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment, AlignmentSource Source=AlignmentSource::Type)
Same as MakeAddrLValue above except that the pointer is known to be unsigned.
void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy, AggValueSlot::Overlap_t MayOverlap, bool isVolatile=false)
EmitAggregateCopy - Emit an aggregate copy.
LValue MakeNaturalAlignRawAddrLValue(llvm::Value *V, QualType T)
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
llvm::CallInst * EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
ASTContext & getContext() const
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy)
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
void EmitBranchThroughCleanup(JumpDest Dest)
EmitBranchThroughCleanup - Emit a branch from the current insert block through the normal cleanup han...
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
void pushDestroy(QualType::DestructionKind dtorKind, Address addr, QualType type)
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant,...
void EmitAutoVarCleanups(const AutoVarEmission &emission)
bool needsEHCleanup(QualType::DestructionKind kind)
Determines whether an EH cleanup is required to destroy a type with the given destruction kind.
llvm::DenseMap< const ValueDecl *, FieldDecl * > LambdaCaptureFields
llvm::CallInst * EmitRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
llvm::Type * ConvertType(QualType T)
CodeGenTypes & getTypes() const
LValue EmitArraySectionExpr(const ArraySectionExpr *E, bool IsLowerBound=true)
static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Emit device code for the target teams distribute parallel for simd directive.
LValue EmitStringLiteralLValue(const StringLiteral *E)
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
Address LoadCXXThisAddress()
LValue EmitOMPSharedLValue(const Expr *E)
Emits the lvalue for the expression with possibly captured variable.
llvm::Value * EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr, ArrayRef< llvm::Value * > IdxList, bool SignedIndices, bool IsSubtraction, SourceLocation Loc, const Twine &Name="")
Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to detect undefined behavior whe...
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S, SourceLocation Loc)
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
static void EmitOMPTargetParallelGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelGenericLoopDirective &S)
Emit device code for the target parallel loop directive.
LValue EmitLoadOfReferenceLValue(LValue RefLVal)
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler's counter for the given statement by StepV.
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
This class organizes the cross-function state that is used while generating LLVM code.
void handleCUDALaunchBoundsAttr(llvm::Function *F, const CUDALaunchBoundsAttr *A, int32_t *MaxThreadsVal=nullptr, int32_t *MinBlocksVal=nullptr, int32_t *MaxClusterRankVal=nullptr)
Emit the IR encoding to attach the CUDA launch bounds attribute to F.
void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
llvm::Module & getModule() const
void addCompilerUsedGlobal(llvm::GlobalValue *GV)
Add a global to a list to be added to the llvm.compiler.used metadata.
CharUnits GetTargetTypeStoreSize(llvm::Type *Ty) const
Return the store size, in character units, of the given LLVM type.
void handleAMDGPUWavesPerEUAttr(llvm::Function *F, const AMDGPUWavesPerEUAttr *A)
Emit the IR encoding to attach the AMD GPU waves-per-eu attribute to F.
DiagnosticsEngine & getDiags() const
const LangOptions & getLangOpts() const
CharUnits getNaturalTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, bool forPointeeType=false)
CodeGenTypes & getTypes()
const TargetInfo & getTarget() const
void EmitGlobal(GlobalDecl D)
Emit code for a single global function or var decl.
void handleAMDGPUFlatWorkGroupSizeAttr(llvm::Function *F, const AMDGPUFlatWorkGroupSizeAttr *A, const ReqdWorkGroupSizeAttr *ReqdWGS=nullptr, int32_t *MinThreadsVal=nullptr, int32_t *MaxThreadsVal=nullptr)
Emit the IR encoding to attach the AMD GPU flat-work-group-size attribute to F.
llvm::GlobalValue::LinkageTypes getLLVMLinkageVarDefinition(const VarDecl *VD)
Returns LLVM linkage for a declarator.
CGCXXABI & getCXXABI() const
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
const llvm::Triple & getTriple() const
TBAAAccessInfo getTBAAInfoForSubobject(LValue Base, QualType AccessType)
getTBAAInfoForSubobject - Get TBAA information for an access with a given base lvalue.
llvm::Constant * GetAddrOfGlobal(GlobalDecl GD, ForDefinition_t IsForDefinition=NotForDefinition)
ASTContext & getContext() const
const TargetCodeGenInfo & getTargetCodeGenInfo()
const CodeGenOptions & getCodeGenOpts() const
StringRef getMangledName(GlobalDecl GD)
std::optional< CharUnits > getOMPAllocateAlignment(const VarDecl *VD)
Return the alignment specified in an allocate directive, if present.
llvm::GlobalValue * GetGlobalValue(StringRef Ref)
llvm::Constant * EmitNullConstant(QualType T)
Return the result of value-initializing the given type, i.e.
llvm::Function * CreateGlobalInitOrCleanUpFunction(llvm::FunctionType *ty, const Twine &name, const CGFunctionInfo &FI, SourceLocation Loc=SourceLocation(), bool TLS=false, llvm::GlobalVariable::LinkageTypes Linkage=llvm::GlobalVariable::InternalLinkage)
llvm::ConstantInt * getSize(CharUnits numChars)
Emit the given number of characters as a value of type size_t.
llvm::Type * ConvertType(QualType T)
ConvertType - Convert type T into a llvm::Type.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
const CGRecordLayout & getCGRecordLayout(const RecordDecl *)
getCGRecordLayout - Return record layout info for the given record decl.
llvm::Type * ConvertTypeForMem(QualType T, bool ForBitField=false)
ConvertTypeForMem - Convert type T into a llvm::Type.
const CGFunctionInfo & arrangeNullaryFunction()
A nullary function is a freestanding function of type 'void ()'.
A specialization of Address that requires the address to be an LLVM Constant.
static ConstantAddress invalid()
Information for lazily generating a cleanup.
bool requiresLandingPad() const
void popTerminate()
Pops a terminate handler off the stack.
void pushTerminate()
Push a terminate handler on the stack.
FunctionArgList - Type for representing both the decl and type of parameters to a function.
LValue - This represents an lvalue references.
CharUnits getAlignment() const
Address getAddress(CodeGenFunction &CGF) const
const Qualifiers & getQuals() const
LValueBaseInfo getBaseInfo() const
llvm::Value * getPointer(CodeGenFunction &CGF) const
TBAAAccessInfo getTBAAInfo() const
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
virtual void Enter(CodeGenFunction &CGF)
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
static RValue get(llvm::Value *V)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
An abstract representation of an aligned address.
llvm::Type * getElementType() const
Return the type of the values stored in this address.
llvm::Value * getPointer() const
static RawAddress invalid()
Class intended to support codegen of all kind of the reduction clauses.
LValue getSharedLValue(unsigned N) const
Returns LValue for the reduction item.
const Expr * getRefExpr(unsigned N) const
Returns the base declaration of the reduction item.
LValue getOrigLValue(unsigned N) const
Returns LValue for the original reduction item.
bool needCleanups(unsigned N)
Returns true if the private copy requires cleanups.
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
const VarDecl * getBaseDecl(unsigned N) const
Returns the base declaration of the reduction item.
QualType getPrivateType(unsigned N) const
Return the type of the private item.
bool usesReductionInitializer(unsigned N) const
Returns true if the initialization of the reduction item uses initializer from declare reduction cons...
void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N)
Emits lvalue for the shared and original reduction item.
void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, llvm::function_ref< bool(CodeGenFunction &)> DefaultInit)
Performs initialization of the private copy for the reduction item.
std::pair< llvm::Value *, llvm::Value * > getSizes(unsigned N) const
Returns the size of the reduction item (in chars and total number of elements in the item),...
ReductionCodeGen(ArrayRef< const Expr * > Shareds, ArrayRef< const Expr * > Origs, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > ReductionOps)
void emitCleanups(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Emits cleanup code for the reduction item.
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations.
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
void operator()(CodeGenFunction &CGF) const
void setAction(PrePostActionTy &Action) const
virtual void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const
setTargetAttributes - Provides a convenient hook to handle extra target-specific attributes for the g...
ConstStmtVisitor - This class implements a simple visitor for Stmt subclasses.
DeclContext - This is used only as base class of specific decl types that can act as declaration cont...
void addDecl(Decl *D)
Add the declaration D into this context.
A reference to a declared variable, function, enum, etc.
Decl - This represents one declaration (or definition), e.g.
ASTContext & getASTContext() const LLVM_READONLY
llvm::iterator_range< specific_attr_iterator< T > > specific_attrs() const
SourceLocation getLocation() const
DeclContext * getDeclContext()
virtual Decl * getCanonicalDecl()
Retrieves the "canonical" declaration of the given declaration.
SourceLocation getBeginLoc() const LLVM_READONLY
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
unsigned getCustomDiagID(Level L, const char(&FormatString)[N])
Return an ID for a diagnostic with the specified format string and level.
The return type of classify().
This represents one expression.
@ SE_AllowSideEffects
Allow any unmodeled side effect.
@ SE_AllowUndefinedBehavior
Allow UB that we can give a value, but not arbitrary unmodeled side effects.
Expr * IgnoreParenCasts() LLVM_READONLY
Skip past any parentheses and casts which might surround this expression until reaching a fixed point...
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx, SmallVectorImpl< PartialDiagnosticAt > *Diag=nullptr) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
bool isEvaluatable(const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects) const
isEvaluatable - Call EvaluateAsRValue to see if this expression can be constant folded without side-e...
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
bool isIntegerConstantExpr(const ASTContext &Ctx, SourceLocation *Loc=nullptr) const
bool EvaluateAsBooleanCondition(bool &Result, const ASTContext &Ctx, bool InConstantContext=false) const
EvaluateAsBooleanCondition - Return true if this is a constant which we can fold and convert to a boo...
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
std::optional< llvm::APSInt > getIntegerConstantExpr(const ASTContext &Ctx, SourceLocation *Loc=nullptr) const
isIntegerConstantExpr - Return the value if this expression is a valid integer constant expression.
bool hasNonTrivialCall(const ASTContext &Ctx) const
Determine whether this expression involves a call to any function that is not trivial.
Represents a member of a struct/union/class.
static FieldDecl * Create(const ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, const IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, Expr *BW, bool Mutable, InClassInitStyle InitStyle)
Represents a function declaration or definition.
const ParmVarDecl * getParamDecl(unsigned i) const
QualType getReturnType() const
ArrayRef< ParmVarDecl * > parameters() const
FunctionDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
unsigned getNumParams() const
Return the number of parameters this function must have based on its FunctionType.
GlobalDecl - represents a global declaration.
const Decl * getDecl() const
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
static IntegerLiteral * Create(const ASTContext &C, const llvm::APInt &V, QualType type, SourceLocation l)
Returns a new integer literal with value 'V' and type 'type'.
Describes the capture of a variable or of this, or of a C++1y init-capture.
std::string OMPHostIRFile
Name of the IR file that contains the result of the OpenMP target host code generation.
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
virtual void mangleCanonicalTypeName(QualType T, raw_ostream &, bool NormalizeIntegers=false)=0
Generates a unique string for an externally visible type for use with TBAA or type uniquing.
MemberExpr - [C99 6.5.2.3] Structure and Union Members.
ValueDecl * getMemberDecl() const
Retrieve the member declaration to which this expression refers.
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
bool isExternallyVisible() const
This represents clause 'affinity' in the '#pragma omp task'-based directives.
Class that represents a component of a mappable expression.
ArrayRef< MappableComponent > MappableExprComponentListRef
const Stmt * getPreInitStmt() const
Get pre-initialization statement for the clause.
This is a basic class for representing single OpenMP clause.
This represents '#pragma omp declare mapper ...' directive.
Expr * getMapperVarRef()
Get the variable declared in the mapper.
This represents '#pragma omp declare reduction ...' directive.
Expr * getInitializer()
Get initializer expression (if specified) of the declare reduction construct.
Expr * getInitPriv()
Get Priv variable of the initializer.
Expr * getCombinerOut()
Get Out variable of the combiner.
Expr * getCombinerIn()
Get In variable of the combiner.
Expr * getCombiner()
Get combiner expression of the declare reduction construct.
Expr * getInitOrig()
Get Orig variable of the initializer.
OMPDeclareReductionInitKind getInitializerKind() const
Get initializer kind.
This represents implicit clause 'depend' for the '#pragma omp task' directive.
This represents 'detach' clause in the '#pragma omp task' directive.
This represents 'device' clause in the '#pragma omp ...' directive.
This represents the 'doacross' clause for the '#pragma omp ordered' directive.
This is a basic class for representing single OpenMP executable directive.
CapturedStmt * getInnermostCapturedStmt()
Get innermost captured statement for the construct.
const CapturedStmt * getCapturedStmt(OpenMPDirectiveKind RegionKind) const
Returns the captured statement associated with the component region within the (combined) directive.
OpenMPDirectiveKind getDirectiveKind() const
SourceLocation getBeginLoc() const
Returns starting location of directive kind.
bool hasClausesOfKind() const
Returns true if the current directive has one or more clauses of a specific kind.
SourceLocation getEndLoc() const
Returns ending location of directive.
static const SpecificClause * getSingleClause(ArrayRef< OMPClause * > Clauses)
Gets a single clause of the specified kind associated with the current directive iff there is only on...
static llvm::iterator_range< specific_clause_iterator< SpecificClause > > getClausesOfKind(ArrayRef< OMPClause * > Clauses)
This represents clause 'firstprivate' in the '#pragma omp ...' directives.
This represents clause 'has_device_ptr' in the '#pragma omp ...' directives.
This represents 'if' clause in the '#pragma omp ...' directive.
Expr * getCondition() const
Returns condition.
This represents clause 'in_reduction' in the '#pragma omp task' directives.
This represents clause 'is_device_ptr' in the '#pragma omp ...' directives.
OpenMP 5.0 [2.1.6 Iterators] Iterators are identifiers that expand to multiple values in the clause o...
OMPIteratorHelperData & getHelper(unsigned I)
Fetches helper data for the specified iteration space.
unsigned numOfIterators() const
Returns number of iterator definitions.
Decl * getIteratorDecl(unsigned I)
Gets the iterator declaration for the given iterator.
This represents clause 'lastprivate' in the '#pragma omp ...' directives.
This represents clause 'linear' in the '#pragma omp ...' directives.
This is a common base class for loop directives ('omp simd', 'omp for', 'omp for simd' etc....
Expr * getStrideVariable() const
Expr * getUpperBoundVariable() const
Expr * getLowerBoundVariable() const
This represents clause 'map' in the '#pragma omp ...' directives.
This represents clause 'nontemporal' in the '#pragma omp ...' directives.
This represents 'nowait' clause in the '#pragma omp ...' directive.
This represents 'num_teams' clause in the '#pragma omp ...' directive.
This represents 'num_threads' clause in the '#pragma omp ...' directive.
This represents 'ordered' clause in the '#pragma omp ...' directive.
This represents clause 'private' in the '#pragma omp ...' directives.
This represents clause 'reduction' in the '#pragma omp ...' directives.
This represents '#pragma omp requires...' directive.
clauselist_range clauselists()
This represents 'thread_limit' clause in the '#pragma omp ...' directive.
This represents clause 'uses_allocators' in the '#pragma omp target'-based directives.
This represents 'ompx_attribute' clause in a directive that might generate an outlined function.
This represents 'ompx_dyn_cgroup_mem' clause in the '#pragma omp target ...' directive.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class.
Represents a parameter to a function.
PointerType - C99 6.7.5.1 - Pointer Declarators.
Represents an unpacked "presumed" location which can be presented to the user.
unsigned getColumn() const
Return the presumed column number of this location.
const char * getFilename() const
Return the presumed filename of this location.
unsigned getLine() const
Return the presumed line number of this location.
A (possibly-)qualified type.
void addRestrict()
Add the restrict qualifier to this QualType.
QualType withRestrict() const
bool isNull() const
Return true if this QualType doesn't point to a type yet.
const Type * getTypePtr() const
Retrieves a pointer to the underlying (unqualified) type.
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
QualType getCanonicalType() const
DestructionKind isDestructedType() const
Returns a nonzero value if objects of this type require non-trivial work to clean up after.
Represents a struct/union/class.
field_iterator field_end() const
field_range fields() const
virtual void completeDefinition()
Note that the definition of this type is now complete.
field_iterator field_begin() const
RecordDecl * getDecl() const
decl_type * getPreviousDecl()
Return the previous declaration of this declaration or NULL if this is the first declaration.
decl_type * getMostRecentDecl()
Returns the most recent (re)declaration of this declaration.
Base for LValueReferenceType and RValueReferenceType.
Scope - A scope is a transient data structure that is used while parsing the program.
Encodes a location in the source.
static SourceLocation getFromRawEncoding(UIntTy Encoding)
Turn a raw encoding of a SourceLocation object into a real SourceLocation.
bool isValid() const
Return true if this is a valid SourceLocation object.
UIntTy getRawEncoding() const
When a SourceLocation itself cannot be used, this returns an (opaque) 32-bit integer encoding for it.
This class handles loading and caching of source files into memory.
PresumedLoc getPresumedLoc(SourceLocation Loc, bool UseLineDirectives=true) const
Returns the "presumed" location of a SourceLocation specifies.
fileinfo_iterator fileinfo_end() const
SourceLocation translateFileLineCol(const FileEntry *SourceFile, unsigned Line, unsigned Col) const
Get the source location for the given file:line:col triplet.
fileinfo_iterator fileinfo_begin() const
A trivial tuple used to represent a source range.
Stmt - This represents one statement.
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top,...
SourceLocation getBeginLoc() const LLVM_READONLY
void startDefinition()
Starts the definition of this tag declaration.
bool isTLSSupported() const
Whether the target supports thread-local storage.
virtual bool hasFeature(StringRef Feature) const
Determine whether the given target has the given feature.
The base class of the type hierarchy.
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
bool isSignedIntegerOrEnumerationType() const
Determines whether this is an integer type that is signed or an enumeration types whose underlying ty...
const Type * getPointeeOrArrayElementType() const
If this is a pointer type, return the pointee type.
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
bool isPointerType() const
CanQualType getCanonicalTypeUnqualified() const
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
const T * castAs() const
Member-template castAs<specific type>.
bool isReferenceType() const
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
bool isLValueReferenceType() const
QualType getCanonicalTypeInternal() const
const RecordType * getAsStructureType() const
const Type * getBaseElementTypeUnsafe() const
Get the base element type of this type, potentially discarding type qualifiers.
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type.
bool isFloatingType() const
bool isUnsignedIntegerType() const
Return true if this is an integer type that is unsigned, according to C99 6.2.5p6 [which returns true...
bool isAnyPointerType() const
const T * getAs() const
Member-template getAs<specific type>'.
bool isRecordType() const
TagDecl * getAsTagDecl() const
Retrieves the TagDecl that this type refers to, either because the type is a TagType or because it is...
RecordDecl * getAsRecordDecl() const
Retrieves the RecordDecl this type refers to.
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Represents a variable declaration or definition.
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
VarDecl * getDefinition(ASTContext &)
Get the real (not just tentative) definition for this declaration.
bool hasExternalStorage() const
Returns true if a variable has extern or private_extern storage.
bool hasLocalStorage() const
Returns true if a variable with function scope is a non-static local variable.
@ DeclarationOnly
This declaration is only a declaration.
DefinitionKind hasDefinition(ASTContext &) const
Check whether this variable is defined in this translation unit.
bool isLocalVarDeclOrParm() const
Similar to isLocalVarDecl but also includes parameters.
const Expr * getAnyInitializer() const
Get the initializer for this variable, no matter which declaration it is attached to.
Represents a C array with a specified size that is not an integer-constant-expression.
Expr * getSizeExpr() const
specific_attr_iterator - Iterates over a subrange of an AttrVec, only providing attributes that are o...
@ Decl
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
@ LLVM_MARK_AS_BITMASK_ENUM
The JSON file list parser is used to communicate input to InstallAPI.
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
@ Private
'private' clause, allowed on 'parallel', 'serial', 'loop', 'parallel loop', and 'serial loop' constru...
@ Vector
'vector' clause, allowed on 'loop', Combined, and 'routine' directives.
@ Reduction
'reduction' clause, allowed on Parallel, Serial, Loop, and the combined constructs.
@ Present
'present' clause, allowed on Compute and Combined constructs, plus 'data' and 'declare'.
bool needsTaskBasedThreadLimit(OpenMPDirectiveKind DKind)
Checks if the specified target directive, combined or not, needs task based thread_limit.
@ Ctor_Complete
Complete object ctor.
if(T->getSizeExpr()) TRY_TO(TraverseStmt(const_cast< Expr * >(T -> getSizeExpr())))
bool isOpenMPTargetDataManagementDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target data offload directive.
llvm::omp::Directive OpenMPDirectiveKind
OpenMP directives.
@ ICIS_NoInit
No in-class initializer.
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
@ LCK_ByRef
Capturing by reference.
LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE()
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
@ OMPC_SCHEDULE_MODIFIER_last
@ OMPC_SCHEDULE_MODIFIER_unknown
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
OpenMPDistScheduleClauseKind
OpenMP attributes for 'dist_schedule' clause.
bool isOpenMPTaskingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of tasking directives - task, taskloop,...
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
@ Result
The result type of a method or function.
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
OpenMPDependClauseKind
OpenMP attributes for 'depend' clause.
@ Dtor_Complete
Complete object dtor.
@ Union
The "union" keyword.
bool isOpenMPLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a directive with an associated loop construct.
LangAS
Defines the address space values used by the address space qualifier of QualType.
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
@ VK_PRValue
A pr-value expression (in the C++11 taxonomy) produces a temporary value.
@ VK_LValue
An l-value expression is a reference to an object with independent storage.
const FunctionProtoType * T
void getOpenMPCaptureRegions(llvm::SmallVectorImpl< OpenMPDirectiveKind > &CaptureRegions, OpenMPDirectiveKind DKind)
Return the captured regions of an OpenMP directive.
@ OMPC_ATOMIC_DEFAULT_MEM_ORDER_unknown
OpenMPMapModifierKind
OpenMP modifier kind for 'map' clause.
@ OMPC_MAP_MODIFIER_unknown
@ Other
Other implicit parameter.
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
OpenMPMapClauseKind
OpenMP mapping kind for 'map' clause.
Diagnostic wrappers for TextAPI types for error reporting.
Maps the expression for the lastprivate variable to the global copy used to store new value because o...
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
llvm::BasicBlock * getBlock() const
unsigned NumberOfTargetItems
Address BasePointersArray
llvm::PointerType * VoidPtrTy
llvm::IntegerType * Int64Ty
llvm::PointerType * Int8PtrPtrTy
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::CallingConv::ID getRuntimeCC() const
llvm::IntegerType * SizeTy
llvm::PointerType * VoidPtrPtrTy
llvm::IntegerType * Int32Ty
llvm::IntegerType * IntPtrTy
llvm::IntegerType * IntTy
int
llvm::PointerType * Int8PtrTy
CharUnits getPointerAlign() const
OpenMPDependClauseKind DepKind
const Expr * IteratorExpr
SmallVector< const Expr *, 4 > DepExprs
EvalResult is a struct with detailed info about an evaluated expression.
Extra information about a function prototype.
Helper expressions and declaration for OMPIteratorExpr class for each iteration space.
Expr * CounterUpdate
Updater for the internal counter: ++CounterVD;.
Expr * Upper
Normalized upper bound.
Expr * Update
Update expression for the originally specified iteration variable, calculated as VD = Begin + Counter...
VarDecl * CounterVD
Internal normalized counter.
Data for list of allocators.
Expr * AllocatorTraits
Allocator traits.
Expr * Allocator
Allocator.
Scheduling data for loop-based OpenMP directives.
OpenMPScheduleClauseModifier M2
OpenMPScheduleClauseModifier M1
OpenMPScheduleClauseKind Schedule
Describes how types, statements, expressions, and declarations should be printed.