30 #include "llvm/ADT/ArrayRef.h"
31 #include "llvm/ADT/SetOperations.h"
32 #include "llvm/ADT/SmallBitVector.h"
33 #include "llvm/ADT/StringExtras.h"
34 #include "llvm/Bitcode/BitcodeReader.h"
35 #include "llvm/IR/Constants.h"
36 #include "llvm/IR/DerivedTypes.h"
37 #include "llvm/IR/GlobalValue.h"
38 #include "llvm/IR/InstrTypes.h"
39 #include "llvm/IR/Value.h"
40 #include "llvm/Support/AtomicOrdering.h"
41 #include "llvm/Support/Format.h"
42 #include "llvm/Support/raw_ostream.h"
46 using namespace clang;
47 using namespace CodeGen;
48 using namespace llvm::omp;
55 enum CGOpenMPRegionKind {
58 ParallelOutlinedRegion,
69 const CGOpenMPRegionKind RegionKind,
72 : CGCapturedStmtInfo(CS,
CR_OpenMP), RegionKind(RegionKind),
73 CodeGen(CodeGen),
Kind(
Kind), HasCancel(HasCancel) {}
75 CGOpenMPRegionInfo(
const CGOpenMPRegionKind RegionKind,
78 : CGCapturedStmtInfo(
CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
83 virtual const VarDecl *getThreadIDVariable()
const = 0;
94 CGOpenMPRegionKind getRegionKind()
const {
return RegionKind; }
98 bool hasCancel()
const {
return HasCancel; }
100 static bool classof(
const CGCapturedStmtInfo *Info) {
104 ~CGOpenMPRegionInfo()
override =
default;
107 CGOpenMPRegionKind RegionKind;
114 class CGOpenMPOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
119 StringRef HelperName)
120 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen,
Kind,
122 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
123 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
128 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
131 StringRef getHelperName()
const override {
return HelperName; }
133 static bool classof(
const CGCapturedStmtInfo *Info) {
134 return CGOpenMPRegionInfo::classof(Info) &&
135 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
136 ParallelOutlinedRegion;
143 StringRef HelperName;
147 class CGOpenMPTaskOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
153 llvm::SwitchInst *UntiedSwitch =
nullptr;
156 UntiedTaskActionTy(
bool Tied,
const VarDecl *PartIDVar,
158 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
168 UntiedSwitch = CGF.
Builder.CreateSwitch(Res, DoneBB);
172 UntiedSwitch->addCase(CGF.
Builder.getInt32(0),
174 emitUntiedSwitch(CGF);
189 UntiedSwitch->addCase(CGF.
Builder.getInt32(UntiedSwitch->getNumCases()),
195 unsigned getNumberOfParts()
const {
return UntiedSwitch->getNumCases(); }
201 const UntiedTaskActionTy &Action)
202 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen,
Kind, HasCancel),
203 ThreadIDVar(ThreadIDVar), Action(Action) {
204 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
209 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
215 StringRef getHelperName()
const override {
return ".omp_outlined."; }
218 Action.emitUntiedSwitch(CGF);
221 static bool classof(
const CGCapturedStmtInfo *Info) {
222 return CGOpenMPRegionInfo::classof(Info) &&
223 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
232 const UntiedTaskActionTy &Action;
237 class CGOpenMPInlinedRegionInfo :
public CGOpenMPRegionInfo {
242 : CGOpenMPRegionInfo(InlinedRegion, CodeGen,
Kind, HasCancel),
244 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
247 llvm::Value *getContextValue()
const override {
250 llvm_unreachable(
"No context value for inlined OpenMP region");
253 void setContextValue(llvm::Value *
V)
override {
254 if (OuterRegionInfo) {
255 OuterRegionInfo->setContextValue(
V);
258 llvm_unreachable(
"No context value for inlined OpenMP region");
264 return OuterRegionInfo->lookup(VD);
270 FieldDecl *getThisFieldDecl()
const override {
272 return OuterRegionInfo->getThisFieldDecl();
278 const VarDecl *getThreadIDVariable()
const override {
280 return OuterRegionInfo->getThreadIDVariable();
287 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
288 llvm_unreachable(
"No LValue for inlined OpenMP construct");
292 StringRef getHelperName()
const override {
293 if (
auto *OuterRegionInfo = getOldCSI())
294 return OuterRegionInfo->getHelperName();
295 llvm_unreachable(
"No helper name for inlined OpenMP construct");
300 OuterRegionInfo->emitUntiedSwitch(CGF);
305 static bool classof(
const CGCapturedStmtInfo *Info) {
306 return CGOpenMPRegionInfo::classof(Info) &&
307 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
310 ~CGOpenMPInlinedRegionInfo()
override =
default;
315 CGOpenMPRegionInfo *OuterRegionInfo;
323 class CGOpenMPTargetRegionInfo final :
public CGOpenMPRegionInfo {
327 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
329 HelperName(HelperName) {}
333 const VarDecl *getThreadIDVariable()
const override {
return nullptr; }
336 StringRef getHelperName()
const override {
return HelperName; }
338 static bool classof(
const CGCapturedStmtInfo *Info) {
339 return CGOpenMPRegionInfo::classof(Info) &&
340 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
344 StringRef HelperName;
348 llvm_unreachable(
"No codegen for expressions");
352 class CGOpenMPInnerExprInfo final :
public CGOpenMPInlinedRegionInfo {
355 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
362 for (
const auto &C : CS.
captures()) {
363 if (!
C.capturesVariable() && !
C.capturesVariableByCopy())
366 const VarDecl *VD =
C.getCapturedVar();
376 (void)PrivScope.Privatize();
381 if (
const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
388 llvm_unreachable(
"No body for expressions");
393 const VarDecl *getThreadIDVariable()
const override {
394 llvm_unreachable(
"No thread id for expressions");
398 StringRef getHelperName()
const override {
399 llvm_unreachable(
"No helper name for expressions");
402 static bool classof(
const CGCapturedStmtInfo *Info) {
return false; }
410 class InlinedOpenMPRegionRAII {
412 llvm::DenseMap<const VarDecl *, FieldDecl *> LambdaCaptureFields;
413 FieldDecl *LambdaThisCaptureField =
nullptr;
415 bool NoInheritance =
false;
424 bool NoInheritance =
true)
425 : CGF(CGF), NoInheritance(NoInheritance) {
438 ~InlinedOpenMPRegionRAII() {
455 enum OpenMPLocationFlags :
unsigned {
457 OMP_IDENT_IMD = 0x01,
459 OMP_IDENT_KMPC = 0x02,
461 OMP_ATOMIC_REDUCE = 0x10,
463 OMP_IDENT_BARRIER_EXPL = 0x20,
465 OMP_IDENT_BARRIER_IMPL = 0x40,
467 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
469 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
471 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
473 OMP_IDENT_WORK_LOOP = 0x200,
475 OMP_IDENT_WORK_SECTIONS = 0x400,
477 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
486 OMP_REQ_UNDEFINED = 0x000,
488 OMP_REQ_NONE = 0x001,
490 OMP_REQ_REVERSE_OFFLOAD = 0x002,
492 OMP_REQ_UNIFIED_ADDRESS = 0x004,
494 OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008,
496 OMP_REQ_DYNAMIC_ALLOCATORS = 0x010,
497 LLVM_MARK_AS_BITMASK_ENUM(OMP_REQ_DYNAMIC_ALLOCATORS)
503 OMP_DEVICEID_UNDEF = -1,
533 enum IdentFieldIndex {
535 IdentField_Reserved_1,
539 IdentField_Reserved_2,
541 IdentField_Reserved_3,
550 enum OpenMPSchedType {
553 OMP_sch_static_chunked = 33,
555 OMP_sch_dynamic_chunked = 35,
556 OMP_sch_guided_chunked = 36,
557 OMP_sch_runtime = 37,
560 OMP_sch_static_balanced_chunked = 45,
563 OMP_ord_static_chunked = 65,
565 OMP_ord_dynamic_chunked = 67,
566 OMP_ord_guided_chunked = 68,
567 OMP_ord_runtime = 69,
569 OMP_sch_default = OMP_sch_static,
571 OMP_dist_sch_static_chunked = 91,
572 OMP_dist_sch_static = 92,
575 OMP_sch_modifier_monotonic = (1 << 29),
577 OMP_sch_modifier_nonmonotonic = (1 << 30),
600 Callback(CodeGen, CGF, *PrePostAction);
603 Callback(CodeGen, CGF, Action);
611 if (
const auto *CE = dyn_cast<CallExpr>(ReductionOp))
612 if (
const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
613 if (
const auto *DRE =
614 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
615 if (
const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
626 std::pair<llvm::Function *, llvm::Function *> Reduction =
628 const auto *CE = cast<CallExpr>(InitOp);
629 const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
633 cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
635 cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
637 PrivateScope.
addPrivate(cast<VarDecl>(LHSDRE->getDecl()), Private);
638 PrivateScope.
addPrivate(cast<VarDecl>(RHSDRE->getDecl()), Original);
646 auto *GV =
new llvm::GlobalVariable(
648 llvm::GlobalValue::PrivateLinkage, Init, Name);
689 llvm::Value *NumElements = CGF.
emitArrayLength(ArrayTy, ElementTy, DestAddr);
694 llvm::Value *SrcBegin =
nullptr;
697 llvm::Value *DestBegin = DestAddr.
getPointer();
699 llvm::Value *DestEnd =
704 llvm::Value *IsEmpty =
705 CGF.
Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arrayinit.isempty");
706 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
709 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
714 llvm::PHINode *SrcElementPHI =
nullptr;
717 SrcElementPHI = CGF.
Builder.CreatePHI(SrcBegin->getType(), 2,
718 "omp.arraycpy.srcElementPast");
719 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
721 Address(SrcElementPHI, SrcAddr.getElementType(),
722 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
724 llvm::PHINode *DestElementPHI = CGF.
Builder.CreatePHI(
725 DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
726 DestElementPHI->addIncoming(DestBegin, EntryBB);
734 if (EmitDeclareReductionInit) {
736 SrcElementCurrent, ElementTy);
744 llvm::Value *SrcElementNext = CGF.
Builder.CreateConstGEP1_32(
745 SrcAddr.getElementType(), SrcElementPHI, 1,
746 "omp.arraycpy.dest.element");
747 SrcElementPHI->addIncoming(SrcElementNext, CGF.
Builder.GetInsertBlock());
751 llvm::Value *DestElementNext = CGF.
Builder.CreateConstGEP1_32(
753 "omp.arraycpy.dest.element");
756 CGF.
Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
757 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
758 DestElementPHI->addIncoming(DestElementNext, CGF.
Builder.GetInsertBlock());
770 if (
const auto *OASE = dyn_cast<OMPArraySectionExpr>(E))
775 void ReductionCodeGen::emitAggregateInitialization(
781 const auto *PrivateVD =
782 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
783 bool EmitDeclareReductionInit =
786 EmitDeclareReductionInit,
787 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
788 : PrivateVD->getInit(),
796 ClausesData.reserve(Shareds.size());
797 SharedAddresses.reserve(Shareds.size());
798 Sizes.reserve(Shareds.size());
799 BaseDecls.reserve(Shareds.size());
800 const auto *IOrig = Origs.begin();
801 const auto *IPriv = Privates.begin();
802 const auto *IRed = ReductionOps.begin();
803 for (
const Expr *Ref : Shareds) {
804 ClausesData.emplace_back(Ref, *IOrig, *IPriv, *IRed);
812 assert(SharedAddresses.size() == N && OrigAddresses.size() == N &&
813 "Number of generated lvalues must be exactly N.");
814 LValue First = emitSharedLValue(CGF, ClausesData[N].Shared);
815 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Shared);
816 SharedAddresses.emplace_back(
First, Second);
817 if (ClausesData[N].Shared == ClausesData[N].Ref) {
818 OrigAddresses.emplace_back(
First, Second);
820 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
821 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
822 OrigAddresses.emplace_back(
First, Second);
827 QualType PrivateType = getPrivateType(N);
828 bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
831 CGF.
getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()),
836 llvm::Value *SizeInChars;
837 auto *ElemType = OrigAddresses[N].first.getAddress(CGF).getElementType();
838 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
839 if (AsArraySection) {
840 Size = CGF.
Builder.CreatePtrDiff(ElemType,
841 OrigAddresses[N].second.getPointer(CGF),
842 OrigAddresses[N].first.getPointer(CGF));
843 Size = CGF.
Builder.CreateNUWAdd(
844 Size, llvm::ConstantInt::get(Size->getType(), 1));
845 SizeInChars = CGF.
Builder.CreateNUWMul(Size, ElemSizeOf);
848 CGF.
getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType());
849 Size = CGF.
Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
851 Sizes.emplace_back(SizeInChars, Size);
854 cast<OpaqueValueExpr>(
862 QualType PrivateType = getPrivateType(N);
864 assert(!Size && !Sizes[N].second &&
865 "Size should be nullptr for non-variably modified reduction "
871 cast<OpaqueValueExpr>(
880 assert(SharedAddresses.size() > N &&
"No variable was generated");
881 const auto *PrivateVD =
882 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
887 (void)DefaultInit(CGF);
888 emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD);
889 }
else if (DRD && (DRD->
getInitializer() || !PrivateVD->hasInit())) {
890 (void)DefaultInit(CGF);
891 QualType SharedType = SharedAddresses[N].first.getType();
893 PrivateAddr, SharedAddr, SharedType);
894 }
else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
897 PrivateVD->
getType().getQualifiers(),
903 QualType PrivateType = getPrivateType(N);
910 QualType PrivateType = getPrivateType(N);
912 if (needCleanups(N)) {
915 CGF.
pushDestroy(DTorKind, PrivateAddr, PrivateType);
940 Address OriginalBaseAddress, llvm::Value *Addr) {
964 Addr, OriginalBaseAddress.
getType());
969 const VarDecl *OrigVD =
nullptr;
970 if (
const auto *OASE = dyn_cast<OMPArraySectionExpr>(Ref)) {
971 const Expr *
Base = OASE->getBase()->IgnoreParenImpCasts();
972 while (
const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(
Base))
973 Base = TempOASE->getBase()->IgnoreParenImpCasts();
974 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
975 Base = TempASE->getBase()->IgnoreParenImpCasts();
976 DE = cast<DeclRefExpr>(
Base);
977 OrigVD = cast<VarDecl>(DE->
getDecl());
978 }
else if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
979 const Expr *
Base = ASE->getBase()->IgnoreParenImpCasts();
980 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
981 Base = TempASE->getBase()->IgnoreParenImpCasts();
982 DE = cast<DeclRefExpr>(
Base);
983 OrigVD = cast<VarDecl>(DE->
getDecl());
992 BaseDecls.emplace_back(OrigVD);
995 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
997 Address SharedAddr = SharedAddresses[N].first.getAddress(CGF);
998 llvm::Value *Adjustment = CGF.
Builder.CreatePtrDiff(
1001 llvm::Value *PrivatePointer =
1007 SharedAddresses[N].first.getType(),
1010 BaseDecls.emplace_back(
1011 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
1024 getThreadIDVariable()->getType()->castAs<PointerType>());
1042 LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1045 getThreadIDVariable()->getType(),
1061 StringRef Separator)
1062 : CGM(CGM), FirstSeparator(FirstSeparator), Separator(Separator),
1063 OMPBuilder(CGM.getModule()), OffloadEntriesInfoManager(CGM) {
1064 KmpCriticalNameTy = llvm::ArrayType::get(
CGM.
Int32Ty, 8);
1068 loadOffloadInfoMetadata();
1072 InternalVars.clear();
1074 for (
const auto &Data : EmittedNonTargetVariables) {
1075 if (!Data.getValue().pointsToAliveValue())
1077 auto *GV = dyn_cast<llvm::GlobalVariable>(Data.getValue());
1080 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1082 GV->eraseFromParent();
1088 llvm::raw_svector_ostream OS(Buffer);
1090 for (StringRef Part : Parts) {
1097 static llvm::Function *
1099 const Expr *CombinerInitializer,
const VarDecl *In,
1100 const VarDecl *Out,
bool IsCombiner) {
1103 QualType PtrTy = C.getPointerType(Ty).withRestrict();
1109 Args.push_back(&OmpOutParm);
1110 Args.push_back(&OmpInParm);
1115 {IsCombiner ?
"omp_combiner" :
"omp_initializer",
""});
1120 Fn->removeFnAttr(llvm::Attribute::NoInline);
1121 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1122 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1128 Out->getLocation());
1138 (void)
Scope.Privatize();
1139 if (!IsCombiner && Out->hasInit() &&
1142 Out->getType().getQualifiers(),
1145 if (CombinerInitializer)
1147 Scope.ForceCleanup();
1154 if (UDRMap.count(D) > 0)
1158 cast<VarDecl>(cast<DeclRefExpr>(D->
getCombinerIn())->getDecl()),
1159 cast<VarDecl>(cast<DeclRefExpr>(D->
getCombinerOut())->getDecl()),
1167 cast<VarDecl>(cast<DeclRefExpr>(D->
getInitOrig())->getDecl()),
1168 cast<VarDecl>(cast<DeclRefExpr>(D->
getInitPriv())->getDecl()),
1173 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->
CurFn);
1174 Decls.second.push_back(D);
1178 std::pair<llvm::Function *, llvm::Function *>
1180 auto I = UDRMap.find(D);
1181 if (I != UDRMap.end())
1184 return UDRMap.lookup(D);
1190 struct PushAndPopStackRAII {
1191 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder,
CodeGenFunction &CGF,
1193 : OMPBuilder(OMPBuilder) {
1209 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1210 assert(IP.getBlock()->end() == IP.getPoint() &&
1211 "Clang CG should cause non-terminated block!");
1212 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1221 llvm::OpenMPIRBuilder::FinalizationInfo FI({FiniCB,
Kind, HasCancel});
1222 OMPBuilder->pushFinalizationCB(std::move(FI));
1224 ~PushAndPopStackRAII() {
1226 OMPBuilder->popFinalizationCB();
1228 llvm::OpenMPIRBuilder *OMPBuilder;
1237 "thread id variable must be of type kmp_int32 *");
1239 bool HasCancel =
false;
1240 if (
const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1241 HasCancel = OPD->hasCancel();
1242 else if (
const auto *OPD = dyn_cast<OMPTargetParallelDirective>(&D))
1243 HasCancel = OPD->hasCancel();
1244 else if (
const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1245 HasCancel = OPSD->hasCancel();
1246 else if (
const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1247 HasCancel = OPFD->hasCancel();
1248 else if (
const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1249 HasCancel = OPFD->hasCancel();
1250 else if (
const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1251 HasCancel = OPFD->hasCancel();
1252 else if (
const auto *OPFD =
1253 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1254 HasCancel = OPFD->hasCancel();
1255 else if (
const auto *OPFD =
1256 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1257 HasCancel = OPFD->hasCancel();
1262 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind);
1263 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1264 HasCancel, OutlinedHelperName);
1289 bool Tied,
unsigned &NumberOfParts) {
1294 llvm::Value *TaskArgs[] = {
1296 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1299 CGF.EmitRuntimeCall(
OMPBuilder.getOrCreateRuntimeFunction(
1303 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1307 "thread id variable must be of type kmp_int32 for tasks");
1312 bool HasCancel =
false;
1313 if (
const auto *TD = dyn_cast<OMPTaskDirective>(&D))
1314 HasCancel = TD->hasCancel();
1315 else if (
const auto *TD = dyn_cast<OMPTaskLoopDirective>(&D))
1316 HasCancel = TD->hasCancel();
1317 else if (
const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(&D))
1318 HasCancel = TD->hasCancel();
1319 else if (
const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(&D))
1320 HasCancel = TD->hasCancel();
1323 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1324 InnermostKind, HasCancel, Action);
1328 NumberOfParts = Action.getNumberOfParts();
1333 bool AtCurrentPoint) {
1334 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.
CurFn);
1335 assert(!Elem.second.ServiceInsertPt &&
"Insert point is set already.");
1337 llvm::Value *Undef = llvm::UndefValue::get(CGF.
Int32Ty);
1338 if (AtCurrentPoint) {
1339 Elem.second.ServiceInsertPt =
new llvm::BitCastInst(
1342 Elem.second.ServiceInsertPt =
1343 new llvm::BitCastInst(Undef, CGF.
Int32Ty,
"svcpt");
1349 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.
CurFn);
1350 if (Elem.second.ServiceInsertPt) {
1351 llvm::Instruction *Ptr = Elem.second.ServiceInsertPt;
1352 Elem.second.ServiceInsertPt =
nullptr;
1353 Ptr->eraseFromParent();
1360 llvm::raw_svector_ostream OS(Buffer);
1364 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.
CurFuncDecl))
1365 OS << FD->getQualifiedNameAsString();
1373 uint32_t SrcLocStrSize;
1374 llvm::Constant *SrcLocStr;
1377 SrcLocStr =
OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1380 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.
CurFuncDecl))
1381 FunctionName = FD->getQualifiedNameAsString();
1386 SrcLocStr =
OMPBuilder.getOrCreateSrcLocStr(FunctionName, FileName,
Line,
1387 Column, SrcLocStrSize);
1391 SrcLocStr, SrcLocStrSize, llvm::omp::IdentFlag(Flags), Reserved2Flags);
1396 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1402 uint32_t SrcLocStrSize;
1403 auto *SrcLocStr =
OMPBuilder.getOrCreateSrcLocStr(
1406 OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize));
1409 llvm::Value *ThreadID =
nullptr;
1412 auto I = OpenMPLocThreadIDMap.find(CGF.
CurFn);
1413 if (I != OpenMPLocThreadIDMap.end()) {
1414 ThreadID = I->second.ThreadID;
1415 if (ThreadID !=
nullptr)
1419 if (
auto *OMPRegionInfo =
1421 if (OMPRegionInfo->getThreadIDVariable()) {
1423 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1427 CGF.
Builder.GetInsertBlock() == TopBlock ||
1428 !isa<llvm::Instruction>(LVal.
getPointer(CGF)) ||
1429 cast<llvm::Instruction>(LVal.
getPointer(CGF))->getParent() ==
1431 cast<llvm::Instruction>(LVal.
getPointer(CGF))->getParent() ==
1432 CGF.
Builder.GetInsertBlock()) {
1436 if (CGF.
Builder.GetInsertBlock() == TopBlock) {
1437 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.
CurFn);
1438 Elem.second.ThreadID = ThreadID;
1449 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.
CurFn);
1450 if (!Elem.second.ServiceInsertPt)
1452 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1453 CGF.
Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1454 llvm::CallInst *Call = CGF.
Builder.CreateCall(
1456 OMPRTL___kmpc_global_thread_num),
1459 Elem.second.ThreadID = Call;
1464 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1465 if (OpenMPLocThreadIDMap.count(CGF.
CurFn)) {
1467 OpenMPLocThreadIDMap.erase(CGF.
CurFn);
1469 if (FunctionUDRMap.count(CGF.
CurFn) > 0) {
1470 for(
const auto *D : FunctionUDRMap[CGF.
CurFn])
1472 FunctionUDRMap.erase(CGF.
CurFn);
1474 auto I = FunctionUDMMap.find(CGF.
CurFn);
1475 if (I != FunctionUDMMap.end()) {
1476 for(
const auto *D : I->second)
1478 FunctionUDMMap.erase(I);
1480 LastprivateConditionalToTypes.erase(CGF.
CurFn);
1481 FunctionToUntiedTaskStackMap.erase(CGF.
CurFn);
1488 llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() {
1489 if (!Kmpc_MicroTy) {
1491 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(
CGM.
Int32Ty),
1493 Kmpc_MicroTy = llvm::FunctionType::get(
CGM.
VoidTy, MicroParams,
true);
1495 return llvm::PointerType::getUnqual(Kmpc_MicroTy);
1498 llvm::FunctionCallee
1499 CGOpenMPRuntime::createForStaticInitFunction(
unsigned IVSize,
bool IVSigned,
1500 bool IsGPUDistribute) {
1501 assert((IVSize == 32 || IVSize == 64) &&
1502 "IV size is not compatible with the omp runtime");
1504 if (IsGPUDistribute)
1505 Name = IVSize == 32 ? (IVSigned ?
"__kmpc_distribute_static_init_4"
1506 :
"__kmpc_distribute_static_init_4u")
1507 : (IVSigned ?
"__kmpc_distribute_static_init_8"
1508 :
"__kmpc_distribute_static_init_8u");
1510 Name = IVSize == 32 ? (IVSigned ?
"__kmpc_for_static_init_4"
1511 :
"__kmpc_for_static_init_4u")
1512 : (IVSigned ?
"__kmpc_for_static_init_8"
1513 :
"__kmpc_for_static_init_8u");
1516 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
1517 llvm::Type *TypeParams[] = {
1529 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1533 llvm::FunctionCallee
1534 CGOpenMPRuntime::createDispatchInitFunction(
unsigned IVSize,
bool IVSigned) {
1535 assert((IVSize == 32 || IVSize == 64) &&
1536 "IV size is not compatible with the omp runtime");
1539 ? (IVSigned ?
"__kmpc_dispatch_init_4" :
"__kmpc_dispatch_init_4u")
1540 : (IVSigned ?
"__kmpc_dispatch_init_8" :
"__kmpc_dispatch_init_8u");
1551 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1555 llvm::FunctionCallee
1556 CGOpenMPRuntime::createDispatchFiniFunction(
unsigned IVSize,
bool IVSigned) {
1557 assert((IVSize == 32 || IVSize == 64) &&
1558 "IV size is not compatible with the omp runtime");
1561 ? (IVSigned ?
"__kmpc_dispatch_fini_4" :
"__kmpc_dispatch_fini_4u")
1562 : (IVSigned ?
"__kmpc_dispatch_fini_8" :
"__kmpc_dispatch_fini_8u");
1563 llvm::Type *TypeParams[] = {
1568 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1572 llvm::FunctionCallee
1573 CGOpenMPRuntime::createDispatchNextFunction(
unsigned IVSize,
bool IVSigned) {
1574 assert((IVSize == 32 || IVSize == 64) &&
1575 "IV size is not compatible with the omp runtime");
1578 ? (IVSigned ?
"__kmpc_dispatch_next_4" :
"__kmpc_dispatch_next_4u")
1579 : (IVSigned ?
"__kmpc_dispatch_next_8" :
"__kmpc_dispatch_next_8u");
1581 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
1582 llvm::Type *TypeParams[] = {
1591 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
1599 unsigned &DeviceID,
unsigned &
FileID,
1600 unsigned &LineNum) {
1606 assert(Loc.
isValid() &&
"Source location is expected to be always valid.");
1609 assert(PLoc.
isValid() &&
"Source location is expected to be always valid.");
1611 llvm::sys::fs::UniqueID
ID;
1612 if (
auto EC = llvm::sys::fs::getUniqueID(PLoc.
getFilename(),
ID)) {
1613 PLoc =
SM.getPresumedLoc(Loc,
false);
1614 assert(PLoc.
isValid() &&
"Source location is expected to be always valid.");
1615 if (
auto EC = llvm::sys::fs::getUniqueID(PLoc.
getFilename(),
ID))
1616 SM.getDiagnostics().Report(diag::err_cannot_open_file)
1620 DeviceID =
ID.getDevice();
1629 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1630 if (Res && (*Res == OMPDeclareTargetDeclAttr::MT_Link ||
1631 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
1632 HasRequiresUnifiedSharedMemory))) {
1635 llvm::raw_svector_ostream OS(PtrName);
1642 OS << llvm::format(
"_%x",
FileID);
1644 OS <<
"_decl_tgt_ref_ptr";
1646 llvm::Value *Ptr =
CGM.
getModule().getNamedValue(PtrName);
1650 Ptr = getOrCreateInternalVariable(LlvmPtrTy, PtrName);
1652 auto *GV = cast<llvm::GlobalVariable>(Ptr);
1653 GV->setLinkage(llvm::GlobalValue::WeakAnyLinkage);
1665 CGOpenMPRuntime::getOrCreateThreadPrivateCache(
const VarDecl *VD) {
1670 return getOrCreateInternalVariable(
1683 llvm::Value *Args[] = {
1687 getOrCreateThreadPrivateCache(VD)};
1696 void CGOpenMPRuntime::emitThreadPrivateVarInit(
1707 llvm::Value *Args[] = {
1709 Ctor, CopyCtor, Dtor};
1712 CGM.
getModule(), OMPRTL___kmpc_threadprivate_register),
1727 llvm::Value *Ctor =
nullptr, *CopyCtor =
nullptr, *Dtor =
nullptr;
1737 Args.push_back(&Dst);
1743 llvm::Function *Fn =
1770 Args.push_back(&Dst);
1776 llvm::Function *Fn =
1798 auto *CopyCtorTy = llvm::FunctionType::get(
CGM.
VoidPtrTy, CopyCtorTyArgs,
1804 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
1805 if (Ctor ==
nullptr) {
1809 Ctor = llvm::Constant::getNullValue(CtorTy);
1811 if (Dtor ==
nullptr) {
1815 Dtor = llvm::Constant::getNullValue(DtorTy);
1818 auto *InitFunctionTy =
1819 llvm::FunctionType::get(
CGM.
VoidTy,
false);
1828 emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1830 return InitFunction;
1832 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1838 llvm::GlobalVariable *Addr,
1844 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1845 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
1846 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
1847 HasRequiresUnifiedSharedMemory))
1850 assert(VD &&
"Unknown VarDecl");
1867 llvm::raw_svector_ostream OS(Buffer);
1868 OS <<
"__omp_offloading_" << llvm::format(
"_%x", DeviceID)
1874 llvm::Constant *Ctor;
1884 FTy, Twine(Buffer,
"_ctor"), FI, Loc,
false,
1885 llvm::GlobalValue::WeakODRLinkage);
1887 Fn->setCallingConv(llvm::CallingConv::AMDGPU_KERNEL);
1892 llvm::Constant *AddrInAS0 = Addr;
1893 if (Addr->getAddressSpace() != 0)
1894 AddrInAS0 = llvm::ConstantExpr::getAddrSpaceCast(
1895 Addr, llvm::PointerType::getWithSamePointeeType(
1896 cast<llvm::PointerType>(Addr->getType()), 0));
1898 Address(AddrInAS0, Addr->getValueType(),
1900 Init->getType().getQualifiers(),
1906 Ctor =
new llvm::GlobalVariable(
1908 llvm::GlobalValue::PrivateLinkage,
1909 llvm::Constant::getNullValue(
CGM.
Int8Ty), Twine(Buffer,
"_ctor"));
1915 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
1916 DeviceID,
FileID, Twine(Buffer,
"_ctor").toStringRef(Out),
Line, Ctor,
1917 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryCtor);
1920 llvm::Constant *Dtor;
1930 FTy, Twine(Buffer,
"_dtor"), FI, Loc,
false,
1931 llvm::GlobalValue::WeakODRLinkage);
1933 Fn->setCallingConv(llvm::CallingConv::AMDGPU_KERNEL);
1940 llvm::Constant *AddrInAS0 = Addr;
1941 if (Addr->getAddressSpace() != 0)
1942 AddrInAS0 = llvm::ConstantExpr::getAddrSpaceCast(
1943 Addr, llvm::PointerType::getWithSamePointeeType(
1944 cast<llvm::PointerType>(Addr->getType()), 0));
1953 Dtor =
new llvm::GlobalVariable(
1955 llvm::GlobalValue::PrivateLinkage,
1956 llvm::Constant::getNullValue(
CGM.
Int8Ty), Twine(Buffer,
"_dtor"));
1961 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
1962 DeviceID,
FileID, Twine(Buffer,
"_dtor").toStringRef(Out),
Line, Dtor,
1963 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryDtor);
1973 llvm::GlobalVariable *GAddr =
1974 getOrCreateInternalVariable(VarLVType, Twine(Name).concat(Suffix));
1977 GAddr->setThreadLocal(
true);
1978 return Address(GAddr, GAddr->getValueType(),
1982 llvm::Value *Args[] = {
1988 getOrCreateInternalVariable(
1996 VarLVType->getPointerTo(0)),
2040 llvm::Function *OutlinedFn,
2043 llvm::Value *NumThreads) {
2048 auto &&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc,
2052 llvm::Value *Args[] = {
2054 CGF.Builder.getInt32(CapturedVars.size()),
2055 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())};
2057 RealArgs.append(std::begin(Args), std::end(Args));
2058 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
2060 llvm::FunctionCallee RTLFn =
2061 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_fork_call);
2062 CGF.EmitRuntimeCall(RTLFn, RealArgs);
2064 auto &&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc, Loc,
2070 llvm::Value *Args[] = {RTLoc, ThreadID};
2071 CGF.EmitRuntimeCall(
OMPBuilder.getOrCreateRuntimeFunction(
2072 M, OMPRTL___kmpc_serialized_parallel),
2078 CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
2079 ".bound.zero.addr");
2080 CGF.Builder.CreateStore(CGF.Builder.getInt32( 0), ZeroAddrBound);
2083 OutlinedFnArgs.push_back(ThreadIDAddr.
getPointer());
2084 OutlinedFnArgs.push_back(ZeroAddrBound.
getPointer());
2085 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
2093 OutlinedFn->removeFnAttr(llvm::Attribute::AlwaysInline);
2094 OutlinedFn->addFnAttr(llvm::Attribute::NoInline);
2099 CGF.EmitRuntimeCall(
OMPBuilder.getOrCreateRuntimeFunction(
2100 M, OMPRTL___kmpc_end_serialized_parallel),
2119 if (
auto *OMPRegionInfo =
2121 if (OMPRegionInfo->getThreadIDVariable())
2122 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(CGF);
2131 return ThreadIDTemp;
2134 llvm::GlobalVariable *CGOpenMPRuntime::getOrCreateInternalVariable(
2135 llvm::Type *Ty,
const llvm::Twine &Name,
unsigned AddressSpace) {
2137 llvm::raw_svector_ostream Out(Buffer);
2139 StringRef RuntimeName = Out.str();
2140 auto &Elem = *InternalVars.try_emplace(RuntimeName,
nullptr).first;
2142 assert(Elem.second->getType()->isOpaqueOrPointeeTypeMatches(Ty) &&
2143 "OMP internal variable has different type than requested");
2144 return &*Elem.second;
2147 return Elem.second =
new llvm::GlobalVariable(
2149 llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
2150 Elem.first(),
nullptr,
2151 llvm::GlobalValue::NotThreadLocal, AddressSpace);
2155 std::string Prefix = Twine(
"gomp_critical_user_", CriticalName).str();
2157 return getOrCreateInternalVariable(KmpCriticalNameTy, Name);
2163 llvm::FunctionCallee EnterCallee;
2165 llvm::FunctionCallee ExitCallee;
2168 llvm::BasicBlock *ContBlock =
nullptr;
2171 CommonActionTy(llvm::FunctionCallee EnterCallee,
2173 llvm::FunctionCallee ExitCallee,
2175 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
2180 llvm::Value *CallBool = CGF.
Builder.CreateIsNotNull(EnterRes);
2184 CGF.
Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
2200 StringRef CriticalName,
2214 EnterArgs.push_back(CGF.
Builder.CreateIntCast(
2217 CommonActionTy Action(
2220 Hint ? OMPRTL___kmpc_critical_with_hint : OMPRTL___kmpc_critical),
2223 OMPRTL___kmpc_end_critical),
2240 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2262 llvm::Value *FilterVal =
Filter
2269 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2289 llvm::Value *Args[] = {
2291 llvm::ConstantInt::get(
CGM.
IntTy, 0,
true)};
2297 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
2298 Region->emitUntiedSwitch(CGF);
2311 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2324 unsigned Index,
const VarDecl *Var) {
2332 Ptr, ElemTy->getPointerTo(Ptr->getType()->getPointerAddressSpace())),
2348 Args.push_back(&LHSArg);
2349 Args.push_back(&RHSArg);
2358 Fn->setDoesNotRecurse();
2365 ArgsElemType->getPointerTo()),
2369 ArgsElemType->getPointerTo()),
2375 for (
unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
2376 const auto *DestVar =
2377 cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
2380 const auto *SrcVar =
2381 cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
2384 const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
2386 CGF.
EmitOMPCopy(
Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
2401 assert(CopyprivateVars.size() == SrcExprs.size() &&
2402 CopyprivateVars.size() == DstExprs.size() &&
2403 CopyprivateVars.size() == AssignmentOps.size());
2415 if (!CopyprivateVars.empty()) {
2418 C.getIntTypeForBitwidth(32, 1);
2419 DidIt = CGF.
CreateMemTemp(KmpInt32Ty,
".omp.copyprivate.did_it");
2424 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2441 llvm::APInt ArraySize(32, CopyprivateVars.size());
2442 QualType CopyprivateArrayTy = C.getConstantArrayType(
2447 CGF.
CreateMemTemp(CopyprivateArrayTy,
".omp.copyprivate.cpr_list");
2448 for (
unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
2460 SrcExprs, DstExprs, AssignmentOps, Loc);
2461 llvm::Value *BufSize = CGF.
getTypeSize(CopyprivateArrayTy);
2465 llvm::Value *Args[] = {
2490 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2505 if (
Kind == OMPD_for)
2506 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
2507 else if (
Kind == OMPD_sections)
2508 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
2509 else if (
Kind == OMPD_single)
2510 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
2511 else if (
Kind == OMPD_barrier)
2512 Flags = OMP_IDENT_BARRIER_EXPL;
2514 Flags = OMP_IDENT_BARRIER_IMPL;
2526 ScheduleKind = OMPC_SCHEDULE_static;
2538 bool ForceSimpleCall) {
2540 auto *OMPRegionInfo =
2557 if (OMPRegionInfo) {
2558 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
2561 OMPRTL___kmpc_cancel_barrier),
2569 llvm::Value *Cmp = CGF.
Builder.CreateIsNotNull(Result);
2570 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
2588 bool Chunked,
bool Ordered) {
2589 switch (ScheduleKind) {
2590 case OMPC_SCHEDULE_static:
2591 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
2592 : (Ordered ? OMP_ord_static : OMP_sch_static);
2593 case OMPC_SCHEDULE_dynamic:
2594 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
2595 case OMPC_SCHEDULE_guided:
2596 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
2597 case OMPC_SCHEDULE_runtime:
2598 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
2599 case OMPC_SCHEDULE_auto:
2600 return Ordered ? OMP_ord_auto : OMP_sch_auto;
2602 assert(!Chunked &&
"chunk was specified but schedule kind not known");
2603 return Ordered ? OMP_ord_static : OMP_sch_static;
2605 llvm_unreachable(
"Unexpected runtime schedule");
2609 static OpenMPSchedType
2612 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
2616 bool Chunked)
const {
2617 OpenMPSchedType Schedule =
2619 return Schedule == OMP_sch_static;
2625 return Schedule == OMP_dist_sch_static;
2629 bool Chunked)
const {
2630 OpenMPSchedType Schedule =
2632 return Schedule == OMP_sch_static_chunked;
2638 return Schedule == OMP_dist_sch_static_chunked;
2642 OpenMPSchedType Schedule =
2644 assert(Schedule != OMP_sch_static_chunked &&
"cannot be chunked here");
2645 return Schedule != OMP_sch_static;
2653 case OMPC_SCHEDULE_MODIFIER_monotonic:
2654 Modifier = OMP_sch_modifier_monotonic;
2656 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2657 Modifier = OMP_sch_modifier_nonmonotonic;
2659 case OMPC_SCHEDULE_MODIFIER_simd:
2660 if (Schedule == OMP_sch_static_chunked)
2661 Schedule = OMP_sch_static_balanced_chunked;
2668 case OMPC_SCHEDULE_MODIFIER_monotonic:
2669 Modifier = OMP_sch_modifier_monotonic;
2671 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2672 Modifier = OMP_sch_modifier_nonmonotonic;
2674 case OMPC_SCHEDULE_MODIFIER_simd:
2675 if (Schedule == OMP_sch_static_chunked)
2676 Schedule = OMP_sch_static_balanced_chunked;
2688 if (CGM.
getLangOpts().OpenMP >= 50 && Modifier == 0) {
2689 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
2690 Schedule == OMP_sch_static_balanced_chunked ||
2691 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||
2692 Schedule == OMP_dist_sch_static_chunked ||
2693 Schedule == OMP_dist_sch_static))
2694 Modifier = OMP_sch_modifier_nonmonotonic;
2696 return Schedule | Modifier;
2706 ScheduleKind.
Schedule, DispatchValues.
Chunk !=
nullptr, Ordered);
2708 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
2709 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&
2710 Schedule != OMP_sch_static_balanced_chunked));
2717 llvm::Value *Chunk = DispatchValues.
Chunk ? DispatchValues.
Chunk
2718 : CGF.
Builder.getIntN(IVSize, 1);
2719 llvm::Value *Args[] = {
2723 CGM, Schedule, ScheduleKind.
M1, ScheduleKind.
M2)),
2726 CGF.
Builder.getIntN(IVSize, 1),
2729 CGF.
EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args);
2733 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
2734 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
2741 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||
2742 Schedule == OMP_sch_static_balanced_chunked ||
2743 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||
2744 Schedule == OMP_dist_sch_static ||
2745 Schedule == OMP_dist_sch_static_chunked);
2752 llvm::Value *Chunk = Values.
Chunk;
2753 if (Chunk ==
nullptr) {
2754 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
2755 Schedule == OMP_dist_sch_static) &&
2756 "expected static non-chunked schedule");
2760 assert((Schedule == OMP_sch_static_chunked ||
2761 Schedule == OMP_sch_static_balanced_chunked ||
2762 Schedule == OMP_ord_static_chunked ||
2763 Schedule == OMP_dist_sch_static_chunked) &&
2764 "expected static chunked schedule");
2766 llvm::Value *Args[] = {
2789 "Expected loop-based or sections-based directive.");
2792 ? OMP_IDENT_WORK_LOOP
2793 : OMP_IDENT_WORK_SECTIONS);
2795 llvm::FunctionCallee StaticInitFunction =
2796 createForStaticInitFunction(Values.
IVSize, Values.
IVSigned,
false);
2799 ScheduleNum, ScheduleKind.
M1, ScheduleKind.
M2, Values);
2806 OpenMPSchedType ScheduleNum =
2808 llvm::Value *UpdatedLocation =
2811 llvm::FunctionCallee StaticInitFunction;
2812 bool isGPUDistribute =
2815 StaticInitFunction = createForStaticInitFunction(
2829 llvm::Value *Args[] = {
2832 ? OMP_IDENT_WORK_DISTRIBUTE
2834 ? OMP_IDENT_WORK_LOOP
2835 : OMP_IDENT_WORK_SECTIONS),
2842 CGM.
getModule(), OMPRTL___kmpc_distribute_static_fini),
2858 CGF.
EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args);
2870 llvm::Value *Args[] = {
2879 CGF.
EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args);
2886 llvm::Value *NumThreads,
2891 llvm::Value *Args[] = {
2900 ProcBindKind ProcBind,
2904 assert(ProcBind != OMP_PROC_BIND_unknown &&
"Unsupported proc_bind value.");
2906 llvm::Value *Args[] = {
2908 llvm::ConstantInt::get(
CGM.
IntTy,
unsigned(ProcBind),
true)};
2930 enum KmpTaskTFields {
2954 bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty()
const {
2955 return OffloadEntriesTargetRegion.empty() &&
2956 OffloadEntriesDeviceGlobalVar.empty();
2960 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
2961 initializeTargetRegionEntryInfo(
unsigned DeviceID,
unsigned FileID,
2962 StringRef ParentName,
unsigned LineNum,
2964 assert(CGM.getLangOpts().OpenMPIsDevice &&
"Initialization of entries is "
2965 "only required for the device "
2966 "code generation.");
2967 OffloadEntriesTargetRegion[DeviceID][
FileID][ParentName][LineNum] =
2968 OffloadEntryInfoTargetRegion(Order,
nullptr,
nullptr,
2969 OMPTargetRegionEntryTargetRegion);
2970 ++OffloadingEntriesNum;
2973 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
2974 registerTargetRegionEntryInfo(
unsigned DeviceID,
unsigned FileID,
2975 StringRef ParentName,
unsigned LineNum,
2976 llvm::Constant *Addr, llvm::Constant *
ID,
2977 OMPTargetRegionEntryKind Flags) {
2980 if (CGM.getLangOpts().OpenMPIsDevice) {
2982 if (!hasTargetRegionEntryInfo(DeviceID,
FileID, ParentName, LineNum))
2985 OffloadEntriesTargetRegion[DeviceID][
FileID][ParentName][LineNum];
2986 Entry.setAddress(Addr);
2988 Entry.setFlags(Flags);
2991 OffloadEntriesInfoManagerTy::OMPTargetRegionEntryTargetRegion &&
2992 hasTargetRegionEntryInfo(DeviceID,
FileID, ParentName, LineNum,
2995 assert(!hasTargetRegionEntryInfo(DeviceID,
FileID, ParentName, LineNum) &&
2996 "Target region entry already registered!");
2997 OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr,
ID, Flags);
2998 OffloadEntriesTargetRegion[DeviceID][
FileID][ParentName][LineNum] = Entry;
2999 ++OffloadingEntriesNum;
3003 bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo(
3004 unsigned DeviceID,
unsigned FileID, StringRef ParentName,
unsigned LineNum,
3005 bool IgnoreAddressId)
const {
3006 auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID);
3007 if (PerDevice == OffloadEntriesTargetRegion.end())
3009 auto PerFile = PerDevice->second.find(
FileID);
3010 if (PerFile == PerDevice->second.end())
3012 auto PerParentName = PerFile->second.find(ParentName);
3013 if (PerParentName == PerFile->second.end())
3015 auto PerLine = PerParentName->second.find(LineNum);
3016 if (PerLine == PerParentName->second.end())
3019 if (!IgnoreAddressId &&
3020 (PerLine->second.getAddress() || PerLine->second.getID()))
3025 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo(
3026 const OffloadTargetRegionEntryInfoActTy &Action) {
3028 for (
const auto &D : OffloadEntriesTargetRegion)
3029 for (
const auto &F : D.second)
3030 for (
const auto &
P : F.second)
3031 for (
const auto &L :
P.second)
3032 Action(D.first, F.first,
P.first(), L.first, L.second);
3035 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3036 initializeDeviceGlobalVarEntryInfo(StringRef Name,
3037 OMPTargetGlobalVarEntryKind Flags,
3039 assert(CGM.getLangOpts().OpenMPIsDevice &&
"Initialization of entries is "
3040 "only required for the device "
3041 "code generation.");
3042 OffloadEntriesDeviceGlobalVar.try_emplace(Name, Order, Flags);
3043 ++OffloadingEntriesNum;
3046 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3047 registerDeviceGlobalVarEntryInfo(StringRef VarName, llvm::Constant *Addr,
3049 OMPTargetGlobalVarEntryKind Flags,
3050 llvm::GlobalValue::LinkageTypes
Linkage) {
3051 if (CGM.getLangOpts().OpenMPIsDevice) {
3053 if (!hasDeviceGlobalVarEntryInfo(VarName))
3055 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3056 if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {
3057 if (Entry.getVarSize().isZero()) {
3058 Entry.setVarSize(VarSize);
3063 Entry.setVarSize(VarSize);
3065 Entry.setAddress(Addr);
3067 if (hasDeviceGlobalVarEntryInfo(VarName)) {
3068 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3069 assert(Entry.isValid() && Entry.getFlags() == Flags &&
3070 "Entry not initialized!");
3071 if (Entry.getVarSize().isZero()) {
3072 Entry.setVarSize(VarSize);
3077 OffloadEntriesDeviceGlobalVar.try_emplace(
3078 VarName, OffloadingEntriesNum, Addr, VarSize, Flags,
Linkage);
3079 ++OffloadingEntriesNum;
3083 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3084 actOnDeviceGlobalVarEntriesInfo(
3085 const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
3087 for (
const auto &E : OffloadEntriesDeviceGlobalVar)
3088 Action(E.getKey(), E.getValue());
3092 llvm::Constant *
ID, llvm::Constant *Addr, uint64_t Size, int32_t Flags,
3093 llvm::GlobalValue::LinkageTypes
Linkage) {
3094 OMPBuilder.emitOffloadingEntry(
ID, Addr->getName(), Size, Flags);
3109 if (
CGM.
getLangOpts().OpenMPSimd || OffloadEntriesInfoManager.empty())
3113 llvm::LLVMContext &C = M.getContext();
3117 OrderedEntries(OffloadEntriesInfoManager.size());
3119 OffloadEntriesInfoManager.size());
3122 auto &&GetMDInt = [
this](
unsigned V) {
3123 return llvm::ConstantAsMetadata::get(
3127 auto &&GetMDString = [&C](StringRef
V) {
return llvm::MDString::get(C,
V); };
3130 llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata(
"omp_offload.info");
3133 auto &&TargetRegionMetadataEmitter =
3134 [
this, &C, MD, &OrderedEntries, &ParentFunctions, &GetMDInt,
3136 unsigned DeviceID,
unsigned FileID, StringRef ParentName,
3149 llvm::Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDInt(DeviceID),
3150 GetMDInt(
FileID), GetMDString(ParentName),
3151 GetMDInt(
Line), GetMDInt(E.getOrder())};
3157 if (I->getFirst()->getUniqueID().getDevice() == DeviceID &&
3158 I->getFirst()->getUniqueID().getFile() ==
FileID) {
3160 I->getFirst(),
Line, 1);
3165 OrderedEntries[E.getOrder()] = std::make_tuple(&E, Loc, ParentName);
3166 ParentFunctions[E.getOrder()] = ParentName;
3169 MD->addOperand(llvm::MDNode::get(C, Ops));
3172 OffloadEntriesInfoManager.actOnTargetRegionEntriesInfo(
3173 TargetRegionMetadataEmitter);
3176 auto &&DeviceGlobalVarMetadataEmitter =
3177 [&C, &OrderedEntries, &GetMDInt, &GetMDString,
3178 MD](StringRef MangledName,
3188 llvm::Metadata *Ops[] = {
3189 GetMDInt(E.getKind()), GetMDString(MangledName),
3190 GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
3193 OrderedEntries[E.getOrder()] =
3197 MD->addOperand(llvm::MDNode::get(C, Ops));
3200 OffloadEntriesInfoManager.actOnDeviceGlobalVarEntriesInfo(
3201 DeviceGlobalVarMetadataEmitter);
3203 for (
const auto &E : OrderedEntries) {
3204 assert(std::get<0>(E) &&
"All ordered entries must exist!");
3205 if (
const auto *CE =
3206 dyn_cast<OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion>(
3208 if (!CE->getID() || !CE->getAddress()) {
3210 StringRef FnName = ParentFunctions[CE->getOrder()];
3215 "Offloading entry for target region in %0 is incorrect: either the "
3216 "address or the ID is invalid.");
3221 CE->getFlags(), llvm::GlobalValue::WeakAnyLinkage);
3222 }
else if (
const auto *CE = dyn_cast<OffloadEntriesInfoManagerTy::
3223 OffloadEntryInfoDeviceGlobalVar>(
3225 OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind Flags =
3226 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind
>(
3229 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo: {
3233 if (!CE->getAddress()) {
3236 "variable %0 is incorrect: the "
3237 "address is invalid.");
3242 if (CE->getVarSize().isZero())
3246 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryLink:
3247 assert(((
CGM.
getLangOpts().OpenMPIsDevice && !CE->getAddress()) ||
3249 "Declaret target link address is set.");
3252 if (!CE->getAddress()) {
3255 "Offloading entry for declare target variable is incorrect: the "
3256 "address is invalid.");
3265 if (
auto *GV = dyn_cast<llvm::GlobalValue>(CE->getAddress()))
3266 if (GV->hasLocalLinkage() || GV->hasHiddenVisibility())
3270 CE->getVarSize().getQuantity(), Flags,
3273 llvm_unreachable(
"Unsupported entry kind.");
3280 void CGOpenMPRuntime::loadOffloadInfoMetadata() {
3291 if (
auto EC = Buf.getError()) {
3297 llvm::LLVMContext C;
3298 auto ME = expectedToErrorOrAndEmitErrors(
3299 C, llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(), C));
3301 if (
auto EC = ME.getError()) {
3309 llvm::NamedMDNode *MD = ME.get()->getNamedMetadata(
"omp_offload.info");
3313 for (llvm::MDNode *MN : MD->operands()) {
3314 auto &&GetMDInt = [MN](
unsigned Idx) {
3315 auto *
V = cast<llvm::ConstantAsMetadata>(MN->getOperand(Idx));
3316 return cast<llvm::ConstantInt>(
V->getValue())->getZExtValue();
3319 auto &&GetMDString = [MN](
unsigned Idx) {
3320 auto *
V = cast<llvm::MDString>(MN->getOperand(Idx));
3321 return V->getString();
3324 switch (GetMDInt(0)) {
3326 llvm_unreachable(
"Unexpected metadata!");
3330 OffloadEntriesInfoManager.initializeTargetRegionEntryInfo(
3331 GetMDInt(1), GetMDInt(2),
3332 GetMDString(3), GetMDInt(4),
3337 OffloadEntriesInfoManager.initializeDeviceGlobalVarEntryInfo(
3339 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind
>(
3347 void CGOpenMPRuntime::emitKmpRoutineEntryT(
QualType KmpInt32Ty) {
3348 if (!KmpRoutineEntryPtrTy) {
3351 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty,
C.VoidPtrTy};
3353 KmpRoutineEntryPtrQTy =
C.getPointerType(
3354 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
3360 struct PrivateHelpersTy {
3361 PrivateHelpersTy(
const Expr *OriginalRef,
const VarDecl *Original,
3363 : OriginalRef(OriginalRef), Original(Original), PrivateCopy(PrivateCopy),
3364 PrivateElemInit(PrivateElemInit) {}
3365 PrivateHelpersTy(
const VarDecl *Original) : Original(Original) {}
3366 const Expr *OriginalRef =
nullptr;
3367 const VarDecl *Original =
nullptr;
3368 const VarDecl *PrivateCopy =
nullptr;
3369 const VarDecl *PrivateElemInit =
nullptr;
3370 bool isLocalPrivate()
const {
3371 return !OriginalRef && !PrivateCopy && !PrivateElemInit;
3374 typedef std::pair<
CharUnits , PrivateHelpersTy> PrivateDataTy;
3379 if (!CVD->
hasAttr<OMPAllocateDeclAttr>())
3381 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
3383 return !(AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc &&
3384 !AA->getAllocator());
3389 if (!Privates.empty()) {
3394 RecordDecl *RD = C.buildImplicitRecord(
".kmp_privates.t");
3396 for (
const auto &Pair : Privates) {
3397 const VarDecl *VD = Pair.second.Original;
3401 if (Pair.second.isLocalPrivate()) {
3424 QualType KmpRoutineEntryPointerQTy) {
3444 QualType KmpCmplrdataTy = C.getRecordType(UD);
3445 RecordDecl *RD = C.buildImplicitRecord(
"kmp_task_t");
3475 RecordDecl *RD = C.buildImplicitRecord(
"kmp_task_t_with_privates");
3495 static llvm::Function *
3498 QualType KmpTaskTWithPrivatesPtrQTy,
3500 QualType SharedsPtrTy, llvm::Function *TaskFunction,
3501 llvm::Value *TaskPrivatesMap) {
3509 Args.push_back(&GtidArg);
3510 Args.push_back(&TaskTypeArg);
3511 const auto &TaskEntryFnInfo =
3513 llvm::FunctionType *TaskEntryTy =
3516 auto *TaskEntry = llvm::Function::Create(
3519 TaskEntry->setDoesNotRecurse();
3534 const auto *KmpTaskTWithPrivatesQTyRD =
3535 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
3538 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->
getAsTagDecl());
3539 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
3541 llvm::Value *PartidParam = PartIdLVal.
getPointer(CGF);
3543 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
3549 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
3550 llvm::Value *PrivatesParam;
3551 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
3556 PrivatesParam = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
3559 llvm::Value *CommonArgs[] = {
3560 GtidParam, PartidParam, PrivatesParam, TaskPrivatesMap,
3566 std::end(CommonArgs));
3568 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
3571 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
3574 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
3577 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
3580 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
3583 CallArgs.push_back(LBParam);
3584 CallArgs.push_back(UBParam);
3585 CallArgs.push_back(StParam);
3586 CallArgs.push_back(LIParam);
3587 CallArgs.push_back(RParam);
3589 CallArgs.push_back(SharedsParam);
3602 QualType KmpTaskTWithPrivatesPtrQTy,
3603 QualType KmpTaskTWithPrivatesQTy) {
3611 Args.push_back(&GtidArg);
3612 Args.push_back(&TaskTypeArg);
3613 const auto &DestructorFnInfo =
3615 llvm::FunctionType *DestructorFnTy =
3619 auto *DestructorFn =
3624 DestructorFn->setDoesNotRecurse();
3632 const auto *KmpTaskTWithPrivatesQTyRD =
3633 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
3634 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
3636 for (
const auto *Field :
3637 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
3639 Field->getType().isDestructedType()) {
3645 return DestructorFn;
3658 static llvm::Value *
3665 C,
nullptr, Loc,
nullptr,
3666 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
3668 Args.push_back(&TaskPrivatesArg);
3669 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>,
unsigned> PrivateVarsPos;
3670 unsigned Counter = 1;
3673 C,
nullptr, Loc,
nullptr,
3674 C.getPointerType(C.getPointerType(E->
getType()))
3678 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3679 PrivateVarsPos[VD] = Counter;
3684 C,
nullptr, Loc,
nullptr,
3685 C.getPointerType(C.getPointerType(E->
getType()))
3689 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3690 PrivateVarsPos[VD] = Counter;
3695 C,
nullptr, Loc,
nullptr,
3696 C.getPointerType(C.getPointerType(E->
getType()))
3700 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3701 PrivateVarsPos[VD] = Counter;
3707 Ty = C.getPointerType(Ty);
3709 Ty = C.getPointerType(Ty);
3711 C,
nullptr, Loc,
nullptr,
3712 C.getPointerType(C.getPointerType(Ty)).withConst().withRestrict(),
3714 PrivateVarsPos[VD] = Counter;
3717 const auto &TaskPrivatesMapFnInfo =
3719 llvm::FunctionType *TaskPrivatesMapTy =
3723 auto *TaskPrivatesMap = llvm::Function::Create(
3727 TaskPrivatesMapFnInfo);
3729 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
3730 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
3731 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
3735 TaskPrivatesMapFnInfo, Args, Loc, Loc);
3741 const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->
getAsTagDecl());
3743 for (
const FieldDecl *Field : PrivatesQTyRD->fields()) {
3745 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
3754 return TaskPrivatesMap;
3766 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
3781 (IsTargetTask && KmpTaskSharedsPtr.
isValid())) {
3788 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
3789 for (
const PrivateDataTy &Pair : Privates) {
3791 if (Pair.second.isLocalPrivate()) {
3795 const VarDecl *VD = Pair.second.PrivateCopy;
3797 if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&
3800 if (
const VarDecl *Elem = Pair.second.PrivateElemInit) {
3801 const VarDecl *OriginalVD = Pair.second.Original;
3807 if (IsTargetTask && !SharedField) {
3808 assert(isa<ImplicitParamDecl>(OriginalVD) &&
3811 ->getNumParams() == 0 &&
3812 isa<TranslationUnitDecl>(
3814 ->getDeclContext()) &&
3815 "Expected artificial target data variable.");
3818 }
else if (ForDup) {
3822 C.getDeclAlign(OriginalVD)),
3826 Pair.second.Original->getCanonicalDecl()) > 0 ||
3828 SharedRefLValue = CGF.
EmitLValue(Pair.second.OriginalRef);
3831 InlinedOpenMPRegionRAII Region(
3834 SharedRefLValue = CGF.
EmitLValue(Pair.second.OriginalRef);
3847 [&CGF, Elem, Init, &CapturesInfo](
Address DestElement,
3850 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3851 InitScope.addPrivate(Elem, SrcElement);
3852 (void)InitScope.Privatize();
3854 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
3855 CGF, &CapturesInfo);
3856 CGF.EmitAnyExprToMem(Init, DestElement,
3857 Init->getType().getQualifiers(),
3880 bool InitRequired =
false;
3881 for (
const PrivateDataTy &Pair : Privates) {
3882 if (Pair.second.isLocalPrivate())
3884 const VarDecl *VD = Pair.second.PrivateCopy;
3886 InitRequired = InitRequired || (isa_and_nonnull<CXXConstructExpr>(Init) &&
3891 return InitRequired;
3905 static llvm::Value *
3908 QualType KmpTaskTWithPrivatesPtrQTy,
3916 KmpTaskTWithPrivatesPtrQTy,
3919 KmpTaskTWithPrivatesPtrQTy,
3923 Args.push_back(&DstArg);
3924 Args.push_back(&SrcArg);
3925 Args.push_back(&LastprivArg);
3926 const auto &TaskDupFnInfo =
3930 auto *TaskDup = llvm::Function::Create(
3933 TaskDup->setDoesNotRecurse();
3943 auto LIFI = std::next(KmpTaskTQTyRD->
field_begin(), KmpTaskTLastIter);
3945 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
3953 assert(!Privates.empty());
3960 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
3968 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
3969 SharedsTy, SharedsPtrTy, Data, Privates,
true);
3979 for (
const PrivateDataTy &
P : Privates) {
3980 if (
P.second.isLocalPrivate())
3982 QualType Ty =
P.second.Original->getType().getNonReferenceType();
3991 class OMPIteratorGeneratorScope final
3997 OMPIteratorGeneratorScope() =
delete;
3998 OMPIteratorGeneratorScope(OMPIteratorGeneratorScope &) =
delete;
4031 llvm::Value *N = Uppers[I];
4039 ? CGF.
Builder.CreateICmpSLT(CVal, N)
4040 : CGF.
Builder.CreateICmpULT(CVal, N);
4049 ~OMPIteratorGeneratorScope() {
4059 CGF.
EmitBlock(ExitDests[I - 1].getBlock(), I == 1);
4065 static std::pair<llvm::Value *, llvm::Value *>
4067 const auto *OASE = dyn_cast<OMPArrayShapingExpr>(E);
4070 const Expr *
Base = OASE->getBase();
4075 llvm::Value *SizeVal;
4078 SizeVal = CGF.
getTypeSize(OASE->getBase()->getType()->getPointeeType());
4079 for (
const Expr *SE : OASE->getDimensions()) {
4083 SizeVal = CGF.
Builder.CreateNUWMul(SizeVal, Sz);
4085 }
else if (
const auto *ASE =
4090 llvm::Value *UpAddr = CGF.
Builder.CreateConstGEP1_32(
4092 llvm::Value *LowIntPtr = CGF.
Builder.CreatePtrToInt(Addr, CGF.
SizeTy);
4093 llvm::Value *UpIntPtr = CGF.
Builder.CreatePtrToInt(UpAddr, CGF.
SizeTy);
4094 SizeVal = CGF.
Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
4098 return std::make_pair(Addr, SizeVal);
4103 QualType FlagsTy = C.getIntTypeForBitwidth(32,
false);
4104 if (KmpTaskAffinityInfoTy.
isNull()) {
4106 C.buildImplicitRecord(
"kmp_task_affinity_info_t");
4112 KmpTaskAffinityInfoTy = C.getRecordType(KmpAffinityInfoRD);
4116 CGOpenMPRuntime::TaskResultTy
4119 llvm::Function *TaskFunction,
QualType SharedsTy,
4126 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4127 Privates.emplace_back(
4129 PrivateHelpersTy(E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4136 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4137 Privates.emplace_back(
4140 E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4141 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
4147 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4148 Privates.emplace_back(
4150 PrivateHelpersTy(E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4158 Privates.emplace_back(
C.getDeclAlign(VD), PrivateHelpersTy(VD));
4160 llvm::stable_sort(Privates,
4161 [](
const PrivateDataTy &L,
const PrivateDataTy &R) {
4162 return L.first > R.first;
4164 QualType KmpInt32Ty =
C.getIntTypeForBitwidth(32, 1);
4166 emitKmpRoutineEntryT(KmpInt32Ty);
4169 if (SavedKmpTaskloopTQTy.
isNull()) {
4173 KmpTaskTQTy = SavedKmpTaskloopTQTy;
4178 "Expected taskloop, task or target directive");
4179 if (SavedKmpTaskTQTy.
isNull()) {
4183 KmpTaskTQTy = SavedKmpTaskTQTy;
4185 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->
getAsTagDecl());
4187 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
4189 QualType KmpTaskTWithPrivatesQTy =
C.getRecordType(KmpTaskTWithPrivatesQTyRD);
4190 QualType KmpTaskTWithPrivatesPtrQTy =
4191 C.getPointerType(KmpTaskTWithPrivatesQTy);
4192 llvm::Type *KmpTaskTWithPrivatesTy = CGF.
ConvertType(KmpTaskTWithPrivatesQTy);
4193 llvm::Type *KmpTaskTWithPrivatesPtrTy =
4194 KmpTaskTWithPrivatesTy->getPointerTo();
4195 llvm::Value *KmpTaskTWithPrivatesTySize =
4197 QualType SharedsPtrTy =
C.getPointerType(SharedsTy);
4200 llvm::Value *TaskPrivatesMap =
nullptr;
4201 llvm::Type *TaskPrivatesMapTy =
4202 std::next(TaskFunction->arg_begin(), 3)->getType();
4203 if (!Privates.empty()) {
4204 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
4208 TaskPrivatesMap, TaskPrivatesMapTy);
4210 TaskPrivatesMap = llvm::ConstantPointerNull::get(
4211 cast<llvm::PointerType>(TaskPrivatesMapTy));
4217 KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction,
4229 DestructorsFlag = 0x8,
4230 PriorityFlag = 0x20,
4231 DetachableFlag = 0x40,
4233 unsigned Flags = Data.
Tied ? TiedFlag : 0;
4234 bool NeedsCleanup =
false;
4235 if (!Privates.empty()) {
4239 Flags = Flags | DestructorsFlag;
4242 Flags = Flags | PriorityFlag;
4244 Flags = Flags | DetachableFlag;
4245 llvm::Value *TaskFlags =
4246 Data.
Final.getPointer()
4248 CGF.
Builder.getInt32(FinalFlag),
4250 : CGF.
Builder.getInt32(Data.
Final.getInt() ? FinalFlag : 0);
4251 TaskFlags = CGF.
Builder.CreateOr(TaskFlags, CGF.
Builder.getInt32(Flags));
4252 llvm::Value *SharedsSize =
CGM.
getSize(
C.getTypeSizeInChars(SharedsTy));
4254 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
4256 TaskEntry, KmpRoutineEntryPtrTy)};
4257 llvm::Value *NewTask;
4260 const Expr *Device =
nullptr;
4262 Device =
C->getDevice();
4264 llvm::Value *DeviceID;
4269 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
4270 AllocArgs.push_back(DeviceID);
4291 llvm::Value *Tid =
getThreadID(CGF, DC->getBeginLoc());
4292 Tid = CGF.
Builder.CreateIntCast(Tid, CGF.
IntTy,
false);
4295 CGM.
getModule(), OMPRTL___kmpc_task_allow_completion_event),
4296 {Loc, Tid, NewTask});
4307 llvm::Value *NumOfElements =
nullptr;
4308 unsigned NumAffinities = 0;
4310 if (
const Expr *Modifier =
C->getModifier()) {
4311 const auto *IE = cast<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts());
4312 for (
unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4316 NumOfElements ? CGF.
Builder.CreateNUWMul(NumOfElements, Sz) : Sz;
4319 NumAffinities +=
C->varlist_size();
4324 enum RTLAffinityInfoFieldsTy { BaseAddr, Len, Flags };
4326 QualType KmpTaskAffinityInfoArrayTy;
4327 if (NumOfElements) {
4328 NumOfElements = CGF.
Builder.CreateNUWAdd(
4329 llvm::ConstantInt::get(CGF.
SizeTy, NumAffinities), NumOfElements);
4332 C.getIntTypeForBitwidth(
C.getTypeSize(
C.getSizeType()), 0),
4336 KmpTaskAffinityInfoArrayTy =
4344 NumOfElements = CGF.
Builder.CreateIntCast(NumOfElements, CGF.
Int32Ty,
4347 KmpTaskAffinityInfoArrayTy =
C.getConstantArrayType(
4348 KmpTaskAffinityInfoTy,
4349 llvm::APInt(
C.getTypeSize(
C.getSizeType()), NumAffinities),
nullptr,
4352 CGF.
CreateMemTemp(KmpTaskAffinityInfoArrayTy,
".affs.arr.addr");
4354 NumOfElements = llvm::ConstantInt::get(
CGM.
Int32Ty, NumAffinities,
4358 const auto *KmpAffinityInfoRD = KmpTaskAffinityInfoTy->
getAsRecordDecl();
4361 bool HasIterator =
false;
4363 if (
C->getModifier()) {
4367 for (
const Expr *E :
C->varlists()) {
4373 KmpTaskAffinityInfoTy);
4376 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
4381 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
4395 const Expr *Modifier =
C->getModifier();
4398 OMPIteratorGeneratorScope IteratorScope(
4400 for (
const Expr *E :
C->varlists()) {
4409 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
4414 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
4416 Idx = CGF.
Builder.CreateNUWAdd(
4417 Idx, llvm::ConstantInt::get(Idx->getType(), 1));
4432 CGM.
getModule(), OMPRTL___kmpc_omp_reg_task_with_affinity),
4433 {LocRef, GTid, NewTask, NumOfElements, AffinListPtr});
4435 llvm::Value *NewTaskNewTaskTTy =
4437 NewTask, KmpTaskTWithPrivatesPtrTy);
4439 KmpTaskTWithPrivatesQTy);
4450 *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds)),
4458 TaskResultTy Result;
4459 if (!Privates.empty()) {
4461 SharedsTy, SharedsPtrTy, Data, Privates,
4466 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
4467 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
4472 enum {
Priority = 0, Destructors = 1 };
4474 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
4476 (*FI)->getType()->getAsUnionType()->getDecl();
4479 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4480 KmpTaskTWithPrivatesQTy);
4483 Data1LV, *std::next(KmpCmplrdataUD->
field_begin(), Destructors));
4485 DestructorFn, KmpRoutineEntryPtrTy),
4491 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
4496 Result.NewTask = NewTask;
4497 Result.TaskEntry = TaskEntry;
4498 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
4499 Result.TDBase = TDBase;
4500 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
4506 enum RTLDependenceKindTy {
4509 DepMutexInOutSet = 0x4,
4513 enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags };
4518 RTLDependenceKindTy DepKind;
4520 case OMPC_DEPEND_in:
4524 case OMPC_DEPEND_out:
4525 case OMPC_DEPEND_inout:
4528 case OMPC_DEPEND_mutexinoutset:
4529 DepKind = DepMutexInOutSet;
4531 case OMPC_DEPEND_inoutset:
4532 DepKind = DepInOutSet;
4534 case OMPC_DEPEND_source:
4535 case OMPC_DEPEND_sink:
4536 case OMPC_DEPEND_depobj:
4538 llvm_unreachable(
"Unknown task dependence type");
4546 FlagsTy = C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy),
false);
4547 if (KmpDependInfoTy.
isNull()) {
4548 RecordDecl *KmpDependInfoRD = C.buildImplicitRecord(
"kmp_depend_info");
4554 KmpDependInfoTy = C.getRecordType(KmpDependInfoRD);
4558 std::pair<llvm::Value *, LValue>
4566 QualType KmpDependInfoPtrTy =
C.getPointerType(KmpDependInfoTy);
4573 Base.getAddress(CGF),
4574 llvm::ConstantInt::get(CGF.
IntPtrTy, -1,
true));
4576 DepObjAddr, KmpDependInfoTy,
Base.getBaseInfo(),
Base.getTBAAInfo());
4579 NumDepsBase, *std::next(KmpDependInfoRD->
field_begin(), BaseAddr));
4581 return std::make_pair(NumDeps,
Base);
4585 llvm::PointerUnion<unsigned *, LValue *> Pos,
4596 OMPIteratorGeneratorScope IteratorScope(
4597 CGF, cast_or_null<OMPIteratorExpr>(
4605 if (
unsigned *
P = Pos.dyn_cast<
unsigned *>()) {
4629 if (
unsigned *
P = Pos.dyn_cast<
unsigned *>()) {
4634 Idx = CGF.
Builder.CreateNUWAdd(Idx,
4635 llvm::ConstantInt::get(Idx->getType(), 1));
4644 assert(Data.
DepKind == OMPC_DEPEND_depobj &&
4645 "Expected depobj dependecy kind.");
4650 OMPIteratorGeneratorScope IteratorScope(
4651 CGF, cast_or_null<OMPIteratorExpr>(
4655 llvm::Value *NumDeps;
4658 std::tie(NumDeps,
Base) =
4659 getDepobjElements(CGF, DepobjLVal, E->
getExprLoc());
4662 C.getUIntPtrType());
4666 llvm::Value *
Add = CGF.
Builder.CreateNUWAdd(PrevVal, NumDeps);
4668 SizeLVals.push_back(NumLVal);
4671 for (
unsigned I = 0, E = SizeLVals.size(); I < E; ++I) {
4674 Sizes.push_back(Size);
4684 assert(Data.
DepKind == OMPC_DEPEND_depobj &&
4685 "Expected depobj dependecy kind.");
4686 llvm::Value *ElSize = CGF.
getTypeSize(KmpDependInfoTy);
4688 OMPIteratorGeneratorScope IteratorScope(
4689 CGF, cast_or_null<OMPIteratorExpr>(
4694 llvm::Value *NumDeps;
4697 std::tie(NumDeps,
Base) =
4698 getDepobjElements(CGF, DepobjLVal, E->
getExprLoc());
4710 llvm::Value *
Add = CGF.
Builder.CreateNUWAdd(Pos, NumDeps);
4726 llvm::Value *NumOfElements =
nullptr;
4727 unsigned NumDependencies = std::accumulate(
4728 Dependencies.begin(), Dependencies.end(), 0,
4730 return D.DepKind == OMPC_DEPEND_depobj
4732 : (V + (D.IteratorExpr ? 0 : D.DepExprs.size()));
4736 bool HasDepobjDeps =
false;
4737 bool HasRegularWithIterators =
false;
4738 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4739 llvm::Value *NumOfRegularWithIterators =
4740 llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4743 if (D.
DepKind == OMPC_DEPEND_depobj) {
4745 emitDepobjElementsSizes(CGF, KmpDependInfoTy, D);
4746 for (llvm::Value *Size : Sizes) {
4747 NumOfDepobjElements =
4748 CGF.
Builder.CreateNUWAdd(NumOfDepobjElements, Size);
4750 HasDepobjDeps =
true;
4755 if (
const auto *IE = cast_or_null<OMPIteratorExpr>(D.
IteratorExpr)) {
4756 for (
unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4759 llvm::Value *NumClauseDeps = CGF.
Builder.CreateNUWMul(
4761 NumOfRegularWithIterators =
4762 CGF.
Builder.CreateNUWAdd(NumOfRegularWithIterators, NumClauseDeps);
4764 HasRegularWithIterators =
true;
4770 if (HasDepobjDeps || HasRegularWithIterators) {
4771 NumOfElements = llvm::ConstantInt::get(
CGM.
IntPtrTy, NumDependencies,
4773 if (HasDepobjDeps) {
4775 CGF.
Builder.CreateNUWAdd(NumOfDepobjElements, NumOfElements);
4777 if (HasRegularWithIterators) {
4779 CGF.
Builder.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements);
4782 Loc, C.getIntTypeForBitwidth(64, 0),
4786 KmpDependInfoArrayTy =
4795 NumOfElements = CGF.
Builder.CreateIntCast(NumOfElements, CGF.
Int32Ty,
4798 KmpDependInfoArrayTy = C.getConstantArrayType(
4799 KmpDependInfoTy,
llvm::APInt(64, NumDependencies),
nullptr,
4804 NumOfElements = llvm::ConstantInt::get(
CGM.
Int32Ty, NumDependencies,
4808 for (
unsigned I = 0,
End = Dependencies.size(); I <
End; ++I) {
4809 if (Dependencies[I].DepKind == OMPC_DEPEND_depobj ||
4810 Dependencies[I].IteratorExpr)
4817 CGF.
CreateMemTemp(C.getSizeType(),
"dep.counter.addr"), C.getSizeType());
4819 for (
unsigned I = 0,
End = Dependencies.size(); I <
End; ++I) {
4820 if (Dependencies[I].DepKind == OMPC_DEPEND_depobj ||
4821 !Dependencies[I].IteratorExpr)
4827 if (HasDepobjDeps) {
4828 for (
unsigned I = 0,
End = Dependencies.size(); I <
End; ++I) {
4829 if (Dependencies[I].DepKind != OMPC_DEPEND_depobj)
4831 emitDepobjElements(CGF, KmpDependInfoTy, PosLVal, Dependencies[I],
4837 return std::make_pair(NumOfElements, DependenciesArray);
4848 unsigned NumDependencies = Dependencies.
DepExprs.size();
4859 llvm::Value *NumDepsVal;
4860 CharUnits Align = C.getTypeAlignInChars(KmpDependInfoTy);
4861 if (
const auto *IE =
4862 cast_or_null<OMPIteratorExpr>(Dependencies.
IteratorExpr)) {
4863 NumDepsVal = llvm::ConstantInt::get(CGF.
SizeTy, 1);
4864 for (
unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4867 NumDepsVal = CGF.
Builder.CreateNUWMul(NumDepsVal, Sz);
4869 Size = CGF.
Builder.CreateNUWAdd(llvm::ConstantInt::get(CGF.
SizeTy, 1),
4872 C.getTypeSizeInChars(KmpDependInfoTy).alignTo(Align);
4873 llvm::Value *RecSize =
CGM.
getSize(SizeInBytes);
4874 Size = CGF.
Builder.CreateNUWMul(Size, RecSize);
4878 QualType KmpDependInfoArrayTy = C.getConstantArrayType(
4879 KmpDependInfoTy,
llvm::APInt(64, NumDependencies + 1),
4881 CharUnits Sz = C.getTypeSizeInChars(KmpDependInfoArrayTy);
4883 NumDepsVal = llvm::ConstantInt::get(CGF.
IntPtrTy, NumDependencies);
4888 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4889 llvm::Value *Args[] = {ThreadID, Size, Allocator};
4894 Args,
".dep.arr.addr");
4897 Addr, KmpDependInfoLlvmTy->getPointerTo());
4898 DependenciesArray =
Address(Addr, KmpDependInfoLlvmTy, Align);
4905 llvm::PointerUnion<unsigned *, LValue *> Pos;
4910 CGF.
CreateMemTemp(C.getSizeType(),
"iterator.counter.addr"),
4918 emitDependData(CGF, KmpDependInfoTy, Pos, Dependencies, DependenciesArray);
4922 return DependenciesArray;
4932 QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);
4938 llvm::ConstantInt::get(CGF.
IntPtrTy, -1,
true));
4943 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4944 llvm::Value *Args[] = {ThreadID, DepObjAddr, Allocator};
4961 llvm::Value *NumDeps;
4963 std::tie(NumDeps,
Base) = getDepobjElements(CGF, DepobjLVal, Loc);
4968 Begin.getElementType(),
Begin.getPointer(), NumDeps);