30#include "llvm/ADT/ArrayRef.h"
31#include "llvm/ADT/SetOperations.h"
32#include "llvm/ADT/SmallBitVector.h"
33#include "llvm/ADT/StringExtras.h"
34#include "llvm/Bitcode/BitcodeReader.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DerivedTypes.h"
37#include "llvm/IR/GlobalValue.h"
38#include "llvm/IR/InstrTypes.h"
39#include "llvm/IR/Value.h"
40#include "llvm/Support/AtomicOrdering.h"
41#include "llvm/Support/Format.h"
42#include "llvm/Support/raw_ostream.h"
49using namespace CodeGen;
50using namespace llvm::omp;
54class CGOpenMPRegionInfo :
public CodeGenFunction::CGCapturedStmtInfo {
57 enum CGOpenMPRegionKind {
60 ParallelOutlinedRegion,
71 const CGOpenMPRegionKind RegionKind,
74 : CGCapturedStmtInfo(CS,
CR_OpenMP), RegionKind(RegionKind),
75 CodeGen(CodeGen),
Kind(
Kind), HasCancel(HasCancel) {}
77 CGOpenMPRegionInfo(
const CGOpenMPRegionKind RegionKind,
80 : CGCapturedStmtInfo(
CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
85 virtual const VarDecl *getThreadIDVariable()
const = 0;
96 CGOpenMPRegionKind getRegionKind()
const {
return RegionKind; }
100 bool hasCancel()
const {
return HasCancel; }
102 static bool classof(
const CGCapturedStmtInfo *Info) {
106 ~CGOpenMPRegionInfo()
override =
default;
109 CGOpenMPRegionKind RegionKind;
116class CGOpenMPOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
121 StringRef HelperName)
122 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen,
Kind,
124 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
125 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
130 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
133 StringRef getHelperName()
const override {
return HelperName; }
135 static bool classof(
const CGCapturedStmtInfo *Info) {
136 return CGOpenMPRegionInfo::classof(Info) &&
137 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
138 ParallelOutlinedRegion;
145 StringRef HelperName;
149class CGOpenMPTaskOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
155 llvm::SwitchInst *UntiedSwitch =
nullptr;
158 UntiedTaskActionTy(
bool Tied,
const VarDecl *PartIDVar,
160 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
170 UntiedSwitch = CGF.
Builder.CreateSwitch(Res, DoneBB);
174 UntiedSwitch->addCase(CGF.
Builder.getInt32(0),
176 emitUntiedSwitch(CGF);
187 CodeGenFunction::JumpDest CurPoint =
191 UntiedSwitch->addCase(CGF.
Builder.getInt32(UntiedSwitch->getNumCases()),
197 unsigned getNumberOfParts()
const {
return UntiedSwitch->getNumCases(); }
203 const UntiedTaskActionTy &Action)
204 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen,
Kind, HasCancel),
205 ThreadIDVar(ThreadIDVar), Action(Action) {
206 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
211 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
217 StringRef getHelperName()
const override {
return ".omp_outlined."; }
220 Action.emitUntiedSwitch(CGF);
223 static bool classof(
const CGCapturedStmtInfo *Info) {
224 return CGOpenMPRegionInfo::classof(Info) &&
225 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
234 const UntiedTaskActionTy &Action;
239class CGOpenMPInlinedRegionInfo :
public CGOpenMPRegionInfo {
241 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
244 : CGOpenMPRegionInfo(InlinedRegion, CodeGen,
Kind, HasCancel),
246 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
249 llvm::Value *getContextValue()
const override {
251 return OuterRegionInfo->getContextValue();
252 llvm_unreachable(
"No context value for inlined OpenMP region");
255 void setContextValue(llvm::Value *
V)
override {
256 if (OuterRegionInfo) {
257 OuterRegionInfo->setContextValue(
V);
260 llvm_unreachable(
"No context value for inlined OpenMP region");
266 return OuterRegionInfo->lookup(VD);
272 FieldDecl *getThisFieldDecl()
const override {
274 return OuterRegionInfo->getThisFieldDecl();
280 const VarDecl *getThreadIDVariable()
const override {
282 return OuterRegionInfo->getThreadIDVariable();
289 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
290 llvm_unreachable(
"No LValue for inlined OpenMP construct");
294 StringRef getHelperName()
const override {
295 if (
auto *OuterRegionInfo = getOldCSI())
296 return OuterRegionInfo->getHelperName();
297 llvm_unreachable(
"No helper name for inlined OpenMP construct");
302 OuterRegionInfo->emitUntiedSwitch(CGF);
305 CodeGenFunction::CGCapturedStmtInfo *getOldCSI()
const {
return OldCSI; }
307 static bool classof(
const CGCapturedStmtInfo *Info) {
308 return CGOpenMPRegionInfo::classof(Info) &&
309 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
312 ~CGOpenMPInlinedRegionInfo()
override =
default;
316 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
317 CGOpenMPRegionInfo *OuterRegionInfo;
325class CGOpenMPTargetRegionInfo final :
public CGOpenMPRegionInfo {
329 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
331 HelperName(HelperName) {}
335 const VarDecl *getThreadIDVariable()
const override {
return nullptr; }
338 StringRef getHelperName()
const override {
return HelperName; }
340 static bool classof(
const CGCapturedStmtInfo *Info) {
341 return CGOpenMPRegionInfo::classof(Info) &&
342 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
346 StringRef HelperName;
350 llvm_unreachable(
"No codegen for expressions");
354class CGOpenMPInnerExprInfo final :
public CGOpenMPInlinedRegionInfo {
357 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
365 if (!
C.capturesVariable() && !
C.capturesVariableByCopy())
368 const VarDecl *VD =
C.getCapturedVar();
378 (void)PrivScope.Privatize();
383 if (
const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
390 llvm_unreachable(
"No body for expressions");
395 const VarDecl *getThreadIDVariable()
const override {
396 llvm_unreachable(
"No thread id for expressions");
400 StringRef getHelperName()
const override {
401 llvm_unreachable(
"No helper name for expressions");
404 static bool classof(
const CGCapturedStmtInfo *Info) {
return false; }
408 CodeGenFunction::OMPPrivateScope PrivScope;
412class InlinedOpenMPRegionRAII {
414 llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
415 FieldDecl *LambdaThisCaptureField =
nullptr;
417 bool NoInheritance =
false;
426 bool NoInheritance =
true)
427 : CGF(CGF), NoInheritance(NoInheritance) {
440 ~InlinedOpenMPRegionRAII() {
457enum OpenMPLocationFlags :
unsigned {
459 OMP_IDENT_IMD = 0x01,
461 OMP_IDENT_KMPC = 0x02,
463 OMP_ATOMIC_REDUCE = 0x10,
465 OMP_IDENT_BARRIER_EXPL = 0x20,
467 OMP_IDENT_BARRIER_IMPL = 0x40,
469 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
471 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
473 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
475 OMP_IDENT_WORK_LOOP = 0x200,
477 OMP_IDENT_WORK_SECTIONS = 0x400,
479 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
509enum IdentFieldIndex {
511 IdentField_Reserved_1,
515 IdentField_Reserved_2,
517 IdentField_Reserved_3,
526enum OpenMPSchedType {
529 OMP_sch_static_chunked = 33,
531 OMP_sch_dynamic_chunked = 35,
532 OMP_sch_guided_chunked = 36,
533 OMP_sch_runtime = 37,
536 OMP_sch_static_balanced_chunked = 45,
539 OMP_ord_static_chunked = 65,
541 OMP_ord_dynamic_chunked = 67,
542 OMP_ord_guided_chunked = 68,
543 OMP_ord_runtime = 69,
545 OMP_sch_default = OMP_sch_static,
547 OMP_dist_sch_static_chunked = 91,
548 OMP_dist_sch_static = 92,
551 OMP_sch_modifier_monotonic = (1 << 29),
553 OMP_sch_modifier_nonmonotonic = (1 << 30),
576 Callback(CodeGen, CGF, *PrePostAction);
579 Callback(CodeGen, CGF, Action);
587 if (
const auto *CE = dyn_cast<CallExpr>(ReductionOp))
588 if (
const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
589 if (
const auto *DRE =
590 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
591 if (
const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
602 std::pair<llvm::Function *, llvm::Function *> Reduction =
604 const auto *CE = cast<CallExpr>(InitOp);
605 const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
609 cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
611 cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
612 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
613 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()), Private);
614 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()), Original);
615 (void)PrivateScope.Privatize();
617 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
622 auto *GV =
new llvm::GlobalVariable(
624 llvm::GlobalValue::PrivateLinkage, Init, Name);
637 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, LV);
644 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
665 llvm::Value *NumElements = CGF.
emitArrayLength(ArrayTy, ElementTy, DestAddr);
669 llvm::Value *SrcBegin =
nullptr;
671 SrcBegin = SrcAddr.getPointer();
672 llvm::Value *DestBegin = DestAddr.
getPointer();
674 llvm::Value *DestEnd =
679 llvm::Value *IsEmpty =
680 CGF.
Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arrayinit.isempty");
681 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
684 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
689 llvm::PHINode *SrcElementPHI =
nullptr;
692 SrcElementPHI = CGF.
Builder.CreatePHI(SrcBegin->getType(), 2,
693 "omp.arraycpy.srcElementPast");
694 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
696 Address(SrcElementPHI, SrcAddr.getElementType(),
697 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
699 llvm::PHINode *DestElementPHI = CGF.
Builder.CreatePHI(
700 DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
701 DestElementPHI->addIncoming(DestBegin, EntryBB);
708 CodeGenFunction::RunCleanupsScope InitScope(CGF);
709 if (EmitDeclareReductionInit) {
711 SrcElementCurrent, ElementTy);
719 llvm::Value *SrcElementNext = CGF.
Builder.CreateConstGEP1_32(
720 SrcAddr.getElementType(), SrcElementPHI, 1,
721 "omp.arraycpy.dest.element");
722 SrcElementPHI->addIncoming(SrcElementNext, CGF.
Builder.GetInsertBlock());
726 llvm::Value *DestElementNext = CGF.
Builder.CreateConstGEP1_32(
728 "omp.arraycpy.dest.element");
731 CGF.
Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
732 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
733 DestElementPHI->addIncoming(DestElementNext, CGF.
Builder.GetInsertBlock());
745 if (
const auto *OASE = dyn_cast<OMPArraySectionExpr>(E))
750void ReductionCodeGen::emitAggregateInitialization(
756 const auto *PrivateVD =
757 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
758 bool EmitDeclareReductionInit =
761 EmitDeclareReductionInit,
762 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
763 : PrivateVD->getInit(),
771 ClausesData.reserve(Shareds.size());
772 SharedAddresses.reserve(Shareds.size());
773 Sizes.reserve(Shareds.size());
774 BaseDecls.reserve(Shareds.size());
775 const auto *IOrig = Origs.begin();
776 const auto *IPriv = Privates.begin();
777 const auto *IRed = ReductionOps.begin();
778 for (
const Expr *Ref : Shareds) {
779 ClausesData.emplace_back(Ref, *IOrig, *IPriv, *IRed);
780 std::advance(IOrig, 1);
781 std::advance(IPriv, 1);
782 std::advance(IRed, 1);
787 assert(SharedAddresses.size() == N && OrigAddresses.size() == N &&
788 "Number of generated lvalues must be exactly N.");
789 LValue First = emitSharedLValue(CGF, ClausesData[N].Shared);
790 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Shared);
791 SharedAddresses.emplace_back(
First, Second);
792 if (ClausesData[N].Shared == ClausesData[N].Ref) {
793 OrigAddresses.emplace_back(
First, Second);
795 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
796 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
797 OrigAddresses.emplace_back(
First, Second);
803 bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
806 CGF.
getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()),
811 llvm::Value *SizeInChars;
812 auto *ElemType = OrigAddresses[N].first.getAddress(CGF).getElementType();
813 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
814 if (AsArraySection) {
815 Size = CGF.
Builder.CreatePtrDiff(ElemType,
816 OrigAddresses[N].second.getPointer(CGF),
817 OrigAddresses[N].first.getPointer(CGF));
818 Size = CGF.
Builder.CreateNUWAdd(
819 Size, llvm::ConstantInt::get(Size->getType(), 1));
820 SizeInChars = CGF.
Builder.CreateNUWMul(Size, ElemSizeOf);
823 CGF.
getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType());
824 Size = CGF.
Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
826 Sizes.emplace_back(SizeInChars, Size);
829 cast<OpaqueValueExpr>(
839 assert(!Size && !Sizes[N].second &&
840 "Size should be nullptr for non-variably modified reduction "
846 cast<OpaqueValueExpr>(
855 assert(SharedAddresses.size() > N &&
"No variable was generated");
856 const auto *PrivateVD =
857 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
862 (void)DefaultInit(CGF);
863 emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD);
864 }
else if (DRD && (DRD->
getInitializer() || !PrivateVD->hasInit())) {
865 (void)DefaultInit(CGF);
866 QualType SharedType = SharedAddresses[N].first.getType();
868 PrivateAddr, SharedAddr, SharedType);
869 }
else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
872 PrivateVD->
getType().getQualifiers(),
890 CGF.
pushDestroy(DTorKind, PrivateAddr, PrivateType);
914 Address OriginalBaseAddress, llvm::Value *Addr) {
938 Addr, OriginalBaseAddress.
getType());
943 const VarDecl *OrigVD =
nullptr;
944 if (
const auto *OASE = dyn_cast<OMPArraySectionExpr>(Ref)) {
945 const Expr *
Base = OASE->getBase()->IgnoreParenImpCasts();
946 while (
const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(
Base))
947 Base = TempOASE->getBase()->IgnoreParenImpCasts();
948 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
949 Base = TempASE->getBase()->IgnoreParenImpCasts();
950 DE = cast<DeclRefExpr>(
Base);
951 OrigVD = cast<VarDecl>(DE->
getDecl());
952 }
else if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
953 const Expr *
Base = ASE->getBase()->IgnoreParenImpCasts();
954 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
955 Base = TempASE->getBase()->IgnoreParenImpCasts();
956 DE = cast<DeclRefExpr>(
Base);
957 OrigVD = cast<VarDecl>(DE->
getDecl());
966 BaseDecls.emplace_back(OrigVD);
969 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
971 Address SharedAddr = SharedAddresses[N].first.getAddress(CGF);
972 llvm::Value *Adjustment = CGF.
Builder.CreatePtrDiff(
975 llvm::Value *PrivatePointer =
981 SharedAddresses[N].first.getType(),
984 BaseDecls.emplace_back(
985 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
998 getThreadIDVariable()->getType()->castAs<PointerType>());
1016LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1019 getThreadIDVariable()->getType(),
1035 : CGM(CGM), OMPBuilder(CGM.getModule()) {
1037 llvm::OpenMPIRBuilderConfig Config(
1053 if (!
Data.getValue().pointsToAliveValue())
1055 auto *GV = dyn_cast<llvm::GlobalVariable>(
Data.getValue());
1058 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1060 GV->eraseFromParent();
1065 return OMPBuilder.createPlatformSpecificName(Parts);
1068static llvm::Function *
1070 const Expr *CombinerInitializer,
const VarDecl *In,
1071 const VarDecl *Out,
bool IsCombiner) {
1074 QualType PtrTy =
C.getPointerType(Ty).withRestrict();
1080 Args.push_back(&OmpOutParm);
1081 Args.push_back(&OmpInParm);
1086 {IsCombiner ?
"omp_combiner" :
"omp_initializer",
""});
1087 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1091 Fn->removeFnAttr(llvm::Attribute::NoInline);
1092 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1093 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1099 Out->getLocation());
1100 CodeGenFunction::OMPPrivateScope
Scope(CGF);
1109 (void)
Scope.Privatize();
1110 if (!IsCombiner && Out->hasInit() &&
1113 Out->getType().getQualifiers(),
1116 if (CombinerInitializer)
1118 Scope.ForceCleanup();
1129 cast<VarDecl>(cast<DeclRefExpr>(D->
getCombinerIn())->getDecl()),
1130 cast<VarDecl>(cast<DeclRefExpr>(D->
getCombinerOut())->getDecl()),
1138 cast<VarDecl>(cast<DeclRefExpr>(D->
getInitOrig())->getDecl()),
1139 cast<VarDecl>(cast<DeclRefExpr>(D->
getInitPriv())->getDecl()),
1145 Decls.second.push_back(D);
1149std::pair<llvm::Function *, llvm::Function *>
1161struct PushAndPopStackRAII {
1162 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder,
CodeGenFunction &CGF,
1163 bool HasCancel, llvm::omp::Directive Kind)
1164 : OMPBuilder(OMPBuilder) {
1180 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1181 assert(IP.getBlock()->end() == IP.getPoint() &&
1182 "Clang CG should cause non-terminated block!");
1183 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1192 llvm::OpenMPIRBuilder::FinalizationInfo FI({FiniCB, Kind, HasCancel});
1193 OMPBuilder->pushFinalizationCB(std::move(FI));
1195 ~PushAndPopStackRAII() {
1197 OMPBuilder->popFinalizationCB();
1199 llvm::OpenMPIRBuilder *OMPBuilder;
1208 "thread id variable must be of type kmp_int32 *");
1210 bool HasCancel =
false;
1211 if (
const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1212 HasCancel = OPD->hasCancel();
1213 else if (
const auto *OPD = dyn_cast<OMPTargetParallelDirective>(&D))
1214 HasCancel = OPD->hasCancel();
1215 else if (
const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1216 HasCancel = OPSD->hasCancel();
1217 else if (
const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1218 HasCancel = OPFD->hasCancel();
1219 else if (
const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1220 HasCancel = OPFD->hasCancel();
1221 else if (
const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1222 HasCancel = OPFD->hasCancel();
1223 else if (
const auto *OPFD =
1224 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1225 HasCancel = OPFD->hasCancel();
1226 else if (
const auto *OPFD =
1227 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1228 HasCancel = OPFD->hasCancel();
1233 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind);
1234 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1235 HasCancel, OutlinedHelperName);
1236 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1241 std::string Suffix =
getName({
"omp_outlined"});
1242 return (Name + Suffix).str();
1250 std::string Suffix =
getName({
"omp",
"reduction",
"reduction_func"});
1251 return (Name + Suffix).str();
1278 bool Tied,
unsigned &NumberOfParts) {
1283 llvm::Value *TaskArgs[] = {
1285 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1288 CGF.EmitRuntimeCall(
OMPBuilder.getOrCreateRuntimeFunction(
1292 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1296 "thread id variable must be of type kmp_int32 for tasks");
1301 bool HasCancel =
false;
1302 if (
const auto *TD = dyn_cast<OMPTaskDirective>(&D))
1303 HasCancel = TD->hasCancel();
1304 else if (
const auto *TD = dyn_cast<OMPTaskLoopDirective>(&D))
1305 HasCancel = TD->hasCancel();
1306 else if (
const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(&D))
1307 HasCancel = TD->hasCancel();
1308 else if (
const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(&D))
1309 HasCancel = TD->hasCancel();
1312 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1313 InnermostKind, HasCancel, Action);
1315 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1317 NumberOfParts = Action.getNumberOfParts();
1322 bool AtCurrentPoint) {
1324 assert(!Elem.second.ServiceInsertPt &&
"Insert point is set already.");
1326 llvm::Value *Undef = llvm::UndefValue::get(CGF.
Int32Ty);
1327 if (AtCurrentPoint) {
1328 Elem.second.ServiceInsertPt =
new llvm::BitCastInst(
1331 Elem.second.ServiceInsertPt =
1332 new llvm::BitCastInst(Undef, CGF.
Int32Ty,
"svcpt");
1339 if (Elem.second.ServiceInsertPt) {
1340 llvm::Instruction *Ptr = Elem.second.ServiceInsertPt;
1341 Elem.second.ServiceInsertPt =
nullptr;
1342 Ptr->eraseFromParent();
1349 llvm::raw_svector_ostream OS(Buffer);
1353 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.
CurFuncDecl))
1354 OS << FD->getQualifiedNameAsString();
1361 unsigned Flags,
bool EmitLoc) {
1362 uint32_t SrcLocStrSize;
1363 llvm::Constant *SrcLocStr;
1365 llvm::codegenoptions::NoDebugInfo) ||
1367 SrcLocStr =
OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1369 std::string FunctionName;
1370 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.
CurFuncDecl))
1371 FunctionName = FD->getQualifiedNameAsString();
1374 unsigned Line = PLoc.
getLine();
1376 SrcLocStr =
OMPBuilder.getOrCreateSrcLocStr(FunctionName, FileName, Line,
1377 Column, SrcLocStrSize);
1381 SrcLocStr, SrcLocStrSize, llvm::omp::IdentFlag(Flags), Reserved2Flags);
1386 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1392 uint32_t SrcLocStrSize;
1393 auto *SrcLocStr =
OMPBuilder.getOrCreateSrcLocStr(
1396 OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize));
1399 llvm::Value *ThreadID =
nullptr;
1404 ThreadID = I->second.ThreadID;
1405 if (ThreadID !=
nullptr)
1409 if (
auto *OMPRegionInfo =
1411 if (OMPRegionInfo->getThreadIDVariable()) {
1413 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1417 CGF.
Builder.GetInsertBlock() == TopBlock ||
1418 !isa<llvm::Instruction>(LVal.
getPointer(CGF)) ||
1419 cast<llvm::Instruction>(LVal.
getPointer(CGF))->getParent() ==
1421 cast<llvm::Instruction>(LVal.
getPointer(CGF))->getParent() ==
1422 CGF.
Builder.GetInsertBlock()) {
1426 if (CGF.
Builder.GetInsertBlock() == TopBlock) {
1428 Elem.second.ThreadID = ThreadID;
1440 if (!Elem.second.ServiceInsertPt)
1442 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1443 CGF.
Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1444 llvm::CallInst *Call = CGF.
Builder.CreateCall(
1446 OMPRTL___kmpc_global_thread_num),
1449 Elem.second.ThreadID = Call;
1454 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1466 for(
const auto *D : I->second)
1481 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(
CGM.
Int32Ty),
1488llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind
1490 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
1491 OMPDeclareTargetDeclAttr::getDeviceType(VD);
1493 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1495 switch ((
int)*DevTy) {
1496 case OMPDeclareTargetDeclAttr::DT_Host:
1497 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseHost;
1499 case OMPDeclareTargetDeclAttr::DT_NoHost:
1500 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNoHost;
1502 case OMPDeclareTargetDeclAttr::DT_Any:
1503 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny;
1506 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1511llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind
1513 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> MapType =
1514 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1516 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1517 switch ((
int)*MapType) {
1518 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_To:
1519 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
1521 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Enter:
1522 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter;
1524 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Link:
1525 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
1528 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1537 auto FileInfoCallBack = [&]() {
1541 llvm::sys::fs::UniqueID ID;
1542 if (llvm::sys::fs::getUniqueID(PLoc.
getFilename(), ID)) {
1543 PLoc =
SM.getPresumedLoc(BeginLoc,
false);
1549 return OMPBuilder.getTargetEntryUniqueInfo(FileInfoCallBack, ParentName);
1555 auto LinkageForVariable = [&VD,
this]() {
1559 std::vector<llvm::GlobalVariable *> GeneratedRefs;
1563 llvm::Constant *addr =
OMPBuilder.getAddrOfDeclareTargetVar(
1571 LinkageForVariable);
1583 std::string Suffix =
getName({
"cache",
""});
1584 return OMPBuilder.getOrCreateInternalVariable(
1597 llvm::Value *Args[] = {
1621 llvm::Value *Args[] = {
1623 Ctor, CopyCtor, Dtor};
1626 CGM.
getModule(), OMPRTL___kmpc_threadprivate_register),
1641 llvm::Value *Ctor =
nullptr, *CopyCtor =
nullptr, *Dtor =
nullptr;
1651 Args.push_back(&Dst);
1656 std::string Name =
getName({
"__kmpc_global_ctor_",
""});
1657 llvm::Function *Fn =
1683 Args.push_back(&Dst);
1688 std::string Name =
getName({
"__kmpc_global_dtor_",
""});
1689 llvm::Function *Fn =
1711 auto *CopyCtorTy = llvm::FunctionType::get(
CGM.
VoidPtrTy, CopyCtorTyArgs,
1717 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
1718 if (Ctor ==
nullptr) {
1722 Ctor = llvm::Constant::getNullValue(CtorTy);
1724 if (Dtor ==
nullptr) {
1728 Dtor = llvm::Constant::getNullValue(DtorTy);
1731 auto *InitFunctionTy =
1732 llvm::FunctionType::get(
CGM.
VoidTy,
false);
1733 std::string Name =
getName({
"__omp_threadprivate_init_",
""});
1743 return InitFunction;
1751 llvm::GlobalVariable *Addr,
1756 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
1757 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1758 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
1759 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
1760 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
1764 assert(VD &&
"Unknown VarDecl");
1775 llvm::TargetRegionEntryInfo EntryInfo =
1778 OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(Buffer, EntryInfo);
1782 llvm::Constant *Ctor;
1792 FTy, Twine(Buffer,
"_ctor"), FI, Loc,
false,
1793 llvm::GlobalValue::WeakODRLinkage);
1794 Fn->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1796 Fn->setCallingConv(llvm::CallingConv::AMDGPU_KERNEL);
1801 llvm::Constant *AddrInAS0 = Addr;
1802 if (Addr->getAddressSpace() != 0)
1803 AddrInAS0 = llvm::ConstantExpr::getAddrSpaceCast(
1806 Address(AddrInAS0, Addr->getValueType(),
1808 Init->getType().getQualifiers(),
1812 ID = llvm::ConstantExpr::getBitCast(Fn,
CGM.
Int8PtrTy);
1814 Ctor =
new llvm::GlobalVariable(
1816 llvm::GlobalValue::PrivateLinkage,
1817 llvm::Constant::getNullValue(
CGM.
Int8Ty), Twine(Buffer,
"_ctor"));
1823 auto CtorEntryInfo = EntryInfo;
1824 CtorEntryInfo.ParentName = Twine(Buffer,
"_ctor").toStringRef(Out);
1825 OMPBuilder.OffloadInfoManager.registerTargetRegionEntryInfo(
1826 CtorEntryInfo, Ctor, ID,
1827 llvm::OffloadEntriesInfoManager::OMPTargetRegionEntryCtor);
1830 llvm::Constant *Dtor;
1840 FTy, Twine(Buffer,
"_dtor"), FI, Loc,
false,
1841 llvm::GlobalValue::WeakODRLinkage);
1842 Fn->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1844 Fn->setCallingConv(llvm::CallingConv::AMDGPU_KERNEL);
1851 llvm::Constant *AddrInAS0 = Addr;
1852 if (Addr->getAddressSpace() != 0)
1853 AddrInAS0 = llvm::ConstantExpr::getAddrSpaceCast(
1861 ID = llvm::ConstantExpr::getBitCast(Fn,
CGM.
Int8PtrTy);
1863 Dtor =
new llvm::GlobalVariable(
1865 llvm::GlobalValue::PrivateLinkage,
1866 llvm::Constant::getNullValue(
CGM.
Int8Ty), Twine(Buffer,
"_dtor"));
1871 auto DtorEntryInfo = EntryInfo;
1872 DtorEntryInfo.ParentName = Twine(Buffer,
"_dtor").toStringRef(Out);
1873 OMPBuilder.OffloadInfoManager.registerTargetRegionEntryInfo(
1874 DtorEntryInfo, Dtor, ID,
1875 llvm::OffloadEntriesInfoManager::OMPTargetRegionEntryDtor);
1881 llvm::GlobalValue *GV) {
1882 std::optional<OMPDeclareTargetDeclAttr *> ActiveAttr =
1883 OMPDeclareTargetDeclAttr::getActiveAttr(FD);
1886 if (!ActiveAttr || !(*ActiveAttr)->getIndirect())
1893 OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(Name, EntryInfo);
1899 llvm::GlobalValue *Addr = GV;
1901 Addr =
new llvm::GlobalVariable(
1903 true, llvm::GlobalValue::ExternalLinkage, GV, Name,
1904 nullptr, llvm::GlobalValue::NotThreadLocal,
1905 CGM.
getModule().getDataLayout().getDefaultGlobalsAddressSpace());
1906 Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1909 OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(
1911 llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect,
1912 llvm::GlobalValue::WeakODRLinkage);
1918 std::string Suffix =
getName({
"artificial",
""});
1920 llvm::GlobalVariable *GAddr =
OMPBuilder.getOrCreateInternalVariable(
1921 VarLVType, Twine(Name).concat(Suffix).str());
1924 GAddr->setThreadLocal(
true);
1925 return Address(GAddr, GAddr->getValueType(),
1928 std::string CacheSuffix =
getName({
"cache",
""});
1929 llvm::Value *Args[] = {
1937 Twine(Name).concat(Suffix).concat(CacheSuffix).str())};
1944 VarLVType->getPointerTo(0)),
1988 llvm::Function *OutlinedFn,
1991 llvm::Value *NumThreads) {
1996 auto &&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc,
2000 llvm::Value *Args[] = {
2002 CGF.
Builder.getInt32(CapturedVars.size()),
2005 RealArgs.append(std::begin(Args), std::end(Args));
2006 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
2008 llvm::FunctionCallee RTLFn =
2009 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_fork_call);
2012 auto &&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc, Loc,
2018 llvm::Value *Args[] = {RTLoc, ThreadID};
2020 M, OMPRTL___kmpc_serialized_parallel),
2027 ".bound.zero.addr");
2031 OutlinedFnArgs.push_back(ThreadIDAddr.
getPointer());
2032 OutlinedFnArgs.push_back(ZeroAddrBound.
getPointer());
2033 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
2041 OutlinedFn->removeFnAttr(llvm::Attribute::AlwaysInline);
2042 OutlinedFn->addFnAttr(llvm::Attribute::NoInline);
2048 M, OMPRTL___kmpc_end_serialized_parallel),
2067 if (
auto *OMPRegionInfo =
2069 if (OMPRegionInfo->getThreadIDVariable())
2070 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(CGF);
2079 return ThreadIDTemp;
2083 std::string Prefix = Twine(
"gomp_critical_user_", CriticalName).str();
2084 std::string Name =
getName({Prefix,
"var"});
2091 llvm::FunctionCallee EnterCallee;
2093 llvm::FunctionCallee ExitCallee;
2096 llvm::BasicBlock *ContBlock =
nullptr;
2099 CommonActionTy(llvm::FunctionCallee EnterCallee,
2101 llvm::FunctionCallee ExitCallee,
2103 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
2104 ExitArgs(ExitArgs), Conditional(Conditional) {}
2108 llvm::Value *CallBool = CGF.
Builder.CreateIsNotNull(EnterRes);
2112 CGF.
Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
2128 StringRef CriticalName,
2142 EnterArgs.push_back(CGF.
Builder.CreateIntCast(
2145 CommonActionTy Action(
2148 Hint ? OMPRTL___kmpc_critical_with_hint : OMPRTL___kmpc_critical),
2151 OMPRTL___kmpc_end_critical),
2168 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2190 llvm::Value *FilterVal = Filter
2197 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2217 llvm::Value *Args[] = {
2219 llvm::ConstantInt::get(
CGM.
IntTy, 0,
true)};
2225 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
2226 Region->emitUntiedSwitch(CGF);
2239 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2252 unsigned Index,
const VarDecl *Var) {
2260 Ptr, ElemTy->getPointerTo(Ptr->getType()->getPointerAddressSpace())),
2276 Args.push_back(&LHSArg);
2277 Args.push_back(&RHSArg);
2283 llvm::GlobalValue::InternalLinkage, Name,
2286 Fn->setDoesNotRecurse();
2293 ArgsElemType->getPointerTo()),
2297 ArgsElemType->getPointerTo()),
2303 for (
unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
2304 const auto *DestVar =
2305 cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
2308 const auto *SrcVar =
2309 cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
2312 const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
2314 CGF.
EmitOMPCopy(
Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
2329 assert(CopyprivateVars.size() == SrcExprs.size() &&
2330 CopyprivateVars.size() == DstExprs.size() &&
2331 CopyprivateVars.size() == AssignmentOps.size());
2343 if (!CopyprivateVars.empty()) {
2346 C.getIntTypeForBitwidth(32, 1);
2347 DidIt = CGF.
CreateMemTemp(KmpInt32Ty,
".omp.copyprivate.did_it");
2352 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2369 llvm::APInt ArraySize(32, CopyprivateVars.size());
2370 QualType CopyprivateArrayTy =
C.getConstantArrayType(
2375 CGF.
CreateMemTemp(CopyprivateArrayTy,
".omp.copyprivate.cpr_list");
2376 for (
unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
2388 SrcExprs, DstExprs, AssignmentOps, Loc);
2389 llvm::Value *BufSize = CGF.
getTypeSize(CopyprivateArrayTy);
2393 llvm::Value *Args[] = {
2418 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2433 if (Kind == OMPD_for)
2434 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
2435 else if (Kind == OMPD_sections)
2436 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
2437 else if (Kind == OMPD_single)
2438 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
2439 else if (Kind == OMPD_barrier)
2440 Flags = OMP_IDENT_BARRIER_EXPL;
2442 Flags = OMP_IDENT_BARRIER_IMPL;
2454 ScheduleKind = OMPC_SCHEDULE_static;
2456 llvm::APInt ChunkSize(32, 1);
2466 bool ForceSimpleCall) {
2468 auto *OMPRegionInfo =
2472 CGF.
Builder, Kind, ForceSimpleCall, EmitChecks));
2485 if (OMPRegionInfo) {
2486 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
2489 OMPRTL___kmpc_cancel_barrier),
2498 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
2515 Expr *ME,
bool IsFatal) {
2518 : llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
2521 llvm::Value *Args[] = {
2523 llvm::ConstantInt::get(
CGM.
Int32Ty, IsFatal ? 2 : 1),
2532 bool Chunked,
bool Ordered) {
2533 switch (ScheduleKind) {
2534 case OMPC_SCHEDULE_static:
2535 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
2536 : (Ordered ? OMP_ord_static : OMP_sch_static);
2537 case OMPC_SCHEDULE_dynamic:
2538 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
2539 case OMPC_SCHEDULE_guided:
2540 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
2541 case OMPC_SCHEDULE_runtime:
2542 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
2543 case OMPC_SCHEDULE_auto:
2544 return Ordered ? OMP_ord_auto : OMP_sch_auto;
2546 assert(!Chunked &&
"chunk was specified but schedule kind not known");
2547 return Ordered ? OMP_ord_static : OMP_sch_static;
2549 llvm_unreachable(
"Unexpected runtime schedule");
2553static OpenMPSchedType
2556 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
2560 bool Chunked)
const {
2561 OpenMPSchedType Schedule =
2563 return Schedule == OMP_sch_static;
2569 return Schedule == OMP_dist_sch_static;
2573 bool Chunked)
const {
2574 OpenMPSchedType Schedule =
2576 return Schedule == OMP_sch_static_chunked;
2582 return Schedule == OMP_dist_sch_static_chunked;
2586 OpenMPSchedType Schedule =
2588 assert(Schedule != OMP_sch_static_chunked &&
"cannot be chunked here");
2589 return Schedule != OMP_sch_static;
2597 case OMPC_SCHEDULE_MODIFIER_monotonic:
2598 Modifier = OMP_sch_modifier_monotonic;
2600 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2601 Modifier = OMP_sch_modifier_nonmonotonic;
2603 case OMPC_SCHEDULE_MODIFIER_simd:
2604 if (Schedule == OMP_sch_static_chunked)
2605 Schedule = OMP_sch_static_balanced_chunked;
2612 case OMPC_SCHEDULE_MODIFIER_monotonic:
2613 Modifier = OMP_sch_modifier_monotonic;
2615 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2616 Modifier = OMP_sch_modifier_nonmonotonic;
2618 case OMPC_SCHEDULE_MODIFIER_simd:
2619 if (Schedule == OMP_sch_static_chunked)
2620 Schedule = OMP_sch_static_balanced_chunked;
2632 if (CGM.
getLangOpts().OpenMP >= 50 && Modifier == 0) {
2633 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
2634 Schedule == OMP_sch_static_balanced_chunked ||
2635 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||
2636 Schedule == OMP_dist_sch_static_chunked ||
2637 Schedule == OMP_dist_sch_static))
2638 Modifier = OMP_sch_modifier_nonmonotonic;
2640 return Schedule | Modifier;
2650 ScheduleKind.
Schedule, DispatchValues.
Chunk !=
nullptr, Ordered);
2652 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
2653 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&
2654 Schedule != OMP_sch_static_balanced_chunked));
2661 llvm::Value *Chunk = DispatchValues.
Chunk ? DispatchValues.
Chunk
2662 : CGF.
Builder.getIntN(IVSize, 1);
2663 llvm::Value *Args[] = {
2667 CGM, Schedule, ScheduleKind.
M1, ScheduleKind.
M2)),
2670 CGF.
Builder.getIntN(IVSize, 1),
2678 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
2679 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
2686 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||
2687 Schedule == OMP_sch_static_balanced_chunked ||
2688 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||
2689 Schedule == OMP_dist_sch_static ||
2690 Schedule == OMP_dist_sch_static_chunked);
2697 llvm::Value *Chunk = Values.
Chunk;
2698 if (Chunk ==
nullptr) {
2699 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
2700 Schedule == OMP_dist_sch_static) &&
2701 "expected static non-chunked schedule");
2705 assert((Schedule == OMP_sch_static_chunked ||
2706 Schedule == OMP_sch_static_balanced_chunked ||
2707 Schedule == OMP_ord_static_chunked ||
2708 Schedule == OMP_dist_sch_static_chunked) &&
2709 "expected static chunked schedule");
2711 llvm::Value *Args[] = {
2734 "Expected loop-based or sections-based directive.");
2737 ? OMP_IDENT_WORK_LOOP
2738 : OMP_IDENT_WORK_SECTIONS);
2740 llvm::FunctionCallee StaticInitFunction =
2745 ScheduleNum, ScheduleKind.
M1, ScheduleKind.
M2, Values);
2752 OpenMPSchedType ScheduleNum =
2754 llvm::Value *UpdatedLocation =
2757 llvm::FunctionCallee StaticInitFunction;
2758 bool isGPUDistribute =
2761 StaticInitFunction =
OMPBuilder.createForStaticInitFunction(
2775 llvm::Value *Args[] = {
2778 ? OMP_IDENT_WORK_DISTRIBUTE
2780 ? OMP_IDENT_WORK_LOOP
2781 : OMP_IDENT_WORK_SECTIONS),
2789 CGM.
getModule(), OMPRTL___kmpc_distribute_static_fini),
2818 llvm::Value *Args[] = {
2827 OMPBuilder.createDispatchNextFunction(IVSize, IVSigned), Args);
2834 llvm::Value *NumThreads,
2839 llvm::Value *Args[] = {
2848 ProcBindKind ProcBind,
2852 assert(ProcBind != OMP_PROC_BIND_unknown &&
"Unsupported proc_bind value.");
2854 llvm::Value *Args[] = {
2856 llvm::ConstantInt::get(
CGM.
IntTy,
unsigned(ProcBind),
true)};
2878enum KmpTaskTFields {
2908 llvm::OpenMPIRBuilder::EmitMetadataErrorReportFunctionTy &&ErrorReportFn =
2909 [
this](llvm::OpenMPIRBuilder::EmitMetadataErrorKind Kind,
2910 const llvm::TargetRegionEntryInfo &EntryInfo) ->
void {
2912 if (Kind != llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR) {
2916 if (I->getFirst().getUniqueID().getDevice() == EntryInfo.DeviceID &&
2917 I->getFirst().getUniqueID().getFile() == EntryInfo.FileID) {
2919 I->getFirst(), EntryInfo.Line, 1);
2925 case llvm::OpenMPIRBuilder::EMIT_MD_TARGET_REGION_ERROR: {
2928 "%0 is incorrect: either the "
2929 "address or the ID is invalid.");
2932 case llvm::OpenMPIRBuilder::EMIT_MD_DECLARE_TARGET_ERROR: {
2935 "variable %0 is incorrect: the "
2936 "address is invalid.");
2939 case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR: {
2942 "Offloading entry for declare target variable is incorrect: the "
2943 "address is invalid.");
2949 OMPBuilder.createOffloadEntriesAndInfoMetadata(ErrorReportFn);
2956 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty,
C.VoidPtrTy};
2959 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
2965struct PrivateHelpersTy {
2966 PrivateHelpersTy(
const Expr *OriginalRef,
const VarDecl *Original,
2968 : OriginalRef(OriginalRef), Original(Original), PrivateCopy(PrivateCopy),
2969 PrivateElemInit(PrivateElemInit) {}
2970 PrivateHelpersTy(
const VarDecl *Original) : Original(Original) {}
2971 const Expr *OriginalRef =
nullptr;
2972 const VarDecl *Original =
nullptr;
2973 const VarDecl *PrivateCopy =
nullptr;
2974 const VarDecl *PrivateElemInit =
nullptr;
2975 bool isLocalPrivate()
const {
2976 return !OriginalRef && !PrivateCopy && !PrivateElemInit;
2979typedef std::pair<
CharUnits , PrivateHelpersTy> PrivateDataTy;
2984 if (!CVD->
hasAttr<OMPAllocateDeclAttr>())
2986 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
2988 return !(AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc &&
2989 !AA->getAllocator());
2994 if (!Privates.empty()) {
2999 RecordDecl *RD =
C.buildImplicitRecord(
".kmp_privates.t");
3001 for (
const auto &Pair : Privates) {
3002 const VarDecl *VD = Pair.second.Original;
3006 if (Pair.second.isLocalPrivate()) {
3029 QualType KmpRoutineEntryPointerQTy) {
3049 QualType KmpCmplrdataTy =
C.getRecordType(UD);
3050 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_task_t");
3080 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_task_t_with_privates");
3100static llvm::Function *
3103 QualType KmpTaskTWithPrivatesPtrQTy,
3105 QualType SharedsPtrTy, llvm::Function *TaskFunction,
3106 llvm::Value *TaskPrivatesMap) {
3114 Args.push_back(&GtidArg);
3115 Args.push_back(&TaskTypeArg);
3116 const auto &TaskEntryFnInfo =
3118 llvm::FunctionType *TaskEntryTy =
3121 auto *TaskEntry = llvm::Function::Create(
3122 TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.
getModule());
3124 TaskEntry->setDoesNotRecurse();
3139 const auto *KmpTaskTWithPrivatesQTyRD =
3140 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
3143 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->
getAsTagDecl());
3144 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
3146 llvm::Value *PartidParam = PartIdLVal.
getPointer(CGF);
3148 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
3154 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
3155 llvm::Value *PrivatesParam;
3156 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
3161 PrivatesParam = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
3164 llvm::Value *CommonArgs[] = {
3165 GtidParam, PartidParam, PrivatesParam, TaskPrivatesMap,
3171 std::end(CommonArgs));
3173 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
3176 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
3179 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
3182 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
3185 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
3188 CallArgs.push_back(LBParam);
3189 CallArgs.push_back(UBParam);
3190 CallArgs.push_back(StParam);
3191 CallArgs.push_back(LIParam);
3192 CallArgs.push_back(RParam);
3194 CallArgs.push_back(SharedsParam);
3207 QualType KmpTaskTWithPrivatesPtrQTy,
3208 QualType KmpTaskTWithPrivatesQTy) {
3216 Args.push_back(&GtidArg);
3217 Args.push_back(&TaskTypeArg);
3218 const auto &DestructorFnInfo =
3220 llvm::FunctionType *DestructorFnTy =
3224 auto *DestructorFn =
3225 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
3229 DestructorFn->setDoesNotRecurse();
3237 const auto *KmpTaskTWithPrivatesQTyRD =
3238 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
3239 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
3241 for (
const auto *Field :
3242 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
3244 Field->getType().isDestructedType()) {
3250 return DestructorFn;
3270 C,
nullptr, Loc,
nullptr,
3271 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
3273 Args.push_back(&TaskPrivatesArg);
3274 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>,
unsigned> PrivateVarsPos;
3275 unsigned Counter = 1;
3276 for (
const Expr *E :
Data.PrivateVars) {
3278 C,
nullptr, Loc,
nullptr,
3279 C.getPointerType(
C.getPointerType(E->
getType()))
3283 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3284 PrivateVarsPos[VD] = Counter;
3287 for (
const Expr *E :
Data.FirstprivateVars) {
3289 C,
nullptr, Loc,
nullptr,
3290 C.getPointerType(
C.getPointerType(E->
getType()))
3294 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3295 PrivateVarsPos[VD] = Counter;
3298 for (
const Expr *E :
Data.LastprivateVars) {
3300 C,
nullptr, Loc,
nullptr,
3301 C.getPointerType(
C.getPointerType(E->
getType()))
3305 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3306 PrivateVarsPos[VD] = Counter;
3312 Ty =
C.getPointerType(Ty);
3314 Ty =
C.getPointerType(Ty);
3316 C,
nullptr, Loc,
nullptr,
3317 C.getPointerType(
C.getPointerType(Ty)).withConst().withRestrict(),
3319 PrivateVarsPos[VD] = Counter;
3322 const auto &TaskPrivatesMapFnInfo =
3324 llvm::FunctionType *TaskPrivatesMapTy =
3328 auto *TaskPrivatesMap = llvm::Function::Create(
3329 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
3332 TaskPrivatesMapFnInfo);
3334 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
3335 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
3336 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
3340 TaskPrivatesMapFnInfo, Args, Loc, Loc);
3346 const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->
getAsTagDecl());
3348 for (
const FieldDecl *Field : PrivatesQTyRD->fields()) {
3350 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
3359 return TaskPrivatesMap;
3371 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
3377 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
3385 if ((!IsTargetTask && !
Data.FirstprivateVars.empty() && ForDup) ||
3386 (IsTargetTask && KmpTaskSharedsPtr.
isValid())) {
3393 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
3394 for (
const PrivateDataTy &Pair : Privates) {
3396 if (Pair.second.isLocalPrivate()) {
3400 const VarDecl *VD = Pair.second.PrivateCopy;
3402 if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&
3405 if (
const VarDecl *Elem = Pair.second.PrivateElemInit) {
3406 const VarDecl *OriginalVD = Pair.second.Original;
3411 const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);
3412 if (IsTargetTask && !SharedField) {
3413 assert(isa<ImplicitParamDecl>(OriginalVD) &&
3416 ->getNumParams() == 0 &&
3417 isa<TranslationUnitDecl>(
3419 ->getDeclContext()) &&
3420 "Expected artificial target data variable.");
3423 }
else if (ForDup) {
3427 C.getDeclAlign(OriginalVD)),
3431 Pair.second.Original->getCanonicalDecl()) > 0 ||
3433 SharedRefLValue = CGF.
EmitLValue(Pair.second.OriginalRef);
3436 InlinedOpenMPRegionRAII Region(
3439 SharedRefLValue = CGF.
EmitLValue(Pair.second.OriginalRef);
3452 [&CGF, Elem, Init, &CapturesInfo](
Address DestElement,
3455 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3456 InitScope.addPrivate(Elem, SrcElement);
3457 (void)InitScope.Privatize();
3459 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
3460 CGF, &CapturesInfo);
3461 CGF.EmitAnyExprToMem(Init, DestElement,
3462 Init->getType().getQualifiers(),
3467 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3468 InitScope.addPrivate(Elem, SharedRefLValue.
getAddress(CGF));
3469 (void)InitScope.Privatize();
3470 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
3485 bool InitRequired =
false;
3486 for (
const PrivateDataTy &Pair : Privates) {
3487 if (Pair.second.isLocalPrivate())
3489 const VarDecl *VD = Pair.second.PrivateCopy;
3491 InitRequired = InitRequired || (isa_and_nonnull<CXXConstructExpr>(Init) &&
3496 return InitRequired;
3513 QualType KmpTaskTWithPrivatesPtrQTy,
3521 KmpTaskTWithPrivatesPtrQTy,
3524 KmpTaskTWithPrivatesPtrQTy,
3528 Args.push_back(&DstArg);
3529 Args.push_back(&SrcArg);
3530 Args.push_back(&LastprivArg);
3531 const auto &TaskDupFnInfo =
3535 auto *TaskDup = llvm::Function::Create(
3536 TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.
getModule());
3538 TaskDup->setDoesNotRecurse();
3548 auto LIFI = std::next(KmpTaskTQTyRD->
field_begin(), KmpTaskTLastIter);
3550 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
3558 assert(!Privates.empty());
3560 if (!
Data.FirstprivateVars.empty()) {
3565 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
3573 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
3574 SharedsTy, SharedsPtrTy,
Data, Privates,
true);
3584 for (
const PrivateDataTy &
P : Privates) {
3585 if (
P.second.isLocalPrivate())
3587 QualType Ty =
P.second.Original->getType().getNonReferenceType();
3596class OMPIteratorGeneratorScope final
3597 :
public CodeGenFunction::OMPPrivateScope {
3602 OMPIteratorGeneratorScope() =
delete;
3603 OMPIteratorGeneratorScope(OMPIteratorGeneratorScope &) =
delete;
3631 CodeGenFunction::JumpDest &ContDest =
3633 CodeGenFunction::JumpDest &ExitDest =
3636 llvm::Value *N = Uppers[I];
3644 ? CGF.
Builder.CreateICmpSLT(CVal, N)
3645 : CGF.
Builder.CreateICmpULT(CVal, N);
3647 CGF.
Builder.CreateCondBr(Cmp, BodyBB, ExitDest.getBlock());
3654 ~OMPIteratorGeneratorScope() {
3664 CGF.
EmitBlock(ExitDests[I - 1].getBlock(), I == 1);
3670static std::pair<llvm::Value *, llvm::Value *>
3672 const auto *OASE = dyn_cast<OMPArrayShapingExpr>(E);
3675 const Expr *
Base = OASE->getBase();
3680 llvm::Value *SizeVal;
3683 SizeVal = CGF.
getTypeSize(OASE->getBase()->getType()->getPointeeType());
3684 for (
const Expr *SE : OASE->getDimensions()) {
3688 SizeVal = CGF.
Builder.CreateNUWMul(SizeVal, Sz);
3690 }
else if (
const auto *ASE =
3695 llvm::Value *UpAddr = CGF.
Builder.CreateConstGEP1_32(
3697 llvm::Value *LowIntPtr = CGF.
Builder.CreatePtrToInt(Addr, CGF.
SizeTy);
3698 llvm::Value *UpIntPtr = CGF.
Builder.CreatePtrToInt(UpAddr, CGF.
SizeTy);
3699 SizeVal = CGF.
Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
3703 return std::make_pair(Addr, SizeVal);
3708 QualType FlagsTy =
C.getIntTypeForBitwidth(32,
false);
3709 if (KmpTaskAffinityInfoTy.
isNull()) {
3711 C.buildImplicitRecord(
"kmp_task_affinity_info_t");
3717 KmpTaskAffinityInfoTy =
C.getRecordType(KmpAffinityInfoRD);
3724 llvm::Function *TaskFunction,
QualType SharedsTy,
3729 const auto *I =
Data.PrivateCopies.begin();
3730 for (
const Expr *E :
Data.PrivateVars) {
3731 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3732 Privates.emplace_back(
3734 PrivateHelpersTy(E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
3738 I =
Data.FirstprivateCopies.begin();
3739 const auto *IElemInitRef =
Data.FirstprivateInits.begin();
3740 for (
const Expr *E :
Data.FirstprivateVars) {
3741 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3742 Privates.emplace_back(
3745 E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
3746 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
3750 I =
Data.LastprivateCopies.begin();
3751 for (
const Expr *E :
Data.LastprivateVars) {
3752 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3753 Privates.emplace_back(
3755 PrivateHelpersTy(E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
3763 Privates.emplace_back(
C.getDeclAlign(VD), PrivateHelpersTy(VD));
3765 llvm::stable_sort(Privates,
3766 [](
const PrivateDataTy &L,
const PrivateDataTy &R) {
3767 return L.first > R.first;
3769 QualType KmpInt32Ty =
C.getIntTypeForBitwidth(32, 1);
3783 "Expected taskloop, task or target directive");
3792 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
3794 QualType KmpTaskTWithPrivatesQTy =
C.getRecordType(KmpTaskTWithPrivatesQTyRD);
3795 QualType KmpTaskTWithPrivatesPtrQTy =
3796 C.getPointerType(KmpTaskTWithPrivatesQTy);
3797 llvm::Type *KmpTaskTWithPrivatesTy = CGF.
ConvertType(KmpTaskTWithPrivatesQTy);
3798 llvm::Type *KmpTaskTWithPrivatesPtrTy =
3799 KmpTaskTWithPrivatesTy->getPointerTo();
3800 llvm::Value *KmpTaskTWithPrivatesTySize =
3802 QualType SharedsPtrTy =
C.getPointerType(SharedsTy);
3805 llvm::Value *TaskPrivatesMap =
nullptr;
3806 llvm::Type *TaskPrivatesMapTy =
3807 std::next(TaskFunction->arg_begin(), 3)->getType();
3808 if (!Privates.empty()) {
3809 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
3813 TaskPrivatesMap, TaskPrivatesMapTy);
3815 TaskPrivatesMap = llvm::ConstantPointerNull::get(
3816 cast<llvm::PointerType>(TaskPrivatesMapTy));
3822 KmpTaskTWithPrivatesQTy,
KmpTaskTQTy, SharedsPtrTy, TaskFunction,
3834 DestructorsFlag = 0x8,
3835 PriorityFlag = 0x20,
3836 DetachableFlag = 0x40,
3838 unsigned Flags =
Data.Tied ? TiedFlag : 0;
3839 bool NeedsCleanup =
false;
3840 if (!Privates.empty()) {
3844 Flags = Flags | DestructorsFlag;
3846 if (
Data.Priority.getInt())
3847 Flags = Flags | PriorityFlag;
3849 Flags = Flags | DetachableFlag;
3850 llvm::Value *TaskFlags =
3851 Data.Final.getPointer()
3852 ? CGF.
Builder.CreateSelect(
Data.Final.getPointer(),
3853 CGF.
Builder.getInt32(FinalFlag),
3855 : CGF.
Builder.getInt32(
Data.Final.getInt() ? FinalFlag : 0);
3856 TaskFlags = CGF.
Builder.CreateOr(TaskFlags, CGF.
Builder.getInt32(Flags));
3857 llvm::Value *SharedsSize =
CGM.
getSize(
C.getTypeSizeInChars(SharedsTy));
3859 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
3862 llvm::Value *NewTask;
3865 const Expr *Device =
nullptr;
3867 Device =
C->getDevice();
3869 llvm::Value *DeviceID;
3874 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
3875 AllocArgs.push_back(DeviceID);
3896 llvm::Value *Tid =
getThreadID(CGF, DC->getBeginLoc());
3897 Tid = CGF.
Builder.CreateIntCast(Tid, CGF.
IntTy,
false);
3900 CGM.
getModule(), OMPRTL___kmpc_task_allow_completion_event),
3901 {Loc, Tid, NewTask});
3912 llvm::Value *NumOfElements =
nullptr;
3913 unsigned NumAffinities = 0;
3915 if (
const Expr *Modifier =
C->getModifier()) {
3916 const auto *IE = cast<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts());
3917 for (
unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
3921 NumOfElements ? CGF.
Builder.CreateNUWMul(NumOfElements, Sz) : Sz;
3924 NumAffinities +=
C->varlist_size();
3929 enum RTLAffinityInfoFieldsTy { BaseAddr, Len, Flags };
3931 QualType KmpTaskAffinityInfoArrayTy;
3932 if (NumOfElements) {
3933 NumOfElements = CGF.
Builder.CreateNUWAdd(
3934 llvm::ConstantInt::get(CGF.
SizeTy, NumAffinities), NumOfElements);
3937 C.getIntTypeForBitwidth(
C.getTypeSize(
C.getSizeType()), 0),
3941 KmpTaskAffinityInfoArrayTy =
3949 NumOfElements = CGF.
Builder.CreateIntCast(NumOfElements, CGF.
Int32Ty,
3952 KmpTaskAffinityInfoArrayTy =
C.getConstantArrayType(
3954 llvm::APInt(
C.getTypeSize(
C.getSizeType()), NumAffinities),
nullptr,
3957 CGF.
CreateMemTemp(KmpTaskAffinityInfoArrayTy,
".affs.arr.addr");
3959 NumOfElements = llvm::ConstantInt::get(
CGM.
Int32Ty, NumAffinities,
3966 bool HasIterator =
false;
3968 if (
C->getModifier()) {
3972 for (
const Expr *E :
C->varlists()) {
3981 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
3986 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
4000 const Expr *Modifier =
C->getModifier();
4003 OMPIteratorGeneratorScope IteratorScope(
4005 for (
const Expr *E :
C->varlists()) {
4014 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
4019 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
4021 Idx = CGF.
Builder.CreateNUWAdd(
4022 Idx, llvm::ConstantInt::get(Idx->getType(), 1));
4037 CGM.
getModule(), OMPRTL___kmpc_omp_reg_task_with_affinity),
4038 {LocRef, GTid, NewTask, NumOfElements, AffinListPtr});
4040 llvm::Value *NewTaskNewTaskTTy =
4042 NewTask, KmpTaskTWithPrivatesPtrTy);
4044 KmpTaskTWithPrivatesQTy);
4055 *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds)),
4064 if (!Privates.empty()) {
4066 SharedsTy, SharedsPtrTy,
Data, Privates,
4071 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
4072 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy,
Data, Privates,
4073 !
Data.LastprivateVars.empty());
4077 enum {
Priority = 0, Destructors = 1 };
4079 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
4081 (*FI)->getType()->getAsUnionType()->getDecl();
4084 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4085 KmpTaskTWithPrivatesQTy);
4088 Data1LV, *std::next(KmpCmplrdataUD->
field_begin(), Destructors));
4094 if (
Data.Priority.getInt()) {
4096 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
4101 Result.NewTask = NewTask;
4102 Result.TaskEntry = TaskEntry;
4103 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
4105 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
4111 RTLDependenceKindTy DepKind;
4113 case OMPC_DEPEND_in:
4114 DepKind = RTLDependenceKindTy::DepIn;
4117 case OMPC_DEPEND_out:
4118 case OMPC_DEPEND_inout:
4119 DepKind = RTLDependenceKindTy::DepInOut;
4121 case OMPC_DEPEND_mutexinoutset:
4122 DepKind = RTLDependenceKindTy::DepMutexInOutSet;
4124 case OMPC_DEPEND_inoutset:
4125 DepKind = RTLDependenceKindTy::DepInOutSet;
4127 case OMPC_DEPEND_outallmemory:
4128 DepKind = RTLDependenceKindTy::DepOmpAllMem;
4130 case OMPC_DEPEND_source:
4131 case OMPC_DEPEND_sink:
4132 case OMPC_DEPEND_depobj:
4133 case OMPC_DEPEND_inoutallmemory:
4135 llvm_unreachable(
"Unknown task dependence type");
4143 FlagsTy =
C.getIntTypeForBitwidth(
C.getTypeSize(
C.BoolTy),
false);
4144 if (KmpDependInfoTy.
isNull()) {
4145 RecordDecl *KmpDependInfoRD =
C.buildImplicitRecord(
"kmp_depend_info");
4151 KmpDependInfoTy =
C.getRecordType(KmpDependInfoRD);
4155std::pair<llvm::Value *, LValue>
4169 Base.getAddress(CGF),
4170 llvm::ConstantInt::get(CGF.
IntPtrTy, -1,
true));
4177 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4179 return std::make_pair(NumDeps,
Base);
4183 llvm::PointerUnion<unsigned *, LValue *> Pos,
4194 OMPIteratorGeneratorScope IteratorScope(
4195 CGF, cast_or_null<OMPIteratorExpr>(
4196 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4198 for (
const Expr *E :
Data.DepExprs) {
4207 Addr = llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4208 Size = llvm::ConstantInt::get(CGF.
SizeTy, 0);
4211 if (
unsigned *
P = Pos.dyn_cast<
unsigned *>()) {
4215 assert(E &&
"Expected a non-null expression");
4225 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4230 static_cast<unsigned int>(RTLDependInfoFields::Len)));
4237 static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4239 llvm::ConstantInt::get(LLVMFlagsTy,
static_cast<unsigned int>(DepKind)),
4241 if (
unsigned *
P = Pos.dyn_cast<
unsigned *>()) {
4246 Idx = CGF.
Builder.CreateNUWAdd(Idx,
4247 llvm::ConstantInt::get(Idx->getType(), 1));
4256 assert(
Data.DepKind == OMPC_DEPEND_depobj &&
4257 "Expected depobj dependency kind.");
4262 OMPIteratorGeneratorScope IteratorScope(
4263 CGF, cast_or_null<OMPIteratorExpr>(
4264 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4266 for (
const Expr *E :
Data.DepExprs) {
4267 llvm::Value *NumDeps;
4270 std::tie(NumDeps,
Base) =
4274 C.getUIntPtrType());
4278 llvm::Value *Add = CGF.
Builder.CreateNUWAdd(PrevVal, NumDeps);
4280 SizeLVals.push_back(NumLVal);
4283 for (
unsigned I = 0, E = SizeLVals.size(); I < E; ++I) {
4286 Sizes.push_back(Size);
4296 assert(
Data.DepKind == OMPC_DEPEND_depobj &&
4297 "Expected depobj dependency kind.");
4300 OMPIteratorGeneratorScope IteratorScope(
4301 CGF, cast_or_null<OMPIteratorExpr>(
4302 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4304 for (
unsigned I = 0, End =
Data.DepExprs.size(); I < End; ++I) {
4306 llvm::Value *NumDeps;
4309 std::tie(NumDeps,
Base) =
4313 llvm::Value *Size = CGF.
Builder.CreateNUWMul(
4322 llvm::Value *Add = CGF.
Builder.CreateNUWAdd(Pos, NumDeps);
4338 llvm::Value *NumOfElements =
nullptr;
4339 unsigned NumDependencies = std::accumulate(
4340 Dependencies.begin(), Dependencies.end(), 0,
4342 return D.DepKind == OMPC_DEPEND_depobj
4344 : (V + (D.IteratorExpr ? 0 : D.DepExprs.size()));
4348 bool HasDepobjDeps =
false;
4349 bool HasRegularWithIterators =
false;
4350 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4351 llvm::Value *NumOfRegularWithIterators =
4352 llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4356 if (D.
DepKind == OMPC_DEPEND_depobj) {
4359 for (llvm::Value *Size : Sizes) {
4360 NumOfDepobjElements =
4361 CGF.
Builder.CreateNUWAdd(NumOfDepobjElements, Size);
4363 HasDepobjDeps =
true;
4368 if (
const auto *IE = cast_or_null<OMPIteratorExpr>(D.
IteratorExpr)) {
4372 llvm::Value *NumClauseDeps = CGF.
Builder.CreateNUWMul(
4374 NumOfRegularWithIterators =
4375 CGF.
Builder.CreateNUWAdd(NumOfRegularWithIterators, NumClauseDeps);
4377 HasRegularWithIterators =
true;
4383 if (HasDepobjDeps || HasRegularWithIterators) {
4384 NumOfElements = llvm::ConstantInt::get(
CGM.
IntPtrTy, NumDependencies,
4386 if (HasDepobjDeps) {
4388 CGF.
Builder.CreateNUWAdd(NumOfDepobjElements, NumOfElements);
4390 if (HasRegularWithIterators) {
4392 CGF.
Builder.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements);
4395 Loc,
C.getIntTypeForBitwidth(64, 0),
4399 KmpDependInfoArrayTy =
4408 NumOfElements = CGF.
Builder.CreateIntCast(NumOfElements, CGF.
Int32Ty,
4411 KmpDependInfoArrayTy =
C.getConstantArrayType(
4417 NumOfElements = llvm::ConstantInt::get(
CGM.
Int32Ty, NumDependencies,
4421 for (
unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4422 if (Dependencies[I].DepKind == OMPC_DEPEND_depobj ||
4423 Dependencies[I].IteratorExpr)
4430 CGF.
CreateMemTemp(
C.getSizeType(),
"dep.counter.addr"),
C.getSizeType());
4432 for (
unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4433 if (Dependencies[I].DepKind == OMPC_DEPEND_depobj ||
4434 !Dependencies[I].IteratorExpr)
4440 if (HasDepobjDeps) {
4441 for (
unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4442 if (Dependencies[I].DepKind != OMPC_DEPEND_depobj)
4450 return std::make_pair(NumOfElements, DependenciesArray);
4461 unsigned NumDependencies = Dependencies.
DepExprs.size();
4472 llvm::Value *NumDepsVal;
4474 if (
const auto *IE =
4475 cast_or_null<OMPIteratorExpr>(Dependencies.
IteratorExpr)) {
4476 NumDepsVal = llvm::ConstantInt::get(CGF.
SizeTy, 1);
4480 NumDepsVal = CGF.
Builder.CreateNUWMul(NumDepsVal, Sz);
4482 Size = CGF.
Builder.CreateNUWAdd(llvm::ConstantInt::get(CGF.
SizeTy, 1),
4486 llvm::Value *RecSize =
CGM.
getSize(SizeInBytes);
4487 Size = CGF.
Builder.CreateNUWMul(Size, RecSize);
4491 QualType KmpDependInfoArrayTy =
C.getConstantArrayType(
4494 CharUnits Sz =
C.getTypeSizeInChars(KmpDependInfoArrayTy);
4496 NumDepsVal = llvm::ConstantInt::get(CGF.
IntPtrTy, NumDependencies);
4501 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4502 llvm::Value *Args[] = {ThreadID, Size, Allocator};
4507 Args,
".dep.arr.addr");
4510 Addr, KmpDependInfoLlvmTy->getPointerTo());
4511 DependenciesArray =
Address(Addr, KmpDependInfoLlvmTy, Align);
4518 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4520 llvm::PointerUnion<unsigned *, LValue *> Pos;