30#include "llvm/ADT/ArrayRef.h"
31#include "llvm/ADT/SetOperations.h"
32#include "llvm/ADT/SmallBitVector.h"
33#include "llvm/ADT/StringExtras.h"
34#include "llvm/Bitcode/BitcodeReader.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DerivedTypes.h"
37#include "llvm/IR/GlobalValue.h"
38#include "llvm/IR/InstrTypes.h"
39#include "llvm/IR/Value.h"
40#include "llvm/Support/AtomicOrdering.h"
41#include "llvm/Support/Format.h"
42#include "llvm/Support/raw_ostream.h"
48using namespace CodeGen;
49using namespace llvm::omp;
53class CGOpenMPRegionInfo :
public CodeGenFunction::CGCapturedStmtInfo {
56 enum CGOpenMPRegionKind {
59 ParallelOutlinedRegion,
70 const CGOpenMPRegionKind RegionKind,
73 : CGCapturedStmtInfo(CS,
CR_OpenMP), RegionKind(RegionKind),
74 CodeGen(CodeGen),
Kind(
Kind), HasCancel(HasCancel) {}
76 CGOpenMPRegionInfo(
const CGOpenMPRegionKind RegionKind,
79 : CGCapturedStmtInfo(
CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
84 virtual const VarDecl *getThreadIDVariable()
const = 0;
95 CGOpenMPRegionKind getRegionKind()
const {
return RegionKind; }
99 bool hasCancel()
const {
return HasCancel; }
101 static bool classof(
const CGCapturedStmtInfo *Info) {
105 ~CGOpenMPRegionInfo()
override =
default;
108 CGOpenMPRegionKind RegionKind;
115class CGOpenMPOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
120 StringRef HelperName)
121 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen,
Kind,
123 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
124 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
129 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
132 StringRef getHelperName()
const override {
return HelperName; }
134 static bool classof(
const CGCapturedStmtInfo *Info) {
135 return CGOpenMPRegionInfo::classof(Info) &&
136 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
137 ParallelOutlinedRegion;
144 StringRef HelperName;
148class CGOpenMPTaskOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
154 llvm::SwitchInst *UntiedSwitch =
nullptr;
157 UntiedTaskActionTy(
bool Tied,
const VarDecl *PartIDVar,
159 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
169 UntiedSwitch = CGF.
Builder.CreateSwitch(Res, DoneBB);
173 UntiedSwitch->addCase(CGF.
Builder.getInt32(0),
175 emitUntiedSwitch(CGF);
186 CodeGenFunction::JumpDest CurPoint =
190 UntiedSwitch->addCase(CGF.
Builder.getInt32(UntiedSwitch->getNumCases()),
196 unsigned getNumberOfParts()
const {
return UntiedSwitch->getNumCases(); }
202 const UntiedTaskActionTy &Action)
203 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen,
Kind, HasCancel),
204 ThreadIDVar(ThreadIDVar), Action(Action) {
205 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
210 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
216 StringRef getHelperName()
const override {
return ".omp_outlined."; }
219 Action.emitUntiedSwitch(CGF);
222 static bool classof(
const CGCapturedStmtInfo *Info) {
223 return CGOpenMPRegionInfo::classof(Info) &&
224 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
233 const UntiedTaskActionTy &Action;
238class CGOpenMPInlinedRegionInfo :
public CGOpenMPRegionInfo {
240 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
243 : CGOpenMPRegionInfo(InlinedRegion, CodeGen,
Kind, HasCancel),
245 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
248 llvm::Value *getContextValue()
const override {
250 return OuterRegionInfo->getContextValue();
251 llvm_unreachable(
"No context value for inlined OpenMP region");
254 void setContextValue(llvm::Value *
V)
override {
255 if (OuterRegionInfo) {
256 OuterRegionInfo->setContextValue(
V);
259 llvm_unreachable(
"No context value for inlined OpenMP region");
265 return OuterRegionInfo->lookup(VD);
271 FieldDecl *getThisFieldDecl()
const override {
273 return OuterRegionInfo->getThisFieldDecl();
279 const VarDecl *getThreadIDVariable()
const override {
281 return OuterRegionInfo->getThreadIDVariable();
288 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
289 llvm_unreachable(
"No LValue for inlined OpenMP construct");
293 StringRef getHelperName()
const override {
294 if (
auto *OuterRegionInfo = getOldCSI())
295 return OuterRegionInfo->getHelperName();
296 llvm_unreachable(
"No helper name for inlined OpenMP construct");
301 OuterRegionInfo->emitUntiedSwitch(CGF);
304 CodeGenFunction::CGCapturedStmtInfo *getOldCSI()
const {
return OldCSI; }
306 static bool classof(
const CGCapturedStmtInfo *Info) {
307 return CGOpenMPRegionInfo::classof(Info) &&
308 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
311 ~CGOpenMPInlinedRegionInfo()
override =
default;
315 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
316 CGOpenMPRegionInfo *OuterRegionInfo;
324class CGOpenMPTargetRegionInfo final :
public CGOpenMPRegionInfo {
328 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
330 HelperName(HelperName) {}
334 const VarDecl *getThreadIDVariable()
const override {
return nullptr; }
337 StringRef getHelperName()
const override {
return HelperName; }
339 static bool classof(
const CGCapturedStmtInfo *Info) {
340 return CGOpenMPRegionInfo::classof(Info) &&
341 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
345 StringRef HelperName;
349 llvm_unreachable(
"No codegen for expressions");
353class CGOpenMPInnerExprInfo final :
public CGOpenMPInlinedRegionInfo {
356 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
364 if (!
C.capturesVariable() && !
C.capturesVariableByCopy())
367 const VarDecl *VD =
C.getCapturedVar();
377 (void)PrivScope.Privatize();
382 if (
const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
389 llvm_unreachable(
"No body for expressions");
394 const VarDecl *getThreadIDVariable()
const override {
395 llvm_unreachable(
"No thread id for expressions");
399 StringRef getHelperName()
const override {
400 llvm_unreachable(
"No helper name for expressions");
403 static bool classof(
const CGCapturedStmtInfo *Info) {
return false; }
407 CodeGenFunction::OMPPrivateScope PrivScope;
411class InlinedOpenMPRegionRAII {
413 llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
414 FieldDecl *LambdaThisCaptureField =
nullptr;
416 bool NoInheritance =
false;
425 bool NoInheritance =
true)
426 : CGF(CGF), NoInheritance(NoInheritance) {
439 ~InlinedOpenMPRegionRAII() {
456enum OpenMPLocationFlags :
unsigned {
458 OMP_IDENT_IMD = 0x01,
460 OMP_IDENT_KMPC = 0x02,
462 OMP_ATOMIC_REDUCE = 0x10,
464 OMP_IDENT_BARRIER_EXPL = 0x20,
466 OMP_IDENT_BARRIER_IMPL = 0x40,
468 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
470 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
472 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
474 OMP_IDENT_WORK_LOOP = 0x200,
476 OMP_IDENT_WORK_SECTIONS = 0x400,
478 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
485enum OpenMPOffloadingRequiresDirFlags :
int64_t {
487 OMP_REQ_UNDEFINED = 0x000,
489 OMP_REQ_NONE = 0x001,
491 OMP_REQ_REVERSE_OFFLOAD = 0x002,
493 OMP_REQ_UNIFIED_ADDRESS = 0x004,
495 OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008,
497 OMP_REQ_DYNAMIC_ALLOCATORS = 0x010,
529enum IdentFieldIndex {
531 IdentField_Reserved_1,
535 IdentField_Reserved_2,
537 IdentField_Reserved_3,
546enum OpenMPSchedType {
549 OMP_sch_static_chunked = 33,
551 OMP_sch_dynamic_chunked = 35,
552 OMP_sch_guided_chunked = 36,
553 OMP_sch_runtime = 37,
556 OMP_sch_static_balanced_chunked = 45,
559 OMP_ord_static_chunked = 65,
561 OMP_ord_dynamic_chunked = 67,
562 OMP_ord_guided_chunked = 68,
563 OMP_ord_runtime = 69,
565 OMP_sch_default = OMP_sch_static,
567 OMP_dist_sch_static_chunked = 91,
568 OMP_dist_sch_static = 92,
571 OMP_sch_modifier_monotonic = (1 << 29),
573 OMP_sch_modifier_nonmonotonic = (1 << 30),
596 Callback(CodeGen, CGF, *PrePostAction);
599 Callback(CodeGen, CGF, Action);
607 if (
const auto *CE = dyn_cast<CallExpr>(ReductionOp))
608 if (
const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
609 if (
const auto *DRE =
610 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
611 if (
const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
622 std::pair<llvm::Function *, llvm::Function *> Reduction =
624 const auto *CE = cast<CallExpr>(InitOp);
625 const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
629 cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
631 cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
632 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
633 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()), Private);
634 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()), Original);
635 (void)PrivateScope.Privatize();
637 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
642 auto *GV =
new llvm::GlobalVariable(
644 llvm::GlobalValue::PrivateLinkage, Init, Name);
657 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, LV);
664 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
685 llvm::Value *NumElements = CGF.
emitArrayLength(ArrayTy, ElementTy, DestAddr);
690 llvm::Value *SrcBegin =
nullptr;
693 llvm::Value *DestBegin = DestAddr.
getPointer();
695 llvm::Value *DestEnd =
700 llvm::Value *IsEmpty =
701 CGF.
Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arrayinit.isempty");
702 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
705 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
710 llvm::PHINode *SrcElementPHI =
nullptr;
713 SrcElementPHI = CGF.
Builder.CreatePHI(SrcBegin->getType(), 2,
714 "omp.arraycpy.srcElementPast");
715 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
717 Address(SrcElementPHI, SrcAddr.getElementType(),
718 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
720 llvm::PHINode *DestElementPHI = CGF.
Builder.CreatePHI(
721 DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
722 DestElementPHI->addIncoming(DestBegin, EntryBB);
729 CodeGenFunction::RunCleanupsScope InitScope(CGF);
730 if (EmitDeclareReductionInit) {
732 SrcElementCurrent, ElementTy);
740 llvm::Value *SrcElementNext = CGF.
Builder.CreateConstGEP1_32(
741 SrcAddr.getElementType(), SrcElementPHI, 1,
742 "omp.arraycpy.dest.element");
743 SrcElementPHI->addIncoming(SrcElementNext, CGF.
Builder.GetInsertBlock());
747 llvm::Value *DestElementNext = CGF.
Builder.CreateConstGEP1_32(
749 "omp.arraycpy.dest.element");
752 CGF.
Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
753 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
754 DestElementPHI->addIncoming(DestElementNext, CGF.
Builder.GetInsertBlock());
766 if (
const auto *OASE = dyn_cast<OMPArraySectionExpr>(E))
771void ReductionCodeGen::emitAggregateInitialization(
777 const auto *PrivateVD =
778 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
779 bool EmitDeclareReductionInit =
782 EmitDeclareReductionInit,
783 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
784 : PrivateVD->getInit(),
792 ClausesData.reserve(Shareds.size());
793 SharedAddresses.reserve(Shareds.size());
794 Sizes.reserve(Shareds.size());
795 BaseDecls.reserve(Shareds.size());
796 const auto *IOrig = Origs.begin();
797 const auto *IPriv = Privates.begin();
798 const auto *IRed = ReductionOps.begin();
799 for (
const Expr *Ref : Shareds) {
800 ClausesData.emplace_back(Ref, *IOrig, *IPriv, *IRed);
801 std::advance(IOrig, 1);
802 std::advance(IPriv, 1);
803 std::advance(IRed, 1);
808 assert(SharedAddresses.size() == N && OrigAddresses.size() == N &&
809 "Number of generated lvalues must be exactly N.");
810 LValue First = emitSharedLValue(CGF, ClausesData[N].Shared);
811 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Shared);
812 SharedAddresses.emplace_back(
First, Second);
813 if (ClausesData[N].Shared == ClausesData[N].Ref) {
814 OrigAddresses.emplace_back(
First, Second);
816 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
817 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
818 OrigAddresses.emplace_back(
First, Second);
824 bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
827 CGF.
getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()),
832 llvm::Value *SizeInChars;
833 auto *ElemType = OrigAddresses[N].first.getAddress(CGF).getElementType();
834 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
835 if (AsArraySection) {
836 Size = CGF.
Builder.CreatePtrDiff(ElemType,
837 OrigAddresses[N].second.getPointer(CGF),
838 OrigAddresses[N].first.getPointer(CGF));
839 Size = CGF.
Builder.CreateNUWAdd(
840 Size, llvm::ConstantInt::get(Size->getType(), 1));
841 SizeInChars = CGF.
Builder.CreateNUWMul(Size, ElemSizeOf);
844 CGF.
getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType());
845 Size = CGF.
Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
847 Sizes.emplace_back(SizeInChars, Size);
850 cast<OpaqueValueExpr>(
860 assert(!Size && !Sizes[N].second &&
861 "Size should be nullptr for non-variably modified reduction "
867 cast<OpaqueValueExpr>(
876 assert(SharedAddresses.size() > N &&
"No variable was generated");
877 const auto *PrivateVD =
878 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
883 (void)DefaultInit(CGF);
884 emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD);
885 }
else if (DRD && (DRD->
getInitializer() || !PrivateVD->hasInit())) {
886 (void)DefaultInit(CGF);
887 QualType SharedType = SharedAddresses[N].first.getType();
889 PrivateAddr, SharedAddr, SharedType);
890 }
else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
893 PrivateVD->
getType().getQualifiers(),
911 CGF.
pushDestroy(DTorKind, PrivateAddr, PrivateType);
936 Address OriginalBaseAddress, llvm::Value *Addr) {
960 Addr, OriginalBaseAddress.
getType());
965 const VarDecl *OrigVD =
nullptr;
966 if (
const auto *OASE = dyn_cast<OMPArraySectionExpr>(Ref)) {
967 const Expr *
Base = OASE->getBase()->IgnoreParenImpCasts();
968 while (
const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(
Base))
969 Base = TempOASE->getBase()->IgnoreParenImpCasts();
970 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
971 Base = TempASE->getBase()->IgnoreParenImpCasts();
972 DE = cast<DeclRefExpr>(
Base);
973 OrigVD = cast<VarDecl>(DE->
getDecl());
974 }
else if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
975 const Expr *
Base = ASE->getBase()->IgnoreParenImpCasts();
976 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
977 Base = TempASE->getBase()->IgnoreParenImpCasts();
978 DE = cast<DeclRefExpr>(
Base);
979 OrigVD = cast<VarDecl>(DE->
getDecl());
988 BaseDecls.emplace_back(OrigVD);
991 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
993 Address SharedAddr = SharedAddresses[N].first.getAddress(CGF);
994 llvm::Value *Adjustment = CGF.
Builder.CreatePtrDiff(
997 llvm::Value *PrivatePointer =
1003 SharedAddresses[N].first.getType(),
1006 BaseDecls.emplace_back(
1007 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
1020 getThreadIDVariable()->getType()->castAs<PointerType>());
1038LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1041 getThreadIDVariable()->getType(),
1057 : CGM(CGM), OMPBuilder(CGM.getModule()) {
1059 llvm::OpenMPIRBuilderConfig Config(
CGM.
getLangOpts().OpenMPIsDevice,
false,
1072 if (!
Data.getValue().pointsToAliveValue())
1074 auto *GV = dyn_cast<llvm::GlobalVariable>(
Data.getValue());
1077 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1079 GV->eraseFromParent();
1084 return OMPBuilder.createPlatformSpecificName(Parts);
1087static llvm::Function *
1089 const Expr *CombinerInitializer,
const VarDecl *In,
1090 const VarDecl *Out,
bool IsCombiner) {
1093 QualType PtrTy =
C.getPointerType(Ty).withRestrict();
1099 Args.push_back(&OmpOutParm);
1100 Args.push_back(&OmpInParm);
1105 {IsCombiner ?
"omp_combiner" :
"omp_initializer",
""});
1106 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1110 Fn->removeFnAttr(llvm::Attribute::NoInline);
1111 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1112 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1118 Out->getLocation());
1119 CodeGenFunction::OMPPrivateScope
Scope(CGF);
1128 (void)
Scope.Privatize();
1129 if (!IsCombiner && Out->hasInit() &&
1132 Out->getType().getQualifiers(),
1135 if (CombinerInitializer)
1137 Scope.ForceCleanup();
1148 cast<VarDecl>(cast<DeclRefExpr>(D->
getCombinerIn())->getDecl()),
1149 cast<VarDecl>(cast<DeclRefExpr>(D->
getCombinerOut())->getDecl()),
1157 cast<VarDecl>(cast<DeclRefExpr>(D->
getInitOrig())->getDecl()),
1158 cast<VarDecl>(cast<DeclRefExpr>(D->
getInitPriv())->getDecl()),
1164 Decls.second.push_back(D);
1168std::pair<llvm::Function *, llvm::Function *>
1180struct PushAndPopStackRAII {
1181 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder,
CodeGenFunction &CGF,
1182 bool HasCancel, llvm::omp::Directive Kind)
1183 : OMPBuilder(OMPBuilder) {
1199 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1200 assert(IP.getBlock()->end() == IP.getPoint() &&
1201 "Clang CG should cause non-terminated block!");
1202 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1211 llvm::OpenMPIRBuilder::FinalizationInfo FI({FiniCB, Kind, HasCancel});
1212 OMPBuilder->pushFinalizationCB(std::move(FI));
1214 ~PushAndPopStackRAII() {
1216 OMPBuilder->popFinalizationCB();
1218 llvm::OpenMPIRBuilder *OMPBuilder;
1227 "thread id variable must be of type kmp_int32 *");
1229 bool HasCancel =
false;
1230 if (
const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1231 HasCancel = OPD->hasCancel();
1232 else if (
const auto *OPD = dyn_cast<OMPTargetParallelDirective>(&D))
1233 HasCancel = OPD->hasCancel();
1234 else if (
const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1235 HasCancel = OPSD->hasCancel();
1236 else if (
const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1237 HasCancel = OPFD->hasCancel();
1238 else if (
const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1239 HasCancel = OPFD->hasCancel();
1240 else if (
const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1241 HasCancel = OPFD->hasCancel();
1242 else if (
const auto *OPFD =
1243 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1244 HasCancel = OPFD->hasCancel();
1245 else if (
const auto *OPFD =
1246 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1247 HasCancel = OPFD->hasCancel();
1252 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind);
1253 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1254 HasCancel, OutlinedHelperName);
1255 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1279 bool Tied,
unsigned &NumberOfParts) {
1284 llvm::Value *TaskArgs[] = {
1286 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1289 CGF.EmitRuntimeCall(
OMPBuilder.getOrCreateRuntimeFunction(
1293 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1297 "thread id variable must be of type kmp_int32 for tasks");
1302 bool HasCancel =
false;
1303 if (
const auto *TD = dyn_cast<OMPTaskDirective>(&D))
1304 HasCancel = TD->hasCancel();
1305 else if (
const auto *TD = dyn_cast<OMPTaskLoopDirective>(&D))
1306 HasCancel = TD->hasCancel();
1307 else if (
const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(&D))
1308 HasCancel = TD->hasCancel();
1309 else if (
const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(&D))
1310 HasCancel = TD->hasCancel();
1313 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1314 InnermostKind, HasCancel, Action);
1316 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1318 NumberOfParts = Action.getNumberOfParts();
1323 bool AtCurrentPoint) {
1325 assert(!Elem.second.ServiceInsertPt &&
"Insert point is set already.");
1327 llvm::Value *Undef = llvm::UndefValue::get(CGF.
Int32Ty);
1328 if (AtCurrentPoint) {
1329 Elem.second.ServiceInsertPt =
new llvm::BitCastInst(
1332 Elem.second.ServiceInsertPt =
1333 new llvm::BitCastInst(Undef, CGF.
Int32Ty,
"svcpt");
1340 if (Elem.second.ServiceInsertPt) {
1341 llvm::Instruction *Ptr = Elem.second.ServiceInsertPt;
1342 Elem.second.ServiceInsertPt =
nullptr;
1343 Ptr->eraseFromParent();
1350 llvm::raw_svector_ostream
OS(Buffer);
1354 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.
CurFuncDecl))
1355 OS << FD->getQualifiedNameAsString();
1362 unsigned Flags,
bool EmitLoc) {
1363 uint32_t SrcLocStrSize;
1364 llvm::Constant *SrcLocStr;
1366 llvm::codegenoptions::NoDebugInfo) ||
1368 SrcLocStr =
OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1370 std::string FunctionName;
1371 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.
CurFuncDecl))
1372 FunctionName = FD->getQualifiedNameAsString();
1375 unsigned Line = PLoc.
getLine();
1377 SrcLocStr =
OMPBuilder.getOrCreateSrcLocStr(FunctionName, FileName, Line,
1378 Column, SrcLocStrSize);
1382 SrcLocStr, SrcLocStrSize, llvm::omp::IdentFlag(Flags), Reserved2Flags);
1387 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1393 uint32_t SrcLocStrSize;
1394 auto *SrcLocStr =
OMPBuilder.getOrCreateSrcLocStr(
1397 OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize));
1400 llvm::Value *ThreadID =
nullptr;
1405 ThreadID = I->second.ThreadID;
1406 if (ThreadID !=
nullptr)
1410 if (
auto *OMPRegionInfo =
1412 if (OMPRegionInfo->getThreadIDVariable()) {
1414 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1418 CGF.
Builder.GetInsertBlock() == TopBlock ||
1419 !isa<llvm::Instruction>(LVal.
getPointer(CGF)) ||
1420 cast<llvm::Instruction>(LVal.
getPointer(CGF))->getParent() ==
1422 cast<llvm::Instruction>(LVal.
getPointer(CGF))->getParent() ==
1423 CGF.
Builder.GetInsertBlock()) {
1427 if (CGF.
Builder.GetInsertBlock() == TopBlock) {
1429 Elem.second.ThreadID = ThreadID;
1441 if (!Elem.second.ServiceInsertPt)
1443 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1444 CGF.
Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1445 llvm::CallInst *Call = CGF.
Builder.CreateCall(
1447 OMPRTL___kmpc_global_thread_num),
1450 Elem.second.ThreadID = Call;
1455 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1467 for(
const auto *D : I->second)
1482 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(
CGM.
Int32Ty),
1491 bool IsGPUDistribute) {
1492 assert((IVSize == 32 || IVSize == 64) &&
1493 "IV size is not compatible with the omp runtime");
1495 if (IsGPUDistribute)
1496 Name = IVSize == 32 ? (IVSigned ?
"__kmpc_distribute_static_init_4"
1497 :
"__kmpc_distribute_static_init_4u")
1498 : (IVSigned ?
"__kmpc_distribute_static_init_8"
1499 :
"__kmpc_distribute_static_init_8u");
1501 Name = IVSize == 32 ? (IVSigned ?
"__kmpc_for_static_init_4"
1502 :
"__kmpc_for_static_init_4u")
1503 : (IVSigned ?
"__kmpc_for_static_init_8"
1504 :
"__kmpc_for_static_init_8u");
1507 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
1508 llvm::Type *TypeParams[] = {
1520 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1526 assert((IVSize == 32 || IVSize == 64) &&
1527 "IV size is not compatible with the omp runtime");
1530 ? (IVSigned ?
"__kmpc_dispatch_init_4" :
"__kmpc_dispatch_init_4u")
1531 : (IVSigned ?
"__kmpc_dispatch_init_8" :
"__kmpc_dispatch_init_8u");
1542 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1548 assert((IVSize == 32 || IVSize == 64) &&
1549 "IV size is not compatible with the omp runtime");
1552 ? (IVSigned ?
"__kmpc_dispatch_fini_4" :
"__kmpc_dispatch_fini_4u")
1553 : (IVSigned ?
"__kmpc_dispatch_fini_8" :
"__kmpc_dispatch_fini_8u");
1554 llvm::Type *TypeParams[] = {
1559 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1565 assert((IVSize == 32 || IVSize == 64) &&
1566 "IV size is not compatible with the omp runtime");
1569 ? (IVSigned ?
"__kmpc_dispatch_next_4" :
"__kmpc_dispatch_next_4u")
1570 : (IVSigned ?
"__kmpc_dispatch_next_8" :
"__kmpc_dispatch_next_8u");
1572 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
1573 llvm::Type *TypeParams[] = {
1582 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
1589static llvm::TargetRegionEntryInfo
1591 StringRef ParentName =
"") {
1597 assert(Loc.
isValid() &&
"Source location is expected to be always valid.");
1600 assert(PLoc.
isValid() &&
"Source location is expected to be always valid.");
1602 llvm::sys::fs::UniqueID ID;
1603 if (
auto EC = llvm::sys::fs::getUniqueID(PLoc.
getFilename(), ID)) {
1604 PLoc =
SM.getPresumedLoc(Loc,
false);
1605 assert(PLoc.
isValid() &&
"Source location is expected to be always valid.");
1606 if (
auto EC = llvm::sys::fs::getUniqueID(PLoc.
getFilename(), ID))
1607 SM.getDiagnostics().Report(diag::err_cannot_open_file)
1611 return llvm::TargetRegionEntryInfo(ParentName, ID.getDevice(), ID.getFile(),
1618 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
1619 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1620 if (Res && (*Res == OMPDeclareTargetDeclAttr::MT_Link ||
1621 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
1622 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
1626 llvm::raw_svector_ostream
OS(PtrName);
1631 OS << llvm::format(
"_%x", EntryInfo.FileID);
1633 OS <<
"_decl_tgt_ref_ptr";
1635 llvm::Value *Ptr =
CGM.
getModule().getNamedValue(PtrName);
1639 Ptr =
OMPBuilder.getOrCreateInternalVariable(LlvmPtrTy, PtrName);
1641 auto *GV = cast<llvm::GlobalVariable>(Ptr);
1642 GV->setLinkage(llvm::GlobalValue::WeakAnyLinkage);
1658 std::string Suffix =
getName({
"cache",
""});
1659 return OMPBuilder.getOrCreateInternalVariable(
1672 llvm::Value *Args[] = {
1696 llvm::Value *Args[] = {
1698 Ctor, CopyCtor, Dtor};
1701 CGM.
getModule(), OMPRTL___kmpc_threadprivate_register),
1716 llvm::Value *Ctor =
nullptr, *CopyCtor =
nullptr, *Dtor =
nullptr;
1726 Args.push_back(&Dst);
1731 std::string Name =
getName({
"__kmpc_global_ctor_",
""});
1732 llvm::Function *Fn =
1759 Args.push_back(&Dst);
1764 std::string Name =
getName({
"__kmpc_global_dtor_",
""});
1765 llvm::Function *Fn =
1787 auto *CopyCtorTy = llvm::FunctionType::get(
CGM.
VoidPtrTy, CopyCtorTyArgs,
1793 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
1794 if (Ctor ==
nullptr) {
1798 Ctor = llvm::Constant::getNullValue(CtorTy);
1800 if (Dtor ==
nullptr) {
1804 Dtor = llvm::Constant::getNullValue(DtorTy);
1807 auto *InitFunctionTy =
1808 llvm::FunctionType::get(
CGM.
VoidTy,
false);
1809 std::string Name =
getName({
"__omp_threadprivate_init_",
""});
1819 return InitFunction;
1827 llvm::GlobalVariable *Addr,
1832 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
1833 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1834 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
1835 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
1836 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
1840 assert(VD &&
"Unknown VarDecl");
1854 OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(Buffer, EntryInfo);
1858 llvm::Constant *Ctor;
1868 FTy, Twine(Buffer,
"_ctor"), FI, Loc,
false,
1869 llvm::GlobalValue::WeakODRLinkage);
1870 Fn->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1872 Fn->setCallingConv(llvm::CallingConv::AMDGPU_KERNEL);
1877 llvm::Constant *AddrInAS0 = Addr;
1878 if (Addr->getAddressSpace() != 0)
1879 AddrInAS0 = llvm::ConstantExpr::getAddrSpaceCast(
1880 Addr, llvm::PointerType::getWithSamePointeeType(
1881 cast<llvm::PointerType>(Addr->getType()), 0));
1883 Address(AddrInAS0, Addr->getValueType(),
1885 Init->getType().getQualifiers(),
1889 ID = llvm::ConstantExpr::getBitCast(Fn,
CGM.
Int8PtrTy);
1891 Ctor =
new llvm::GlobalVariable(
1893 llvm::GlobalValue::PrivateLinkage,
1894 llvm::Constant::getNullValue(
CGM.
Int8Ty), Twine(Buffer,
"_ctor"));
1900 auto CtorEntryInfo = EntryInfo;
1901 CtorEntryInfo.ParentName = Twine(Buffer,
"_ctor").toStringRef(Out);
1902 OMPBuilder.OffloadInfoManager.registerTargetRegionEntryInfo(
1903 CtorEntryInfo, Ctor, ID,
1904 llvm::OffloadEntriesInfoManager::OMPTargetRegionEntryCtor);
1907 llvm::Constant *Dtor;
1917 FTy, Twine(Buffer,
"_dtor"), FI, Loc,
false,
1918 llvm::GlobalValue::WeakODRLinkage);
1919 Fn->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1921 Fn->setCallingConv(llvm::CallingConv::AMDGPU_KERNEL);
1928 llvm::Constant *AddrInAS0 = Addr;
1929 if (Addr->getAddressSpace() != 0)
1930 AddrInAS0 = llvm::ConstantExpr::getAddrSpaceCast(
1931 Addr, llvm::PointerType::getWithSamePointeeType(
1932 cast<llvm::PointerType>(Addr->getType()), 0));
1939 ID = llvm::ConstantExpr::getBitCast(Fn,
CGM.
Int8PtrTy);
1941 Dtor =
new llvm::GlobalVariable(
1943 llvm::GlobalValue::PrivateLinkage,
1944 llvm::Constant::getNullValue(
CGM.
Int8Ty), Twine(Buffer,
"_dtor"));
1949 auto DtorEntryInfo = EntryInfo;
1950 DtorEntryInfo.ParentName = Twine(Buffer,
"_dtor").toStringRef(Out);
1951 OMPBuilder.OffloadInfoManager.registerTargetRegionEntryInfo(
1952 DtorEntryInfo, Dtor, ID,
1953 llvm::OffloadEntriesInfoManager::OMPTargetRegionEntryDtor);
1961 std::string Suffix =
getName({
"artificial",
""});
1963 llvm::GlobalVariable *GAddr =
OMPBuilder.getOrCreateInternalVariable(
1964 VarLVType, Twine(Name).concat(Suffix).str());
1967 GAddr->setThreadLocal(
true);
1968 return Address(GAddr, GAddr->getValueType(),
1971 std::string CacheSuffix =
getName({
"cache",
""});
1972 llvm::Value *Args[] = {
1980 Twine(Name).concat(Suffix).concat(CacheSuffix).str())};
1987 VarLVType->getPointerTo(0)),
2031 llvm::Function *OutlinedFn,
2034 llvm::Value *NumThreads) {
2039 auto &&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc,
2043 llvm::Value *Args[] = {
2045 CGF.
Builder.getInt32(CapturedVars.size()),
2048 RealArgs.append(std::begin(Args), std::end(Args));
2049 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
2051 llvm::FunctionCallee RTLFn =
2052 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_fork_call);
2055 auto &&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc, Loc,
2061 llvm::Value *Args[] = {RTLoc, ThreadID};
2063 M, OMPRTL___kmpc_serialized_parallel),
2070 ".bound.zero.addr");
2074 OutlinedFnArgs.push_back(ThreadIDAddr.
getPointer());
2075 OutlinedFnArgs.push_back(ZeroAddrBound.
getPointer());
2076 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
2084 OutlinedFn->removeFnAttr(llvm::Attribute::AlwaysInline);
2085 OutlinedFn->addFnAttr(llvm::Attribute::NoInline);
2091 M, OMPRTL___kmpc_end_serialized_parallel),
2110 if (
auto *OMPRegionInfo =
2112 if (OMPRegionInfo->getThreadIDVariable())
2113 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(CGF);
2122 return ThreadIDTemp;
2126 std::string Prefix = Twine(
"gomp_critical_user_", CriticalName).str();
2127 std::string Name =
getName({Prefix,
"var"});
2134 llvm::FunctionCallee EnterCallee;
2136 llvm::FunctionCallee ExitCallee;
2139 llvm::BasicBlock *ContBlock =
nullptr;
2142 CommonActionTy(llvm::FunctionCallee EnterCallee,
2144 llvm::FunctionCallee ExitCallee,
2146 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
2147 ExitArgs(ExitArgs), Conditional(Conditional) {}
2151 llvm::Value *CallBool = CGF.
Builder.CreateIsNotNull(EnterRes);
2155 CGF.
Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
2171 StringRef CriticalName,
2185 EnterArgs.push_back(CGF.
Builder.CreateIntCast(
2188 CommonActionTy Action(
2191 Hint ? OMPRTL___kmpc_critical_with_hint : OMPRTL___kmpc_critical),
2194 OMPRTL___kmpc_end_critical),
2211 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2233 llvm::Value *FilterVal = Filter
2240 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2260 llvm::Value *Args[] = {
2262 llvm::ConstantInt::get(
CGM.
IntTy, 0,
true)};
2268 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
2269 Region->emitUntiedSwitch(CGF);
2282 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2295 unsigned Index,
const VarDecl *Var) {
2303 Ptr, ElemTy->getPointerTo(Ptr->getType()->getPointerAddressSpace())),
2319 Args.push_back(&LHSArg);
2320 Args.push_back(&RHSArg);
2326 llvm::GlobalValue::InternalLinkage, Name,
2329 Fn->setDoesNotRecurse();
2336 ArgsElemType->getPointerTo()),
2340 ArgsElemType->getPointerTo()),
2346 for (
unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
2347 const auto *DestVar =
2348 cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
2351 const auto *SrcVar =
2352 cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
2355 const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
2357 CGF.
EmitOMPCopy(
Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
2372 assert(CopyprivateVars.size() == SrcExprs.size() &&
2373 CopyprivateVars.size() == DstExprs.size() &&
2374 CopyprivateVars.size() == AssignmentOps.size());
2386 if (!CopyprivateVars.empty()) {
2389 C.getIntTypeForBitwidth(32, 1);
2390 DidIt = CGF.
CreateMemTemp(KmpInt32Ty,
".omp.copyprivate.did_it");
2395 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2412 llvm::APInt ArraySize(32, CopyprivateVars.size());
2413 QualType CopyprivateArrayTy =
C.getConstantArrayType(
2418 CGF.
CreateMemTemp(CopyprivateArrayTy,
".omp.copyprivate.cpr_list");
2419 for (
unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
2431 SrcExprs, DstExprs, AssignmentOps, Loc);
2432 llvm::Value *BufSize = CGF.
getTypeSize(CopyprivateArrayTy);
2436 llvm::Value *Args[] = {
2461 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2476 if (Kind == OMPD_for)
2477 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
2478 else if (Kind == OMPD_sections)
2479 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
2480 else if (Kind == OMPD_single)
2481 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
2482 else if (Kind == OMPD_barrier)
2483 Flags = OMP_IDENT_BARRIER_EXPL;
2485 Flags = OMP_IDENT_BARRIER_IMPL;
2497 ScheduleKind = OMPC_SCHEDULE_static;
2499 llvm::APInt ChunkSize(32, 1);
2509 bool ForceSimpleCall) {
2511 auto *OMPRegionInfo =
2515 CGF.
Builder, Kind, ForceSimpleCall, EmitChecks));
2528 if (OMPRegionInfo) {
2529 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
2532 OMPRTL___kmpc_cancel_barrier),
2541 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
2558 Expr *ME,
bool IsFatal) {
2561 : llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
2564 llvm::Value *Args[] = {
2566 llvm::ConstantInt::get(
CGM.
Int32Ty, IsFatal ? 2 : 1),
2575 bool Chunked,
bool Ordered) {
2576 switch (ScheduleKind) {
2577 case OMPC_SCHEDULE_static:
2578 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
2579 : (Ordered ? OMP_ord_static : OMP_sch_static);
2580 case OMPC_SCHEDULE_dynamic:
2581 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
2582 case OMPC_SCHEDULE_guided:
2583 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
2584 case OMPC_SCHEDULE_runtime:
2585 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
2586 case OMPC_SCHEDULE_auto:
2587 return Ordered ? OMP_ord_auto : OMP_sch_auto;
2589 assert(!Chunked &&
"chunk was specified but schedule kind not known");
2590 return Ordered ? OMP_ord_static : OMP_sch_static;
2592 llvm_unreachable(
"Unexpected runtime schedule");
2596static OpenMPSchedType
2599 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
2603 bool Chunked)
const {
2604 OpenMPSchedType Schedule =
2606 return Schedule == OMP_sch_static;
2612 return Schedule == OMP_dist_sch_static;
2616 bool Chunked)
const {
2617 OpenMPSchedType Schedule =
2619 return Schedule == OMP_sch_static_chunked;
2625 return Schedule == OMP_dist_sch_static_chunked;
2629 OpenMPSchedType Schedule =
2631 assert(Schedule != OMP_sch_static_chunked &&
"cannot be chunked here");
2632 return Schedule != OMP_sch_static;
2640 case OMPC_SCHEDULE_MODIFIER_monotonic:
2641 Modifier = OMP_sch_modifier_monotonic;
2643 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2644 Modifier = OMP_sch_modifier_nonmonotonic;
2646 case OMPC_SCHEDULE_MODIFIER_simd:
2647 if (Schedule == OMP_sch_static_chunked)
2648 Schedule = OMP_sch_static_balanced_chunked;
2655 case OMPC_SCHEDULE_MODIFIER_monotonic:
2656 Modifier = OMP_sch_modifier_monotonic;
2658 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2659 Modifier = OMP_sch_modifier_nonmonotonic;
2661 case OMPC_SCHEDULE_MODIFIER_simd:
2662 if (Schedule == OMP_sch_static_chunked)
2663 Schedule = OMP_sch_static_balanced_chunked;
2675 if (CGM.
getLangOpts().OpenMP >= 50 && Modifier == 0) {
2676 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
2677 Schedule == OMP_sch_static_balanced_chunked ||
2678 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||
2679 Schedule == OMP_dist_sch_static_chunked ||
2680 Schedule == OMP_dist_sch_static))
2681 Modifier = OMP_sch_modifier_nonmonotonic;
2683 return Schedule | Modifier;
2693 ScheduleKind.
Schedule, DispatchValues.
Chunk !=
nullptr, Ordered);
2695 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
2696 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&
2697 Schedule != OMP_sch_static_balanced_chunked));
2704 llvm::Value *Chunk = DispatchValues.
Chunk ? DispatchValues.
Chunk
2705 : CGF.
Builder.getIntN(IVSize, 1);
2706 llvm::Value *Args[] = {
2710 CGM, Schedule, ScheduleKind.
M1, ScheduleKind.
M2)),
2713 CGF.
Builder.getIntN(IVSize, 1),
2720 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
2721 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
2728 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||
2729 Schedule == OMP_sch_static_balanced_chunked ||
2730 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||
2731 Schedule == OMP_dist_sch_static ||
2732 Schedule == OMP_dist_sch_static_chunked);
2739 llvm::Value *Chunk = Values.
Chunk;
2740 if (Chunk ==
nullptr) {
2741 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
2742 Schedule == OMP_dist_sch_static) &&
2743 "expected static non-chunked schedule");
2747 assert((Schedule == OMP_sch_static_chunked ||
2748 Schedule == OMP_sch_static_balanced_chunked ||
2749 Schedule == OMP_ord_static_chunked ||
2750 Schedule == OMP_dist_sch_static_chunked) &&
2751 "expected static chunked schedule");
2753 llvm::Value *Args[] = {
2776 "Expected loop-based or sections-based directive.");
2779 ? OMP_IDENT_WORK_LOOP
2780 : OMP_IDENT_WORK_SECTIONS);
2782 llvm::FunctionCallee StaticInitFunction =
2786 ScheduleNum, ScheduleKind.
M1, ScheduleKind.
M2, Values);
2793 OpenMPSchedType ScheduleNum =
2795 llvm::Value *UpdatedLocation =
2798 llvm::FunctionCallee StaticInitFunction;
2799 bool isGPUDistribute =
2816 llvm::Value *Args[] = {
2819 ? OMP_IDENT_WORK_DISTRIBUTE
2821 ? OMP_IDENT_WORK_LOOP
2822 : OMP_IDENT_WORK_SECTIONS),
2829 CGM.
getModule(), OMPRTL___kmpc_distribute_static_fini),
2857 llvm::Value *Args[] = {
2873 llvm::Value *NumThreads,
2878 llvm::Value *Args[] = {
2887 ProcBindKind ProcBind,
2891 assert(ProcBind != OMP_PROC_BIND_unknown &&
"Unsupported proc_bind value.");
2893 llvm::Value *Args[] = {
2895 llvm::ConstantInt::get(
CGM.
IntTy,
unsigned(ProcBind),
true)};
2917enum KmpTaskTFields {
2947 llvm::OpenMPIRBuilder::EmitMetadataErrorReportFunctionTy &&ErrorReportFn =
2948 [
this](llvm::OpenMPIRBuilder::EmitMetadataErrorKind Kind,
2949 const llvm::TargetRegionEntryInfo &EntryInfo) ->
void {
2951 if (Kind != llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR) {
2955 if (I->getFirst()->getUniqueID().getDevice() == EntryInfo.DeviceID &&
2956 I->getFirst()->getUniqueID().getFile() == EntryInfo.FileID) {
2958 I->getFirst(), EntryInfo.Line, 1);
2964 case llvm::OpenMPIRBuilder::EMIT_MD_TARGET_REGION_ERROR: {
2967 "%0 is incorrect: either the "
2968 "address or the ID is invalid.");
2971 case llvm::OpenMPIRBuilder::EMIT_MD_DECLARE_TARGET_ERROR: {
2974 "variable %0 is incorrect: the "
2975 "address is invalid.");
2978 case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR: {
2981 "Offloading entry for declare target variable is incorrect: the "
2982 "address is invalid.");
2988 OMPBuilder.createOffloadEntriesAndInfoMetadata(ErrorReportFn);
3004 if (
auto EC = Buf.getError()) {
3010 llvm::LLVMContext
C;
3011 auto ME = expectedToErrorOrAndEmitErrors(
3012 C, llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(),
C));
3014 if (
auto EC = ME.getError()) {
3022 OMPBuilder.loadOffloadInfoMetadata(*ME.get());
3029 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty,
C.VoidPtrTy};
3032 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
3038struct PrivateHelpersTy {
3039 PrivateHelpersTy(
const Expr *OriginalRef,
const VarDecl *Original,
3041 : OriginalRef(OriginalRef), Original(Original), PrivateCopy(PrivateCopy),
3042 PrivateElemInit(PrivateElemInit) {}
3043 PrivateHelpersTy(
const VarDecl *Original) : Original(Original) {}
3044 const Expr *OriginalRef =
nullptr;
3045 const VarDecl *Original =
nullptr;
3046 const VarDecl *PrivateCopy =
nullptr;
3047 const VarDecl *PrivateElemInit =
nullptr;
3048 bool isLocalPrivate()
const {
3049 return !OriginalRef && !PrivateCopy && !PrivateElemInit;
3052typedef std::pair<
CharUnits , PrivateHelpersTy> PrivateDataTy;
3057 if (!CVD->
hasAttr<OMPAllocateDeclAttr>())
3059 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
3061 return !(AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc &&
3062 !AA->getAllocator());
3067 if (!Privates.empty()) {
3072 RecordDecl *RD =
C.buildImplicitRecord(
".kmp_privates.t");
3074 for (
const auto &Pair : Privates) {
3075 const VarDecl *VD = Pair.second.Original;
3079 if (Pair.second.isLocalPrivate()) {
3102 QualType KmpRoutineEntryPointerQTy) {
3122 QualType KmpCmplrdataTy =
C.getRecordType(UD);
3123 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_task_t");
3153 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_task_t_with_privates");
3173static llvm::Function *
3176 QualType KmpTaskTWithPrivatesPtrQTy,
3178 QualType SharedsPtrTy, llvm::Function *TaskFunction,
3179 llvm::Value *TaskPrivatesMap) {
3187 Args.push_back(&GtidArg);
3188 Args.push_back(&TaskTypeArg);
3189 const auto &TaskEntryFnInfo =
3191 llvm::FunctionType *TaskEntryTy =
3194 auto *TaskEntry = llvm::Function::Create(
3195 TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.
getModule());
3197 TaskEntry->setDoesNotRecurse();
3212 const auto *KmpTaskTWithPrivatesQTyRD =
3213 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
3216 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->
getAsTagDecl());
3217 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
3219 llvm::Value *PartidParam = PartIdLVal.
getPointer(CGF);
3221 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
3227 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
3228 llvm::Value *PrivatesParam;
3229 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
3234 PrivatesParam = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
3237 llvm::Value *CommonArgs[] = {
3238 GtidParam, PartidParam, PrivatesParam, TaskPrivatesMap,
3244 std::end(CommonArgs));
3246 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
3249 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
3252 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
3255 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
3258 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
3261 CallArgs.push_back(LBParam);
3262 CallArgs.push_back(UBParam);
3263 CallArgs.push_back(StParam);
3264 CallArgs.push_back(LIParam);
3265 CallArgs.push_back(RParam);
3267 CallArgs.push_back(SharedsParam);
3280 QualType KmpTaskTWithPrivatesPtrQTy,
3281 QualType KmpTaskTWithPrivatesQTy) {
3289 Args.push_back(&GtidArg);
3290 Args.push_back(&TaskTypeArg);
3291 const auto &DestructorFnInfo =
3293 llvm::FunctionType *DestructorFnTy =
3297 auto *DestructorFn =
3298 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
3302 DestructorFn->setDoesNotRecurse();
3310 const auto *KmpTaskTWithPrivatesQTyRD =
3311 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
3312 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
3314 for (
const auto *Field :
3315 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
3317 Field->getType().isDestructedType()) {
3323 return DestructorFn;
3343 C,
nullptr, Loc,
nullptr,
3344 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
3346 Args.push_back(&TaskPrivatesArg);
3347 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>,
unsigned> PrivateVarsPos;
3348 unsigned Counter = 1;
3349 for (
const Expr *E :
Data.PrivateVars) {
3351 C,
nullptr, Loc,
nullptr,
3352 C.getPointerType(
C.getPointerType(E->
getType()))
3356 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3357 PrivateVarsPos[VD] = Counter;
3360 for (
const Expr *E :
Data.FirstprivateVars) {
3362 C,
nullptr, Loc,
nullptr,
3363 C.getPointerType(
C.getPointerType(E->
getType()))
3367 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3368 PrivateVarsPos[VD] = Counter;
3371 for (
const Expr *E :
Data.LastprivateVars) {
3373 C,
nullptr, Loc,
nullptr,
3374 C.getPointerType(
C.getPointerType(E->
getType()))
3378 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3379 PrivateVarsPos[VD] = Counter;
3385 Ty =
C.getPointerType(Ty);
3387 Ty =
C.getPointerType(Ty);
3389 C,
nullptr, Loc,
nullptr,
3390 C.getPointerType(
C.getPointerType(Ty)).withConst().withRestrict(),
3392 PrivateVarsPos[VD] = Counter;
3395 const auto &TaskPrivatesMapFnInfo =
3397 llvm::FunctionType *TaskPrivatesMapTy =
3401 auto *TaskPrivatesMap = llvm::Function::Create(
3402 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
3405 TaskPrivatesMapFnInfo);
3407 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
3408 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
3409 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
3413 TaskPrivatesMapFnInfo, Args, Loc, Loc);
3419 const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->
getAsTagDecl());
3421 for (
const FieldDecl *Field : PrivatesQTyRD->fields()) {
3423 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
3432 return TaskPrivatesMap;
3444 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
3450 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
3458 if ((!IsTargetTask && !
Data.FirstprivateVars.empty() && ForDup) ||
3459 (IsTargetTask && KmpTaskSharedsPtr.
isValid())) {
3466 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
3467 for (
const PrivateDataTy &Pair : Privates) {
3469 if (Pair.second.isLocalPrivate()) {
3473 const VarDecl *VD = Pair.second.PrivateCopy;
3475 if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&
3478 if (
const VarDecl *Elem = Pair.second.PrivateElemInit) {
3479 const VarDecl *OriginalVD = Pair.second.Original;
3484 const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);
3485 if (IsTargetTask && !SharedField) {
3486 assert(isa<ImplicitParamDecl>(OriginalVD) &&
3489 ->getNumParams() == 0 &&
3490 isa<TranslationUnitDecl>(
3492 ->getDeclContext()) &&
3493 "Expected artificial target data variable.");
3496 }
else if (ForDup) {
3500 C.getDeclAlign(OriginalVD)),
3504 Pair.second.Original->getCanonicalDecl()) > 0 ||
3506 SharedRefLValue = CGF.
EmitLValue(Pair.second.OriginalRef);
3509 InlinedOpenMPRegionRAII Region(
3512 SharedRefLValue = CGF.
EmitLValue(Pair.second.OriginalRef);
3525 [&CGF, Elem, Init, &CapturesInfo](
Address DestElement,
3528 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3529 InitScope.addPrivate(Elem, SrcElement);
3530 (void)InitScope.Privatize();
3532 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
3533 CGF, &CapturesInfo);
3534 CGF.EmitAnyExprToMem(Init, DestElement,
3535 Init->getType().getQualifiers(),
3540 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3541 InitScope.addPrivate(Elem, SharedRefLValue.
getAddress(CGF));
3542 (void)InitScope.Privatize();
3543 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
3558 bool InitRequired =
false;
3559 for (
const PrivateDataTy &Pair : Privates) {
3560 if (Pair.second.isLocalPrivate())
3562 const VarDecl *VD = Pair.second.PrivateCopy;
3564 InitRequired = InitRequired || (isa_and_nonnull<CXXConstructExpr>(Init) &&
3569 return InitRequired;
3586 QualType KmpTaskTWithPrivatesPtrQTy,
3594 KmpTaskTWithPrivatesPtrQTy,
3597 KmpTaskTWithPrivatesPtrQTy,
3601 Args.push_back(&DstArg);
3602 Args.push_back(&SrcArg);
3603 Args.push_back(&LastprivArg);
3604 const auto &TaskDupFnInfo =
3608 auto *TaskDup = llvm::Function::Create(
3609 TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.
getModule());
3611 TaskDup->setDoesNotRecurse();
3621 auto LIFI = std::next(KmpTaskTQTyRD->
field_begin(), KmpTaskTLastIter);
3623 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
3631 assert(!Privates.empty());
3633 if (!
Data.FirstprivateVars.empty()) {
3638 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
3646 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
3647 SharedsTy, SharedsPtrTy,
Data, Privates,
true);
3657 for (
const PrivateDataTy &
P : Privates) {
3658 if (
P.second.isLocalPrivate())
3660 QualType Ty =
P.second.Original->getType().getNonReferenceType();
3669class OMPIteratorGeneratorScope final
3670 :
public CodeGenFunction::OMPPrivateScope {
3675 OMPIteratorGeneratorScope() =
delete;
3676 OMPIteratorGeneratorScope(OMPIteratorGeneratorScope &) =
delete;
3704 CodeGenFunction::JumpDest &ContDest =
3706 CodeGenFunction::JumpDest &ExitDest =
3709 llvm::Value *N = Uppers[I];
3717 ? CGF.
Builder.CreateICmpSLT(CVal, N)
3718 : CGF.
Builder.CreateICmpULT(CVal, N);
3720 CGF.
Builder.CreateCondBr(Cmp, BodyBB, ExitDest.getBlock());
3727 ~OMPIteratorGeneratorScope() {
3737 CGF.
EmitBlock(ExitDests[I - 1].getBlock(), I == 1);
3743static std::pair<llvm::Value *, llvm::Value *>
3745 const auto *OASE = dyn_cast<OMPArrayShapingExpr>(E);
3748 const Expr *
Base = OASE->getBase();
3753 llvm::Value *SizeVal;
3756 SizeVal = CGF.
getTypeSize(OASE->getBase()->getType()->getPointeeType());
3757 for (
const Expr *SE : OASE->getDimensions()) {
3761 SizeVal = CGF.
Builder.CreateNUWMul(SizeVal, Sz);
3763 }
else if (
const auto *ASE =
3768 llvm::Value *UpAddr = CGF.
Builder.CreateConstGEP1_32(
3770 llvm::Value *LowIntPtr = CGF.
Builder.CreatePtrToInt(Addr, CGF.
SizeTy);
3771 llvm::Value *UpIntPtr = CGF.
Builder.CreatePtrToInt(UpAddr, CGF.
SizeTy);
3772 SizeVal = CGF.
Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
3776 return std::make_pair(Addr, SizeVal);
3781 QualType FlagsTy =
C.getIntTypeForBitwidth(32,
false);
3782 if (KmpTaskAffinityInfoTy.
isNull()) {
3784 C.buildImplicitRecord(
"kmp_task_affinity_info_t");
3790 KmpTaskAffinityInfoTy =
C.getRecordType(KmpAffinityInfoRD);
3797 llvm::Function *TaskFunction,
QualType SharedsTy,
3802 const auto *I =
Data.PrivateCopies.begin();
3803 for (
const Expr *E :
Data.PrivateVars) {
3804 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3805 Privates.emplace_back(
3807 PrivateHelpersTy(E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
3811 I =
Data.FirstprivateCopies.begin();
3812 const auto *IElemInitRef =
Data.FirstprivateInits.begin();
3813 for (
const Expr *E :
Data.FirstprivateVars) {
3814 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3815 Privates.emplace_back(
3818 E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
3819 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
3823 I =
Data.LastprivateCopies.begin();
3824 for (
const Expr *E :
Data.LastprivateVars) {
3825 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3826 Privates.emplace_back(
3828 PrivateHelpersTy(E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
3836 Privates.emplace_back(
C.getDeclAlign(VD), PrivateHelpersTy(VD));
3838 llvm::stable_sort(Privates,
3839 [](
const PrivateDataTy &L,
const PrivateDataTy &R) {
3840 return L.first > R.first;
3842 QualType KmpInt32Ty =
C.getIntTypeForBitwidth(32, 1);
3856 "Expected taskloop, task or target directive");
3865 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
3867 QualType KmpTaskTWithPrivatesQTy =
C.getRecordType(KmpTaskTWithPrivatesQTyRD);
3868 QualType KmpTaskTWithPrivatesPtrQTy =
3869 C.getPointerType(KmpTaskTWithPrivatesQTy);
3870 llvm::Type *KmpTaskTWithPrivatesTy = CGF.
ConvertType(KmpTaskTWithPrivatesQTy);
3871 llvm::Type *KmpTaskTWithPrivatesPtrTy =
3872 KmpTaskTWithPrivatesTy->getPointerTo();
3873 llvm::Value *KmpTaskTWithPrivatesTySize =
3875 QualType SharedsPtrTy =
C.getPointerType(SharedsTy);
3878 llvm::Value *TaskPrivatesMap =
nullptr;
3879 llvm::Type *TaskPrivatesMapTy =
3880 std::next(TaskFunction->arg_begin(), 3)->getType();
3881 if (!Privates.empty()) {
3882 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
3886 TaskPrivatesMap, TaskPrivatesMapTy);
3888 TaskPrivatesMap = llvm::ConstantPointerNull::get(
3889 cast<llvm::PointerType>(TaskPrivatesMapTy));
3895 KmpTaskTWithPrivatesQTy,
KmpTaskTQTy, SharedsPtrTy, TaskFunction,
3907 DestructorsFlag = 0x8,
3908 PriorityFlag = 0x20,
3909 DetachableFlag = 0x40,
3911 unsigned Flags =
Data.Tied ? TiedFlag : 0;
3912 bool NeedsCleanup =
false;
3913 if (!Privates.empty()) {
3917 Flags = Flags | DestructorsFlag;
3919 if (
Data.Priority.getInt())
3920 Flags = Flags | PriorityFlag;
3922 Flags = Flags | DetachableFlag;
3923 llvm::Value *TaskFlags =
3924 Data.Final.getPointer()
3925 ? CGF.
Builder.CreateSelect(
Data.Final.getPointer(),
3926 CGF.
Builder.getInt32(FinalFlag),
3928 : CGF.
Builder.getInt32(
Data.Final.getInt() ? FinalFlag : 0);
3929 TaskFlags = CGF.
Builder.CreateOr(TaskFlags, CGF.
Builder.getInt32(Flags));
3930 llvm::Value *SharedsSize =
CGM.
getSize(
C.getTypeSizeInChars(SharedsTy));
3932 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
3935 llvm::Value *NewTask;
3938 const Expr *Device =
nullptr;
3940 Device =
C->getDevice();
3942 llvm::Value *DeviceID;
3947 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
3948 AllocArgs.push_back(DeviceID);
3969 llvm::Value *Tid =
getThreadID(CGF, DC->getBeginLoc());
3970 Tid = CGF.
Builder.CreateIntCast(Tid, CGF.
IntTy,
false);
3973 CGM.
getModule(), OMPRTL___kmpc_task_allow_completion_event),
3974 {Loc, Tid, NewTask});
3985 llvm::Value *NumOfElements =
nullptr;
3986 unsigned NumAffinities = 0;
3988 if (
const Expr *Modifier =
C->getModifier()) {
3989 const auto *IE = cast<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts());
3990 for (
unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
3994 NumOfElements ? CGF.
Builder.CreateNUWMul(NumOfElements, Sz) : Sz;
3997 NumAffinities +=
C->varlist_size();
4002 enum RTLAffinityInfoFieldsTy { BaseAddr, Len, Flags };
4004 QualType KmpTaskAffinityInfoArrayTy;
4005 if (NumOfElements) {
4006 NumOfElements = CGF.
Builder.CreateNUWAdd(
4007 llvm::ConstantInt::get(CGF.
SizeTy, NumAffinities), NumOfElements);
4010 C.getIntTypeForBitwidth(
C.getTypeSize(
C.getSizeType()), 0),
4014 KmpTaskAffinityInfoArrayTy =
4022 NumOfElements = CGF.
Builder.CreateIntCast(NumOfElements, CGF.
Int32Ty,
4025 KmpTaskAffinityInfoArrayTy =
C.getConstantArrayType(
4027 llvm::APInt(
C.getTypeSize(
C.getSizeType()), NumAffinities),
nullptr,
4030 CGF.
CreateMemTemp(KmpTaskAffinityInfoArrayTy,
".affs.arr.addr");
4032 NumOfElements = llvm::ConstantInt::get(
CGM.
Int32Ty, NumAffinities,
4039 bool HasIterator =
false;
4041 if (
C->getModifier()) {
4045 for (
const Expr *E :
C->varlists()) {
4054 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
4059 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
4073 const Expr *Modifier =
C->getModifier();
4076 OMPIteratorGeneratorScope IteratorScope(
4078 for (
const Expr *E :
C->varlists()) {
4087 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
4092 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
4094 Idx = CGF.
Builder.CreateNUWAdd(
4095 Idx, llvm::ConstantInt::get(Idx->getType(), 1));
4110 CGM.
getModule(), OMPRTL___kmpc_omp_reg_task_with_affinity),
4111 {LocRef, GTid, NewTask, NumOfElements, AffinListPtr});
4113 llvm::Value *NewTaskNewTaskTTy =
4115 NewTask, KmpTaskTWithPrivatesPtrTy);
4117 KmpTaskTWithPrivatesQTy);
4128 *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds)),
4137 if (!Privates.empty()) {
4139 SharedsTy, SharedsPtrTy,
Data, Privates,
4144 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
4145 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy,
Data, Privates,
4146 !
Data.LastprivateVars.empty());
4150 enum {
Priority = 0, Destructors = 1 };
4152 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
4154 (*FI)->getType()->getAsUnionType()->getDecl();
4157 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4158 KmpTaskTWithPrivatesQTy);
4161 Data1LV, *std::next(KmpCmplrdataUD->
field_begin(), Destructors));
4167 if (
Data.Priority.getInt()) {
4169 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
4174 Result.NewTask = NewTask;
4175 Result.TaskEntry = TaskEntry;
4176 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
4178 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
4184 RTLDependenceKindTy DepKind;
4186 case OMPC_DEPEND_in:
4187 DepKind = RTLDependenceKindTy::DepIn;
4190 case OMPC_DEPEND_out:
4191 case OMPC_DEPEND_inout:
4192 DepKind = RTLDependenceKindTy::DepInOut;
4194 case OMPC_DEPEND_mutexinoutset:
4195 DepKind = RTLDependenceKindTy::DepMutexInOutSet;
4197 case OMPC_DEPEND_inoutset:
4198 DepKind = RTLDependenceKindTy::DepInOutSet;
4200 case OMPC_DEPEND_outallmemory:
4201 DepKind = RTLDependenceKindTy::DepOmpAllMem;
4203 case OMPC_DEPEND_source:
4204 case OMPC_DEPEND_sink:
4205 case OMPC_DEPEND_depobj:
4206 case OMPC_DEPEND_inoutallmemory:
4208 llvm_unreachable(
"Unknown task dependence type");
4216 FlagsTy =
C.getIntTypeForBitwidth(
C.getTypeSize(
C.BoolTy),
false);
4217 if (KmpDependInfoTy.
isNull()) {
4218 RecordDecl *KmpDependInfoRD =
C.buildImplicitRecord(
"kmp_depend_info");
4224 KmpDependInfoTy =
C.getRecordType(KmpDependInfoRD);
4228std::pair<llvm::Value *, LValue>
4243 Base.getAddress(CGF),
4244 llvm::ConstantInt::get(CGF.
IntPtrTy, -1,
true));
4251 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4253 return std::make_pair(NumDeps,
Base);
4257 llvm::PointerUnion<unsigned *, LValue *> Pos,
4268 OMPIteratorGeneratorScope IteratorScope(
4269 CGF, cast_or_null<OMPIteratorExpr>(
4270 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4272 for (
const Expr *E :
Data.DepExprs) {
4281 Addr = llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4282 Size = llvm::ConstantInt::get(CGF.
SizeTy, 0);
4285 if (
unsigned *
P = Pos.dyn_cast<
unsigned *>()) {
4289 assert(E &&
"Expected a non-null expression");
4299 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4304 static_cast<unsigned int>(RTLDependInfoFields::Len)));
4311 static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4313 llvm::ConstantInt::get(LLVMFlagsTy,
static_cast<unsigned int>(DepKind)),
4315 if (
unsigned *
P = Pos.dyn_cast<
unsigned *>()) {
4320 Idx = CGF.
Builder.CreateNUWAdd(Idx,
4321 llvm::ConstantInt::get(Idx->getType(), 1));
4330 assert(
Data.DepKind == OMPC_DEPEND_depobj &&
4331 "Expected depobj dependency kind.");
4336 OMPIteratorGeneratorScope IteratorScope(
4337 CGF, cast_or_null<OMPIteratorExpr>(
4338 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4340 for (
const Expr *E :
Data.DepExprs) {
4341 llvm::Value *NumDeps;
4344 std::tie(NumDeps,
Base) =
4348 C.getUIntPtrType());
4352 llvm::Value *Add = CGF.
Builder.CreateNUWAdd(PrevVal, NumDeps);
4354 SizeLVals.push_back(NumLVal);
4357 for (
unsigned I = 0, E = SizeLVals.size(); I < E; ++I) {
4360 Sizes.push_back(Size);
4370 assert(
Data.DepKind == OMPC_DEPEND_depobj &&
4371 "Expected depobj dependency kind.");
4374 OMPIteratorGeneratorScope IteratorScope(
4375 CGF, cast_or_null<OMPIteratorExpr>(
4376 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4378 for (
unsigned I = 0, End =
Data.DepExprs.size(); I < End; ++I) {
4380 llvm::Value *NumDeps;
4383 std::tie(NumDeps,
Base) =
4387 llvm::Value *Size = CGF.
Builder.CreateNUWMul(
4396 llvm::Value *Add = CGF.
Builder.CreateNUWAdd(Pos, NumDeps);
4412 llvm::Value *NumOfElements =
nullptr;
4413 unsigned NumDependencies = std::accumulate(
4414 Dependencies.begin(), Dependencies.end(), 0,
4416 return D.DepKind == OMPC_DEPEND_depobj
4418 : (V + (D.IteratorExpr ? 0 : D.DepExprs.size()));
4422 bool HasDepobjDeps =
false;
4423 bool HasRegularWithIterators =
false;
4424 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4425 llvm::Value *NumOfRegularWithIterators =
4426 llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4430 if (D.
DepKind == OMPC_DEPEND_depobj) {
4433 for (llvm::Value *Size : Sizes) {
4434 NumOfDepobjElements =
4435 CGF.
Builder.CreateNUWAdd(NumOfDepobjElements, Size);
4437 HasDepobjDeps =
true;
4442 if (
const auto *IE = cast_or_null<OMPIteratorExpr>(D.
IteratorExpr)) {
4446 llvm::Value *NumClauseDeps = CGF.
Builder.CreateNUWMul(
4448 NumOfRegularWithIterators =
4449 CGF.
Builder.CreateNUWAdd(NumOfRegularWithIterators, NumClauseDeps);
4451 HasRegularWithIterators =
true;
4457 if (HasDepobjDeps || HasRegularWithIterators) {
4458 NumOfElements = llvm::ConstantInt::get(
CGM.
IntPtrTy, NumDependencies,
4460 if (HasDepobjDeps) {
4462 CGF.
Builder.CreateNUWAdd(NumOfDepobjElements, NumOfElements);
4464 if (HasRegularWithIterators) {
4466 CGF.
Builder.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements);
4469 Loc,
C.getIntTypeForBitwidth(64, 0),
4473 KmpDependInfoArrayTy =
4482 NumOfElements = CGF.
Builder.CreateIntCast(NumOfElements, CGF.
Int32Ty,
4485 KmpDependInfoArrayTy =
C.getConstantArrayType(
4491 NumOfElements = llvm::ConstantInt::get(
CGM.
Int32Ty, NumDependencies,
4495 for (
unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4496 if (Dependencies[I].DepKind == OMPC_DEPEND_depobj ||
4497 Dependencies[I].IteratorExpr)
4504 CGF.
CreateMemTemp(
C.getSizeType(),
"dep.counter.addr"),
C.getSizeType());
4506 for (
unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4507 if (Dependencies[I].DepKind == OMPC_DEPEND_depobj ||
4508 !Dependencies[I].IteratorExpr)
4514 if (HasDepobjDeps) {
4515 for (
unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4516 if (Dependencies[I].DepKind != OMPC_DEPEND_depobj)
4524 return std::make_pair(NumOfElements, DependenciesArray);