29#include "llvm/ADT/ArrayRef.h"
30#include "llvm/ADT/SmallVector.h"
31#include "llvm/ADT/StringExtras.h"
32#include "llvm/Bitcode/BitcodeReader.h"
33#include "llvm/IR/Constants.h"
34#include "llvm/IR/DerivedTypes.h"
35#include "llvm/IR/GlobalValue.h"
36#include "llvm/IR/InstrTypes.h"
37#include "llvm/IR/Value.h"
38#include "llvm/Support/AtomicOrdering.h"
39#include "llvm/Support/raw_ostream.h"
46using namespace CodeGen;
47using namespace llvm::omp;
51class CGOpenMPRegionInfo :
public CodeGenFunction::CGCapturedStmtInfo {
54 enum CGOpenMPRegionKind {
57 ParallelOutlinedRegion,
68 const CGOpenMPRegionKind RegionKind,
71 : CGCapturedStmtInfo(CS,
CR_OpenMP), RegionKind(RegionKind),
72 CodeGen(CodeGen),
Kind(
Kind), HasCancel(HasCancel) {}
74 CGOpenMPRegionInfo(
const CGOpenMPRegionKind RegionKind,
77 : CGCapturedStmtInfo(
CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
82 virtual const VarDecl *getThreadIDVariable()
const = 0;
93 CGOpenMPRegionKind getRegionKind()
const {
return RegionKind; }
97 bool hasCancel()
const {
return HasCancel; }
99 static bool classof(
const CGCapturedStmtInfo *Info) {
103 ~CGOpenMPRegionInfo()
override =
default;
106 CGOpenMPRegionKind RegionKind;
113class CGOpenMPOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
118 StringRef HelperName)
119 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen,
Kind,
121 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
122 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
127 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
130 StringRef getHelperName()
const override {
return HelperName; }
132 static bool classof(
const CGCapturedStmtInfo *Info) {
133 return CGOpenMPRegionInfo::classof(Info) &&
134 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
135 ParallelOutlinedRegion;
142 StringRef HelperName;
146class CGOpenMPTaskOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
152 llvm::SwitchInst *UntiedSwitch =
nullptr;
155 UntiedTaskActionTy(
bool Tied,
const VarDecl *PartIDVar,
157 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
167 UntiedSwitch = CGF.
Builder.CreateSwitch(Res, DoneBB);
171 UntiedSwitch->addCase(CGF.
Builder.getInt32(0),
173 emitUntiedSwitch(CGF);
184 CodeGenFunction::JumpDest CurPoint =
188 UntiedSwitch->addCase(CGF.
Builder.getInt32(UntiedSwitch->getNumCases()),
194 unsigned getNumberOfParts()
const {
return UntiedSwitch->getNumCases(); }
200 const UntiedTaskActionTy &Action)
201 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen,
Kind, HasCancel),
202 ThreadIDVar(ThreadIDVar), Action(Action) {
203 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
208 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
214 StringRef getHelperName()
const override {
return ".omp_outlined."; }
217 Action.emitUntiedSwitch(CGF);
220 static bool classof(
const CGCapturedStmtInfo *Info) {
221 return CGOpenMPRegionInfo::classof(Info) &&
222 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
231 const UntiedTaskActionTy &Action;
236class CGOpenMPInlinedRegionInfo :
public CGOpenMPRegionInfo {
238 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
241 : CGOpenMPRegionInfo(InlinedRegion, CodeGen,
Kind, HasCancel),
243 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
246 llvm::Value *getContextValue()
const override {
248 return OuterRegionInfo->getContextValue();
249 llvm_unreachable(
"No context value for inlined OpenMP region");
252 void setContextValue(llvm::Value *
V)
override {
253 if (OuterRegionInfo) {
254 OuterRegionInfo->setContextValue(
V);
257 llvm_unreachable(
"No context value for inlined OpenMP region");
263 return OuterRegionInfo->lookup(VD);
269 FieldDecl *getThisFieldDecl()
const override {
271 return OuterRegionInfo->getThisFieldDecl();
277 const VarDecl *getThreadIDVariable()
const override {
279 return OuterRegionInfo->getThreadIDVariable();
286 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
287 llvm_unreachable(
"No LValue for inlined OpenMP construct");
291 StringRef getHelperName()
const override {
292 if (
auto *OuterRegionInfo = getOldCSI())
293 return OuterRegionInfo->getHelperName();
294 llvm_unreachable(
"No helper name for inlined OpenMP construct");
299 OuterRegionInfo->emitUntiedSwitch(CGF);
302 CodeGenFunction::CGCapturedStmtInfo *getOldCSI()
const {
return OldCSI; }
304 static bool classof(
const CGCapturedStmtInfo *Info) {
305 return CGOpenMPRegionInfo::classof(Info) &&
306 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
309 ~CGOpenMPInlinedRegionInfo()
override =
default;
313 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
314 CGOpenMPRegionInfo *OuterRegionInfo;
322class CGOpenMPTargetRegionInfo final :
public CGOpenMPRegionInfo {
326 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
328 HelperName(HelperName) {}
332 const VarDecl *getThreadIDVariable()
const override {
return nullptr; }
335 StringRef getHelperName()
const override {
return HelperName; }
337 static bool classof(
const CGCapturedStmtInfo *Info) {
338 return CGOpenMPRegionInfo::classof(Info) &&
339 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
343 StringRef HelperName;
347 llvm_unreachable(
"No codegen for expressions");
351class CGOpenMPInnerExprInfo final :
public CGOpenMPInlinedRegionInfo {
354 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
362 if (!
C.capturesVariable() && !
C.capturesVariableByCopy())
365 const VarDecl *VD =
C.getCapturedVar();
375 (void)PrivScope.Privatize();
380 if (
const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
387 llvm_unreachable(
"No body for expressions");
392 const VarDecl *getThreadIDVariable()
const override {
393 llvm_unreachable(
"No thread id for expressions");
397 StringRef getHelperName()
const override {
398 llvm_unreachable(
"No helper name for expressions");
401 static bool classof(
const CGCapturedStmtInfo *Info) {
return false; }
405 CodeGenFunction::OMPPrivateScope PrivScope;
409class InlinedOpenMPRegionRAII {
411 llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
412 FieldDecl *LambdaThisCaptureField =
nullptr;
414 bool NoInheritance =
false;
423 bool NoInheritance =
true)
424 : CGF(CGF), NoInheritance(NoInheritance) {
437 ~InlinedOpenMPRegionRAII() {
454enum OpenMPLocationFlags :
unsigned {
456 OMP_IDENT_IMD = 0x01,
458 OMP_IDENT_KMPC = 0x02,
460 OMP_ATOMIC_REDUCE = 0x10,
462 OMP_IDENT_BARRIER_EXPL = 0x20,
464 OMP_IDENT_BARRIER_IMPL = 0x40,
466 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
468 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
470 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
472 OMP_IDENT_WORK_LOOP = 0x200,
474 OMP_IDENT_WORK_SECTIONS = 0x400,
476 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
506enum IdentFieldIndex {
508 IdentField_Reserved_1,
512 IdentField_Reserved_2,
514 IdentField_Reserved_3,
523enum OpenMPSchedType {
526 OMP_sch_static_chunked = 33,
528 OMP_sch_dynamic_chunked = 35,
529 OMP_sch_guided_chunked = 36,
530 OMP_sch_runtime = 37,
533 OMP_sch_static_balanced_chunked = 45,
536 OMP_ord_static_chunked = 65,
538 OMP_ord_dynamic_chunked = 67,
539 OMP_ord_guided_chunked = 68,
540 OMP_ord_runtime = 69,
542 OMP_sch_default = OMP_sch_static,
544 OMP_dist_sch_static_chunked = 91,
545 OMP_dist_sch_static = 92,
548 OMP_sch_modifier_monotonic = (1 << 29),
550 OMP_sch_modifier_nonmonotonic = (1 << 30),
573 Callback(CodeGen, CGF, *PrePostAction);
576 Callback(CodeGen, CGF, Action);
584 if (
const auto *CE = dyn_cast<CallExpr>(ReductionOp))
585 if (
const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
586 if (
const auto *DRE =
587 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
588 if (
const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
599 std::pair<llvm::Function *, llvm::Function *>
Reduction =
601 const auto *CE = cast<CallExpr>(InitOp);
602 const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
606 cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
608 cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
609 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
610 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
Private);
611 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()), Original);
612 (void)PrivateScope.Privatize();
614 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE,
Func);
619 auto *GV =
new llvm::GlobalVariable(
621 llvm::GlobalValue::PrivateLinkage,
Init, Name);
634 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, LV);
641 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
662 llvm::Value *NumElements = CGF.
emitArrayLength(ArrayTy, ElementTy, DestAddr);
666 llvm::Value *SrcBegin =
nullptr;
668 SrcBegin = SrcAddr.emitRawPointer(CGF);
671 llvm::Value *DestEnd =
676 llvm::Value *IsEmpty =
677 CGF.
Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arrayinit.isempty");
678 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
681 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
686 llvm::PHINode *SrcElementPHI =
nullptr;
689 SrcElementPHI = CGF.
Builder.CreatePHI(SrcBegin->getType(), 2,
690 "omp.arraycpy.srcElementPast");
691 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
693 Address(SrcElementPHI, SrcAddr.getElementType(),
694 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
696 llvm::PHINode *DestElementPHI = CGF.
Builder.CreatePHI(
697 DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
698 DestElementPHI->addIncoming(DestBegin, EntryBB);
705 CodeGenFunction::RunCleanupsScope InitScope(CGF);
706 if (EmitDeclareReductionInit) {
708 SrcElementCurrent, ElementTy);
716 llvm::Value *SrcElementNext = CGF.
Builder.CreateConstGEP1_32(
717 SrcAddr.getElementType(), SrcElementPHI, 1,
718 "omp.arraycpy.dest.element");
719 SrcElementPHI->addIncoming(SrcElementNext, CGF.
Builder.GetInsertBlock());
723 llvm::Value *DestElementNext = CGF.
Builder.CreateConstGEP1_32(
725 "omp.arraycpy.dest.element");
728 CGF.
Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
729 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
730 DestElementPHI->addIncoming(DestElementNext, CGF.
Builder.GetInsertBlock());
742 if (
const auto *OASE = dyn_cast<ArraySectionExpr>(
E))
747void ReductionCodeGen::emitAggregateInitialization(
753 const auto *PrivateVD =
754 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].
Private)->getDecl());
755 bool EmitDeclareReductionInit =
758 EmitDeclareReductionInit,
759 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
760 : PrivateVD->getInit(),
768 ClausesData.reserve(Shareds.size());
769 SharedAddresses.reserve(Shareds.size());
770 Sizes.reserve(Shareds.size());
771 BaseDecls.reserve(Shareds.size());
772 const auto *IOrig = Origs.begin();
773 const auto *IPriv = Privates.begin();
774 const auto *IRed = ReductionOps.begin();
775 for (
const Expr *Ref : Shareds) {
776 ClausesData.emplace_back(Ref, *IOrig, *IPriv, *IRed);
777 std::advance(IOrig, 1);
778 std::advance(IPriv, 1);
779 std::advance(IRed, 1);
784 assert(SharedAddresses.size() == N && OrigAddresses.size() == N &&
785 "Number of generated lvalues must be exactly N.");
786 LValue First = emitSharedLValue(CGF, ClausesData[N].Shared);
787 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Shared);
788 SharedAddresses.emplace_back(
First, Second);
789 if (ClausesData[N].Shared == ClausesData[N].Ref) {
790 OrigAddresses.emplace_back(
First, Second);
792 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
793 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
794 OrigAddresses.emplace_back(
First, Second);
800 bool AsArraySection = isa<ArraySectionExpr>(ClausesData[N].Ref);
803 CGF.
getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()),
808 llvm::Value *SizeInChars;
809 auto *ElemType = OrigAddresses[N].first.getAddress().getElementType();
810 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
811 if (AsArraySection) {
812 Size = CGF.
Builder.CreatePtrDiff(ElemType,
813 OrigAddresses[N].second.getPointer(CGF),
814 OrigAddresses[N].first.getPointer(CGF));
815 Size = CGF.
Builder.CreateNUWAdd(
816 Size, llvm::ConstantInt::get(Size->getType(), 1));
817 SizeInChars = CGF.
Builder.CreateNUWMul(Size, ElemSizeOf);
820 CGF.
getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType());
821 Size = CGF.
Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
823 Sizes.emplace_back(SizeInChars, Size);
826 cast<OpaqueValueExpr>(
836 assert(!Size && !Sizes[N].second &&
837 "Size should be nullptr for non-variably modified reduction "
843 cast<OpaqueValueExpr>(
852 assert(SharedAddresses.size() > N &&
"No variable was generated");
853 const auto *PrivateVD =
854 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].
Private)->getDecl());
859 (void)DefaultInit(CGF);
860 emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD);
861 }
else if (DRD && (DRD->
getInitializer() || !PrivateVD->hasInit())) {
862 (void)DefaultInit(CGF);
863 QualType SharedType = SharedAddresses[N].first.getType();
865 PrivateAddr, SharedAddr, SharedType);
866 }
else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
869 PrivateVD->
getType().getQualifiers(),
887 CGF.
pushDestroy(DTorKind, PrivateAddr, PrivateType);
911 Address OriginalBaseAddress, llvm::Value *Addr) {
935 Addr, OriginalBaseAddress.
getType());
940 const VarDecl *OrigVD =
nullptr;
941 if (
const auto *OASE = dyn_cast<ArraySectionExpr>(Ref)) {
942 const Expr *
Base = OASE->getBase()->IgnoreParenImpCasts();
943 while (
const auto *TempOASE = dyn_cast<ArraySectionExpr>(
Base))
944 Base = TempOASE->getBase()->IgnoreParenImpCasts();
945 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
946 Base = TempASE->getBase()->IgnoreParenImpCasts();
947 DE = cast<DeclRefExpr>(
Base);
948 OrigVD = cast<VarDecl>(DE->
getDecl());
949 }
else if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
950 const Expr *
Base = ASE->getBase()->IgnoreParenImpCasts();
951 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
952 Base = TempASE->getBase()->IgnoreParenImpCasts();
953 DE = cast<DeclRefExpr>(
Base);
954 OrigVD = cast<VarDecl>(DE->
getDecl());
963 BaseDecls.emplace_back(OrigVD);
966 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
968 Address SharedAddr = SharedAddresses[N].first.getAddress();
969 llvm::Value *Adjustment = CGF.
Builder.CreatePtrDiff(
972 llvm::Value *PrivatePointer =
978 SharedAddresses[N].first.getType(),
981 BaseDecls.emplace_back(
982 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
995 getThreadIDVariable()->getType()->castAs<PointerType>());
1013LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1016 getThreadIDVariable()->getType(),
1032 : CGM(CGM), OMPBuilder(CGM.getModule()) {
1034 llvm::OpenMPIRBuilderConfig Config(
1049 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(
true);
1057 if (!
Data.getValue().pointsToAliveValue())
1059 auto *GV = dyn_cast<llvm::GlobalVariable>(
Data.getValue());
1062 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1064 GV->eraseFromParent();
1069 return OMPBuilder.createPlatformSpecificName(Parts);
1072static llvm::Function *
1074 const Expr *CombinerInitializer,
const VarDecl *In,
1075 const VarDecl *Out,
bool IsCombiner) {
1078 QualType PtrTy =
C.getPointerType(Ty).withRestrict();
1084 Args.push_back(&OmpOutParm);
1085 Args.push_back(&OmpInParm);
1090 {IsCombiner ?
"omp_combiner" :
"omp_initializer",
""});
1091 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1095 Fn->removeFnAttr(llvm::Attribute::NoInline);
1096 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1097 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1103 Out->getLocation());
1104 CodeGenFunction::OMPPrivateScope
Scope(CGF);
1113 (void)
Scope.Privatize();
1114 if (!IsCombiner && Out->hasInit() &&
1117 Out->getType().getQualifiers(),
1120 if (CombinerInitializer)
1122 Scope.ForceCleanup();
1132 CGM,
D->getType(),
D->getCombiner(),
1133 cast<VarDecl>(cast<DeclRefExpr>(
D->getCombinerIn())->getDecl()),
1134 cast<VarDecl>(cast<DeclRefExpr>(
D->getCombinerOut())->getDecl()),
1137 if (
const Expr *
Init =
D->getInitializer()) {
1142 cast<VarDecl>(cast<DeclRefExpr>(
D->getInitOrig())->getDecl()),
1143 cast<VarDecl>(cast<DeclRefExpr>(
D->getInitPriv())->getDecl()),
1151std::pair<llvm::Function *, llvm::Function *>
1163struct PushAndPopStackRAII {
1164 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder,
CodeGenFunction &CGF,
1165 bool HasCancel, llvm::omp::Directive
Kind)
1166 : OMPBuilder(OMPBuilder) {
1182 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1183 assert(IP.getBlock()->end() == IP.getPoint() &&
1184 "Clang CG should cause non-terminated block!");
1185 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1190 return llvm::Error::success();
1195 llvm::OpenMPIRBuilder::FinalizationInfo FI({FiniCB,
Kind, HasCancel});
1196 OMPBuilder->pushFinalizationCB(std::move(FI));
1198 ~PushAndPopStackRAII() {
1200 OMPBuilder->popFinalizationCB();
1202 llvm::OpenMPIRBuilder *OMPBuilder;
1211 "thread id variable must be of type kmp_int32 *");
1213 bool HasCancel =
false;
1214 if (
const auto *OPD = dyn_cast<OMPParallelDirective>(&
D))
1215 HasCancel = OPD->hasCancel();
1216 else if (
const auto *OPD = dyn_cast<OMPTargetParallelDirective>(&
D))
1217 HasCancel = OPD->hasCancel();
1218 else if (
const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&
D))
1219 HasCancel = OPSD->hasCancel();
1220 else if (
const auto *OPFD = dyn_cast<OMPParallelForDirective>(&
D))
1221 HasCancel = OPFD->hasCancel();
1222 else if (
const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&
D))
1223 HasCancel = OPFD->hasCancel();
1224 else if (
const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&
D))
1225 HasCancel = OPFD->hasCancel();
1226 else if (
const auto *OPFD =
1227 dyn_cast<OMPTeamsDistributeParallelForDirective>(&
D))
1228 HasCancel = OPFD->hasCancel();
1229 else if (
const auto *OPFD =
1230 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&
D))
1231 HasCancel = OPFD->hasCancel();
1236 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind);
1237 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1238 HasCancel, OutlinedHelperName);
1239 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1244 std::string Suffix =
getName({
"omp_outlined"});
1245 return (Name + Suffix).str();
1253 std::string Suffix =
getName({
"omp",
"reduction",
"reduction_func"});
1254 return (Name + Suffix).str();
1281 bool Tied,
unsigned &NumberOfParts) {
1286 llvm::Value *TaskArgs[] = {
1288 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1291 CGF.EmitRuntimeCall(
OMPBuilder.getOrCreateRuntimeFunction(
1295 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1299 "thread id variable must be of type kmp_int32 for tasks");
1304 bool HasCancel =
false;
1305 if (
const auto *TD = dyn_cast<OMPTaskDirective>(&
D))
1306 HasCancel = TD->hasCancel();
1307 else if (
const auto *TD = dyn_cast<OMPTaskLoopDirective>(&
D))
1308 HasCancel = TD->hasCancel();
1309 else if (
const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(&
D))
1310 HasCancel = TD->hasCancel();
1311 else if (
const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(&
D))
1312 HasCancel = TD->hasCancel();
1315 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1316 InnermostKind, HasCancel, Action);
1318 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1320 NumberOfParts = Action.getNumberOfParts();
1325 bool AtCurrentPoint) {
1327 assert(!Elem.ServiceInsertPt &&
"Insert point is set already.");
1329 llvm::Value *Undef = llvm::UndefValue::get(CGF.
Int32Ty);
1330 if (AtCurrentPoint) {
1331 Elem.ServiceInsertPt =
new llvm::BitCastInst(Undef, CGF.
Int32Ty,
"svcpt",
1332 CGF.
Builder.GetInsertBlock());
1334 Elem.ServiceInsertPt =
new llvm::BitCastInst(Undef, CGF.
Int32Ty,
"svcpt");
1341 if (Elem.ServiceInsertPt) {
1342 llvm::Instruction *Ptr = Elem.ServiceInsertPt;
1343 Elem.ServiceInsertPt =
nullptr;
1344 Ptr->eraseFromParent();
1351 llvm::raw_svector_ostream OS(Buffer);
1355 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.
CurFuncDecl))
1356 OS << FD->getQualifiedNameAsString();
1363 unsigned Flags,
bool EmitLoc) {
1364 uint32_t SrcLocStrSize;
1365 llvm::Constant *SrcLocStr;
1367 llvm::codegenoptions::NoDebugInfo) ||
1369 SrcLocStr =
OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1371 std::string FunctionName;
1372 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.
CurFuncDecl))
1373 FunctionName = FD->getQualifiedNameAsString();
1383 SrcLocStr, SrcLocStrSize, llvm::omp::IdentFlag(Flags), Reserved2Flags);
1388 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1394 uint32_t SrcLocStrSize;
1395 auto *SrcLocStr =
OMPBuilder.getOrCreateSrcLocStr(
1398 OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize));
1401 llvm::Value *ThreadID =
nullptr;
1406 ThreadID = I->second.ThreadID;
1407 if (ThreadID !=
nullptr)
1411 if (
auto *OMPRegionInfo =
1413 if (OMPRegionInfo->getThreadIDVariable()) {
1415 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1419 CGF.
Builder.GetInsertBlock() == TopBlock ||
1420 !isa<llvm::Instruction>(LVal.
getPointer(CGF)) ||
1421 cast<llvm::Instruction>(LVal.
getPointer(CGF))->getParent() ==
1423 cast<llvm::Instruction>(LVal.
getPointer(CGF))->getParent() ==
1424 CGF.
Builder.GetInsertBlock()) {
1428 if (CGF.
Builder.GetInsertBlock() == TopBlock)
1440 if (!Elem.ServiceInsertPt)
1442 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1443 CGF.
Builder.SetInsertPoint(Elem.ServiceInsertPt);
1447 OMPRTL___kmpc_global_thread_num),
1450 Elem.ThreadID =
Call;
1455 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1467 for(
const auto *
D : I->second)
1482 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(
CGM.
Int32Ty),
1489static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind
1491 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
1492 OMPDeclareTargetDeclAttr::getDeviceType(VD);
1494 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1496 switch ((
int)*DevTy) {
1497 case OMPDeclareTargetDeclAttr::DT_Host:
1498 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseHost;
1500 case OMPDeclareTargetDeclAttr::DT_NoHost:
1501 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNoHost;
1503 case OMPDeclareTargetDeclAttr::DT_Any:
1504 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny;
1507 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1512static llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind
1514 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> MapType =
1515 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1517 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1518 switch ((
int)*MapType) {
1519 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_To:
1520 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
1522 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Enter:
1523 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter;
1525 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Link:
1526 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
1529 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1538 auto FileInfoCallBack = [&]() {
1542 llvm::sys::fs::UniqueID ID;
1543 if (llvm::sys::fs::getUniqueID(PLoc.
getFilename(), ID)) {
1544 PLoc =
SM.getPresumedLoc(BeginLoc,
false);
1550 return OMPBuilder.getTargetEntryUniqueInfo(FileInfoCallBack, ParentName);
1556 auto LinkageForVariable = [&VD,
this]() {
1560 std::vector<llvm::GlobalVariable *> GeneratedRefs;
1564 llvm::Constant *addr =
OMPBuilder.getAddrOfDeclareTargetVar(
1572 LinkageForVariable);
1584 std::string Suffix =
getName({
"cache",
""});
1585 return OMPBuilder.getOrCreateInternalVariable(
1598 llvm::Value *Args[] = {
1622 llvm::Value *Args[] = {
1625 Ctor, CopyCtor, Dtor};
1628 CGM.
getModule(), OMPRTL___kmpc_threadprivate_register),
1643 llvm::Value *Ctor =
nullptr, *CopyCtor =
nullptr, *Dtor =
nullptr;
1653 Args.push_back(&Dst);
1658 std::string Name =
getName({
"__kmpc_global_ctor_",
""});
1659 llvm::Function *Fn =
1685 Args.push_back(&Dst);
1690 std::string Name =
getName({
"__kmpc_global_dtor_",
""});
1691 llvm::Function *Fn =
1716 if (Ctor ==
nullptr) {
1719 if (Dtor ==
nullptr) {
1723 auto *InitFunctionTy =
1724 llvm::FunctionType::get(
CGM.
VoidTy,
false);
1725 std::string Name =
getName({
"__omp_threadprivate_init_",
""});
1735 return InitFunction;
1743 llvm::GlobalValue *GV) {
1744 std::optional<OMPDeclareTargetDeclAttr *> ActiveAttr =
1745 OMPDeclareTargetDeclAttr::getActiveAttr(FD);
1748 if (!ActiveAttr || !(*ActiveAttr)->getIndirect())
1755 OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(Name, EntryInfo);
1761 llvm::GlobalValue *Addr = GV;
1763 Addr =
new llvm::GlobalVariable(
1765 true, llvm::GlobalValue::ExternalLinkage, GV, Name,
1766 nullptr, llvm::GlobalValue::NotThreadLocal,
1767 CGM.
getModule().getDataLayout().getDefaultGlobalsAddressSpace());
1768 Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1771 OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(
1773 llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect,
1774 llvm::GlobalValue::WeakODRLinkage);
1780 std::string Suffix =
getName({
"artificial",
""});
1782 llvm::GlobalVariable *GAddr =
OMPBuilder.getOrCreateInternalVariable(
1783 VarLVType, Twine(Name).concat(Suffix).str());
1786 GAddr->setThreadLocal(
true);
1787 return Address(GAddr, GAddr->getValueType(),
1790 std::string CacheSuffix =
getName({
"cache",
""});
1791 llvm::Value *Args[] = {
1799 Twine(Name).concat(Suffix).concat(CacheSuffix).str())};
1850 llvm::Function *OutlinedFn,
1853 llvm::Value *NumThreads) {
1858 auto &&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc,
1862 llvm::Value *Args[] = {
1864 CGF.
Builder.getInt32(CapturedVars.size()),
1867 RealArgs.append(std::begin(Args), std::end(Args));
1868 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
1870 llvm::FunctionCallee RTLFn =
1871 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_fork_call);
1874 auto &&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc,
Loc,
1880 llvm::Value *Args[] = {RTLoc, ThreadID};
1882 M, OMPRTL___kmpc_serialized_parallel),
1889 ".bound.zero.addr");
1894 OutlinedFnArgs.push_back(ZeroAddrBound.
getPointer());
1895 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
1903 OutlinedFn->removeFnAttr(llvm::Attribute::AlwaysInline);
1904 OutlinedFn->addFnAttr(llvm::Attribute::NoInline);
1910 M, OMPRTL___kmpc_end_serialized_parallel),
1929 if (
auto *OMPRegionInfo =
1931 if (OMPRegionInfo->getThreadIDVariable())
1932 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
1941 return ThreadIDTemp;
1945 std::string Prefix = Twine(
"gomp_critical_user_", CriticalName).str();
1946 std::string Name =
getName({Prefix,
"var"});
1953 llvm::FunctionCallee EnterCallee;
1955 llvm::FunctionCallee ExitCallee;
1958 llvm::BasicBlock *ContBlock =
nullptr;
1961 CommonActionTy(llvm::FunctionCallee EnterCallee,
1963 llvm::FunctionCallee ExitCallee,
1965 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
1966 ExitArgs(ExitArgs), Conditional(Conditional) {}
1970 llvm::Value *CallBool = CGF.
Builder.CreateIsNotNull(EnterRes);
1974 CGF.
Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
1990 StringRef CriticalName,
2004 EnterArgs.push_back(CGF.
Builder.CreateIntCast(
2007 CommonActionTy Action(
2010 Hint ? OMPRTL___kmpc_critical_with_hint : OMPRTL___kmpc_critical),
2013 OMPRTL___kmpc_end_critical),
2030 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2052 llvm::Value *FilterVal = Filter
2059 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2079 llvm::Value *Args[] = {
2081 llvm::ConstantInt::get(
CGM.
IntTy, 0,
true)};
2087 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
2088 Region->emitUntiedSwitch(CGF);
2101 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2114 unsigned Index,
const VarDecl *Var) {
2135 Args.push_back(&LHSArg);
2136 Args.push_back(&RHSArg);
2142 llvm::GlobalValue::InternalLinkage, Name,
2145 Fn->setDoesNotRecurse();
2162 for (
unsigned I = 0,
E = AssignmentOps.size(); I <
E; ++I) {
2163 const auto *DestVar =
2164 cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
2167 const auto *SrcVar =
2168 cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
2171 const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
2173 CGF.
EmitOMPCopy(
Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
2188 assert(CopyprivateVars.size() == SrcExprs.size() &&
2189 CopyprivateVars.size() == DstExprs.size() &&
2190 CopyprivateVars.size() == AssignmentOps.size());
2202 if (!CopyprivateVars.empty()) {
2205 C.getIntTypeForBitwidth(32, 1);
2206 DidIt = CGF.
CreateMemTemp(KmpInt32Ty,
".omp.copyprivate.did_it");
2211 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2228 llvm::APInt ArraySize(32, CopyprivateVars.size());
2229 QualType CopyprivateArrayTy =
C.getConstantArrayType(
2234 CGF.
CreateMemTemp(CopyprivateArrayTy,
".omp.copyprivate.cpr_list");
2235 for (
unsigned I = 0,
E = CopyprivateVars.size(); I <
E; ++I) {
2247 SrcExprs, DstExprs, AssignmentOps,
Loc);
2248 llvm::Value *BufSize = CGF.
getTypeSize(CopyprivateArrayTy);
2252 llvm::Value *Args[] = {
2277 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2292 if (Kind == OMPD_for)
2293 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
2294 else if (Kind == OMPD_sections)
2295 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
2296 else if (Kind == OMPD_single)
2297 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
2298 else if (Kind == OMPD_barrier)
2299 Flags = OMP_IDENT_BARRIER_EXPL;
2301 Flags = OMP_IDENT_BARRIER_IMPL;
2313 ScheduleKind = OMPC_SCHEDULE_static;
2315 llvm::APInt ChunkSize(32, 1);
2325 bool ForceSimpleCall) {
2327 auto *OMPRegionInfo =
2330 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
2333 assert(AfterIP &&
"unexpected error creating barrier");
2334 CGF.
Builder.restoreIP(*AfterIP);
2347 if (OMPRegionInfo) {
2348 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
2351 OMPRTL___kmpc_cancel_barrier),
2360 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
2377 Expr *ME,
bool IsFatal) {
2380 : llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
2383 llvm::Value *Args[] = {
2385 llvm::ConstantInt::get(
CGM.
Int32Ty, IsFatal ? 2 : 1),
2394 bool Chunked,
bool Ordered) {
2395 switch (ScheduleKind) {
2396 case OMPC_SCHEDULE_static:
2397 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
2398 : (Ordered ? OMP_ord_static : OMP_sch_static);
2399 case OMPC_SCHEDULE_dynamic:
2400 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
2401 case OMPC_SCHEDULE_guided:
2402 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
2403 case OMPC_SCHEDULE_runtime:
2404 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
2405 case OMPC_SCHEDULE_auto:
2406 return Ordered ? OMP_ord_auto : OMP_sch_auto;
2408 assert(!Chunked &&
"chunk was specified but schedule kind not known");
2409 return Ordered ? OMP_ord_static : OMP_sch_static;
2411 llvm_unreachable(
"Unexpected runtime schedule");
2415static OpenMPSchedType
2418 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
2422 bool Chunked)
const {
2423 OpenMPSchedType Schedule =
2425 return Schedule == OMP_sch_static;
2431 return Schedule == OMP_dist_sch_static;
2435 bool Chunked)
const {
2436 OpenMPSchedType Schedule =
2438 return Schedule == OMP_sch_static_chunked;
2444 return Schedule == OMP_dist_sch_static_chunked;
2448 OpenMPSchedType Schedule =
2450 assert(Schedule != OMP_sch_static_chunked &&
"cannot be chunked here");
2451 return Schedule != OMP_sch_static;
2459 case OMPC_SCHEDULE_MODIFIER_monotonic:
2460 Modifier = OMP_sch_modifier_monotonic;
2462 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2463 Modifier = OMP_sch_modifier_nonmonotonic;
2465 case OMPC_SCHEDULE_MODIFIER_simd:
2466 if (Schedule == OMP_sch_static_chunked)
2467 Schedule = OMP_sch_static_balanced_chunked;
2474 case OMPC_SCHEDULE_MODIFIER_monotonic:
2475 Modifier = OMP_sch_modifier_monotonic;
2477 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2478 Modifier = OMP_sch_modifier_nonmonotonic;
2480 case OMPC_SCHEDULE_MODIFIER_simd:
2481 if (Schedule == OMP_sch_static_chunked)
2482 Schedule = OMP_sch_static_balanced_chunked;
2494 if (CGM.
getLangOpts().OpenMP >= 50 && Modifier == 0) {
2495 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
2496 Schedule == OMP_sch_static_balanced_chunked ||
2497 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||
2498 Schedule == OMP_dist_sch_static_chunked ||
2499 Schedule == OMP_dist_sch_static))
2500 Modifier = OMP_sch_modifier_nonmonotonic;
2502 return Schedule | Modifier;
2512 ScheduleKind.
Schedule, DispatchValues.
Chunk !=
nullptr, Ordered);
2514 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
2515 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&
2516 Schedule != OMP_sch_static_balanced_chunked));
2523 llvm::Value *Chunk = DispatchValues.
Chunk ? DispatchValues.
Chunk
2524 : CGF.
Builder.getIntN(IVSize, 1);
2525 llvm::Value *Args[] = {
2529 CGM, Schedule, ScheduleKind.
M1, ScheduleKind.
M2)),
2532 CGF.
Builder.getIntN(IVSize, 1),
2549 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
2550 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
2557 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||
2558 Schedule == OMP_sch_static_balanced_chunked ||
2559 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||
2560 Schedule == OMP_dist_sch_static ||
2561 Schedule == OMP_dist_sch_static_chunked);
2568 llvm::Value *Chunk = Values.
Chunk;
2569 if (Chunk ==
nullptr) {
2570 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
2571 Schedule == OMP_dist_sch_static) &&
2572 "expected static non-chunked schedule");
2576 assert((Schedule == OMP_sch_static_chunked ||
2577 Schedule == OMP_sch_static_balanced_chunked ||
2578 Schedule == OMP_ord_static_chunked ||
2579 Schedule == OMP_dist_sch_static_chunked) &&
2580 "expected static chunked schedule");
2582 llvm::Value *Args[] = {
2605 "Expected loop-based or sections-based directive.");
2608 ? OMP_IDENT_WORK_LOOP
2609 : OMP_IDENT_WORK_SECTIONS);
2611 llvm::FunctionCallee StaticInitFunction =
2616 ScheduleNum, ScheduleKind.
M1, ScheduleKind.
M2, Values);
2623 OpenMPSchedType ScheduleNum =
2625 llvm::Value *UpdatedLocation =
2628 llvm::FunctionCallee StaticInitFunction;
2629 bool isGPUDistribute =
2632 StaticInitFunction =
OMPBuilder.createForStaticInitFunction(
2643 assert((DKind == OMPD_distribute || DKind == OMPD_for ||
2644 DKind == OMPD_sections) &&
2645 "Expected distribute, for, or sections directive kind");
2649 llvm::Value *Args[] = {
2652 (DKind == OMPD_target_teams_loop)
2653 ? OMP_IDENT_WORK_DISTRIBUTE
2655 ? OMP_IDENT_WORK_LOOP
2656 : OMP_IDENT_WORK_SECTIONS),
2664 CGM.
getModule(), OMPRTL___kmpc_distribute_static_fini),
2693 llvm::Value *Args[] = {
2701 OMPBuilder.createDispatchNextFunction(IVSize, IVSigned), Args);
2708 llvm::Value *NumThreads,
2713 llvm::Value *Args[] = {
2722 ProcBindKind ProcBind,
2726 assert(ProcBind != OMP_PROC_BIND_unknown &&
"Unsupported proc_bind value.");
2728 llvm::Value *Args[] = {
2730 llvm::ConstantInt::get(
CGM.
IntTy,
unsigned(ProcBind),
true)};
2752enum KmpTaskTFields {
2782 llvm::OpenMPIRBuilder::EmitMetadataErrorReportFunctionTy &&ErrorReportFn =
2783 [
this](llvm::OpenMPIRBuilder::EmitMetadataErrorKind Kind,
2784 const llvm::TargetRegionEntryInfo &EntryInfo) ->
void {
2786 if (Kind != llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR) {
2790 if (I->getFirst().getUniqueID().getDevice() == EntryInfo.DeviceID &&
2791 I->getFirst().getUniqueID().getFile() == EntryInfo.FileID) {
2793 I->getFirst(), EntryInfo.Line, 1);
2799 case llvm::OpenMPIRBuilder::EMIT_MD_TARGET_REGION_ERROR: {
2802 "%0 is incorrect: either the "
2803 "address or the ID is invalid.");
2806 case llvm::OpenMPIRBuilder::EMIT_MD_DECLARE_TARGET_ERROR: {
2809 "variable %0 is incorrect: the "
2810 "address is invalid.");
2813 case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR: {
2816 "Offloading entry for declare target variable is incorrect: the "
2817 "address is invalid.");
2823 OMPBuilder.createOffloadEntriesAndInfoMetadata(ErrorReportFn);
2830 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty,
C.VoidPtrTy};
2833 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
2839struct PrivateHelpersTy {
2840 PrivateHelpersTy(
const Expr *OriginalRef,
const VarDecl *Original,
2842 : OriginalRef(OriginalRef), Original(Original), PrivateCopy(PrivateCopy),
2843 PrivateElemInit(PrivateElemInit) {}
2844 PrivateHelpersTy(
const VarDecl *Original) : Original(Original) {}
2845 const Expr *OriginalRef =
nullptr;
2846 const VarDecl *Original =
nullptr;
2847 const VarDecl *PrivateCopy =
nullptr;
2848 const VarDecl *PrivateElemInit =
nullptr;
2849 bool isLocalPrivate()
const {
2850 return !OriginalRef && !PrivateCopy && !PrivateElemInit;
2853typedef std::pair<
CharUnits , PrivateHelpersTy> PrivateDataTy;
2858 if (!CVD->
hasAttr<OMPAllocateDeclAttr>())
2860 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
2862 return !(AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc &&
2863 !AA->getAllocator());
2868 if (!Privates.empty()) {
2873 RecordDecl *RD =
C.buildImplicitRecord(
".kmp_privates.t");
2875 for (
const auto &Pair : Privates) {
2876 const VarDecl *VD = Pair.second.Original;
2880 if (Pair.second.isLocalPrivate()) {
2903 QualType KmpRoutineEntryPointerQTy) {
2923 QualType KmpCmplrdataTy =
C.getRecordType(UD);
2924 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_task_t");
2954 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_task_t_with_privates");
2974static llvm::Function *
2977 QualType KmpTaskTWithPrivatesPtrQTy,
2979 QualType SharedsPtrTy, llvm::Function *TaskFunction,
2980 llvm::Value *TaskPrivatesMap) {
2988 Args.push_back(&GtidArg);
2989 Args.push_back(&TaskTypeArg);
2990 const auto &TaskEntryFnInfo =
2992 llvm::FunctionType *TaskEntryTy =
2995 auto *TaskEntry = llvm::Function::Create(
2996 TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.
getModule());
2998 TaskEntry->setDoesNotRecurse();
3013 const auto *KmpTaskTWithPrivatesQTyRD =
3014 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
3017 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->
getAsTagDecl());
3018 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
3020 llvm::Value *PartidParam = PartIdLVal.
getPointer(CGF);
3022 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
3028 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
3029 llvm::Value *PrivatesParam;
3030 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
3035 PrivatesParam = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
3038 llvm::Value *CommonArgs[] = {
3039 GtidParam, PartidParam, PrivatesParam, TaskPrivatesMap,
3045 std::end(CommonArgs));
3047 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
3050 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
3053 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
3056 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
3059 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
3062 CallArgs.push_back(LBParam);
3063 CallArgs.push_back(UBParam);
3064 CallArgs.push_back(StParam);
3065 CallArgs.push_back(LIParam);
3066 CallArgs.push_back(RParam);
3068 CallArgs.push_back(SharedsParam);
3081 QualType KmpTaskTWithPrivatesPtrQTy,
3082 QualType KmpTaskTWithPrivatesQTy) {
3090 Args.push_back(&GtidArg);
3091 Args.push_back(&TaskTypeArg);
3092 const auto &DestructorFnInfo =
3094 llvm::FunctionType *DestructorFnTy =
3098 auto *DestructorFn =
3099 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
3103 DestructorFn->setDoesNotRecurse();
3111 const auto *KmpTaskTWithPrivatesQTyRD =
3112 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
3113 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
3115 for (
const auto *Field :
3116 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
3118 Field->getType().isDestructedType()) {
3124 return DestructorFn;
3144 C,
nullptr,
Loc,
nullptr,
3145 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
3147 Args.push_back(&TaskPrivatesArg);
3148 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>,
unsigned> PrivateVarsPos;
3149 unsigned Counter = 1;
3150 for (
const Expr *
E :
Data.PrivateVars) {
3152 C,
nullptr,
Loc,
nullptr,
3153 C.getPointerType(
C.getPointerType(
E->
getType()))
3157 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
3158 PrivateVarsPos[VD] = Counter;
3161 for (
const Expr *
E :
Data.FirstprivateVars) {
3163 C,
nullptr,
Loc,
nullptr,
3164 C.getPointerType(
C.getPointerType(
E->
getType()))
3168 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
3169 PrivateVarsPos[VD] = Counter;
3172 for (
const Expr *
E :
Data.LastprivateVars) {
3174 C,
nullptr,
Loc,
nullptr,
3175 C.getPointerType(
C.getPointerType(
E->
getType()))
3179 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
3180 PrivateVarsPos[VD] = Counter;
3186 Ty =
C.getPointerType(Ty);
3188 Ty =
C.getPointerType(Ty);
3190 C,
nullptr,
Loc,
nullptr,
3191 C.getPointerType(
C.getPointerType(Ty)).withConst().withRestrict(),
3193 PrivateVarsPos[VD] = Counter;
3196 const auto &TaskPrivatesMapFnInfo =
3198 llvm::FunctionType *TaskPrivatesMapTy =
3202 auto *TaskPrivatesMap = llvm::Function::Create(
3203 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
3206 TaskPrivatesMapFnInfo);
3208 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
3209 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
3210 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
3214 TaskPrivatesMapFnInfo, Args,
Loc,
Loc);
3220 const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->
getAsTagDecl());
3222 for (
const FieldDecl *Field : PrivatesQTyRD->fields()) {
3224 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
3233 return TaskPrivatesMap;
3245 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
3251 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
3259 if ((!IsTargetTask && !
Data.FirstprivateVars.empty() && ForDup) ||
3260 (IsTargetTask && KmpTaskSharedsPtr.
isValid())) {
3267 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
3268 for (
const PrivateDataTy &Pair : Privates) {
3270 if (Pair.second.isLocalPrivate()) {
3274 const VarDecl *VD = Pair.second.PrivateCopy;
3276 if (
Init && (!ForDup || (isa<CXXConstructExpr>(
Init) &&
3279 if (
const VarDecl *Elem = Pair.second.PrivateElemInit) {
3280 const VarDecl *OriginalVD = Pair.second.Original;
3285 const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);
3286 if (IsTargetTask && !SharedField) {
3287 assert(isa<ImplicitParamDecl>(OriginalVD) &&
3290 ->getNumParams() == 0 &&
3291 isa<TranslationUnitDecl>(
3293 ->getDeclContext()) &&
3294 "Expected artificial target data variable.");
3297 }
else if (ForDup) {
3301 C.getDeclAlign(OriginalVD)),
3305 Pair.second.Original->getCanonicalDecl()) > 0 ||
3307 SharedRefLValue = CGF.
EmitLValue(Pair.second.OriginalRef);
3310 InlinedOpenMPRegionRAII Region(
3313 SharedRefLValue = CGF.
EmitLValue(Pair.second.OriginalRef);
3325 [&CGF, Elem,
Init, &CapturesInfo](
Address DestElement,
3328 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3329 InitScope.addPrivate(Elem, SrcElement);
3330 (void)InitScope.Privatize();
3332 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
3333 CGF, &CapturesInfo);
3334 CGF.EmitAnyExprToMem(Init, DestElement,
3335 Init->getType().getQualifiers(),
3340 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3341 InitScope.addPrivate(Elem, SharedRefLValue.
getAddress());
3342 (void)InitScope.Privatize();
3343 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
3358 bool InitRequired =
false;
3359 for (
const PrivateDataTy &Pair : Privates) {
3360 if (Pair.second.isLocalPrivate())
3362 const VarDecl *VD = Pair.second.PrivateCopy;
3364 InitRequired = InitRequired || (isa_and_nonnull<CXXConstructExpr>(
Init) &&
3369 return InitRequired;
3386 QualType KmpTaskTWithPrivatesPtrQTy,
3394 KmpTaskTWithPrivatesPtrQTy,
3397 KmpTaskTWithPrivatesPtrQTy,
3401 Args.push_back(&DstArg);
3402 Args.push_back(&SrcArg);
3403 Args.push_back(&LastprivArg);
3404 const auto &TaskDupFnInfo =
3408 auto *TaskDup = llvm::Function::Create(
3409 TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.
getModule());
3411 TaskDup->setDoesNotRecurse();
3421 auto LIFI = std::next(KmpTaskTQTyRD->
field_begin(), KmpTaskTLastIter);
3423 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
3431 assert(!Privates.empty());
3433 if (!
Data.FirstprivateVars.empty()) {
3438 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
3447 SharedsTy, SharedsPtrTy,
Data, Privates,
true);
3457 for (
const PrivateDataTy &
P : Privates) {
3458 if (
P.second.isLocalPrivate())
3460 QualType Ty =
P.second.Original->getType().getNonReferenceType();
3469class OMPIteratorGeneratorScope final
3470 :
public CodeGenFunction::OMPPrivateScope {
3475 OMPIteratorGeneratorScope() =
delete;
3476 OMPIteratorGeneratorScope(OMPIteratorGeneratorScope &) =
delete;
3484 for (
unsigned I = 0, End =
E->numOfIterators(); I < End; ++I) {
3486 const auto *VD = cast<VarDecl>(
E->getIteratorDecl(I));
3495 for (
unsigned I = 0, End =
E->numOfIterators(); I < End; ++I) {
3504 CodeGenFunction::JumpDest &ContDest =
3506 CodeGenFunction::JumpDest &ExitDest =
3509 llvm::Value *N = Uppers[I];
3517 ? CGF.
Builder.CreateICmpSLT(CVal, N)
3518 : CGF.
Builder.CreateICmpULT(CVal, N);
3520 CGF.
Builder.CreateCondBr(Cmp, BodyBB, ExitDest.getBlock());
3527 ~OMPIteratorGeneratorScope() {
3530 for (
unsigned I =
E->numOfIterators(); I > 0; --I) {
3537 CGF.
EmitBlock(ExitDests[I - 1].getBlock(), I == 1);
3543static std::pair<llvm::Value *, llvm::Value *>
3545 const auto *OASE = dyn_cast<OMPArrayShapingExpr>(
E);
3548 const Expr *
Base = OASE->getBase();
3553 llvm::Value *SizeVal;
3556 SizeVal = CGF.
getTypeSize(OASE->getBase()->getType()->getPointeeType());
3557 for (
const Expr *SE : OASE->getDimensions()) {
3561 SizeVal = CGF.
Builder.CreateNUWMul(SizeVal, Sz);
3563 }
else if (
const auto *ASE =
3567 llvm::Value *UpAddr = CGF.
Builder.CreateConstGEP1_32(
3570 llvm::Value *LowIntPtr = CGF.
Builder.CreatePtrToInt(Addr, CGF.
SizeTy);
3571 llvm::Value *UpIntPtr = CGF.
Builder.CreatePtrToInt(UpAddr, CGF.
SizeTy);
3572 SizeVal = CGF.
Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
3576 return std::make_pair(Addr, SizeVal);
3581 QualType FlagsTy =
C.getIntTypeForBitwidth(32,
false);
3582 if (KmpTaskAffinityInfoTy.
isNull()) {
3584 C.buildImplicitRecord(
"kmp_task_affinity_info_t");
3590 KmpTaskAffinityInfoTy =
C.getRecordType(KmpAffinityInfoRD);
3597 llvm::Function *TaskFunction,
QualType SharedsTy,
3602 const auto *I =
Data.PrivateCopies.begin();
3603 for (
const Expr *
E :
Data.PrivateVars) {
3604 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
3605 Privates.emplace_back(
3607 PrivateHelpersTy(
E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
3611 I =
Data.FirstprivateCopies.begin();
3612 const auto *IElemInitRef =
Data.FirstprivateInits.begin();
3613 for (
const Expr *
E :
Data.FirstprivateVars) {
3614 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
3615 Privates.emplace_back(
3618 E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
3619 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
3623 I =
Data.LastprivateCopies.begin();
3624 for (
const Expr *
E :
Data.LastprivateVars) {
3625 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
3626 Privates.emplace_back(
3628 PrivateHelpersTy(
E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
3636 Privates.emplace_back(
C.getDeclAlign(VD), PrivateHelpersTy(VD));
3638 llvm::stable_sort(Privates,
3639 [](
const PrivateDataTy &L,
const PrivateDataTy &R) {
3640 return L.first > R.first;
3642 QualType KmpInt32Ty =
C.getIntTypeForBitwidth(32, 1);
3653 assert((
D.getDirectiveKind() == OMPD_task ||
3656 "Expected taskloop, task or target directive");
3665 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
3667 QualType KmpTaskTWithPrivatesQTy =
C.getRecordType(KmpTaskTWithPrivatesQTyRD);
3668 QualType KmpTaskTWithPrivatesPtrQTy =
3669 C.getPointerType(KmpTaskTWithPrivatesQTy);
3670 llvm::Type *KmpTaskTWithPrivatesPtrTy = CGF.
Builder.getPtrTy(0);
3671 llvm::Value *KmpTaskTWithPrivatesTySize =
3673 QualType SharedsPtrTy =
C.getPointerType(SharedsTy);
3676 llvm::Value *TaskPrivatesMap =
nullptr;
3677 llvm::Type *TaskPrivatesMapTy =
3678 std::next(TaskFunction->arg_begin(), 3)->getType();
3679 if (!Privates.empty()) {
3680 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
3684 TaskPrivatesMap, TaskPrivatesMapTy);
3686 TaskPrivatesMap = llvm::ConstantPointerNull::get(
3687 cast<llvm::PointerType>(TaskPrivatesMapTy));
3692 CGM,
Loc,
D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
3693 KmpTaskTWithPrivatesQTy,
KmpTaskTQTy, SharedsPtrTy, TaskFunction,
3705 DestructorsFlag = 0x8,
3706 PriorityFlag = 0x20,
3707 DetachableFlag = 0x40,
3709 unsigned Flags =
Data.Tied ? TiedFlag : 0;
3710 bool NeedsCleanup =
false;
3711 if (!Privates.empty()) {
3715 Flags = Flags | DestructorsFlag;
3717 if (
Data.Priority.getInt())
3718 Flags = Flags | PriorityFlag;
3720 Flags = Flags | DetachableFlag;
3721 llvm::Value *TaskFlags =
3722 Data.Final.getPointer()
3723 ? CGF.
Builder.CreateSelect(
Data.Final.getPointer(),
3724 CGF.
Builder.getInt32(FinalFlag),
3726 : CGF.
Builder.getInt32(
Data.Final.getInt() ? FinalFlag : 0);
3727 TaskFlags = CGF.
Builder.CreateOr(TaskFlags, CGF.
Builder.getInt32(Flags));
3728 llvm::Value *SharedsSize =
CGM.
getSize(
C.getTypeSizeInChars(SharedsTy));
3733 llvm::Value *NewTask;
3740 llvm::Value *DeviceID;
3745 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
3746 AllocArgs.push_back(DeviceID);
3767 llvm::Value *Tid =
getThreadID(CGF, DC->getBeginLoc());
3768 Tid = CGF.
Builder.CreateIntCast(Tid, CGF.
IntTy,
false);
3771 CGM.
getModule(), OMPRTL___kmpc_task_allow_completion_event),
3772 {Loc, Tid, NewTask});
3783 llvm::Value *NumOfElements =
nullptr;
3784 unsigned NumAffinities = 0;
3786 if (
const Expr *Modifier =
C->getModifier()) {
3787 const auto *IE = cast<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts());
3788 for (
unsigned I = 0,
E = IE->numOfIterators(); I <
E; ++I) {
3792 NumOfElements ? CGF.
Builder.CreateNUWMul(NumOfElements, Sz) : Sz;
3795 NumAffinities +=
C->varlist_size();
3800 enum RTLAffinityInfoFieldsTy { BaseAddr, Len, Flags };
3802 QualType KmpTaskAffinityInfoArrayTy;
3803 if (NumOfElements) {
3804 NumOfElements = CGF.
Builder.CreateNUWAdd(
3805 llvm::ConstantInt::get(CGF.
SizeTy, NumAffinities), NumOfElements);
3808 C.getIntTypeForBitwidth(
C.getTypeSize(
C.getSizeType()), 0),
3812 KmpTaskAffinityInfoArrayTy =
C.getVariableArrayType(
3820 NumOfElements = CGF.
Builder.CreateIntCast(NumOfElements, CGF.
Int32Ty,
3823 KmpTaskAffinityInfoArrayTy =
C.getConstantArrayType(
3825 llvm::APInt(
C.getTypeSize(
C.getSizeType()), NumAffinities),
nullptr,
3828 CGF.
CreateMemTemp(KmpTaskAffinityInfoArrayTy,
".affs.arr.addr");
3830 NumOfElements = llvm::ConstantInt::get(
CGM.
Int32Ty, NumAffinities,
3837 bool HasIterator =
false;
3839 if (
C->getModifier()) {
3843 for (
const Expr *
E :
C->varlist()) {
3852 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
3857 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
3871 const Expr *Modifier =
C->getModifier();
3874 OMPIteratorGeneratorScope IteratorScope(
3876 for (
const Expr *
E :
C->varlist()) {
3886 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
3891 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
3893 Idx = CGF.
Builder.CreateNUWAdd(
3894 Idx, llvm::ConstantInt::get(Idx->getType(), 1));
3909 CGM.
getModule(), OMPRTL___kmpc_omp_reg_task_with_affinity),
3910 {LocRef, GTid, NewTask, NumOfElements, AffinListPtr});
3912 llvm::Value *NewTaskNewTaskTTy =
3914 NewTask, KmpTaskTWithPrivatesPtrTy);
3916 KmpTaskTWithPrivatesQTy);
3927 *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds)),
3936 if (!Privates.empty()) {
3938 SharedsTy, SharedsPtrTy,
Data, Privates,
3943 CGM,
Loc,
D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
3944 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy,
Data, Privates,
3945 !
Data.LastprivateVars.empty());
3949 enum {
Priority = 0, Destructors = 1 };
3951 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
3953 (*FI)->getType()->getAsUnionType()->getDecl();
3956 CGM,
Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
3957 KmpTaskTWithPrivatesQTy);
3960 Data1LV, *std::next(KmpCmplrdataUD->
field_begin(), Destructors));
3966 if (
Data.Priority.getInt()) {
3968 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
3973 Result.NewTask = NewTask;
3974 Result.TaskEntry = TaskEntry;
3975 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
3977 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
3983 RTLDependenceKindTy DepKind;
3985 case OMPC_DEPEND_in:
3986 DepKind = RTLDependenceKindTy::DepIn;
3989 case OMPC_DEPEND_out:
3990 case OMPC_DEPEND_inout:
3991 DepKind = RTLDependenceKindTy::DepInOut;
3993 case OMPC_DEPEND_mutexinoutset:
3994 DepKind = RTLDependenceKindTy::DepMutexInOutSet;
3996 case OMPC_DEPEND_inoutset:
3997 DepKind = RTLDependenceKindTy::DepInOutSet;
3999 case OMPC_DEPEND_outallmemory:
4000 DepKind = RTLDependenceKindTy::DepOmpAllMem;
4002 case OMPC_DEPEND_source:
4003 case OMPC_DEPEND_sink:
4004 case OMPC_DEPEND_depobj:
4005 case OMPC_DEPEND_inoutallmemory:
4007 llvm_unreachable(
"Unknown task dependence type");
4015 FlagsTy =
C.getIntTypeForBitwidth(
C.getTypeSize(
C.BoolTy),
false);
4016 if (KmpDependInfoTy.
isNull()) {
4017 RecordDecl *KmpDependInfoRD =
C.buildImplicitRecord(
"kmp_depend_info");
4023 KmpDependInfoTy =
C.getRecordType(KmpDependInfoRD);
4027std::pair<llvm::Value *, LValue>
4041 CGF,
Base.getAddress(),
4042 llvm::ConstantInt::get(CGF.
IntPtrTy, -1,
true));
4049 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4051 return std::make_pair(NumDeps,
Base);
4055 llvm::PointerUnion<unsigned *, LValue *> Pos,
4066 OMPIteratorGeneratorScope IteratorScope(
4067 CGF, cast_or_null<OMPIteratorExpr>(
4068 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4079 Addr = llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4080 Size = llvm::ConstantInt::get(CGF.
SizeTy, 0);
4083 if (
unsigned *
P = Pos.dyn_cast<
unsigned *>()) {
4087 assert(
E &&
"Expected a non-null expression");
4088 LValue &PosLVal = *cast<LValue *>(Pos);
4097 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4102 static_cast<unsigned int>(RTLDependInfoFields::Len)));
4109 static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4111 llvm::ConstantInt::get(LLVMFlagsTy,
static_cast<unsigned int>(DepKind)),
4113 if (
unsigned *
P = Pos.dyn_cast<
unsigned *>()) {
4116 LValue &PosLVal = *cast<LValue *>(Pos);
4118 Idx = CGF.
Builder.CreateNUWAdd(Idx,
4119 llvm::ConstantInt::get(Idx->getType(), 1));
4128 assert(
Data.DepKind == OMPC_DEPEND_depobj &&
4129 "Expected depobj dependency kind.");
4134 OMPIteratorGeneratorScope IteratorScope(
4135 CGF, cast_or_null<OMPIteratorExpr>(
4136 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4139 llvm::Value *NumDeps;
4142 std::tie(NumDeps,
Base) =
4146 C.getUIntPtrType());
4150 llvm::Value *Add = CGF.
Builder.CreateNUWAdd(PrevVal, NumDeps);
4152 SizeLVals.push_back(NumLVal);
4155 for (
unsigned I = 0,
E = SizeLVals.size(); I <
E; ++I) {
4158 Sizes.push_back(Size);
4168 assert(
Data.DepKind == OMPC_DEPEND_depobj &&
4169 "Expected depobj dependency kind.");
4172 OMPIteratorGeneratorScope IteratorScope(
4173 CGF, cast_or_null<OMPIteratorExpr>(
4174 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4176 for (
unsigned I = 0, End =
Data.DepExprs.size(); I < End; ++I) {
4178 llvm::Value *NumDeps;
4181 std::tie(NumDeps,
Base) =
4185 llvm::Value *Size = CGF.
Builder.CreateNUWMul(
4194 llvm::Value *Add = CGF.
Builder.CreateNUWAdd(Pos, NumDeps);
4204 return D.DepExprs.empty();
4210 llvm::Value *NumOfElements =
nullptr;
4211 unsigned NumDependencies = std::accumulate(
4212 Dependencies.begin(), Dependencies.end(), 0,
4214 return D.DepKind == OMPC_DEPEND_depobj
4216 : (V + (D.IteratorExpr ? 0 : D.DepExprs.size()));
4220 bool HasDepobjDeps =
false;
4221 bool HasRegularWithIterators =
false;
4222 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4223 llvm::Value *NumOfRegularWithIterators =
4224 llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4228 if (
D.DepKind == OMPC_DEPEND_depobj) {
4231 for (llvm::Value *Size : Sizes) {
4232 NumOfDepobjElements =
4233 CGF.
Builder.CreateNUWAdd(NumOfDepobjElements, Size);
4235 HasDepobjDeps =
true;
4240 if (
const auto *IE = cast_or_null<OMPIteratorExpr>(
D.IteratorExpr)) {
4241 llvm::Value *ClauseIteratorSpace =
4242 llvm::ConstantInt::get(CGF.
IntPtrTy, 1);
4243 for (
unsigned I = 0,
E = IE->numOfIterators(); I <
E; ++I) {
4246 ClauseIteratorSpace = CGF.
Builder.CreateNUWMul(Sz, ClauseIteratorSpace);
4248 llvm::Value *NumClauseDeps = CGF.
Builder.CreateNUWMul(
4249 ClauseIteratorSpace,
4250 llvm::ConstantInt::get(CGF.
IntPtrTy,
D.DepExprs.size()));
4251 NumOfRegularWithIterators =
4252 CGF.
Builder.CreateNUWAdd(NumOfRegularWithIterators, NumClauseDeps);
4253 HasRegularWithIterators =
true;
4259 if (HasDepobjDeps || HasRegularWithIterators) {
4260 NumOfElements = llvm::ConstantInt::get(
CGM.
IntPtrTy, NumDependencies,
4262 if (HasDepobjDeps) {
4264 CGF.
Builder.CreateNUWAdd(NumOfDepobjElements, NumOfElements);
4266 if (HasRegularWithIterators) {
4268 CGF.
Builder.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements);
4271 Loc,
C.getIntTypeForBitwidth(64, 0),
4275 KmpDependInfoArrayTy =
4284 NumOfElements = CGF.
Builder.CreateIntCast(NumOfElements, CGF.
Int32Ty,
4287 KmpDependInfoArrayTy =
C.getConstantArrayType(
4293 NumOfElements = llvm::ConstantInt::get(
CGM.
Int32Ty, NumDependencies,
4297 for (
unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4298 if (Dependencies[I].DepKind == OMPC_DEPEND_depobj ||
4299 Dependencies[I].IteratorExpr)
4306 CGF.
CreateMemTemp(
C.getSizeType(),
"dep.counter.addr"),
C.getSizeType());
4308 for (
unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4309 if (Dependencies[I].DepKind == OMPC_DEPEND_depobj ||
4310 !Dependencies[I].IteratorExpr)
4316 if (HasDepobjDeps) {
4317 for (
unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4318 if (Dependencies[I].DepKind != OMPC_DEPEND_depobj)
4326 return std::make_pair(NumOfElements, DependenciesArray);
4337 unsigned NumDependencies = Dependencies.
DepExprs.size();
4348 llvm::Value *NumDepsVal;
4350 if (
const auto *IE =
4351 cast_or_null<OMPIteratorExpr>(Dependencies.
IteratorExpr)) {
4352 NumDepsVal = llvm::ConstantInt::get(CGF.
SizeTy, 1);
4353 for (
unsigned I = 0,
E = IE->numOfIterators(); I <
E; ++I) {
4356 NumDepsVal = CGF.
Builder.CreateNUWMul(NumDepsVal, Sz);
4358 Size = CGF.
Builder.CreateNUWAdd(llvm::ConstantInt::get(CGF.
SizeTy, 1),
4362 llvm::Value *RecSize =
CGM.
getSize(SizeInBytes);
4363 Size = CGF.
Builder.CreateNUWMul(Size, RecSize);
4367 QualType KmpDependInfoArrayTy =
C.getConstantArrayType(
4370 CharUnits Sz =
C.getTypeSizeInChars(KmpDependInfoArrayTy);
4372 NumDepsVal = llvm::ConstantInt::get(CGF.
IntPtrTy, NumDependencies);
4377 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4378 llvm::Value *Args[] = {ThreadID, Size, Allocator};
4383 Args,
".dep.arr.addr");
4386 Addr, CGF.
Builder.getPtrTy(0));
4387 DependenciesArray =
Address(Addr, KmpDependInfoLlvmTy, Align);
4394 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4396 llvm::PointerUnion<unsigned *, LValue *> Pos;