31#include "llvm/ADT/ArrayRef.h"
32#include "llvm/ADT/SetOperations.h"
33#include "llvm/ADT/SmallBitVector.h"
34#include "llvm/ADT/SmallVector.h"
35#include "llvm/ADT/StringExtras.h"
36#include "llvm/Bitcode/BitcodeReader.h"
37#include "llvm/IR/Constants.h"
38#include "llvm/IR/DerivedTypes.h"
39#include "llvm/IR/GlobalValue.h"
40#include "llvm/IR/InstrTypes.h"
41#include "llvm/IR/Value.h"
42#include "llvm/Support/AtomicOrdering.h"
43#include "llvm/Support/Format.h"
44#include "llvm/Support/raw_ostream.h"
51using namespace CodeGen;
52using namespace llvm::omp;
56class CGOpenMPRegionInfo :
public CodeGenFunction::CGCapturedStmtInfo {
59 enum CGOpenMPRegionKind {
62 ParallelOutlinedRegion,
73 const CGOpenMPRegionKind RegionKind,
76 : CGCapturedStmtInfo(CS,
CR_OpenMP), RegionKind(RegionKind),
77 CodeGen(CodeGen),
Kind(
Kind), HasCancel(HasCancel) {}
79 CGOpenMPRegionInfo(
const CGOpenMPRegionKind RegionKind,
82 : CGCapturedStmtInfo(
CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
87 virtual const VarDecl *getThreadIDVariable()
const = 0;
98 CGOpenMPRegionKind getRegionKind()
const {
return RegionKind; }
102 bool hasCancel()
const {
return HasCancel; }
104 static bool classof(
const CGCapturedStmtInfo *Info) {
108 ~CGOpenMPRegionInfo()
override =
default;
111 CGOpenMPRegionKind RegionKind;
118class CGOpenMPOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
123 StringRef HelperName)
124 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen,
Kind,
126 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
127 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
132 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
135 StringRef getHelperName()
const override {
return HelperName; }
137 static bool classof(
const CGCapturedStmtInfo *Info) {
138 return CGOpenMPRegionInfo::classof(Info) &&
139 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
140 ParallelOutlinedRegion;
147 StringRef HelperName;
151class CGOpenMPTaskOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
157 llvm::SwitchInst *UntiedSwitch =
nullptr;
160 UntiedTaskActionTy(
bool Tied,
const VarDecl *PartIDVar,
162 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
172 UntiedSwitch = CGF.
Builder.CreateSwitch(Res, DoneBB);
176 UntiedSwitch->addCase(CGF.
Builder.getInt32(0),
178 emitUntiedSwitch(CGF);
189 CodeGenFunction::JumpDest CurPoint =
193 UntiedSwitch->addCase(CGF.
Builder.getInt32(UntiedSwitch->getNumCases()),
199 unsigned getNumberOfParts()
const {
return UntiedSwitch->getNumCases(); }
205 const UntiedTaskActionTy &Action)
206 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen,
Kind, HasCancel),
207 ThreadIDVar(ThreadIDVar), Action(Action) {
208 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
213 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
219 StringRef getHelperName()
const override {
return ".omp_outlined."; }
222 Action.emitUntiedSwitch(CGF);
225 static bool classof(
const CGCapturedStmtInfo *Info) {
226 return CGOpenMPRegionInfo::classof(Info) &&
227 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
236 const UntiedTaskActionTy &Action;
241class CGOpenMPInlinedRegionInfo :
public CGOpenMPRegionInfo {
243 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
246 : CGOpenMPRegionInfo(InlinedRegion, CodeGen,
Kind, HasCancel),
248 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
251 llvm::Value *getContextValue()
const override {
253 return OuterRegionInfo->getContextValue();
254 llvm_unreachable(
"No context value for inlined OpenMP region");
257 void setContextValue(llvm::Value *
V)
override {
258 if (OuterRegionInfo) {
259 OuterRegionInfo->setContextValue(
V);
262 llvm_unreachable(
"No context value for inlined OpenMP region");
268 return OuterRegionInfo->lookup(VD);
274 FieldDecl *getThisFieldDecl()
const override {
276 return OuterRegionInfo->getThisFieldDecl();
282 const VarDecl *getThreadIDVariable()
const override {
284 return OuterRegionInfo->getThreadIDVariable();
291 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
292 llvm_unreachable(
"No LValue for inlined OpenMP construct");
296 StringRef getHelperName()
const override {
297 if (
auto *OuterRegionInfo = getOldCSI())
298 return OuterRegionInfo->getHelperName();
299 llvm_unreachable(
"No helper name for inlined OpenMP construct");
304 OuterRegionInfo->emitUntiedSwitch(CGF);
307 CodeGenFunction::CGCapturedStmtInfo *getOldCSI()
const {
return OldCSI; }
309 static bool classof(
const CGCapturedStmtInfo *Info) {
310 return CGOpenMPRegionInfo::classof(Info) &&
311 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
314 ~CGOpenMPInlinedRegionInfo()
override =
default;
318 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
319 CGOpenMPRegionInfo *OuterRegionInfo;
327class CGOpenMPTargetRegionInfo final :
public CGOpenMPRegionInfo {
331 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
333 HelperName(HelperName) {}
337 const VarDecl *getThreadIDVariable()
const override {
return nullptr; }
340 StringRef getHelperName()
const override {
return HelperName; }
342 static bool classof(
const CGCapturedStmtInfo *Info) {
343 return CGOpenMPRegionInfo::classof(Info) &&
344 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
348 StringRef HelperName;
352 llvm_unreachable(
"No codegen for expressions");
356class CGOpenMPInnerExprInfo final :
public CGOpenMPInlinedRegionInfo {
359 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
367 if (!
C.capturesVariable() && !
C.capturesVariableByCopy())
370 const VarDecl *VD =
C.getCapturedVar();
380 (void)PrivScope.Privatize();
385 if (
const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
392 llvm_unreachable(
"No body for expressions");
397 const VarDecl *getThreadIDVariable()
const override {
398 llvm_unreachable(
"No thread id for expressions");
402 StringRef getHelperName()
const override {
403 llvm_unreachable(
"No helper name for expressions");
406 static bool classof(
const CGCapturedStmtInfo *Info) {
return false; }
410 CodeGenFunction::OMPPrivateScope PrivScope;
414class InlinedOpenMPRegionRAII {
416 llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
417 FieldDecl *LambdaThisCaptureField =
nullptr;
419 bool NoInheritance =
false;
428 bool NoInheritance =
true)
429 : CGF(CGF), NoInheritance(NoInheritance) {
442 ~InlinedOpenMPRegionRAII() {
459enum OpenMPLocationFlags :
unsigned {
461 OMP_IDENT_IMD = 0x01,
463 OMP_IDENT_KMPC = 0x02,
465 OMP_ATOMIC_REDUCE = 0x10,
467 OMP_IDENT_BARRIER_EXPL = 0x20,
469 OMP_IDENT_BARRIER_IMPL = 0x40,
471 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
473 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
475 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
477 OMP_IDENT_WORK_LOOP = 0x200,
479 OMP_IDENT_WORK_SECTIONS = 0x400,
481 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
511enum IdentFieldIndex {
513 IdentField_Reserved_1,
517 IdentField_Reserved_2,
519 IdentField_Reserved_3,
528enum OpenMPSchedType {
531 OMP_sch_static_chunked = 33,
533 OMP_sch_dynamic_chunked = 35,
534 OMP_sch_guided_chunked = 36,
535 OMP_sch_runtime = 37,
538 OMP_sch_static_balanced_chunked = 45,
541 OMP_ord_static_chunked = 65,
543 OMP_ord_dynamic_chunked = 67,
544 OMP_ord_guided_chunked = 68,
545 OMP_ord_runtime = 69,
547 OMP_sch_default = OMP_sch_static,
549 OMP_dist_sch_static_chunked = 91,
550 OMP_dist_sch_static = 92,
553 OMP_sch_modifier_monotonic = (1 << 29),
555 OMP_sch_modifier_nonmonotonic = (1 << 30),
578 Callback(CodeGen, CGF, *PrePostAction);
581 Callback(CodeGen, CGF, Action);
589 if (
const auto *CE = dyn_cast<CallExpr>(ReductionOp))
590 if (
const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
591 if (
const auto *DRE =
592 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
593 if (
const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
604 std::pair<llvm::Function *, llvm::Function *>
Reduction =
606 const auto *CE = cast<CallExpr>(InitOp);
607 const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
611 cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
613 cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
614 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
615 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
Private);
616 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()), Original);
617 (void)PrivateScope.Privatize();
619 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE,
Func);
624 auto *GV =
new llvm::GlobalVariable(
626 llvm::GlobalValue::PrivateLinkage,
Init, Name);
639 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, LV);
646 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
667 llvm::Value *NumElements = CGF.
emitArrayLength(ArrayTy, ElementTy, DestAddr);
671 llvm::Value *SrcBegin =
nullptr;
673 SrcBegin = SrcAddr.emitRawPointer(CGF);
676 llvm::Value *DestEnd =
681 llvm::Value *IsEmpty =
682 CGF.
Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arrayinit.isempty");
683 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
686 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
691 llvm::PHINode *SrcElementPHI =
nullptr;
694 SrcElementPHI = CGF.
Builder.CreatePHI(SrcBegin->getType(), 2,
695 "omp.arraycpy.srcElementPast");
696 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
698 Address(SrcElementPHI, SrcAddr.getElementType(),
699 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
701 llvm::PHINode *DestElementPHI = CGF.
Builder.CreatePHI(
702 DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
703 DestElementPHI->addIncoming(DestBegin, EntryBB);
710 CodeGenFunction::RunCleanupsScope InitScope(CGF);
711 if (EmitDeclareReductionInit) {
713 SrcElementCurrent, ElementTy);
721 llvm::Value *SrcElementNext = CGF.
Builder.CreateConstGEP1_32(
722 SrcAddr.getElementType(), SrcElementPHI, 1,
723 "omp.arraycpy.dest.element");
724 SrcElementPHI->addIncoming(SrcElementNext, CGF.
Builder.GetInsertBlock());
728 llvm::Value *DestElementNext = CGF.
Builder.CreateConstGEP1_32(
730 "omp.arraycpy.dest.element");
733 CGF.
Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
734 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
735 DestElementPHI->addIncoming(DestElementNext, CGF.
Builder.GetInsertBlock());
747 if (
const auto *OASE = dyn_cast<ArraySectionExpr>(
E))
752void ReductionCodeGen::emitAggregateInitialization(
758 const auto *PrivateVD =
759 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].
Private)->getDecl());
760 bool EmitDeclareReductionInit =
763 EmitDeclareReductionInit,
764 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
765 : PrivateVD->getInit(),
773 ClausesData.reserve(Shareds.size());
774 SharedAddresses.reserve(Shareds.size());
775 Sizes.reserve(Shareds.size());
776 BaseDecls.reserve(Shareds.size());
777 const auto *IOrig = Origs.begin();
778 const auto *IPriv = Privates.begin();
779 const auto *IRed = ReductionOps.begin();
780 for (
const Expr *Ref : Shareds) {
781 ClausesData.emplace_back(Ref, *IOrig, *IPriv, *IRed);
782 std::advance(IOrig, 1);
783 std::advance(IPriv, 1);
784 std::advance(IRed, 1);
789 assert(SharedAddresses.size() == N && OrigAddresses.size() == N &&
790 "Number of generated lvalues must be exactly N.");
791 LValue First = emitSharedLValue(CGF, ClausesData[N].Shared);
792 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Shared);
793 SharedAddresses.emplace_back(
First, Second);
794 if (ClausesData[N].Shared == ClausesData[N].Ref) {
795 OrigAddresses.emplace_back(
First, Second);
797 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
798 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
799 OrigAddresses.emplace_back(
First, Second);
805 bool AsArraySection = isa<ArraySectionExpr>(ClausesData[N].Ref);
808 CGF.
getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()),
813 llvm::Value *SizeInChars;
814 auto *ElemType = OrigAddresses[N].first.getAddress().getElementType();
815 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
816 if (AsArraySection) {
817 Size = CGF.
Builder.CreatePtrDiff(ElemType,
818 OrigAddresses[N].second.getPointer(CGF),
819 OrigAddresses[N].first.getPointer(CGF));
820 Size = CGF.
Builder.CreateNUWAdd(
821 Size, llvm::ConstantInt::get(Size->getType(), 1));
822 SizeInChars = CGF.
Builder.CreateNUWMul(Size, ElemSizeOf);
825 CGF.
getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType());
826 Size = CGF.
Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
828 Sizes.emplace_back(SizeInChars, Size);
831 cast<OpaqueValueExpr>(
841 assert(!Size && !Sizes[N].second &&
842 "Size should be nullptr for non-variably modified reduction "
848 cast<OpaqueValueExpr>(
857 assert(SharedAddresses.size() > N &&
"No variable was generated");
858 const auto *PrivateVD =
859 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].
Private)->getDecl());
864 (void)DefaultInit(CGF);
865 emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD);
866 }
else if (DRD && (DRD->
getInitializer() || !PrivateVD->hasInit())) {
867 (void)DefaultInit(CGF);
868 QualType SharedType = SharedAddresses[N].first.getType();
870 PrivateAddr, SharedAddr, SharedType);
871 }
else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
874 PrivateVD->
getType().getQualifiers(),
892 CGF.
pushDestroy(DTorKind, PrivateAddr, PrivateType);
916 Address OriginalBaseAddress, llvm::Value *Addr) {
940 Addr, OriginalBaseAddress.
getType());
945 const VarDecl *OrigVD =
nullptr;
946 if (
const auto *OASE = dyn_cast<ArraySectionExpr>(Ref)) {
947 const Expr *
Base = OASE->getBase()->IgnoreParenImpCasts();
948 while (
const auto *TempOASE = dyn_cast<ArraySectionExpr>(
Base))
949 Base = TempOASE->getBase()->IgnoreParenImpCasts();
950 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
951 Base = TempASE->getBase()->IgnoreParenImpCasts();
952 DE = cast<DeclRefExpr>(
Base);
953 OrigVD = cast<VarDecl>(DE->
getDecl());
954 }
else if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
955 const Expr *
Base = ASE->getBase()->IgnoreParenImpCasts();
956 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
957 Base = TempASE->getBase()->IgnoreParenImpCasts();
958 DE = cast<DeclRefExpr>(
Base);
959 OrigVD = cast<VarDecl>(DE->
getDecl());
968 BaseDecls.emplace_back(OrigVD);
971 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
973 Address SharedAddr = SharedAddresses[N].first.getAddress();
974 llvm::Value *Adjustment = CGF.
Builder.CreatePtrDiff(
977 llvm::Value *PrivatePointer =
983 SharedAddresses[N].first.getType(),
986 BaseDecls.emplace_back(
987 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
1000 getThreadIDVariable()->getType()->castAs<PointerType>());
1018LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1021 getThreadIDVariable()->getType(),
1037 : CGM(CGM), OMPBuilder(CGM.getModule()) {
1039 llvm::OpenMPIRBuilderConfig Config(
1054 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(
true);
1062 if (!
Data.getValue().pointsToAliveValue())
1064 auto *GV = dyn_cast<llvm::GlobalVariable>(
Data.getValue());
1067 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1069 GV->eraseFromParent();
1074 return OMPBuilder.createPlatformSpecificName(Parts);
1077static llvm::Function *
1079 const Expr *CombinerInitializer,
const VarDecl *In,
1080 const VarDecl *Out,
bool IsCombiner) {
1083 QualType PtrTy =
C.getPointerType(Ty).withRestrict();
1089 Args.push_back(&OmpOutParm);
1090 Args.push_back(&OmpInParm);
1095 {IsCombiner ?
"omp_combiner" :
"omp_initializer",
""});
1096 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1100 Fn->removeFnAttr(llvm::Attribute::NoInline);
1101 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1102 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1108 Out->getLocation());
1109 CodeGenFunction::OMPPrivateScope
Scope(CGF);
1118 (void)
Scope.Privatize();
1119 if (!IsCombiner && Out->hasInit() &&
1122 Out->getType().getQualifiers(),
1125 if (CombinerInitializer)
1127 Scope.ForceCleanup();
1137 CGM,
D->getType(),
D->getCombiner(),
1138 cast<VarDecl>(cast<DeclRefExpr>(
D->getCombinerIn())->getDecl()),
1139 cast<VarDecl>(cast<DeclRefExpr>(
D->getCombinerOut())->getDecl()),
1142 if (
const Expr *
Init =
D->getInitializer()) {
1147 cast<VarDecl>(cast<DeclRefExpr>(
D->getInitOrig())->getDecl()),
1148 cast<VarDecl>(cast<DeclRefExpr>(
D->getInitPriv())->getDecl()),
1154 Decls.second.push_back(
D);
1158std::pair<llvm::Function *, llvm::Function *>
1170struct PushAndPopStackRAII {
1171 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder,
CodeGenFunction &CGF,
1172 bool HasCancel, llvm::omp::Directive
Kind)
1173 : OMPBuilder(OMPBuilder) {
1189 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1190 assert(IP.getBlock()->end() == IP.getPoint() &&
1191 "Clang CG should cause non-terminated block!");
1192 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1201 llvm::OpenMPIRBuilder::FinalizationInfo FI({FiniCB,
Kind, HasCancel});
1202 OMPBuilder->pushFinalizationCB(std::move(FI));
1204 ~PushAndPopStackRAII() {
1206 OMPBuilder->popFinalizationCB();
1208 llvm::OpenMPIRBuilder *OMPBuilder;
1217 "thread id variable must be of type kmp_int32 *");
1219 bool HasCancel =
false;
1220 if (
const auto *OPD = dyn_cast<OMPParallelDirective>(&
D))
1221 HasCancel = OPD->hasCancel();
1222 else if (
const auto *OPD = dyn_cast<OMPTargetParallelDirective>(&
D))
1223 HasCancel = OPD->hasCancel();
1224 else if (
const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&
D))
1225 HasCancel = OPSD->hasCancel();
1226 else if (
const auto *OPFD = dyn_cast<OMPParallelForDirective>(&
D))
1227 HasCancel = OPFD->hasCancel();
1228 else if (
const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&
D))
1229 HasCancel = OPFD->hasCancel();
1230 else if (
const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&
D))
1231 HasCancel = OPFD->hasCancel();
1232 else if (
const auto *OPFD =
1233 dyn_cast<OMPTeamsDistributeParallelForDirective>(&
D))
1234 HasCancel = OPFD->hasCancel();
1235 else if (
const auto *OPFD =
1236 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&
D))
1237 HasCancel = OPFD->hasCancel();
1242 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind);
1243 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1244 HasCancel, OutlinedHelperName);
1245 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1250 std::string Suffix =
getName({
"omp_outlined"});
1251 return (Name + Suffix).str();
1259 std::string Suffix =
getName({
"omp",
"reduction",
"reduction_func"});
1260 return (Name + Suffix).str();
1287 bool Tied,
unsigned &NumberOfParts) {
1292 llvm::Value *TaskArgs[] = {
1294 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1297 CGF.EmitRuntimeCall(
OMPBuilder.getOrCreateRuntimeFunction(
1301 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1305 "thread id variable must be of type kmp_int32 for tasks");
1310 bool HasCancel =
false;
1311 if (
const auto *TD = dyn_cast<OMPTaskDirective>(&
D))
1312 HasCancel = TD->hasCancel();
1313 else if (
const auto *TD = dyn_cast<OMPTaskLoopDirective>(&
D))
1314 HasCancel = TD->hasCancel();
1315 else if (
const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(&
D))
1316 HasCancel = TD->hasCancel();
1317 else if (
const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(&
D))
1318 HasCancel = TD->hasCancel();
1321 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1322 InnermostKind, HasCancel, Action);
1324 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1326 NumberOfParts = Action.getNumberOfParts();
1331 bool AtCurrentPoint) {
1333 assert(!Elem.second.ServiceInsertPt &&
"Insert point is set already.");
1335 llvm::Value *Undef = llvm::UndefValue::get(CGF.
Int32Ty);
1336 if (AtCurrentPoint) {
1337 Elem.second.ServiceInsertPt =
new llvm::BitCastInst(
1340 Elem.second.ServiceInsertPt =
1341 new llvm::BitCastInst(Undef, CGF.
Int32Ty,
"svcpt");
1348 if (Elem.second.ServiceInsertPt) {
1349 llvm::Instruction *Ptr = Elem.second.ServiceInsertPt;
1350 Elem.second.ServiceInsertPt =
nullptr;
1351 Ptr->eraseFromParent();
1358 llvm::raw_svector_ostream OS(Buffer);
1362 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.
CurFuncDecl))
1363 OS << FD->getQualifiedNameAsString();
1370 unsigned Flags,
bool EmitLoc) {
1371 uint32_t SrcLocStrSize;
1372 llvm::Constant *SrcLocStr;
1374 llvm::codegenoptions::NoDebugInfo) ||
1376 SrcLocStr =
OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1378 std::string FunctionName;
1379 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.
CurFuncDecl))
1380 FunctionName = FD->getQualifiedNameAsString();
1390 SrcLocStr, SrcLocStrSize, llvm::omp::IdentFlag(Flags), Reserved2Flags);
1395 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1401 uint32_t SrcLocStrSize;
1402 auto *SrcLocStr =
OMPBuilder.getOrCreateSrcLocStr(
1405 OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize));
1408 llvm::Value *ThreadID =
nullptr;
1413 ThreadID = I->second.ThreadID;
1414 if (ThreadID !=
nullptr)
1418 if (
auto *OMPRegionInfo =
1420 if (OMPRegionInfo->getThreadIDVariable()) {
1422 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1426 CGF.
Builder.GetInsertBlock() == TopBlock ||
1427 !isa<llvm::Instruction>(LVal.
getPointer(CGF)) ||
1428 cast<llvm::Instruction>(LVal.
getPointer(CGF))->getParent() ==
1430 cast<llvm::Instruction>(LVal.
getPointer(CGF))->getParent() ==
1431 CGF.
Builder.GetInsertBlock()) {
1435 if (CGF.
Builder.GetInsertBlock() == TopBlock) {
1437 Elem.second.ThreadID = ThreadID;
1449 if (!Elem.second.ServiceInsertPt)
1451 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1452 CGF.
Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1456 OMPRTL___kmpc_global_thread_num),
1459 Elem.second.ThreadID =
Call;
1464 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1476 for(
const auto *
D : I->second)
1491 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(
CGM.
Int32Ty),
1498llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind
1500 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
1501 OMPDeclareTargetDeclAttr::getDeviceType(VD);
1503 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1505 switch ((
int)*DevTy) {
1506 case OMPDeclareTargetDeclAttr::DT_Host:
1507 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseHost;
1509 case OMPDeclareTargetDeclAttr::DT_NoHost:
1510 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNoHost;
1512 case OMPDeclareTargetDeclAttr::DT_Any:
1513 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny;
1516 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1521llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind
1523 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> MapType =
1524 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1526 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1527 switch ((
int)*MapType) {
1528 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_To:
1529 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
1531 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Enter:
1532 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter;
1534 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Link:
1535 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
1538 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1547 auto FileInfoCallBack = [&]() {
1551 llvm::sys::fs::UniqueID ID;
1552 if (llvm::sys::fs::getUniqueID(PLoc.
getFilename(), ID)) {
1553 PLoc =
SM.getPresumedLoc(BeginLoc,
false);
1559 return OMPBuilder.getTargetEntryUniqueInfo(FileInfoCallBack, ParentName);
1565 auto LinkageForVariable = [&VD,
this]() {
1569 std::vector<llvm::GlobalVariable *> GeneratedRefs;
1573 llvm::Constant *addr =
OMPBuilder.getAddrOfDeclareTargetVar(
1581 LinkageForVariable);
1593 std::string Suffix =
getName({
"cache",
""});
1594 return OMPBuilder.getOrCreateInternalVariable(
1607 llvm::Value *Args[] = {
1631 llvm::Value *Args[] = {
1634 Ctor, CopyCtor, Dtor};
1637 CGM.
getModule(), OMPRTL___kmpc_threadprivate_register),
1652 llvm::Value *Ctor =
nullptr, *CopyCtor =
nullptr, *Dtor =
nullptr;
1662 Args.push_back(&Dst);
1667 std::string Name =
getName({
"__kmpc_global_ctor_",
""});
1668 llvm::Function *Fn =
1694 Args.push_back(&Dst);
1699 std::string Name =
getName({
"__kmpc_global_dtor_",
""});
1700 llvm::Function *Fn =
1722 auto *CopyCtorTy = llvm::FunctionType::get(
CGM.
VoidPtrTy, CopyCtorTyArgs,
1728 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
1729 if (Ctor ==
nullptr) {
1733 Ctor = llvm::Constant::getNullValue(CtorTy);
1735 if (Dtor ==
nullptr) {
1739 Dtor = llvm::Constant::getNullValue(DtorTy);
1742 auto *InitFunctionTy =
1743 llvm::FunctionType::get(
CGM.
VoidTy,
false);
1744 std::string Name =
getName({
"__omp_threadprivate_init_",
""});
1754 return InitFunction;
1762 llvm::GlobalValue *GV) {
1763 std::optional<OMPDeclareTargetDeclAttr *> ActiveAttr =
1764 OMPDeclareTargetDeclAttr::getActiveAttr(FD);
1767 if (!ActiveAttr || !(*ActiveAttr)->getIndirect())
1774 OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(Name, EntryInfo);
1780 llvm::GlobalValue *Addr = GV;
1782 Addr =
new llvm::GlobalVariable(
1784 true, llvm::GlobalValue::ExternalLinkage, GV, Name,
1785 nullptr, llvm::GlobalValue::NotThreadLocal,
1786 CGM.
getModule().getDataLayout().getDefaultGlobalsAddressSpace());
1787 Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1790 OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(
1792 llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect,
1793 llvm::GlobalValue::WeakODRLinkage);
1799 std::string Suffix =
getName({
"artificial",
""});
1801 llvm::GlobalVariable *GAddr =
OMPBuilder.getOrCreateInternalVariable(
1802 VarLVType, Twine(Name).concat(Suffix).str());
1805 GAddr->setThreadLocal(
true);
1806 return Address(GAddr, GAddr->getValueType(),
1809 std::string CacheSuffix =
getName({
"cache",
""});
1810 llvm::Value *Args[] = {
1818 Twine(Name).concat(Suffix).concat(CacheSuffix).str())};
1825 VarLVType->getPointerTo(0)),
1869 llvm::Function *OutlinedFn,
1872 llvm::Value *NumThreads) {
1877 auto &&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc,
1881 llvm::Value *Args[] = {
1883 CGF.
Builder.getInt32(CapturedVars.size()),
1886 RealArgs.append(std::begin(Args), std::end(Args));
1887 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
1889 llvm::FunctionCallee RTLFn =
1890 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_fork_call);
1893 auto &&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc,
Loc,
1899 llvm::Value *Args[] = {RTLoc, ThreadID};
1901 M, OMPRTL___kmpc_serialized_parallel),
1908 ".bound.zero.addr");
1913 OutlinedFnArgs.push_back(ZeroAddrBound.
getPointer());
1914 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
1922 OutlinedFn->removeFnAttr(llvm::Attribute::AlwaysInline);
1923 OutlinedFn->addFnAttr(llvm::Attribute::NoInline);
1929 M, OMPRTL___kmpc_end_serialized_parallel),
1948 if (
auto *OMPRegionInfo =
1950 if (OMPRegionInfo->getThreadIDVariable())
1951 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
1960 return ThreadIDTemp;
1964 std::string Prefix = Twine(
"gomp_critical_user_", CriticalName).str();
1965 std::string Name =
getName({Prefix,
"var"});
1972 llvm::FunctionCallee EnterCallee;
1974 llvm::FunctionCallee ExitCallee;
1977 llvm::BasicBlock *ContBlock =
nullptr;
1980 CommonActionTy(llvm::FunctionCallee EnterCallee,
1982 llvm::FunctionCallee ExitCallee,
1984 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
1985 ExitArgs(ExitArgs), Conditional(Conditional) {}
1989 llvm::Value *CallBool = CGF.
Builder.CreateIsNotNull(EnterRes);
1993 CGF.
Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
2009 StringRef CriticalName,
2023 EnterArgs.push_back(CGF.
Builder.CreateIntCast(
2026 CommonActionTy Action(
2029 Hint ? OMPRTL___kmpc_critical_with_hint : OMPRTL___kmpc_critical),
2032 OMPRTL___kmpc_end_critical),
2049 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2071 llvm::Value *FilterVal = Filter
2078 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2098 llvm::Value *Args[] = {
2100 llvm::ConstantInt::get(
CGM.
IntTy, 0,
true)};
2106 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
2107 Region->emitUntiedSwitch(CGF);
2120 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2133 unsigned Index,
const VarDecl *Var) {
2141 Ptr, ElemTy->getPointerTo(Ptr->getType()->getPointerAddressSpace())),
2157 Args.push_back(&LHSArg);
2158 Args.push_back(&RHSArg);
2164 llvm::GlobalValue::InternalLinkage, Name,
2167 Fn->setDoesNotRecurse();
2174 ArgsElemType->getPointerTo()),
2178 ArgsElemType->getPointerTo()),
2184 for (
unsigned I = 0,
E = AssignmentOps.size(); I <
E; ++I) {
2185 const auto *DestVar =
2186 cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
2189 const auto *SrcVar =
2190 cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
2193 const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
2195 CGF.
EmitOMPCopy(
Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
2210 assert(CopyprivateVars.size() == SrcExprs.size() &&
2211 CopyprivateVars.size() == DstExprs.size() &&
2212 CopyprivateVars.size() == AssignmentOps.size());
2224 if (!CopyprivateVars.empty()) {
2227 C.getIntTypeForBitwidth(32, 1);
2228 DidIt = CGF.
CreateMemTemp(KmpInt32Ty,
".omp.copyprivate.did_it");
2233 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2250 llvm::APInt ArraySize(32, CopyprivateVars.size());
2251 QualType CopyprivateArrayTy =
C.getConstantArrayType(
2256 CGF.
CreateMemTemp(CopyprivateArrayTy,
".omp.copyprivate.cpr_list");
2257 for (
unsigned I = 0,
E = CopyprivateVars.size(); I <
E; ++I) {
2269 SrcExprs, DstExprs, AssignmentOps,
Loc);
2270 llvm::Value *BufSize = CGF.
getTypeSize(CopyprivateArrayTy);
2274 llvm::Value *Args[] = {
2299 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2314 if (Kind == OMPD_for)
2315 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
2316 else if (Kind == OMPD_sections)
2317 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
2318 else if (Kind == OMPD_single)
2319 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
2320 else if (Kind == OMPD_barrier)
2321 Flags = OMP_IDENT_BARRIER_EXPL;
2323 Flags = OMP_IDENT_BARRIER_IMPL;
2335 ScheduleKind = OMPC_SCHEDULE_static;
2337 llvm::APInt ChunkSize(32, 1);
2347 bool ForceSimpleCall) {
2349 auto *OMPRegionInfo =
2353 CGF.
Builder, Kind, ForceSimpleCall, EmitChecks));
2366 if (OMPRegionInfo) {
2367 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
2370 OMPRTL___kmpc_cancel_barrier),
2379 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
2396 Expr *ME,
bool IsFatal) {
2399 : llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
2402 llvm::Value *Args[] = {
2404 llvm::ConstantInt::get(
CGM.
Int32Ty, IsFatal ? 2 : 1),
2413 bool Chunked,
bool Ordered) {
2414 switch (ScheduleKind) {
2415 case OMPC_SCHEDULE_static:
2416 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
2417 : (Ordered ? OMP_ord_static : OMP_sch_static);
2418 case OMPC_SCHEDULE_dynamic:
2419 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
2420 case OMPC_SCHEDULE_guided:
2421 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
2422 case OMPC_SCHEDULE_runtime:
2423 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
2424 case OMPC_SCHEDULE_auto:
2425 return Ordered ? OMP_ord_auto : OMP_sch_auto;
2427 assert(!Chunked &&
"chunk was specified but schedule kind not known");
2428 return Ordered ? OMP_ord_static : OMP_sch_static;
2430 llvm_unreachable(
"Unexpected runtime schedule");
2434static OpenMPSchedType
2437 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
2441 bool Chunked)
const {
2442 OpenMPSchedType Schedule =
2444 return Schedule == OMP_sch_static;
2450 return Schedule == OMP_dist_sch_static;
2454 bool Chunked)
const {
2455 OpenMPSchedType Schedule =
2457 return Schedule == OMP_sch_static_chunked;
2463 return Schedule == OMP_dist_sch_static_chunked;
2467 OpenMPSchedType Schedule =
2469 assert(Schedule != OMP_sch_static_chunked &&
"cannot be chunked here");
2470 return Schedule != OMP_sch_static;
2478 case OMPC_SCHEDULE_MODIFIER_monotonic:
2479 Modifier = OMP_sch_modifier_monotonic;
2481 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2482 Modifier = OMP_sch_modifier_nonmonotonic;
2484 case OMPC_SCHEDULE_MODIFIER_simd:
2485 if (Schedule == OMP_sch_static_chunked)
2486 Schedule = OMP_sch_static_balanced_chunked;
2493 case OMPC_SCHEDULE_MODIFIER_monotonic:
2494 Modifier = OMP_sch_modifier_monotonic;
2496 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2497 Modifier = OMP_sch_modifier_nonmonotonic;
2499 case OMPC_SCHEDULE_MODIFIER_simd:
2500 if (Schedule == OMP_sch_static_chunked)
2501 Schedule = OMP_sch_static_balanced_chunked;
2513 if (CGM.
getLangOpts().OpenMP >= 50 && Modifier == 0) {
2514 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
2515 Schedule == OMP_sch_static_balanced_chunked ||
2516 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||
2517 Schedule == OMP_dist_sch_static_chunked ||
2518 Schedule == OMP_dist_sch_static))
2519 Modifier = OMP_sch_modifier_nonmonotonic;
2521 return Schedule | Modifier;
2531 ScheduleKind.
Schedule, DispatchValues.
Chunk !=
nullptr, Ordered);
2533 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
2534 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&
2535 Schedule != OMP_sch_static_balanced_chunked));
2542 llvm::Value *Chunk = DispatchValues.
Chunk ? DispatchValues.
Chunk
2543 : CGF.
Builder.getIntN(IVSize, 1);
2544 llvm::Value *Args[] = {
2548 CGM, Schedule, ScheduleKind.
M1, ScheduleKind.
M2)),
2551 CGF.
Builder.getIntN(IVSize, 1),
2568 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
2569 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
2576 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||
2577 Schedule == OMP_sch_static_balanced_chunked ||
2578 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||
2579 Schedule == OMP_dist_sch_static ||
2580 Schedule == OMP_dist_sch_static_chunked);
2587 llvm::Value *Chunk = Values.
Chunk;
2588 if (Chunk ==
nullptr) {
2589 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
2590 Schedule == OMP_dist_sch_static) &&
2591 "expected static non-chunked schedule");
2595 assert((Schedule == OMP_sch_static_chunked ||
2596 Schedule == OMP_sch_static_balanced_chunked ||
2597 Schedule == OMP_ord_static_chunked ||
2598 Schedule == OMP_dist_sch_static_chunked) &&
2599 "expected static chunked schedule");
2601 llvm::Value *Args[] = {
2624 "Expected loop-based or sections-based directive.");
2627 ? OMP_IDENT_WORK_LOOP
2628 : OMP_IDENT_WORK_SECTIONS);
2630 llvm::FunctionCallee StaticInitFunction =
2635 ScheduleNum, ScheduleKind.
M1, ScheduleKind.
M2, Values);
2642 OpenMPSchedType ScheduleNum =
2644 llvm::Value *UpdatedLocation =
2647 llvm::FunctionCallee StaticInitFunction;
2648 bool isGPUDistribute =
2651 StaticInitFunction =
OMPBuilder.createForStaticInitFunction(
2662 assert((DKind == OMPD_distribute || DKind == OMPD_for ||
2663 DKind == OMPD_sections) &&
2664 "Expected distribute, for, or sections directive kind");
2668 llvm::Value *Args[] = {
2671 (DKind == OMPD_target_teams_loop)
2672 ? OMP_IDENT_WORK_DISTRIBUTE
2674 ? OMP_IDENT_WORK_LOOP
2675 : OMP_IDENT_WORK_SECTIONS),
2683 CGM.
getModule(), OMPRTL___kmpc_distribute_static_fini),
2712 llvm::Value *Args[] = {
2720 OMPBuilder.createDispatchNextFunction(IVSize, IVSigned), Args);
2727 llvm::Value *NumThreads,
2732 llvm::Value *Args[] = {
2741 ProcBindKind ProcBind,
2745 assert(ProcBind != OMP_PROC_BIND_unknown &&
"Unsupported proc_bind value.");
2747 llvm::Value *Args[] = {
2749 llvm::ConstantInt::get(
CGM.
IntTy,
unsigned(ProcBind),
true)};
2771enum KmpTaskTFields {
2801 llvm::OpenMPIRBuilder::EmitMetadataErrorReportFunctionTy &&ErrorReportFn =
2802 [
this](llvm::OpenMPIRBuilder::EmitMetadataErrorKind Kind,
2803 const llvm::TargetRegionEntryInfo &EntryInfo) ->
void {
2805 if (Kind != llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR) {
2809 if (I->getFirst().getUniqueID().getDevice() == EntryInfo.DeviceID &&
2810 I->getFirst().getUniqueID().getFile() == EntryInfo.FileID) {
2812 I->getFirst(), EntryInfo.Line, 1);
2818 case llvm::OpenMPIRBuilder::EMIT_MD_TARGET_REGION_ERROR: {
2821 "%0 is incorrect: either the "
2822 "address or the ID is invalid.");
2825 case llvm::OpenMPIRBuilder::EMIT_MD_DECLARE_TARGET_ERROR: {
2828 "variable %0 is incorrect: the "
2829 "address is invalid.");
2832 case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR: {
2835 "Offloading entry for declare target variable is incorrect: the "
2836 "address is invalid.");
2842 OMPBuilder.createOffloadEntriesAndInfoMetadata(ErrorReportFn);
2849 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty,
C.VoidPtrTy};
2852 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
2858struct PrivateHelpersTy {
2859 PrivateHelpersTy(
const Expr *OriginalRef,
const VarDecl *Original,
2861 : OriginalRef(OriginalRef), Original(Original), PrivateCopy(PrivateCopy),
2862 PrivateElemInit(PrivateElemInit) {}
2863 PrivateHelpersTy(
const VarDecl *Original) : Original(Original) {}
2864 const Expr *OriginalRef =
nullptr;
2865 const VarDecl *Original =
nullptr;
2866 const VarDecl *PrivateCopy =
nullptr;
2867 const VarDecl *PrivateElemInit =
nullptr;
2868 bool isLocalPrivate()
const {
2869 return !OriginalRef && !PrivateCopy && !PrivateElemInit;
2872typedef std::pair<
CharUnits , PrivateHelpersTy> PrivateDataTy;
2877 if (!CVD->
hasAttr<OMPAllocateDeclAttr>())
2879 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
2881 return !(AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc &&
2882 !AA->getAllocator());
2887 if (!Privates.empty()) {
2892 RecordDecl *RD =
C.buildImplicitRecord(
".kmp_privates.t");
2894 for (
const auto &Pair : Privates) {
2895 const VarDecl *VD = Pair.second.Original;
2899 if (Pair.second.isLocalPrivate()) {
2922 QualType KmpRoutineEntryPointerQTy) {
2942 QualType KmpCmplrdataTy =
C.getRecordType(UD);
2943 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_task_t");
2973 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_task_t_with_privates");
2993static llvm::Function *
2996 QualType KmpTaskTWithPrivatesPtrQTy,
2998 QualType SharedsPtrTy, llvm::Function *TaskFunction,
2999 llvm::Value *TaskPrivatesMap) {
3007 Args.push_back(&GtidArg);
3008 Args.push_back(&TaskTypeArg);
3009 const auto &TaskEntryFnInfo =
3011 llvm::FunctionType *TaskEntryTy =
3014 auto *TaskEntry = llvm::Function::Create(
3015 TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.
getModule());
3017 TaskEntry->setDoesNotRecurse();
3032 const auto *KmpTaskTWithPrivatesQTyRD =
3033 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
3036 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->
getAsTagDecl());
3037 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
3039 llvm::Value *PartidParam = PartIdLVal.
getPointer(CGF);
3041 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
3047 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
3048 llvm::Value *PrivatesParam;
3049 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
3054 PrivatesParam = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
3057 llvm::Value *CommonArgs[] = {
3058 GtidParam, PartidParam, PrivatesParam, TaskPrivatesMap,
3064 std::end(CommonArgs));
3066 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
3069 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
3072 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
3075 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
3078 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
3081 CallArgs.push_back(LBParam);
3082 CallArgs.push_back(UBParam);
3083 CallArgs.push_back(StParam);
3084 CallArgs.push_back(LIParam);
3085 CallArgs.push_back(RParam);
3087 CallArgs.push_back(SharedsParam);
3100 QualType KmpTaskTWithPrivatesPtrQTy,
3101 QualType KmpTaskTWithPrivatesQTy) {
3109 Args.push_back(&GtidArg);
3110 Args.push_back(&TaskTypeArg);
3111 const auto &DestructorFnInfo =
3113 llvm::FunctionType *DestructorFnTy =
3117 auto *DestructorFn =
3118 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
3122 DestructorFn->setDoesNotRecurse();
3130 const auto *KmpTaskTWithPrivatesQTyRD =
3131 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
3132 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
3134 for (
const auto *Field :
3135 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
3137 Field->getType().isDestructedType()) {
3143 return DestructorFn;
3163 C,
nullptr,
Loc,
nullptr,
3164 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
3166 Args.push_back(&TaskPrivatesArg);
3167 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>,
unsigned> PrivateVarsPos;
3168 unsigned Counter = 1;
3169 for (
const Expr *
E :
Data.PrivateVars) {
3171 C,
nullptr,
Loc,
nullptr,
3172 C.getPointerType(
C.getPointerType(
E->
getType()))
3176 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
3177 PrivateVarsPos[VD] = Counter;
3180 for (
const Expr *
E :
Data.FirstprivateVars) {
3182 C,
nullptr,
Loc,
nullptr,
3183 C.getPointerType(
C.getPointerType(
E->
getType()))
3187 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
3188 PrivateVarsPos[VD] = Counter;
3191 for (
const Expr *
E :
Data.LastprivateVars) {
3193 C,
nullptr,
Loc,
nullptr,
3194 C.getPointerType(
C.getPointerType(
E->
getType()))
3198 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
3199 PrivateVarsPos[VD] = Counter;
3205 Ty =
C.getPointerType(Ty);
3207 Ty =
C.getPointerType(Ty);
3209 C,
nullptr,
Loc,
nullptr,
3210 C.getPointerType(
C.getPointerType(Ty)).withConst().withRestrict(),
3212 PrivateVarsPos[VD] = Counter;
3215 const auto &TaskPrivatesMapFnInfo =
3217 llvm::FunctionType *TaskPrivatesMapTy =
3221 auto *TaskPrivatesMap = llvm::Function::Create(
3222 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
3225 TaskPrivatesMapFnInfo);
3227 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
3228 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
3229 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
3233 TaskPrivatesMapFnInfo, Args,
Loc,
Loc);
3239 const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->
getAsTagDecl());
3241 for (
const FieldDecl *Field : PrivatesQTyRD->fields()) {
3243 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
3252 return TaskPrivatesMap;
3264 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
3270 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
3278 if ((!IsTargetTask && !
Data.FirstprivateVars.empty() && ForDup) ||
3279 (IsTargetTask && KmpTaskSharedsPtr.
isValid())) {
3286 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
3287 for (
const PrivateDataTy &Pair : Privates) {
3289 if (Pair.second.isLocalPrivate()) {
3293 const VarDecl *VD = Pair.second.PrivateCopy;
3295 if (
Init && (!ForDup || (isa<CXXConstructExpr>(
Init) &&
3298 if (
const VarDecl *Elem = Pair.second.PrivateElemInit) {
3299 const VarDecl *OriginalVD = Pair.second.Original;
3304 const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);
3305 if (IsTargetTask && !SharedField) {
3306 assert(isa<ImplicitParamDecl>(OriginalVD) &&
3309 ->getNumParams() == 0 &&
3310 isa<TranslationUnitDecl>(
3312 ->getDeclContext()) &&
3313 "Expected artificial target data variable.");
3316 }
else if (ForDup) {
3320 C.getDeclAlign(OriginalVD)),
3324 Pair.second.Original->getCanonicalDecl()) > 0 ||
3326 SharedRefLValue = CGF.
EmitLValue(Pair.second.OriginalRef);
3329 InlinedOpenMPRegionRAII Region(
3332 SharedRefLValue = CGF.
EmitLValue(Pair.second.OriginalRef);
3344 [&CGF, Elem,
Init, &CapturesInfo](
Address DestElement,
3347 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3348 InitScope.addPrivate(Elem, SrcElement);
3349 (void)InitScope.Privatize();
3351 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
3352 CGF, &CapturesInfo);
3353 CGF.EmitAnyExprToMem(Init, DestElement,
3354 Init->getType().getQualifiers(),
3359 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3360 InitScope.addPrivate(Elem, SharedRefLValue.
getAddress());
3361 (void)InitScope.Privatize();
3362 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
3377 bool InitRequired =
false;
3378 for (
const PrivateDataTy &Pair : Privates) {
3379 if (Pair.second.isLocalPrivate())
3381 const VarDecl *VD = Pair.second.PrivateCopy;
3383 InitRequired = InitRequired || (isa_and_nonnull<CXXConstructExpr>(
Init) &&
3388 return InitRequired;
3405 QualType KmpTaskTWithPrivatesPtrQTy,
3413 KmpTaskTWithPrivatesPtrQTy,
3416 KmpTaskTWithPrivatesPtrQTy,
3420 Args.push_back(&DstArg);
3421 Args.push_back(&SrcArg);
3422 Args.push_back(&LastprivArg);
3423 const auto &TaskDupFnInfo =
3427 auto *TaskDup = llvm::Function::Create(
3428 TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.
getModule());
3430 TaskDup->setDoesNotRecurse();
3440 auto LIFI = std::next(KmpTaskTQTyRD->
field_begin(), KmpTaskTLastIter);
3442 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
3450 assert(!Privates.empty());
3452 if (!
Data.FirstprivateVars.empty()) {
3457 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
3466 SharedsTy, SharedsPtrTy,
Data, Privates,
true);
3476 for (
const PrivateDataTy &
P : Privates) {
3477 if (
P.second.isLocalPrivate())
3479 QualType Ty =
P.second.Original->getType().getNonReferenceType();
3488class OMPIteratorGeneratorScope final
3489 :
public CodeGenFunction::OMPPrivateScope {
3494 OMPIteratorGeneratorScope() =
delete;
3495 OMPIteratorGeneratorScope(OMPIteratorGeneratorScope &) =
delete;
3503 for (
unsigned I = 0, End =
E->numOfIterators(); I < End; ++I) {
3505 const auto *VD = cast<VarDecl>(
E->getIteratorDecl(I));
3514 for (
unsigned I = 0, End =
E->numOfIterators(); I < End; ++I) {
3523 CodeGenFunction::JumpDest &ContDest =
3525 CodeGenFunction::JumpDest &ExitDest =
3528 llvm::Value *N = Uppers[I];
3536 ? CGF.
Builder.CreateICmpSLT(CVal, N)
3537 : CGF.
Builder.CreateICmpULT(CVal, N);
3539 CGF.
Builder.CreateCondBr(Cmp, BodyBB, ExitDest.getBlock());
3546 ~OMPIteratorGeneratorScope() {
3549 for (
unsigned I =
E->numOfIterators(); I > 0; --I) {
3556 CGF.
EmitBlock(ExitDests[I - 1].getBlock(), I == 1);
3562static std::pair<llvm::Value *, llvm::Value *>
3564 const auto *OASE = dyn_cast<OMPArrayShapingExpr>(
E);
3567 const Expr *
Base = OASE->getBase();
3572 llvm::Value *SizeVal;
3575 SizeVal = CGF.
getTypeSize(OASE->getBase()->getType()->getPointeeType());
3576 for (
const Expr *SE : OASE->getDimensions()) {
3580 SizeVal = CGF.
Builder.CreateNUWMul(SizeVal, Sz);
3582 }
else if (
const auto *ASE =
3586 llvm::Value *UpAddr = CGF.
Builder.CreateConstGEP1_32(
3589 llvm::Value *LowIntPtr = CGF.
Builder.CreatePtrToInt(Addr, CGF.
SizeTy);
3590 llvm::Value *UpIntPtr = CGF.
Builder.CreatePtrToInt(UpAddr, CGF.
SizeTy);
3591 SizeVal = CGF.
Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
3595 return std::make_pair(Addr, SizeVal);
3600 QualType FlagsTy =
C.getIntTypeForBitwidth(32,
false);
3601 if (KmpTaskAffinityInfoTy.
isNull()) {
3603 C.buildImplicitRecord(
"kmp_task_affinity_info_t");
3609 KmpTaskAffinityInfoTy =
C.getRecordType(KmpAffinityInfoRD);
3616 llvm::Function *TaskFunction,
QualType SharedsTy,
3621 const auto *I =
Data.PrivateCopies.begin();
3622 for (
const Expr *
E :
Data.PrivateVars) {
3623 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
3624 Privates.emplace_back(
3626 PrivateHelpersTy(
E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
3630 I =
Data.FirstprivateCopies.begin();
3631 const auto *IElemInitRef =
Data.FirstprivateInits.begin();
3632 for (
const Expr *
E :
Data.FirstprivateVars) {
3633 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
3634 Privates.emplace_back(
3637 E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
3638 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
3642 I =
Data.LastprivateCopies.begin();
3643 for (
const Expr *
E :
Data.LastprivateVars) {
3644 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
3645 Privates.emplace_back(
3647 PrivateHelpersTy(
E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
3655 Privates.emplace_back(
C.getDeclAlign(VD), PrivateHelpersTy(VD));
3657 llvm::stable_sort(Privates,
3658 [](
const PrivateDataTy &L,
const PrivateDataTy &R) {
3659 return L.first > R.first;
3661 QualType KmpInt32Ty =
C.getIntTypeForBitwidth(32, 1);
3672 assert((
D.getDirectiveKind() == OMPD_task ||
3675 "Expected taskloop, task or target directive");
3684 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
3686 QualType KmpTaskTWithPrivatesQTy =
C.getRecordType(KmpTaskTWithPrivatesQTyRD);
3687 QualType KmpTaskTWithPrivatesPtrQTy =
3688 C.getPointerType(KmpTaskTWithPrivatesQTy);
3689 llvm::Type *KmpTaskTWithPrivatesTy = CGF.
ConvertType(KmpTaskTWithPrivatesQTy);
3690 llvm::Type *KmpTaskTWithPrivatesPtrTy =
3691 KmpTaskTWithPrivatesTy->getPointerTo();
3692 llvm::Value *KmpTaskTWithPrivatesTySize =
3694 QualType SharedsPtrTy =
C.getPointerType(SharedsTy);
3697 llvm::Value *TaskPrivatesMap =
nullptr;
3698 llvm::Type *TaskPrivatesMapTy =
3699 std::next(TaskFunction->arg_begin(), 3)->getType();
3700 if (!Privates.empty()) {
3701 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
3705 TaskPrivatesMap, TaskPrivatesMapTy);
3707 TaskPrivatesMap = llvm::ConstantPointerNull::get(
3708 cast<llvm::PointerType>(TaskPrivatesMapTy));
3713 CGM,
Loc,
D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
3714 KmpTaskTWithPrivatesQTy,
KmpTaskTQTy, SharedsPtrTy, TaskFunction,
3726 DestructorsFlag = 0x8,
3727 PriorityFlag = 0x20,
3728 DetachableFlag = 0x40,
3730 unsigned Flags =
Data.Tied ? TiedFlag : 0;
3731 bool NeedsCleanup =
false;
3732 if (!Privates.empty()) {
3736 Flags = Flags | DestructorsFlag;
3738 if (
Data.Priority.getInt())
3739 Flags = Flags | PriorityFlag;
3741 Flags = Flags | DetachableFlag;
3742 llvm::Value *TaskFlags =
3743 Data.Final.getPointer()
3744 ? CGF.
Builder.CreateSelect(
Data.Final.getPointer(),
3745 CGF.
Builder.getInt32(FinalFlag),
3747 : CGF.
Builder.getInt32(
Data.Final.getInt() ? FinalFlag : 0);
3748 TaskFlags = CGF.
Builder.CreateOr(TaskFlags, CGF.
Builder.getInt32(Flags));
3749 llvm::Value *SharedsSize =
CGM.
getSize(
C.getTypeSizeInChars(SharedsTy));
3754 llvm::Value *NewTask;
3761 llvm::Value *DeviceID;
3766 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
3767 AllocArgs.push_back(DeviceID);
3788 llvm::Value *Tid =
getThreadID(CGF, DC->getBeginLoc());
3789 Tid = CGF.
Builder.CreateIntCast(Tid, CGF.
IntTy,
false);
3792 CGM.
getModule(), OMPRTL___kmpc_task_allow_completion_event),
3793 {Loc, Tid, NewTask});
3804 llvm::Value *NumOfElements =
nullptr;
3805 unsigned NumAffinities = 0;
3807 if (
const Expr *Modifier =
C->getModifier()) {
3808 const auto *IE = cast<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts());
3809 for (
unsigned I = 0,
E = IE->numOfIterators(); I <
E; ++I) {
3813 NumOfElements ? CGF.
Builder.CreateNUWMul(NumOfElements, Sz) : Sz;
3816 NumAffinities +=
C->varlist_size();
3821 enum RTLAffinityInfoFieldsTy { BaseAddr, Len, Flags };
3823 QualType KmpTaskAffinityInfoArrayTy;
3824 if (NumOfElements) {
3825 NumOfElements = CGF.
Builder.CreateNUWAdd(
3826 llvm::ConstantInt::get(CGF.
SizeTy, NumAffinities), NumOfElements);
3829 C.getIntTypeForBitwidth(
C.getTypeSize(
C.getSizeType()), 0),
3833 KmpTaskAffinityInfoArrayTy =
C.getVariableArrayType(
3841 NumOfElements = CGF.
Builder.CreateIntCast(NumOfElements, CGF.
Int32Ty,
3844 KmpTaskAffinityInfoArrayTy =
C.getConstantArrayType(
3846 llvm::APInt(
C.getTypeSize(
C.getSizeType()), NumAffinities),
nullptr,
3849 CGF.
CreateMemTemp(KmpTaskAffinityInfoArrayTy,
".affs.arr.addr");
3851 NumOfElements = llvm::ConstantInt::get(
CGM.
Int32Ty, NumAffinities,
3858 bool HasIterator =
false;
3860 if (
C->getModifier()) {
3864 for (
const Expr *
E :
C->varlists()) {
3873 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
3878 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
3892 const Expr *Modifier =
C->getModifier();
3895 OMPIteratorGeneratorScope IteratorScope(
3897 for (
const Expr *
E :
C->varlists()) {
3907 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
3912 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
3914 Idx = CGF.
Builder.CreateNUWAdd(
3915 Idx, llvm::ConstantInt::get(Idx->getType(), 1));
3930 CGM.
getModule(), OMPRTL___kmpc_omp_reg_task_with_affinity),
3931 {LocRef, GTid, NewTask, NumOfElements, AffinListPtr});
3933 llvm::Value *NewTaskNewTaskTTy =
3935 NewTask, KmpTaskTWithPrivatesPtrTy);
3937 KmpTaskTWithPrivatesQTy);
3948 *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds)),
3957 if (!Privates.empty()) {
3959 SharedsTy, SharedsPtrTy,
Data, Privates,
3964 CGM,
Loc,
D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
3965 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy,
Data, Privates,
3966 !
Data.LastprivateVars.empty());
3970 enum {
Priority = 0, Destructors = 1 };
3972 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
3974 (*FI)->getType()->getAsUnionType()->getDecl();
3977 CGM,
Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
3978 KmpTaskTWithPrivatesQTy);
3981 Data1LV, *std::next(KmpCmplrdataUD->
field_begin(), Destructors));
3987 if (
Data.Priority.getInt()) {
3989 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
3994 Result.NewTask = NewTask;
3995 Result.TaskEntry = TaskEntry;
3996 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
3998 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
4004 RTLDependenceKindTy DepKind;
4006 case OMPC_DEPEND_in:
4007 DepKind = RTLDependenceKindTy::DepIn;
4010 case OMPC_DEPEND_out:
4011 case OMPC_DEPEND_inout:
4012 DepKind = RTLDependenceKindTy::DepInOut;
4014 case OMPC_DEPEND_mutexinoutset:
4015 DepKind = RTLDependenceKindTy::DepMutexInOutSet;
4017 case OMPC_DEPEND_inoutset:
4018 DepKind = RTLDependenceKindTy::DepInOutSet;
4020 case OMPC_DEPEND_outallmemory:
4021 DepKind = RTLDependenceKindTy::DepOmpAllMem;
4023 case OMPC_DEPEND_source:
4024 case OMPC_DEPEND_sink:
4025 case OMPC_DEPEND_depobj:
4026 case OMPC_DEPEND_inoutallmemory:
4028 llvm_unreachable(
"Unknown task dependence type");
4036 FlagsTy =
C.getIntTypeForBitwidth(
C.getTypeSize(
C.BoolTy),
false);
4037 if (KmpDependInfoTy.
isNull()) {
4038 RecordDecl *KmpDependInfoRD =
C.buildImplicitRecord(
"kmp_depend_info");
4044 KmpDependInfoTy =
C.getRecordType(KmpDependInfoRD);
4048std::pair<llvm::Value *, LValue>
4062 CGF,
Base.getAddress(),
4063 llvm::ConstantInt::get(CGF.
IntPtrTy, -1,
true));
4070 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4072 return std::make_pair(NumDeps,
Base);
4076 llvm::PointerUnion<unsigned *, LValue *> Pos,
4087 OMPIteratorGeneratorScope IteratorScope(
4088 CGF, cast_or_null<OMPIteratorExpr>(
4089 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4100 Addr = llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4101 Size = llvm::ConstantInt::get(CGF.
SizeTy, 0);
4104 if (
unsigned *
P = Pos.dyn_cast<
unsigned *>()) {
4108 assert(
E &&
"Expected a non-null expression");
4118 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4123 static_cast<unsigned int>(RTLDependInfoFields::Len)));
4130 static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4132 llvm::ConstantInt::get(LLVMFlagsTy,
static_cast<unsigned int>(DepKind)),
4134 if (
unsigned *
P = Pos.dyn_cast<
unsigned *>()) {
4139 Idx = CGF.
Builder.CreateNUWAdd(Idx,
4140 llvm::ConstantInt::get(Idx->getType(), 1));
4149 assert(
Data.DepKind == OMPC_DEPEND_depobj &&
4150 "Expected depobj dependency kind.");
4155 OMPIteratorGeneratorScope IteratorScope(
4156 CGF, cast_or_null<OMPIteratorExpr>(
4157 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4160 llvm::Value *NumDeps;
4163 std::tie(NumDeps,
Base) =
4167 C.getUIntPtrType());
4171 llvm::Value *Add = CGF.
Builder.CreateNUWAdd(PrevVal, NumDeps);
4173 SizeLVals.push_back(NumLVal);
4176 for (
unsigned I = 0,
E = SizeLVals.size(); I <
E; ++I) {
4179 Sizes.push_back(Size);
4189 assert(
Data.DepKind == OMPC_DEPEND_depobj &&
4190 "Expected depobj dependency kind.");
4193 OMPIteratorGeneratorScope IteratorScope(
4194 CGF, cast_or_null<OMPIteratorExpr>(
4195 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4197 for (
unsigned I = 0, End =
Data.DepExprs.size(); I < End; ++I) {
4199 llvm::Value *NumDeps;
4202 std::tie(NumDeps,
Base) =
4206 llvm::Value *Size = CGF.
Builder.CreateNUWMul(
4215 llvm::Value *Add = CGF.
Builder.CreateNUWAdd(Pos, NumDeps);
4225 return D.DepExprs.empty();
4231 llvm::Value *NumOfElements =
nullptr;
4232 unsigned NumDependencies = std::accumulate(
4233 Dependencies.begin(), Dependencies.end(), 0,
4235 return D.DepKind == OMPC_DEPEND_depobj
4237 : (V + (D.IteratorExpr ? 0 : D.DepExprs.size()));
4241 bool HasDepobjDeps =
false;
4242 bool HasRegularWithIterators =
false;
4243 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4244 llvm::Value *NumOfRegularWithIterators =
4245 llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4249 if (
D.DepKind == OMPC_DEPEND_depobj) {
4252 for (llvm::Value *Size : Sizes) {
4253 NumOfDepobjElements =
4254 CGF.
Builder.CreateNUWAdd(NumOfDepobjElements, Size);
4256 HasDepobjDeps =
true;
4261 if (
const auto *IE = cast_or_null<OMPIteratorExpr>(
D.IteratorExpr)) {
4262 llvm::Value *ClauseIteratorSpace =
4263 llvm::ConstantInt::get(CGF.
IntPtrTy, 1);
4264 for (
unsigned I = 0,
E = IE->numOfIterators(); I <
E; ++I) {
4267 ClauseIteratorSpace = CGF.
Builder.CreateNUWMul(Sz, ClauseIteratorSpace);
4269 llvm::Value *NumClauseDeps = CGF.
Builder.CreateNUWMul(
4270 ClauseIteratorSpace,
4271 llvm::ConstantInt::get(CGF.
IntPtrTy,
D.DepExprs.size()));
4272 NumOfRegularWithIterators =
4273 CGF.
Builder.CreateNUWAdd(NumOfRegularWithIterators, NumClauseDeps);
4274 HasRegularWithIterators =
true;
4280 if (HasDepobjDeps || HasRegularWithIterators) {
4281 NumOfElements = llvm::ConstantInt::get(
CGM.
IntPtrTy, NumDependencies,
4283 if (HasDepobjDeps) {
4285 CGF.
Builder.CreateNUWAdd(NumOfDepobjElements, NumOfElements);
4287 if (HasRegularWithIterators) {
4289 CGF.
Builder.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements);
4292 Loc,
C.getIntTypeForBitwidth(64, 0),
4296 KmpDependInfoArrayTy =
4305 NumOfElements = CGF.
Builder.CreateIntCast(NumOfElements, CGF.
Int32Ty,
4308 KmpDependInfoArrayTy =
C.getConstantArrayType(
4314 NumOfElements = llvm::ConstantInt::get(
CGM.
Int32Ty, NumDependencies,
4318 for (
unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4319 if (Dependencies[I].DepKind == OMPC_DEPEND_depobj ||
4320 Dependencies[I].IteratorExpr)
4327 CGF.
CreateMemTemp(
C.getSizeType(),
"dep.counter.addr"),
C.getSizeType());
4329 for (
unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4330 if (Dependencies[I].DepKind == OMPC_DEPEND_depobj ||
4331 !Dependencies[I].IteratorExpr)
4337 if (HasDepobjDeps) {
4338 for (
unsigned I = 0, End = Dependencies.size(); I < End; ++I) {
4339 if (Dependencies[I].DepKind != OMPC_DEPEND_depobj)
4347 return std::make_pair(NumOfElements, DependenciesArray);
4358 unsigned NumDependencies = Dependencies.
DepExprs.size();
4369 llvm::Value *NumDepsVal;
4371 if (
const auto *IE =
4372 cast_or_null<OMPIteratorExpr>(Dependencies.
IteratorExpr)) {
4373 NumDepsVal = llvm::ConstantInt::get(CGF.
SizeTy, 1);
4374 for (
unsigned I = 0,
E = IE->numOfIterators(); I <
E; ++I) {
4377 NumDepsVal = CGF.
Builder.CreateNUWMul(NumDepsVal, Sz);
4379 Size = CGF.
Builder.CreateNUWAdd(llvm::ConstantInt::get(CGF.
SizeTy, 1),
4383 llvm::Value *RecSize =
CGM.
getSize(SizeInBytes);
4384 Size = CGF.
Builder.CreateNUWMul(Size, RecSize);
4388 QualType KmpDependInfoArrayTy =
C.getConstantArrayType(
4391 CharUnits Sz =
C.getTypeSizeInChars(KmpDependInfoArrayTy);
4393 NumDepsVal = llvm::ConstantInt::get(CGF.
IntPtrTy, NumDependencies);
4398 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4399 llvm::Value *Args[] = {ThreadID, Size, Allocator};
4404 Args,
".dep.arr.addr");
4407 Addr, KmpDependInfoLlvmTy->getPointerTo());
4408 DependenciesArray =
Address(Addr, KmpDependInfoLlvmTy, Align);
4415 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4417 llvm::PointerUnion<unsigned *, LValue *> Pos;
4434 return DependenciesArray;
4450 llvm::ConstantInt::get(CGF.
IntPtrTy, -1,
true));
4455 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4456 llvm::Value *Args[] = {ThreadID, DepObjAddr, Allocator};
4473 llvm::Value *NumDeps;
4480 Begin.emitRawPointer(CGF), NumDeps);
4484 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
4486 llvm::PHINode *ElementPHI =
4487 CGF.
Builder.CreatePHI(
Begin.getType(), 2,
"omp.elementPast");
4488 ElementPHI->addIncoming(
Begin.emitRawPointer(CGF), EntryBB);
4491 Base.getTBAAInfo());
4496 static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4498 llvm::ConstantInt::get(LLVMFlagsTy,
static_cast<unsigned int>(DepKind)),
4502 llvm::Value *ElementNext =
4505 ElementPHI->addIncoming(ElementNext, CGF.
Builder.GetInsertBlock());
4506 llvm::Value *IsEmpty =
4507 CGF.
Builder.CreateICmpEQ(ElementNext, End,
"omp.isempty");
4508 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
4515 llvm::Function *TaskFunction,
4524 llvm::Value *NewTask =
Result.NewTask;
4525 llvm::Function *TaskEntry =
Result.TaskEntry;
4526 llvm::Value *NewTaskNewTaskTTy =
Result.NewTaskNewTaskTTy;
4531 llvm::Value *NumOfElements;
4532 std::tie(NumOfElements, DependenciesArray) =
4543 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
4544 llvm::Value *DepTaskArgs[7];
4545 if (!
Data.Dependences.empty()) {
4546 DepTaskArgs[0] = UpLoc;
4547 DepTaskArgs[1] = ThreadID;
4548 DepTaskArgs[2] = NewTask;
4549 DepTaskArgs[3] = NumOfElements;
4551 DepTaskArgs[5] = CGF.
Builder.getInt32(0);
4552 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4554 auto &&ThenCodeGen = [
this, &
Data, TDBase, KmpTaskTQTyRD, &TaskArgs,
4557 auto PartIdFI = std::next(KmpTaskTQTyRD->
field_begin(), KmpTaskTPartId);
4561 if (!
Data.Dependences.empty()) {
4574 Region->emitUntiedSwitch(CGF);
4577 llvm::Value *DepWaitTaskArgs[7];
4578 if (!
Data.Dependences.empty()) {
4579 DepWaitTaskArgs[0] = UpLoc;
4580 DepWaitTaskArgs[1] = ThreadID;
4581 DepWaitTaskArgs[2] = NumOfElements;
4583 DepWaitTaskArgs[4] = CGF.
Builder.getInt32(0);
4584 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4585 DepWaitTaskArgs[6] =
4586 llvm::ConstantInt::get(CGF.
Int32Ty,
Data.HasNowaitClause);
4589 auto &&ElseCodeGen = [
this, &M, &TaskArgs, ThreadID, NewTaskNewTaskTTy,
4590 TaskEntry, &
Data, &DepWaitTaskArgs,
4597 if (!
Data.Dependences.empty())
4599 M, OMPRTL___kmpc_omp_taskwait_deps_51),
4602 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
4605 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
4606 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF,
Loc, TaskEntry,
4615 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
4616 M, OMPRTL___kmpc_omp_task_begin_if0),
4619 M, OMPRTL___kmpc_omp_task_complete_if0),
4635 llvm::Function *TaskFunction,
4655 IfVal = llvm::ConstantInt::getSigned(CGF.
IntTy, 1);
4660 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
4662 cast<VarDecl>(cast<DeclRefExpr>(
D.getLowerBoundVariable())->getDecl());
4667 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
4669 cast<VarDecl>(cast<DeclRefExpr>(
D.getUpperBoundVariable())->getDecl());
4674 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
4676 cast<VarDecl>(cast<DeclRefExpr>(
D.getStrideVariable())->getDecl());
4682 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
4683 if (
Data.Reductions) {
4689 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
4690 llvm::Value *TaskArgs[] = {
4698 llvm::ConstantInt::getSigned(
4700 llvm::ConstantInt::getSigned(
4702 ?
Data.Schedule.getInt() ? NumTasks : Grainsize
4704 Data.Schedule.getPointer()
4707 : llvm::ConstantInt::get(CGF.
Int64Ty, 0),
4710 : llvm::ConstantPointerNull::get(CGF.
VoidPtrTy)};
4729 const Expr *,
const Expr *)> &RedOpGen,
4730 const Expr *XExpr =
nullptr,
const Expr *EExpr =
nullptr,
4731 const Expr *UpExpr =
nullptr) {
4739 llvm::Value *NumElements = CGF.
emitArrayLength(ArrayTy, ElementTy, LHSAddr);
4744 llvm::Value *LHSEnd =
4749 llvm::Value *IsEmpty =
4750 CGF.
Builder.CreateICmpEQ(LHSBegin, LHSEnd,
"omp.arraycpy.isempty");
4751 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
4754 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
4759 llvm::PHINode *RHSElementPHI = CGF.
Builder.CreatePHI(
4760 RHSBegin->getType(), 2,
"omp.arraycpy.srcElementPast");
4761 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
4766 llvm::PHINode *LHSElementPHI = CGF.
Builder.CreatePHI(
4767 LHSBegin->getType(), 2,
"omp.arraycpy.destElementPast");
4768 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
4774 CodeGenFunction::OMPPrivateScope
Scope(CGF);
4775 Scope.addPrivate(LHSVar, LHSElementCurrent);
4776 Scope.addPrivate(RHSVar, RHSElementCurrent);
4778 RedOpGen(CGF, XExpr, EExpr, UpExpr);
4779 Scope.ForceCleanup();
4782 llvm::Value *LHSElementNext = CGF.
Builder.CreateConstGEP1_32(
4784 "omp.arraycpy.dest.element");
4785 llvm::Value *RHSElementNext = CGF.
Builder.CreateConstGEP1_32(
4787 "omp.arraycpy.src.element");
4790 CGF.
Builder.CreateICmpEQ(LHSElementNext, LHSEnd,
"omp.arraycpy.done");
4791 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
4792 LHSElementPHI->addIncoming(LHSElementNext, CGF.
Builder.GetInsertBlock());
4793 RHSElementPHI->addIncoming(RHSElementNext, CGF.
Builder.GetInsertBlock());
4803 const Expr *ReductionOp) {
4804 if (
const auto *CE = dyn_cast<CallExpr>(ReductionOp))
4805 if (
const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
4806 if (
const auto *DRE =
4807 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
4808 if (
const auto *DRD =
4809 dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
4810 std::pair<llvm::Function *, llvm::Function *>
Reduction =
4813 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE,
Func);
4832 Args.push_back(&LHSArg);
4833 Args.push_back(&RHSArg);
4838 llvm::GlobalValue::InternalLinkage, Name,
4841 Fn->setDoesNotRecurse();
4849 ArgsElemType->getPointerTo()),
4853 ArgsElemType->getPointerTo()),
4860 const auto *IPriv = Privates.begin();
4862 for (
unsigned I = 0,
E = ReductionOps.size(); I <
E; ++I, ++IPriv, ++Idx) {
4863 const auto *RHSVar =
4864 cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl());
4866 const auto *LHSVar =
4867 cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl());
4869 QualType PrivTy = (*IPriv)->getType();
4877 const auto *OVE = cast<OpaqueValueExpr>(VLA->
getSizeExpr());
4884 IPriv = Privates.begin();
4885 const auto *ILHS = LHSExprs.begin();
4886 const auto *IRHS = RHSExprs.begin();
4887 for (
const Expr *
E : ReductionOps) {
4888 if ((*IPriv)->getType()->isArrayType()) {
4890 const auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
4891 const auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
4893 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
4895 emitReductionCombiner(CGF, E);
4905 Scope.ForceCleanup();
4911 const Expr *ReductionOp,
4912 const Expr *PrivateRef,
4917 const auto *LHSVar = cast<VarDecl>(LHS->
getDecl());
4918 const auto *RHSVar = cast<VarDecl>(RHS->
getDecl());
4920 CGF, PrivateRef->
getType(), LHSVar, RHSVar,
4922 emitReductionCombiner(CGF, ReductionOp);
4939 bool WithNowait = Options.WithNowait;
4940 bool SimpleReduction = Options.SimpleReduction;
4979 if (SimpleReduction) {
4981 const auto *IPriv = Privates.begin();
4982 const auto *ILHS = LHSExprs.begin();
4983 const auto *IRHS = RHSExprs.begin();
4984 for (
const Expr *
E : ReductionOps) {
4986 cast<DeclRefExpr>(*IRHS));
4996 auto Size = RHSExprs.size();
4997 for (
const Expr *
E : Privates) {
5002 llvm::APInt ArraySize(32, Size);
5003 QualType ReductionArrayTy =
C.getConstantArrayType(
5007 CGF.
CreateMemTemp(ReductionArrayTy,
".omp.reduction.red_list");
5008 const auto *IPriv = Privates.begin();
5010 for (
unsigned I = 0,
E = RHSExprs.size(); I <
E; ++I, ++IPriv, ++Idx) {
5016 if ((*IPriv)->getType()->isVariablyModifiedType()) {
5020 llvm::Value *Size = CGF.
Builder.CreateIntCast(
5033 Privates, LHSExprs, RHSExprs, ReductionOps);
5036 std::string Name =
getName({
"reduction"});
5043 llvm::Value *ReductionArrayTySize = CGF.
getTypeSize(ReductionArrayTy);
5046 llvm::Value *Args[] = {
5049 CGF.
Builder.getInt32(RHSExprs.size()),
5050 ReductionArrayTySize,
5058 WithNowait ? OMPRTL___kmpc_reduce_nowait : OMPRTL___kmpc_reduce),
5062 llvm::BasicBlock *DefaultBB = CGF.
createBasicBlock(
".omp.reduction.default");
5063 llvm::SwitchInst *SwInst =
5064 CGF.
Builder.CreateSwitch(Res, DefaultBB, 2);
5073 SwInst->addCase(CGF.
Builder.getInt32(1), Case1BB);
5077 llvm::Value *EndArgs[] = {
5082 auto &&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps](
5085 const auto *IPriv = Privates.begin();
5086 const auto *ILHS = LHSExprs.begin();
5087 const auto *IRHS = RHSExprs.begin();
5088 for (
const Expr *
E : ReductionOps) {
5090 cast<DeclRefExpr>(*IRHS));
5097 CommonActionTy Action(
5098 nullptr, std::nullopt,
5100 CGM.
getModule(), WithNowait ? OMPRTL___kmpc_end_reduce_nowait
5101 : OMPRTL___kmpc_end_reduce),
5114 SwInst->addCase(CGF.
Builder.getInt32(2), Case2BB);
5117 auto &&AtomicCodeGen = [
Loc, Privates, LHSExprs, RHSExprs, ReductionOps](
5119 const auto *ILHS = LHSExprs.begin();
5120 const auto *IRHS = RHSExprs.begin();
5121 const auto *IPriv = Privates.begin();
5122 for (
const Expr *
E : ReductionOps) {
5123 const Expr *XExpr =
nullptr;
5124 const Expr *EExpr =
nullptr;
5125 const Expr *UpExpr =
nullptr;
5127 if (
const auto *BO = dyn_cast<BinaryOperator>(
E)) {
5128 if (BO->getOpcode() == BO_Assign) {
5129 XExpr = BO->getLHS();
5130 UpExpr = BO->getRHS();
5134 const Expr *RHSExpr = UpExpr;
5137 if (
const auto *ACO = dyn_cast<AbstractConditionalOperator>(
5141 RHSExpr = ACO->getCond();
5143 if (
const auto *BORHS =
5145 EExpr = BORHS->getRHS();
5146 BO = BORHS->getOpcode();
5150 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5151 auto &&AtomicRedGen = [BO, VD,
5153 const Expr *EExpr,
const Expr *UpExpr) {
5154 LValue X = CGF.EmitLValue(XExpr);
5157 E = CGF.EmitAnyExpr(EExpr);
5158 CGF.EmitOMPAtomicSimpleUpdateExpr(
5160 llvm::AtomicOrdering::Monotonic,
Loc,
5161 [&CGF, UpExpr, VD,
Loc](
RValue XRValue) {
5163 Address LHSTemp = CGF.CreateMemTemp(VD->getType());
5164 CGF.emitOMPSimpleStore(
5165 CGF.MakeAddrLValue(LHSTemp, VD->
getType()), XRValue,
5166 VD->getType().getNonReferenceType(),
Loc);
5169 return CGF.EmitAnyExpr(UpExpr);
5172 if ((*IPriv)->getType()->isArrayType()) {
5174 const auto *RHSVar =
5175 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5177 AtomicRedGen, XExpr, EExpr, UpExpr);
5180 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5187 std::string Name = RT.
getName({
"atomic_reduction"});
5196 if ((*IPriv)->getType()->isArrayType()) {
5197 const auto *LHSVar =
5198 cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5199 const auto *RHSVar =
5200 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5204 CritRedGen(CGF,
nullptr,
nullptr,
nullptr);
5215 llvm::Value *EndArgs[] = {
5220 CommonActionTy Action(
nullptr, std::nullopt,
5239 llvm::raw_svector_ostream Out(Buffer);
5243 D = cast<VarDecl>(cast<DeclRefExpr>(Ref)->getDecl());
5247 Out << Prefix << Name <<
"_"
5249 return std::string(Out.str());
5271 Args.emplace_back(&Param);
5272 Args.emplace_back(&ParamOrig);
5273 const auto &FnInfo =
5277 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5280 Fn->setDoesNotRecurse();
5288 llvm::Value *Size =
nullptr;
5331 const Expr *ReductionOp,
5333 const Expr *PrivateRef) {
5335 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(LHS)->getDecl());
5336 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(RHS)->getDecl());
5342 Args.emplace_back(&ParamInOut);
5343 Args.emplace_back(&ParamIn);
5344 const auto &FnInfo =
5348 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5351 Fn->setDoesNotRecurse();
5354 llvm::Value *Size =
nullptr;
5368 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5369 PrivateScope.addPrivate(
5376 C.getPointerType(LHSVD->getType())->castAs<
PointerType>()));
5377 PrivateScope.addPrivate(
5383 C.getPointerType(RHSVD->getType())->castAs<
PointerType>()));
5384 PrivateScope.Privatize();
5389 CGF, ReductionOp, PrivateRef, cast<DeclRefExpr>(LHS),
5390 cast<DeclRefExpr>(RHS));
5412 Args.emplace_back(&Param);
5413 const auto &FnInfo =
5417 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5420 Fn->setDoesNotRecurse();
5425 llvm::Value *Size =
nullptr;
5460 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_taskred_input_t");
5469 C, RD,
C.getIntTypeForBitwidth(32,
false));
5472 unsigned Size =
Data.ReductionVars.size();
5473 llvm::APInt ArraySize(64, Size);
5475 C.getConstantArrayType(RDType, ArraySize,
nullptr,
5480 Data.ReductionCopies,
Data.ReductionOps);
5481 for (
unsigned Cnt = 0; Cnt < Size; ++Cnt) {
5483 llvm::Value *Idxs[] = {llvm::ConstantInt::get(
CGM.
SizeTy, 0),
5484 llvm::ConstantInt::get(
CGM.
SizeTy, Cnt)};
5500 llvm::Value *SizeValInChars;
5501 llvm::Value *SizeVal;
5502 std::tie(SizeValInChars, SizeVal) = RCG.
getSizes(Cnt);
5508 bool DelayedCreation = !!SizeVal;
5520 llvm::Value *FiniAddr =
5521 Fini ? Fini : llvm::ConstantPointerNull::get(
CGM.
VoidPtrTy);
5526 CGM,
Loc, RCG, Cnt,
Data.ReductionOps[Cnt], LHSExprs[Cnt],
5527 RHSExprs[Cnt],
Data.ReductionCopies[Cnt]);
5531 if (DelayedCreation) {
5533 llvm::ConstantInt::get(
CGM.
Int32Ty, 1,
true),
5538 if (
Data.IsReductionWithTaskMod) {
5544 llvm::Value *Args[] = {
5546 llvm::ConstantInt::get(
CGM.
IntTy,
Data.IsWorksharingReduction ? 1 : 0,
5548 llvm::ConstantInt::get(
CGM.
IntTy, Size,
true),
5557 llvm::Value *Args[] = {
5560 llvm::ConstantInt::get(
CGM.
IntTy, Size,
true),
5570 bool IsWorksharingReduction) {
5576 llvm::Value *Args[] = {IdentTLoc, GTid,
5578 IsWorksharingReduction ? 1 : 0,
5582 CGM.
getModule(), OMPRTL___kmpc_task_reduction_modifier_fini),
5605 llvm::Value *ReductionsPtr,
5618 CGM.
getModule(), OMPRTL___kmpc_task_reduction_get_th_data),
5636 llvm::Value *NumOfElements;
5637 std::tie(NumOfElements, DependenciesArray) =
5639 if (!
Data.Dependences.empty()) {
5640 llvm::Value *DepWaitTaskArgs[7];
5641 DepWaitTaskArgs[0] = UpLoc;
5642 DepWaitTaskArgs[1] = ThreadID;
5643 DepWaitTaskArgs[2] = NumOfElements;
5645 DepWaitTaskArgs[4] = CGF.
Builder.getInt32(0);
5646 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
5647 DepWaitTaskArgs[6] =
5648 llvm::ConstantInt::get(CGF.
Int32Ty,
Data.HasNowaitClause);
5657 M, OMPRTL___kmpc_omp_taskwait_deps_51),
5664 llvm::Value *Args[] = {UpLoc, ThreadID};
5667 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_omp_taskwait),
5672 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
5673 Region->emitUntiedSwitch(CGF);
5682 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel,
5683 InnerKind != OMPD_critical &&
5684 InnerKind != OMPD_master &&
5685 InnerKind != OMPD_masked);
5700 RTCancelKind CancelKind = CancelNoreq;
5701 if (CancelRegion == OMPD_parallel)
5702 CancelKind = CancelParallel;
5703 else if (CancelRegion == OMPD_for)
5704 CancelKind = CancelLoop;
5705 else if (CancelRegion == OMPD_sections)
5706 CancelKind = CancelSections;
5708 assert(CancelRegion == OMPD_taskgroup);
5709 CancelKind = CancelTaskgroup;
5721 if (
auto *OMPRegionInfo =
5725 if (CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) {
5726 llvm::Value *Args[] = {
5741 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
5743 if (CancelRegion == OMPD_parallel)
5762 if (
auto *OMPRegionInfo =
5764 auto &&ThenGen = [
this, &M,
Loc, CancelRegion,
5767 llvm::Value *Args[] = {
5771 llvm::Value *
Result = CGF.EmitRuntimeCall(
5772 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_cancel), Args);
5777 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
".cancel.exit");
5778 llvm::BasicBlock *ContBB = CGF.createBasicBlock(
".cancel.continue");
5779 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(
Result);
5780 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
5781 CGF.EmitBlock(ExitBB);
5782 if (CancelRegion == OMPD_parallel)
5786 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
5787 CGF.EmitBranchThroughCleanup(CancelDest);
5788 CGF.EmitBlock(ContBB,
true);
5806 OMPUsesAllocatorsActionTy(
5807 ArrayRef<std::pair<const Expr *, const Expr *>> Allocators)
5808 : Allocators(Allocators) {}
5812 for (
const auto &AllocatorData : Allocators) {
5814 CGF, AllocatorData.first, AllocatorData.second);
5820 for (
const auto &AllocatorData : Allocators) {
5822 AllocatorData.first);
5830 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
5832 assert(!ParentName.empty() &&
"Invalid target entry parent name!");
5836 for (
unsigned I = 0,
E =
C->getNumberOfAllocators(); I <
E; ++I) {
5838 if (!
D.AllocatorTraits)
5840 Allocators.emplace_back(
D.Allocator,
D.AllocatorTraits);
5843 OMPUsesAllocatorsActionTy UsesAllocatorAction(Allocators);
5846 IsOffloadEntry, CodeGen);
5850 const Expr *Allocator,
5851 const Expr *AllocatorTraits) {
5852 llvm::Value *ThreadId =
getThreadID(CGF, Allocator->getExprLoc());
5853 ThreadId = CGF.
Builder.CreateIntCast(ThreadId, CGF.
IntTy,
true);
5855 llvm::Value *MemSpaceHandle = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
5856 llvm::Value *NumTraits = llvm::ConstantInt::get(
5857 CGF.
IntTy, cast<ConstantArrayType>(
5860 .getLimitedValue());
5869 llvm::Value *AllocatorVal =
5872 {ThreadId, MemSpaceHandle, NumTraits, Traits});
5875 cast<DeclRefExpr>(Allocator->IgnoreParenImpCasts())->getDecl()));
5879 Allocator->getType(), Allocator->getExprLoc());
5884 const Expr *Allocator) {
5885 llvm::Value *ThreadId =
getThreadID(CGF, Allocator->getExprLoc());
5886 ThreadId = CGF.
Builder.CreateIntCast(ThreadId, CGF.
IntTy,
true);
5888 llvm::Value *AllocatorVal =
5892 Allocator->getExprLoc());
5895 OMPRTL___kmpc_destroy_allocator),
5896 {ThreadId, AllocatorVal});
5901 int32_t &MinThreadsVal, int32_t &MaxThreadsVal, int32_t &MinTeamsVal,
5902 int32_t &MaxTeamsVal) {
5909 for (
auto *A :
C->getAttrs()) {
5910 int32_t AttrMinThreadsVal = 1, AttrMaxThreadsVal = -1;
5911 int32_t AttrMinBlocksVal = 1, AttrMaxBlocksVal = -1;
5912 if (
auto *
Attr = dyn_cast<CUDALaunchBoundsAttr>(A))
5914 &AttrMinBlocksVal, &AttrMaxBlocksVal);
5915 else if (
auto *
Attr = dyn_cast<AMDGPUFlatWorkGroupSizeAttr>(A))
5917 nullptr,
Attr,
nullptr, &AttrMinThreadsVal,
5918 &AttrMaxThreadsVal);
5922 MinThreadsVal = std::max(MinThreadsVal, AttrMinThreadsVal);
5923 if (AttrMaxThreadsVal > 0)
5924 MaxThreadsVal = MaxThreadsVal > 0
5925 ? std::min(MaxThreadsVal, AttrMaxThreadsVal)
5926 : AttrMaxThreadsVal;
5927 MinTeamsVal = std::max(MinTeamsVal, AttrMinBlocksVal);
5928 if (AttrMaxBlocksVal > 0)
5929 MaxTeamsVal = MaxTeamsVal > 0 ? std::min(MaxTeamsVal, AttrMaxBlocksVal)
5937 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
5940 llvm::TargetRegionEntryInfo EntryInfo =
5944 llvm::OpenMPIRBuilder::FunctionGenCallback &&GenerateOutlinedFunction =
5945 [&CGF, &
D, &CodeGen](StringRef EntryFnName) {
5948 CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName);
5953 OMPBuilder.emitTargetRegionFunction(EntryInfo, GenerateOutlinedFunction,
5954 IsOffloadEntry, OutlinedFn, OutlinedFnID);
5962 for (
auto *A :
C->getAttrs()) {
5963 if (
auto *
Attr = dyn_cast<AMDGPUWavesPerEUAttr>(A))
5982 while (
const auto *
C = dyn_cast_or_null<CompoundStmt>(Child)) {
5984 for (
const Stmt *S :
C->body()) {
5985 if (
const auto *
E = dyn_cast<Expr>(S)) {
5990 if (isa<AsmStmt>(S) || isa<NullStmt>(S) || isa<OMPFlushDirective>(S) ||
5991 isa<OMPBarrierDirective>(S) || isa<OMPTaskyieldDirective>(S))
5994 if (
const auto *DS = dyn_cast<DeclStmt>(S)) {
5995 if (llvm::all_of(DS->decls(), [](
const Decl *
D) {
5996 if (isa<EmptyDecl>(D) || isa<DeclContext>(D) ||
5997 isa<TypeDecl>(D) || isa<PragmaCommentDecl>(D) ||
5998 isa<PragmaDetectMismatchDecl>(D) || isa<UsingDecl>(D) ||
5999 isa<UsingDirectiveDecl>(D) ||
6000 isa<OMPDeclareReductionDecl>(D) ||
6001 isa<OMPThreadPrivateDecl>(D) || isa<OMPAllocateDecl>(D))
6003 const auto *VD = dyn_cast<VarDecl>(D);
6006 return VD->hasGlobalStorage() || !VD->isUsed();
6016 Child = Child->IgnoreContainers();
6023 int32_t &MaxTeamsVal) {
6027 "Expected target-based executable directive.");
6028 switch (DirectiveKind) {
6030 const auto *CS =
D.getInnermostCapturedStmt();
6033 const Stmt *ChildStmt =
6035 if (
const auto *NestedDir =
6036 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
6039 const Expr *NumTeams =
6044 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();
6047 MinTeamsVal = MaxTeamsVal = 0;
6050 MinTeamsVal = MaxTeamsVal = 1;
6054 MinTeamsVal = MaxTeamsVal = -1;
6057 case OMPD_target_teams_loop:
6058 case OMPD_target_teams:
6059 case OMPD_target_teams_distribute:
6060 case OMPD_target_teams_distribute_simd:
6061 case OMPD_target_teams_distribute_parallel_for:
6062 case OMPD_target_teams_distribute_parallel_for_simd: {
6064 const Expr *NumTeams =
6068 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();
6071 MinTeamsVal = MaxTeamsVal = 0;
6074 case OMPD_target_parallel:
6075 case OMPD_target_parallel_for:
6076 case OMPD_target_parallel_for_simd:
6077 case OMPD_target_parallel_loop:
6078 case OMPD_target_simd:
6079 MinTeamsVal = MaxTeamsVal = 1;
6083 case OMPD_parallel_for:
6084 case OMPD_parallel_loop:
6085 case OMPD_parallel_master:
6086 case OMPD_parallel_sections:
6088 case OMPD_parallel_for_simd:
6090 case OMPD_cancellation_point:
6092 case OMPD_threadprivate:
6103 case OMPD_taskyield:
6106 case OMPD_taskgroup:
6112 case OMPD_target_data:
6113 case OMPD_target_exit_data:
6114 case OMPD_target_enter_data:
6115 case OMPD_distribute:
6116 case OMPD_distribute_simd:
6117 case OMPD_distribute_parallel_for:
6118 case OMPD_distribute_parallel_for_simd:
6119 case OMPD_teams_distribute:
6120 case OMPD_teams_distribute_simd:
6121 case OMPD_teams_distribute_parallel_for:
6122 case OMPD_teams_distribute_parallel_for_simd:
6123 case OMPD_target_update:
6124 case OMPD_declare_simd:
6125 case OMPD_declare_variant:
6126 case OMPD_begin_declare_variant:
6127 case OMPD_end_declare_variant:
6128 case OMPD_declare_target:
6129 case OMPD_end_declare_target:
6130 case OMPD_declare_reduction:
6131 case OMPD_declare_mapper:
6133 case OMPD_taskloop_simd:
6134 case OMPD_master_taskloop:
6135 case OMPD_master_taskloop_simd:
6136 case OMPD_parallel_master_taskloop:
6137 case OMPD_parallel_master_taskloop_simd:
6139 case OMPD_metadirective:
6145 llvm_unreachable(
"Unexpected directive kind.");
6151 "Clauses associated with the teams directive expected to be emitted "
6152 "only for the host!");
6154 int32_t MinNT = -1, MaxNT = -1;
6155 const Expr *NumTeams =
6157 if (NumTeams !=
nullptr) {
6160 switch (DirectiveKind) {
6162 const auto *CS =
D.getInnermostCapturedStmt();
6163 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6167 return Bld.CreateIntCast(NumTeamsVal, CGF.
Int32Ty,
6170 case OMPD_target_teams:
6171 case OMPD_target_teams_distribute:
6172 case OMPD_target_teams_distribute_simd:
6173 case OMPD_target_teams_distribute_parallel_for:
6174 case OMPD_target_teams_distribute_parallel_for_simd: {
6178 return Bld.CreateIntCast(NumTeamsVal, CGF.
Int32Ty,
6186 assert(MinNT == MaxNT &&
"Num threads ranges require handling here.");
6187 return llvm::ConstantInt::get(CGF.
Int32Ty, MinNT);
6195 const Expr **
E, int32_t &UpperBound,
6196 bool UpperBoundOnly, llvm::Value **CondVal) {
6199 const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child);
6206 if (CondVal && Dir->hasClausesOfKind<
OMPIfClause>()) {
6207 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6208 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6210 for (
const auto *
C : Dir->getClausesOfKind<
OMPIfClause>()) {
6211 if (
C->getNameModifier() == OMPD_unknown ||
6212 C->getNameModifier() == OMPD_parallel) {
6227 if (
const auto *PreInit =
6229 for (
const auto *I : PreInit->decls()) {
6230 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6233 CodeGenFunction::AutoVarEmission Emission =
6246 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6247 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6248 const auto *NumThreadsClause =
6250 const Expr *NTExpr = NumThreadsClause->getNumThreads();
6251 if (NTExpr->isIntegerConstantExpr(CGF.
getContext()))
6252 if (
auto Constant = NTExpr->getIntegerConstantExpr(CGF.
getContext()))
6255 ? Constant->getZExtValue()
6256 : std::min(UpperBound,
6257 static_cast<int32_t
>(Constant->getZExtValue()));
6260 if (UpperBound == -1)
6264 CodeGenFunction::LexicalScope
Scope(CGF, NTExpr->getSourceRange());
6265 if (
const auto *PreInit =
6266 cast_or_null<DeclStmt>(NumThreadsClause->getPreInitStmt())) {
6267 for (
const auto *I : PreInit->decls()) {
6268 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6271 CodeGenFunction::AutoVarEmission Emission =
6287 bool UpperBoundOnly, llvm::Value **CondVal,
const Expr **ThreadLimitExpr) {
6288 assert((!CGF.
getLangOpts().OpenMPIsTargetDevice || UpperBoundOnly) &&
6289 "Clauses associated with the teams directive expected to be emitted "
6290 "only for the host!");
6293 "Expected target-based executable directive.");
6295 const Expr *NT =
nullptr;
6296 const Expr **NTPtr = UpperBoundOnly ? nullptr : &NT;
6298 auto CheckForConstExpr = [&](
const Expr *
E,
const Expr **EPtr) {
6301 UpperBound = UpperBound ? Constant->getZExtValue()
6302 : std::min(UpperBound,
6303 int32_t(Constant->getZExtValue()));
6307 if (UpperBound == -1)
6313 auto ReturnSequential = [&]() {
6318 switch (DirectiveKind) {
6321 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6327 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6329 ThreadLimitClause = TLC;
6330 if (ThreadLimitExpr) {
6331 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6334 CGF, ThreadLimitClause->getThreadLimit()->getSourceRange());
6335 if (
const auto *PreInit =
6336 cast_or_null<DeclStmt>(ThreadLimitClause->getPreInitStmt())) {
6337 for (
const auto *I : PreInit->decls()) {
6338 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6350 if (ThreadLimitClause)
6351 CheckForConstExpr(ThreadLimitClause->getThreadLimit(), ThreadLimitExpr);
6352 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6355 CS = Dir->getInnermostCapturedStmt();
6358 Dir = dyn_cast_or_null<OMPExecutableDirective>(Child);
6361 CS = Dir->getInnermostCapturedStmt();
6362 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6364 return ReturnSequential();
6368 case OMPD_target_teams: {
6372 CheckForConstExpr(ThreadLimitClause->getThreadLimit(), ThreadLimitExpr);
6375 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6378 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6379 if (Dir->getDirectiveKind() == OMPD_distribute) {
6380 CS = Dir->getInnermostCapturedStmt();
6381 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6386 case OMPD_target_teams_distribute:
6390 CheckForConstExpr(ThreadLimitClause->getThreadLimit(), ThreadLimitExpr);
6392 getNumThreads(CGF,
D.getInnermostCapturedStmt(), NTPtr, UpperBound,
6393 UpperBoundOnly, CondVal);
6395 case OMPD_target_teams_loop:
6396 case OMPD_target_parallel_loop:
6397 case OMPD_target_parallel:
6398 case OMPD_target_parallel_for:
6399 case OMPD_target_parallel_for_simd:
6400 case OMPD_target_teams_distribute_parallel_for:
6401 case OMPD_target_teams_distribute_parallel_for_simd: {
6405 if (
C->getNameModifier() == OMPD_unknown ||
6406 C->getNameModifier() == OMPD_parallel) {
6416 return ReturnSequential();
6426 CheckForConstExpr(ThreadLimitClause->getThreadLimit(), ThreadLimitExpr);
6431 CheckForConstExpr(NumThreadsClause->getNumThreads(),
nullptr);
6432 return NumThreadsClause->getNumThreads();
6436 case OMPD_target_teams_distribute_simd:
6437 case OMPD_target_simd:
6438 return ReturnSequential();
6442 llvm_unreachable(
"Unsupported directive kind.");
6447 llvm::Value *NumThreadsVal =
nullptr;
6448 llvm::Value *CondVal =
nullptr;
6449 llvm::Value *ThreadLimitVal =
nullptr;
6450 const Expr *ThreadLimitExpr =
nullptr;
6451 int32_t UpperBound = -1;
6454 CGF,
D, UpperBound,
false, &CondVal,
6458 if (ThreadLimitExpr) {
6461 ThreadLimitVal = CGF.
Builder.CreateIntCast(ThreadLimitVal, CGF.
Int32Ty,
6466 if (UpperBound == 1) {
6467 NumThreadsVal = CGF.
Builder.getInt32(UpperBound);
6470 NumThreadsVal = CGF.
Builder.CreateIntCast(NumThreadsVal, CGF.
Int32Ty,
6472 }
else if (ThreadLimitVal) {
6475 NumThreadsVal = ThreadLimitVal;
6476 ThreadLimitVal =
nullptr;
6479 assert(!ThreadLimitVal &&
"Default not applicable with thread limit value");
6480 NumThreadsVal = CGF.
Builder.getInt32(0);
6487 NumThreadsVal = CGF.
Builder.CreateSelect(CondVal, NumThreadsVal,
6493 if (ThreadLimitVal) {
6494 NumThreadsVal = CGF.
Builder.CreateSelect(
6495 CGF.
Builder.CreateICmpULT(ThreadLimitVal, NumThreadsVal),
6496 ThreadLimitVal, NumThreadsVal);
6499 return NumThreadsVal;
6509class MappableExprsHandler {
6512 static unsigned getFlagMemberOffset() {
6513 unsigned Offset = 0;
6514 for (uint64_t Remain =
6515 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
6516 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
6517 !(Remain & 1); Remain = Remain >> 1)
6524 class MappingExprInfo {
6529 const Expr *MapExpr =
nullptr;
6532 MappingExprInfo(
const ValueDecl *MapDecl,
const Expr *MapExpr =
nullptr)
6533 : MapDecl(MapDecl), MapExpr(MapExpr) {}
6535 const ValueDecl *getMapDecl()
const {
return MapDecl; }
6536 const Expr *getMapExpr()
const {
return MapExpr; }
6539 using DeviceInfoTy = llvm::OpenMPIRBuilder::DeviceInfoTy;
6540 using MapBaseValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
6541 using MapValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
6542 using MapFlagsArrayTy = llvm::OpenMPIRBuilder::MapFlagsArrayTy;
6543 using MapDimArrayTy = llvm::OpenMPIRBuilder::MapDimArrayTy;
6544 using MapNonContiguousArrayTy =
6545 llvm::OpenMPIRBuilder::MapNonContiguousArrayTy;
6552 struct MapCombinedInfoTy : llvm::OpenMPIRBuilder::MapInfosTy {
6553 MapExprsArrayTy Exprs;
6554 MapValueDeclsArrayTy Mappers;
6555 MapValueDeclsArrayTy DevicePtrDecls;
6558 void append(MapCombinedInfoTy &CurInfo) {
6559 Exprs.append(CurInfo.Exprs.begin(), CurInfo.Exprs.end());
6560 DevicePtrDecls.append(CurInfo.DevicePtrDecls.begin(),
6561 CurInfo.DevicePtrDecls.end());
6562 Mappers.append(CurInfo.Mappers.begin(), CurInfo.Mappers.end());
6563 llvm::OpenMPIRBuilder::MapInfosTy::append(CurInfo);
6571 struct StructRangeInfoTy {
6572 MapCombinedInfoTy PreliminaryMapData;
6573 std::pair<
unsigned ,
Address > LowestElem = {
6575 std::pair<
unsigned ,
Address > HighestElem = {
6579 bool IsArraySection =
false;
6580 bool HasCompleteRecord =
false;
6590 bool ReturnDevicePointer =
false;
6591 bool IsImplicit =
false;
6593 const Expr *VarRef =
nullptr;
6594 bool ForDeviceAddr =
false;
6596 MapInfo() =
default;
6602 bool ReturnDevicePointer,
bool IsImplicit,
6603 const ValueDecl *Mapper =
nullptr,
const Expr *VarRef =
nullptr,
6604 bool ForDeviceAddr =
false)
6605 : Components(Components), MapType(MapType), MapModifiers(MapModifiers),
6606 MotionModifiers(MotionModifiers),
6607 ReturnDevicePointer(ReturnDevicePointer), IsImplicit(IsImplicit),
6608 Mapper(Mapper), VarRef(VarRef), ForDeviceAddr(ForDeviceAddr) {}
6614 struct DeferredDevicePtrEntryTy {
6615 const Expr *IE =
nullptr;
6617 bool ForDeviceAddr =
false;
6619 DeferredDevicePtrEntryTy(
const Expr *IE,
const ValueDecl *VD,
6621 : IE(IE), VD(VD), ForDeviceAddr(ForDeviceAddr) {}
6636 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>,
bool> FirstPrivateDecls;
6653 llvm::DenseMap<const ValueDecl *, const OMPMapClause *> LambdasMap;
6655 llvm::Value *getExprTypeSize(
const Expr *
E)
const {
6659 if (
const auto *OAE = dyn_cast<OMPArrayShapingExpr>(
E)) {
6661 CGF.
getTypeSize(OAE->getBase()->getType()->getPointeeType());
6662 for (
const Expr *SE : OAE->getDimensions()) {
6679 if (
const auto *OAE = dyn_cast<ArraySectionExpr>(
E)) {
6681 OAE->getBase()->IgnoreParenImpCasts())
6687 if (!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
6688 !OAE->getLowerBound())
6691 llvm::Value *ElemSize;
6693 ElemSize = CGF.
getTypeSize(PTy->getPointeeType().getCanonicalType());
6695 const auto *ATy = cast<ArrayType>(BaseTy.
getTypePtr());
6696 assert(ATy &&
"Expecting array type if not a pointer type.");
6697 ElemSize = CGF.
getTypeSize(ATy->getElementType().getCanonicalType());
6702 if (!OAE->getLength() && OAE->getColonLocFirst().isInvalid())
6705 if (
const Expr *LenExpr = OAE->getLength()) {
6709 LenExpr->getExprLoc());
6710 return CGF.
Builder.CreateNUWMul(LengthVal, ElemSize);
6712 assert(!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
6713 OAE->getLowerBound() &&
"expected array_section[lb:].");
6719 OAE->getLowerBound()->getExprLoc());
6720 LBVal = CGF.
Builder.CreateNUWMul(LBVal, ElemSize);
6721 llvm::Value *Cmp = CGF.
Builder.CreateICmpUGT(LengthVal, LBVal);
6722 llvm::Value *TrueVal = CGF.
Builder.CreateNUWSub(LengthVal, LBVal);
6723 LengthVal = CGF.
Builder.CreateSelect(
6724 Cmp, TrueVal, llvm::ConstantInt::get(CGF.
SizeTy, 0));
6734 OpenMPOffloadMappingFlags getMapTypeBits(
6737 bool AddPtrFlag,
bool AddIsTargetParamFlag,
bool IsNonContiguous)
const {
6738 OpenMPOffloadMappingFlags Bits =
6739 IsImplicit ? OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT
6740 : OpenMPOffloadMappingFlags::OMP_MAP_NONE;
6742 case OMPC_MAP_alloc:
6743 case OMPC_MAP_release:
6750 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO;
6753 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_FROM;
6755 case OMPC_MAP_tofrom:
6756 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO |
6757 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
6759 case OMPC_MAP_delete:
6760 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_DELETE;
6763 llvm_unreachable(
"Unexpected map type!");
6766 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
6767 if (AddIsTargetParamFlag)
6768 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
6769 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_always))
6770 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS;
6771 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_close))
6772 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_CLOSE;
6773 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_present) ||
6774 llvm::is_contained(MotionModifiers, OMPC_MOTION_MODIFIER_present))
6775 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
6776 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_ompx_hold))
6777 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
6778 if (IsNonContiguous)
6779 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG;
6785 bool isFinalArraySectionExpression(
const Expr *
E)
const {
6786 const auto *OASE = dyn_cast<ArraySectionExpr>(
E);
6793 if (OASE->getColonLocFirst().isInvalid())
6796 const Expr *Length = OASE->getLength();
6803 OASE->getBase()->IgnoreParenImpCasts())
6805 if (
const auto *ATy = dyn_cast<ConstantArrayType>(BaseQTy.
getTypePtr()))
6806 return ATy->getSExtSize() != 1;
6815 if (!Length->EvaluateAsInt(Result, CGF.
getContext()))
6818 llvm::APSInt ConstLength = Result.Val.getInt();
6819 return ConstLength.getSExtValue() != 1;
6827 void generateInfoForComponentList(
6831 MapCombinedInfoTy &CombinedInfo,
6832 MapCombinedInfoTy &StructBaseCombinedInfo,
6833 StructRangeInfoTy &PartialStruct,
bool IsFirstComponentList,
6834 bool IsImplicit,
bool GenerateAllInfoForClauses,
6835 const ValueDecl *Mapper =
nullptr,
bool ForDeviceAddr =
false,
6836 const ValueDecl *BaseDecl =
nullptr,
const Expr *MapExpr =
nullptr,
6838 OverlappedElements = std::nullopt,
6839 bool AreBothBasePtrAndPteeMapped =
false)
const {
7021 bool IsCaptureFirstInfo = IsFirstComponentList;
7025 bool RequiresReference =
false;
7028 auto CI = Components.rbegin();
7029 auto CE = Components.rend();
7034 bool IsExpressionFirstInfo =
true;
7035 bool FirstPointerInComplexData =
false;
7037 const Expr *AssocExpr = I->getAssociatedExpression();
7038 const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr);
7039 const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr);
7040 const auto *OAShE = dyn_cast<OMPArrayShapingExpr>(AssocExpr);
7042 if (AreBothBasePtrAndPteeMapped && std::next(I) == CE)
7044 if (isa<MemberExpr>(AssocExpr)) {
7048 }
else if ((AE && isa<CXXThisExpr>(AE->getBase()->IgnoreParenImpCasts())) ||
7050 isa<CXXThisExpr>(OASE->getBase()->IgnoreParenImpCasts()))) {
7053 isa<CXXThisExpr>(OAShE->getBase()->IgnoreParenCasts())) {
7062 if (
const auto *VD =
7063 dyn_cast_or_null<VarDecl>(I->getAssociatedDeclaration())) {
7064 if (std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
7065 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) {
7066 if ((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
7067 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
7068 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
7070 RequiresReference =
true;
7080 I->getAssociatedDeclaration()->getType().getNonReferenceType();
7085 const auto *VD = dyn_cast<VarDecl>(I->getAssociatedDeclaration());
7086 if (!AreBothBasePtrAndPteeMapped &&
7091 FirstPointerInComplexData =
true;
7110 bool ShouldBeMemberOf =
false;
7125 bool IsNonContiguous = CombinedInfo.NonContigInfo.IsNonContiguous;
7126 bool IsPrevMemberReference =
false;
7133 bool IsMappingWholeStruct =
true;
7134 if (!GenerateAllInfoForClauses) {
7135 IsMappingWholeStruct =
false;
7137 for (
auto TempI = I; TempI != CE; ++TempI) {
7139 dyn_cast<MemberExpr>(TempI->getAssociatedExpression());
7141 IsMappingWholeStruct =
false;
7147 for (; I != CE; ++I) {
7149 if (!EncounteredME) {
7150 EncounteredME = dyn_cast<MemberExpr>(I->getAssociatedExpression());
7153 if (EncounteredME) {
7154 ShouldBeMemberOf =
true;
7157 if (FirstPointerInComplexData) {
7159 ->getAssociatedDeclaration()
7161 .getNonReferenceType();
7163 FirstPointerInComplexData =
false;
7168 auto Next = std::next(I);
7178 bool IsFinalArraySection =
7180 isFinalArraySectionExpression(I->getAssociatedExpression());
7184 const ValueDecl *MapDecl = (I->getAssociatedDeclaration())
7185 ? I->getAssociatedDeclaration()
7187 MapExpr = (I->getAssociatedExpression()) ? I->getAssociatedExpression()
7194 dyn_cast<ArraySectionExpr>(I->getAssociatedExpression());
7196 dyn_cast<OMPArrayShapingExpr>(I->getAssociatedExpression());
7197 const auto *UO = dyn_cast<UnaryOperator>(I->getAssociatedExpression());
7198 const auto *BO = dyn_cast<BinaryOperator>(I->getAssociatedExpression());
7204 I->getAssociatedExpression()->getType()->isAnyPointerType();
7205 bool IsMemberReference = isa<MemberExpr>(I->getAssociatedExpression()) &&
7208 bool IsNonDerefPointer = IsPointer &&
7209 !(UO && UO->getOpcode() != UO_Deref) && !BO &&
7215 if (Next == CE || IsMemberReference || IsNonDerefPointer ||
7216 IsFinalArraySection) {
7219 assert((Next == CE ||
7220 isa<MemberExpr>(Next->getAssociatedExpression()) ||
7221 isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) ||
7222 isa<ArraySectionExpr>(Next->getAssociatedExpression()) ||
7223 isa<OMPArrayShapingExpr>(Next->getAssociatedExpression()) ||
7224 isa<UnaryOperator>(Next->getAssociatedExpression()) ||
7225 isa<BinaryOperator>(Next->getAssociatedExpression())) &&
7226 "Unexpected expression");
7232 const Expr *BaseExpr =
E->getBase();
7252 OAShE->getBase()->getType()->getPointeeType()),
7254 OAShE->getBase()->getType()));
7255 }
else if (IsMemberReference) {
7256 const auto *ME = cast<MemberExpr>(I->getAssociatedExpression());
7257 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
7259 BaseLVal, cast<FieldDecl>(MapDecl))
7272 bool IsMemberPointerOrAddr =
7274 (((IsPointer || ForDeviceAddr) &&
7275 I->getAssociatedExpression() == EncounteredME) ||
7276 (IsPrevMemberReference && !IsPointer) ||
7277 (IsMemberReference && Next != CE &&
7278 !Next->getAssociatedExpression()->getType()->isPointerType()));
7279 if (!OverlappedElements.empty() && Next == CE) {
7281 assert(!PartialStruct.Base.isValid() &&
"The base element is set.");
7282 assert(!IsPointer &&
7283 "Unexpected base element with the pointer type.");
7286 PartialStruct.LowestElem = {0, LowestElem};
7288 I->getAssociatedExpression()->getType());
7293 PartialStruct.HighestElem = {
7294 std::numeric_limits<
decltype(
7295 PartialStruct.HighestElem.first)>
::max(),
7297 PartialStruct.Base = BP;
7298 PartialStruct.LB = LB;
7300 PartialStruct.PreliminaryMapData.BasePointers.empty() &&
7301 "Overlapped elements must be used only once for the variable.");
7302 std::swap(PartialStruct.PreliminaryMapData, CombinedInfo);
7304 OpenMPOffloadMappingFlags Flags =
7305 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
7306 getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,
7308 false, IsNonContiguous);
7309 llvm::Value *
Size =
nullptr;
7312 Component : OverlappedElements) {
7316 if (
const ValueDecl *VD = MC.getAssociatedDeclaration()) {
7317 const auto *FD = dyn_cast<FieldDecl>(VD);
7318 if (FD && FD->getType()->isLValueReferenceType()) {
7320 cast<MemberExpr>(MC.getAssociatedExpression());
7321 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
7337 assert(Size &&
"Failed to determine structure size");
7338 CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
7340 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
7341 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
7343 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
7345 CombinedInfo.Types.push_back(Flags);
7346 CombinedInfo.Mappers.push_back(
nullptr);
7347 CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize
7351 CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
7353 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
7354 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
7360 CombinedInfo.Sizes.push_back(
7362 CombinedInfo.Types.push_back(Flags);
7363 CombinedInfo.Mappers.push_back(
nullptr);
7364 CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize
7368 llvm::Value *
Size = getExprTypeSize(I->getAssociatedExpression());
7373 if (!IsMemberPointerOrAddr ||
7375 if (!IsMappingWholeStruct) {
7376 CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
7378 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
7379 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
7381 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
7383 CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize
7386 StructBaseCombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
7387 StructBaseCombinedInfo.BasePointers.push_back(
7389 StructBaseCombinedInfo.DevicePtrDecls.push_back(
nullptr);
7390 StructBaseCombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
7391 StructBaseCombinedInfo.Pointers.push_back(LB.
emitRawPointer(CGF));
7392 StructBaseCombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
7394 StructBaseCombinedInfo.NonContigInfo.Dims.push_back(
7395 IsNonContiguous ? DimSize : 1);
7399 bool HasMapper = Mapper && Next == CE;
7400 if (!IsMappingWholeStruct)
7401 CombinedInfo.Mappers.push_back(HasMapper ? Mapper :
nullptr);
7403 StructBaseCombinedInfo.Mappers.push_back(HasMapper ? Mapper
7410 OpenMPOffloadMappingFlags Flags =
7411 getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,
7412 !IsExpressionFirstInfo || RequiresReference ||
7413 FirstPointerInComplexData || IsMemberReference,
7414 AreBothBasePtrAndPteeMapped ||
7415 (IsCaptureFirstInfo && !RequiresReference),
7418 if (!IsExpressionFirstInfo || IsMemberReference) {
7421 if (IsPointer || (IsMemberReference && Next != CE))
7422 Flags &= ~(OpenMPOffloadMappingFlags::OMP_MAP_TO |
7423 OpenMPOffloadMappingFlags::OMP_MAP_FROM |
7424 OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS |
7425 OpenMPOffloadMappingFlags::OMP_MAP_DELETE |
7426 OpenMPOffloadMappingFlags::OMP_MAP_CLOSE);
7428 if (ShouldBeMemberOf) {
7431 Flags |= OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;
7434 ShouldBeMemberOf =
false;
7438 if (!IsMappingWholeStruct)
7439 CombinedInfo.Types.push_back(Flags);
7441 StructBaseCombinedInfo.Types.push_back(Flags);
7447 if (EncounteredME) {
7448 const auto *FD = cast<FieldDecl>(EncounteredME->
getMemberDecl());
7449 unsigned FieldIndex = FD->getFieldIndex();
7452 if (!PartialStruct.Base.isValid()) {
7453 PartialStruct.LowestElem = {FieldIndex, LowestElem};
7454 if (IsFinalArraySection) {
7458 PartialStruct.HighestElem = {FieldIndex, HB};
7460 PartialStruct.HighestElem = {FieldIndex, LowestElem};
7462 PartialStruct.Base = BP;
7463 PartialStruct.LB = BP;
7464 }
else if (FieldIndex < PartialStruct.LowestElem.first) {
7465 PartialStruct.LowestElem = {FieldIndex, LowestElem};
7466 }
else if (FieldIndex > PartialStruct.HighestElem.first) {
7467 if (IsFinalArraySection) {
7471 PartialStruct.HighestElem = {FieldIndex, HB};
7473 PartialStruct.HighestElem = {FieldIndex, LowestElem};
7479 if (IsFinalArraySection || IsNonContiguous)
7480 PartialStruct.IsArraySection =
true;
7483 if (IsFinalArraySection)
7488 BP = IsMemberReference ? LowestElem : LB;
7490 IsExpressionFirstInfo =
false;
7491 IsCaptureFirstInfo =
false;
7492 FirstPointerInComplexData =
false;
7493 IsPrevMemberReference = IsMemberReference;
7494 }
else if (FirstPointerInComplexData) {
7496 ->getAssociatedDeclaration()
7498 .getNonReferenceType();
7500 FirstPointerInComplexData =
false;
7506 PartialStruct.HasCompleteRecord =
true;
7508 if (!IsNonContiguous)
7515 MapValuesArrayTy CurOffsets = {llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, 0)};
7516 MapValuesArrayTy CurCounts = {llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, 1)};
7517 MapValuesArrayTy CurStrides;
7518 MapValuesArrayTy DimSizes{llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, 1)};
7526 const Expr *AssocExpr = Component.getAssociatedExpression();
7527 const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr);
7537 assert((VAT || CAT || &Component == &*Components.begin()) &&
7538 "Should be either ConstantArray or VariableArray if not the "
7542 if (CurStrides.empty()) {
7543 const Type *ElementType =
nullptr;
7545 ElementType = CAT->getElementType().getTypePtr();
7547 ElementType = VAT->getElementType().getTypePtr();
7549 assert(&Component == &*Components.begin() &&
7550 "Only expect pointer (non CAT or VAT) when this is the "
7558 if (&Component != &*Components.begin())
7562 CurStrides.push_back(
7563 llvm::ConstantInt::get(CGF.
Int64Ty, ElementTypeSize));
7568 if (DimSizes.size() < Components.size() - 1) {
7571 llvm::ConstantInt::get(CGF.
Int64Ty, CAT->getZExtSize()));
7573 DimSizes.push_back(CGF.
Builder.CreateIntCast(
7580 auto *DI = DimSizes.begin() + 1;
7582 llvm::Value *DimProd =
7583 llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, ElementTypeSize);
7594 const Expr *AssocExpr = Component.getAssociatedExpression();
7596 if (
const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr)) {
7597 llvm::Value *Offset = CGF.
Builder.CreateIntCast(
7600 CurOffsets.push_back(Offset);
7601 CurCounts.push_back(llvm::ConstantInt::get(CGF.
Int64Ty, 1));
7602 CurStrides.push_back(CurStrides.back());
7606 const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr);
7612 const Expr *OffsetExpr = OASE->getLowerBound();
7613 llvm::Value *Offset =
nullptr;
7616 Offset = llvm::ConstantInt::get(CGF.
Int64Ty, 0);
7622 CurOffsets.push_back(Offset);
7625 const Expr *CountExpr = OASE->getLength();
7626 llvm::Value *Count =
nullptr;
7632 if (!OASE->getColonLocFirst().isValid() &&
7633 !OASE->getColonLocSecond().isValid()) {
7634 Count = llvm::ConstantInt::get(CGF.
Int64Ty, 1);
7640 const Expr *StrideExpr = OASE->getStride();
7641 llvm::Value *Stride =
7647 Count = CGF.
Builder.CreateUDiv(
7648 CGF.
Builder.CreateNUWSub(*DI, Offset), Stride);
7650 Count = CGF.
Builder.CreateNUWSub(*DI, Offset);
7656 CurCounts.push_back(Count);
7665 const Expr *StrideExpr = OASE->getStride();
7666 llvm::Value *Stride =
7671 DimProd = CGF.
Builder.CreateNUWMul(DimProd, *(DI - 1));
7673 CurStrides.push_back(CGF.
Builder.CreateNUWMul(DimProd, Stride));
7675 CurStrides.push_back(DimProd);
7676 if (DI != DimSizes.end())
7680 CombinedInfo.NonContigInfo.Offsets.push_back(CurOffsets);
7681 CombinedInfo.NonContigInfo.Counts.push_back(CurCounts);
7682 CombinedInfo.NonContigInfo.Strides.push_back(CurStrides);
7688 OpenMPOffloadMappingFlags
7697 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
7698 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
7699 return OpenMPOffloadMappingFlags::OMP_MAP_PRIVATE |
7700 OpenMPOffloadMappingFlags::OMP_MAP_TO;
7703 if (I != LambdasMap.end())
7705 return getMapTypeBits(
7706 I->getSecond()->getMapType(), I->getSecond()->getMapTypeModifiers(),
7707 std::nullopt, I->getSecond()->isImplicit(),
7711 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
7712 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
7717 bool AsBase)
const {
7720 llvm::StructType *St =
7723 unsigned NumElements = St->getNumElements();
7725 llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>, 4>
7726 RecordLayout(NumElements);
7729 for (
const auto &I : RD->
bases()) {
7744 RecordLayout[FieldIndex] =
Base;
7747 for (
const auto &I : RD->
vbases()) {
7755 if (RecordLayout[FieldIndex])
7757 RecordLayout[FieldIndex] =
Base;
7760 assert(!RD->
isUnion() &&
"Unexpected union.");
7761 for (
const auto *Field : RD->
fields()) {
7764 if (!
Field->isBitField() &&
7767 RecordLayout[FieldIndex] =
Field;
7770 for (
const llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>
7771 &
Data : RecordLayout) {
7775 getPlainLayout(
Base, Layout,
true);
7786 void generateAllInfoForClauses(
7788 llvm::OpenMPIRBuilder &OMPBuilder,
7795 llvm::MapVector<CanonicalDeclPtr<const Decl>,
7802 [&Info, &SkipVarSet](
7808 bool ReturnDevicePointer,
bool IsImplicit,
const ValueDecl *Mapper,
7809 const Expr *VarRef =
nullptr,
bool ForDeviceAddr =
false) {
7810 if (SkipVarSet.contains(
D))
7812 auto It = Info.find(
D);
7813 if (It == Info.end())
7815 .insert(std::make_pair(
7818 It->second[
Kind].emplace_back(
7819 L, MapType, MapModifiers, MotionModifiers, ReturnDevicePointer,
7820 IsImplicit, Mapper, VarRef, ForDeviceAddr);
7823 for (
const auto *
Cl : Clauses) {
7824 const auto *
C = dyn_cast<OMPMapClause>(
Cl);
7828 if (llvm::is_contained(
C->getMapTypeModifiers(),
7829 OMPC_MAP_MODIFIER_present))
7831 else if (
C->getMapType() == OMPC_MAP_alloc)
7833 const auto *EI =
C->getVarRefs().begin();
7834 for (
const auto L :
C->component_lists()) {
7835 const Expr *
E = (
C->getMapLoc().isValid()) ? *EI :
nullptr;
7836 InfoGen(std::get<0>(L),
Kind, std::get<1>(L),
C->getMapType(),
7837 C->getMapTypeModifiers(), std::nullopt,
7838 false,
C->isImplicit(), std::get<2>(L),
7843 for (
const auto *
Cl : Clauses) {
7844 const auto *
C = dyn_cast<OMPToClause>(
Cl);
7848 if (llvm::is_contained(
C->getMotionModifiers(),
7849 OMPC_MOTION_MODIFIER_present))
7851 const auto *EI =
C->getVarRefs().begin();
7852 for (
const auto L :
C->component_lists()) {
7853 InfoGen(std::get<0>(L),
Kind, std::get<1>(L), OMPC_MAP_to, std::nullopt,
7854 C->getMotionModifiers(),
false,
7855 C->isImplicit(), std::get<2>(L), *EI);
7859 for (
const auto *
Cl : Clauses) {
7860 const auto *
C = dyn_cast<OMPFromClause>(
Cl);
7864 if (llvm::is_contained(
C->getMotionModifiers(),
7865 OMPC_MOTION_MODIFIER_present))
7867 const auto *EI =
C->getVarRefs().begin();
7868 for (
const auto L :
C->component_lists()) {
7869 InfoGen(std::get<0>(L),
Kind, std::get<1>(L), OMPC_MAP_from,
7870 std::nullopt,
C->getMotionModifiers(),
7871 false,
C->isImplicit(), std::get<2>(L),
7884 llvm::MapVector<CanonicalDeclPtr<const Decl>,
7887 MapCombinedInfoTy UseDeviceDataCombinedInfo;
7889 auto &&UseDeviceDataCombinedInfoGen =
7890 [&UseDeviceDataCombinedInfo](
const ValueDecl *VD, llvm::Value *Ptr,
7892 UseDeviceDataCombinedInfo.Exprs.push_back(VD);
7893 UseDeviceDataCombinedInfo.BasePointers.emplace_back(Ptr);
7894 UseDeviceDataCombinedInfo.DevicePtrDecls.emplace_back(VD);
7895 UseDeviceDataCombinedInfo.DevicePointers.emplace_back(
7896 IsDevAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer);
7897 UseDeviceDataCombinedInfo.Pointers.push_back(Ptr);
7898 UseDeviceDataCombinedInfo.Sizes.push_back(
7899 llvm::Constant::getNullValue(CGF.Int64Ty));
7900 UseDeviceDataCombinedInfo.Types.push_back(
7901 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM);
7902 UseDeviceDataCombinedInfo.Mappers.push_back(
nullptr);
7906 [&DeferredInfo, &UseDeviceDataCombinedInfoGen,
7910 bool IsImplicit,
bool IsDevAddr) {
7914 if (isa<MemberExpr>(IE)) {
7923 std::nullopt,
false, IsImplicit,
7924 nullptr,
nullptr, IsDevAddr);
7925 DeferredInfo[
nullptr].emplace_back(IE, VD, IsDevAddr);
7929 if (IE->isGLValue())
7930 Ptr = CGF.EmitLValue(IE).getPointer(CGF);
7932 Ptr = CGF.EmitScalarExpr(IE);
7934 Ptr = CGF.EmitLoadOfScalar(CGF.EmitLValue(IE), IE->getExprLoc());
7936 UseDeviceDataCombinedInfoGen(VD, Ptr, CGF, IsDevAddr);
7941 const Expr *IE,
bool IsDevAddr) ->
bool {
7948 auto It = Info.find(isa<MemberExpr>(IE) ? nullptr : VD);
7949 if (It != Info.end()) {
7951 for (
auto &
Data : It->second) {
7952 auto *CI = llvm::find_if(
Data, [VD](
const MapInfo &MI) {
7953 return MI.Components.back().getAssociatedDeclaration() == VD;
7961 if (CI !=
Data.end()) {
7963 CI->ForDeviceAddr = IsDevAddr;
7964 CI->ReturnDevicePointer =
true;
7968 auto PrevCI = std::next(CI->Components.rbegin());
7969 const auto *VarD = dyn_cast<VarDecl>(VD);
7971 isa<MemberExpr>(IE) ||
7972 !VD->getType().getNonReferenceType()->isPointerType() ||
7973 PrevCI == CI->Components.rend() ||
7974 isa<MemberExpr>(PrevCI->getAssociatedExpression()) || !VarD ||
7975 VarD->hasLocalStorage()) {
7976 CI->ForDeviceAddr = IsDevAddr;
7977 CI->ReturnDevicePointer =
true;
7995 for (
const auto *
Cl : Clauses) {
7996 const auto *
C = dyn_cast<OMPUseDevicePtrClause>(
Cl);
7999 for (
const auto L :
C->component_lists()) {
8002 assert(!Components.empty() &&
8003 "Not expecting empty list of components!");
8004 const ValueDecl *VD = Components.back().getAssociatedDeclaration();
8006 const Expr *IE = Components.back().getAssociatedExpression();
8007 if (IsMapInfoExist(CGF, VD, IE,
false))
8009 MapInfoGen(CGF, IE, VD, Components,
C->isImplicit(),
8014 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
8015 for (
const auto *
Cl : Clauses) {
8016 const auto *
C = dyn_cast<OMPUseDeviceAddrClause>(
Cl);
8019 for (
const auto L :
C->component_lists()) {
8022 assert(!std::get<1>(L).empty() &&
8023 "Not expecting empty list of components!");
8024 const ValueDecl *VD = std::get<1>(L).back().getAssociatedDeclaration();
8025 if (!Processed.insert(VD).second)
8028 const Expr *IE = std::get<1>(L).back().getAssociatedExpression();
8029 if (IsMapInfoExist(CGF, VD, IE,
true))
8031 MapInfoGen(CGF, IE, VD, Components,
C->isImplicit(),
8036 for (
const auto &
Data : Info) {
8037 StructRangeInfoTy PartialStruct;
8039 MapCombinedInfoTy CurInfo;
8041 MapCombinedInfoTy StructBaseCurInfo;
8043 const ValueDecl *VD = cast_or_null<ValueDecl>(
D);
8044 bool HasMapBasePtr =
false;
8045 bool HasMapArraySec =
false;
8047 for (
const auto &M :
Data.second) {
8048 HasMapBasePtr = any_of(M, [](
const MapInfo &L) {
8049 return isa_and_present<DeclRefExpr>(L.VarRef);
8051 HasMapArraySec = any_of(M, [](
const MapInfo &L) {
8052 return isa_and_present<ArraySectionExpr, ArraySubscriptExpr>(
8055 if (HasMapBasePtr && HasMapArraySec)
8059 for (
const auto &M :
Data.second) {
8060 for (
const MapInfo &L : M) {
8061 assert(!L.Components.empty() &&
8062 "Not expecting declaration with no component lists.");
8065 unsigned CurrentBasePointersIdx = CurInfo.BasePointers.size();
8066 unsigned StructBasePointersIdx =
8067 StructBaseCurInfo.BasePointers.size();
8068 CurInfo.NonContigInfo.IsNonContiguous =
8069 L.Components.back().isNonContiguous();
8070 generateInfoForComponentList(
8071 L.MapType, L.MapModifiers, L.MotionModifiers, L.Components,
8072 CurInfo, StructBaseCurInfo, PartialStruct,
8073 false, L.IsImplicit,
8074 true, L.Mapper, L.ForDeviceAddr, VD,
8075 L.VarRef, std::nullopt,
8076 HasMapBasePtr && HasMapArraySec);
8080 if (L.ReturnDevicePointer) {
8084 assert((CurrentBasePointersIdx < CurInfo.BasePointers.size() ||
8085 StructBasePointersIdx <
8086 StructBaseCurInfo.BasePointers.size()) &&
8087 "Unexpected number of mapped base pointers.");
8091 L.Components.back().getAssociatedDeclaration();
8092 assert(RelevantVD &&
8093 "No relevant declaration related with device pointer??");
8100 if (StructBasePointersIdx < StructBaseCurInfo.BasePointers.size()) {
8101 StructBaseCurInfo.DevicePtrDecls[StructBasePointersIdx] =
8103 StructBaseCurInfo.DevicePointers[StructBasePointersIdx] =
8104 L.ForDeviceAddr ? DeviceInfoTy::Address
8105 : DeviceInfoTy::Pointer;
8106 StructBaseCurInfo.Types[StructBasePointersIdx] |=
8107 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
8109 CurInfo.DevicePtrDecls[CurrentBasePointersIdx] = RelevantVD;
8110 CurInfo.DevicePointers[CurrentBasePointersIdx] =
8111 L.ForDeviceAddr ? DeviceInfoTy::Address
8112 : DeviceInfoTy::Pointer;
8113 CurInfo.Types[CurrentBasePointersIdx] |=
8114 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
8122 auto CI = DeferredInfo.find(
Data.first);
8123 if (CI != DeferredInfo.end()) {
8124 for (
const DeferredDevicePtrEntryTy &L : CI->second) {
8125 llvm::Value *BasePtr;
8127 if (L.ForDeviceAddr) {
8128 if (L.IE->isGLValue())
8136 CurInfo.Types.push_back(
8137 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM |
8138 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
8142 L.IE->getExprLoc());
8146 CurInfo.Types.push_back(
8147 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
8148 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM |
8149 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
8151 CurInfo.Exprs.push_back(L.VD);
8152 CurInfo.BasePointers.emplace_back(BasePtr);
8153 CurInfo.DevicePtrDecls.emplace_back(L.VD);
8154 CurInfo.DevicePointers.emplace_back(
8155 L.ForDeviceAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer);
8156 CurInfo.Pointers.push_back(Ptr);
8157 CurInfo.Sizes.push_back(
8158 llvm::Constant::getNullValue(this->CGF.
Int64Ty));
8159 CurInfo.Mappers.push_back(
nullptr);
8165 MapCombinedInfoTy UnionCurInfo;
8166 UnionCurInfo.append(StructBaseCurInfo);
8167 UnionCurInfo.append(CurInfo);
8171 if (PartialStruct.Base.isValid()) {
8172 UnionCurInfo.NonContigInfo.Dims.push_back(0);
8174 emitCombinedEntry(CombinedInfo, UnionCurInfo.Types, PartialStruct,
8175 !VD, OMPBuilder, VD);
8179 CombinedInfo.append(UnionCurInfo);
8182 CombinedInfo.append(UseDeviceDataCombinedInfo);
8187 : CurDir(&Dir), CGF(CGF) {
8190 for (
const auto *
D :
C->varlists())
8191 FirstPrivateDecls.try_emplace(
8192 cast<VarDecl>(cast<DeclRefExpr>(
D)->getDecl()),
C->isImplicit());
8195 for (
unsigned I = 0,
E =
C->getNumberOfAllocators(); I <
E; ++I) {
8197 if (
const auto *DRE = dyn_cast_or_null<DeclRefExpr>(
D.AllocatorTraits))
8198 FirstPrivateDecls.try_emplace(cast<VarDecl>(DRE->getDecl()),
8200 else if (
const auto *VD = dyn_cast<VarDecl>(
8201 cast<DeclRefExpr>(
D.Allocator->IgnoreParenImpCasts())
8203 FirstPrivateDecls.try_emplace(VD,
true);
8208 for (
auto L :
C->component_lists())
8209 DevPointersMap[std::get<0>(L)].push_back(std::get<1>(L));
8212 for (
auto L :
C->component_lists())
8213 HasDevAddrsMap[std::get<0>(L)].push_back(std::get<1>(L));
8216 if (
C->getMapType() != OMPC_MAP_to)
8218 for (
auto L :
C->component_lists()) {
8220 const auto *RD = VD ? VD->
getType()
8226 LambdasMap.try_emplace(std::get<0>(L),
C);
8233 : CurDir(&Dir), CGF(CGF) {}
8238 void emitCombinedEntry(MapCombinedInfoTy &CombinedInfo,
8239 MapFlagsArrayTy &CurTypes,
8240 const StructRangeInfoTy &PartialStruct,
bool IsMapThis,
8241 llvm::OpenMPIRBuilder &OMPBuilder,
8243 bool NotTargetParams =
true)
const {
8244 if (CurTypes.size() == 1 &&
8245 ((CurTypes.back() & OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) !=
8246 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) &&
8247 !PartialStruct.IsArraySection)
8249 Address LBAddr = PartialStruct.LowestElem.second;
8250 Address HBAddr = PartialStruct.HighestElem.second;
8251 if (PartialStruct.HasCompleteRecord) {
8252 LBAddr = PartialStruct.LB;
8253 HBAddr = PartialStruct.LB;
8255 CombinedInfo.Exprs.push_back(VD);
8257 CombinedInfo.BasePointers.push_back(PartialStruct.Base.emitRawPointer(CGF));
8258 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8259 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8265 bool HasBaseClass = RD && IsMapThis ? RD->
getNumBases() > 0 :
false;
8275 CombinedInfo.Pointers.push_back(PartialStruct.Base.emitRawPointer(CGF));
8280 CombinedInfo.Sizes.push_back(Size);
8282 CombinedInfo.Pointers.push_back(LB);
8285 llvm::Value *HAddr = CGF.
Builder.CreateConstGEP1_32(
8289 llvm::Value *Diff = CGF.
Builder.CreatePtrDiff(CGF.
Int8Ty, CHAddr, CLAddr);
8292 CombinedInfo.Sizes.push_back(Size);
8294 CombinedInfo.Mappers.push_back(
nullptr);
8296 CombinedInfo.Types.push_back(
8297 NotTargetParams ? OpenMPOffloadMappingFlags::OMP_MAP_NONE
8298 : OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
8301 if (CurTypes.end() !=
8302 llvm::find_if(CurTypes, [](OpenMPOffloadMappingFlags
Type) {
8303 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
8304 Type & OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
8306 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
8308 (*CurTypes.begin()) &= ~OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
8315 if (CurTypes.end() !=
8316 llvm::find_if(CurTypes, [](OpenMPOffloadMappingFlags
Type) {
8317 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
8318 Type & OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD);
8320 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
8321 for (
auto &M : CurTypes)
8322 M |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
8328 OpenMPOffloadMappingFlags MemberOfFlag =
8329 OMPBuilder.getMemberOfFlag(CombinedInfo.BasePointers.size() - 1);
8330 for (
auto &M : CurTypes)
8331 OMPBuilder.setCorrectMemberOfFlag(M, MemberOfFlag);
8339 void generateAllInfo(
8340 MapCombinedInfoTy &CombinedInfo, llvm::OpenMPIRBuilder &OMPBuilder,
8344 "Expect a executable directive");
8346 generateAllInfoForClauses(CurExecDir->clauses(), CombinedInfo, OMPBuilder,
8353 void generateAllInfoForMapper(MapCombinedInfoTy &CombinedInfo,
8354 llvm::OpenMPIRBuilder &OMPBuilder)
const {
8356 "Expect a declare mapper directive");
8358 generateAllInfoForClauses(CurMapperDir->clauses(), CombinedInfo,
8363 void generateInfoForLambdaCaptures(
8364 const ValueDecl *VD, llvm::Value *Arg, MapCombinedInfoTy &CombinedInfo,
8365 llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers)
const {
8373 llvm::DenseMap<const ValueDecl *, FieldDecl *> Captures;
8380 LambdaPointers.try_emplace(ThisLVal.
getPointer(CGF),
8382 CombinedInfo.Exprs.push_back(VD);
8383 CombinedInfo.BasePointers.push_back(ThisLVal.
getPointer(CGF));
8384 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8385 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8386 CombinedInfo.Pointers.push_back(ThisLValVal.
getPointer(CGF));
8387 CombinedInfo.Sizes.push_back(
8390 CombinedInfo.Types.push_back(
8391 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
8392 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
8393 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
8394 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
8395 CombinedInfo.Mappers.push_back(
nullptr);
8398 if (!LC.capturesVariable())
8400 const VarDecl *VD = cast<VarDecl>(LC.getCapturedVar());
8403 auto It = Captures.find(VD);
8404 assert(It != Captures.end() &&
"Found lambda capture without field.");
8408 LambdaPointers.try_emplace(VarLVal.
getPointer(CGF),
8410 CombinedInfo.Exprs.push_back(VD);
8411 CombinedInfo.BasePointers.push_back(VarLVal.
getPointer(CGF));
8412 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8413 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8414 CombinedInfo.Pointers.push_back(VarLValVal.
getPointer(CGF));
8415 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
8421 LambdaPointers.try_emplace(VarLVal.
getPointer(CGF),
8423 CombinedInfo.Exprs.push_back(VD);
8424 CombinedInfo.BasePointers.push_back(VarLVal.
getPointer(CGF));
8425 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8426 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8427 CombinedInfo.Pointers.push_back(VarRVal.
getScalarVal());
8428 CombinedInfo.Sizes.push_back(llvm::ConstantInt::get(CGF.
Int64Ty, 0));
8430 CombinedInfo.Types.push_back(
8431 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
8432 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
8433 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
8434 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
8435 CombinedInfo.Mappers.push_back(
nullptr);
8440 void adjustMemberOfForLambdaCaptures(
8441 llvm::OpenMPIRBuilder &OMPBuilder,
8442 const llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers,
8443 MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
8444 MapFlagsArrayTy &Types)
const {
8445 for (
unsigned I = 0,
E = Types.size(); I <
E; ++I) {
8447 if (Types[I] != (OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
8448 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
8449 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
8450 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT))
8452 llvm::Value *BasePtr = LambdaPointers.lookup(BasePointers[I]);
8453 assert(BasePtr &&
"Unable to find base lambda address.");
8455 for (
unsigned J = I; J > 0; --J) {
8456 unsigned Idx = J - 1;
8457 if (Pointers[Idx] != BasePtr)
8462 assert(TgtIdx != -1 &&
"Unable to find parent lambda.");
8466 OpenMPOffloadMappingFlags MemberOfFlag =
8467 OMPBuilder.getMemberOfFlag(TgtIdx);
8468 OMPBuilder.setCorrectMemberOfFlag(Types[I], MemberOfFlag);
8475 llvm::Value *Arg, MapCombinedInfoTy &CombinedInfo,
8476 StructRangeInfoTy &PartialStruct)
const {
8478 "Not expecting to generate map info for a variable array type!");
8487 if (LambdasMap.count(VD))
8493 if (VD && (DevPointersMap.count(VD) || HasDevAddrsMap.count(VD))) {
8494 CombinedInfo.Exprs.push_back(VD);
8495 CombinedInfo.BasePointers.emplace_back(Arg);
8496 CombinedInfo.DevicePtrDecls.emplace_back(VD);
8497 CombinedInfo.DevicePointers.emplace_back(DeviceInfoTy::Pointer);
8498 CombinedInfo.Pointers.push_back(Arg);
8499 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
8502 CombinedInfo.Types.push_back(
8503 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
8504 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
8505 CombinedInfo.Mappers.push_back(
nullptr);
8517 auto It = DevPointersMap.find(VD);
8518 if (It != DevPointersMap.end())
8519 for (
const auto &MCL : It->second)
8520 DeclComponentLists.emplace_back(MCL, OMPC_MAP_to,
Unknown,
8523 auto I = HasDevAddrsMap.find(VD);
8524 if (I != HasDevAddrsMap.end())
8525 for (
const auto &MCL : I->second)
8526 DeclComponentLists.emplace_back(MCL, OMPC_MAP_tofrom,
Unknown,
8530 "Expect a executable directive");
8532 bool HasMapBasePtr =
false;
8533 bool HasMapArraySec =
false;
8534 for (
const auto *
C : CurExecDir->getClausesOfKind<
OMPMapClause>()) {
8535 const auto *EI =
C->getVarRefs().begin();
8536 for (
const auto L :
C->decl_component_lists(VD)) {
8539 const Expr *
E = (
C->getMapLoc().isValid()) ? *EI :
nullptr;
8541 std::tie(VDecl, Components, Mapper) = L;
8542 assert(VDecl == VD &&
"We got information for the wrong declaration??");
8543 assert(!Components.empty() &&
8544 "Not expecting declaration with no component lists.");
8546 HasMapBasePtr =
true;
8548 (isa<ArraySectionExpr>(
E) || isa<ArraySubscriptExpr>(
E)))
8549 HasMapArraySec =
true;
8550 DeclComponentLists.emplace_back(Components,
C->getMapType(),
8551 C->getMapTypeModifiers(),
8552 C->isImplicit(), Mapper,
E);
8556 llvm::stable_sort(DeclComponentLists, [](
const MapData &LHS,
8557 const MapData &RHS) {
8561 llvm::is_contained(MapModifiers, clang::OMPC_MAP_MODIFIER_present);
8562 bool HasAllocs = MapType == OMPC_MAP_alloc;
8563 MapModifiers = std::get<2>(RHS);
8564 MapType = std::get<1>(LHS);
8566 llvm::is_contained(MapModifiers, clang::OMPC_MAP_MODIFIER_present);
8567 bool HasAllocsR = MapType == OMPC_MAP_alloc;
8568 return (HasPresent && !HasPresentR) || (HasAllocs && !HasAllocsR);
8572 llvm::SmallDenseMap<
8579 for (
const MapData &L : DeclComponentLists) {
8586 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =
8589 for (
const MapData &L1 :
ArrayRef(DeclComponentLists).slice(Count)) {
8591 std::tie(Components1, MapType, MapModifiers, IsImplicit, Mapper,
8593 auto CI = Components.rbegin();
8594 auto CE = Components.rend();
8595 auto SI = Components1.rbegin();
8596 auto SE = Components1.rend();
8597 for (; CI != CE && SI != SE; ++CI, ++SI) {
8598 if (CI->getAssociatedExpression()->getStmtClass() !=
8599 SI->getAssociatedExpression()->getStmtClass())
8602 if (CI->getAssociatedDeclaration() != SI->getAssociatedDeclaration())
8607 if (CI == CE || SI == SE) {
8609 if (CI == CE && SI == SE)
8611 const auto It = (SI == SE) ? CI : SI;
8617 if (!isa<MemberExpr>(It->getAssociatedExpression()) ||
8618 (std::prev(It)->getAssociatedDeclaration() &&
8620 ->getAssociatedDeclaration()
8622 ->isPointerType()) ||
8623 (It->getAssociatedDeclaration() &&
8624 It->getAssociatedDeclaration()->getType()->isPointerType() &&
8625 std::next(It) != CE && std::next(It) != SE))
8627 const MapData &BaseData = CI == CE ? L : L1;
8629 SI == SE ? Components : Components1;
8630 auto &OverlappedElements = OverlappedData.FindAndConstruct(&BaseData);
8631 OverlappedElements.getSecond().push_back(SubData);
8637 if (!OverlappedData.empty()) {
8640 while (BaseType != OrigType) {
8646 getPlainLayout(CRD, Layout,
false);
8652 for (
auto &Pair : OverlappedData) {
8659 auto CI = First.rbegin();
8660 auto CE = First.rend();
8661 auto SI = Second.rbegin();
8662 auto SE = Second.rend();
8663 for (; CI != CE && SI != SE; ++CI, ++SI) {
8664 if (CI->getAssociatedExpression()->getStmtClass() !=
8665 SI->getAssociatedExpression()->getStmtClass())
8668 if (CI->getAssociatedDeclaration() !=
8669 SI->getAssociatedDeclaration())
8674 if (CI == CE && SI == SE)
8678 if (CI == CE || SI == SE)
8681 const auto *FD1 = cast<FieldDecl>(CI->getAssociatedDeclaration());
8682 const auto *FD2 = cast<FieldDecl>(SI->getAssociatedDeclaration());
8683 if (FD1->getParent() == FD2->getParent())
8684 return FD1->getFieldIndex() < FD2->getFieldIndex();
8686 llvm::find_if(Layout, [FD1, FD2](
const FieldDecl *FD) {
8687 return FD == FD1 || FD == FD2;
8695 bool IsFirstComponentList =
true;
8696 MapCombinedInfoTy StructBaseCombinedInfo;
8697 for (
const auto &Pair : OverlappedData) {
8698 const MapData &L = *Pair.getFirst();
8705 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =
8708 OverlappedComponents = Pair.getSecond();
8709 generateInfoForComponentList(
8710 MapType, MapModifiers, std::nullopt, Components, CombinedInfo,
8711 StructBaseCombinedInfo, PartialStruct, IsFirstComponentList,
8712 IsImplicit,
false, Mapper,
8713 false, VD, VarRef, OverlappedComponents);
8714 IsFirstComponentList =
false;
8717 for (
const MapData &L : DeclComponentLists) {
8724 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =
8726 auto It = OverlappedData.find(&L);
8727 if (It == OverlappedData.end())
8728 generateInfoForComponentList(
8729 MapType, MapModifiers, std::nullopt, Components, CombinedInfo,
8730 StructBaseCombinedInfo, PartialStruct, IsFirstComponentList,
8731 IsImplicit,
false, Mapper,
8734 HasMapBasePtr && HasMapArraySec);
8735 IsFirstComponentList =
false;
8743 MapCombinedInfoTy &CombinedInfo)
const {
8744 bool IsImplicit =
true;
8747 CombinedInfo.Exprs.push_back(
nullptr);
8748 CombinedInfo.BasePointers.push_back(CV);
8749 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8750 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8751 CombinedInfo.Pointers.push_back(CV);
8753 CombinedInfo.Sizes.push_back(
8757 CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_TO |
8758 OpenMPOffloadMappingFlags::OMP_MAP_FROM);
8762 CombinedInfo.BasePointers.push_back(CV);
8763 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8764 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8765 CombinedInfo.Pointers.push_back(CV);
8769 CombinedInfo.Types.push_back(
8770 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
8771 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
8776 CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_NONE);
8777 CombinedInfo.Sizes.push_back(llvm::Constant::getNullValue(CGF.
Int64Ty));
8779 auto I = FirstPrivateDecls.find(VD);
8780 if (I != FirstPrivateDecls.end())
8781 IsImplicit = I->getSecond();
8786 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
8791 CombinedInfo.Types.push_back(getMapModifiersForPrivateClauses(CI));
8793 auto I = FirstPrivateDecls.find(VD);
8795 CombinedInfo.BasePointers.push_back(CV);
8796 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8797 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8804 CombinedInfo.Pointers.push_back(CV);
8806 if (I != FirstPrivateDecls.end())
8807 IsImplicit = I->getSecond();
8810 CombinedInfo.Types.back() |=
8811 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
8815 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
8818 CombinedInfo.Mappers.push_back(
nullptr);
8830 dyn_cast<MemberExpr>(OASE->getBase()->IgnoreParenImpCasts()))
8831 return ME->getMemberDecl();
8839 MappableExprsHandler::MappingExprInfo &MapExprs) {
8841 uint32_t SrcLocStrSize;
8842 if (!MapExprs.getMapDecl() && !MapExprs.getMapExpr())
8843 return OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
8846 if (!MapExprs.getMapDecl() && MapExprs.getMapExpr()) {
8850 Loc = MapExprs.getMapExpr()->getExprLoc();
8852 Loc = MapExprs.getMapDecl()->getLocation();
8855 std::string ExprName;
8856 if (MapExprs.getMapExpr()) {
8858 llvm::raw_string_ostream OS(ExprName);
8859 MapExprs.getMapExpr()->printPretty(OS,
nullptr,
P);
8862 ExprName = MapExprs.getMapDecl()->getNameAsString();
8866 return OMPBuilder.getOrCreateSrcLocStr(PLoc.
getFilename(), ExprName,
8875 CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
8877 bool IsNonContiguous =
false) {
8881 Info.clearArrayInfo();
8882 Info.NumberOfPtrs = CombinedInfo.BasePointers.size();
8884 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
8887 InsertPointTy CodeGenIP(CGF.
Builder.GetInsertBlock(),
8888 CGF.
Builder.GetInsertPoint());
8890 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
8894 llvm::codegenoptions::NoDebugInfo) {
8895 CombinedInfo.Names.resize(CombinedInfo.Exprs.size());
8896 llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),
8900 auto DeviceAddrCB = [&](
unsigned int I, llvm::Value *NewDecl) {
8901 if (
const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
8906 auto CustomMapperCB = [&](
unsigned int I) {
8907 llvm::Value *MFunc =
nullptr;
8908 if (CombinedInfo.Mappers[I]) {
8909 Info.HasMapper =
true;
8911 cast<OMPDeclareMapperDecl>(CombinedInfo.Mappers[I]));
8915 OMPBuilder.emitOffloadingArrays(AllocaIP, CodeGenIP, CombinedInfo, Info,
8923 const auto *CS =
D.getInnermostCapturedStmt();
8926 const Stmt *ChildStmt =
8929 if (
const auto *NestedDir =
8930 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
8932 switch (
D.getDirectiveKind()) {
8938 if (DKind == OMPD_teams) {
8939 Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers(
8944 if (
const auto *NND =
8945 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
8946 DKind = NND->getDirectiveKind();
8952 case OMPD_target_teams:
8956 case OMPD_target_parallel:
8957 case OMPD_target_simd:
8958 case OMPD_target_parallel_for:
8959 case OMPD_target_parallel_for_simd:
8961 case OMPD_target_teams_distribute:
8962 case OMPD_target_teams_distribute_simd:
8963 case OMPD_target_teams_distribute_parallel_for:
8964 case OMPD_target_teams_distribute_parallel_for_simd:
8967 case OMPD_parallel_for:
8968 case OMPD_parallel_master:
8969 case OMPD_parallel_sections:
8971 case OMPD_parallel_for_simd:
8973 case OMPD_cancellation_point:
8975 case OMPD_threadprivate:
8986 case OMPD_taskyield:
8989 case OMPD_taskgroup:
8995 case OMPD_target_data:
8996 case OMPD_target_exit_data:
8997 case OMPD_target_enter_data:
8998 case OMPD_distribute:
8999 case OMPD_distribute_simd:
9000 case OMPD_distribute_parallel_for:
9001 case OMPD_distribute_parallel_for_simd:
9002 case OMPD_teams_distribute:
9003 case OMPD_teams_distribute_simd:
9004 case OMPD_teams_distribute_parallel_for:
9005 case OMPD_teams_distribute_parallel_for_simd:
9006 case OMPD_target_update:
9007 case OMPD_declare_simd:
9008 case OMPD_declare_variant:
9009 case OMPD_begin_declare_variant:
9010 case OMPD_end_declare_variant:
9011 case OMPD_declare_target:
9012 case OMPD_end_declare_target:
9013 case OMPD_declare_reduction:
9014 case OMPD_declare_mapper:
9016 case OMPD_taskloop_simd:
9017 case OMPD_master_taskloop:
9018 case OMPD_master_taskloop_simd:
9019 case OMPD_parallel_master_taskloop:
9020 case OMPD_parallel_master_taskloop_simd:
9022 case OMPD_metadirective:
9025 llvm_unreachable(
"Unexpected directive.");
9070 QualType PtrTy =
C.getPointerType(Ty).withRestrict();
9071 QualType Int64Ty =
C.getIntTypeForBitwidth(64,
true);
9072 auto *MapperVarDecl =
9073 cast<VarDecl>(cast<DeclRefExpr>(
D->getMapperVarRef())->getDecl());
9075 CharUnits ElementSize =
C.getTypeSizeInChars(Ty);
9092 Args.push_back(&HandleArg);
9093 Args.push_back(&BaseArg);
9094 Args.push_back(&BeginArg);
9095 Args.push_back(&SizeArg);
9096 Args.push_back(&TypeArg);
9097 Args.push_back(&NameArg);
9102 llvm::raw_svector_ostream Out(TyStr);
9104 std::string Name =
getName({
"omp_mapper", TyStr,
D->getName()});
9105 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
9108 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
9115 C.getPointerType(Int64Ty),
Loc);
9119 false,
C.getPointerType(
C.VoidPtrTy),
Loc);
9122 false,
C.getPointerType(
C.VoidPtrTy),
Loc);
9125 false,
C.getPointerType(
C.VoidPtrTy),
Loc);
9127 Size = MapperCGF.
Builder.CreateExactUDiv(
9129 llvm::Value *PtrBegin = MapperCGF.
Builder.CreateBitCast(
9134 C.getPointerType(Int64Ty),
Loc);
9137 false,
C.getPointerType(
C.VoidPtrTy),
Loc);
9141 llvm::BasicBlock *HeadBB = MapperCGF.
createBasicBlock(
"omp.arraymap.head");
9143 MapName, ElementSize, HeadBB,
true);
9149 llvm::BasicBlock *BodyBB = MapperCGF.
createBasicBlock(
"omp.arraymap.body");
9152 llvm::Value *IsEmpty =
9153 MapperCGF.
Builder.CreateICmpEQ(PtrBegin, PtrEnd,
"omp.arraymap.isempty");
9154 MapperCGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
9155 llvm::BasicBlock *EntryBB = MapperCGF.
Builder.GetInsertBlock();
9159 llvm::BasicBlock *LastBB = BodyBB;
9160 llvm::PHINode *PtrPHI = MapperCGF.
Builder.CreatePHI(
9161 PtrBegin->getType(), 2,
"omp.arraymap.ptrcurrent");
9162 PtrPHI->addIncoming(PtrBegin, EntryBB);
9163 Address PtrCurrent(PtrPHI, ElemTy,
9169 Scope.addPrivate(MapperVarDecl, PtrCurrent);
9170 (void)
Scope.Privatize();
9173 MappableExprsHandler::MapCombinedInfoTy Info;
9174 MappableExprsHandler MEHandler(*
D, MapperCGF);
9175 MEHandler.generateAllInfoForMapper(Info,
OMPBuilder);
9179 llvm::Value *OffloadingArgs[] = {Handle};
9182 OMPRTL___tgt_mapper_num_components),
9184 llvm::Value *ShiftedPreviousSize = MapperCGF.
Builder.CreateShl(
9186 MapperCGF.
Builder.getInt64(MappableExprsHandler::getFlagMemberOffset()));
9189 for (
unsigned I = 0; I < Info.BasePointers.size(); ++I) {
9190 llvm::Value *CurBaseArg = MapperCGF.
Builder.CreateBitCast(
9192 llvm::Value *CurBeginArg = MapperCGF.
Builder.CreateBitCast(
9194 llvm::Value *CurSizeArg = Info.Sizes[I];
9195 llvm::Value *CurNameArg =
9197 llvm::codegenoptions::NoDebugInfo)
9202 llvm::Value *OriMapType = MapperCGF.
Builder.getInt64(
9203 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9205 llvm::Value *MemberMapType =
9206 MapperCGF.
Builder.CreateNUWAdd(OriMapType, ShiftedPreviousSize);
9220 llvm::Value *LeftToFrom = MapperCGF.
Builder.CreateAnd(
9223 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9224 OpenMPOffloadMappingFlags::OMP_MAP_TO |
9225 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
9227 llvm::BasicBlock *AllocElseBB =
9230 llvm::BasicBlock *ToElseBB = MapperCGF.
createBasicBlock(
"omp.type.to.else");
9234 MapperCGF.
Builder.CreateCondBr(IsAlloc, AllocBB, AllocElseBB);
9237 llvm::Value *AllocMapType = MapperCGF.
Builder.CreateAnd(
9240 ~
static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9241 OpenMPOffloadMappingFlags::OMP_MAP_TO |
9242 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
9243 MapperCGF.
Builder.CreateBr(EndBB);
9245 llvm::Value *IsTo = MapperCGF.
Builder.CreateICmpEQ(
9248 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9249 OpenMPOffloadMappingFlags::OMP_MAP_TO)));
9250 MapperCGF.
Builder.CreateCondBr(IsTo, ToBB, ToElseBB);
9253 llvm::Value *ToMapType = MapperCGF.
Builder.CreateAnd(
9256 ~
static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9257 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
9258 MapperCGF.
Builder.CreateBr(EndBB);
9260 llvm::Value *IsFrom = MapperCGF.
Builder.CreateICmpEQ(
9263 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9264 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
9265 MapperCGF.
Builder.CreateCondBr(IsFrom, FromBB, EndBB);
9268 llvm::Value *FromMapType = MapperCGF.
Builder.CreateAnd(
9271 ~
static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9272 OpenMPOffloadMappingFlags::OMP_MAP_TO)));
9276 llvm::PHINode *CurMapType =
9278 CurMapType->addIncoming(AllocMapType, AllocBB);
9279 CurMapType->addIncoming(ToMapType, ToBB);
9280 CurMapType->addIncoming(FromMapType, FromBB);
9281 CurMapType->addIncoming(MemberMapType, ToElseBB);
9283 llvm::Value *OffloadingArgs[] = {Handle, CurBaseArg, CurBeginArg,
9284 CurSizeArg, CurMapType, CurNameArg};
9285 if (Info.Mappers[I]) {
9288 cast<OMPDeclareMapperDecl>(Info.Mappers[I]));
9289 assert(MapperFunc &&
"Expect a valid mapper function is available.");
9303 llvm::Value *PtrNext = MapperCGF.
Builder.CreateConstGEP1_32(
9304 ElemTy, PtrPHI, 1,
"omp.arraymap.next");
9305 PtrPHI->addIncoming(PtrNext, LastBB);
9306 llvm::Value *IsDone =
9307 MapperCGF.
Builder.CreateICmpEQ(PtrNext, PtrEnd,
"omp.arraymap.isdone");
9308 llvm::BasicBlock *ExitBB = MapperCGF.
createBasicBlock(
"omp.arraymap.exit");
9309 MapperCGF.
Builder.CreateCondBr(IsDone, ExitBB, BodyBB);
9315 MapName, ElementSize, DoneBB,
false);
9323 Decls.second.push_back(
D);
9335 llvm::Value *
Begin, llvm::Value *Size, llvm::Value *MapType,
9336 llvm::Value *MapName,
CharUnits ElementSize, llvm::BasicBlock *ExitBB,
9338 StringRef Prefix = IsInit ?
".init" :
".del";
9341 llvm::BasicBlock *BodyBB =
9343 llvm::Value *IsArray = MapperCGF.
Builder.CreateICmpSGT(
9344 Size, MapperCGF.
Builder.getInt64(1),
"omp.arrayinit.isarray");
9345 llvm::Value *DeleteBit = MapperCGF.
Builder.CreateAnd(
9348 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9349 OpenMPOffloadMappingFlags::OMP_MAP_DELETE)));
9350 llvm::Value *DeleteCond;
9356 llvm::Value *PtrAndObjBit = MapperCGF.
Builder.CreateAnd(
9359 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9360 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ)));
9361 PtrAndObjBit = MapperCGF.
Builder.CreateIsNotNull(PtrAndObjBit);
9362 BaseIsBegin = MapperCGF.
Builder.CreateAnd(BaseIsBegin, PtrAndObjBit);
9363 Cond = MapperCGF.
Builder.CreateOr(IsArray, BaseIsBegin);
9365 DeleteBit,
getName({
"omp.array", Prefix,
".delete"}));
9368 DeleteCond = MapperCGF.
Builder.CreateIsNotNull(
9369 DeleteBit,
getName({
"omp.array", Prefix,
".delete"}));
9371 Cond = MapperCGF.
Builder.CreateAnd(Cond, DeleteCond);
9372 MapperCGF.
Builder.CreateCondBr(Cond, BodyBB, ExitBB);
9377 llvm::Value *ArraySize = MapperCGF.
Builder.CreateNUWMul(
9381 llvm::Value *MapTypeArg = MapperCGF.
Builder.CreateAnd(
9384 ~
static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9385 OpenMPOffloadMappingFlags::OMP_MAP_TO |
9386 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
9387 MapTypeArg = MapperCGF.
Builder.CreateOr(
9390 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
9391 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT)));
9395 llvm::Value *OffloadingArgs[] = {Handle,
Base,
Begin,
9396 ArraySize, MapTypeArg, MapName};
9399 OMPRTL___tgt_push_mapper_component),
9422 Kind != OMPD_target_teams_loop)
9425 return llvm::ConstantInt::get(CGF.
Int64Ty, 0);
9427 const auto *LD = cast<OMPLoopDirective>(TD);
9428 if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD))
9429 return NumIterations;
9430 return llvm::ConstantInt::get(CGF.
Int64Ty, 0);
9439 if (OffloadingMandatory) {
9440 CGF.
Builder.CreateUnreachable();
9442 if (RequiresOuterTask) {
9443 CapturedVars.clear();
9452 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
9455 llvm::Value *DeviceID;
9456 if (
Device.getPointer()) {
9458 Device.getInt() == OMPC_DEVICE_device_num) &&
9459 "Expected device_num modifier.");
9464 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
9471 llvm::Value *DynCGroupMem = CGF.
Builder.getInt32(0);
9474 CodeGenFunction::RunCleanupsScope DynCGroupMemScope(CGF);
9476 DynMemClause->getSize(),
true);
9477 DynCGroupMem = CGF.
Builder.CreateIntCast(DynCGroupMemVal, CGF.
Int32Ty,
9480 return DynCGroupMem;
9488 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
9489 llvm::Value *OutlinedFnID, CodeGenFunction::OMPTargetDataInfo &InputInfo,
9490 llvm::Value *&MapTypesArray, llvm::Value *&MapNamesArray,
9495 llvm::OpenMPIRBuilder &OMPBuilder = OMPRuntime->
getOMPBuilder();
9498 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
9501 MappableExprsHandler MEHandler(
D, CGF);
9502 llvm::DenseMap<llvm::Value *, llvm::Value *> LambdaPointers;
9506 auto *CV = CapturedVars.begin();
9509 CI != CE; ++CI, ++RI, ++CV) {
9510 MappableExprsHandler::MapCombinedInfoTy CurInfo;
9511 MappableExprsHandler::StructRangeInfoTy PartialStruct;
9516 CurInfo.Exprs.push_back(
nullptr);
9517 CurInfo.BasePointers.push_back(*CV);
9518 CurInfo.DevicePtrDecls.push_back(
nullptr);
9519 CurInfo.DevicePointers.push_back(
9520 MappableExprsHandler::DeviceInfoTy::None);
9521 CurInfo.Pointers.push_back(*CV);
9522 CurInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
9525 CurInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9526 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM |
9527 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
9528 CurInfo.Mappers.push_back(
nullptr);
9532 MEHandler.generateInfoForCapture(CI, *CV, CurInfo, PartialStruct);
9536 MappedVarSet.insert(
nullptr);
9537 if (CurInfo.BasePointers.empty() && !PartialStruct.Base.isValid())
9538 MEHandler.generateDefaultMapInfo(*CI, **RI, *CV, CurInfo);
9542 MEHandler.generateInfoForLambdaCaptures(CI->
getCapturedVar(), *CV,
9543 CurInfo, LambdaPointers);
9546 assert((!CurInfo.BasePointers.empty() || PartialStruct.Base.isValid()) &&
9547 "Non-existing map pointer for capture!");
9548 assert(CurInfo.BasePointers.size() == CurInfo.Pointers.size() &&
9549 CurInfo.BasePointers.size() == CurInfo.Sizes.size() &&
9550 CurInfo.BasePointers.size() == CurInfo.Types.size() &&
9551 CurInfo.BasePointers.size() == CurInfo.Mappers.size() &&
9552 "Inconsistent map information sizes!");
9556 if (PartialStruct.Base.isValid()) {
9557 CombinedInfo.append(PartialStruct.PreliminaryMapData);
9558 MEHandler.emitCombinedEntry(
9559 CombinedInfo, CurInfo.Types, PartialStruct, CI->
capturesThis(),
9560 OMPBuilder,
nullptr,
9561 !PartialStruct.PreliminaryMapData.BasePointers.empty());
9565 CombinedInfo.append(CurInfo);
9568 MEHandler.adjustMemberOfForLambdaCaptures(
9569 OMPBuilder, LambdaPointers, CombinedInfo.BasePointers,
9570 CombinedInfo.Pointers, CombinedInfo.Types);
9573 MEHandler.generateAllInfo(CombinedInfo, OMPBuilder, MappedVarSet);
9579 llvm::codegenoptions::NoDebugInfo;
9580 OMPBuilder.emitOffloadingArraysArgument(CGF.
Builder, Info.RTArgs, Info,
9584 InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
9585 InputInfo.BasePointersArray =
Address(Info.RTArgs.BasePointersArray,
9587 InputInfo.PointersArray =
9589 InputInfo.SizesArray =
9591 InputInfo.MappersArray =
9593 MapTypesArray = Info.RTArgs.MapTypesArray;
9594 MapNamesArray = Info.RTArgs.MapNamesArray;
9596 auto &&ThenGen = [&OMPRuntime, OutlinedFn, &
D, &CapturedVars,
9597 RequiresOuterTask, &CS, OffloadingMandatory,
Device,
9598 OutlinedFnID, &InputInfo, &MapTypesArray, &MapNamesArray,
9600 bool IsReverseOffloading =
Device.getInt() == OMPC_DEVICE_ancestor;
9602 if (IsReverseOffloading) {
9608 RequiresOuterTask, CS, OffloadingMandatory, CGF);
9613 unsigned NumTargetItems = InputInfo.NumberOfTargetItems;
9615 llvm::Value *BasePointersArray =
9616 InputInfo.BasePointersArray.emitRawPointer(CGF);
9617 llvm::Value *PointersArray = InputInfo.PointersArray.emitRawPointer(CGF);
9618 llvm::Value *SizesArray = InputInfo.SizesArray.emitRawPointer(CGF);
9619 llvm::Value *MappersArray = InputInfo.MappersArray.emitRawPointer(CGF);
9621 auto &&EmitTargetCallFallbackCB =
9622 [&OMPRuntime, OutlinedFn, &
D, &CapturedVars, RequiresOuterTask, &CS,
9623 OffloadingMandatory, &CGF](llvm::OpenMPIRBuilder::InsertPointTy IP)
9624 -> llvm::OpenMPIRBuilder::InsertPointTy {
9627 RequiresOuterTask, CS, OffloadingMandatory, CGF);
9633 llvm::Value *NumThreads =
9636 llvm::Value *NumIterations =
9639 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
9642 llvm::OpenMPIRBuilder::TargetDataRTArgs RTArgs(
9643 BasePointersArray, PointersArray, SizesArray, MapTypesArray,
9644 nullptr , MappersArray, MapNamesArray);
9646 llvm::OpenMPIRBuilder::TargetKernelArgs Args(
9647 NumTargetItems, RTArgs, NumIterations, NumTeams, NumThreads,
9648 DynCGGroupMem, HasNoWait);
9651 CGF.
Builder, OutlinedFn, OutlinedFnID, EmitTargetCallFallbackCB, Args,
9652 DeviceID, RTLoc, AllocaIP));
9655 if (RequiresOuterTask)
9670 [&OMPRuntime, OutlinedFn, &
D, &CapturedVars, RequiresOuterTask, &CS,
9673 RequiresOuterTask, CS, OffloadingMandatory, CGF);
9676 if (RequiresOuterTask) {
9677 CodeGenFunction::OMPTargetDataInfo InputInfo;
9686 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID,
const Expr *IfCond,
9687 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
9694 const bool OffloadingMandatory = !
CGM.
getLangOpts().OpenMPIsTargetDevice &&
9697 assert((OffloadingMandatory || OutlinedFn) &&
"Invalid outlined function!");
9699 const bool RequiresOuterTask =
9715 llvm::Value *MapTypesArray =
nullptr;
9716 llvm::Value *MapNamesArray =
nullptr;
9718 auto &&TargetThenGen = [
this, OutlinedFn, &
D, &CapturedVars,
9719 RequiresOuterTask, &CS, OffloadingMandatory,
Device,
9720 OutlinedFnID, &InputInfo, &MapTypesArray,
9724 RequiresOuterTask, CS, OffloadingMandatory,
9725 Device, OutlinedFnID, InputInfo, MapTypesArray,
9726 MapNamesArray, SizeEmitter, CGF,
CGM);
9729 auto &&TargetElseGen =
9730 [
this, OutlinedFn, &
D, &CapturedVars, RequiresOuterTask, &CS,
9733 CS, OffloadingMandatory, CGF);
9742 emitIfClause(CGF, IfCond, TargetThenGen, TargetElseGen);
9754 StringRef ParentName) {
9759 bool RequiresDeviceCodegen =
9760 isa<OMPExecutableDirective>(S) &&
9762 cast<OMPExecutableDirective>(S)->getDirectiveKind());
9764 if (RequiresDeviceCodegen) {
9765 const auto &
E = *cast<OMPExecutableDirective>(S);
9772 if (!
OMPBuilder.OffloadInfoManager.hasTargetRegionEntryInfo(EntryInfo))
9775 switch (
E.getDirectiveKind()) {
9778 cast<OMPTargetDirective>(
E));
9780 case OMPD_target_parallel:
9782 CGM, ParentName, cast<OMPTargetParallelDirective>(
E));
9784 case OMPD_target_teams:
9786 CGM, ParentName, cast<OMPTargetTeamsDirective>(
E));
9788 case OMPD_target_teams_distribute:
9790 CGM, ParentName, cast<OMPTargetTeamsDistributeDirective>(
E));
9792 case OMPD_target_teams_distribute_simd:
9794 CGM, ParentName, cast<OMPTargetTeamsDistributeSimdDirective>(
E));
9796 case OMPD_target_parallel_for:
9798 CGM, ParentName, cast<OMPTargetParallelForDirective>(
E));
9800 case OMPD_target_parallel_for_simd:
9802 CGM, ParentName, cast<OMPTargetParallelForSimdDirective>(
E));
9804 case OMPD_target_simd:
9806 CGM, ParentName, cast<OMPTargetSimdDirective>(
E));
9808 case OMPD_target_teams_distribute_parallel_for:
9811 cast<OMPTargetTeamsDistributeParallelForDirective>(
E));
9813 case OMPD_target_teams_distribute_parallel_for_simd:
9817 cast<OMPTargetTeamsDistributeParallelForSimdDirective>(
E));
9819 case OMPD_target_teams_loop:
9821 CGM, ParentName, cast<OMPTargetTeamsGenericLoopDirective>(
E));
9823 case OMPD_target_parallel_loop:
9825 CGM, ParentName, cast<OMPTargetParallelGenericLoopDirective>(
E));
9829 case OMPD_parallel_for:
9830 case OMPD_parallel_master:
9831 case OMPD_parallel_sections:
9833 case OMPD_parallel_for_simd:
9835 case OMPD_cancellation_point:
9837 case OMPD_threadprivate:
9848 case OMPD_taskyield:
9851 case OMPD_taskgroup:
9857 case OMPD_target_data:
9858 case OMPD_target_exit_data:
9859 case OMPD_target_enter_data:
9860 case OMPD_distribute:
9861 case OMPD_distribute_simd:
9862 case OMPD_distribute_parallel_for:
9863 case OMPD_distribute_parallel_for_simd:
9864 case OMPD_teams_distribute:
9865 case OMPD_teams_distribute_simd:
9866 case OMPD_teams_distribute_parallel_for:
9867 case OMPD_teams_distribute_parallel_for_simd:
9868 case OMPD_target_update:
9869 case OMPD_declare_simd:
9870 case OMPD_declare_variant:
9871 case OMPD_begin_declare_variant:
9872 case OMPD_end_declare_variant:
9873 case OMPD_declare_target:
9874 case OMPD_end_declare_target:
9875 case OMPD_declare_reduction:
9876 case OMPD_declare_mapper:
9878 case OMPD_taskloop_simd:
9879 case OMPD_master_taskloop:
9880 case OMPD_master_taskloop_simd:
9881 case OMPD_parallel_master_taskloop:
9882 case OMPD_parallel_master_taskloop_simd:
9884 case OMPD_metadirective:
9887 llvm_unreachable(
"Unknown target directive for OpenMP device codegen.");
9892 if (
const auto *
E = dyn_cast<OMPExecutableDirective>(S)) {
9893 if (!
E->hasAssociatedStmt() || !
E->getAssociatedStmt())
9901 if (
const auto *L = dyn_cast<LambdaExpr>(S))
9905 for (
const Stmt *II : S->children())
9910 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
9911 OMPDeclareTargetDeclAttr::getDeviceType(VD);
9915 if (!IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_NoHost)
9918 if (IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_Host)
9927 if (
const auto *FD = dyn_cast<FunctionDecl>(GD.
getDecl()))
9936 if (
const auto *FD = dyn_cast<FunctionDecl>(VD)) {
9945 return !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD) &&
9963 StringRef ParentName =
9968 StringRef ParentName =
9975 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
9976 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(
9978 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
9979 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
9980 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
9989 llvm::Constant *Addr) {
9994 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
9995 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
9999 if (Res && *Res != OMPDeclareTargetDeclAttr::MT_Link &&
10014 auto LinkageForVariable = [&VD,
this]() {
10018 std::vector<llvm::GlobalVariable *> GeneratedRefs;
10031 for (
auto *ref : GeneratedRefs)
10036 if (isa<FunctionDecl>(GD.
getDecl()) ||
10037 isa<OMPDeclareReductionDecl>(GD.
getDecl()))
10045 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
10046 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
10049 if ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
10050 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
10054 assert((*Res == OMPDeclareTargetDeclAttr::MT_Link ||
10055 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
10056 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
10058 "Expected link clause or to clause with unified memory.");
10067 " Expected target-based directive.");
10071 for (
const OMPClause *Clause :
D->clauselists()) {
10072 if (Clause->getClauseKind() == OMPC_unified_shared_memory) {
10074 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(
true);
10075 }
else if (
const auto *AC =
10076 dyn_cast<OMPAtomicDefaultMemOrderClause>(Clause)) {
10077 switch (AC->getAtomicDefaultMemOrderKind()) {
10078 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_acq_rel:
10081 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_seq_cst:
10084 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_relaxed:
10100 if (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())
10102 const auto *A = VD->
getAttr<OMPAllocateDeclAttr>();
10103 switch(A->getAllocatorType()) {
10104 case OMPAllocateDeclAttr::OMPNullMemAlloc:
10105 case OMPAllocateDeclAttr::OMPDefaultMemAlloc:
10107 case OMPAllocateDeclAttr::OMPLargeCapMemAlloc:
10108 case OMPAllocateDeclAttr::OMPCGroupMemAlloc:
10109 case OMPAllocateDeclAttr::OMPHighBWMemAlloc:
10110 case OMPAllocateDeclAttr::OMPLowLatMemAlloc:
10111 case OMPAllocateDeclAttr::OMPThreadMemAlloc:
10112 case OMPAllocateDeclAttr::OMPConstMemAlloc:
10113 case OMPAllocateDeclAttr::OMPPTeamMemAlloc:
10116 case OMPAllocateDeclAttr::OMPUserDefinedMemAlloc:
10117 llvm_unreachable(
"Expected predefined allocator for the variables with the "
10118 "static storage.");
10145 const auto *
D = cast<FunctionDecl>(GD.
getDecl());
10148 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(
D)) {
10150 if (
auto *F = dyn_cast_or_null<llvm::Function>(
10152 return !F->isDeclaration();
10164 llvm::Function *OutlinedFn,
10173 llvm::Value *Args[] = {
10175 CGF.
Builder.getInt32(CapturedVars.size()),
10178 RealArgs.append(std::begin(Args), std::end(Args));
10179 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
10181 llvm::FunctionCallee RTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
10187 const Expr *NumTeams,
10188 const Expr *ThreadLimit,
10195 llvm::Value *NumTeamsVal =
10201 llvm::Value *ThreadLimitVal =
10208 llvm::Value *PushNumTeamsArgs[] = {RTLoc,
getThreadID(CGF,
Loc), NumTeamsVal,
10216 const Expr *ThreadLimit,
10219 llvm::Value *ThreadLimitVal =
10244 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
10246 llvm::Value *IfCondVal =
nullptr;
10251 llvm::Value *DeviceID =
nullptr;
10256 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
10260 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
10261 auto GenMapInfoCB =
10262 [&](InsertPointTy CodeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {
10263 CGF.
Builder.restoreIP(CodeGenIP);
10265 MappableExprsHandler MEHandler(
D, CGF);
10266 MEHandler.generateAllInfo(CombinedInfo,
OMPBuilder);
10268 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
10272 llvm::codegenoptions::NoDebugInfo) {
10273 CombinedInfo.Names.resize(CombinedInfo.Exprs.size());
10274 llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),
10278 return CombinedInfo;
10280 using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;
10281 auto BodyCB = [&](InsertPointTy CodeGenIP, BodyGenTy BodyGenType) {
10282 CGF.
Builder.restoreIP(CodeGenIP);
10283 switch (BodyGenType) {
10284 case BodyGenTy::Priv:
10288 case BodyGenTy::DupNoPriv:
10294 case BodyGenTy::NoPriv:
10301 return InsertPointTy(CGF.
Builder.GetInsertBlock(),
10302 CGF.
Builder.GetInsertPoint());
10305 auto DeviceAddrCB = [&](
unsigned int I, llvm::Value *NewDecl) {
10306 if (
const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
10311 auto CustomMapperCB = [&](
unsigned int I) {
10312 llvm::Value *MFunc =
nullptr;
10313 if (CombinedInfo.Mappers[I]) {
10314 Info.HasMapper =
true;
10316 cast<OMPDeclareMapperDecl>(CombinedInfo.Mappers[I]));
10326 InsertPointTy CodeGenIP(CGF.
Builder.GetInsertBlock(),
10327 CGF.
Builder.GetInsertPoint());
10328 llvm::OpenMPIRBuilder::LocationDescription OmpLoc(CodeGenIP);
10330 OmpLoc, AllocaIP, CodeGenIP, DeviceID, IfCondVal, Info, GenMapInfoCB,
10331 nullptr, BodyCB, DeviceAddrCB, CustomMapperCB, RTLoc));
10340 assert((isa<OMPTargetEnterDataDirective>(
D) ||
10341 isa<OMPTargetExitDataDirective>(
D) ||
10342 isa<OMPTargetUpdateDirective>(
D)) &&
10343 "Expecting either target enter, exit data, or update directives.");
10346 llvm::Value *MapTypesArray =
nullptr;
10347 llvm::Value *MapNamesArray =
nullptr;
10349 auto &&ThenGen = [
this, &
D,
Device, &InputInfo, &MapTypesArray,
10352 llvm::Value *DeviceID =
nullptr;
10357 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
10361 llvm::Constant *PointerNum =
10368 {RTLoc, DeviceID, PointerNum,
10377 RuntimeFunction RTLFn;
10378 switch (
D.getDirectiveKind()) {
10379 case OMPD_target_enter_data:
10380 RTLFn = HasNowait ? OMPRTL___tgt_target_data_begin_nowait_mapper
10381 : OMPRTL___tgt_target_data_begin_mapper;
10383 case OMPD_target_exit_data:
10384 RTLFn = HasNowait ? OMPRTL___tgt_target_data_end_nowait_mapper
10385 : OMPRTL___tgt_target_data_end_mapper;
10387 case OMPD_target_update:
10388 RTLFn = HasNowait ? OMPRTL___tgt_target_data_update_nowait_mapper
10389 : OMPRTL___tgt_target_data_update_mapper;
10391 case OMPD_parallel:
10393 case OMPD_parallel_for:
10394 case OMPD_parallel_master:
10395 case OMPD_parallel_sections:
10396 case OMPD_for_simd:
10397 case OMPD_parallel_for_simd:
10399 case OMPD_cancellation_point:
10401 case OMPD_threadprivate:
10402 case OMPD_allocate:
10407 case OMPD_sections:
10411 case OMPD_critical:
10412 case OMPD_taskyield:
10414 case OMPD_taskwait:
10415 case OMPD_taskgroup:
10421 case OMPD_target_data:
10422 case OMPD_distribute:
10423 case OMPD_distribute_simd:
10424 case OMPD_distribute_parallel_for:
10425 case OMPD_distribute_parallel_for_simd:
10426 case OMPD_teams_distribute:
10427 case OMPD_teams_distribute_simd:
10428 case OMPD_teams_distribute_parallel_for:
10429 case OMPD_teams_distribute_parallel_for_simd:
10430 case OMPD_declare_simd:
10431 case OMPD_declare_variant:
10432 case OMPD_begin_declare_variant:
10433 case OMPD_end_declare_variant:
10434 case OMPD_declare_target:
10435 case OMPD_end_declare_target:
10436 case OMPD_declare_reduction:
10437 case OMPD_declare_mapper:
10438 case OMPD_taskloop:
10439 case OMPD_taskloop_simd:
10440 case OMPD_master_taskloop:
10441 case OMPD_master_taskloop_simd:
10442 case OMPD_parallel_master_taskloop:
10443 case OMPD_parallel_master_taskloop_simd:
10445 case OMPD_target_simd:
10446 case OMPD_target_teams_distribute:
10447 case OMPD_target_teams_distribute_simd:
10448 case OMPD_target_teams_distribute_parallel_for:
10449 case OMPD_target_teams_distribute_parallel_for_simd:
10450 case OMPD_target_teams:
10451 case OMPD_target_parallel:
10452 case OMPD_target_parallel_for:
10453 case OMPD_target_parallel_for_simd:
10454 case OMPD_requires:
10455 case OMPD_metadirective:
10458 llvm_unreachable(
"Unexpected standalone target data directive.");
10462 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.
Int32Ty));
10463 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.
VoidPtrTy));
10464 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.
Int32Ty));
10465 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.
VoidPtrTy));
10472 auto &&TargetThenGen = [
this, &ThenGen, &
D, &InputInfo, &MapTypesArray,
10476 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
10479 MappableExprsHandler MEHandler(
D, CGF);
10480 MEHandler.generateAllInfo(CombinedInfo,
OMPBuilder);
10489 llvm::codegenoptions::NoDebugInfo;
10502 MapTypesArray = Info.RTArgs.MapTypesArray;
10503 MapNamesArray = Info.RTArgs.MapNamesArray;
10504 if (RequiresOuterTask)
10530struct ParamAttrTy {
10532 llvm::APSInt StrideOrArg;
10533 llvm::APSInt Alignment;
10534 bool HasVarStride =
false;
10567 unsigned Offset = 0;
10568 if (
const auto *MD = dyn_cast<CXXMethodDecl>(FD)) {
10569 if (ParamAttrs[Offset].Kind ==
Vector)
10570 CDT =
C.getPointerType(
C.getRecordType(MD->
getParent()));
10575 if (ParamAttrs[I + Offset].Kind ==
Vector) {
10587 return C.getTypeSize(CDT);
10595 llvm::raw_svector_ostream Out(Buffer);
10596 for (
const auto &ParamAttr : ParamAttrs) {
10597 switch (ParamAttr.Kind) {
10617 if (ParamAttr.HasVarStride)
10618 Out <<
"s" << ParamAttr.StrideOrArg;
10619 else if (ParamAttr.Kind == Linear || ParamAttr.Kind == LinearRef ||
10620 ParamAttr.Kind == LinearUVal || ParamAttr.Kind == LinearVal) {
10623 if (ParamAttr.StrideOrArg < 0)
10624 Out <<
'n' << -ParamAttr.StrideOrArg;
10625 else if (ParamAttr.StrideOrArg != 1)
10626 Out << ParamAttr.StrideOrArg;
10629 if (!!ParamAttr.Alignment)
10630 Out <<
'a' << ParamAttr.Alignment;
10633 return std::string(Out.str());
10638 const llvm::APSInt &VLENVal,
10640 OMPDeclareSimdDeclAttr::BranchStateTy State) {
10643 unsigned VecRegSize;
10645 ISADataTy ISAData[] = {
10661 case OMPDeclareSimdDeclAttr::BS_Undefined:
10662 Masked.push_back(
'N');
10663 Masked.push_back(
'M');
10665 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
10666 Masked.push_back(
'N');
10668 case OMPDeclareSimdDeclAttr::BS_Inbranch:
10669 Masked.push_back(
'M');
10672 for (
char Mask : Masked) {
10673 for (
const ISADataTy &
Data : ISAData) {
10675 llvm::raw_svector_ostream Out(Buffer);
10676 Out <<
"_ZGV" <<
Data.ISA << Mask;
10679 assert(NumElts &&
"Non-zero simdlen/cdtsize expected");
10680 Out << llvm::APSInt::getUnsigned(
Data.VecRegSize / NumElts);
10685 Out <<
'_' << Fn->getName();
10686 Fn->addFnAttr(Out.str());
10704 if (Kind == ParamKindTy::Uniform)
10707 if (Kind == ParamKindTy::LinearUVal || Kind == ParamKindTy::LinearRef)
10710 if ((Kind == ParamKindTy::Linear || Kind == ParamKindTy::LinearVal) &&
10720 unsigned Size =
C.getTypeSize(QT);
10723 if (Size != 8 && Size != 16 && Size != 32 && Size != 64 && Size != 128)
10747 return C.getTypeSize(PTy);
10750 return C.getTypeSize(QT);
10752 return C.getTypeSize(
C.getUIntPtrType());
10758static std::tuple<unsigned, unsigned, bool>
10764 bool OutputBecomesInput =
false;
10768 Sizes.push_back(
getAArch64LS(RetType, ParamKindTy::Vector,
C));
10770 OutputBecomesInput =
true;
10777 assert(!Sizes.empty() &&
"Unable to determine NDS and WDS.");
10780 assert(llvm::all_of(Sizes,
10781 [](
unsigned Size) {
10782 return Size == 8 || Size == 16 || Size == 32 ||
10783 Size == 64 || Size == 128;
10787 return std::make_tuple(*std::min_element(std::begin(Sizes), std::end(Sizes)),
10788 *std::max_element(std::begin(Sizes), std::end(Sizes)),
10789 OutputBecomesInput);
10795template <
typename T>
10797 char ISA, StringRef ParSeq,
10798 StringRef MangledName,
bool OutputBecomesInput,
10799 llvm::Function *Fn) {
10801 llvm::raw_svector_ostream Out(Buffer);
10802 Out << Prefix << ISA << LMask << VLEN;
10803 if (OutputBecomesInput)
10805 Out << ParSeq <<
"_" << MangledName;
10806 Fn->addFnAttr(Out.str());
10812 StringRef Prefix,
char ISA,
10813 StringRef ParSeq, StringRef MangledName,
10814 bool OutputBecomesInput,
10815 llvm::Function *Fn) {
10819 OutputBecomesInput, Fn);
10821 OutputBecomesInput, Fn);
10825 OutputBecomesInput, Fn);
10827 OutputBecomesInput, Fn);
10831 OutputBecomesInput, Fn);
10833 OutputBecomesInput, Fn);
10838 OutputBecomesInput, Fn);
10841 llvm_unreachable(
"Scalar type is too wide.");
10849 OMPDeclareSimdDeclAttr::BranchStateTy State, StringRef MangledName,
10850 char ISA,
unsigned VecRegSize, llvm::Function *Fn,
SourceLocation SLoc) {
10854 const unsigned NDS = std::get<0>(
Data);
10855 const unsigned WDS = std::get<1>(
Data);
10856 const bool OutputBecomesInput = std::get<2>(
Data);
10860 if (UserVLEN == 1) {
10863 "The clause simdlen(1) has no effect when targeting aarch64.");
10870 if (ISA ==
'n' && UserVLEN && !llvm::isPowerOf2_32(UserVLEN)) {
10873 "power of 2 when targeting Advanced SIMD.");
10880 if (ISA ==
's' && UserVLEN != 0) {
10881 if ((UserVLEN * WDS > 2048) || (UserVLEN * WDS % 128 != 0)) {
10884 "lanes in the architectural constraints "
10885 "for SVE (min is 128-bit, max is "
10886 "2048-bit, by steps of 128-bit)");
10894 StringRef Prefix =
"_ZGV";
10900 OutputBecomesInput, Fn);
10902 assert(ISA ==
'n' &&
"Expected ISA either 's' or 'n'.");
10906 case OMPDeclareSimdDeclAttr::BS_Undefined:
10908 OutputBecomesInput, Fn);
10910 OutputBecomesInput, Fn);
10912 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
10914 OutputBecomesInput, Fn);
10916 case OMPDeclareSimdDeclAttr::BS_Inbranch:
10918 OutputBecomesInput, Fn);
10928 OutputBecomesInput, Fn);
10930 assert(ISA ==
'n' &&
"Expected ISA either 's' or 'n'.");
10935 case OMPDeclareSimdDeclAttr::BS_Undefined:
10937 OutputBecomesInput, Fn);
10939 OutputBecomesInput, Fn);
10941 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
10943 OutputBecomesInput, Fn);
10945 case OMPDeclareSimdDeclAttr::BS_Inbranch:
10947 OutputBecomesInput, Fn);
10955 llvm::Function *Fn) {
10960 llvm::DenseMap<const Decl *, unsigned> ParamPositions;
10961 if (isa<CXXMethodDecl>(FD))
10962 ParamPositions.try_emplace(FD, 0);
10963 unsigned ParamPos = ParamPositions.size();
10965 ParamPositions.try_emplace(
P->getCanonicalDecl(), ParamPos);
10971 for (
const Expr *
E :
Attr->uniforms()) {
10974 if (isa<CXXThisExpr>(
E)) {
10975 Pos = ParamPositions[FD];
10977 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(
E)->getDecl())
10978 ->getCanonicalDecl();
10979 auto It = ParamPositions.find(PVD);
10980 assert(It != ParamPositions.end() &&
"Function parameter not found");
10983 ParamAttrs[Pos].Kind = Uniform;
10986 auto *NI =
Attr->alignments_begin();
10987 for (
const Expr *
E :
Attr->aligneds()) {
10991 if (isa<CXXThisExpr>(
E)) {
10992 Pos = ParamPositions[FD];
10995 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(
E)->getDecl())
10996 ->getCanonicalDecl();
10997 auto It = ParamPositions.find(PVD);
10998 assert(It != ParamPositions.end() &&
"Function parameter not found");
11000 ParmTy = PVD->getType();
11002 ParamAttrs[Pos].Alignment =
11004 ? (*NI)->EvaluateKnownConstInt(
C)
11005 : llvm::APSInt::getUnsigned(
11006 C.toCharUnitsFromBits(
C.getOpenMPDefaultSimdAlign(ParmTy))
11011 auto *SI =
Attr->steps_begin();
11012 auto *MI =
Attr->modifiers_begin();
11013 for (
const Expr *
E :
Attr->linears()) {
11016 bool IsReferenceType =
false;
11019 unsigned PtrRescalingFactor = 1;
11020 if (isa<CXXThisExpr>(
E)) {
11021 Pos = ParamPositions[FD];
11022 auto *
P = cast<PointerType>(
E->
getType());
11027 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(
E)->getDecl())
11028 ->getCanonicalDecl();
11029 auto It = ParamPositions.find(PVD);
11030 assert(It != ParamPositions.end() &&
"Function parameter not found");
11032 if (
auto *
P = dyn_cast<PointerType>(PVD->getType()))
11036 else if (PVD->getType()->isReferenceType()) {
11037 IsReferenceType =
true;
11038 PtrRescalingFactor =
11044 ParamAttrTy &ParamAttr = ParamAttrs[Pos];
11045 if (*MI == OMPC_LINEAR_ref)
11046 ParamAttr.Kind = LinearRef;
11047 else if (*MI == OMPC_LINEAR_uval)
11048 ParamAttr.Kind = LinearUVal;
11049 else if (IsReferenceType)
11050 ParamAttr.Kind = LinearVal;
11052 ParamAttr.Kind = Linear;
11054 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(1);
11058 if (
const auto *DRE =
11059 cast<DeclRefExpr>((*SI)->IgnoreParenImpCasts())) {
11060 if (
const auto *StridePVD =
11061 dyn_cast<ParmVarDecl>(DRE->getDecl())) {
11062 ParamAttr.HasVarStride =
true;
11063 auto It = ParamPositions.find(StridePVD->getCanonicalDecl());
11064 assert(It != ParamPositions.end() &&
11065 "Function parameter not found");
11066 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(It->second);
11070 ParamAttr.StrideOrArg =
Result.Val.getInt();
11076 if (!ParamAttr.HasVarStride &&
11077 (ParamAttr.Kind == Linear || ParamAttr.Kind == LinearRef))
11078 ParamAttr.StrideOrArg = ParamAttr.StrideOrArg * PtrRescalingFactor;
11082 llvm::APSInt VLENVal;
11084 const Expr *VLENExpr =
Attr->getSimdlen();
11089 OMPDeclareSimdDeclAttr::BranchStateTy State =
Attr->getBranchState();
11092 }
else if (
CGM.
getTriple().getArch() == llvm::Triple::aarch64) {
11093 unsigned VLEN = VLENVal.getExtValue();
11094 StringRef MangledName = Fn->getName();
11097 MangledName,
's', 128, Fn, ExprLoc);
11100 MangledName,
'n', 128, Fn, ExprLoc);
11111 static const int DoacrossFinArgs = 2;
11114 llvm::FunctionCallee RTLFn;
11115 llvm::Value *Args[DoacrossFinArgs];
11118 DoacrossCleanupTy(llvm::FunctionCallee RTLFn,
11121 assert(CallArgs.size() == DoacrossFinArgs);
11122 std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args));
11139 QualType Int64Ty =
C.getIntTypeForBitwidth(64,
true);
11147 RD =
C.buildImplicitRecord(
"kmp_dim");
11157 llvm::APInt Size(32, NumIterations.size());
11163 enum { LowerFD = 0, UpperFD, StrideFD };
11165 for (
unsigned I = 0,
E = NumIterations.size(); I <
E; ++I) {
11170 DimsLVal, *std::next(RD->
field_begin(), UpperFD));
11172 CGF.
EmitScalarExpr(NumIterations[I]), NumIterations[I]->getType(),
11173 Int64Ty, NumIterations[I]->getExprLoc());
11177 DimsLVal, *std::next(RD->
field_begin(), StrideFD));
11184 llvm::Value *Args[] = {
11187 llvm::ConstantInt::getSigned(
CGM.
Int32Ty, NumIterations.size()),
11192 llvm::FunctionCallee RTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
11195 llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = {
11197 llvm::FunctionCallee FiniRTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
11203template <
typename T>
11205 const T *
C, llvm::Value *ULoc,
11206 llvm::Value *ThreadID) {
11209 llvm::APInt Size(32,
C->getNumLoops());
11213 for (
unsigned I = 0,
E =
C->getNumLoops(); I <
E; ++I) {
11214 const Expr *CounterVal =
C->getLoopData(I);
11215 assert(CounterVal);
11222 llvm::Value *Args[] = {
11225 llvm::FunctionCallee RTLFn;
11227 OMPDoacrossKind<T> ODK;
11228 if (ODK.isSource(
C)) {
11230 OMPRTL___kmpc_doacross_post);
11232 assert(ODK.isSink(
C) &&
"Expect sink modifier.");
11234 OMPRTL___kmpc_doacross_wait);
11241 return EmitDoacrossOrdered<OMPDependClause>(
11248 return EmitDoacrossOrdered<OMPDoacrossClause>(
11254 llvm::FunctionCallee Callee,
11256 assert(
Loc.
isValid() &&
"Outlined function call location must be valid.");
11259 if (
auto *Fn = dyn_cast<llvm::Function>(Callee.getCallee())) {
11260 if (Fn->doesNotThrow()) {
11275 if (
const auto *FD = dyn_cast<FunctionDecl>(
D))
11276 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(FD))
11282 const VarDecl *TargetParam)
const {
11289 const Expr *Allocator) {
11290 llvm::Value *AllocVal;
11297 Allocator->getExprLoc());
11300 AllocVal = llvm::Constant::getNullValue(
11310 if (!AllocateAlignment)
11313 return llvm::ConstantInt::get(
CGM.
SizeTy, AllocateAlignment->getQuantity());
11326 auto I = UntiedData.find(VD);
11327 if (I != UntiedData.end()) {
11328 UntiedAddr = I->second.first;
11329 UntiedRealAddr = I->second.second;
11333 if (CVD->
hasAttr<OMPAllocateDeclAttr>()) {
11342 Size = CGF.
Builder.CreateNUWAdd(
11351 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
11352 const Expr *Allocator = AA->getAllocator();
11356 Args.push_back(ThreadID);
11358 Args.push_back(Alignment);
11359 Args.push_back(Size);
11360 Args.push_back(AllocVal);
11361 llvm::omp::RuntimeFunction FnID =
11362 Alignment ? OMPRTL___kmpc_aligned_alloc : OMPRTL___kmpc_alloc;
11366 llvm::FunctionCallee FiniRTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
11376 llvm::FunctionCallee RTLFn;
11379 const Expr *AllocExpr;
11382 OMPAllocateCleanupTy(llvm::FunctionCallee RTLFn,
11384 const Expr *AllocExpr)
11385 : RTLFn(RTLFn), LocEncoding(LocEncoding), Addr(Addr),
11386 AllocExpr(AllocExpr) {}
11390 llvm::Value *Args[3];
11396 Args[2] = AllocVal;
11404 CGF.
EHStack.pushCleanup<OMPAllocateCleanupTy>(
11406 VDAddr, Allocator);
11407 if (UntiedRealAddr.
isValid())
11410 Region->emitUntiedSwitch(CGF);
11427 assert(CGM.
getLangOpts().OpenMP &&
"Not in OpenMP mode.");
11433 for (
const Stmt *Ref :
C->private_refs()) {
11434 const auto *SimpleRefExpr = cast<Expr>(Ref)->IgnoreParenImpCasts();
11436 if (
const auto *DRE = dyn_cast<DeclRefExpr>(SimpleRefExpr)) {
11437 VD = DRE->getDecl();
11439 const auto *ME = cast<MemberExpr>(SimpleRefExpr);
11440 assert((ME->isImplicitCXXThis() ||
11441 isa<CXXThisExpr>(ME->getBase()->IgnoreParenImpCasts())) &&
11442 "Expected member of current class.");
11443 VD = ME->getMemberDecl();
11459 std::pair<Address, Address>> &LocalVars)
11460 :
CGM(CGF.
CGM), NeedToPush(!LocalVars.empty()) {
11477 return llvm::any_of(
11482void CGOpenMPRuntime::LastprivateConditionalRAII::tryToDisableInnerAnalysis(
11492 const CapturedStmt *CS = S.getCapturedStmt(CaptureRegions.front());
11500 for (
const Expr *Ref :
C->varlists()) {
11501 if (!Ref->getType()->isScalarType())
11503 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
11506 NeedToCheckForLPCs.insert(DRE->getDecl());
11510 for (
const Expr *Ref :
C->varlists()) {
11511 if (!Ref->getType()->isScalarType())
11513 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
11516 NeedToCheckForLPCs.insert(DRE->getDecl());
11520 for (
const Expr *Ref :
C->varlists()) {
11521 if (!Ref->getType()->isScalarType())
11523 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
11526 NeedToCheckForLPCs.insert(DRE->getDecl());
11530 for (
const Expr *Ref :
C->varlists()) {
11531 if (!Ref->getType()->isScalarType())
11533 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
11536 NeedToCheckForLPCs.insert(DRE->getDecl());
11540 for (
const Expr *Ref :
C->varlists()) {
11541 if (!Ref->getType()->isScalarType())
11543 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
11546 NeedToCheckForLPCs.insert(DRE->getDecl());
11549 for (
const Decl *VD : NeedToCheckForLPCs) {
11550 for (
const LastprivateConditionalData &
Data :
11552 if (
Data.DeclToUniqueName.count(VD) > 0) {
11553 if (!
Data.Disabled)
11554 NeedToAddForLPCsAsDisabled.insert(VD);
11561CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
11564 Action((
CGM.getLangOpts().OpenMP >= 50 &&
11567 return C->getKind() ==
11568 OMPC_LASTPRIVATE_conditional;
11570 ? ActionToDo::PushAsLastprivateConditional
11571 : ActionToDo::DoNotPush) {
11573 if (
CGM.
getLangOpts().OpenMP < 50 || Action == ActionToDo::DoNotPush)
11575 assert(Action == ActionToDo::PushAsLastprivateConditional &&
11576 "Expected a push action.");
11580 if (
C->getKind() != OMPC_LASTPRIVATE_conditional)
11583 for (
const Expr *Ref :
C->varlists()) {
11584 Data.DeclToUniqueName.insert(std::make_pair(
11585 cast<DeclRefExpr>(Ref->IgnoreParenImpCasts())->getDecl(),
11589 Data.IVLVal = IVLVal;
11593CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
11595 :
CGM(CGF.
CGM), Action(ActionToDo::DoNotPush) {
11596 assert(CGM.
getLangOpts().OpenMP &&
"Not in OpenMP mode.");
11600 tryToDisableInnerAnalysis(S, NeedToAddForLPCsAsDisabled);
11601 if (!NeedToAddForLPCsAsDisabled.empty()) {
11602 Action = ActionToDo::DisableLastprivateConditional;
11603 LastprivateConditionalData &
Data =
11605 for (
const Decl *VD : NeedToAddForLPCsAsDisabled)
11608 Data.Disabled =
true;
11621 if (Action == ActionToDo::DisableLastprivateConditional) {
11623 "Expected list of disabled private vars.");
11626 if (Action == ActionToDo::PushAsLastprivateConditional) {
11629 "Expected list of lastprivate conditional vars.");
11644 auto VI = I->getSecond().find(VD);
11645 if (VI == I->getSecond().end()) {
11646 RecordDecl *RD =
C.buildImplicitRecord(
"lasprivate.conditional");
11651 NewType =
C.getRecordType(RD);
11654 I->getSecond().try_emplace(VD, NewType, VDField, FiredField, BaseLVal);
11656 NewType = std::get<0>(VI->getSecond());
11657 VDField = std::get<1>(VI->getSecond());
11658 FiredField = std::get<2>(VI->getSecond());
11659 BaseLVal = std::get<3>(VI->getSecond());
11671class LastprivateConditionalRefChecker final
11674 const Expr *FoundE =
nullptr;
11675 const Decl *FoundD =
nullptr;
11676 StringRef UniqueDeclName;
11678 llvm::Function *FoundFn =
nullptr;
11684 llvm::reverse(LPM)) {
11685 auto It =
D.DeclToUniqueName.find(
E->getDecl());
11686 if (It ==
D.DeclToUniqueName.end())
11691 FoundD =
E->getDecl()->getCanonicalDecl();
11692 UniqueDeclName = It->second;
11697 return FoundE ==
E;
11700 if (!CodeGenFunction::IsWrappedCXXThis(
E->getBase()))
11703 llvm::reverse(LPM)) {
11704 auto It =
D.DeclToUniqueName.find(
E->getMemberDecl());
11705 if (It ==
D.DeclToUniqueName.end())
11710 FoundD =
E->getMemberDecl()->getCanonicalDecl();
11711 UniqueDeclName = It->second;
11716 return FoundE ==
E;
11718 bool VisitStmt(
const Stmt *S) {
11719 for (
const Stmt *Child : S->children()) {
11722 if (
const auto *
E = dyn_cast<Expr>(Child))
11730 explicit LastprivateConditionalRefChecker(
11733 std::tuple<const Expr *, const Decl *, StringRef, LValue, llvm::Function *>
11734 getFoundData()
const {
11735 return std::make_tuple(FoundE, FoundD, UniqueDeclName, IVLVal, FoundFn);
11742 StringRef UniqueDeclName,
11748 llvm::Constant *LastIV =
OMPBuilder.getOrCreateInternalVariable(
11749 LLIVTy,
getName({UniqueDeclName,
"iv"}));
11750 cast<llvm::GlobalVariable>(LastIV)->setAlignment(
11757 llvm::GlobalVariable *
Last =
OMPBuilder.getOrCreateInternalVariable(
11759 cast<llvm::GlobalVariable>(
Last)->setAlignment(
11773 auto &&CodeGen = [&LastIVLVal, &IVLVal, IVVal, &LVal, &LastLVal,
11779 llvm::Value *CmpRes;
11781 CmpRes = CGF.
Builder.CreateICmpSLE(LastIVVal, IVVal);
11784 "Loop iteration variable must be integer.");
11785 CmpRes = CGF.
Builder.CreateICmpULE(LastIVVal, IVVal);
11789 CGF.
Builder.CreateCondBr(CmpRes, ThenBB, ExitBB);
11810 "Aggregates are not supported in lastprivate conditional.");
11833 if (!Checker.Visit(LHS))
11835 const Expr *FoundE;
11836 const Decl *FoundD;
11837 StringRef UniqueDeclName;
11839 llvm::Function *FoundFn;
11840 std::tie(FoundE, FoundD, UniqueDeclName, IVLVal, FoundFn) =
11841 Checker.getFoundData();
11842 if (FoundFn != CGF.
CurFn) {
11847 "Lastprivate conditional is not found in outer region.");
11848 QualType StructTy = std::get<0>(It->getSecond());
11849 const FieldDecl* FiredDecl = std::get<2>(It->getSecond());
11860 FiredLVal, llvm::AtomicOrdering::Unordered,
11878 auto It = llvm::find_if(
11880 if (It ==
Range.end() || It->Fn != CGF.
CurFn)
11884 "Lastprivates must be registered already.");
11887 const CapturedStmt *CS =
D.getCapturedStmt(CaptureRegions.back());
11888 for (
const auto &Pair : It->DeclToUniqueName) {
11889 const auto *VD = cast<VarDecl>(Pair.first->getCanonicalDecl());
11892 auto I = LPCI->getSecond().find(Pair.first);
11893 assert(I != LPCI->getSecond().end() &&
11894 "Lastprivate must be rehistered already.");
11896 LValue BaseLVal = std::get<3>(I->getSecond());
11900 llvm::Value *Cmp = CGF.
Builder.CreateIsNotNull(Res);
11904 CGF.
Builder.CreateCondBr(Cmp, ThenBB, DoneBB);
11929 "Unknown lastprivate conditional variable.");
11930 StringRef UniqueName = It->second;
11931 llvm::GlobalVariable *GV =
CGM.
getModule().getNamedGlobal(UniqueName);
11945 llvm_unreachable(
"Not supported in SIMD-only mode");
11952 llvm_unreachable(
"Not supported in SIMD-only mode");
11959 bool Tied,
unsigned &NumberOfParts) {
11960 llvm_unreachable(
"Not supported in SIMD-only mode");
11965 llvm::Function *OutlinedFn,
11967 const Expr *IfCond,
11968 llvm::Value *NumThreads) {
11969 llvm_unreachable(
"Not supported in SIMD-only mode");
11975 const Expr *Hint) {
11976 llvm_unreachable(
"Not supported in SIMD-only mode");
11982 llvm_unreachable(
"Not supported in SIMD-only mode");
11988 const Expr *Filter) {
11989 llvm_unreachable(
"Not supported in SIMD-only mode");
11994 llvm_unreachable(
"Not supported in SIMD-only mode");
12000 llvm_unreachable(
"Not supported in SIMD-only mode");
12008 llvm_unreachable(
"Not supported in SIMD-only mode");
12015 llvm_unreachable(
"Not supported in SIMD-only mode");
12022 bool ForceSimpleCall) {
12023 llvm_unreachable(
"Not supported in SIMD-only mode");
12030 llvm_unreachable(
"Not supported in SIMD-only mode");
12035 llvm_unreachable(
"Not supported in SIMD-only mode");
12041 llvm_unreachable(
"Not supported in SIMD-only mode");
12047 llvm_unreachable(
"Not supported in SIMD-only mode");
12054 llvm_unreachable(
"Not supported in SIMD-only mode");
12060 llvm_unreachable(
"Not supported in SIMD-only mode");
12065 unsigned IVSize,
bool IVSigned,
12068 llvm_unreachable(
"Not supported in SIMD-only mode");
12072 llvm::Value *NumThreads,
12074 llvm_unreachable(
"Not supported in SIMD-only mode");
12078 ProcBindKind ProcBind,
12080 llvm_unreachable(
"Not supported in SIMD-only mode");
12087 llvm_unreachable(
"Not supported in SIMD-only mode");
12093 llvm_unreachable(
"Not supported in SIMD-only mode");
12098 llvm_unreachable(
"Not supported in SIMD-only mode");
12104 llvm::AtomicOrdering AO) {
12105 llvm_unreachable(
"Not supported in SIMD-only mode");
12110 llvm::Function *TaskFunction,
12112 const Expr *IfCond,
12114 llvm_unreachable(
"Not supported in SIMD-only mode");
12121 llvm_unreachable(
"Not supported in SIMD-only mode");
12128 assert(Options.SimpleReduction &&
"Only simple reduction is expected.");
12130 ReductionOps, Options);
12136 llvm_unreachable(
"Not supported in SIMD-only mode");
12141 bool IsWorksharingReduction) {
12142 llvm_unreachable(
"Not supported in SIMD-only mode");
12149 llvm_unreachable(
"Not supported in SIMD-only mode");
12154 llvm::Value *ReductionsPtr,
12156 llvm_unreachable(
"Not supported in SIMD-only mode");
12162 llvm_unreachable(
"Not supported in SIMD-only mode");
12168 llvm_unreachable(
"Not supported in SIMD-only mode");
12174 llvm_unreachable(
"Not supported in SIMD-only mode");
12179 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
12181 llvm_unreachable(
"Not supported in SIMD-only mode");
12186 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID,
const Expr *IfCond,
12187 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
12191 llvm_unreachable(
"Not supported in SIMD-only mode");
12195 llvm_unreachable(
"Not supported in SIMD-only mode");
12199 llvm_unreachable(
"Not supported in SIMD-only mode");
12209 llvm::Function *OutlinedFn,
12211 llvm_unreachable(
"Not supported in SIMD-only mode");
12215 const Expr *NumTeams,
12216 const Expr *ThreadLimit,
12218 llvm_unreachable(
"Not supported in SIMD-only mode");
12225 llvm_unreachable(
"Not supported in SIMD-only mode");
12231 llvm_unreachable(
"Not supported in SIMD-only mode");
12237 llvm_unreachable(
"Not supported in SIMD-only mode");
12242 llvm_unreachable(
"Not supported in SIMD-only mode");
12247 llvm_unreachable(
"Not supported in SIMD-only mode");
12252 const VarDecl *NativeParam)
const {
12253 llvm_unreachable(
"Not supported in SIMD-only mode");
12259 const VarDecl *TargetParam)
const {
12260 llvm_unreachable(
"Not supported in SIMD-only mode");
Provides LLVM's BitmaskEnum facility to enumeration types declared in namespace clang.
static llvm::Value * emitCopyprivateCopyFunction(CodeGenModule &CGM, llvm::Type *ArgsElemType, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps, SourceLocation Loc)
static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF, SourceLocation Loc, SmallString< 128 > &Buffer)
llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind convertCaptureClause(const VarDecl *VD)
static void emitOffloadingArrays(CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo, CGOpenMPRuntime::TargetDataInfo &Info, llvm::OpenMPIRBuilder &OMPBuilder, bool IsNonContiguous=false)
Emit the arrays used to pass the captures and map information to the offloading runtime library.
static RecordDecl * createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy, ArrayRef< PrivateDataTy > Privates)
llvm::Value * emitDynCGGroupMem(const OMPExecutableDirective &D, CodeGenFunction &CGF)
static void emitInitWithReductionInitializer(CodeGenFunction &CGF, const OMPDeclareReductionDecl *DRD, const Expr *InitOp, Address Private, Address Original, QualType Ty)
static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, Address OriginalBaseAddress, llvm::Value *Addr)
static void emitPrivatesInit(CodeGenFunction &CGF, const OMPExecutableDirective &D, Address KmpTaskSharedsPtr, LValue TDBase, const RecordDecl *KmpTaskTWithPrivatesQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef< PrivateDataTy > Privates, bool ForDup)
Emit initialization for private variables in task-based directives.
static llvm::Value * emitDestructorsFunction(CodeGenModule &CGM, SourceLocation Loc, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy)
static unsigned evaluateCDTSize(const FunctionDecl *FD, ArrayRef< ParamAttrTy > ParamAttrs)
static void EmitOMPAggregateReduction(CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar, const VarDecl *RHSVar, const llvm::function_ref< void(CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *)> &RedOpGen, const Expr *XExpr=nullptr, const Expr *EExpr=nullptr, const Expr *UpExpr=nullptr)
Emit reduction operation for each element of array (required for array sections) LHS op = RHS.
static void emitTargetCallFallback(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, CodeGenFunction &CGF)
static llvm::Value * emitReduceInitFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Emits reduction initializer function:
static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion)
static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy, llvm::PointerUnion< unsigned *, LValue * > Pos, const OMPTaskDataTy::DependData &Data, Address DependenciesArray)
static llvm::Value * emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc, const OMPTaskDataTy &Data, QualType PrivatesQTy, ArrayRef< PrivateDataTy > Privates)
Emit a privates mapping function for correct handling of private and firstprivate variables.
static llvm::Value * emitReduceCombFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N, const Expr *ReductionOp, const Expr *LHS, const Expr *RHS, const Expr *PrivateRef)
Emits reduction combiner function:
static RecordDecl * createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef< PrivateDataTy > Privates)
static llvm::Value * getAllocatorVal(CodeGenFunction &CGF, const Expr *Allocator)
Return allocator value from expression, or return a null allocator (default when no allocator specifi...
static llvm::Function * emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy, QualType SharedsPtrTy, llvm::Function *TaskFunction, llvm::Value *TaskPrivatesMap)
Emit a proxy function which accepts kmp_task_t as the second argument.
static void addAArch64VectorName(T VLEN, StringRef LMask, StringRef Prefix, char ISA, StringRef ParSeq, StringRef MangledName, bool OutputBecomesInput, llvm::Function *Fn)
static bool isAllocatableDecl(const VarDecl *VD)
static llvm::Value * getAlignmentValue(CodeGenModule &CGM, const VarDecl *VD)
Return the alignment from an allocate directive if present.
static void emitTargetCallKernelLaunch(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::Value *OutlinedFnID, CodeGenFunction::OMPTargetDataInfo &InputInfo, llvm::Value *&MapTypesArray, llvm::Value *&MapNamesArray, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter, CodeGenFunction &CGF, CodeGenModule &CGM)
static std::tuple< unsigned, unsigned, bool > getNDSWDS(const FunctionDecl *FD, ArrayRef< ParamAttrTy > ParamAttrs)
static const OMPExecutableDirective * getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D)
Check for inner distribute directive.
static std::pair< llvm::Value *, llvm::Value * > getPointerAndSize(CodeGenFunction &CGF, const Expr *E)
static const VarDecl * getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE)
static bool isTrivial(ASTContext &Ctx, const Expr *E)
Checks if the expression is constant or does not have non-trivial function calls.
static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind, bool Chunked, bool Ordered)
Map the OpenMP loop schedule to the runtime enumeration.
static void getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS, const Expr **E, int32_t &UpperBound, bool UpperBoundOnly, llvm::Value **CondVal)
Check for a num threads constant value (stored in DefaultVal), or expression (stored in E).
static llvm::Value * emitDeviceID(llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, CodeGenFunction &CGF)
static const OMPDeclareReductionDecl * getReductionInit(const Expr *ReductionOp)
Check if the combiner is a call to UDR combiner and if it is so return the UDR decl used for reductio...
static bool checkInitIsRequired(CodeGenFunction &CGF, ArrayRef< PrivateDataTy > Privates)
Check if duplication function is required for taskloops.
static bool checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD, ArrayRef< PrivateDataTy > Privates)
Checks if destructor function is required to be generated.
static llvm::TargetRegionEntryInfo getEntryInfoFromPresumedLoc(CodeGenModule &CGM, llvm::OpenMPIRBuilder &OMPBuilder, SourceLocation BeginLoc, llvm::StringRef ParentName="")
static std::string generateUniqueName(CodeGenModule &CGM, StringRef Prefix, const Expr *Ref)
Generates unique name for artificial threadprivate variables.
static void emitForStaticInitCall(CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId, llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2, const CGOpenMPRuntime::StaticRTInput &Values)
static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, LValue BaseLV)
static void getKmpAffinityType(ASTContext &C, QualType &KmpTaskAffinityInfoTy)
Builds kmp_depend_info, if it is not built yet, and builds flags type.
static void getDependTypes(ASTContext &C, QualType &KmpDependInfoTy, QualType &FlagsTy)
Builds kmp_depend_info, if it is not built yet, and builds flags type.
static llvm::Value * emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc, const OMPExecutableDirective &D, QualType KmpTaskTWithPrivatesPtrQTy, const RecordDecl *KmpTaskTWithPrivatesQTyRD, const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef< PrivateDataTy > Privates, bool WithLastIter)
Emit task_dup function (for initialization of private/firstprivate/lastprivate vars and last_iter fla...
static llvm::Value * emitReduceFiniFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Emits reduction finalizer function:
static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, QualType Type, bool EmitDeclareReductionInit, const Expr *Init, const OMPDeclareReductionDecl *DRD, Address SrcAddr=Address::invalid())
Emit initialization of arrays of complex types.
static bool getAArch64PBV(QualType QT, ASTContext &C)
Pass By Value (PBV), as defined in 3.1.2 of the AAVFABI.
static unsigned getAArch64LS(QualType QT, ParamKindTy Kind, ASTContext &C)
Computes the lane size (LS) of a return type or of an input parameter, as defined by LS(P) in 3....
static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM, const T *C, llvm::Value *ULoc, llvm::Value *ThreadID)
static RTLDependenceKindTy translateDependencyKind(OpenMPDependClauseKind K)
Translates internal dependency kind into the runtime kind.
static void emitTargetCallElse(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, CodeGenFunction &CGF)
static llvm::Function * emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty, const Expr *CombinerInitializer, const VarDecl *In, const VarDecl *Out, bool IsCombiner)
static void emitX86DeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn, const llvm::APSInt &VLENVal, ArrayRef< ParamAttrTy > ParamAttrs, OMPDeclareSimdDeclAttr::BranchStateTy State)
static void emitReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp)
Emit reduction combiner.
static std::string mangleVectorParameters(ArrayRef< ParamAttrTy > ParamAttrs)
Mangle the parameter part of the vector function name according to their OpenMP classification.
static llvm::Function * emitParallelOrTeamsOutlinedFunction(CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen)
static void emitAArch64DeclareSimdFunction(CodeGenModule &CGM, const FunctionDecl *FD, unsigned UserVLEN, ArrayRef< ParamAttrTy > ParamAttrs, OMPDeclareSimdDeclAttr::BranchStateTy State, StringRef MangledName, char ISA, unsigned VecRegSize, llvm::Function *Fn, SourceLocation SLoc)
Emit vector function attributes for AArch64, as defined in the AAVFABI.
static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array, unsigned Index, const VarDecl *Var)
Given an array of pointers to variables, project the address of a given variable.
llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind convertDeviceClause(const VarDecl *VD)
static bool isAssumedToBeNotEmitted(const ValueDecl *VD, bool IsDevice)
static void addAArch64AdvSIMDNDSNames(unsigned NDS, StringRef Mask, StringRef Prefix, char ISA, StringRef ParSeq, StringRef MangledName, bool OutputBecomesInput, llvm::Function *Fn)
static FieldDecl * addFieldToRecordDecl(ASTContext &C, DeclContext *DC, QualType FieldTy)
static ValueDecl * getDeclFromThisExpr(const Expr *E)
llvm::Constant * emitMappingInformation(CodeGenFunction &CGF, llvm::OpenMPIRBuilder &OMPBuilder, MappableExprsHandler::MappingExprInfo &MapExprs)
Emit a string constant containing the names of the values mapped to the offloading runtime library.
static RecordDecl * createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpRoutineEntryPointerQTy)
static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2)
static bool getAArch64MTV(QualType QT, ParamKindTy Kind)
Maps To Vector (MTV), as defined in 4.1.1 of the AAVFABI (2021Q1).
enum clang::sema::@1651::IndirectLocalPathEntry::EntryKind Kind
Defines the clang::FileManager interface and associated types.
This file defines OpenMP AST classes for clauses.
Defines some OpenMP-specific enums and functions.
Defines the SourceManager interface.
This file defines OpenMP AST classes for executable directives and clauses.
__DEVICE__ int max(int __a, int __b)
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
SourceManager & getSourceManager()
const ConstantArrayType * getAsConstantArrayType(QualType T) const
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
const ASTRecordLayout & getASTRecordLayout(const RecordDecl *D) const
Get or compute information about the layout of the specified record (struct/union/class) D,...
bool hasSameType(QualType T1, QualType T2) const
Determine whether the given types T1 and T2 are equivalent.
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
const LangOptions & getLangOpts() const
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth,...
CanQualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
const VariableArrayType * getAsVariableArrayType(QualType T) const
const TargetInfo & getTargetInfo() const
CharUnits getNonVirtualSize() const
getNonVirtualSize - Get the non-virtual size (in chars) of an object, which is the size of the object...
static QualType getBaseOriginalType(const Expr *Base)
Return original type of the base expression for array section.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Attr - This represents one attribute.
Represents a C++ constructor within a class.
Represents a C++ destructor within a class.
Represents a static or instance method of a struct/union/class.
const CXXRecordDecl * getParent() const
Return the parent of this method declaration, which is the class in which this method is defined.
QualType getFunctionObjectParameterType() const
Represents a C++ struct/union/class.
bool isLambda() const
Determine whether this class describes a lambda function object.
void getCaptureFields(llvm::DenseMap< const ValueDecl *, FieldDecl * > &Captures, FieldDecl *&ThisCapture) const
For a closure type, retrieve the mapping from captured variables and this to the non-static data memb...
unsigned getNumBases() const
Retrieves the number of base classes of this class.
base_class_range vbases()
capture_const_range captures() const
CXXDestructorDecl * getDestructor() const
Returns the destructor decl for this class.
CanProxy< U > castAs() const
A wrapper class around a pointer that always points to its canonical declaration.
Describes the capture of either a variable, or 'this', or variable-length array type.
bool capturesVariableByCopy() const
Determine whether this capture handles a variable by copy.
VarDecl * getCapturedVar() const
Retrieve the declaration of the variable being captured.
bool capturesVariableArrayType() const
Determine whether this capture handles a variable-length array type.
bool capturesThis() const
Determine whether this capture handles the C++ 'this' pointer.
bool capturesVariable() const
Determine whether this capture handles a variable (by reference).
This captures a statement into a function.
capture_iterator capture_end() const
Retrieve an iterator pointing past the end of the sequence of captures.
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Stmt * getCapturedStmt()
Retrieve the statement being captured.
bool capturesVariable(const VarDecl *Var) const
True if this variable has been captured.
capture_iterator capture_begin()
Retrieve an iterator pointing to the first capture.
CharUnits - This is an opaque type for sizes expressed in character units.
bool isZero() const
isZero - Test whether the quantity equals zero.
llvm::Align getAsAlign() const
getAsAlign - Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit b...
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
CharUnits alignTo(const CharUnits &Align) const
alignTo - Returns the next integer (mod 2**64) that is greater than or equal to this quantity and is ...
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
CharUnits getAlignment() const
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Address withPointer(llvm::Value *NewPointer, KnownNonNull_t IsKnownNonNull) const
Return address with different pointer, but same element type and alignment.
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Address withAlignment(CharUnits NewAlignment) const
Return address with different alignment, but same pointer and element type.
llvm::PointerType * getType() const
Return the type of the pointer value.
static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateDefaultArtificial(CodeGenFunction &CGF, SourceLocation TemporaryLocation)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF)
Set the IRBuilder to not attach debug locations.
CGBlockInfo - Information to generate a block literal.
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
llvm::Value * CreateIsNull(Address Addr, const Twine &Name="")
Address CreateGEP(CodeGenFunction &CGF, Address Addr, llvm::Value *Index, const llvm::Twine &Name="")
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Address CreateConstArrayGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = [n x T]* ... produce name = getelementptr inbounds addr, i64 0, i64 index where i64 is a...
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
Address CreateConstGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = T* ... produce name = getelementptr inbounds addr, i64 index where i64 is actually the t...
MangleContext & getMangleContext()
Gets the mangle context.
CGFunctionInfo - Class to encapsulate the information about a function definition.
DisableAutoDeclareTargetRAII(CodeGenModule &CGM)
~DisableAutoDeclareTargetRAII()
Manages list of lastprivate conditional decls for the specified directive.
~LastprivateConditionalRAII()
static LastprivateConditionalRAII disable(CodeGenFunction &CGF, const OMPExecutableDirective &S)
NontemporalDeclsRAII(CodeGenModule &CGM, const OMPLoopDirective &S)
Struct that keeps all the relevant information that should be kept throughout a 'target data' region.
llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap
Map between the a declaration of a capture and the corresponding new llvm address where the runtime r...
~UntiedTaskLocalDeclsRAII()
UntiedTaskLocalDeclsRAII(CodeGenFunction &CGF, const llvm::MapVector< CanonicalDeclPtr< const VarDecl >, std::pair< Address, Address > > &LocalVars)
virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc)
Emits address of the word in a memory where current thread id is stored.
llvm::FunctionType * Kmpc_MicroTy
The type for a microtask which gets passed to __kmpc_fork_call().
llvm::StringSet ThreadPrivateWithDefinition
Set of threadprivate variables with the generated initializer.
CGOpenMPRuntime(CodeGenModule &CGM)
virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
Emit task region for the task directive.
void createOffloadEntriesAndInfoMetadata()
Creates all the offload entries in the current compilation unit along with the associated metadata.
const Expr * getNumTeamsExprForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &MinTeamsVal, int32_t &MaxTeamsVal)
Emit the number of teams for a target directive.
virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc)
Returns address of the threadprivate variable for the current thread.
void emitDeferredTargetDecls() const
Emit deferred declare target variables marked for deferred emission.
virtual llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST)
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
bool markAsGlobalTarget(GlobalDecl GD)
Marks the declaration as already emitted for the device code and returns true, if it was marked alrea...
virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device)
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
virtual void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc)
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)...
QualType SavedKmpTaskloopTQTy
Saved kmp_task_t for taskloop-based directive.
virtual void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps)
Emits a single region.
virtual bool emitTargetGlobal(GlobalDecl GD)
Emit the global GD if it is meaningful for the target.
void setLocThreadIdInsertPt(CodeGenFunction &CGF, bool AtCurrentPoint=false)
std::string getOutlinedHelperName(StringRef Name) const
Get the function name of an outlined region.
bool HasEmittedDeclareTargetRegion
Flag for keeping track of weather a device routine has been emitted.
llvm::Constant * getOrCreateThreadPrivateCache(const VarDecl *VD)
If the specified mangled name is not in the module, create and return threadprivate cache object.
virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal)
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
virtual void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc)
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF, const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const
Choose default schedule type and chunk value for the schedule clause.
virtual std::pair< llvm::Function *, llvm::Function * > getUserDefinedReduction(const OMPDeclareReductionDecl *D)
Get combiner/initializer for the specified user-defined reduction, if any.
virtual bool isGPU() const
Returns true if the current target is a GPU.
static const Stmt * getSingleCompoundChild(ASTContext &Ctx, const Stmt *Body)
Checks if the Body is the CompoundStmt and returns its child statement iff there is only one that is ...
virtual void emitDeclareTargetFunction(const FunctionDecl *FD, llvm::GlobalValue *GV)
Emit code for handling declare target functions in the runtime.
llvm::Type * getKmpc_MicroPointerTy()
Returns pointer to kmpc_micro type.
bool HasRequiresUnifiedSharedMemory
Flag for keeping track of weather a requires unified_shared_memory directive is present.
llvm::Value * emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc, unsigned Flags=0, bool EmitLoc=false)
Emits object of ident_t type with info for source location.
bool isLocalVarInUntiedTask(CodeGenFunction &CGF, const VarDecl *VD) const
Returns true if the variable is a local variable in untied task.
virtual void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars)
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
virtual void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancellation point' construct.
void emitUDMapperArrayInitOrDel(CodeGenFunction &MapperCGF, llvm::Value *Handle, llvm::Value *BasePtr, llvm::Value *Ptr, llvm::Value *Size, llvm::Value *MapType, llvm::Value *MapName, CharUnits ElementSize, llvm::BasicBlock *ExitBB, bool IsInit)
Emit the array initialization or deletion portion for user-defined mapper code generation.
virtual llvm::Function * emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF=nullptr)
Emit a code for initialization of threadprivate variable.
FunctionUDMMapTy FunctionUDMMap
virtual ConstantAddress getAddrOfDeclareTargetVar(const VarDecl *VD)
Returns the address of the variable marked as declare target with link clause OR as declare target wi...
llvm::MapVector< CanonicalDeclPtr< const VarDecl >, std::pair< Address, Address > > UntiedLocalVarsAddressesMap
llvm::Function * getOrCreateUserDefinedMapperFunc(const OMPDeclareMapperDecl *D)
Get the function for the specified user-defined mapper.
OpenMPLocThreadIDMapTy OpenMPLocThreadIDMap
virtual void functionFinished(CodeGenFunction &CGF)
Cleans up references to the objects in finished function.
void emitCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee Callee, ArrayRef< llvm::Value * > Args=std::nullopt) const
Emits Callee function call with arguments Args with location Loc.
virtual llvm::Function * emitTeamsOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP teams directive D.
QualType KmpTaskTQTy
Type typedef struct kmp_task { void * shareds; /**< pointer to block of pointers to shared vars / k...
llvm::OpenMPIRBuilder OMPBuilder
An OpenMP-IR-Builder instance.
virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr * > NumIterations)
Emit initialization for doacross loop nesting support.
virtual void adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF, const OMPExecutableDirective &D) const
Adjust some parameters for the target-based directives, like addresses of the variables captured by r...
FunctionUDRMapTy FunctionUDRMap
virtual void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, CGOpenMPRuntime::TargetDataInfo &Info)
Emit the target data mapping code associated with D.
virtual unsigned getDefaultLocationReserved2Flags() const
Returns additional flags that can be stored in reserved_2 field of the default location.
void computeMinAndMaxThreadsAndTeams(const OMPExecutableDirective &D, CodeGenFunction &CGF, int32_t &MinThreadsVal, int32_t &MaxThreadsVal, int32_t &MinTeamsVal, int32_t &MaxTeamsVal)
Helper to determine the min/max number of threads/teams for D.
virtual Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const
Gets the address of the native argument basing on the address of the target-specific parameter.
void emitUsesAllocatorsFini(CodeGenFunction &CGF, const Expr *Allocator)
Destroys user defined allocators specified in the uses_allocators clause.
QualType KmpTaskAffinityInfoTy
Type typedef struct kmp_task_affinity_info { kmp_intptr_t base_addr; size_t len; struct { bool flag1 ...
llvm::SmallVector< NontemporalDeclsSet, 4 > NontemporalDeclsStack
Stack for list of declarations in current context marked as nontemporal.
llvm::Value * emitNumTeamsForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D)
virtual void emitTargetOutlinedFunctionHelper(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Helper to emit outlined function for 'target' directive.
void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName)
Start scanning from statement S and emit all target regions found along the way.
SmallVector< llvm::Value *, 4 > emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy, const OMPTaskDataTy::DependData &Data)
virtual void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc)
Emit a taskgroup region.
llvm::DenseMap< llvm::Function *, llvm::DenseMap< CanonicalDeclPtr< const Decl >, std::tuple< QualType, const FieldDecl *, const FieldDecl *, LValue > > > LastprivateConditionalToTypes
Maps local variables marked as lastprivate conditional to their internal types.
virtual bool emitTargetGlobalVariable(GlobalDecl GD)
Emit the global variable if it is a valid device global variable.
virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams,...
bool hasRequiresUnifiedSharedMemory() const
Return whether the unified_shared_memory has been specified.
virtual Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, QualType VarType, StringRef Name)
Creates artificial threadprivate variable with name Name and type VarType.
void emitUserDefinedMapper(const OMPDeclareMapperDecl *D, CodeGenFunction *CGF=nullptr)
Emit the function for the user defined mapper construct.
bool HasEmittedTargetRegion
Flag for keeping track of weather a target region has been emitted.
void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy, LValue PosLVal, const OMPTaskDataTy::DependData &Data, Address DependenciesArray)
std::string getReductionFuncName(StringRef Name) const
Get the function name of a reduction function.
virtual void processRequiresDirective(const OMPRequiresDecl *D)
Perform check on requires decl to ensure that target architecture supports unified addressing.
llvm::DenseSet< CanonicalDeclPtr< const Decl > > AlreadyEmittedTargetDecls
List of the emitted declarations.
virtual llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data)
Emit a code for initialization of task reduction clause.
llvm::Value * getThreadID(CodeGenFunction &CGF, SourceLocation Loc)
Gets thread id value for the current thread.
void emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, OpenMPDependClauseKind NewDepKind, SourceLocation Loc)
Updates the dependency kind in the specified depobj object.
virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD, SourceLocation Loc)
Gets the address of the global copy used for lastprivate conditional update, if any.
virtual void emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, Expr *ME, bool IsFatal)
Emit __kmpc_error call for error directive extern void __kmpc_error(ident_t *loc, int severity,...
void clearLocThreadIdInsertPt(CodeGenFunction &CGF)
virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc)
Emits code for a taskyield directive.
std::string getName(ArrayRef< StringRef > Parts) const
Get the platform-specific name separator.
QualType KmpRoutineEntryPtrQTy
virtual void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc, llvm::AtomicOrdering AO)
Emit flush of the variables specified in 'omp flush' directive.
virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPTaskDataTy &Data)
Emit code for 'taskwait' directive.
virtual void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc)
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, int proc_bind) to generat...
void emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LValue IVLVal, StringRef UniqueDeclName, LValue LVal, SourceLocation Loc)
Emit update for lastprivate conditional data.
virtual void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
Emit task region for the taskloop directive.
virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false)
Emit an implicit/explicit barrier for OpenMP threads.
static unsigned getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind)
Returns default flags for the barriers depending on the directive, for which this barier is going to ...
virtual bool emitTargetFunctions(GlobalDecl GD)
Emit the target regions enclosed in GD function definition or the function itself in case it is a val...
TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const OMPTaskDataTy &Data)
Emit task region for the task directive.
llvm::Value * emitTargetNumIterationsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Return the trip count of loops associated with constructs / 'target teams distribute' and 'teams dist...
llvm::StringMap< llvm::AssertingVH< llvm::GlobalVariable >, llvm::BumpPtrAllocator > InternalVars
An ordered map of auto-generated variables to their unique names.
virtual void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values)
llvm::SmallVector< UntiedLocalVarsAddressesMap, 4 > UntiedLocalVarsStack
virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind)
Call the appropriate runtime routine to notify that we finished all the work with current loop.
virtual void emitThreadLimitClause(CodeGenFunction &CGF, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_set_thread_limit(ident_t *loc, kmp_int32 global_tid, kmp_int32 thread_limit...
void emitIfClause(CodeGenFunction &CGF, const Expr *Cond, const RegionCodeGenTy &ThenGen, const RegionCodeGenTy &ElseGen)
Emits code for OpenMP 'if' clause using specified CodeGen function.
Address emitDepobjDependClause(CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies, SourceLocation Loc)
Emits list of dependecies based on the provided data (array of dependence/expression pairs) for depob...
bool isNontemporalDecl(const ValueDecl *VD) const
Checks if the VD variable is marked as nontemporal declaration in current context.
virtual llvm::Function * emitParallelOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP parallel directive D.
const Expr * getNumThreadsExprForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &UpperBound, bool UpperBoundOnly, llvm::Value **CondExpr=nullptr, const Expr **ThreadLimitExpr=nullptr)
Check for a number of threads upper bound constant value (stored in UpperBound), or expression (retur...
llvm::SmallVector< LastprivateConditionalData, 4 > LastprivateConditionalStack
Stack for list of addresses of declarations in current context marked as lastprivate conditional.
virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values)
Call the appropriate runtime routine to initialize it before start of loop.
virtual void emitDeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn)
Marks function Fn with properly mangled versions of vector functions.
llvm::AtomicOrdering getDefaultMemoryOrdering() const
Gets default memory ordering as specified in requires directive.
llvm::SmallDenseSet< CanonicalDeclPtr< const Decl > > NontemporalDeclsSet
virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static non-chunked.
llvm::Value * getCriticalRegionLock(StringRef CriticalName)
Returns corresponding lock object for the specified critical region name.
virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancel' construct.
QualType SavedKmpTaskTQTy
Saved kmp_task_t for task directive.
virtual void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc)
Emits a master region.
virtual llvm::Function * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts)
Emits outlined function for the OpenMP task directive D.
llvm::DenseMap< llvm::Function *, unsigned > FunctionToUntiedTaskStackMap
Maps function to the position of the untied task locals stack.
void emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal, SourceLocation Loc)
Emits the code to destroy the dependency object provided in depobj directive.
virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Required to resolve existing problems in the runtime.
llvm::ArrayType * KmpCriticalNameTy
Type kmp_critical_name, originally defined as typedef kmp_int32 kmp_critical_name[8];.
virtual void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C)
Emit code for doacross ordered directive with 'depend' clause.
llvm::DenseMap< const OMPDeclareMapperDecl *, llvm::Function * > UDMMap
Map from the user-defined mapper declaration to its corresponding functions.
virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF, const Expr *LHS)
Checks if the provided LVal is lastprivate conditional and emits the code to update the value of the ...
std::pair< llvm::Value *, LValue > getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal, SourceLocation Loc)
Returns the number of the elements and the address of the depobj dependency array.
llvm::SmallDenseSet< const VarDecl * > DeferredGlobalVariables
List of variables that can become declare target implicitly and, thus, must be emitted.
void emitUsesAllocatorsInit(CodeGenFunction &CGF, const Expr *Allocator, const Expr *AllocatorTraits)
Initializes user defined allocators specified in the uses_allocators clauses.
llvm::Type * KmpRoutineEntryPtrTy
Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *);.
llvm::Type * getIdentTyPointerTy()
Returns pointer to ident_t type.
void emitSingleReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp, const Expr *PrivateRef, const DeclRefExpr *LHS, const DeclRefExpr *RHS)
Emits single reduction combiner.
llvm::OpenMPIRBuilder & getOMPBuilder()
virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Emit outilined function for 'target' directive.
virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr)
Emits a critical region.
virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned)
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
llvm::Value * emitNumThreadsForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D)
Emit an expression that denotes the number of threads a target region shall use.
void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor, llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc)
Emits initialization code for the threadprivate variables.
virtual void emitUserDefinedReduction(CodeGenFunction *CGF, const OMPDeclareReductionDecl *D)
Emit code for the specified user defined reduction construct.
virtual void checkAndEmitSharedLastprivateConditional(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::DenseSet< CanonicalDeclPtr< const VarDecl > > &IgnoredDecls)
Checks if the lastprivate conditional was updated in inner region and writes the value.
QualType KmpDimTy
struct kmp_dim { // loop bounds info casted to kmp_int64 kmp_int64 lo; // lower kmp_int64 up; // uppe...
virtual void emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool HasCancel=false)
Emit code for the directive that does not require outlining.
virtual void registerTargetGlobalVariable(const VarDecl *VD, llvm::Constant *Addr)
Checks if the provided global decl GD is a declare target variable and registers it when emitting cod...
virtual void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D)
Emits OpenMP-specific function prolog.
virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond, llvm::Value *NumThreads)
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
void emitKmpRoutineEntryT(QualType KmpInt32Ty)
Build type kmp_routine_entry_t (if not built yet).
virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static chunked.
virtual void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Emit the target offloading code associated with D.
virtual bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS)
Checks if the variable has associated OMPAllocateDeclAttr attribute with the predefined allocator and...
llvm::AtomicOrdering RequiresAtomicOrdering
Atomic ordering from the omp requires directive.
virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps, ReductionOptionsTy Options)
Emit a code for reduction clause.
std::pair< llvm::Value *, Address > emitDependClause(CodeGenFunction &CGF, ArrayRef< OMPTaskDataTy::DependData > Dependencies, SourceLocation Loc)
Emits list of dependecies based on the provided data (array of dependence/expression pairs).
llvm::StringMap< llvm::WeakTrackingVH > EmittedNonTargetVariables
List of the global variables with their addresses that should not be emitted for the target.
virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const
Check if the specified ScheduleKind is dynamic.
Address emitLastprivateConditionalInit(CodeGenFunction &CGF, const VarDecl *VD)
Create specialized alloca to handle lastprivate conditionals.
virtual void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads)
Emit an ordered region.
virtual void emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, ArrayRef< llvm::Value * > Args=std::nullopt) const
Emits call of the outlined function with the provided arguments, translating these arguments to corre...
virtual Address getAddressOfLocalVariable(CodeGenFunction &CGF, const VarDecl *VD)
Gets the OpenMP-specific address of the local variable.
virtual void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, bool IsWorksharingReduction)
Emits the following code for reduction clause with task modifier:
virtual void emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter=nullptr)
Emits a masked region.
QualType KmpDependInfoTy
Type typedef struct kmp_depend_info { kmp_intptr_t base_addr; size_t len; struct { bool in:1; bool ou...
llvm::Function * emitReductionFunction(StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps)
Emits reduction function.
virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues)
Call the appropriate runtime routine to initialize it before start of loop.
Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal) override
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr) override
Emits a critical region.
void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values) override
void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) override
Call the appropriate runtime routine to initialize it before start of loop.
bool emitTargetGlobalVariable(GlobalDecl GD) override
Emit the global variable if it is a valid device global variable.
llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST) override
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
llvm::Function * emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF=nullptr) override
Emit a code for initialization of threadprivate variable.
void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device) override
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
llvm::Function * emitTeamsOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) override
Emits outlined function for the specified OpenMP teams directive D.
void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps, ReductionOptionsTy Options) override
Emit a code for reduction clause.
void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc, llvm::AtomicOrdering AO) override
Emit flush of the variables specified in 'omp flush' directive.
void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C) override
Emit code for doacross ordered directive with 'depend' clause.
void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc) override
Emits a masked region.
Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, QualType VarType, StringRef Name) override
Creates artificial threadprivate variable with name Name and type VarType.
Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc) override
Returns address of the threadprivate variable for the current thread.
void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps) override
Emits a single region.
void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N) override
Required to resolve existing problems in the runtime.
llvm::Function * emitParallelOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) override
Emits outlined function for the specified OpenMP parallel directive D.
void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion) override
Emit code for 'cancellation point' construct.
void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false) override
Emit an implicit/explicit barrier for OpenMP threads.
Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const override
Gets the address of the native argument basing on the address of the target-specific parameter.
void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars) override
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned) override
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
bool emitTargetGlobal(GlobalDecl GD) override
Emit the global GD if it is meaningful for the target.
void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, bool IsWorksharingReduction) override
Emits the following code for reduction clause with task modifier:
void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads) override
Emit an ordered region.
void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind) override
Call the appropriate runtime routine to notify that we finished all the work with current loop.
llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data) override
Emit a code for initialization of task reduction clause.
void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond, llvm::Value *NumThreads) override
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc) override
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, int proc_bind) to generat...
void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) override
Emit outilined function for 'target' directive.
void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc) override
Emits a master region.
void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc) override
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams,...
void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc) override
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
const VarDecl * translateParameter(const FieldDecl *FD, const VarDecl *NativeParam) const override
Translates the native parameter of outlined function if this is required for target.
void emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter=nullptr) override
Emits a masked region.
void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data) override
Emit task region for the task directive.
void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter) override
Emit the target offloading code associated with D.
bool emitTargetFunctions(GlobalDecl GD) override
Emit the target regions enclosed in GD function definition or the function itself in case it is a val...
void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc) override
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)...
void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr * > NumIterations) override
Emit initialization for doacross loop nesting support.
void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion) override
Emit code for 'cancel' construct.
void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPTaskDataTy &Data) override
Emit code for 'taskwait' directive.
void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc) override
Emit a taskgroup region.
void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, CGOpenMPRuntime::TargetDataInfo &Info) override
Emit the target data mapping code associated with D.
void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues) override
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
llvm::Function * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts) override
Emits outlined function for the OpenMP task directive D.
void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data) override
Emit task region for the taskloop directive.
CGRecordLayout - This class handles struct and union layout info while lowering AST types to LLVM typ...
unsigned getNonVirtualBaseLLVMFieldNo(const CXXRecordDecl *RD) const
llvm::StructType * getLLVMType() const
Return the "complete object" LLVM type associated with this record.
llvm::StructType * getBaseSubobjectLLVMType() const
Return the "base subobject" LLVM type associated with this record.
unsigned getLLVMFieldNo(const FieldDecl *FD) const
Return llvm::StructType element number that corresponds to the field FD.
unsigned getVirtualBaseIndex(const CXXRecordDecl *base) const
Return the LLVM field index corresponding to the given virtual base.
virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S)
Emit the captured statement body.
RAII for correct setting/restoring of CapturedStmtInfo.
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
bool Privatize()
Privatizes local variables previously registered as private.
bool addPrivate(const VarDecl *LocalVD, Address Addr)
Registers LocalVD variable as a private with Addr as the address of the corresponding private variabl...
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited.
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
void EmitNullInitialization(Address DestPtr, QualType Ty)
EmitNullInitialization - Generate code to set a value of the given type to null, If the type contains...
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount, Stmt::Likelihood LH=Stmt::LH_None, const Expr *ConditionalOp=nullptr)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target)
The given basic block lies in the current EH scope, but may be a target of a potentially scope-crossi...
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
FieldDecl * LambdaThisCaptureField
CGCapturedStmtInfo * CapturedStmtInfo
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Load a pointer with type PtrTy stored at address Ptr.
RawAddress CreateDefaultAlignTempAlloca(llvm::Type *Ty, const Twine &Name="tmp")
CreateDefaultAlignedTempAlloca - This creates an alloca with the default ABI alignment of the given L...
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
llvm::Value * emitArrayLength(const ArrayType *arrayType, QualType &baseType, Address &addr)
emitArrayLength - Compute the length of an array, even if it's a VLA, and drill down to the base elem...
VlaSizePair getVLASize(const VariableArrayType *vla)
Returns an LLVM value that corresponds to the size, in non-variably-sized elements,...
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
void EmitVariablyModifiedType(QualType Ty)
EmitVLASize - Capture all the sizes for the VLA expressions in the given variably-modified type and s...
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
const LangOptions & getLangOpts() const
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S)
Emit device code for the target simd directive.
LValue EmitLValueForFieldInitialization(LValue Base, const FieldDecl *Field)
EmitLValueForFieldInitialization - Like EmitLValueForField, except that if the Field is a reference,...
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
const CodeGen::CGBlockInfo * BlockInfo
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit)
EmitExprAsInit - Emits the code necessary to initialize a location in memory with the given initializ...
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
llvm::Type * ConvertTypeForMem(QualType T)
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
llvm::AssertingVH< llvm::Instruction > AllocaInsertPt
AllocaInsertPoint - This is an instruction in the entry block before which we prefer to insert alloca...
void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy)
Emit an aggregate assignment.
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value * > &CapturedVars)
JumpDest ReturnBlock
ReturnBlock - Unified return block.
static void EmitOMPTargetTeamsGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsGenericLoopDirective &S)
Emit device code for the target teams loop directive.
LValue EmitLValueForField(LValue Base, const FieldDecl *Field)
RawAddress CreateMemTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Destroyer * getDestroyer(QualType::DestructionKind destructionKind)
static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
void emitDestroy(Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray)
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc)
EmitLoadOfComplex - Load a complex number from the specified l-value.
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
bool isTrivialInitializer(const Expr *Init)
Determine whether the given initializer is trivial in the sense that it requires no code to be genera...
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block,...
LValue MakeRawAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment, AlignmentSource Source=AlignmentSource::Type)
Same as MakeAddrLValue above except that the pointer is known to be unsigned.
void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy, AggValueSlot::Overlap_t MayOverlap, bool isVolatile=false)
EmitAggregateCopy - Emit an aggregate copy.
LValue MakeNaturalAlignRawAddrLValue(llvm::Value *V, QualType T)
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
llvm::CallInst * EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
ASTContext & getContext() const
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy)
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
void EmitBranchThroughCleanup(JumpDest Dest)
EmitBranchThroughCleanup - Emit a branch from the current insert block through the normal cleanup han...
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
void pushDestroy(QualType::DestructionKind dtorKind, Address addr, QualType type)
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant,...
void EmitAutoVarCleanups(const AutoVarEmission &emission)
bool needsEHCleanup(QualType::DestructionKind kind)
Determines whether an EH cleanup is required to destroy a type with the given destruction kind.
llvm::DenseMap< const ValueDecl *, FieldDecl * > LambdaCaptureFields
llvm::CallInst * EmitRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
llvm::Type * ConvertType(QualType T)
CodeGenTypes & getTypes() const
LValue EmitArraySectionExpr(const ArraySectionExpr *E, bool IsLowerBound=true)
static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Emit device code for the target teams distribute parallel for simd directive.
LValue EmitStringLiteralLValue(const StringLiteral *E)
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
Address LoadCXXThisAddress()
LValue EmitOMPSharedLValue(const Expr *E)
Emits the lvalue for the expression with possibly captured variable.
llvm::Value * EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr, ArrayRef< llvm::Value * > IdxList, bool SignedIndices, bool IsSubtraction, SourceLocation Loc, const Twine &Name="")
Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to detect undefined behavior whe...
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S, SourceLocation Loc)
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
static void EmitOMPTargetParallelGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelGenericLoopDirective &S)
Emit device code for the target parallel loop directive.
LValue EmitLoadOfReferenceLValue(LValue RefLVal)
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler's counter for the given statement by StepV.
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
This class organizes the cross-function state that is used while generating LLVM code.
void handleCUDALaunchBoundsAttr(llvm::Function *F, const CUDALaunchBoundsAttr *A, int32_t *MaxThreadsVal=nullptr, int32_t *MinBlocksVal=nullptr, int32_t *MaxClusterRankVal=nullptr)
Emit the IR encoding to attach the CUDA launch bounds attribute to F.
void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
llvm::Module & getModule() const
void addCompilerUsedGlobal(llvm::GlobalValue *GV)
Add a global to a list to be added to the llvm.compiler.used metadata.
CharUnits GetTargetTypeStoreSize(llvm::Type *Ty) const
Return the store size, in character units, of the given LLVM type.
void handleAMDGPUWavesPerEUAttr(llvm::Function *F, const AMDGPUWavesPerEUAttr *A)
Emit the IR encoding to attach the AMD GPU waves-per-eu attribute to F.
DiagnosticsEngine & getDiags() const
const LangOptions & getLangOpts() const
CharUnits getNaturalTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, bool forPointeeType=false)
CodeGenTypes & getTypes()
const TargetInfo & getTarget() const
void EmitGlobal(GlobalDecl D)
Emit code for a single global function or var decl.
void handleAMDGPUFlatWorkGroupSizeAttr(llvm::Function *F, const AMDGPUFlatWorkGroupSizeAttr *A, const ReqdWorkGroupSizeAttr *ReqdWGS=nullptr, int32_t *MinThreadsVal=nullptr, int32_t *MaxThreadsVal=nullptr)
Emit the IR encoding to attach the AMD GPU flat-work-group-size attribute to F.
llvm::GlobalValue::LinkageTypes getLLVMLinkageVarDefinition(const VarDecl *VD)
Returns LLVM linkage for a declarator.
CGCXXABI & getCXXABI() const
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
const llvm::Triple & getTriple() const
TBAAAccessInfo getTBAAInfoForSubobject(LValue Base, QualType AccessType)
getTBAAInfoForSubobject - Get TBAA information for an access with a given base lvalue.
llvm::Constant * GetAddrOfGlobal(GlobalDecl GD, ForDefinition_t IsForDefinition=NotForDefinition)
ASTContext & getContext() const
const TargetCodeGenInfo & getTargetCodeGenInfo()
const CodeGenOptions & getCodeGenOpts() const
StringRef getMangledName(GlobalDecl GD)
std::optional< CharUnits > getOMPAllocateAlignment(const VarDecl *VD)
Return the alignment specified in an allocate directive, if present.
llvm::GlobalValue * GetGlobalValue(StringRef Ref)
llvm::Constant * EmitNullConstant(QualType T)
Return the result of value-initializing the given type, i.e.
llvm::Function * CreateGlobalInitOrCleanUpFunction(llvm::FunctionType *ty, const Twine &name, const CGFunctionInfo &FI, SourceLocation Loc=SourceLocation(), bool TLS=false, llvm::GlobalVariable::LinkageTypes Linkage=llvm::GlobalVariable::InternalLinkage)
llvm::ConstantInt * getSize(CharUnits numChars)
Emit the given number of characters as a value of type size_t.
llvm::Type * ConvertType(QualType T)
ConvertType - Convert type T into a llvm::Type.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
const CGRecordLayout & getCGRecordLayout(const RecordDecl *)
getCGRecordLayout - Return record layout info for the given record decl.
llvm::Type * ConvertTypeForMem(QualType T)
ConvertTypeForMem - Convert type T into a llvm::Type.
const CGFunctionInfo & arrangeNullaryFunction()
A nullary function is a freestanding function of type 'void ()'.
A specialization of Address that requires the address to be an LLVM Constant.
static ConstantAddress invalid()
Information for lazily generating a cleanup.
bool requiresLandingPad() const
void popTerminate()
Pops a terminate handler off the stack.
void pushTerminate()
Push a terminate handler on the stack.
FunctionArgList - Type for representing both the decl and type of parameters to a function.
LValue - This represents an lvalue references.
CharUnits getAlignment() const
llvm::Value * getPointer(CodeGenFunction &CGF) const
const Qualifiers & getQuals() const
Address getAddress() const
LValueBaseInfo getBaseInfo() const
TBAAAccessInfo getTBAAInfo() const
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
virtual void Enter(CodeGenFunction &CGF)
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
static RValue get(llvm::Value *V)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
An abstract representation of an aligned address.
llvm::Type * getElementType() const
Return the type of the values stored in this address.
llvm::Value * getPointer() const
static RawAddress invalid()
Class intended to support codegen of all kind of the reduction clauses.
LValue getSharedLValue(unsigned N) const
Returns LValue for the reduction item.
const Expr * getRefExpr(unsigned N) const
Returns the base declaration of the reduction item.
LValue getOrigLValue(unsigned N) const
Returns LValue for the original reduction item.
bool needCleanups(unsigned N)
Returns true if the private copy requires cleanups.
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
const VarDecl * getBaseDecl(unsigned N) const
Returns the base declaration of the reduction item.
QualType getPrivateType(unsigned N) const
Return the type of the private item.
bool usesReductionInitializer(unsigned N) const
Returns true if the initialization of the reduction item uses initializer from declare reduction cons...
void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N)
Emits lvalue for the shared and original reduction item.
void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, llvm::function_ref< bool(CodeGenFunction &)> DefaultInit)
Performs initialization of the private copy for the reduction item.
std::pair< llvm::Value *, llvm::Value * > getSizes(unsigned N) const
Returns the size of the reduction item (in chars and total number of elements in the item),...
ReductionCodeGen(ArrayRef< const Expr * > Shareds, ArrayRef< const Expr * > Origs, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > ReductionOps)
void emitCleanups(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Emits cleanup code for the reduction item.
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations.
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
void operator()(CodeGenFunction &CGF) const
void setAction(PrePostActionTy &Action) const
virtual void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const
setTargetAttributes - Provides a convenient hook to handle extra target-specific attributes for the g...
ConstStmtVisitor - This class implements a simple visitor for Stmt subclasses.
DeclContext - This is used only as base class of specific decl types that can act as declaration cont...
void addDecl(Decl *D)
Add the declaration D into this context.
A reference to a declared variable, function, enum, etc.
Decl - This represents one declaration (or definition), e.g.
SourceLocation getEndLoc() const LLVM_READONLY
ASTContext & getASTContext() const LLVM_READONLY
virtual bool hasBody() const
Returns true if this Decl represents a declaration for a body of code, such as a function or method d...
llvm::iterator_range< specific_attr_iterator< T > > specific_attrs() const
SourceLocation getLocation() const
DeclContext * getDeclContext()
SourceLocation getBeginLoc() const LLVM_READONLY
virtual Decl * getCanonicalDecl()
Retrieves the "canonical" declaration of the given declaration.
SourceLocation getBeginLoc() const LLVM_READONLY
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
unsigned getCustomDiagID(Level L, const char(&FormatString)[N])
Return an ID for a diagnostic with the specified format string and level.
The return type of classify().
This represents one expression.
@ SE_AllowSideEffects
Allow any unmodeled side effect.
@ SE_AllowUndefinedBehavior
Allow UB that we can give a value, but not arbitrary unmodeled side effects.
Expr * IgnoreParenCasts() LLVM_READONLY
Skip past any parentheses and casts which might surround this expression until reaching a fixed point...
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx, SmallVectorImpl< PartialDiagnosticAt > *Diag=nullptr) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
bool isEvaluatable(const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects) const
isEvaluatable - Call EvaluateAsRValue to see if this expression can be constant folded without side-e...
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
bool isIntegerConstantExpr(const ASTContext &Ctx, SourceLocation *Loc=nullptr) const
bool EvaluateAsBooleanCondition(bool &Result, const ASTContext &Ctx, bool InConstantContext=false) const
EvaluateAsBooleanCondition - Return true if this is a constant which we can fold and convert to a boo...
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
std::optional< llvm::APSInt > getIntegerConstantExpr(const ASTContext &Ctx, SourceLocation *Loc=nullptr) const
isIntegerConstantExpr - Return the value if this expression is a valid integer constant expression.
bool hasNonTrivialCall(const ASTContext &Ctx) const
Determine whether this expression involves a call to any function that is not trivial.
Represents a member of a struct/union/class.
static FieldDecl * Create(const ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, const IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, Expr *BW, bool Mutable, InClassInitStyle InitStyle)
Represents a function declaration or definition.
const ParmVarDecl * getParamDecl(unsigned i) const
QualType getReturnType() const
ArrayRef< ParmVarDecl * > parameters() const
FunctionDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
unsigned getNumParams() const
Return the number of parameters this function must have based on its FunctionType.
GlobalDecl - represents a global declaration.
const Decl * getDecl() const
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
static IntegerLiteral * Create(const ASTContext &C, const llvm::APInt &V, QualType type, SourceLocation l)
Returns a new integer literal with value 'V' and type 'type'.
Describes the capture of a variable or of this, or of a C++1y init-capture.
std::string OMPHostIRFile
Name of the IR file that contains the result of the OpenMP target host code generation.
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
virtual void mangleCanonicalTypeName(QualType T, raw_ostream &, bool NormalizeIntegers=false)=0
Generates a unique string for an externally visible type for use with TBAA or type uniquing.
MemberExpr - [C99 6.5.2.3] Structure and Union Members.
ValueDecl * getMemberDecl() const
Retrieve the member declaration to which this expression refers.
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
bool isExternallyVisible() const
This represents clause 'affinity' in the '#pragma omp task'-based directives.
Class that represents a component of a mappable expression.
ArrayRef< MappableComponent > MappableExprComponentListRef
const Stmt * getPreInitStmt() const
Get pre-initialization statement for the clause.
This is a basic class for representing single OpenMP clause.
This represents '#pragma omp declare mapper ...' directive.
This represents '#pragma omp declare reduction ...' directive.
Expr * getInitializer()
Get initializer expression (if specified) of the declare reduction construct.
This represents implicit clause 'depend' for the '#pragma omp task' directive.
This represents 'detach' clause in the '#pragma omp task' directive.
This represents 'device' clause in the '#pragma omp ...' directive.
This represents the 'doacross' clause for the '#pragma omp ordered' directive.
This is a basic class for representing single OpenMP executable directive.
static llvm::iterator_range< specific_clause_iterator< SpecificClause > > getClausesOfKind(ArrayRef< OMPClause * > Clauses)
This represents clause 'firstprivate' in the '#pragma omp ...' directives.
This represents clause 'has_device_ptr' in the '#pragma omp ...' directives.
This represents 'if' clause in the '#pragma omp ...' directive.
Expr * getCondition() const
Returns condition.
This represents clause 'in_reduction' in the '#pragma omp task' directives.
This represents clause 'is_device_ptr' in the '#pragma omp ...' directives.
OpenMP 5.0 [2.1.6 Iterators] Iterators are identifiers that expand to multiple values in the clause o...
This represents clause 'lastprivate' in the '#pragma omp ...' directives.
This represents clause 'linear' in the '#pragma omp ...' directives.
This is a common base class for loop directives ('omp simd', 'omp for', 'omp for simd' etc....
This represents clause 'map' in the '#pragma omp ...' directives.
This represents clause 'nontemporal' in the '#pragma omp ...' directives.
This represents 'nowait' clause in the '#pragma omp ...' directive.
This represents 'num_teams' clause in the '#pragma omp ...' directive.
This represents 'num_threads' clause in the '#pragma omp ...' directive.
This represents 'ordered' clause in the '#pragma omp ...' directive.
This represents clause 'private' in the '#pragma omp ...' directives.
This represents clause 'reduction' in the '#pragma omp ...' directives.
This represents '#pragma omp requires...' directive.
This represents 'thread_limit' clause in the '#pragma omp ...' directive.
This represents clause 'uses_allocators' in the '#pragma omp target'-based directives.
This represents 'ompx_attribute' clause in a directive that might generate an outlined function.
This represents 'ompx_dyn_cgroup_mem' clause in the '#pragma omp target ...' directive.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class.
Represents a parameter to a function.
PointerType - C99 6.7.5.1 - Pointer Declarators.
Represents an unpacked "presumed" location which can be presented to the user.
unsigned getColumn() const
Return the presumed column number of this location.
const char * getFilename() const
Return the presumed filename of this location.
unsigned getLine() const
Return the presumed line number of this location.
A (possibly-)qualified type.
void addRestrict()
Add the restrict qualifier to this QualType.
QualType withRestrict() const
bool isNull() const
Return true if this QualType doesn't point to a type yet.
const Type * getTypePtr() const
Retrieves a pointer to the underlying (unqualified) type.
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
QualType getCanonicalType() const
DestructionKind isDestructedType() const
Returns a nonzero value if objects of this type require non-trivial work to clean up after.
Represents a struct/union/class.
field_iterator field_end() const
field_range fields() const
virtual void completeDefinition()
Note that the definition of this type is now complete.
field_iterator field_begin() const
RecordDecl * getDecl() const
decl_type * getPreviousDecl()
Return the previous declaration of this declaration or NULL if this is the first declaration.
decl_type * getMostRecentDecl()
Returns the most recent (re)declaration of this declaration.
Base for LValueReferenceType and RValueReferenceType.
Scope - A scope is a transient data structure that is used while parsing the program.
Encodes a location in the source.
static SourceLocation getFromRawEncoding(UIntTy Encoding)
Turn a raw encoding of a SourceLocation object into a real SourceLocation.
bool isValid() const
Return true if this is a valid SourceLocation object.
UIntTy getRawEncoding() const
When a SourceLocation itself cannot be used, this returns an (opaque) 32-bit integer encoding for it.
This class handles loading and caching of source files into memory.
PresumedLoc getPresumedLoc(SourceLocation Loc, bool UseLineDirectives=true) const
Returns the "presumed" location of a SourceLocation specifies.
fileinfo_iterator fileinfo_end() const
SourceLocation translateFileLineCol(const FileEntry *SourceFile, unsigned Line, unsigned Col) const
Get the source location for the given file:line:col triplet.
fileinfo_iterator fileinfo_begin() const
A trivial tuple used to represent a source range.
Stmt - This represents one statement.
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top,...
SourceLocation getBeginLoc() const LLVM_READONLY
void startDefinition()
Starts the definition of this tag declaration.
bool isTLSSupported() const
Whether the target supports thread-local storage.
virtual bool hasFeature(StringRef Feature) const
Determine whether the given target has the given feature.
The base class of the type hierarchy.
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
bool isSignedIntegerOrEnumerationType() const
Determines whether this is an integer type that is signed or an enumeration types whose underlying ty...
const Type * getPointeeOrArrayElementType() const
If this is a pointer type, return the pointee type.
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
bool isPointerType() const
CanQualType getCanonicalTypeUnqualified() const
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
const T * castAs() const
Member-template castAs<specific type>.
bool isReferenceType() const
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
bool isLValueReferenceType() const
QualType getCanonicalTypeInternal() const
const RecordType * getAsStructureType() const
const Type * getBaseElementTypeUnsafe() const
Get the base element type of this type, potentially discarding type qualifiers.
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type.
bool isFloatingType() const
bool isUnsignedIntegerType() const
Return true if this is an integer type that is unsigned, according to C99 6.2.5p6 [which returns true...
bool isAnyPointerType() const
const T * getAs() const
Member-template getAs<specific type>'.
bool isRecordType() const
TagDecl * getAsTagDecl() const
Retrieves the TagDecl that this type refers to, either because the type is a TagType or because it is...
RecordDecl * getAsRecordDecl() const
Retrieves the RecordDecl this type refers to.
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Represents a variable declaration or definition.
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
VarDecl * getDefinition(ASTContext &)
Get the real (not just tentative) definition for this declaration.
bool hasExternalStorage() const
Returns true if a variable has extern or private_extern storage.
bool hasLocalStorage() const
Returns true if a variable with function scope is a non-static local variable.
@ DeclarationOnly
This declaration is only a declaration.
DefinitionKind hasDefinition(ASTContext &) const
Check whether this variable is defined in this translation unit.
bool isLocalVarDeclOrParm() const
Similar to isLocalVarDecl but also includes parameters.
const Expr * getAnyInitializer() const
Get the initializer for this variable, no matter which declaration it is attached to.
Represents a C array with a specified size that is not an integer-constant-expression.
Expr * getSizeExpr() const
specific_attr_iterator - Iterates over a subrange of an AttrVec, only providing attributes that are o...
bool isEmptyRecordForLayout(const ASTContext &Context, QualType T)
isEmptyRecordForLayout - Return true iff a structure contains only empty base classes (per isEmptyRec...
@ Decl
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
bool isEmptyFieldForLayout(const ASTContext &Context, const FieldDecl *FD)
isEmptyFieldForLayout - Return true iff the field is "empty", that is, either a zero-width bit-field ...
@ LLVM_MARK_AS_BITMASK_ENUM
The JSON file list parser is used to communicate input to InstallAPI.
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
@ Private
'private' clause, allowed on 'parallel', 'serial', 'loop', 'parallel loop', and 'serial loop' constru...
@ Vector
'vector' clause, allowed on 'loop', Combined, and 'routine' directives.
@ Reduction
'reduction' clause, allowed on Parallel, Serial, Loop, and the combined constructs.
@ Present
'present' clause, allowed on Compute and Combined constructs, plus 'data' and 'declare'.
bool needsTaskBasedThreadLimit(OpenMPDirectiveKind DKind)
Checks if the specified target directive, combined or not, needs task based thread_limit.
@ Ctor_Complete
Complete object ctor.
if(T->getSizeExpr()) TRY_TO(TraverseStmt(const_cast< Expr * >(T -> getSizeExpr())))
bool isOpenMPTargetDataManagementDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target data offload directive.
llvm::omp::Directive OpenMPDirectiveKind
OpenMP directives.
@ ICIS_NoInit
No in-class initializer.
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
@ LCK_ByRef
Capturing by reference.
LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE()
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
@ OMPC_SCHEDULE_MODIFIER_last
@ OMPC_SCHEDULE_MODIFIER_unknown
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
OpenMPDistScheduleClauseKind
OpenMP attributes for 'dist_schedule' clause.
bool isOpenMPTaskingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of tasking directives - task, taskloop,...
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
@ Result
The result type of a method or function.
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
OpenMPDependClauseKind
OpenMP attributes for 'depend' clause.
@ Dtor_Complete
Complete object dtor.
@ Union
The "union" keyword.
bool isOpenMPLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a directive with an associated loop construct.
LangAS
Defines the address space values used by the address space qualifier of QualType.
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
@ VK_PRValue
A pr-value expression (in the C++11 taxonomy) produces a temporary value.
@ VK_LValue
An l-value expression is a reference to an object with independent storage.
const FunctionProtoType * T
void getOpenMPCaptureRegions(llvm::SmallVectorImpl< OpenMPDirectiveKind > &CaptureRegions, OpenMPDirectiveKind DKind)
Return the captured regions of an OpenMP directive.
@ OMPC_ATOMIC_DEFAULT_MEM_ORDER_unknown
OpenMPMapModifierKind
OpenMP modifier kind for 'map' clause.
@ OMPC_MAP_MODIFIER_unknown
@ Other
Other implicit parameter.
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
OpenMPMapClauseKind
OpenMP mapping kind for 'map' clause.
Diagnostic wrappers for TextAPI types for error reporting.
Maps the expression for the lastprivate variable to the global copy used to store new value because o...
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
llvm::BasicBlock * getBlock() const
unsigned NumberOfTargetItems
Address BasePointersArray
llvm::PointerType * VoidPtrTy
llvm::IntegerType * Int64Ty
llvm::PointerType * Int8PtrPtrTy
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::CallingConv::ID getRuntimeCC() const
llvm::IntegerType * SizeTy
llvm::PointerType * VoidPtrPtrTy
llvm::IntegerType * Int32Ty
llvm::IntegerType * IntPtrTy
llvm::IntegerType * IntTy
int
llvm::PointerType * Int8PtrTy
CharUnits getPointerAlign() const
const Expr * IteratorExpr
SmallVector< const Expr *, 4 > DepExprs
EvalResult is a struct with detailed info about an evaluated expression.
Extra information about a function prototype.
Helper expressions and declaration for OMPIteratorExpr class for each iteration space.
Expr * CounterUpdate
Updater for the internal counter: ++CounterVD;.
Expr * Update
Update expression for the originally specified iteration variable, calculated as VD = Begin + Counter...
VarDecl * CounterVD
Internal normalized counter.
Data for list of allocators.
Scheduling data for loop-based OpenMP directives.
OpenMPScheduleClauseModifier M2
OpenMPScheduleClauseModifier M1
OpenMPScheduleClauseKind Schedule
Describes how types, statements, expressions, and declarations should be printed.