32#include "llvm/ADT/ArrayRef.h"
33#include "llvm/ADT/STLExtras.h"
34#include "llvm/ADT/SmallSet.h"
35#include "llvm/ADT/SmallVector.h"
36#include "llvm/ADT/StringExtras.h"
37#include "llvm/Analysis/ValueTracking.h"
38#include "llvm/Bitcode/BitcodeReader.h"
39#include "llvm/IR/Constants.h"
40#include "llvm/IR/DerivedTypes.h"
41#include "llvm/IR/GlobalValue.h"
42#include "llvm/IR/InstrTypes.h"
43#include "llvm/IR/Value.h"
44#include "llvm/Support/AtomicOrdering.h"
45#include "llvm/Support/VirtualFileSystem.h"
46#include "llvm/Support/raw_ostream.h"
54using namespace llvm::omp;
61 enum CGOpenMPRegionKind {
64 ParallelOutlinedRegion,
74 CGOpenMPRegionInfo(
const CapturedStmt &CS,
75 const CGOpenMPRegionKind RegionKind,
78 : CGCapturedStmtInfo(CS,
CR_OpenMP), RegionKind(RegionKind),
79 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
81 CGOpenMPRegionInfo(
const CGOpenMPRegionKind RegionKind,
84 : CGCapturedStmtInfo(
CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
85 Kind(Kind), HasCancel(HasCancel) {}
89 virtual const VarDecl *getThreadIDVariable()
const = 0;
92 void EmitBody(CodeGenFunction &CGF,
const Stmt *S)
override;
96 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
98 virtual void emitUntiedSwitch(CodeGenFunction & ) {}
100 CGOpenMPRegionKind getRegionKind()
const {
return RegionKind; }
104 bool hasCancel()
const {
return HasCancel; }
106 static bool classof(
const CGCapturedStmtInfo *Info) {
110 ~CGOpenMPRegionInfo()
override =
default;
113 CGOpenMPRegionKind RegionKind;
114 RegionCodeGenTy CodeGen;
120class CGOpenMPOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
122 CGOpenMPOutlinedRegionInfo(
const CapturedStmt &CS,
const VarDecl *ThreadIDVar,
123 const RegionCodeGenTy &CodeGen,
125 StringRef HelperName)
126 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen,
Kind,
128 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
129 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
134 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
137 StringRef getHelperName()
const override {
return HelperName; }
139 static bool classof(
const CGCapturedStmtInfo *Info) {
140 return CGOpenMPRegionInfo::classof(Info) &&
142 ParallelOutlinedRegion;
148 const VarDecl *ThreadIDVar;
149 StringRef HelperName;
153class CGOpenMPTaskOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
155 class UntiedTaskActionTy final :
public PrePostActionTy {
157 const VarDecl *PartIDVar;
158 const RegionCodeGenTy UntiedCodeGen;
159 llvm::SwitchInst *UntiedSwitch =
nullptr;
162 UntiedTaskActionTy(
bool Tied,
const VarDecl *PartIDVar,
163 const RegionCodeGenTy &UntiedCodeGen)
164 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
165 void Enter(CodeGenFunction &CGF)
override {
170 PartIDVar->
getType()->castAs<PointerType>());
174 UntiedSwitch = CGF.
Builder.CreateSwitch(Res, DoneBB);
178 UntiedSwitch->addCase(CGF.
Builder.getInt32(0),
180 emitUntiedSwitch(CGF);
183 void emitUntiedSwitch(CodeGenFunction &CGF)
const {
187 PartIDVar->
getType()->castAs<PointerType>());
191 CodeGenFunction::JumpDest CurPoint =
195 UntiedSwitch->addCase(CGF.
Builder.getInt32(UntiedSwitch->getNumCases()),
201 unsigned getNumberOfParts()
const {
return UntiedSwitch->getNumCases(); }
203 CGOpenMPTaskOutlinedRegionInfo(
const CapturedStmt &CS,
204 const VarDecl *ThreadIDVar,
205 const RegionCodeGenTy &CodeGen,
207 const UntiedTaskActionTy &Action)
208 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen,
Kind, HasCancel),
209 ThreadIDVar(ThreadIDVar), Action(Action) {
210 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
215 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
218 LValue getThreadIDVariableLValue(CodeGenFunction &CGF)
override;
221 StringRef getHelperName()
const override {
return ".omp_outlined."; }
223 void emitUntiedSwitch(CodeGenFunction &CGF)
override {
224 Action.emitUntiedSwitch(CGF);
227 static bool classof(
const CGCapturedStmtInfo *Info) {
228 return CGOpenMPRegionInfo::classof(Info) &&
236 const VarDecl *ThreadIDVar;
238 const UntiedTaskActionTy &Action;
243class CGOpenMPInlinedRegionInfo :
public CGOpenMPRegionInfo {
245 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
246 const RegionCodeGenTy &CodeGen,
248 : CGOpenMPRegionInfo(InlinedRegion, CodeGen,
Kind, HasCancel),
250 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
253 llvm::Value *getContextValue()
const override {
255 return OuterRegionInfo->getContextValue();
256 llvm_unreachable(
"No context value for inlined OpenMP region");
259 void setContextValue(llvm::Value *
V)
override {
260 if (OuterRegionInfo) {
261 OuterRegionInfo->setContextValue(
V);
264 llvm_unreachable(
"No context value for inlined OpenMP region");
268 const FieldDecl *lookup(
const VarDecl *VD)
const override {
270 return OuterRegionInfo->lookup(VD);
276 FieldDecl *getThisFieldDecl()
const override {
278 return OuterRegionInfo->getThisFieldDecl();
284 const VarDecl *getThreadIDVariable()
const override {
286 return OuterRegionInfo->getThreadIDVariable();
291 LValue getThreadIDVariableLValue(CodeGenFunction &CGF)
override {
293 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
294 llvm_unreachable(
"No LValue for inlined OpenMP construct");
298 StringRef getHelperName()
const override {
299 if (
auto *OuterRegionInfo = getOldCSI())
300 return OuterRegionInfo->getHelperName();
301 llvm_unreachable(
"No helper name for inlined OpenMP construct");
304 void emitUntiedSwitch(CodeGenFunction &CGF)
override {
306 OuterRegionInfo->emitUntiedSwitch(CGF);
309 CodeGenFunction::CGCapturedStmtInfo *getOldCSI()
const {
return OldCSI; }
311 static bool classof(
const CGCapturedStmtInfo *Info) {
312 return CGOpenMPRegionInfo::classof(Info) &&
316 ~CGOpenMPInlinedRegionInfo()
override =
default;
320 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
321 CGOpenMPRegionInfo *OuterRegionInfo;
329class CGOpenMPTargetRegionInfo final :
public CGOpenMPRegionInfo {
331 CGOpenMPTargetRegionInfo(
const CapturedStmt &CS,
332 const RegionCodeGenTy &CodeGen, StringRef HelperName)
333 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
335 HelperName(HelperName) {}
339 const VarDecl *getThreadIDVariable()
const override {
return nullptr; }
342 StringRef getHelperName()
const override {
return HelperName; }
344 static bool classof(
const CGCapturedStmtInfo *Info) {
345 return CGOpenMPRegionInfo::classof(Info) &&
350 StringRef HelperName;
354 llvm_unreachable(
"No codegen for expressions");
358class CGOpenMPInnerExprInfo final :
public CGOpenMPInlinedRegionInfo {
360 CGOpenMPInnerExprInfo(CodeGenFunction &CGF,
const CapturedStmt &CS)
361 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
369 if (!C.capturesVariable() && !C.capturesVariableByCopy())
372 const VarDecl *VD = C.getCapturedVar();
373 if (VD->isLocalVarDeclOrParm())
376 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
378 VD->getType().getNonReferenceType(), VK_LValue,
380 PrivScope.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress());
382 (
void)PrivScope.Privatize();
386 const FieldDecl *lookup(
const VarDecl *VD)
const override {
387 if (
const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
393 void EmitBody(CodeGenFunction &CGF,
const Stmt *S)
override {
394 llvm_unreachable(
"No body for expressions");
399 const VarDecl *getThreadIDVariable()
const override {
400 llvm_unreachable(
"No thread id for expressions");
404 StringRef getHelperName()
const override {
405 llvm_unreachable(
"No helper name for expressions");
408 static bool classof(
const CGCapturedStmtInfo *Info) {
return false; }
412 CodeGenFunction::OMPPrivateScope PrivScope;
416class InlinedOpenMPRegionRAII {
417 CodeGenFunction &CGF;
418 llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
419 FieldDecl *LambdaThisCaptureField =
nullptr;
420 const CodeGen::CGBlockInfo *BlockInfo =
nullptr;
421 bool NoInheritance =
false;
428 InlinedOpenMPRegionRAII(CodeGenFunction &CGF,
const RegionCodeGenTy &CodeGen,
430 bool NoInheritance =
true)
431 : CGF(CGF), NoInheritance(NoInheritance) {
433 CGF.CapturedStmtInfo =
new CGOpenMPInlinedRegionInfo(
434 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
436 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
437 LambdaThisCaptureField = CGF.LambdaThisCaptureField;
438 CGF.LambdaThisCaptureField =
nullptr;
439 BlockInfo = CGF.BlockInfo;
440 CGF.BlockInfo =
nullptr;
444 ~InlinedOpenMPRegionRAII() {
448 delete CGF.CapturedStmtInfo;
449 CGF.CapturedStmtInfo = OldCSI;
451 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
452 CGF.LambdaThisCaptureField = LambdaThisCaptureField;
453 CGF.BlockInfo = BlockInfo;
461enum OpenMPLocationFlags :
unsigned {
463 OMP_IDENT_IMD = 0x01,
465 OMP_IDENT_KMPC = 0x02,
467 OMP_ATOMIC_REDUCE = 0x10,
469 OMP_IDENT_BARRIER_EXPL = 0x20,
471 OMP_IDENT_BARRIER_IMPL = 0x40,
473 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
475 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
477 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
479 OMP_IDENT_WORK_LOOP = 0x200,
481 OMP_IDENT_WORK_SECTIONS = 0x400,
483 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
513enum IdentFieldIndex {
515 IdentField_Reserved_1,
519 IdentField_Reserved_2,
521 IdentField_Reserved_3,
530enum OpenMPSchedType {
533 OMP_sch_static_chunked = 33,
535 OMP_sch_dynamic_chunked = 35,
536 OMP_sch_guided_chunked = 36,
537 OMP_sch_runtime = 37,
540 OMP_sch_static_balanced_chunked = 45,
543 OMP_ord_static_chunked = 65,
545 OMP_ord_dynamic_chunked = 67,
546 OMP_ord_guided_chunked = 68,
547 OMP_ord_runtime = 69,
549 OMP_sch_default = OMP_sch_static,
551 OMP_dist_sch_static_chunked = 91,
552 OMP_dist_sch_static = 92,
558 OMP_dist_sch_static_chunked_sch_static_chunkone = 93,
561 OMP_sch_modifier_monotonic = (1 << 29),
563 OMP_sch_modifier_nonmonotonic = (1 << 30),
568class CleanupTy final :
public EHScopeStack::Cleanup {
569 PrePostActionTy *Action;
572 explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
573 void Emit(CodeGenFunction &CGF, Flags )
override {
586 Callback(CodeGen, CGF, *PrePostAction);
597 if (
const auto *CE = dyn_cast<CallExpr>(ReductionOp))
598 if (
const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
599 if (
const auto *DRE =
600 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
601 if (
const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
612 std::pair<llvm::Function *, llvm::Function *>
Reduction =
632 auto *GV =
new llvm::GlobalVariable(
634 llvm::GlobalValue::PrivateLinkage,
Init, Name);
676 assert((!UseSrc || SrcAddr.isValid()) &&
677 "Declare reduction initializer requires the original item address.");
681 llvm::Value *NumElements = CGF.
emitArrayLength(ArrayTy, ElementTy, DestAddr);
685 llvm::Value *SrcBegin =
nullptr;
687 SrcBegin = SrcAddr.emitRawPointer(CGF);
690 llvm::Value *DestEnd =
695 llvm::Value *IsEmpty =
696 CGF.
Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arrayinit.isempty");
697 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
700 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
705 llvm::PHINode *SrcElementPHI =
nullptr;
708 SrcElementPHI = CGF.
Builder.CreatePHI(SrcBegin->getType(), 2,
709 "omp.arraycpy.srcElementPast");
710 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
712 Address(SrcElementPHI, SrcAddr.getElementType(),
713 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
715 llvm::PHINode *DestElementPHI = CGF.
Builder.CreatePHI(
716 DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
717 DestElementPHI->addIncoming(DestBegin, EntryBB);
725 if (EmitDeclareReductionInit) {
727 SrcElementCurrent, ElementTy);
735 llvm::Value *SrcElementNext = CGF.
Builder.CreateConstGEP1_32(
736 SrcAddr.getElementType(), SrcElementPHI, 1,
737 "omp.arraycpy.dest.element");
738 SrcElementPHI->addIncoming(SrcElementNext, CGF.
Builder.GetInsertBlock());
742 llvm::Value *DestElementNext = CGF.
Builder.CreateConstGEP1_32(
744 "omp.arraycpy.dest.element");
747 CGF.
Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
748 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
749 DestElementPHI->addIncoming(DestElementNext, CGF.
Builder.GetInsertBlock());
761 if (
const auto *OASE = dyn_cast<ArraySectionExpr>(E))
766void ReductionCodeGen::emitAggregateInitialization(
768 const OMPDeclareReductionDecl *DRD) {
772 const auto *PrivateVD =
774 bool EmitDeclareReductionInit =
777 EmitDeclareReductionInit,
778 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
779 : PrivateVD->getInit(),
787 ClausesData.reserve(Shareds.size());
788 SharedAddresses.reserve(Shareds.size());
789 Sizes.reserve(Shareds.size());
790 BaseDecls.reserve(Shareds.size());
791 const auto *IOrig = Origs.begin();
792 const auto *IPriv =
Privates.begin();
793 const auto *IRed = ReductionOps.begin();
794 for (
const Expr *Ref : Shareds) {
795 ClausesData.emplace_back(Ref, *IOrig, *IPriv, *IRed);
796 std::advance(IOrig, 1);
797 std::advance(IPriv, 1);
798 std::advance(IRed, 1);
803 assert(SharedAddresses.size() == N && OrigAddresses.size() == N &&
804 "Number of generated lvalues must be exactly N.");
805 LValue First = emitSharedLValue(CGF, ClausesData[N].Shared);
806 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Shared);
807 SharedAddresses.emplace_back(
First, Second);
808 if (ClausesData[N].Shared == ClausesData[N].Ref) {
809 OrigAddresses.emplace_back(
First, Second);
811 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
812 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
813 OrigAddresses.emplace_back(
First, Second);
822 CGF.
getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()),
827 llvm::Value *SizeInChars;
828 auto *ElemType = OrigAddresses[N].first.getAddress().getElementType();
829 auto *ElemSizeOf = llvm::ConstantInt::get(
831 if (AsArraySection) {
833 CGF.
Builder.CreatePtrDiff(OrigAddresses[N].second.getPointer(CGF),
834 OrigAddresses[N].first.getPointer(CGF));
835 SizeInChars = CGF.
Builder.CreateNUWAdd(SizeInChars, ElemSizeOf);
838 CGF.
getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType());
840 Size = ElemSizeOf->isOne()
842 : CGF.
Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
843 Sizes.emplace_back(SizeInChars, Size);
856 assert(!Size && !Sizes[N].second &&
857 "Size should be nullptr for non-variably modified reduction "
872 assert(SharedAddresses.size() > N &&
"No variable was generated");
873 const auto *PrivateVD =
879 (void)DefaultInit(CGF);
880 emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD);
881 }
else if (DRD && (DRD->
getInitializer() || !PrivateVD->hasInit())) {
882 (void)DefaultInit(CGF);
883 QualType SharedType = SharedAddresses[N].first.getType();
885 PrivateAddr, SharedAddr, SharedType);
886 }
else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
889 PrivateVD->
getType().getQualifiers(),
907 CGF.
pushDestroy(DTorKind, PrivateAddr, PrivateType);
926 BaseLV.getType(), BaseLV.getBaseInfo(),
960 const VarDecl *OrigVD =
nullptr;
961 if (
const auto *OASE = dyn_cast<ArraySectionExpr>(Ref)) {
962 const Expr *
Base = OASE->getBase()->IgnoreParenImpCasts();
963 while (
const auto *TempOASE = dyn_cast<ArraySectionExpr>(
Base))
964 Base = TempOASE->getBase()->IgnoreParenImpCasts();
965 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
966 Base = TempASE->getBase()->IgnoreParenImpCasts();
969 }
else if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
970 const Expr *
Base = ASE->getBase()->IgnoreParenImpCasts();
971 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
972 Base = TempASE->getBase()->IgnoreParenImpCasts();
983 BaseDecls.emplace_back(OrigVD);
986 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
988 Address SharedAddr = SharedAddresses[N].first.getAddress();
989 llvm::Value *Adjustment = CGF.
Builder.CreatePtrDiff(
992 llvm::Value *PrivatePointer =
998 SharedAddresses[N].first.getType(),
1001 BaseDecls.emplace_back(
1015 getThreadIDVariable()->
getType()->castAs<PointerType>());
1033LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1036 getThreadIDVariable()->
getType(),
1054 llvm::OpenMPIRBuilderConfig Config(
1055 CGM.getLangOpts().OpenMPIsTargetDevice,
isGPU(),
1056 CGM.getLangOpts().OpenMPOffloadMandatory,
1059 Config.setDefaultTargetAS(
1061 Config.setRuntimeCC(
CGM.getRuntimeCC());
1066 CGM.getLangOpts().OpenMPIsTargetDevice
1067 ?
CGM.getLangOpts().OMPHostIRFile
1072 if (
CGM.getLangOpts().OpenMPForceUSM) {
1074 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(
true);
1082 if (!
Data.getValue().pointsToAliveValue())
1084 auto *GV = dyn_cast<llvm::GlobalVariable>(
Data.getValue());
1087 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1089 GV->eraseFromParent();
1094 return OMPBuilder.createPlatformSpecificName(Parts);
1097static llvm::Function *
1099 const Expr *CombinerInitializer,
const VarDecl *In,
1100 const VarDecl *Out,
bool IsCombiner) {
1103 QualType PtrTy =
C.getPointerType(Ty).withRestrict();
1105 C,
nullptr, Out->getLocation(),
1108 C,
nullptr, In->getLocation(),
1115 {IsCombiner ?
"omp_combiner" :
"omp_initializer",
""});
1116 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1120 Fn->addFnAttr(
"sample-profile-suffix-elision-policy",
"selected");
1122 Fn->removeFnAttr(llvm::Attribute::NoInline);
1123 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1124 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1130 Out->getLocation());
1140 (void)
Scope.Privatize();
1141 if (!IsCombiner && Out->hasInit() &&
1144 Out->getType().getQualifiers(),
1147 if (CombinerInitializer)
1149 Scope.ForceCleanup();
1178std::pair<llvm::Function *, llvm::Function *>
1190struct PushAndPopStackRAII {
1191 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder,
CodeGenFunction &CGF,
1192 bool HasCancel, llvm::omp::Directive Kind)
1193 : OMPBuilder(OMPBuilder) {
1209 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1210 assert(IP == IP.getNodeParent()->end() &&
1211 "Clang CG should cause non-terminated block!");
1212 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1217 return llvm::Error::success();
1222 llvm::OpenMPIRBuilder::FinalizationInfo FI({FiniCB, Kind, HasCancel});
1223 OMPBuilder->pushFinalizationCB(std::move(FI));
1225 ~PushAndPopStackRAII() {
1227 OMPBuilder->popFinalizationCB();
1229 llvm::OpenMPIRBuilder *OMPBuilder;
1238 "thread id variable must be of type kmp_int32 *");
1240 bool HasCancel =
false;
1241 if (
const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1242 HasCancel = OPD->hasCancel();
1243 else if (
const auto *OPD = dyn_cast<OMPTargetParallelDirective>(&D))
1244 HasCancel = OPD->hasCancel();
1245 else if (
const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1246 HasCancel = OPSD->hasCancel();
1247 else if (
const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1248 HasCancel = OPFD->hasCancel();
1249 else if (
const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1250 HasCancel = OPFD->hasCancel();
1251 else if (
const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1252 HasCancel = OPFD->hasCancel();
1253 else if (
const auto *OPFD =
1254 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1255 HasCancel = OPFD->hasCancel();
1256 else if (
const auto *OPFD =
1257 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1258 HasCancel = OPFD->hasCancel();
1263 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind);
1264 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar,
CodeGen, InnermostKind,
1265 HasCancel, OutlinedHelperName);
1271 std::string Suffix =
getName({
"omp_outlined"});
1272 return (Name + Suffix).str();
1280 std::string Suffix =
getName({
"omp",
"reduction",
"reduction_func"});
1281 return (Name + Suffix).str();
1288 const CapturedStmt *CS = D.getCapturedStmt(OMPD_parallel);
1298 const CapturedStmt *CS = D.getCapturedStmt(OMPD_teams);
1305 OutlinedFn->setDoesNotRecurse();
1313 bool Tied,
unsigned &NumberOfParts) {
1316 llvm::Value *ThreadID =
getThreadID(CGF, D.getBeginLoc());
1318 llvm::Value *TaskArgs[] = {
1320 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1323 CGF.EmitRuntimeCall(
OMPBuilder.getOrCreateRuntimeFunction(
1324 CGM.getModule(), OMPRTL___kmpc_omp_task),
1327 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1331 "thread id variable must be of type kmp_int32 for tasks");
1336 bool HasCancel =
false;
1337 if (
const auto *TD = dyn_cast<OMPTaskDirective>(&D))
1338 HasCancel = TD->hasCancel();
1339 else if (
const auto *TD = dyn_cast<OMPTaskLoopDirective>(&D))
1340 HasCancel = TD->hasCancel();
1341 else if (
const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(&D))
1342 HasCancel = TD->hasCancel();
1343 else if (
const auto *TD = dyn_cast<OMPMaskedTaskLoopDirective>(&D))
1344 HasCancel = TD->hasCancel();
1345 else if (
const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(&D))
1346 HasCancel = TD->hasCancel();
1347 else if (
const auto *TD = dyn_cast<OMPParallelMaskedTaskLoopDirective>(&D))
1348 HasCancel = TD->hasCancel();
1351 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar,
CodeGen,
1352 InnermostKind, HasCancel, Action);
1354 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1356 NumberOfParts = Action.getNumberOfParts();
1361 bool AtCurrentPoint) {
1363 assert(!Elem.ServiceInsertPt &&
"Insert point is set already.");
1365 llvm::Value *Undef = llvm::UndefValue::get(CGF.
Int32Ty);
1366 if (AtCurrentPoint) {
1367 Elem.ServiceInsertPt =
new llvm::BitCastInst(Undef, CGF.
Int32Ty,
"svcpt",
1368 CGF.
Builder.GetInsertBlock());
1370 Elem.ServiceInsertPt =
new llvm::BitCastInst(Undef, CGF.
Int32Ty,
"svcpt");
1371 Elem.ServiceInsertPt->insertAfter(CGF.
AllocaInsertPt->getIterator());
1377 if (Elem.ServiceInsertPt) {
1378 llvm::Instruction *Ptr = Elem.ServiceInsertPt;
1379 Elem.ServiceInsertPt =
nullptr;
1380 Ptr->eraseFromParent();
1387 llvm::raw_svector_ostream OS(Buffer);
1396 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.
CurFuncDecl))
1397 OS << FD->getQualifiedNameAsString();
1404 unsigned Flags,
bool EmitLoc) {
1405 uint32_t SrcLocStrSize;
1406 llvm::Constant *SrcLocStr;
1407 if ((!EmitLoc &&
CGM.getCodeGenOpts().getDebugInfo() ==
1408 llvm::codegenoptions::NoDebugInfo) ||
1410 SrcLocStr =
OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1412 std::string FunctionName;
1414 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.
CurFuncDecl))
1415 FunctionName = FD->getQualifiedNameAsString();
1428 SrcLocStr, SrcLocStrSize, llvm::omp::IdentFlag(Flags), Reserved2Flags);
1433 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1436 if (
CGM.getLangOpts().OpenMPIRBuilder) {
1439 uint32_t SrcLocStrSize;
1440 auto *SrcLocStr =
OMPBuilder.getOrCreateSrcLocStr(
1443 OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize));
1446 llvm::Value *ThreadID =
nullptr;
1451 ThreadID = I->second.ThreadID;
1452 if (ThreadID !=
nullptr)
1456 if (
auto *OMPRegionInfo =
1458 if (OMPRegionInfo->getThreadIDVariable()) {
1460 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1464 CGF.
Builder.GetInsertBlock() == TopBlock ||
1469 CGF.
Builder.GetInsertBlock()) {
1473 if (CGF.
Builder.GetInsertBlock() == TopBlock)
1485 if (!Elem.ServiceInsertPt)
1487 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1488 CGF.
Builder.SetInsertPoint(Elem.ServiceInsertPt);
1492 OMPRTL___kmpc_global_thread_num),
1495 Elem.ThreadID =
Call;
1500 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1506 for (
const auto *D : I->second)
1511 for (
const auto *D : I->second)
1523static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind
1525 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
1526 OMPDeclareTargetDeclAttr::getDeviceType(VD);
1528 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1530 switch ((
int)*DevTy) {
1531 case OMPDeclareTargetDeclAttr::DT_Host:
1532 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseHost;
1534 case OMPDeclareTargetDeclAttr::DT_NoHost:
1535 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNoHost;
1537 case OMPDeclareTargetDeclAttr::DT_Any:
1538 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny;
1541 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1546static llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind
1548 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> MapType =
1549 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1551 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1552 switch ((
int)*MapType) {
1553 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_To:
1554 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
1556 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Enter:
1557 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter;
1558 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Link:
1559 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
1561 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Local:
1563 llvm_unreachable(
"MT_Local should not reach convertCaptureClause");
1566 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1575 auto FileInfoCallBack = [&]() {
1585 return OMPBuilder.getTargetEntryUniqueInfo(FileInfoCallBack,
1590 auto AddrOfGlobal = [&VD,
this]() {
return CGM.GetAddrOfGlobal(VD); };
1592 auto LinkageForVariable = [&VD,
this]() {
1593 return CGM.getLLVMLinkageVarDefinition(VD);
1596 std::vector<llvm::GlobalVariable *> GeneratedRefs;
1598 llvm::Type *LlvmPtrTy =
CGM.getTypes().ConvertTypeForMem(
1599 CGM.getContext().getPointerType(VD->
getType()));
1600 llvm::Constant *addr =
OMPBuilder.getAddrOfDeclareTargetVar(
1606 CGM.getMangledName(VD), GeneratedRefs,
CGM.getLangOpts().OpenMPSimd,
1607 CGM.getLangOpts().OMPTargetTriples, LlvmPtrTy, AddrOfGlobal,
1608 LinkageForVariable);
1617 assert(!
CGM.getLangOpts().OpenMPUseTLS ||
1618 !
CGM.getContext().getTargetInfo().isTLSSupported());
1620 std::string Suffix =
getName({
"cache",
""});
1621 return OMPBuilder.getOrCreateInternalVariable(
1622 CGM.Int8PtrPtrTy, Twine(
CGM.getMangledName(VD)).concat(Suffix).str());
1629 if (
CGM.getLangOpts().OpenMPUseTLS &&
1630 CGM.getContext().getTargetInfo().isTLSSupported())
1634 llvm::Value *Args[] = {
1637 CGM.getSize(
CGM.GetTargetTypeStoreSize(VarTy)),
1642 CGM.getModule(), OMPRTL___kmpc_threadprivate_cached),
1654 CGM.getModule(), OMPRTL___kmpc_global_thread_num),
1658 llvm::Value *Args[] = {
1661 Ctor, CopyCtor, Dtor};
1664 CGM.getModule(), OMPRTL___kmpc_threadprivate_register),
1671 if (
CGM.getLangOpts().OpenMPUseTLS &&
1672 CGM.getContext().getTargetInfo().isTLSSupported())
1679 llvm::Value *Ctor =
nullptr, *CopyCtor =
nullptr, *Dtor =
nullptr;
1681 if (
CGM.getLangOpts().CPlusPlus && PerformInit) {
1686 CGM.getContext(),
nullptr, Loc,
1690 const auto &FI =
CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1691 CGM.getContext().VoidPtrTy, Args);
1692 llvm::FunctionType *FTy =
CGM.getTypes().GetFunctionType(FI);
1693 std::string Name =
getName({
"__kmpc_global_ctor_",
""});
1694 llvm::Function *Fn =
1695 CGM.CreateGlobalInitOrCleanUpFunction(FTy, Name, FI, Loc);
1700 CGM.getContext().VoidPtrTy, Dst->getLocation());
1707 CGM.getContext().VoidPtrTy, Dst->getLocation());
1717 CGM.getContext(),
nullptr, Loc,
1721 const auto &FI =
CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1722 CGM.getContext().VoidTy, Args);
1723 llvm::FunctionType *FTy =
CGM.getTypes().GetFunctionType(FI);
1724 std::string Name =
getName({
"__kmpc_global_dtor_",
""});
1725 llvm::Function *Fn =
1726 CGM.CreateGlobalInitOrCleanUpFunction(FTy, Name, FI, Loc);
1734 false,
CGM.getContext().VoidPtrTy, Dst->getLocation());
1749 CopyCtor = llvm::Constant::getNullValue(
CGM.DefaultPtrTy);
1750 if (Ctor ==
nullptr) {
1751 Ctor = llvm::Constant::getNullValue(
CGM.DefaultPtrTy);
1753 if (Dtor ==
nullptr) {
1754 Dtor = llvm::Constant::getNullValue(
CGM.DefaultPtrTy);
1757 auto *InitFunctionTy =
1758 llvm::FunctionType::get(
CGM.VoidTy,
false);
1759 std::string Name =
getName({
"__omp_threadprivate_init_",
""});
1760 llvm::Function *InitFunction =
CGM.CreateGlobalInitOrCleanUpFunction(
1761 InitFunctionTy, Name,
CGM.getTypes().arrangeNullaryFunction());
1765 CGM.getTypes().arrangeNullaryFunction(), ArgList,
1769 return InitFunction;
1777 llvm::GlobalValue *GV) {
1778 std::optional<OMPDeclareTargetDeclAttr *> ActiveAttr =
1779 OMPDeclareTargetDeclAttr::getActiveAttr(FD);
1782 if (!ActiveAttr || !(*ActiveAttr)->getIndirect())
1789 OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(Name, EntryInfo);
1795 llvm::GlobalValue *
Addr = GV;
1796 if (
CGM.getLangOpts().OpenMPIsTargetDevice) {
1797 llvm::PointerType *FnPtrTy = llvm::PointerType::get(
1798 CGM.getLLVMContext(),
1799 CGM.getModule().getDataLayout().getProgramAddressSpace());
1800 Addr =
new llvm::GlobalVariable(
1801 CGM.getModule(), FnPtrTy,
1802 true, llvm::GlobalValue::ExternalLinkage, GV, Name,
1803 nullptr, llvm::GlobalValue::NotThreadLocal,
1804 CGM.getModule().getDataLayout().getDefaultGlobalsAddressSpace());
1805 Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1812 OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(
1813 Name,
Addr,
CGM.GetTargetTypeStoreSize(
CGM.VoidPtrTy).getQuantity(),
1814 llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect,
1815 llvm::GlobalValue::WeakODRLinkage);
1828 llvm::OpenMPIRBuilder &
OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
1844 llvm::GlobalVariable *
Addr = VTable;
1846 OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(AddrName, EntryInfo);
1847 AddrName.append(
"addr");
1849 if (
CGM.getLangOpts().OpenMPIsTargetDevice) {
1850 Addr =
new llvm::GlobalVariable(
1851 CGM.getModule(), VTable->getType(),
1852 true, llvm::GlobalValue::ExternalLinkage, VTable,
1854 nullptr, llvm::GlobalValue::NotThreadLocal,
1855 CGM.getModule().getDataLayout().getDefaultGlobalsAddressSpace());
1856 Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1858 OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(
1860 CGM.getDataLayout().getTypeAllocSize(VTable->getInitializer()->getType()),
1861 llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirectVTable,
1862 llvm::GlobalValue::WeakODRLinkage);
1871 !
CGM.getOpenMPRuntime().VTableDeclMap.contains(
CXXRecord)) {
1872 auto Res =
CGM.getOpenMPRuntime().VTableDeclMap.try_emplace(
CXXRecord, VD);
1877 assert(VTablesAddr &&
"Expected non-null VTable address");
1879 if (VTablesAddr->hasExternalLinkage())
1880 VTablesAddr->setLinkage(llvm::GlobalValue::WeakODRLinkage);
1881 CGM.getOpenMPRuntime().registerVTableOffloadEntry(VTablesAddr, VD);
1899 auto GetVTableDecl = [](
const Expr *E) {
1910 if (
auto *DRE = dyn_cast<DeclRefExpr>(E)) {
1912 if (
auto *BD = dyn_cast<BindingDecl>(DRE->getDecl()))
1916 }
else if (
auto *MRE = dyn_cast<MemberExpr>(E)) {
1917 if (
auto *BaseDRE = dyn_cast<DeclRefExpr>(MRE->getBase())) {
1918 if (
auto *BaseVD = dyn_cast<VarDecl>(BaseDRE->getDecl()))
1922 return std::pair<CXXRecordDecl *, const VarDecl *>(
CXXRecord, VD);
1926 for (
const auto *E :
C->varlist()) {
1927 auto DeclPair = GetVTableDecl(E);
1929 if (DeclPair.second)
1938 std::string Suffix =
getName({
"artificial",
""});
1940 llvm::GlobalVariable *GAddr =
OMPBuilder.getOrCreateInternalVariable(
1941 VarLVType, Twine(Name).concat(Suffix).str());
1942 if (
CGM.getLangOpts().OpenMP &&
CGM.getLangOpts().OpenMPUseTLS &&
1943 CGM.getTarget().isTLSSupported()) {
1944 GAddr->setThreadLocal(
true);
1945 return Address(GAddr, GAddr->getValueType(),
1946 CGM.getContext().getTypeAlignInChars(VarType));
1948 std::string CacheSuffix =
getName({
"cache",
""});
1949 llvm::Value *Args[] = {
1957 Twine(Name).concat(Suffix).concat(CacheSuffix).str())};
1962 CGM.getModule(), OMPRTL___kmpc_threadprivate_cached),
1965 VarLVType,
CGM.getContext().getTypeAlignInChars(VarType));
2022 if (!
CGM.getLangOpts().OpenMPIsTargetDevice) {
2023 assert(OutlinedFn->arg_size() == CapturedVars.size() + 2 &&
2024 "Expected tid and bound tid before the captured arguments");
2025 for (
auto [I, Captured] : llvm::enumerate(CapturedVars)) {
2026 if (Captured->getType()->isPointerTy() &&
2027 llvm::isa<llvm::AllocaInst, llvm::GlobalVariable>(
2028 llvm::getUnderlyingObject(Captured)))
2029 OutlinedFn->addParamAttr(I + 2, llvm::Attribute::NoFreeObj);
2034 auto &M =
CGM.getModule();
2035 auto &&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc,
2038 llvm::Value *Args[] = {
2040 CGF.
Builder.getInt32(CapturedVars.size()),
2043 RealArgs.append(std::begin(Args), std::end(Args));
2044 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
2046 llvm::FunctionCallee RTLFn =
2047 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_fork_call);
2050 auto &&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc, Loc,
2056 llvm::Value *Args[] = {RTLoc, ThreadID};
2058 M, OMPRTL___kmpc_serialized_parallel),
2065 ".bound.zero.addr");
2070 OutlinedFnArgs.push_back(ZeroAddrBound.
getPointer());
2071 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
2079 OutlinedFn->removeFnAttr(llvm::Attribute::AlwaysInline);
2080 OutlinedFn->addFnAttr(llvm::Attribute::NoInline);
2086 M, OMPRTL___kmpc_end_serialized_parallel),
2105 if (
auto *OMPRegionInfo =
2107 if (OMPRegionInfo->getThreadIDVariable())
2108 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
2118 return ThreadIDTemp;
2122 std::string Prefix = Twine(
"gomp_critical_user_", CriticalName).str();
2123 std::string Name =
getName({Prefix,
"var"});
2124 llvm::GlobalVariable *GV =
2126 CGM.setDSOLocal(GV);
2133 llvm::FunctionCallee EnterCallee;
2135 llvm::FunctionCallee ExitCallee;
2138 llvm::BasicBlock *ContBlock =
nullptr;
2141 CommonActionTy(llvm::FunctionCallee EnterCallee,
2143 llvm::FunctionCallee ExitCallee,
2145 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
2150 llvm::Value *CallBool = CGF.
Builder.CreateIsNotNull(EnterRes);
2154 CGF.
Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
2158 void Done(CodeGenFunction &CGF) {
2163 void Exit(CodeGenFunction &CGF)
override {
2170 StringRef CriticalName,
2179 llvm::FunctionCallee RuntimeFcn =
OMPBuilder.getOrCreateRuntimeFunction(
2181 Hint ? OMPRTL___kmpc_critical_with_hint : OMPRTL___kmpc_critical);
2183 unsigned LockVarArgIdx = 2;
2185 RuntimeFcn.getFunctionType()
2186 ->getParamType(LockVarArgIdx)
2187 ->getPointerAddressSpace())
2189 LockVar, RuntimeFcn.getFunctionType()->getParamType(LockVarArgIdx));
2195 EnterArgs.push_back(CGF.
Builder.CreateIntCast(
2198 CommonActionTy Action(RuntimeFcn, EnterArgs,
2200 CGM.getModule(), OMPRTL___kmpc_end_critical),
2217 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2218 CGM.getModule(), OMPRTL___kmpc_master),
2221 CGM.getModule(), OMPRTL___kmpc_end_master),
2239 llvm::Value *FilterVal = Filter
2241 : llvm::ConstantInt::get(
CGM.Int32Ty, 0);
2246 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2247 CGM.getModule(), OMPRTL___kmpc_masked),
2250 CGM.getModule(), OMPRTL___kmpc_end_masked),
2266 llvm::Value *Args[] = {
2268 llvm::ConstantInt::get(
CGM.IntTy, 0,
true)};
2270 CGM.getModule(), OMPRTL___kmpc_omp_taskyield),
2274 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
2275 Region->emitUntiedSwitch(CGF);
2288 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2289 CGM.getModule(), OMPRTL___kmpc_taskgroup),
2292 CGM.getModule(), OMPRTL___kmpc_end_taskgroup),
2301 unsigned Index,
const VarDecl *Var) {
2330 llvm::GlobalValue::InternalLinkage, Name,
2334 Fn->addFnAttr(
"sample-profile-suffix-elision-policy",
"selected");
2335 Fn->setDoesNotRecurse();
2352 for (
unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
2353 const auto *DestVar =
2357 const auto *SrcVar =
2363 CGF.
EmitOMPCopy(
Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
2378 assert(CopyprivateVars.size() == SrcExprs.size() &&
2379 CopyprivateVars.size() == DstExprs.size() &&
2380 CopyprivateVars.size() == AssignmentOps.size());
2392 if (!CopyprivateVars.empty()) {
2395 C.getIntTypeForBitwidth(32, 1);
2401 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2402 CGM.getModule(), OMPRTL___kmpc_single),
2405 CGM.getModule(), OMPRTL___kmpc_end_single),
2418 llvm::APInt ArraySize(32, CopyprivateVars.size());
2419 QualType CopyprivateArrayTy =
C.getConstantArrayType(
2424 CopyprivateArrayTy,
".omp.copyprivate.cpr_list");
2425 for (
unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
2437 SrcExprs, DstExprs, AssignmentOps, Loc);
2438 llvm::Value *BufSize = CGF.
getTypeSize(CopyprivateArrayTy);
2442 llvm::Value *Args[] = {
2446 CL.emitRawPointer(CGF),
2451 CGM.getModule(), OMPRTL___kmpc_copyprivate),
2467 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2468 CGM.getModule(), OMPRTL___kmpc_ordered),
2471 CGM.getModule(), OMPRTL___kmpc_end_ordered),
2482 if (Kind == OMPD_for)
2483 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
2484 else if (Kind == OMPD_sections)
2485 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
2486 else if (Kind == OMPD_single)
2487 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
2488 else if (Kind == OMPD_barrier)
2489 Flags = OMP_IDENT_BARRIER_EXPL;
2491 Flags = OMP_IDENT_BARRIER_IMPL;
2501 S.getClausesOfKind<OMPOrderedClause>(),
2502 [](
const OMPOrderedClause *
C) { return C->getNumForLoops(); })) {
2503 ScheduleKind = OMPC_SCHEDULE_static;
2505 llvm::APInt ChunkSize(32, 1);
2515 bool ForceSimpleCall) {
2517 auto *OMPRegionInfo =
2520 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
2523 CGF.
Builder.restoreIP(AfterIP);
2536 if (OMPRegionInfo) {
2537 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
2540 OMPRTL___kmpc_cancel_barrier),
2549 CGF.
Builder.CreateCondBr(
Cmp, ExitBB, ContBB);
2561 CGM.getModule(), OMPRTL___kmpc_barrier),
2566 Expr *ME,
bool IsFatal) {
2568 : llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
2571 llvm::Value *Args[] = {
2573 llvm::ConstantInt::get(
CGM.Int32Ty, IsFatal ? 2 : 1),
2574 CGF.
Builder.CreatePointerCast(MVL,
CGM.Int8PtrTy)};
2576 CGM.getModule(), OMPRTL___kmpc_error),
2582 bool Chunked,
bool Ordered) {
2583 switch (ScheduleKind) {
2584 case OMPC_SCHEDULE_static:
2585 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
2586 : (Ordered ? OMP_ord_static : OMP_sch_static);
2587 case OMPC_SCHEDULE_dynamic:
2588 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
2589 case OMPC_SCHEDULE_guided:
2590 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
2591 case OMPC_SCHEDULE_runtime:
2592 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
2593 case OMPC_SCHEDULE_auto:
2594 return Ordered ? OMP_ord_auto : OMP_sch_auto;
2596 assert(!Chunked &&
"chunk was specified but schedule kind not known");
2597 return Ordered ? OMP_ord_static : OMP_sch_static;
2599 llvm_unreachable(
"Unexpected runtime schedule");
2603static OpenMPSchedType
2606 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
2610 bool Chunked)
const {
2611 OpenMPSchedType Schedule =
2613 return Schedule == OMP_sch_static;
2619 return Schedule == OMP_dist_sch_static;
2623 bool Chunked)
const {
2624 OpenMPSchedType Schedule =
2626 return Schedule == OMP_sch_static_chunked;
2632 return Schedule == OMP_dist_sch_static_chunked;
2636 OpenMPSchedType Schedule =
2638 assert(Schedule != OMP_sch_static_chunked &&
"cannot be chunked here");
2639 return Schedule != OMP_sch_static;
2647 case OMPC_SCHEDULE_MODIFIER_monotonic:
2648 Modifier = OMP_sch_modifier_monotonic;
2650 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2651 Modifier = OMP_sch_modifier_nonmonotonic;
2653 case OMPC_SCHEDULE_MODIFIER_simd:
2654 if (Schedule == OMP_sch_static_chunked)
2655 Schedule = OMP_sch_static_balanced_chunked;
2662 case OMPC_SCHEDULE_MODIFIER_monotonic:
2663 Modifier = OMP_sch_modifier_monotonic;
2665 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2666 Modifier = OMP_sch_modifier_nonmonotonic;
2668 case OMPC_SCHEDULE_MODIFIER_simd:
2669 if (Schedule == OMP_sch_static_chunked)
2670 Schedule = OMP_sch_static_balanced_chunked;
2682 if (CGM.
getLangOpts().OpenMP >= 50 && Modifier == 0) {
2683 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
2684 Schedule == OMP_sch_static_balanced_chunked ||
2685 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||
2686 Schedule == OMP_dist_sch_static_chunked ||
2687 Schedule == OMP_dist_sch_static ||
2688 Schedule == OMP_dist_sch_static_chunked_sch_static_chunkone))
2689 Modifier = OMP_sch_modifier_nonmonotonic;
2691 return Schedule | Modifier;
2701 ScheduleKind.
Schedule, DispatchValues.
Chunk !=
nullptr, Ordered);
2703 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
2704 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&
2705 Schedule != OMP_sch_static_balanced_chunked));
2712 llvm::Value *Chunk = DispatchValues.
Chunk ? DispatchValues.
Chunk
2713 : CGF.
Builder.getIntN(IVSize, 1);
2714 llvm::Value *Args[] = {
2718 CGM, Schedule, ScheduleKind.
M1, ScheduleKind.
M2)),
2721 CGF.
Builder.getIntN(IVSize, 1),
2738 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
2739 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
2746 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||
2747 Schedule == OMP_sch_static_balanced_chunked ||
2748 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||
2749 Schedule == OMP_dist_sch_static ||
2750 Schedule == OMP_dist_sch_static_chunked ||
2751 Schedule == OMP_dist_sch_static_chunked_sch_static_chunkone);
2758 llvm::Value *Chunk = Values.
Chunk;
2759 if (Chunk ==
nullptr) {
2760 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
2761 Schedule == OMP_dist_sch_static) &&
2762 "expected static non-chunked schedule");
2766 assert((Schedule == OMP_sch_static_chunked ||
2767 Schedule == OMP_sch_static_balanced_chunked ||
2768 Schedule == OMP_ord_static_chunked ||
2769 Schedule == OMP_dist_sch_static_chunked ||
2770 Schedule == OMP_dist_sch_static_chunked_sch_static_chunkone) &&
2771 "expected static chunked schedule");
2773 llvm::Value *Args[] = {
2793 OpenMPSchedType ScheduleNum =
2795 ? OMP_dist_sch_static_chunked_sch_static_chunkone
2799 "Expected loop-based or sections-based directive.");
2802 ? OMP_IDENT_WORK_LOOP
2803 : OMP_IDENT_WORK_SECTIONS);
2805 llvm::FunctionCallee StaticInitFunction =
2810 ScheduleNum, ScheduleKind.
M1, ScheduleKind.
M2, Values);
2817 OpenMPSchedType ScheduleNum =
2819 llvm::Value *UpdatedLocation =
2822 llvm::FunctionCallee StaticInitFunction;
2823 bool isGPUDistribute =
2824 CGM.getLangOpts().OpenMPIsTargetDevice &&
CGM.getTriple().isGPU();
2825 StaticInitFunction =
OMPBuilder.createForStaticInitFunction(
2836 assert((DKind == OMPD_distribute || DKind == OMPD_for ||
2837 DKind == OMPD_sections) &&
2838 "Expected distribute, for, or sections directive kind");
2842 llvm::Value *Args[] = {
2845 (DKind == OMPD_target_teams_loop)
2846 ? OMP_IDENT_WORK_DISTRIBUTE
2848 ? OMP_IDENT_WORK_LOOP
2849 : OMP_IDENT_WORK_SECTIONS),
2853 CGM.getLangOpts().OpenMPIsTargetDevice &&
CGM.getTriple().isGPU())
2856 CGM.getModule(), OMPRTL___kmpc_distribute_static_fini),
2860 CGM.getModule(), OMPRTL___kmpc_for_static_fini),
2885 llvm::Value *Args[] = {
2893 OMPBuilder.createDispatchNextFunction(IVSize, IVSigned), Args);
2900 const Expr *Message,
2903 return llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
2912 return llvm::ConstantInt::get(
CGM.Int32Ty,
2913 Severity == OMPC_SEVERITY_warning ? 1 : 2);
2929 RuntimeFunction FnID = OMPRTL___kmpc_push_num_threads;
2930 if (Modifier == OMPC_NUMTHREADS_strict) {
2931 FnID = OMPRTL___kmpc_push_num_threads_strict;
2936 OMPBuilder.getOrCreateRuntimeFunction(
CGM.getModule(), FnID), Args);
2940 ProcBindKind ProcBind,
2944 assert(ProcBind != OMP_PROC_BIND_unknown &&
"Unsupported proc_bind value.");
2946 llvm::Value *Args[] = {
2948 llvm::ConstantInt::get(
CGM.IntTy,
unsigned(ProcBind),
true)};
2950 CGM.getModule(), OMPRTL___kmpc_push_proc_bind),
2963 CGM.getModule(), OMPRTL___kmpc_flush),
2970enum KmpTaskTFields {
2997 if (
CGM.getLangOpts().OpenMPSimd ||
OMPBuilder.OffloadInfoManager.empty())
3000 llvm::OpenMPIRBuilder::EmitMetadataErrorReportFunctionTy &&ErrorReportFn =
3001 [
this](llvm::OpenMPIRBuilder::EmitMetadataErrorKind Kind,
3002 const llvm::TargetRegionEntryInfo &EntryInfo) ->
void {
3004 if (Kind != llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR) {
3005 for (
auto I =
CGM.getContext().getSourceManager().fileinfo_begin(),
3006 E =
CGM.getContext().getSourceManager().fileinfo_end();
3008 if (I->getFirst().getUniqueID().getDevice() == EntryInfo.DeviceID &&
3009 I->getFirst().getUniqueID().getFile() == EntryInfo.FileID) {
3010 Loc =
CGM.getContext().getSourceManager().translateFileLineCol(
3011 I->getFirst(), EntryInfo.Line, 1);
3017 case llvm::OpenMPIRBuilder::EMIT_MD_TARGET_REGION_ERROR: {
3018 CGM.getDiags().Report(Loc,
3019 diag::err_target_region_offloading_entry_incorrect)
3020 << EntryInfo.ParentName;
3022 case llvm::OpenMPIRBuilder::EMIT_MD_DECLARE_TARGET_ERROR: {
3023 CGM.getDiags().Report(
3024 Loc, diag::err_target_var_offloading_entry_incorrect_with_parent)
3025 << EntryInfo.ParentName;
3027 case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR: {
3028 CGM.getDiags().Report(diag::err_target_var_offloading_entry_incorrect);
3030 case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_INDIRECT_ERROR: {
3031 unsigned DiagID =
CGM.getDiags().getCustomDiagID(
3033 "target variable is incorrect: the "
3034 "address is invalid.");
3035 CGM.getDiags().Report(DiagID);
3040 OMPBuilder.createOffloadEntriesAndInfoMetadata(ErrorReportFn);
3047 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty,
C.VoidPtrTy};
3050 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
3056struct PrivateHelpersTy {
3057 PrivateHelpersTy(
const Expr *OriginalRef,
const VarDecl *Original,
3059 : OriginalRef(OriginalRef), Original(Original), PrivateCopy(PrivateCopy),
3060 PrivateElemInit(PrivateElemInit) {}
3061 PrivateHelpersTy(
const VarDecl *Original) : Original(Original) {}
3062 const Expr *OriginalRef =
nullptr;
3063 const VarDecl *Original =
nullptr;
3064 const VarDecl *PrivateCopy =
nullptr;
3065 const VarDecl *PrivateElemInit =
nullptr;
3066 bool isLocalPrivate()
const {
3067 return !OriginalRef && !PrivateCopy && !PrivateElemInit;
3070typedef std::pair<CharUnits , PrivateHelpersTy> PrivateDataTy;
3076 if (
const auto *BD = dyn_cast<BindingDecl>(
Decl))
3083 if (!CVD->
hasAttr<OMPAllocateDeclAttr>())
3085 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
3087 return !(AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc &&
3088 !AA->getAllocator());
3098 RecordDecl *RD =
C.buildImplicitRecord(
".kmp_privates.t");
3100 for (
const auto &Pair :
Privates) {
3101 const VarDecl *VD = Pair.second.Original;
3102 const VarDecl *PrivateCopy = Pair.second.PrivateCopy;
3108 Pair.second.OriginalRef &&
3116 if (Pair.second.isLocalPrivate()) {
3139 QualType KmpRoutineEntryPointerQTy) {
3159 CanQualType KmpCmplrdataTy =
C.getCanonicalTagType(UD);
3160 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_task_t");
3190 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_task_t_with_privates");
3210static llvm::Function *
3213 QualType KmpTaskTWithPrivatesPtrQTy,
3215 QualType SharedsPtrTy, llvm::Function *TaskFunction,
3216 llvm::Value *TaskPrivatesMap) {
3222 C,
nullptr, Loc,
nullptr,
3225 const auto &TaskEntryFnInfo =
3227 llvm::FunctionType *TaskEntryTy =
3230 auto *TaskEntry = llvm::Function::Create(
3231 TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.
getModule());
3234 TaskEntry->addFnAttr(
"sample-profile-suffix-elision-policy",
"selected");
3235 TaskEntry->setDoesNotRecurse();
3250 const auto *KmpTaskTWithPrivatesQTyRD =
3255 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
3257 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF);
3259 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
3265 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
3266 llvm::Value *PrivatesParam;
3267 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
3270 PrivatesLVal.getPointer(CGF), CGF.
VoidPtrTy);
3272 PrivatesParam = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
3275 llvm::Value *CommonArgs[] = {
3276 GtidParam, PartidParam, PrivatesParam, TaskPrivatesMap,
3282 std::end(CommonArgs));
3284 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
3287 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
3290 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
3293 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
3296 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
3299 CallArgs.push_back(LBParam);
3300 CallArgs.push_back(UBParam);
3301 CallArgs.push_back(StParam);
3302 CallArgs.push_back(LIParam);
3303 CallArgs.push_back(RParam);
3305 CallArgs.push_back(SharedsParam);
3318 QualType KmpTaskTWithPrivatesPtrQTy,
3319 QualType KmpTaskTWithPrivatesQTy) {
3325 C,
nullptr, Loc,
nullptr,
3328 const auto &DestructorFnInfo =
3330 llvm::FunctionType *DestructorFnTy =
3334 auto *DestructorFn =
3335 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
3340 DestructorFn->addFnAttr(
"sample-profile-suffix-elision-policy",
"selected");
3341 DestructorFn->setDoesNotRecurse();
3349 const auto *KmpTaskTWithPrivatesQTyRD =
3351 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
3353 for (
const auto *Field : FI->getType()->castAsRecordDecl()->fields()) {
3355 Field->getType().isDestructedType()) {
3357 CGF.
pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType());
3361 return DestructorFn;
3381 C,
nullptr, Loc,
nullptr,
3382 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
3384 Args.push_back(TaskPrivatesArg);
3385 llvm::SmallDenseMap<CanonicalDeclPtr<const VarDecl>,
unsigned> PrivateVarsPos;
3387 llvm::SmallDenseMap<const BindingDecl *, unsigned> BindingDeclPos;
3388 unsigned Counter = 1;
3389 for (
const Expr *E :
Data.PrivateVars) {
3391 C,
nullptr, Loc,
nullptr,
3392 C.getPointerType(
C.getPointerType(E->
getType()))
3397 if (
const auto *BD = dyn_cast<BindingDecl>(VD))
3403 for (
const Expr *E :
Data.FirstprivateVars) {
3405 C,
nullptr, Loc,
nullptr,
3406 C.getPointerType(
C.getPointerType(E->
getType()))
3411 if (
const auto *BD = dyn_cast<BindingDecl>(VD))
3417 for (
const Expr *E :
Data.LastprivateVars) {
3419 C,
nullptr, Loc,
nullptr,
3420 C.getPointerType(
C.getPointerType(E->
getType()))
3425 if (
const auto *BD = dyn_cast<BindingDecl>(VD))
3434 Ty =
C.getPointerType(Ty);
3436 Ty =
C.getPointerType(Ty);
3438 C,
nullptr, Loc,
nullptr,
3439 C.getPointerType(
C.getPointerType(Ty)).withConst().withRestrict(),
3441 PrivateVarsPos[VD] = Counter;
3444 const auto &TaskPrivatesMapFnInfo =
3446 llvm::FunctionType *TaskPrivatesMapTy =
3450 auto *TaskPrivatesMap = llvm::Function::Create(
3451 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
3454 TaskPrivatesMapFnInfo);
3456 TaskPrivatesMap->addFnAttr(
"sample-profile-suffix-elision-policy",
3459 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
3460 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
3461 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
3465 TaskPrivatesMapFnInfo, Args, Loc, Loc);
3473 for (
const FieldDecl *Field : PrivatesQTyRD->fields()) {
3477 if (
Privates[Counter].second.OriginalRef) {
3481 LookupVD =
Privates[Counter].second.Original;
3489 if (
const auto *BD = dyn_cast<BindingDecl>(LookupVD)) {
3492 assert(Position &&
"binding not in privates mapping");
3496 const VarDecl *VD = Args[Position];
3500 RefLVal.getAddress(), RefLVal.getType()->castAs<
PointerType>());
3505 return TaskPrivatesMap;
3511 Address KmpTaskSharedsPtr, LValue TDBase,
3517 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
3531 if ((!IsTargetTask && !
Data.FirstprivateVars.empty() && ForDup) ||
3532 (IsTargetTask && KmpTaskSharedsPtr.
isValid())) {
3539 FI = FI->getType()->castAsRecordDecl()->field_begin();
3540 for (
const PrivateDataTy &Pair :
Privates) {
3542 if (Pair.second.isLocalPrivate()) {
3546 const VarDecl *VD = Pair.second.PrivateCopy;
3551 if (
const VarDecl *Elem = Pair.second.PrivateElemInit) {
3552 const VarDecl *OriginalVD = Pair.second.Original;
3555 LValue SharedRefLValue;
3558 if (IsTargetTask && !SharedField) {
3562 ->getNumParams() == 0 &&
3565 ->getDeclContext()) &&
3566 "Expected artificial target data variable.");
3569 }
else if (ForDup) {
3573 if (Pair.second.OriginalRef) {
3574 if (
const auto *DRE =
3575 dyn_cast<DeclRefExpr>(Pair.second.OriginalRef)) {
3576 if (
const auto *BD = dyn_cast<BindingDecl>(DRE->getDecl())) {
3584 It->second = SharedRefLValue.getAddress();
3587 SharedRefLValue = CGF.
EmitLValue(BD->getBinding());
3590 RestoreIt->second = Saved;
3598 Pair.second.OriginalRef &&
3602 SharedRefLValue.getAddress().withAlignment(
3603 IsBinding ?
C.toCharUnitsFromBits(
3604 C.getTypeAlign(SharedRefLValue.getType()))
3605 :
C.getDeclAlign(OriginalVD)),
3607 SharedRefLValue.getTBAAInfo());
3609 Pair.second.Original->getCanonicalDecl()) > 0 ||
3611 SharedRefLValue = CGF.
EmitLValue(Pair.second.OriginalRef);
3614 InlinedOpenMPRegionRAII Region(
3617 SharedRefLValue = CGF.
EmitLValue(Pair.second.OriginalRef);
3628 PrivateLValue.getAddress(), SharedRefLValue.getAddress(),
Type,
3629 [&CGF, Elem,
Init, &CapturesInfo](
Address DestElement,
3632 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3633 InitScope.addPrivate(Elem, SrcElement);
3634 (void)InitScope.Privatize();
3636 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
3637 CGF, &CapturesInfo);
3638 CGF.EmitAnyExprToMem(Init, DestElement,
3639 Init->getType().getQualifiers(),
3645 InitScope.
addPrivate(Elem, SharedRefLValue.getAddress());
3654 }
else if (
const VarDecl *OriginalVD = Pair.second.Original) {
3659 if (Pair.second.OriginalRef) {
3660 if (
const auto *DRE = dyn_cast<DeclRefExpr>(Pair.second.OriginalRef)) {
3661 BD = dyn_cast<BindingDecl>(DRE->getDecl());
3671 auto EmitBindingWithTempMap = [&CGF](
const BindingDecl *BD,
3678 It->second = DDAddr;
3687 RestoreIt->second = Saved;
3695 LValue SharedRefLValue;
3699 EmitBindingWithTempMap(BD, SharedRefLValue.getAddress());
3701 SharedRefLValue.getAddress().withAlignment(
3702 C.getDeclAlign(OriginalVD)),
3704 SharedRefLValue.getTBAAInfo());
3708 InlinedOpenMPRegionRAII Region(
3715 SharedRefLValue = EmitBindingWithTempMap(BD, DDAddr);
3728 bool InitRequired =
false;
3729 for (
const PrivateDataTy &Pair :
Privates) {
3730 if (Pair.second.isLocalPrivate())
3732 const VarDecl *VD = Pair.second.PrivateCopy;
3734 InitRequired = InitRequired || (isa_and_nonnull<CXXConstructExpr>(
Init) &&
3739 return InitRequired;
3756 QualType KmpTaskTWithPrivatesPtrQTy,
3763 C,
nullptr, Loc,
nullptr, KmpTaskTWithPrivatesPtrQTy,
3766 C,
nullptr, Loc,
nullptr, KmpTaskTWithPrivatesPtrQTy,
3772 const auto &TaskDupFnInfo =
3776 auto *TaskDup = llvm::Function::Create(
3777 TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.
getModule());
3780 TaskDup->addFnAttr(
"sample-profile-suffix-elision-policy",
"selected");
3781 TaskDup->setDoesNotRecurse();
3791 auto LIFI = std::next(KmpTaskTQTyRD->
field_begin(), KmpTaskTLastIter);
3793 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
3803 if (!
Data.FirstprivateVars.empty()) {
3808 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
3816 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
3827 for (
const PrivateDataTy &P :
Privates) {
3828 if (P.second.isLocalPrivate())
3830 QualType Ty = P.second.Original->getType().getNonReferenceType();
3839class OMPIteratorGeneratorScope final
3841 CodeGenFunction &CGF;
3842 const OMPIteratorExpr *E =
nullptr;
3843 SmallVector<CodeGenFunction::JumpDest, 4> ContDests;
3844 SmallVector<CodeGenFunction::JumpDest, 4> ExitDests;
3845 OMPIteratorGeneratorScope() =
delete;
3846 OMPIteratorGeneratorScope(OMPIteratorGeneratorScope &) =
delete;
3849 OMPIteratorGeneratorScope(CodeGenFunction &CGF,
const OMPIteratorExpr *E)
3850 : CodeGenFunction::OMPPrivateScope(CGF), CGF(CGF), E(E) {
3853 SmallVector<llvm::Value *, 4> Uppers;
3855 Uppers.push_back(CGF.EmitScalarExpr(E->getHelper(I).Upper));
3856 const auto *VD = cast<VarDecl>(E->getIteratorDecl(I));
3857 addPrivate(VD, CGF.CreateMemTemp(VD->getType(), VD->getName()));
3858 const OMPIteratorHelperData &HelperData = E->getHelper(I);
3860 HelperData.CounterVD,
3861 CGF.CreateMemTemp(HelperData.CounterVD->getType(),
"counter.addr"));
3866 const OMPIteratorHelperData &HelperData = E->getHelper(I);
3868 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(HelperData.CounterVD),
3869 HelperData.CounterVD->getType());
3871 CGF.EmitStoreOfScalar(
3872 llvm::ConstantInt::get(CLVal.getAddress().getElementType(), 0),
3874 CodeGenFunction::JumpDest &ContDest =
3875 ContDests.emplace_back(CGF.getJumpDestInCurrentScope(
"iter.cont"));
3876 CodeGenFunction::JumpDest &ExitDest =
3877 ExitDests.emplace_back(CGF.getJumpDestInCurrentScope(
"iter.exit"));
3879 llvm::Value *N = Uppers[I];
3882 CGF.EmitBlock(ContDest.getBlock());
3884 CGF.EmitLoadOfScalar(CLVal, HelperData.CounterVD->getLocation());
3886 HelperData.CounterVD->getType()->isSignedIntegerOrEnumerationType()
3887 ? CGF.Builder.CreateICmpSLT(CVal, N)
3888 : CGF.Builder.CreateICmpULT(CVal, N);
3889 llvm::BasicBlock *BodyBB = CGF.createBasicBlock(
"iter.body");
3890 CGF.Builder.CreateCondBr(Cmp, BodyBB, ExitDest.getBlock());
3892 CGF.EmitBlock(BodyBB);
3894 CGF.EmitIgnoredExpr(HelperData.Update);
3897 ~OMPIteratorGeneratorScope() {
3902 const OMPIteratorHelperData &HelperData = E->
getHelper(I - 1);
3907 CGF.
EmitBlock(ExitDests[I - 1].getBlock(), I == 1);
3913static std::pair<llvm::Value *, llvm::Value *>
3915 const auto *OASE = dyn_cast<OMPArrayShapingExpr>(E);
3918 const Expr *
Base = OASE->getBase();
3923 llvm::Value *SizeVal;
3926 SizeVal = CGF.
getTypeSize(OASE->getBase()->getType()->getPointeeType());
3927 for (
const Expr *SE : OASE->getDimensions()) {
3931 SizeVal = CGF.
Builder.CreateNUWMul(SizeVal, Sz);
3933 }
else if (
const auto *ASE =
3936 Address UpAddrAddress = UpAddrLVal.getAddress();
3937 llvm::Value *UpAddr = CGF.
Builder.CreateConstGEP1_32(
3940 SizeVal = CGF.
Builder.CreatePtrDiff(UpAddr,
Addr,
"",
true);
3944 return std::make_pair(
Addr, SizeVal);
3949 QualType FlagsTy =
C.getIntTypeForBitwidth(32,
false);
3950 if (KmpTaskAffinityInfoTy.
isNull()) {
3952 C.buildImplicitRecord(
"kmp_task_affinity_info_t");
3958 KmpTaskAffinityInfoTy =
C.getCanonicalTagType(KmpAffinityInfoRD);
3965 llvm::Function *TaskFunction,
QualType SharedsTy,
3970 const auto *I =
Data.PrivateCopies.begin();
3971 for (
const Expr *E :
Data.PrivateVars) {
3975 Privates.emplace_back(
C.getDeclAlign(VD),
3976 PrivateHelpersTy(E, VD, CopyVD,
3980 I =
Data.FirstprivateCopies.begin();
3981 const auto *IElemInitRef =
Data.FirstprivateInits.begin();
3982 for (
const Expr *E :
Data.FirstprivateVars) {
3986 const auto *InitVD =
3988 Privates.emplace_back(
C.getDeclAlign(VD),
3989 PrivateHelpersTy(E, VD, CopyVD, InitVD));
3993 I =
Data.LastprivateCopies.begin();
3994 for (
const Expr *E :
Data.LastprivateVars) {
3998 Privates.emplace_back(
C.getDeclAlign(VD),
3999 PrivateHelpersTy(E, VD, CopyVD,
4005 Privates.emplace_back(
CGM.getPointerAlign(), PrivateHelpersTy(VD));
4007 Privates.emplace_back(
C.getDeclAlign(VD), PrivateHelpersTy(VD));
4010 [](
const PrivateDataTy &L,
const PrivateDataTy &R) {
4011 return L.first > R.first;
4013 QualType KmpInt32Ty =
C.getIntTypeForBitwidth(32, 1);
4024 assert((D.getDirectiveKind() == OMPD_task ||
4027 "Expected taskloop, task or target directive");
4034 const auto *KmpTaskTQTyRD =
KmpTaskTQTy->castAsRecordDecl();
4036 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
4039 C.getCanonicalTagType(KmpTaskTWithPrivatesQTyRD);
4040 QualType KmpTaskTWithPrivatesPtrQTy =
4041 C.getPointerType(KmpTaskTWithPrivatesQTy);
4042 llvm::Type *KmpTaskTWithPrivatesPtrTy = CGF.
Builder.getPtrTy(0);
4043 llvm::Value *KmpTaskTWithPrivatesTySize =
4045 QualType SharedsPtrTy =
C.getPointerType(SharedsTy);
4048 llvm::Value *TaskPrivatesMap =
nullptr;
4049 llvm::Type *TaskPrivatesMapTy =
4050 std::next(TaskFunction->arg_begin(), 3)->getType();
4052 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
4056 TaskPrivatesMap, TaskPrivatesMapTy);
4058 TaskPrivatesMap = llvm::ConstantPointerNull::get(
4064 CGM, Loc, D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4065 KmpTaskTWithPrivatesQTy,
KmpTaskTQTy, SharedsPtrTy, TaskFunction,
4077 DestructorsFlag = 0x8,
4078 PriorityFlag = 0x20,
4079 DetachableFlag = 0x40,
4080 FreeAgentFlag = 0x80,
4081 TransparentFlag = 0x100,
4083 unsigned Flags =
Data.Tied ? TiedFlag : 0;
4084 bool NeedsCleanup =
false;
4089 Flags = Flags | DestructorsFlag;
4093 if (Kind == OMPC_THREADSET_omp_pool)
4094 Flags = Flags | FreeAgentFlag;
4096 if (D.getSingleClause<OMPTransparentClause>())
4097 Flags |= TransparentFlag;
4099 if (
Data.Priority.getInt())
4100 Flags = Flags | PriorityFlag;
4102 Flags = Flags | DetachableFlag;
4103 llvm::Value *TaskFlags =
4104 Data.Final.getPointer()
4105 ? CGF.
Builder.CreateSelect(
Data.Final.getPointer(),
4106 CGF.
Builder.getInt32(FinalFlag),
4108 : CGF.
Builder.getInt32(
Data.Final.getInt() ? FinalFlag : 0);
4109 TaskFlags = CGF.
Builder.CreateOr(TaskFlags, CGF.
Builder.getInt32(Flags));
4110 llvm::Value *SharedsSize =
CGM.getSize(
C.getTypeSizeInChars(SharedsTy));
4112 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
4115 llvm::Value *NewTask;
4116 if (D.hasClausesOfKind<OMPNowaitClause>()) {
4122 llvm::Value *DeviceID;
4127 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
4128 AllocArgs.push_back(DeviceID);
4131 CGM.getModule(), OMPRTL___kmpc_omp_target_task_alloc),
4136 CGM.getModule(), OMPRTL___kmpc_omp_task_alloc),
4149 llvm::Value *Tid =
getThreadID(CGF, DC->getBeginLoc());
4150 Tid = CGF.
Builder.CreateIntCast(Tid, CGF.
IntTy,
false);
4153 CGM.getModule(), OMPRTL___kmpc_task_allow_completion_event),
4154 {Loc, Tid, NewTask});
4165 llvm::Value *NumOfElements =
nullptr;
4166 unsigned NumAffinities = 0;
4168 if (
const Expr *Modifier =
C->getModifier()) {
4170 for (
unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4174 NumOfElements ? CGF.
Builder.CreateNUWMul(NumOfElements, Sz) : Sz;
4177 NumAffinities +=
C->varlist_size();
4182 enum RTLAffinityInfoFieldsTy { BaseAddr, Len, Flags };
4184 QualType KmpTaskAffinityInfoArrayTy;
4185 if (NumOfElements) {
4186 NumOfElements = CGF.
Builder.CreateNUWAdd(
4187 llvm::ConstantInt::get(CGF.
SizeTy, NumAffinities), NumOfElements);
4190 C.getIntTypeForBitwidth(
C.getTypeSize(
C.getSizeType()), 0),
4194 KmpTaskAffinityInfoArrayTy =
C.getVariableArrayType(
4202 NumOfElements = CGF.
Builder.CreateIntCast(NumOfElements, CGF.
Int32Ty,
4205 KmpTaskAffinityInfoArrayTy =
C.getConstantArrayType(
4207 llvm::APInt(
C.getTypeSize(
C.getSizeType()), NumAffinities),
nullptr,
4212 NumOfElements = llvm::ConstantInt::get(
CGM.Int32Ty, NumAffinities,
4219 bool HasIterator =
false;
4221 if (
C->getModifier()) {
4225 for (
const Expr *E :
C->varlist()) {
4234 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
4239 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
4253 const Expr *Modifier =
C->getModifier();
4256 OMPIteratorGeneratorScope IteratorScope(
4258 for (
const Expr *E :
C->varlist()) {
4268 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
4273 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
4275 Idx = CGF.
Builder.CreateNUWAdd(
4276 Idx, llvm::ConstantInt::get(Idx->getType(), 1));
4291 CGM.getModule(), OMPRTL___kmpc_omp_reg_task_with_affinity),
4292 {LocRef, GTid, NewTask, NumOfElements, AffinListPtr});
4294 llvm::Value *NewTaskNewTaskTTy =
4296 NewTask, KmpTaskTWithPrivatesPtrTy);
4298 KmpTaskTWithPrivatesQTy);
4309 *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds)),
4311 CGF.
Int8Ty,
CGM.getNaturalTypeAlignment(SharedsTy));
4325 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
4326 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy,
Data,
Privates,
4327 !
Data.LastprivateVars.empty());
4331 enum { Priority = 0, Destructors = 1 };
4333 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
4334 const auto *KmpCmplrdataUD = (*FI)->getType()->castAsRecordDecl();
4335 assert(KmpCmplrdataUD->isUnion());
4338 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4339 KmpTaskTWithPrivatesQTy);
4342 Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
4348 if (
Data.Priority.getInt()) {
4350 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
4352 Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority));
4355 Result.NewTask = NewTask;
4356 Result.TaskEntry = TaskEntry;
4357 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
4359 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
4365 RTLDependenceKindTy DepKind;
4367 case OMPC_DEPEND_in:
4368 DepKind = RTLDependenceKindTy::DepIn;
4371 case OMPC_DEPEND_out:
4372 case OMPC_DEPEND_inout:
4373 DepKind = RTLDependenceKindTy::DepInOut;
4375 case OMPC_DEPEND_mutexinoutset:
4376 DepKind = RTLDependenceKindTy::DepMutexInOutSet;
4378 case OMPC_DEPEND_inoutset:
4379 DepKind = RTLDependenceKindTy::DepInOutSet;
4381 case OMPC_DEPEND_outallmemory:
4382 DepKind = RTLDependenceKindTy::DepOmpAllMem;
4384 case OMPC_DEPEND_source:
4385 case OMPC_DEPEND_sink:
4386 case OMPC_DEPEND_depobj:
4387 case OMPC_DEPEND_inoutallmemory:
4389 llvm_unreachable(
"Unknown task dependence type");
4397 FlagsTy =
C.getIntTypeForBitwidth(
C.getTypeSize(
C.BoolTy),
false);
4398 if (KmpDependInfoTy.
isNull()) {
4399 RecordDecl *KmpDependInfoRD =
C.buildImplicitRecord(
"kmp_depend_info");
4405 KmpDependInfoTy =
C.getCanonicalTagType(KmpDependInfoRD);
4409std::pair<llvm::Value *, LValue>
4422 CGF,
Base.getAddress(),
4423 llvm::ConstantInt::get(CGF.
IntPtrTy, -1,
true));
4429 *std::next(KmpDependInfoRD->field_begin(),
4430 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4432 return std::make_pair(NumDeps,
Base);
4436 llvm::PointerUnion<unsigned *, LValue *> Pos,
4446 OMPIteratorGeneratorScope IteratorScope(
4447 CGF, cast_or_null<OMPIteratorExpr>(
4448 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4450 for (
const Expr *E :
Data.DepExprs) {
4460 Size = llvm::ConstantInt::get(CGF.
SizeTy, 0);
4463 if (
unsigned *P = dyn_cast<unsigned *>(Pos)) {
4467 assert(E &&
"Expected a non-null expression");
4476 *std::next(KmpDependInfoRD->field_begin(),
4477 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4481 Base, *std::next(KmpDependInfoRD->field_begin(),
4482 static_cast<unsigned int>(RTLDependInfoFields::Len)));
4488 *std::next(KmpDependInfoRD->field_begin(),
4489 static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4491 llvm::ConstantInt::get(LLVMFlagsTy,
static_cast<unsigned int>(DepKind)),
4493 if (
unsigned *P = dyn_cast<unsigned *>(Pos)) {
4498 Idx = CGF.
Builder.CreateNUWAdd(Idx,
4499 llvm::ConstantInt::get(Idx->getType(), 1));
4508 assert(
Data.DepKind == OMPC_DEPEND_depobj &&
4509 "Expected depobj dependency kind.");
4514 OMPIteratorGeneratorScope IteratorScope(
4515 CGF, cast_or_null<OMPIteratorExpr>(
4516 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4518 for (
const Expr *E :
Data.DepExprs) {
4519 llvm::Value *NumDeps;
4522 std::tie(NumDeps,
Base) =
4526 C.getUIntPtrType());
4530 llvm::Value *Add = CGF.
Builder.CreateNUWAdd(PrevVal, NumDeps);
4532 SizeLVals.push_back(NumLVal);
4535 for (
unsigned I = 0, E = SizeLVals.size(); I < E; ++I) {
4538 Sizes.push_back(Size);
4548 assert(
Data.DepKind == OMPC_DEPEND_depobj &&
4549 "Expected depobj dependency kind.");
4552 OMPIteratorGeneratorScope IteratorScope(
4553 CGF, cast_or_null<OMPIteratorExpr>(
4554 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4556 for (
const Expr *E :
Data.DepExprs) {
4557 llvm::Value *NumDeps;
4560 std::tie(NumDeps,
Base) =
4564 llvm::Value *Size = CGF.
Builder.CreateNUWMul(
4573 llvm::Value *Add = CGF.
Builder.CreateNUWAdd(Pos, NumDeps);
4589 llvm::Value *NumOfElements =
nullptr;
4590 unsigned NumDependencies = std::accumulate(
4591 Dependencies.begin(), Dependencies.end(), 0,
4593 return D.DepKind == OMPC_DEPEND_depobj
4595 : (V + (D.IteratorExpr ? 0 : D.DepExprs.size()));
4599 bool HasDepobjDeps =
false;
4600 bool HasRegularWithIterators =
false;
4601 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4602 llvm::Value *NumOfRegularWithIterators =
4603 llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4607 if (D.
DepKind == OMPC_DEPEND_depobj) {
4610 for (llvm::Value *Size : Sizes) {
4611 NumOfDepobjElements =
4612 CGF.
Builder.CreateNUWAdd(NumOfDepobjElements, Size);
4614 HasDepobjDeps =
true;
4619 if (
const auto *IE = cast_or_null<OMPIteratorExpr>(D.
IteratorExpr)) {
4620 llvm::Value *ClauseIteratorSpace =
4621 llvm::ConstantInt::get(CGF.
IntPtrTy, 1);
4625 ClauseIteratorSpace = CGF.
Builder.CreateNUWMul(Sz, ClauseIteratorSpace);
4627 llvm::Value *NumClauseDeps = CGF.
Builder.CreateNUWMul(
4628 ClauseIteratorSpace,
4630 NumOfRegularWithIterators =
4631 CGF.
Builder.CreateNUWAdd(NumOfRegularWithIterators, NumClauseDeps);
4632 HasRegularWithIterators =
true;
4638 if (HasDepobjDeps || HasRegularWithIterators) {
4639 NumOfElements = llvm::ConstantInt::get(
CGM.IntPtrTy, NumDependencies,
4641 if (HasDepobjDeps) {
4643 CGF.
Builder.CreateNUWAdd(NumOfDepobjElements, NumOfElements);
4645 if (HasRegularWithIterators) {
4647 CGF.
Builder.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements);
4650 Loc,
C.getIntTypeForBitwidth(64, 0),
4654 KmpDependInfoArrayTy =
4663 NumOfElements = CGF.
Builder.CreateIntCast(NumOfElements, CGF.
Int32Ty,
4666 KmpDependInfoArrayTy =
C.getConstantArrayType(
4672 NumOfElements = llvm::ConstantInt::get(
CGM.Int32Ty, NumDependencies,
4677 if (Dep.DepKind == OMPC_DEPEND_depobj || Dep.IteratorExpr)
4687 if (Dep.DepKind == OMPC_DEPEND_depobj || !Dep.IteratorExpr)
4692 if (HasDepobjDeps) {
4694 if (Dep.DepKind != OMPC_DEPEND_depobj)
4701 return std::make_pair(NumOfElements, DependenciesArray);
4712 unsigned NumDependencies = Dependencies.
DepExprs.size();
4722 llvm::Value *NumDepsVal;
4724 if (
const auto *IE =
4725 cast_or_null<OMPIteratorExpr>(Dependencies.
IteratorExpr)) {
4726 NumDepsVal = llvm::ConstantInt::get(CGF.
SizeTy, 1);
4730 NumDepsVal = CGF.
Builder.CreateNUWMul(NumDepsVal, Sz);
4732 Size = CGF.
Builder.CreateNUWAdd(llvm::ConstantInt::get(CGF.
SizeTy, 1),
4736 llvm::Value *RecSize =
CGM.getSize(SizeInBytes);
4737 Size = CGF.
Builder.CreateNUWMul(Size, RecSize);
4741 QualType KmpDependInfoArrayTy =
C.getConstantArrayType(
4744 CharUnits Sz =
C.getTypeSizeInChars(KmpDependInfoArrayTy);
4746 NumDepsVal = llvm::ConstantInt::get(CGF.
IntPtrTy, NumDependencies);
4751 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4752 llvm::Value *Args[] = {ThreadID, Size, Allocator};
4756 CGM.getModule(), OMPRTL___kmpc_alloc),
4757 Args,
".dep.arr.addr");
4761 DependenciesArray =
Address(
Addr, KmpDependInfoLlvmTy, Align);
4767 *std::next(KmpDependInfoRD->field_begin(),
4768 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4770 llvm::PointerUnion<unsigned *, LValue *> Pos;
4787 return DependenciesArray;
4802 Addr.getElementType(),
Addr.emitRawPointer(CGF),
4803 llvm::ConstantInt::get(CGF.
IntPtrTy, -1,
true));
4808 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4809 llvm::Value *Args[] = {ThreadID, DepObjAddr, Allocator};
4813 CGM.getModule(), OMPRTL___kmpc_free),
4825 llvm::Value *NumDeps;
4836 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
4838 llvm::PHINode *ElementPHI =
4843 Base.getTBAAInfo());
4847 Base, *std::next(KmpDependInfoRD->field_begin(),
4848 static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4850 llvm::ConstantInt::get(LLVMFlagsTy,
static_cast<unsigned int>(DepKind)),
4854 llvm::Value *ElementNext =
4857 ElementPHI->addIncoming(ElementNext, CGF.
Builder.GetInsertBlock());
4858 llvm::Value *IsEmpty =
4859 CGF.
Builder.CreateICmpEQ(ElementNext, End,
"omp.isempty");
4860 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
4867 llvm::Function *TaskFunction,
4876 llvm::Value *NewTask =
Result.NewTask;
4877 llvm::Function *TaskEntry =
Result.TaskEntry;
4878 llvm::Value *NewTaskNewTaskTTy =
Result.NewTaskNewTaskTTy;
4883 llvm::Value *NumOfElements;
4884 std::tie(NumOfElements, DependenciesArray) =
4895 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
4896 llvm::Value *DepTaskArgs[7];
4897 if (!
Data.Dependences.empty()) {
4898 DepTaskArgs[0] = UpLoc;
4899 DepTaskArgs[1] = ThreadID;
4900 DepTaskArgs[2] = NewTask;
4901 DepTaskArgs[3] = NumOfElements;
4903 DepTaskArgs[5] = CGF.
Builder.getInt32(0);
4904 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4906 auto &&ThenCodeGen = [
this, &
Data, TDBase, KmpTaskTQTyRD, &TaskArgs,
4909 auto PartIdFI = std::next(KmpTaskTQTyRD->
field_begin(), KmpTaskTPartId);
4913 if (!
Data.Dependences.empty()) {
4916 CGM.getModule(), OMPRTL___kmpc_omp_task_with_deps),
4920 CGM.getModule(), OMPRTL___kmpc_omp_task),
4926 Region->emitUntiedSwitch(CGF);
4929 llvm::Value *DepWaitTaskArgs[7];
4930 if (!
Data.Dependences.empty()) {
4931 DepWaitTaskArgs[0] = UpLoc;
4932 DepWaitTaskArgs[1] = ThreadID;
4933 DepWaitTaskArgs[2] = NumOfElements;
4935 DepWaitTaskArgs[4] = CGF.
Builder.getInt32(0);
4936 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4937 DepWaitTaskArgs[6] =
4938 llvm::ConstantInt::get(CGF.
Int32Ty,
Data.HasNowaitClause);
4940 auto &M =
CGM.getModule();
4941 auto &&ElseCodeGen = [
this, &M, &TaskArgs, ThreadID, NewTaskNewTaskTTy,
4942 TaskEntry, &
Data, &DepWaitTaskArgs,
4949 if (!
Data.Dependences.empty())
4951 M, OMPRTL___kmpc_omp_taskwait_deps_51),
4954 auto &&
CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
4957 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
4958 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
4967 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
4968 M, OMPRTL___kmpc_omp_task_begin_if0),
4971 M, OMPRTL___kmpc_omp_task_complete_if0),
4987 llvm::Function *TaskFunction,
5007 IfVal = llvm::ConstantInt::getSigned(CGF.
IntTy, 1);
5012 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
5019 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
5026 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
5034 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
5035 if (
Data.Reductions) {
5041 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
5050 llvm::ConstantInt::getSigned(
5052 llvm::ConstantInt::getSigned(
5054 ?
Data.Schedule.getInt() ? NumTasks : Grainsize
5056 Data.Schedule.getPointer()
5059 : llvm::ConstantInt::get(CGF.
Int64Ty, 0)};
5060 if (
Data.HasModifier)
5061 TaskArgs.push_back(llvm::ConstantInt::get(CGF.
Int32Ty, 1));
5063 TaskArgs.push_back(
Result.TaskDupFn
5066 : llvm::ConstantPointerNull::get(CGF.
VoidPtrTy));
5068 CGM.getModule(),
Data.HasModifier
5069 ? OMPRTL___kmpc_taskloop_5
5070 : OMPRTL___kmpc_taskloop),
5087 const Expr *,
const Expr *)> &RedOpGen,
5088 const Expr *XExpr =
nullptr,
const Expr *EExpr =
nullptr,
5089 const Expr *UpExpr =
nullptr) {
5097 llvm::Value *NumElements = CGF.
emitArrayLength(ArrayTy, ElementTy, LHSAddr);
5102 llvm::Value *LHSEnd =
5107 llvm::Value *IsEmpty =
5108 CGF.
Builder.CreateICmpEQ(LHSBegin, LHSEnd,
"omp.arraycpy.isempty");
5109 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
5112 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
5117 llvm::PHINode *RHSElementPHI = CGF.
Builder.CreatePHI(
5118 RHSBegin->getType(), 2,
"omp.arraycpy.srcElementPast");
5119 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
5124 llvm::PHINode *LHSElementPHI = CGF.
Builder.CreatePHI(
5125 LHSBegin->getType(), 2,
"omp.arraycpy.destElementPast");
5126 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
5133 Scope.addPrivate(LHSVar, LHSElementCurrent);
5134 Scope.addPrivate(RHSVar, RHSElementCurrent);
5136 RedOpGen(CGF, XExpr, EExpr, UpExpr);
5137 Scope.ForceCleanup();
5140 llvm::Value *LHSElementNext = CGF.
Builder.CreateConstGEP1_32(
5142 "omp.arraycpy.dest.element");
5143 llvm::Value *RHSElementNext = CGF.
Builder.CreateConstGEP1_32(
5145 "omp.arraycpy.src.element");
5148 CGF.
Builder.CreateICmpEQ(LHSElementNext, LHSEnd,
"omp.arraycpy.done");
5149 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
5150 LHSElementPHI->addIncoming(LHSElementNext, CGF.
Builder.GetInsertBlock());
5151 RHSElementPHI->addIncoming(RHSElementNext, CGF.
Builder.GetInsertBlock());
5161 const Expr *ReductionOp) {
5162 if (
const auto *CE = dyn_cast<CallExpr>(ReductionOp))
5163 if (
const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
5164 if (
const auto *DRE =
5165 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
5166 if (
const auto *DRD =
5167 dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
5168 std::pair<llvm::Function *, llvm::Function *>
Reduction =
5179 StringRef ReducerName,
SourceLocation Loc, llvm::Type *ArgsElemType,
5193 CGM.getTypes().arrangeBuiltinFunctionDeclaration(
C.VoidTy, Args);
5195 auto *Fn = llvm::Function::Create(
CGM.getTypes().GetFunctionType(CGFI),
5196 llvm::GlobalValue::InternalLinkage, Name,
5199 if (!
CGM.getCodeGenOpts().SampleProfileFile.empty())
5200 Fn->addFnAttr(
"sample-profile-suffix-elision-policy",
"selected");
5201 Fn->setDoesNotRecurse();
5220 const auto *IPriv =
Privates.begin();
5222 for (
unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
5223 const auto *RHSVar =
5226 const auto *LHSVar =
5229 QualType PrivTy = (*IPriv)->getType();
5245 const auto *ILHS = LHSExprs.begin();
5246 const auto *IRHS = RHSExprs.begin();
5247 for (
const Expr *E : ReductionOps) {
5248 if ((*IPriv)->getType()->isArrayType()) {
5253 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5255 emitReductionCombiner(CGF, E);
5265 Scope.ForceCleanup();
5271 const Expr *ReductionOp,
5272 const Expr *PrivateRef,
5280 CGF, PrivateRef->
getType(), LHSVar, RHSVar,
5282 emitReductionCombiner(CGF, ReductionOp);
5291 llvm::StringRef Prefix,
const Expr *Ref);
5295 const Expr *LHSExprs,
const Expr *RHSExprs,
const Expr *ReductionOps) {
5322 std::string ReductionVarNameStr;
5323 if (
const auto *DRE = dyn_cast<DeclRefExpr>(
Privates->IgnoreParenCasts()))
5324 ReductionVarNameStr =
5327 ReductionVarNameStr =
"unnamed_priv_var";
5330 std::string SharedName =
5331 CGM.getOpenMPRuntime().getName({
"internal_pivate_", ReductionVarNameStr});
5332 llvm::GlobalVariable *SharedVar =
OMPBuilder.getOrCreateInternalVariable(
5333 LLVMType,
".omp.reduction." + SharedName);
5335 SharedVar->setAlignment(
5343 llvm::Value *BarrierArgs[] = {BarrierLoc, ThreadId};
5348 llvm::Value *IsWorker = CGF.
Builder.CreateICmpEQ(
5349 ThreadId, llvm::ConstantInt::get(ThreadId->getType(), 0));
5350 CGF.
Builder.CreateCondBr(IsWorker, InitBB, InitEndBB);
5354 auto EmitSharedInit = [&]() {
5357 std::pair<llvm::Function *, llvm::Function *> FnPair =
5359 llvm::Function *InitializerFn = FnPair.second;
5360 if (InitializerFn) {
5361 if (
const auto *CE =
5362 dyn_cast<CallExpr>(UDRInitExpr->IgnoreParenImpCasts())) {
5369 LocalScope.addPrivate(OutVD, SharedResult);
5371 (void)LocalScope.Privatize();
5372 if (
const auto *OVE = dyn_cast<OpaqueValueExpr>(
5373 CE->getCallee()->IgnoreParenImpCasts())) {
5399 if (
const auto *DRE = dyn_cast<DeclRefExpr>(
Privates)) {
5400 if (
const auto *VD = dyn_cast<VarDecl>(DRE->getDecl())) {
5411 CGF.
Builder.CreateBr(InitEndBB);
5415 CGM.getModule(), OMPRTL___kmpc_barrier),
5418 const Expr *ReductionOp = ReductionOps;
5423 auto EmitCriticalReduction = [&](
auto ReductionGen) {
5424 std::string CriticalName =
getName({
"reduction_critical"});
5432 std::pair<llvm::Function *, llvm::Function *> FnPair =
5435 if (
const auto *CE = dyn_cast<CallExpr>(ReductionOp)) {
5447 (void)LocalScope.Privatize();
5452 EmitCriticalReduction(ReductionGen);
5457 if (
const auto *Cleanup = dyn_cast<ExprWithCleanups>(ReductionClauseExpr))
5460 const Expr *AssignRHS =
nullptr;
5461 if (
const auto *BinOp = dyn_cast<BinaryOperator>(ReductionClauseExpr)) {
5462 if (BinOp->getOpcode() == BO_Assign)
5463 AssignRHS = BinOp->getRHS();
5464 }
else if (
const auto *OpCall =
5465 dyn_cast<CXXOperatorCallExpr>(ReductionClauseExpr)) {
5466 if (OpCall->getOperator() == OO_Equal)
5467 AssignRHS = OpCall->getArg(1);
5471 "Private Variable Reduction : Invalid ReductionOp expression");
5476 const auto *OmpOutDRE =
5478 const auto *OmpInDRE =
5481 OmpOutDRE && OmpInDRE &&
5482 "Private Variable Reduction : LHSExpr/RHSExpr must be DeclRefExprs");
5486 LocalScope.addPrivate(OmpOutVD, SharedLV.
getAddress());
5487 LocalScope.addPrivate(OmpInVD, LHSLV.
getAddress());
5488 (void)LocalScope.Privatize();
5492 EmitCriticalReduction(ReductionGen);
5496 CGM.getModule(), OMPRTL___kmpc_barrier),
5502 llvm::Value *FinalResultVal =
nullptr;
5506 FinalResultAddr = SharedResult;
5520 CGM.getModule(), OMPRTL___kmpc_barrier),
5531 EmitCriticalReduction(OriginalListCombiner);
5583 if (SimpleReduction) {
5585 const auto *IPriv = OrgPrivates.begin();
5586 const auto *ILHS = OrgLHSExprs.begin();
5587 const auto *IRHS = OrgRHSExprs.begin();
5588 for (
const Expr *E : OrgReductionOps) {
5601 FilteredRHSExprs, FilteredReductionOps;
5602 for (
unsigned I : llvm::seq<unsigned>(
5603 std::min(OrgReductionOps.size(), OrgLHSExprs.size()))) {
5605 FilteredPrivates.emplace_back(OrgPrivates[I]);
5606 FilteredLHSExprs.emplace_back(OrgLHSExprs[I]);
5607 FilteredRHSExprs.emplace_back(OrgRHSExprs[I]);
5608 FilteredReductionOps.emplace_back(OrgReductionOps[I]);
5620 auto Size = RHSExprs.size();
5626 llvm::APInt ArraySize(32, Size);
5627 QualType ReductionArrayTy =
C.getConstantArrayType(
5631 CGF.
CreateMemTemp(ReductionArrayTy,
".omp.reduction.red_list");
5632 const auto *IPriv =
Privates.begin();
5634 for (
unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
5640 if ((*IPriv)->getType()->isVariablyModifiedType()) {
5644 llvm::Value *Size = CGF.
Builder.CreateIntCast(
5657 Privates, LHSExprs, RHSExprs, ReductionOps);
5660 std::string Name =
getName({
"reduction"});
5667 llvm::Value *ReductionArrayTySize = CGF.
getTypeSize(ReductionArrayTy);
5670 llvm::Value *Args[] = {
5673 CGF.
Builder.getInt32(RHSExprs.size()),
5674 ReductionArrayTySize,
5682 WithNowait ? OMPRTL___kmpc_reduce_nowait : OMPRTL___kmpc_reduce),
5686 llvm::BasicBlock *DefaultBB = CGF.
createBasicBlock(
".omp.reduction.default");
5687 llvm::SwitchInst *SwInst =
5688 CGF.
Builder.CreateSwitch(Res, DefaultBB, 2);
5697 SwInst->addCase(CGF.
Builder.getInt32(1), Case1BB);
5701 llvm::Value *EndArgs[] = {
5709 const auto *IPriv =
Privates.begin();
5710 const auto *ILHS = LHSExprs.begin();
5711 const auto *IRHS = RHSExprs.begin();
5712 for (
const Expr *E : ReductionOps) {
5721 CommonActionTy Action(
5724 CGM.getModule(), WithNowait ? OMPRTL___kmpc_end_reduce_nowait
5725 : OMPRTL___kmpc_end_reduce),
5738 SwInst->addCase(CGF.
Builder.getInt32(2), Case2BB);
5741 auto &&AtomicCodeGen = [Loc,
Privates, LHSExprs, RHSExprs, ReductionOps](
5743 const auto *ILHS = LHSExprs.begin();
5744 const auto *IRHS = RHSExprs.begin();
5745 const auto *IPriv =
Privates.begin();
5746 for (
const Expr *E : ReductionOps) {
5747 const Expr *XExpr =
nullptr;
5748 const Expr *EExpr =
nullptr;
5749 const Expr *UpExpr =
nullptr;
5751 if (
const auto *BO = dyn_cast<BinaryOperator>(E)) {
5752 if (BO->getOpcode() == BO_Assign) {
5753 XExpr = BO->getLHS();
5754 UpExpr = BO->getRHS();
5758 const Expr *RHSExpr = UpExpr;
5761 if (
const auto *ACO = dyn_cast<AbstractConditionalOperator>(
5765 RHSExpr = ACO->getCond();
5767 if (
const auto *BORHS =
5769 EExpr = BORHS->getRHS();
5770 BO = BORHS->getOpcode();
5775 auto &&AtomicRedGen = [BO, VD,
5777 const Expr *EExpr,
const Expr *UpExpr) {
5778 LValue X = CGF.EmitLValue(XExpr);
5781 E = CGF.EmitAnyExpr(EExpr);
5782 CGF.EmitOMPAtomicSimpleUpdateExpr(
5784 llvm::AtomicOrdering::Monotonic, Loc,
5785 [&CGF, UpExpr, VD, Loc](
RValue XRValue) {
5787 Address LHSTemp = CGF.CreateMemTemp(VD->getType());
5788 CGF.emitOMPSimpleStore(
5789 CGF.MakeAddrLValue(LHSTemp, VD->
getType()), XRValue,
5790 VD->getType().getNonReferenceType(), Loc);
5793 return CGF.EmitAnyExpr(UpExpr);
5796 if ((*IPriv)->getType()->isArrayType()) {
5798 const auto *RHSVar =
5801 AtomicRedGen, XExpr, EExpr, UpExpr);
5804 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5811 std::string Name = RT.
getName({
"atomic_reduction"});
5820 if ((*IPriv)->getType()->isArrayType()) {
5821 const auto *LHSVar =
5823 const auto *RHSVar =
5828 CritRedGen(CGF,
nullptr,
nullptr,
nullptr);
5839 llvm::Value *EndArgs[] = {
5844 CommonActionTy Action(
nullptr, {},
5846 CGM.getModule(), OMPRTL___kmpc_end_reduce),
5856 assert(OrgLHSExprs.size() == OrgPrivates.size() &&
5857 "PrivateVarReduction: Privates size mismatch");
5858 assert(OrgLHSExprs.size() == OrgReductionOps.size() &&
5859 "PrivateVarReduction: ReductionOps size mismatch");
5860 for (
unsigned I : llvm::seq<unsigned>(
5861 std::min(OrgReductionOps.size(), OrgLHSExprs.size()))) {
5864 OrgRHSExprs[I], OrgReductionOps[I]);
5873 llvm::raw_svector_ostream Out(Buffer);
5878 if (
const auto *BD = dyn_cast<BindingDecl>(DRE->getDecl())) {
5888 Out << Prefix << Name <<
"_"
5890 return std::string(Out.str());
5914 Args.emplace_back(Param);
5915 Args.emplace_back(ParamOrig);
5916 const auto &FnInfo =
5920 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5924 Fn->addFnAttr(
"sample-profile-suffix-elision-policy",
"selected");
5925 Fn->setDoesNotRecurse();
5932 llvm::Value *Size =
nullptr;
5975 const Expr *ReductionOp,
5977 const Expr *PrivateRef) {
5988 Args.emplace_back(ParamInOut);
5989 Args.emplace_back(ParamIn);
5990 const auto &FnInfo =
5994 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5998 Fn->addFnAttr(
"sample-profile-suffix-elision-policy",
"selected");
5999 Fn->setDoesNotRecurse();
6002 llvm::Value *Size =
nullptr;
6023 C.getPointerType(LHSVD->getType())->castAs<
PointerType>()));
6030 C.getPointerType(RHSVD->getType())->castAs<
PointerType>()));
6060 Args.emplace_back(Param);
6061 const auto &FnInfo =
6065 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
6069 Fn->addFnAttr(
"sample-profile-suffix-elision-policy",
"selected");
6070 Fn->setDoesNotRecurse();
6075 llvm::Value *Size =
nullptr;
6110 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_taskred_input_t");
6119 C, RD,
C.getIntTypeForBitwidth(32,
false));
6122 unsigned Size =
Data.ReductionVars.size();
6123 llvm::APInt ArraySize(64, Size);
6125 C.getConstantArrayType(RDType, ArraySize,
nullptr,
6130 Data.ReductionCopies,
Data.ReductionOps);
6131 for (
unsigned Cnt = 0; Cnt < Size; ++Cnt) {
6133 llvm::Value *Idxs[] = {llvm::ConstantInt::get(
CGM.SizeTy, 0),
6134 llvm::ConstantInt::get(
CGM.SizeTy, Cnt)};
6150 llvm::Value *SizeValInChars;
6151 llvm::Value *SizeVal;
6152 std::tie(SizeValInChars, SizeVal) = RCG.
getSizes(Cnt);
6158 bool DelayedCreation = !!SizeVal;
6159 SizeValInChars = CGF.
Builder.CreateIntCast(SizeValInChars,
CGM.SizeTy,
6170 llvm::Value *FiniAddr =
6171 Fini ? Fini : llvm::ConstantPointerNull::get(
CGM.VoidPtrTy);
6176 CGM, Loc, RCG, Cnt,
Data.ReductionOps[Cnt], LHSExprs[Cnt],
6177 RHSExprs[Cnt],
Data.ReductionCopies[Cnt]);
6181 if (DelayedCreation) {
6183 llvm::ConstantInt::get(
CGM.Int32Ty, 1,
true),
6188 if (
Data.IsReductionWithTaskMod) {
6194 llvm::Value *Args[] = {
6196 llvm::ConstantInt::get(
CGM.IntTy,
Data.IsWorksharingReduction ? 1 : 0,
6198 llvm::ConstantInt::get(
CGM.IntTy, Size,
true),
6203 CGM.getModule(), OMPRTL___kmpc_taskred_modifier_init),
6207 llvm::Value *Args[] = {
6210 llvm::ConstantInt::get(
CGM.IntTy, Size,
true),
6214 CGM.getModule(), OMPRTL___kmpc_taskred_init),
6220 bool IsWorksharingReduction) {
6226 llvm::Value *Args[] = {IdentTLoc, GTid,
6227 llvm::ConstantInt::get(
CGM.IntTy,
6228 IsWorksharingReduction ? 1 : 0,
6232 CGM.getModule(), OMPRTL___kmpc_task_reduction_modifier_fini),
6244 llvm::Value *SizeVal = CGF.
Builder.CreateIntCast(Sizes.second,
CGM.SizeTy,
6247 CGF,
CGM.getContext().getSizeType(),
6255 llvm::Value *ReductionsPtr,
6268 CGM.getModule(), OMPRTL___kmpc_task_reduction_get_th_data),
6284 auto &M =
CGM.getModule();
6286 llvm::Value *NumOfElements;
6287 std::tie(NumOfElements, DependenciesArray) =
6289 if (!
Data.Dependences.empty()) {
6290 llvm::Value *DepWaitTaskArgs[7];
6291 DepWaitTaskArgs[0] = UpLoc;
6292 DepWaitTaskArgs[1] = ThreadID;
6293 DepWaitTaskArgs[2] = NumOfElements;
6295 DepWaitTaskArgs[4] = CGF.
Builder.getInt32(0);
6296 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
6297 DepWaitTaskArgs[6] =
6298 llvm::ConstantInt::get(CGF.
Int32Ty,
Data.HasNowaitClause);
6307 M, OMPRTL___kmpc_omp_taskwait_deps_51),
6314 llvm::Value *Args[] = {UpLoc, ThreadID};
6317 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_omp_taskwait),
6322 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
6323 Region->emitUntiedSwitch(CGF);
6332 InlinedOpenMPRegionRAII Region(CGF,
CodeGen, InnerKind, HasCancel,
6333 InnerKind != OMPD_critical &&
6334 InnerKind != OMPD_master &&
6335 InnerKind != OMPD_masked);
6350 RTCancelKind CancelKind = CancelNoreq;
6351 if (CancelRegion == OMPD_parallel)
6352 CancelKind = CancelParallel;
6353 else if (CancelRegion == OMPD_for)
6354 CancelKind = CancelLoop;
6355 else if (CancelRegion == OMPD_sections)
6356 CancelKind = CancelSections;
6358 assert(CancelRegion == OMPD_taskgroup);
6359 CancelKind = CancelTaskgroup;
6371 if (
auto *OMPRegionInfo =
6375 if (CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) {
6376 llvm::Value *Args[] = {
6382 CGM.getModule(), OMPRTL___kmpc_cancellationpoint),
6391 CGF.
Builder.CreateCondBr(
Cmp, ExitBB, ContBB);
6393 if (CancelRegion == OMPD_parallel)
6411 auto &M =
CGM.getModule();
6412 if (
auto *OMPRegionInfo =
6414 auto &&ThenGen = [
this, &M, Loc, CancelRegion,
6417 llvm::Value *Args[] = {
6421 llvm::Value *
Result = CGF.EmitRuntimeCall(
6422 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_cancel), Args);
6427 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
".cancel.exit");
6428 llvm::BasicBlock *ContBB = CGF.createBasicBlock(
".cancel.continue");
6429 llvm::Value *
Cmp = CGF.Builder.CreateIsNotNull(
Result);
6430 CGF.Builder.CreateCondBr(
Cmp, ExitBB, ContBB);
6431 CGF.EmitBlock(ExitBB);
6432 if (CancelRegion == OMPD_parallel)
6436 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
6437 CGF.EmitBranchThroughCleanup(CancelDest);
6438 CGF.EmitBlock(ContBB,
true);
6455 bool IsOffloadEntry;
6458 OMPUsesAllocatorsActionTy(
6459 ArrayRef<std::pair<const Expr *, const Expr *>> Allocators,
6461 : Allocators(Allocators), D(D), IsOffloadEntry(IsOffloadEntry) {}
6465 for (
const auto &AllocatorData : Allocators) {
6467 CGF, AllocatorData.first, AllocatorData.second);
6476 void Exit(CodeGenFunction &CGF)
override {
6479 for (
const auto &AllocatorData : Allocators) {
6481 AllocatorData.first);
6489 llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs Attrs;
6490 Attrs.ExecFlags = llvm::omp::OMPTgtExecModeFlags::OMP_TGT_EXEC_MODE_GENERIC;
6499 if (
const auto *Parent =
6502 if (
const auto *Ctor = dyn_cast<CXXConstructorDecl>(FD))
6504 if (
const auto *Dtor = dyn_cast<CXXDestructorDecl>(FD))
6511 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6513 assert(!ParentName.empty() &&
"Invalid target entry parent name!");
6517 for (
unsigned I = 0, E =
C->getNumberOfAllocators(); I < E; ++I) {
6524 OMPUsesAllocatorsActionTy UsesAllocatorAction(Allocators, D, IsOffloadEntry);
6525 CodeGen.setAction(UsesAllocatorAction);
6531 const Expr *Allocator,
6532 const Expr *AllocatorTraits) {
6534 ThreadId = CGF.
Builder.CreateIntCast(ThreadId, CGF.
IntTy,
true);
6536 llvm::Value *MemSpaceHandle = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
6537 llvm::Value *NumTraits = llvm::ConstantInt::get(
6541 .getLimitedValue());
6548 llvm::Value *Traits =
Addr.emitRawPointer(CGF);
6550 llvm::Value *AllocatorVal =
6552 CGM.getModule(), OMPRTL___kmpc_init_allocator),
6553 {ThreadId, MemSpaceHandle, NumTraits, Traits});
6565 const Expr *Allocator) {
6567 ThreadId = CGF.
Builder.CreateIntCast(ThreadId, CGF.
IntTy,
true);
6569 llvm::Value *AllocatorVal =
6576 OMPRTL___kmpc_destroy_allocator),
6577 {ThreadId, AllocatorVal});
6582 llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs) {
6583 assert(Attrs.MaxTeams.size() == 1 && Attrs.MaxThreads.size() == 1 &&
6584 "invalid default attrs structure");
6585 int32_t &MaxTeamsVal = Attrs.MaxTeams.front();
6586 int32_t &MaxThreadsVal = Attrs.MaxThreads.front();
6594 for (
auto *A :
C->getAttrs()) {
6595 int32_t AttrMinThreadsVal = 1, AttrMaxThreadsVal = -1;
6596 int32_t AttrMinBlocksVal = 1, AttrMaxBlocksVal = -1;
6597 if (
auto *
Attr = dyn_cast<CUDALaunchBoundsAttr>(A))
6598 CGM.handleCUDALaunchBoundsAttr(
nullptr,
Attr, &AttrMaxThreadsVal,
6599 &AttrMinBlocksVal, &AttrMaxBlocksVal);
6600 else if (
auto *
Attr = dyn_cast<AMDGPUFlatWorkGroupSizeAttr>(A))
6601 CGM.handleAMDGPUFlatWorkGroupSizeAttr(
6602 nullptr,
Attr,
nullptr, &AttrMinThreadsVal,
6603 &AttrMaxThreadsVal);
6607 Attrs.MinThreads.front() =
6608 std::max(Attrs.MinThreads.front(), AttrMinThreadsVal);
6609 if (AttrMaxThreadsVal > 0)
6610 MaxThreadsVal = MaxThreadsVal > 0
6611 ? std::min(MaxThreadsVal, AttrMaxThreadsVal)
6612 : AttrMaxThreadsVal;
6613 Attrs.MinTeams.front() =
6614 std::max(Attrs.MinTeams.front(), AttrMinBlocksVal);
6615 if (AttrMaxBlocksVal > 0)
6616 MaxTeamsVal = MaxTeamsVal > 0 ? std::min(MaxTeamsVal, AttrMaxBlocksVal)
6624 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6627 llvm::TargetRegionEntryInfo EntryInfo =
6631 llvm::OpenMPIRBuilder::FunctionGenCallback &&GenerateOutlinedFunction =
6632 [&CGF, &D, &
CodeGen,
this](StringRef EntryFnName) {
6633 const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target);
6635 CGOpenMPTargetRegionInfo CGInfo(CS,
CodeGen, EntryFnName);
6637 if (
CGM.getLangOpts().OpenMPIsTargetDevice && !
isGPU())
6642 cantFail(
OMPBuilder.emitTargetRegionFunction(
6643 EntryInfo, GenerateOutlinedFunction, IsOffloadEntry, OutlinedFn,
6651 OutlinedFn->setDoesNotRecurse();
6653 CGM.getTargetCodeGenInfo().setTargetAttributes(
nullptr, OutlinedFn,
CGM);
6656 for (
auto *A :
C->getAttrs()) {
6657 if (
auto *
Attr = dyn_cast<AMDGPUWavesPerEUAttr>(A))
6658 CGM.handleAMDGPUWavesPerEUAttr(OutlinedFn,
Attr);
6677 while (
const auto *
C = dyn_cast_or_null<CompoundStmt>(Child)) {
6679 for (
const Stmt *S :
C->body()) {
6680 if (
const auto *E = dyn_cast<Expr>(S)) {
6689 if (
const auto *DS = dyn_cast<DeclStmt>(S)) {
6690 if (llvm::all_of(DS->decls(), [](
const Decl *D) {
6691 if (isa<EmptyDecl>(D) || isa<DeclContext>(D) ||
6692 isa<TypeDecl>(D) || isa<PragmaCommentDecl>(D) ||
6693 isa<PragmaDetectMismatchDecl>(D) || isa<UsingDecl>(D) ||
6694 isa<UsingDirectiveDecl>(D) ||
6695 isa<OMPDeclareReductionDecl>(D) ||
6696 isa<OMPThreadPrivateDecl>(D) || isa<OMPAllocateDecl>(D))
6698 const auto *VD = dyn_cast<VarDecl>(D);
6701 return VD->hasGlobalStorage() || !VD->isUsed();
6711 Child = Child->IgnoreContainers();
6718 int32_t &MaxTeamsVal) {
6722 "Expected target-based executable directive.");
6723 switch (DirectiveKind) {
6725 const auto *CS = D.getInnermostCapturedStmt();
6728 const Stmt *ChildStmt =
6730 if (
const auto *NestedDir =
6731 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
6740 MinTeamsVal = MaxTeamsVal =
Constant->getExtValue();
6743 MinTeamsVal = MaxTeamsVal = 0;
6746 MinTeamsVal = MaxTeamsVal = 1;
6750 MinTeamsVal = MaxTeamsVal = -1;
6753 case OMPD_target_teams_loop:
6754 case OMPD_target_teams:
6755 case OMPD_target_teams_distribute:
6756 case OMPD_target_teams_distribute_simd:
6757 case OMPD_target_teams_distribute_parallel_for:
6758 case OMPD_target_teams_distribute_parallel_for_simd: {
6760 const Expr *NumTeams =
6764 MinTeamsVal = MaxTeamsVal =
Constant->getExtValue();
6767 MinTeamsVal = MaxTeamsVal = 0;
6770 case OMPD_target_parallel:
6771 case OMPD_target_parallel_for:
6772 case OMPD_target_parallel_for_simd:
6773 case OMPD_target_parallel_loop:
6774 case OMPD_target_simd:
6775 case OMPD_target_loop:
6776 MinTeamsVal = MaxTeamsVal = 1;
6780 case OMPD_parallel_for:
6781 case OMPD_parallel_loop:
6782 case OMPD_parallel_master:
6783 case OMPD_parallel_sections:
6785 case OMPD_parallel_for_simd:
6787 case OMPD_cancellation_point:
6788 case OMPD_ordered_standalone:
6789 case OMPD_ordered_blockassoc:
6790 case OMPD_threadprivate:
6801 case OMPD_taskyield:
6804 case OMPD_taskgroup:
6810 case OMPD_target_data:
6811 case OMPD_target_exit_data:
6812 case OMPD_target_enter_data:
6813 case OMPD_distribute:
6814 case OMPD_distribute_simd:
6815 case OMPD_distribute_parallel_for:
6816 case OMPD_distribute_parallel_for_simd:
6817 case OMPD_teams_distribute:
6818 case OMPD_teams_distribute_simd:
6819 case OMPD_teams_distribute_parallel_for:
6820 case OMPD_teams_distribute_parallel_for_simd:
6821 case OMPD_target_update:
6822 case OMPD_declare_simd:
6823 case OMPD_declare_variant:
6824 case OMPD_begin_declare_variant:
6825 case OMPD_end_declare_variant:
6826 case OMPD_declare_target:
6827 case OMPD_end_declare_target:
6828 case OMPD_declare_reduction:
6829 case OMPD_declare_mapper:
6831 case OMPD_taskloop_simd:
6832 case OMPD_master_taskloop:
6833 case OMPD_master_taskloop_simd:
6834 case OMPD_parallel_master_taskloop:
6835 case OMPD_parallel_master_taskloop_simd:
6837 case OMPD_metadirective:
6843 llvm_unreachable(
"Unexpected directive kind.");
6849 "Clauses associated with the teams directive expected to be emitted "
6850 "only for the host!");
6852 int32_t MinNT = -1, MaxNT = -1;
6853 const Expr *NumTeams =
6855 if (NumTeams !=
nullptr) {
6858 switch (DirectiveKind) {
6860 const auto *CS = D.getInnermostCapturedStmt();
6861 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6865 return Bld.CreateIntCast(NumTeamsVal, CGF.
Int32Ty,
6868 case OMPD_target_teams:
6869 case OMPD_target_teams_distribute:
6870 case OMPD_target_teams_distribute_simd:
6871 case OMPD_target_teams_distribute_parallel_for:
6872 case OMPD_target_teams_distribute_parallel_for_simd: {
6876 return Bld.CreateIntCast(NumTeamsVal, CGF.
Int32Ty,
6884 assert(MinNT == MaxNT &&
"Num threads ranges require handling here.");
6885 return llvm::ConstantInt::getSigned(CGF.
Int32Ty, MinNT);
6899 UpperBound = UpperBound > 0 ? std::min(UpperBound, Val) : Val;
6908 bool UpperBoundOnly, llvm::Value **CondVal) {
6911 const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child);
6918 if (CondVal && Dir->hasClausesOfKind<
OMPIfClause>()) {
6919 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6922 for (
const auto *
C : Dir->getClausesOfKind<
OMPIfClause>()) {
6923 if (
C->getNameModifier() == OMPD_unknown ||
6924 C->getNameModifier() == OMPD_parallel) {
6939 if (
const auto *PreInit =
6941 for (
const auto *I : PreInit->decls()) {
6942 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6958 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6960 const auto *NumThreadsClause =
6962 const Expr *NTExpr = NumThreadsClause->getNumThreads().front();
6963 if (NTExpr->isIntegerConstantExpr(CGF.
getContext()))
6969 if (UpperBound == -1)
6974 if (
const auto *PreInit =
6975 cast_or_null<DeclStmt>(NumThreadsClause->getPreInitStmt())) {
6976 for (
const auto *I : PreInit->decls()) {
6977 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6996 bool UpperBoundOnly, llvm::Value **CondVal,
const Expr **ThreadLimitExpr) {
6997 assert((!CGF.
getLangOpts().OpenMPIsTargetDevice || UpperBoundOnly) &&
6998 "Clauses associated with the teams directive expected to be emitted "
6999 "only for the host!");
7002 "Expected target-based executable directive.");
7004 const Expr *NT =
nullptr;
7005 const Expr **NTPtr = UpperBoundOnly ?
nullptr : &NT;
7007 auto CheckForConstExpr = [&](
const Expr *E,
const Expr **EPtr) {
7011 UpperBound,
static_cast<int32_t
>(
Constant->getZExtValue()));
7015 if (UpperBound == -1)
7021 auto ReturnSequential = [&]() {
7026 switch (DirectiveKind) {
7029 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
7035 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
7037 ThreadLimitClause = TLC;
7038 if (ThreadLimitExpr) {
7039 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
7043 ThreadLimitClause->getThreadLimit().front()->getSourceRange());
7044 if (
const auto *PreInit =
7045 cast_or_null<DeclStmt>(ThreadLimitClause->getPreInitStmt())) {
7046 for (
const auto *I : PreInit->decls()) {
7047 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
7059 if (ThreadLimitClause)
7060 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
7062 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
7065 CS = Dir->getInnermostCapturedStmt();
7079 Dir = dyn_cast_or_null<OMPExecutableDirective>(Child);
7082 CS = Dir->getInnermostCapturedStmt();
7083 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
7085 return ReturnSequential();
7089 case OMPD_target_teams: {
7093 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
7097 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
7100 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
7101 if (Dir->getDirectiveKind() == OMPD_distribute) {
7102 CS = Dir->getInnermostCapturedStmt();
7103 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
7108 case OMPD_target_teams_distribute:
7112 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
7115 getNumThreads(CGF, D.getInnermostCapturedStmt(), NTPtr, UpperBound,
7116 UpperBoundOnly, CondVal);
7118 case OMPD_target_teams_loop:
7119 case OMPD_target_parallel_loop:
7120 case OMPD_target_parallel:
7121 case OMPD_target_parallel_for:
7122 case OMPD_target_parallel_for_simd:
7123 case OMPD_target_teams_distribute_parallel_for:
7124 case OMPD_target_teams_distribute_parallel_for_simd: {
7125 if (CondVal && D.hasClausesOfKind<
OMPIfClause>()) {
7127 for (
const auto *
C : D.getClausesOfKind<
OMPIfClause>()) {
7128 if (
C->getNameModifier() == OMPD_unknown ||
7129 C->getNameModifier() == OMPD_parallel) {
7139 return ReturnSequential();
7149 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
7155 CheckForConstExpr(NumThreadsClause->getNumThreads().front(),
nullptr);
7156 return NumThreadsClause->getNumThreads().front();
7160 case OMPD_target_teams_distribute_simd:
7161 case OMPD_target_simd:
7162 case OMPD_target_loop:
7163 return ReturnSequential();
7167 llvm_unreachable(
"Unsupported directive kind.");
7172 llvm::Value *NumThreadsVal =
nullptr;
7173 llvm::Value *CondVal =
nullptr;
7174 llvm::Value *ThreadLimitVal =
nullptr;
7175 const Expr *ThreadLimitExpr =
nullptr;
7176 int32_t UpperBound = -1;
7179 CGF, D, UpperBound,
false, &CondVal,
7183 if (ThreadLimitExpr) {
7186 ThreadLimitVal = CGF.
Builder.CreateIntCast(ThreadLimitVal, CGF.
Int32Ty,
7191 if (UpperBound == 1) {
7192 NumThreadsVal = CGF.
Builder.getInt32(UpperBound);
7195 NumThreadsVal = CGF.
Builder.CreateIntCast(NumThreadsVal, CGF.
Int32Ty,
7197 }
else if (ThreadLimitVal) {
7200 NumThreadsVal = ThreadLimitVal;
7201 ThreadLimitVal =
nullptr;
7204 assert(!ThreadLimitVal &&
"Default not applicable with thread limit value");
7205 NumThreadsVal = CGF.
Builder.getInt32(0);
7212 NumThreadsVal = CGF.
Builder.CreateSelect(CondVal, NumThreadsVal,
7218 if (ThreadLimitVal) {
7219 NumThreadsVal = CGF.
Builder.CreateSelect(
7220 CGF.
Builder.CreateICmpULT(ThreadLimitVal, NumThreadsVal),
7221 ThreadLimitVal, NumThreadsVal);
7224 return NumThreadsVal;
7234class MappableExprsHandler {
7240 struct AttachPtrExprComparator {
7241 const MappableExprsHandler &Handler;
7243 mutable llvm::DenseMap<std::pair<const Expr *, const Expr *>,
bool>
7244 CachedEqualityComparisons;
7246 AttachPtrExprComparator(
const MappableExprsHandler &H) : Handler(H) {}
7247 AttachPtrExprComparator() =
delete;
7250 bool operator()(
const Expr *LHS,
const Expr *RHS)
const {
7255 const auto ItLHS = Handler.AttachPtrComponentDepthMap.find(LHS);
7256 const auto ItRHS = Handler.AttachPtrComponentDepthMap.find(RHS);
7258 std::optional<size_t> DepthLHS =
7259 (ItLHS != Handler.AttachPtrComponentDepthMap.end()) ? ItLHS->second
7261 std::optional<size_t> DepthRHS =
7262 (ItRHS != Handler.AttachPtrComponentDepthMap.end()) ? ItRHS->second
7266 if (!DepthLHS.has_value() && !DepthRHS.has_value()) {
7268 if (areEqual(LHS, RHS))
7271 return wasComputedBefore(LHS, RHS);
7273 if (!DepthLHS.has_value())
7275 if (!DepthRHS.has_value())
7279 if (DepthLHS.value() != DepthRHS.value())
7280 return DepthLHS.value() < DepthRHS.value();
7283 if (areEqual(LHS, RHS))
7286 return wasComputedBefore(LHS, RHS);
7292 bool areEqual(
const Expr *LHS,
const Expr *RHS)
const {
7294 const auto CachedResultIt = CachedEqualityComparisons.find({LHS, RHS});
7295 if (CachedResultIt != CachedEqualityComparisons.end())
7296 return CachedResultIt->second;
7310 bool wasComputedBefore(
const Expr *LHS,
const Expr *RHS)
const {
7311 const size_t &OrderLHS = Handler.AttachPtrComputationOrderMap.at(LHS);
7312 const size_t &OrderRHS = Handler.AttachPtrComputationOrderMap.at(RHS);
7314 return OrderLHS < OrderRHS;
7323 bool areSemanticallyEqual(
const Expr *LHS,
const Expr *RHS)
const {
7345 if (
const auto *LD = dyn_cast<DeclRefExpr>(LHS)) {
7346 const auto *RD = dyn_cast<DeclRefExpr>(RHS);
7349 return LD->getDecl()->getCanonicalDecl() ==
7350 RD->getDecl()->getCanonicalDecl();
7354 if (
const auto *LA = dyn_cast<ArraySubscriptExpr>(LHS)) {
7355 const auto *RA = dyn_cast<ArraySubscriptExpr>(RHS);
7358 return areSemanticallyEqual(LA->getBase(), RA->getBase()) &&
7359 areSemanticallyEqual(LA->getIdx(), RA->getIdx());
7363 if (
const auto *LM = dyn_cast<MemberExpr>(LHS)) {
7364 const auto *RM = dyn_cast<MemberExpr>(RHS);
7367 if (LM->getMemberDecl()->getCanonicalDecl() !=
7368 RM->getMemberDecl()->getCanonicalDecl())
7370 return areSemanticallyEqual(LM->getBase(), RM->getBase());
7374 if (
const auto *LU = dyn_cast<UnaryOperator>(LHS)) {
7375 const auto *RU = dyn_cast<UnaryOperator>(RHS);
7378 if (LU->getOpcode() != RU->getOpcode())
7380 return areSemanticallyEqual(LU->getSubExpr(), RU->getSubExpr());
7384 if (
const auto *LB = dyn_cast<BinaryOperator>(LHS)) {
7385 const auto *RB = dyn_cast<BinaryOperator>(RHS);
7388 if (LB->getOpcode() != RB->getOpcode())
7390 return areSemanticallyEqual(LB->getLHS(), RB->getLHS()) &&
7391 areSemanticallyEqual(LB->getRHS(), RB->getRHS());
7397 if (
const auto *LAS = dyn_cast<ArraySectionExpr>(LHS)) {
7398 const auto *RAS = dyn_cast<ArraySectionExpr>(RHS);
7401 return areSemanticallyEqual(LAS->getBase(), RAS->getBase()) &&
7402 areSemanticallyEqual(LAS->getLowerBound(),
7403 RAS->getLowerBound()) &&
7404 areSemanticallyEqual(LAS->getLength(), RAS->getLength());
7408 if (
const auto *LC = dyn_cast<CastExpr>(LHS)) {
7409 const auto *RC = dyn_cast<CastExpr>(RHS);
7412 if (LC->getCastKind() != RC->getCastKind())
7414 return areSemanticallyEqual(LC->getSubExpr(), RC->getSubExpr());
7422 if (
const auto *LI = dyn_cast<IntegerLiteral>(LHS)) {
7423 const auto *RI = dyn_cast<IntegerLiteral>(RHS);
7426 return LI->getValue() == RI->getValue();
7430 if (
const auto *LC = dyn_cast<CharacterLiteral>(LHS)) {
7431 const auto *RC = dyn_cast<CharacterLiteral>(RHS);
7434 return LC->getValue() == RC->getValue();
7438 if (
const auto *LF = dyn_cast<FloatingLiteral>(LHS)) {
7439 const auto *RF = dyn_cast<FloatingLiteral>(RHS);
7443 return LF->getValue().bitwiseIsEqual(RF->getValue());
7447 if (
const auto *LS = dyn_cast<StringLiteral>(LHS)) {
7448 const auto *RS = dyn_cast<StringLiteral>(RHS);
7451 return LS->getString() == RS->getString();
7459 if (
const auto *LB = dyn_cast<CXXBoolLiteralExpr>(LHS)) {
7460 const auto *RB = dyn_cast<CXXBoolLiteralExpr>(RHS);
7463 return LB->getValue() == RB->getValue();
7472 static unsigned getFlagMemberOffset() {
7473 unsigned Offset = 0;
7475 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
7476 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
7477 !(Remain & 1); Remain = Remain >> 1)
7484 class MappingExprInfo {
7486 const ValueDecl *MapDecl =
nullptr;
7489 const Expr *MapExpr =
nullptr;
7492 MappingExprInfo(
const ValueDecl *MapDecl,
const Expr *MapExpr =
nullptr)
7493 : MapDecl(MapDecl), MapExpr(MapExpr) {}
7495 const ValueDecl *getMapDecl()
const {
return MapDecl; }
7496 const Expr *getMapExpr()
const {
return MapExpr; }
7499 using DeviceInfoTy = llvm::OpenMPIRBuilder::DeviceInfoTy;
7500 using MapBaseValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
7501 using MapValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
7502 using MapFlagsArrayTy = llvm::OpenMPIRBuilder::MapFlagsArrayTy;
7503 using MapDimArrayTy = llvm::OpenMPIRBuilder::MapDimArrayTy;
7504 using MapNonContiguousArrayTy =
7505 llvm::OpenMPIRBuilder::MapNonContiguousArrayTy;
7506 using MapExprsArrayTy = SmallVector<MappingExprInfo, 4>;
7507 using MapValueDeclsArrayTy = SmallVector<const ValueDecl *, 4>;
7511 bool ,
const ValueDecl *,
const Expr *>;
7512 using MapDataArrayTy = SmallVector<MapData, 4>;
7517 struct MapCombinedInfoTy : llvm::OpenMPIRBuilder::MapInfosTy {
7518 MapExprsArrayTy Exprs;
7519 MapValueDeclsArrayTy Mappers;
7520 MapValueDeclsArrayTy DevicePtrDecls;
7523 void append(MapCombinedInfoTy &CurInfo) {
7524 Exprs.append(CurInfo.Exprs.begin(), CurInfo.Exprs.end());
7525 DevicePtrDecls.append(CurInfo.DevicePtrDecls.begin(),
7526 CurInfo.DevicePtrDecls.end());
7527 Mappers.append(CurInfo.Mappers.begin(), CurInfo.Mappers.end());
7528 llvm::OpenMPIRBuilder::MapInfosTy::append(CurInfo);
7536 struct StructRangeInfoTy {
7537 MapCombinedInfoTy PreliminaryMapData;
7538 std::pair<
unsigned ,
Address > LowestElem = {
7540 std::pair<
unsigned ,
Address > HighestElem = {
7544 bool IsArraySection =
false;
7545 bool HasCompleteRecord =
false;
7550 struct AttachInfoTy {
7553 const ValueDecl *AttachPtrDecl =
nullptr;
7554 const Expr *AttachMapExpr =
nullptr;
7556 bool isValid()
const {
7563 bool hasAttachEntryForCapturedVar(
const ValueDecl *VD)
const {
7564 for (
const auto &AttachEntry : AttachPtrExprMap) {
7565 if (AttachEntry.second) {
7568 if (
const auto *DRE = dyn_cast<DeclRefExpr>(AttachEntry.second))
7569 if (DRE->getDecl() == VD)
7577 const Expr *getAttachPtrExpr(
7580 const auto It = AttachPtrExprMap.find(Components);
7581 if (It != AttachPtrExprMap.end())
7592 ArrayRef<OpenMPMapModifierKind> MapModifiers;
7593 ArrayRef<OpenMPMotionModifierKind> MotionModifiers;
7594 bool ReturnDevicePointer =
false;
7595 bool IsImplicit =
false;
7596 const ValueDecl *Mapper =
nullptr;
7597 const Expr *VarRef =
nullptr;
7598 bool ForDeviceAddr =
false;
7599 bool HasUdpFbNullify =
false;
7601 MapInfo() =
default;
7605 ArrayRef<OpenMPMapModifierKind> MapModifiers,
7606 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
7607 bool ReturnDevicePointer,
bool IsImplicit,
7608 const ValueDecl *Mapper =
nullptr,
const Expr *VarRef =
nullptr,
7609 bool ForDeviceAddr =
false,
bool HasUdpFbNullify =
false)
7610 : Components(Components), MapType(MapType), MapModifiers(MapModifiers),
7611 MotionModifiers(MotionModifiers),
7612 ReturnDevicePointer(ReturnDevicePointer), IsImplicit(IsImplicit),
7613 Mapper(Mapper), VarRef(VarRef), ForDeviceAddr(ForDeviceAddr),
7614 HasUdpFbNullify(HasUdpFbNullify) {}
7619 llvm::PointerUnion<
const OMPExecutableDirective *,
7620 const OMPDeclareMapperDecl *>
7624 CodeGenFunction &CGF;
7629 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>,
bool> FirstPrivateDecls;
7632 llvm::SmallSet<OpenMPDefaultmapClauseKind, 4> DefaultmapFirstprivateKinds;
7638 SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>>
7645 SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>>
7649 llvm::DenseMap<const ValueDecl *, const OMPMapClause *> LambdasMap;
7664 llvm::DenseMap<const Expr *, std::optional<size_t>>
7665 AttachPtrComponentDepthMap = {{
nullptr, std::nullopt}};
7669 llvm::DenseMap<const Expr *, size_t> AttachPtrComputationOrderMap = {
7674 AttachPtrExprComparator AttachPtrComparator;
7676 llvm::Value *getExprTypeSize(
const Expr *E)
const {
7680 if (
const auto *OAE = dyn_cast<OMPArrayShapingExpr>(E)) {
7682 CGF.
getTypeSize(OAE->getBase()->getType()->getPointeeType());
7683 for (
const Expr *SE : OAE->getDimensions()) {
7694 if (
const auto *RefTy =
ExprTy->getAs<ReferenceType>())
7695 ExprTy = RefTy->getPointeeType().getCanonicalType();
7700 if (
const auto *OAE = dyn_cast<ArraySectionExpr>(E)) {
7702 OAE->getBase()->IgnoreParenImpCasts())
7708 if (!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
7709 !OAE->getLowerBound())
7712 llvm::Value *ElemSize;
7713 if (
const auto *PTy = BaseTy->
getAs<PointerType>()) {
7714 ElemSize = CGF.
getTypeSize(PTy->getPointeeType().getCanonicalType());
7717 assert(ATy &&
"Expecting array type if not a pointer type.");
7718 ElemSize = CGF.
getTypeSize(ATy->getElementType().getCanonicalType());
7723 if (!OAE->getLength() && OAE->getColonLocFirst().isInvalid())
7726 if (
const Expr *LenExpr = OAE->getLength()) {
7730 LenExpr->getExprLoc());
7731 return CGF.
Builder.CreateNUWMul(LengthVal, ElemSize);
7733 assert(!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
7734 OAE->getLowerBound() &&
"expected array_section[lb:].");
7740 OAE->getLowerBound()->getExprLoc());
7741 LBVal = CGF.
Builder.CreateNUWMul(LBVal, ElemSize);
7742 llvm::Value *
Cmp = CGF.
Builder.CreateICmpUGT(LengthVal, LBVal);
7743 llvm::Value *TrueVal = CGF.
Builder.CreateNUWSub(LengthVal, LBVal);
7744 LengthVal = CGF.
Builder.CreateSelect(
7745 Cmp, TrueVal, llvm::ConstantInt::get(CGF.
SizeTy, 0));
7755 OpenMPOffloadMappingFlags getMapTypeBits(
7757 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
bool IsImplicit,
7758 bool AddPtrFlag,
bool AddIsTargetParamFlag,
bool IsNonContiguous)
const {
7759 OpenMPOffloadMappingFlags Bits =
7760 IsImplicit ? OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT
7761 : OpenMPOffloadMappingFlags::OMP_MAP_NONE;
7763 case OMPC_MAP_alloc:
7764 case OMPC_MAP_release:
7771 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO;
7774 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_FROM;
7776 case OMPC_MAP_tofrom:
7777 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO |
7778 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
7780 case OMPC_MAP_delete:
7781 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_DELETE;
7784 llvm_unreachable(
"Unexpected map type!");
7787 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
7788 if (AddIsTargetParamFlag)
7789 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
7790 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_always))
7791 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS;
7792 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_close))
7793 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_CLOSE;
7794 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_present) ||
7795 llvm::is_contained(MotionModifiers, OMPC_MOTION_MODIFIER_present))
7796 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
7797 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_ompx_hold))
7798 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
7799 if (IsNonContiguous)
7800 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG;
7806 bool isFinalArraySectionExpression(
const Expr *E)
const {
7807 const auto *OASE = dyn_cast<ArraySectionExpr>(E);
7814 if (OASE->getColonLocFirst().isInvalid())
7817 const Expr *Length = OASE->getLength();
7824 OASE->getBase()->IgnoreParenImpCasts())
7826 if (
const auto *ATy = dyn_cast<ConstantArrayType>(BaseQTy.
getTypePtr()))
7827 return ATy->getSExtSize() != 1;
7839 llvm::APSInt ConstLength =
Result.Val.getInt();
7840 return ConstLength.getSExtValue() != 1;
7847 void emitAttachEntry(CodeGenFunction &CGF, MapCombinedInfoTy &CombinedInfo,
7848 const AttachInfoTy &AttachInfo)
const {
7849 assert(AttachInfo.isValid() &&
7850 "Expected valid attach pointer/pointee information!");
7854 llvm::Value *PointerSize = CGF.
Builder.CreateIntCast(
7855 llvm::ConstantInt::get(
7861 CombinedInfo.Exprs.emplace_back(AttachInfo.AttachPtrDecl,
7862 AttachInfo.AttachMapExpr);
7863 CombinedInfo.BasePointers.push_back(
7864 AttachInfo.AttachPtrAddr.emitRawPointer(CGF));
7865 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
7866 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
7867 CombinedInfo.Pointers.push_back(
7868 AttachInfo.AttachPteeAddr.emitRawPointer(CGF));
7869 CombinedInfo.Sizes.push_back(PointerSize);
7870 CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_ATTACH);
7872 CombinedInfo.HasAttachPtr.push_back(
false);
7873 CombinedInfo.Mappers.push_back(
nullptr);
7874 CombinedInfo.NonContigInfo.Dims.push_back(1);
7881 class CopyOverlappedEntryGaps {
7882 CodeGenFunction &CGF;
7883 MapCombinedInfoTy &CombinedInfo;
7884 OpenMPOffloadMappingFlags Flags = OpenMPOffloadMappingFlags::OMP_MAP_NONE;
7885 const ValueDecl *MapDecl =
nullptr;
7886 const Expr *MapExpr =
nullptr;
7888 bool IsNonContiguous =
false;
7892 const RecordDecl *LastParent =
nullptr;
7894 unsigned LastIndex = -1u;
7898 CopyOverlappedEntryGaps(CodeGenFunction &CGF,
7899 MapCombinedInfoTy &CombinedInfo,
7900 OpenMPOffloadMappingFlags Flags,
7901 const ValueDecl *MapDecl,
const Expr *MapExpr,
7902 Address BP, Address LB,
bool IsNonContiguous,
7904 : CGF(CGF), CombinedInfo(CombinedInfo), Flags(Flags), MapDecl(MapDecl),
7905 MapExpr(MapExpr), BP(BP), IsNonContiguous(IsNonContiguous),
7906 DimSize(DimSize), LB(LB) {}
7909 const OMPClauseMappableExprCommon::MappableComponent &MC,
7910 const FieldDecl *FD,
7911 llvm::function_ref<LValue(CodeGenFunction &,
const MemberExpr *)>
7912 EmitMemberExprBase) {
7922 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
7934 copyUntilField(FD, ComponentLB);
7938 copyUntilField(FD, ComponentLB);
7940 Cursor = FieldOffset + FieldSize;
7945 void copyUntilField(
const FieldDecl *FD, Address ComponentLB) {
7948 llvm::Value *
Size = CGF.
Builder.CreatePtrDiff(ComponentLBPtr, LBPtr);
7949 copySizedChunk(LBPtr, Size);
7952 void copyUntilEnd(Address HB) {
7954 const ASTRecordLayout &RL =
7962 copySizedChunk(LBPtr, Size);
7965 void copySizedChunk(llvm::Value *Base, llvm::Value *Size) {
7966 CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
7968 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
7969 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
7970 CombinedInfo.Pointers.push_back(Base);
7971 CombinedInfo.Sizes.push_back(
7973 CombinedInfo.Types.push_back(Flags);
7974 CombinedInfo.HasAttachPtr.push_back(
false);
7975 CombinedInfo.Mappers.push_back(
nullptr);
7976 CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize : 1);
7985 void generateInfoForComponentList(
7987 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
7989 MapCombinedInfoTy &CombinedInfo,
7990 MapCombinedInfoTy &StructBaseCombinedInfo,
7991 StructRangeInfoTy &PartialStruct, AttachInfoTy &AttachInfo,
7992 bool IsFirstComponentList,
bool IsImplicit,
7993 bool GenerateAllInfoForClauses,
const ValueDecl *Mapper =
nullptr,
7994 bool ForDeviceAddr =
false,
const ValueDecl *BaseDecl =
nullptr,
7995 const Expr *MapExpr =
nullptr,
7996 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
7997 OverlappedElements = {})
const {
8215 bool IsCaptureFirstInfo = IsFirstComponentList;
8219 bool RequiresReference =
false;
8222 auto CI = Components.rbegin();
8223 auto CE = Components.rend();
8228 bool IsExpressionFirstInfo =
true;
8229 bool FirstPointerInComplexData =
false;
8232 const Expr *AssocExpr = I->getAssociatedExpression();
8233 const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr);
8234 const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr);
8235 const auto *OAShE = dyn_cast<OMPArrayShapingExpr>(AssocExpr);
8238 const Expr *AttachPtrExpr = getAttachPtrExpr(Components);
8239 auto [AttachPtrAddr, AttachPteeBaseAddr] =
8240 getAttachPtrAddrAndPteeBaseAddr(AttachPtrExpr, CGF);
8242 bool HasAttachPtr = AttachPtrExpr !=
nullptr;
8243 bool FirstComponentIsForAttachPtr = AssocExpr == AttachPtrExpr;
8244 bool SeenAttachPtr = FirstComponentIsForAttachPtr;
8246 if (FirstComponentIsForAttachPtr) {
8254 }
else if ((AE &&
isa<CXXThisExpr>(AE->getBase()->IgnoreParenImpCasts())) ||
8268 if (
const auto *VD =
8269 dyn_cast_or_null<VarDecl>(I->getAssociatedDeclaration())) {
8270 if (std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
8271 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) {
8272 if ((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
8273 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
8274 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
8276 RequiresReference =
true;
8286 I->getAssociatedDeclaration()->
getType().getNonReferenceType();
8291 const auto *VD = dyn_cast<VarDecl>(I->getAssociatedDeclaration());
8293 !VD || VD->hasLocalStorage() || HasAttachPtr)
8296 FirstPointerInComplexData =
true;
8315 bool ShouldBeMemberOf =
false;
8324 const MemberExpr *EncounteredME =
nullptr;
8336 bool IsNonContiguous =
8337 CombinedInfo.NonContigInfo.IsNonContiguous ||
8338 any_of(Components, [&](
const auto &Component) {
8340 dyn_cast<ArraySectionExpr>(Component.getAssociatedExpression());
8344 const Expr *StrideExpr = OASE->getStride();
8349 "Stride expression must be of integer type");
8362 bool IsPrevMemberReference =
false;
8364 bool IsPartialMapped =
8365 !PartialStruct.PreliminaryMapData.BasePointers.empty();
8372 bool IsMappingWholeStruct =
true;
8373 if (!GenerateAllInfoForClauses) {
8374 IsMappingWholeStruct =
false;
8376 for (
auto TempI = I; TempI != CE; ++TempI) {
8377 const MemberExpr *PossibleME =
8378 dyn_cast<MemberExpr>(TempI->getAssociatedExpression());
8380 IsMappingWholeStruct =
false;
8386 bool SeenFirstNonBinOpExprAfterAttachPtr =
false;
8387 for (; I != CE; ++I) {
8390 if (HasAttachPtr && !SeenAttachPtr) {
8391 SeenAttachPtr = I->getAssociatedExpression() == AttachPtrExpr;
8398 if (HasAttachPtr && !SeenFirstNonBinOpExprAfterAttachPtr) {
8399 const auto *BO = dyn_cast<BinaryOperator>(I->getAssociatedExpression());
8404 SeenFirstNonBinOpExprAfterAttachPtr =
true;
8405 BP = AttachPteeBaseAddr;
8409 if (!EncounteredME) {
8410 EncounteredME = dyn_cast<MemberExpr>(I->getAssociatedExpression());
8413 if (EncounteredME) {
8414 ShouldBeMemberOf =
true;
8417 if (FirstPointerInComplexData) {
8418 QualType Ty = std::prev(I)
8419 ->getAssociatedDeclaration()
8421 .getNonReferenceType();
8423 FirstPointerInComplexData =
false;
8428 auto Next = std::next(I);
8438 bool IsFinalArraySection =
8440 isFinalArraySectionExpression(I->getAssociatedExpression());
8444 const ValueDecl *MapDecl = (I->getAssociatedDeclaration())
8445 ? I->getAssociatedDeclaration()
8447 MapExpr = (I->getAssociatedExpression()) ? I->getAssociatedExpression()
8454 dyn_cast<ArraySectionExpr>(I->getAssociatedExpression());
8456 dyn_cast<OMPArrayShapingExpr>(I->getAssociatedExpression());
8457 const auto *UO = dyn_cast<UnaryOperator>(I->getAssociatedExpression());
8458 const auto *BO = dyn_cast<BinaryOperator>(I->getAssociatedExpression());
8464 I->getAssociatedExpression()->getType()->isAnyPointerType();
8465 bool IsMemberReference =
isa<MemberExpr>(I->getAssociatedExpression()) &&
8468 bool IsNonDerefPointer = IsPointer &&
8469 !(UO && UO->getOpcode() != UO_Deref) && !BO &&
8475 if (
Next == CE || IsMemberReference || IsNonDerefPointer ||
8476 IsFinalArraySection) {
8479 assert((
Next == CE ||
8486 "Unexpected expression");
8490 auto &&EmitMemberExprBase = [](CodeGenFunction &CGF,
8491 const MemberExpr *E) {
8492 const Expr *BaseExpr = E->getBase();
8497 LValueBaseInfo BaseInfo;
8498 TBAAAccessInfo TBAAInfo;
8512 OAShE->getBase()->getType()->getPointeeType()),
8514 OAShE->getBase()->getType()));
8515 }
else if (IsMemberReference) {
8517 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
8532 FinalLowestElem = LowestElem;
8537 bool IsMemberPointerOrAddr =
8539 (((IsPointer || ForDeviceAddr) &&
8540 I->getAssociatedExpression() == EncounteredME) ||
8541 (IsPrevMemberReference && !IsPointer) ||
8542 (IsMemberReference &&
Next != CE &&
8543 !
Next->getAssociatedExpression()->getType()->isPointerType()));
8544 if (!OverlappedElements.empty() &&
Next == CE) {
8546 assert(!PartialStruct.Base.isValid() &&
"The base element is set.");
8547 assert(!IsPointer &&
8548 "Unexpected base element with the pointer type.");
8551 PartialStruct.LowestElem = {0, LowestElem};
8553 I->getAssociatedExpression()->getType());
8558 PartialStruct.HighestElem = {
8559 std::numeric_limits<
decltype(
8560 PartialStruct.HighestElem.first)>
::max(),
8562 PartialStruct.Base = BP;
8563 PartialStruct.LB = LB;
8565 PartialStruct.PreliminaryMapData.BasePointers.empty() &&
8566 "Overlapped elements must be used only once for the variable.");
8567 std::swap(PartialStruct.PreliminaryMapData, CombinedInfo);
8569 OpenMPOffloadMappingFlags Flags =
8570 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
8571 getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,
8573 false, IsNonContiguous);
8574 CopyOverlappedEntryGaps CopyGaps(CGF, CombinedInfo, Flags, MapDecl,
8575 MapExpr, BP, LB, IsNonContiguous,
8579 Component : OverlappedElements) {
8580 for (
const OMPClauseMappableExprCommon::MappableComponent &MC :
8583 if (
const auto *FD = dyn_cast<FieldDecl>(VD)) {
8584 CopyGaps.processField(MC, FD, EmitMemberExprBase);
8589 CopyGaps.copyUntilEnd(HB);
8592 llvm::Value *
Size = getExprTypeSize(I->getAssociatedExpression());
8599 if ((!IsMemberPointerOrAddr && !IsPartialMapped) ||
8601 if (!IsMappingWholeStruct) {
8602 CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
8604 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8605 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8607 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
8609 CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize
8612 StructBaseCombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
8613 StructBaseCombinedInfo.BasePointers.push_back(
8615 StructBaseCombinedInfo.DevicePtrDecls.push_back(
nullptr);
8616 StructBaseCombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8617 StructBaseCombinedInfo.Pointers.push_back(LB.
emitRawPointer(CGF));
8618 StructBaseCombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
8620 StructBaseCombinedInfo.NonContigInfo.Dims.push_back(
8621 IsNonContiguous ? DimSize : 1);
8625 bool HasMapper = Mapper &&
Next == CE;
8626 if (!IsMappingWholeStruct)
8627 CombinedInfo.Mappers.push_back(HasMapper ? Mapper :
nullptr);
8629 StructBaseCombinedInfo.Mappers.push_back(HasMapper ? Mapper
8636 OpenMPOffloadMappingFlags Flags = getMapTypeBits(
8637 MapType, MapModifiers, MotionModifiers, IsImplicit,
8638 !IsExpressionFirstInfo || RequiresReference ||
8639 FirstPointerInComplexData || IsMemberReference,
8640 IsCaptureFirstInfo && !RequiresReference, IsNonContiguous);
8642 if (!IsExpressionFirstInfo || IsMemberReference) {
8645 if (IsPointer || (IsMemberReference &&
Next != CE))
8646 Flags &= ~(OpenMPOffloadMappingFlags::OMP_MAP_TO |
8647 OpenMPOffloadMappingFlags::OMP_MAP_FROM |
8648 OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS |
8649 OpenMPOffloadMappingFlags::OMP_MAP_DELETE |
8650 OpenMPOffloadMappingFlags::OMP_MAP_CLOSE);
8652 if (ShouldBeMemberOf) {
8655 Flags |= OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;
8658 ShouldBeMemberOf =
false;
8662 if (!IsMappingWholeStruct) {
8663 CombinedInfo.Types.push_back(Flags);
8665 CombinedInfo.HasAttachPtr.push_back(HasAttachPtr);
8667 StructBaseCombinedInfo.Types.push_back(Flags);
8668 StructBaseCombinedInfo.HasAttachPtr.push_back(HasAttachPtr);
8675 if (EncounteredME) {
8680 if (!PartialStruct.Base.isValid()) {
8681 PartialStruct.LowestElem = {FieldIndex, LowestElem};
8682 if (IsFinalArraySection && OASE) {
8686 PartialStruct.HighestElem = {FieldIndex, HB};
8688 PartialStruct.HighestElem = {FieldIndex, LowestElem};
8690 PartialStruct.Base = BP;
8691 PartialStruct.LB = BP;
8692 }
else if (FieldIndex < PartialStruct.LowestElem.first) {
8693 PartialStruct.LowestElem = {FieldIndex, LowestElem};
8694 }
else if (FieldIndex > PartialStruct.HighestElem.first) {
8695 if (IsFinalArraySection && OASE) {
8699 PartialStruct.HighestElem = {FieldIndex, HB};
8701 PartialStruct.HighestElem = {FieldIndex, LowestElem};
8707 if (IsFinalArraySection || IsNonContiguous)
8708 PartialStruct.IsArraySection =
true;
8711 if (IsFinalArraySection)
8716 BP = IsMemberReference ? LowestElem : LB;
8717 if (!IsPartialMapped)
8718 IsExpressionFirstInfo =
false;
8719 IsCaptureFirstInfo =
false;
8720 FirstPointerInComplexData =
false;
8721 IsPrevMemberReference = IsMemberReference;
8722 }
else if (FirstPointerInComplexData) {
8723 QualType Ty = Components.rbegin()
8724 ->getAssociatedDeclaration()
8726 .getNonReferenceType();
8728 FirstPointerInComplexData =
false;
8734 PartialStruct.HasCompleteRecord =
true;
8737 if (shouldEmitAttachEntry(AttachPtrExpr, BaseDecl, CGF, CurDir)) {
8738 AttachInfo.AttachPtrAddr = AttachPtrAddr;
8739 AttachInfo.AttachPteeAddr = FinalLowestElem;
8740 AttachInfo.AttachPtrDecl = BaseDecl;
8741 AttachInfo.AttachMapExpr = MapExpr;
8744 if (!IsNonContiguous)
8747 const ASTContext &Context = CGF.
getContext();
8751 MapValuesArrayTy CurOffsets = {llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, 0)};
8752 MapValuesArrayTy CurCounts;
8753 MapValuesArrayTy CurStrides = {llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, 1)};
8754 MapValuesArrayTy DimSizes{llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, 1)};
8760 for (
const OMPClauseMappableExprCommon::MappableComponent &Component :
8762 const Expr *AssocExpr = Component.getAssociatedExpression();
8763 const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr);
8773 assert((VAT || CAT || &Component == &*Components.begin()) &&
8774 "Should be either ConstantArray or VariableArray if not the "
8778 if (CurCounts.empty()) {
8779 const Type *ElementType =
nullptr;
8781 ElementType = CAT->getElementType().getTypePtr();
8783 ElementType = VAT->getElementType().getTypePtr();
8784 else if (&Component == &*Components.begin()) {
8791 if (
const auto *PtrType = Ty->
getAs<PointerType>())
8792 ElementType = PtrType->getPointeeType().getTypePtr();
8798 "Non-first components should not be raw pointers");
8806 if (&Component != &*Components.begin())
8807 ElementType = ElementType->getPointeeOrArrayElementType();
8810 CurCounts.push_back(
8811 llvm::ConstantInt::get(CGF.
Int64Ty, ElementTypeSize));
8816 if (DimSizes.size() < Components.size() - 1) {
8819 llvm::ConstantInt::get(CGF.
Int64Ty, CAT->getZExtSize()));
8821 DimSizes.push_back(CGF.
Builder.CreateIntCast(
8828 auto *DI = DimSizes.begin() + 1;
8830 llvm::Value *DimProd =
8831 llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, ElementTypeSize);
8840 for (
const OMPClauseMappableExprCommon::MappableComponent &Component :
8842 const Expr *AssocExpr = Component.getAssociatedExpression();
8844 if (
const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr)) {
8845 llvm::Value *Offset = CGF.
Builder.CreateIntCast(
8848 CurOffsets.push_back(Offset);
8849 CurCounts.push_back(llvm::ConstantInt::get(CGF.
Int64Ty, 1));
8850 CurStrides.push_back(CurStrides.back());
8854 const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr);
8860 const Expr *OffsetExpr = OASE->getLowerBound();
8861 llvm::Value *Offset =
nullptr;
8864 Offset = llvm::ConstantInt::get(CGF.
Int64Ty, 0);
8872 const Expr *CountExpr = OASE->getLength();
8873 llvm::Value *Count =
nullptr;
8879 if (!OASE->getColonLocFirst().isValid() &&
8880 !OASE->getColonLocSecond().isValid()) {
8881 Count = llvm::ConstantInt::get(CGF.
Int64Ty, 1);
8887 const Expr *StrideExpr = OASE->getStride();
8888 llvm::Value *Stride =
8894 Count = CGF.
Builder.CreateUDiv(
8895 CGF.
Builder.CreateNUWSub(*DI, Offset), Stride);
8897 Count = CGF.
Builder.CreateNUWSub(*DI, Offset);
8903 CurCounts.push_back(Count);
8913 const Expr *StrideExpr = OASE->getStride();
8914 llvm::Value *Stride =
8919 DimProd = CGF.
Builder.CreateNUWMul(DimProd, *(DI - 1));
8921 CurStrides.push_back(CGF.
Builder.CreateNUWMul(DimProd, Stride));
8923 CurStrides.push_back(DimProd);
8925 Offset = CGF.
Builder.CreateNUWMul(DimProd, Offset);
8926 CurOffsets.push_back(Offset);
8928 if (DI != DimSizes.end())
8932 CombinedInfo.NonContigInfo.Offsets.push_back(CurOffsets);
8933 CombinedInfo.NonContigInfo.Counts.push_back(CurCounts);
8934 CombinedInfo.NonContigInfo.Strides.push_back(CurStrides);
8940 OpenMPOffloadMappingFlags
8941 getMapModifiersForPrivateClauses(
const CapturedStmt::Capture &Cap)
const {
8949 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
8950 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
8951 return OpenMPOffloadMappingFlags::OMP_MAP_PRIVATE |
8952 OpenMPOffloadMappingFlags::OMP_MAP_TO;
8955 if (I != LambdasMap.end())
8957 return getMapTypeBits(
8958 I->getSecond()->getMapType(), I->getSecond()->getMapTypeModifiers(),
8959 {}, I->getSecond()->isImplicit(),
8963 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
8964 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
8967 void getPlainLayout(
const CXXRecordDecl *RD,
8968 llvm::SmallVectorImpl<const FieldDecl *> &Layout,
8969 bool AsBase)
const {
8972 llvm::StructType *St =
8975 unsigned NumElements = St->getNumElements();
8977 llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>, 4>
8978 RecordLayout(NumElements);
8981 for (
const auto &I : RD->
bases()) {
8985 QualType BaseTy = I.getType();
8996 RecordLayout[FieldIndex] =
Base;
8999 for (
const auto &I : RD->
vbases()) {
9000 QualType BaseTy = I.getType();
9007 if (RecordLayout[FieldIndex])
9009 RecordLayout[FieldIndex] =
Base;
9012 assert(!RD->
isUnion() &&
"Unexpected union.");
9013 for (
const auto *Field : RD->
fields()) {
9016 if (!
Field->isBitField() &&
9019 RecordLayout[FieldIndex] =
Field;
9022 for (
const llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>
9023 &
Data : RecordLayout) {
9026 if (
const auto *Base = dyn_cast<const CXXRecordDecl *>(
Data))
9027 getPlainLayout(Base, Layout,
true);
9034 static Address getAttachPtrAddr(
const Expr *PointerExpr,
9035 CodeGenFunction &CGF) {
9036 assert(PointerExpr &&
"Cannot get addr from null attach-ptr expr");
9039 if (
auto *DRE = dyn_cast<DeclRefExpr>(PointerExpr)) {
9042 }
else if (
auto *OASE = dyn_cast<ArraySectionExpr>(PointerExpr)) {
9045 }
else if (
auto *ASE = dyn_cast<ArraySubscriptExpr>(PointerExpr)) {
9047 }
else if (
auto *ME = dyn_cast<MemberExpr>(PointerExpr)) {
9049 }
else if (
auto *UO = dyn_cast<UnaryOperator>(PointerExpr)) {
9050 assert(UO->getOpcode() == UO_Deref &&
9051 "Unexpected unary-operator on attach-ptr-expr");
9054 assert(AttachPtrAddr.
isValid() &&
9055 "Failed to get address for attach pointer expression");
9056 return AttachPtrAddr;
9063 static std::pair<Address, Address>
9064 getAttachPtrAddrAndPteeBaseAddr(
const Expr *AttachPtrExpr,
9065 CodeGenFunction &CGF) {
9070 Address AttachPtrAddr = getAttachPtrAddr(AttachPtrExpr, CGF);
9071 assert(AttachPtrAddr.
isValid() &&
"Invalid attach pointer addr");
9073 QualType AttachPtrType =
9078 AttachPtrAddr, AttachPtrType->
castAs<PointerType>());
9079 assert(AttachPteeBaseAddr.
isValid() &&
"Invalid attach pointee base addr");
9081 return {AttachPtrAddr, AttachPteeBaseAddr};
9087 shouldEmitAttachEntry(
const Expr *PointerExpr,
const ValueDecl *MapBaseDecl,
9088 CodeGenFunction &CGF,
9089 llvm::PointerUnion<
const OMPExecutableDirective *,
9090 const OMPDeclareMapperDecl *>
9100 ->getDirectiveKind());
9109 void collectAttachPtrExprInfo(
9111 llvm::PointerUnion<
const OMPExecutableDirective *,
9112 const OMPDeclareMapperDecl *>
9117 ? OMPD_declare_mapper
9120 const auto &[AttachPtrExpr, Depth] =
9124 AttachPtrComputationOrderMap.try_emplace(
9125 AttachPtrExpr, AttachPtrComputationOrderMap.size());
9126 AttachPtrComponentDepthMap.try_emplace(AttachPtrExpr, Depth);
9127 AttachPtrExprMap.try_emplace(Components, AttachPtrExpr);
9135 void generateAllInfoForClauses(
9136 ArrayRef<const OMPClause *> Clauses, MapCombinedInfoTy &CombinedInfo,
9137 llvm::OpenMPIRBuilder &OMPBuilder,
9138 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkipVarSet =
9139 llvm::DenseSet<CanonicalDeclPtr<const Decl>>())
const {
9144 llvm::MapVector<CanonicalDeclPtr<const Decl>,
9145 SmallVector<SmallVector<MapInfo, 8>, 4>>
9151 [&Info, &SkipVarSet](
9152 const ValueDecl *D, MapKind
Kind,
9155 ArrayRef<OpenMPMapModifierKind> MapModifiers,
9156 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
9157 bool ReturnDevicePointer,
bool IsImplicit,
const ValueDecl *Mapper,
9158 const Expr *VarRef =
nullptr,
bool ForDeviceAddr =
false) {
9159 if (SkipVarSet.contains(D))
9161 auto It = Info.try_emplace(D, Total).first;
9162 It->second[
Kind].emplace_back(
9163 L, MapType, MapModifiers, MotionModifiers, ReturnDevicePointer,
9164 IsImplicit, Mapper, VarRef, ForDeviceAddr);
9167 for (
const auto *
Cl : Clauses) {
9168 const auto *
C = dyn_cast<OMPMapClause>(
Cl);
9172 if (llvm::is_contained(
C->getMapTypeModifiers(),
9173 OMPC_MAP_MODIFIER_present))
9175 else if (
C->getMapType() == OMPC_MAP_alloc)
9177 const auto *EI =
C->getVarRefs().begin();
9178 for (
const auto L :
C->component_lists()) {
9179 const Expr *E = (
C->getMapLoc().isValid()) ? *EI :
nullptr;
9180 InfoGen(std::get<0>(L), Kind, std::get<1>(L),
C->getMapType(),
9181 C->getMapTypeModifiers(), {},
9182 false,
C->isImplicit(), std::get<2>(L),
9187 for (
const auto *
Cl : Clauses) {
9188 const auto *
C = dyn_cast<OMPToClause>(
Cl);
9192 if (llvm::is_contained(
C->getMotionModifiers(),
9193 OMPC_MOTION_MODIFIER_present))
9195 if (llvm::is_contained(
C->getMotionModifiers(),
9196 OMPC_MOTION_MODIFIER_iterator)) {
9197 if (
auto *IteratorExpr = dyn_cast<OMPIteratorExpr>(
9198 C->getIteratorModifier()->IgnoreParenImpCasts())) {
9199 const auto *VD =
cast<VarDecl>(IteratorExpr->getIteratorDecl(0));
9204 const auto *EI =
C->getVarRefs().begin();
9205 for (
const auto L :
C->component_lists()) {
9206 InfoGen(std::get<0>(L), Kind, std::get<1>(L), OMPC_MAP_to, {},
9207 C->getMotionModifiers(),
false,
9208 C->isImplicit(), std::get<2>(L), *EI);
9212 for (
const auto *
Cl : Clauses) {
9213 const auto *
C = dyn_cast<OMPFromClause>(
Cl);
9217 if (llvm::is_contained(
C->getMotionModifiers(),
9218 OMPC_MOTION_MODIFIER_present))
9220 if (llvm::is_contained(
C->getMotionModifiers(),
9221 OMPC_MOTION_MODIFIER_iterator)) {
9222 if (
auto *IteratorExpr = dyn_cast<OMPIteratorExpr>(
9223 C->getIteratorModifier()->IgnoreParenImpCasts())) {
9224 const auto *VD =
cast<VarDecl>(IteratorExpr->getIteratorDecl(0));
9229 const auto *EI =
C->getVarRefs().begin();
9230 for (
const auto L :
C->component_lists()) {
9231 InfoGen(std::get<0>(L), Kind, std::get<1>(L), OMPC_MAP_from, {},
9232 C->getMotionModifiers(),
9233 false,
C->isImplicit(), std::get<2>(L),
9246 MapCombinedInfoTy UseDeviceDataCombinedInfo;
9248 auto &&UseDeviceDataCombinedInfoGen =
9249 [&UseDeviceDataCombinedInfo](
const ValueDecl *VD, llvm::Value *
Ptr,
9250 CodeGenFunction &CGF,
bool IsDevAddr,
9251 bool HasUdpFbNullify =
false) {
9252 UseDeviceDataCombinedInfo.Exprs.push_back(VD);
9253 UseDeviceDataCombinedInfo.BasePointers.emplace_back(Ptr);
9254 UseDeviceDataCombinedInfo.DevicePtrDecls.emplace_back(VD);
9255 UseDeviceDataCombinedInfo.DevicePointers.emplace_back(
9256 IsDevAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer);
9262 UseDeviceDataCombinedInfo.Pointers.push_back(Ptr);
9263 UseDeviceDataCombinedInfo.Sizes.push_back(
9264 llvm::Constant::getNullValue(CGF.Int64Ty));
9265 OpenMPOffloadMappingFlags Flags =
9266 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
9267 if (HasUdpFbNullify)
9268 Flags |= OpenMPOffloadMappingFlags::OMP_MAP_FB_NULLIFY;
9269 UseDeviceDataCombinedInfo.Types.push_back(Flags);
9270 UseDeviceDataCombinedInfo.HasAttachPtr.push_back(
false);
9271 UseDeviceDataCombinedInfo.Mappers.push_back(
nullptr);
9275 [&UseDeviceDataCombinedInfoGen](
9276 CodeGenFunction &CGF,
const Expr *IE,
const ValueDecl *VD,
9279 bool IsDevAddr,
bool IEIsAttachPtrForDevAddr =
false,
9280 bool HasUdpFbNullify =
false) {
9284 if (IsDevAddr && !IEIsAttachPtrForDevAddr) {
9285 if (IE->isGLValue())
9292 bool TreatDevAddrAsDevPtr = IEIsAttachPtrForDevAddr;
9299 UseDeviceDataCombinedInfoGen(VD, Ptr, CGF, IsDevAddr &&
9300 !TreatDevAddrAsDevPtr,
9304 auto &&IsMapInfoExist =
9305 [&Info,
this](CodeGenFunction &CGF,
const ValueDecl *VD,
const Expr *IE,
9306 const Expr *DesiredAttachPtrExpr,
bool IsDevAddr,
9307 bool HasUdpFbNullify =
false) ->
bool {
9315 if (It != Info.end()) {
9317 for (
auto &
Data : It->second) {
9318 MapInfo *CI =
nullptr;
9322 auto *It = llvm::find_if(
Data, [&](
const MapInfo &MI) {
9323 if (MI.Components.back().getAssociatedDeclaration() != VD)
9326 const Expr *MapAttachPtr = getAttachPtrExpr(MI.Components);
9327 bool Match = AttachPtrComparator.areEqual(MapAttachPtr,
9328 DesiredAttachPtrExpr);
9332 if (It !=
Data.end())
9337 CI->ForDeviceAddr =
true;
9338 CI->ReturnDevicePointer =
true;
9339 CI->HasUdpFbNullify = HasUdpFbNullify;
9343 auto PrevCI = std::next(CI->Components.rbegin());
9344 const auto *VarD = dyn_cast<VarDecl>(VD);
9345 const Expr *AttachPtrExpr = getAttachPtrExpr(CI->Components);
9346 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
9348 !VD->getType().getNonReferenceType()->isPointerType() ||
9349 PrevCI == CI->Components.rend() ||
9351 VarD->hasLocalStorage() ||
9352 (isa_and_nonnull<DeclRefExpr>(AttachPtrExpr) &&
9354 CI->ForDeviceAddr = IsDevAddr;
9355 CI->ReturnDevicePointer =
true;
9356 CI->HasUdpFbNullify = HasUdpFbNullify;
9374 for (
const auto *
Cl : Clauses) {
9375 const auto *
C = dyn_cast<OMPUseDevicePtrClause>(
Cl);
9378 bool HasUdpFbNullify =
9379 C->getFallbackModifier() == OMPC_USE_DEVICE_PTR_FALLBACK_fb_nullify;
9380 for (
const auto L :
C->component_lists()) {
9383 assert(!Components.empty() &&
9384 "Not expecting empty list of components!");
9385 const ValueDecl *VD = Components.back().getAssociatedDeclaration();
9387 const Expr *IE = Components.back().getAssociatedExpression();
9395 const Expr *UDPOperandExpr =
9396 Components.front().getAssociatedExpression();
9397 if (IsMapInfoExist(CGF, VD, IE,
9399 false, HasUdpFbNullify))
9401 MapInfoGen(CGF, IE, VD, Components,
false,
9402 false, HasUdpFbNullify);
9406 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
9407 for (
const auto *
Cl : Clauses) {
9408 const auto *
C = dyn_cast<OMPUseDeviceAddrClause>(
Cl);
9411 for (
const auto L :
C->component_lists()) {
9414 assert(!std::get<1>(L).empty() &&
9415 "Not expecting empty list of components!");
9416 const ValueDecl *VD = std::get<1>(L).back().getAssociatedDeclaration();
9417 if (!Processed.insert(VD).second)
9438 const Expr *UDAAttachPtrExpr = getAttachPtrExpr(Components);
9439 const Expr *IE = std::get<1>(L).back().getAssociatedExpression();
9440 assert((!UDAAttachPtrExpr || UDAAttachPtrExpr == IE) &&
9441 "use_device_addr operand has an attach-ptr, but does not match "
9442 "last component's expr.");
9443 if (IsMapInfoExist(CGF, VD, IE,
9447 MapInfoGen(CGF, IE, VD, Components,
9449 UDAAttachPtrExpr !=
nullptr);
9453 for (
const auto &
Data : Info) {
9454 MapCombinedInfoTy CurInfo;
9456 const ValueDecl *VD = cast_or_null<ValueDecl>(D);
9463 SmallVector<std::pair<const Expr *, MapInfo>, 16> AttachPtrMapInfoPairs;
9466 for (
const auto &M :
Data.second) {
9467 for (
const MapInfo &L : M) {
9468 assert(!L.Components.empty() &&
9469 "Not expecting declaration with no component lists.");
9471 const Expr *AttachPtrExpr = getAttachPtrExpr(L.Components);
9472 AttachPtrMapInfoPairs.emplace_back(AttachPtrExpr, L);
9477 llvm::stable_sort(AttachPtrMapInfoPairs,
9478 [
this](
const auto &LHS,
const auto &RHS) {
9479 return AttachPtrComparator(LHS.first, RHS.first);
9484 auto *It = AttachPtrMapInfoPairs.begin();
9485 while (It != AttachPtrMapInfoPairs.end()) {
9486 const Expr *AttachPtrExpr = It->first;
9488 SmallVector<MapInfo, 8> GroupLists;
9489 while (It != AttachPtrMapInfoPairs.end() &&
9490 (It->first == AttachPtrExpr ||
9491 AttachPtrComparator.areEqual(It->first, AttachPtrExpr))) {
9492 GroupLists.push_back(It->second);
9495 assert(!GroupLists.empty() &&
"GroupLists should not be empty");
9497 StructRangeInfoTy PartialStruct;
9498 AttachInfoTy AttachInfo;
9499 MapCombinedInfoTy GroupCurInfo;
9501 MapCombinedInfoTy GroupStructBaseCurInfo;
9502 for (
const MapInfo &L : GroupLists) {
9504 unsigned CurrentBasePointersIdx = GroupCurInfo.BasePointers.size();
9505 unsigned StructBasePointersIdx =
9506 GroupStructBaseCurInfo.BasePointers.size();
9508 GroupCurInfo.NonContigInfo.IsNonContiguous =
9509 L.Components.back().isNonContiguous();
9510 generateInfoForComponentList(
9511 L.MapType, L.MapModifiers, L.MotionModifiers, L.Components,
9512 GroupCurInfo, GroupStructBaseCurInfo, PartialStruct, AttachInfo,
9513 false, L.IsImplicit,
9514 true, L.Mapper, L.ForDeviceAddr, VD,
9519 if (L.ReturnDevicePointer) {
9523 assert((CurrentBasePointersIdx < GroupCurInfo.BasePointers.size() ||
9524 StructBasePointersIdx <
9525 GroupStructBaseCurInfo.BasePointers.size()) &&
9526 "Unexpected number of mapped base pointers.");
9529 const ValueDecl *RelevantVD =
9530 L.Components.back().getAssociatedDeclaration();
9531 assert(RelevantVD &&
9532 "No relevant declaration related with device pointer??");
9539 auto SetDevicePointerInfo = [&](MapCombinedInfoTy &Info,
9541 Info.DevicePtrDecls[Idx] = RelevantVD;
9542 Info.DevicePointers[Idx] = L.ForDeviceAddr
9543 ? DeviceInfoTy::Address
9544 : DeviceInfoTy::Pointer;
9546 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
9547 if (L.HasUdpFbNullify)
9549 OpenMPOffloadMappingFlags::OMP_MAP_FB_NULLIFY;
9552 if (StructBasePointersIdx <
9553 GroupStructBaseCurInfo.BasePointers.size())
9554 SetDevicePointerInfo(GroupStructBaseCurInfo,
9555 StructBasePointersIdx);
9557 SetDevicePointerInfo(GroupCurInfo, CurrentBasePointersIdx);
9563 MapCombinedInfoTy GroupUnionCurInfo;
9564 GroupUnionCurInfo.append(GroupStructBaseCurInfo);
9565 GroupUnionCurInfo.append(GroupCurInfo);
9569 if (PartialStruct.Base.isValid()) {
9577 GroupUnionCurInfo.NonContigInfo.Dims.insert(
9578 GroupUnionCurInfo.NonContigInfo.Dims.begin(), 1);
9580 CurInfo, GroupUnionCurInfo.Types, PartialStruct, AttachInfo,
9581 !VD, OMPBuilder, VD,
9582 CombinedInfo.BasePointers.size(),
9588 CurInfo.append(GroupUnionCurInfo);
9589 if (AttachInfo.isValid())
9590 emitAttachEntry(CGF, CurInfo, AttachInfo);
9594 CombinedInfo.append(CurInfo);
9597 CombinedInfo.append(UseDeviceDataCombinedInfo);
9601 MappableExprsHandler(
const OMPExecutableDirective &Dir, CodeGenFunction &CGF)
9602 : CurDir(&Dir), CGF(CGF), AttachPtrComparator(*this) {
9604 for (
const auto *
C : Dir.getClausesOfKind<OMPFirstprivateClause>())
9605 for (
const auto *D :
C->varlist()) {
9606 const ValueDecl *VD = cast<DeclRefExpr>(D)->getDecl();
9607 if (const auto *BD = dyn_cast<BindingDecl>(VD))
9608 VD = cast<VarDecl>(BD->getDecomposedDecl());
9609 FirstPrivateDecls.try_emplace(cast<VarDecl>(VD), C->isImplicit());
9612 for (
const auto *
C : Dir.getClausesOfKind<OMPUsesAllocatorsClause>()) {
9613 for (unsigned I = 0, E = C->getNumberOfAllocators(); I < E; ++I) {
9614 OMPUsesAllocatorsClause::Data D = C->getAllocatorData(I);
9615 if (const auto *DRE = dyn_cast_or_null<DeclRefExpr>(D.AllocatorTraits))
9616 FirstPrivateDecls.try_emplace(cast<VarDecl>(DRE->getDecl()),
9618 else if (const auto *VD = dyn_cast<VarDecl>(
9619 cast<DeclRefExpr>(D.Allocator->IgnoreParenImpCasts())
9621 FirstPrivateDecls.try_emplace(VD, true);
9625 for (
const auto *
C : Dir.getClausesOfKind<OMPDefaultmapClause>())
9626 if (
C->getDefaultmapModifier() == OMPC_DEFAULTMAP_MODIFIER_firstprivate)
9627 DefaultmapFirstprivateKinds.insert(
C->getDefaultmapKind());
9629 for (
const auto *
C : Dir.getClausesOfKind<OMPIsDevicePtrClause>())
9630 for (
auto L :
C->component_lists())
9631 DevPointersMap[std::get<0>(L)].push_back(std::get<1>(L));
9633 for (
const auto *
C : Dir.getClausesOfKind<OMPHasDeviceAddrClause>())
9634 for (
auto L :
C->component_lists())
9635 HasDevAddrsMap[std::get<0>(L)].push_back(std::get<1>(L));
9637 for (
const auto *
C : Dir.getClausesOfKind<OMPMapClause>()) {
9638 if (C->getMapType() != OMPC_MAP_to)
9640 for (auto L : C->component_lists()) {
9641 const ValueDecl *VD = std::get<0>(L);
9642 const auto *RD = VD ? VD->getType()
9644 .getNonReferenceType()
9645 ->getAsCXXRecordDecl()
9647 if (RD && RD->isLambda())
9648 LambdasMap.try_emplace(std::get<0>(L), C);
9652 auto CollectAttachPtrExprsForClauseComponents = [
this](
const auto *
C) {
9653 for (
auto L :
C->component_lists()) {
9656 if (!Components.empty())
9657 collectAttachPtrExprInfo(Components, CurDir);
9663 for (
const auto *
C : Dir.getClausesOfKind<OMPMapClause>())
9664 CollectAttachPtrExprsForClauseComponents(
C);
9665 for (
const auto *
C : Dir.getClausesOfKind<OMPToClause>())
9666 CollectAttachPtrExprsForClauseComponents(
C);
9667 for (
const auto *
C : Dir.getClausesOfKind<OMPFromClause>())
9668 CollectAttachPtrExprsForClauseComponents(
C);
9669 for (
const auto *
C : Dir.getClausesOfKind<OMPUseDevicePtrClause>())
9670 CollectAttachPtrExprsForClauseComponents(
C);
9671 for (
const auto *
C : Dir.getClausesOfKind<OMPUseDeviceAddrClause>())
9672 CollectAttachPtrExprsForClauseComponents(
C);
9673 for (
const auto *
C : Dir.getClausesOfKind<OMPIsDevicePtrClause>())
9674 CollectAttachPtrExprsForClauseComponents(
C);
9675 for (
const auto *
C : Dir.getClausesOfKind<OMPHasDeviceAddrClause>())
9676 CollectAttachPtrExprsForClauseComponents(
C);
9680 MappableExprsHandler(
const OMPDeclareMapperDecl &Dir,
CodeGenFunction &CGF)
9681 : CurDir(&Dir), CGF(CGF), AttachPtrComparator(*this) {
9682 auto CollectAttachPtrExprsForClauseComponents = [
this](
const auto *
C) {
9683 for (
auto L :
C->component_lists()) {
9686 if (!Components.empty())
9687 collectAttachPtrExprInfo(Components, CurDir);
9695 if (const auto *C = dyn_cast<OMPMapClause>(Cl))
9696 CollectAttachPtrExprsForClauseComponents(C);
9697 else if (const auto *C = dyn_cast<OMPToClause>(Cl))
9698 CollectAttachPtrExprsForClauseComponents(C);
9699 else if (const auto *C = dyn_cast<OMPFromClause>(Cl))
9700 CollectAttachPtrExprsForClauseComponents(C);
9712 void emitCombinedEntry(MapCombinedInfoTy &CombinedInfo,
9713 MapFlagsArrayTy &CurTypes,
9714 const StructRangeInfoTy &PartialStruct,
9715 AttachInfoTy &AttachInfo,
bool IsMapThis,
9716 llvm::OpenMPIRBuilder &OMPBuilder,
const ValueDecl *VD,
9717 unsigned OffsetForMemberOfFlag,
9718 bool NotTargetParams)
const {
9719 if (CurTypes.size() == 1 &&
9720 ((CurTypes.back() & OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) !=
9721 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) &&
9722 !PartialStruct.IsArraySection)
9724 Address LBAddr = PartialStruct.LowestElem.second;
9725 Address HBAddr = PartialStruct.HighestElem.second;
9726 if (PartialStruct.HasCompleteRecord) {
9727 LBAddr = PartialStruct.LB;
9728 HBAddr = PartialStruct.LB;
9730 CombinedInfo.Exprs.push_back(VD);
9732 CombinedInfo.BasePointers.push_back(PartialStruct.Base.emitRawPointer(CGF));
9733 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
9734 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
9737 const CXXMethodDecl *MD =
9739 const CXXRecordDecl *RD = MD ? MD->
getParent() :
nullptr;
9740 bool HasBaseClass = RD && IsMapThis ? RD->
getNumBases() > 0 :
false;
9750 CombinedInfo.Pointers.push_back(PartialStruct.Base.emitRawPointer(CGF));
9755 CombinedInfo.Sizes.push_back(Size);
9757 CombinedInfo.Pointers.push_back(LB);
9760 llvm::Value *HAddr = CGF.
Builder.CreateConstGEP1_32(
9764 llvm::Value *Diff = CGF.
Builder.CreatePtrDiff(CHAddr, CLAddr);
9767 CombinedInfo.Sizes.push_back(Size);
9769 CombinedInfo.Mappers.push_back(
nullptr);
9771 CombinedInfo.Types.push_back(
9772 NotTargetParams ? OpenMPOffloadMappingFlags::OMP_MAP_NONE
9773 : !PartialStruct.PreliminaryMapData.BasePointers.empty()
9774 ? OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ
9775 : OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
9783 CombinedInfo.HasAttachPtr.push_back(AttachInfo.isValid());
9786 if (CurTypes.end() !=
9787 llvm::find_if(CurTypes, [](OpenMPOffloadMappingFlags
Type) {
9788 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
9789 Type & OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
9791 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
9793 (*CurTypes.begin()) &= ~OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
9800 if (CurTypes.end() !=
9801 llvm::find_if(CurTypes, [](OpenMPOffloadMappingFlags
Type) {
9802 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
9803 Type & OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD);
9805 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
9806 for (
auto &M : CurTypes)
9807 M |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
9814 OpenMPOffloadMappingFlags MemberOfFlag = OMPBuilder.getMemberOfFlag(
9815 OffsetForMemberOfFlag + CombinedInfo.BasePointers.size() - 1);
9816 for (
auto &M : CurTypes)
9817 OMPBuilder.setCorrectMemberOfFlag(M, MemberOfFlag);
9834 if (AttachInfo.isValid())
9835 AttachInfo.AttachPteeAddr = LBAddr;
9843 void generateAllInfo(
9844 MapCombinedInfoTy &CombinedInfo, llvm::OpenMPIRBuilder &OMPBuilder,
9845 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkipVarSet =
9846 llvm::DenseSet<CanonicalDeclPtr<const Decl>>())
const {
9848 "Expect a executable directive");
9850 generateAllInfoForClauses(CurExecDir->clauses(), CombinedInfo, OMPBuilder,
9857 void generateAllInfoForMapper(MapCombinedInfoTy &CombinedInfo,
9858 llvm::OpenMPIRBuilder &OMPBuilder)
const {
9860 "Expect a declare mapper directive");
9862 generateAllInfoForClauses(CurMapperDir->clauses(), CombinedInfo,
9867 void generateInfoForLambdaCaptures(
9868 const ValueDecl *VD, llvm::Value *Arg, MapCombinedInfoTy &CombinedInfo,
9869 llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers)
const {
9877 llvm::DenseMap<const ValueDecl *, FieldDecl *> Captures;
9878 FieldDecl *ThisCapture =
nullptr;
9884 LambdaPointers.try_emplace(ThisLVal.getPointer(CGF),
9885 VDLVal.getPointer(CGF));
9886 CombinedInfo.Exprs.push_back(VD);
9887 CombinedInfo.BasePointers.push_back(ThisLVal.getPointer(CGF));
9888 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
9889 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
9890 CombinedInfo.Pointers.push_back(ThisLValVal.getPointer(CGF));
9891 CombinedInfo.Sizes.push_back(
9894 CombinedInfo.Types.push_back(
9895 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
9896 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9897 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
9898 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
9899 CombinedInfo.HasAttachPtr.push_back(
false);
9900 CombinedInfo.Mappers.push_back(
nullptr);
9902 for (
const LambdaCapture &LC : RD->
captures()) {
9903 if (!LC.capturesVariable())
9908 auto It = Captures.find(VD);
9909 assert(It != Captures.end() &&
"Found lambda capture without field.");
9913 LambdaPointers.try_emplace(VarLVal.getPointer(CGF),
9914 VDLVal.getPointer(CGF));
9915 CombinedInfo.Exprs.push_back(VD);
9916 CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF));
9917 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
9918 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
9919 CombinedInfo.Pointers.push_back(VarLValVal.getPointer(CGF));
9920 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
9926 LambdaPointers.try_emplace(VarLVal.getPointer(CGF),
9927 VDLVal.getPointer(CGF));
9928 CombinedInfo.Exprs.push_back(VD);
9929 CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF));
9930 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
9931 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
9932 CombinedInfo.Pointers.push_back(VarRVal.
getScalarVal());
9933 CombinedInfo.Sizes.push_back(llvm::ConstantInt::get(CGF.
Int64Ty, 0));
9935 CombinedInfo.Types.push_back(
9936 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
9937 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9938 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
9939 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
9940 CombinedInfo.HasAttachPtr.push_back(
false);
9941 CombinedInfo.Mappers.push_back(
nullptr);
9946 void adjustMemberOfForLambdaCaptures(
9947 llvm::OpenMPIRBuilder &OMPBuilder,
9948 const llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers,
9949 MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
9950 MapFlagsArrayTy &Types)
const {
9951 for (
unsigned I = 0, E = Types.size(); I < E; ++I) {
9953 if (Types[I] != (OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
9954 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9955 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
9956 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT))
9958 llvm::Value *BasePtr = LambdaPointers.lookup(BasePointers[I]);
9959 assert(BasePtr &&
"Unable to find base lambda address.");
9961 for (
unsigned J = I; J > 0; --J) {
9962 unsigned Idx = J - 1;
9963 if (Pointers[Idx] != BasePtr)
9968 assert(TgtIdx != -1 &&
"Unable to find parent lambda.");
9972 OpenMPOffloadMappingFlags MemberOfFlag =
9973 OMPBuilder.getMemberOfFlag(TgtIdx);
9974 OMPBuilder.setCorrectMemberOfFlag(Types[I], MemberOfFlag);
9980 void populateComponentListsForNonLambdaCaptureFromClauses(
9981 const ValueDecl *VD, MapDataArrayTy &DeclComponentLists,
9983 SmallVector<OMPClauseMappableExprCommon::MappableComponent, 8>>
9984 &StorageForImplicitlyAddedComponentLists)
const {
9985 if (VD && LambdasMap.count(VD))
9991 auto It = DevPointersMap.find(VD);
9992 if (It != DevPointersMap.end())
9993 for (
const auto &MCL : It->second)
9994 DeclComponentLists.emplace_back(MCL, OMPC_MAP_to,
Unknown,
9997 auto I = HasDevAddrsMap.find(VD);
9998 if (I != HasDevAddrsMap.end())
9999 for (
const auto &MCL : I->second)
10000 DeclComponentLists.emplace_back(MCL, OMPC_MAP_tofrom,
Unknown,
10004 "Expect a executable directive");
10006 for (
const auto *
C : CurExecDir->getClausesOfKind<OMPMapClause>()) {
10007 const auto *EI =
C->getVarRefs().begin();
10008 for (
const auto L :
C->decl_component_lists(VD)) {
10009 const ValueDecl *VDecl, *Mapper;
10011 const Expr *E = (
C->getMapLoc().isValid()) ? *EI :
nullptr;
10013 std::tie(VDecl, Components, Mapper) = L;
10014 assert(VDecl == VD &&
"We got information for the wrong declaration??");
10015 assert(!Components.empty() &&
10016 "Not expecting declaration with no component lists.");
10017 DeclComponentLists.emplace_back(Components,
C->getMapType(),
10018 C->getMapTypeModifiers(),
10019 C->isImplicit(), Mapper, E);
10028 addImplicitMapForAttachPtrBaseIfMemberOfCapturedVD(
10029 VD, DeclComponentLists, StorageForImplicitlyAddedComponentLists);
10031 llvm::stable_sort(DeclComponentLists, [](
const MapData &LHS,
10032 const MapData &RHS) {
10033 ArrayRef<OpenMPMapModifierKind> MapModifiers = std::get<2>(LHS);
10036 llvm::is_contained(MapModifiers, clang::OMPC_MAP_MODIFIER_present);
10037 bool HasAllocs = MapType == OMPC_MAP_alloc;
10038 MapModifiers = std::get<2>(RHS);
10039 MapType = std::get<1>(LHS);
10041 llvm::is_contained(MapModifiers, clang::OMPC_MAP_MODIFIER_present);
10042 bool HasAllocsR = MapType == OMPC_MAP_alloc;
10043 return (HasPresent && !HasPresentR) || (HasAllocs && !HasAllocsR);
10079 void addImplicitMapForAttachPtrBaseIfMemberOfCapturedVD(
10080 const ValueDecl *CapturedVD, MapDataArrayTy &DeclComponentLists,
10082 SmallVector<OMPClauseMappableExprCommon::MappableComponent, 8>>
10083 &ComponentVectorStorage)
const {
10084 bool IsThisCapture = CapturedVD ==
nullptr;
10086 for (
const auto &ComponentsAndAttachPtr : AttachPtrExprMap) {
10088 ComponentsWithAttachPtr = ComponentsAndAttachPtr.first;
10089 const Expr *AttachPtrExpr = ComponentsAndAttachPtr.second;
10090 if (!AttachPtrExpr)
10093 const auto *ME = dyn_cast<MemberExpr>(AttachPtrExpr);
10097 const Expr *
Base = ME->getBase()->IgnoreParenImpCasts();
10118 bool FoundExistingMap =
false;
10119 for (
const MapData &ExistingL : DeclComponentLists) {
10121 ExistingComponents = std::get<0>(ExistingL);
10123 if (ExistingComponents.empty())
10127 const auto &FirstComponent = ExistingComponents.front();
10128 const Expr *FirstExpr = FirstComponent.getAssociatedExpression();
10134 if (AttachPtrComparator.areEqual(FirstExpr, AttachPtrExpr)) {
10135 FoundExistingMap =
true;
10140 if (IsThisCapture) {
10141 if (
const auto *OASE = dyn_cast<ArraySectionExpr>(FirstExpr)) {
10143 FoundExistingMap =
true;
10152 if (
const auto *DRE = dyn_cast<DeclRefExpr>(FirstExpr)) {
10153 if (DRE->getDecl() == CapturedVD) {
10154 FoundExistingMap =
true;
10160 if (FoundExistingMap)
10166 ComponentVectorStorage.emplace_back();
10167 auto &AttachPtrComponents = ComponentVectorStorage.back();
10170 bool SeenAttachPtrComponent =
false;
10176 for (
size_t i = 0; i < ComponentsWithAttachPtr.size(); ++i) {
10177 const auto &Component = ComponentsWithAttachPtr[i];
10178 const Expr *ComponentExpr = Component.getAssociatedExpression();
10180 if (!SeenAttachPtrComponent && ComponentExpr != AttachPtrExpr)
10182 SeenAttachPtrComponent =
true;
10184 AttachPtrComponents.emplace_back(Component.getAssociatedExpression(),
10185 Component.getAssociatedDeclaration(),
10186 Component.isNonContiguous());
10188 assert(!AttachPtrComponents.empty() &&
10189 "Could not populate component-lists for mapping attach-ptr");
10191 DeclComponentLists.emplace_back(
10192 AttachPtrComponents, OMPC_MAP_tofrom,
Unknown,
10193 true,
nullptr, AttachPtrExpr);
10200 void generateInfoForCaptureFromClauseInfo(
10201 const MapDataArrayTy &DeclComponentListsFromClauses,
10202 const CapturedStmt::Capture *Cap, llvm::Value *Arg,
10203 MapCombinedInfoTy &CurCaptureVarInfo, llvm::OpenMPIRBuilder &OMPBuilder,
10204 unsigned OffsetForMemberOfFlag)
const {
10206 "Not expecting to generate map info for a variable array type!");
10215 if (LambdasMap.count(VD))
10221 if (VD && (DevPointersMap.count(VD) || HasDevAddrsMap.count(VD))) {
10222 CurCaptureVarInfo.Exprs.push_back(VD);
10223 CurCaptureVarInfo.BasePointers.emplace_back(Arg);
10224 CurCaptureVarInfo.DevicePtrDecls.emplace_back(VD);
10225 CurCaptureVarInfo.DevicePointers.emplace_back(DeviceInfoTy::Pointer);
10226 CurCaptureVarInfo.Pointers.push_back(Arg);
10227 CurCaptureVarInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
10230 CurCaptureVarInfo.Types.push_back(
10231 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
10232 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
10233 CurCaptureVarInfo.HasAttachPtr.push_back(
false);
10234 CurCaptureVarInfo.Mappers.push_back(
nullptr);
10238 auto GenerateInfoForComponentLists =
10239 [&](ArrayRef<MapData> DeclComponentListsFromClauses,
10240 bool IsEligibleForTargetParamFlag) {
10241 MapCombinedInfoTy CurInfoForComponentLists;
10242 StructRangeInfoTy PartialStruct;
10243 AttachInfoTy AttachInfo;
10245 if (DeclComponentListsFromClauses.empty())
10248 generateInfoForCaptureFromComponentLists(
10249 VD, DeclComponentListsFromClauses, CurInfoForComponentLists,
10250 PartialStruct, AttachInfo, IsEligibleForTargetParamFlag);
10255 if (PartialStruct.Base.isValid()) {
10256 CurCaptureVarInfo.append(PartialStruct.PreliminaryMapData);
10258 CurCaptureVarInfo, CurInfoForComponentLists.Types,
10259 PartialStruct, AttachInfo, Cap->
capturesThis(), OMPBuilder,
10260 nullptr, OffsetForMemberOfFlag,
10261 !IsEligibleForTargetParamFlag);
10266 CurCaptureVarInfo.append(CurInfoForComponentLists);
10267 if (AttachInfo.isValid())
10268 emitAttachEntry(CGF, CurCaptureVarInfo, AttachInfo);
10292 SmallVector<std::pair<const Expr *, MapData>, 16> AttachPtrMapDataPairs;
10294 for (
const MapData &L : DeclComponentListsFromClauses) {
10297 const Expr *AttachPtrExpr = getAttachPtrExpr(Components);
10298 AttachPtrMapDataPairs.emplace_back(AttachPtrExpr, L);
10302 llvm::stable_sort(AttachPtrMapDataPairs,
10303 [
this](
const auto &LHS,
const auto &RHS) {
10304 return AttachPtrComparator(LHS.first, RHS.first);
10307 bool NoDefaultMappingDoneForVD = CurCaptureVarInfo.BasePointers.empty();
10308 bool IsFirstGroup =
true;
10312 auto *It = AttachPtrMapDataPairs.begin();
10313 while (It != AttachPtrMapDataPairs.end()) {
10314 const Expr *AttachPtrExpr = It->first;
10316 MapDataArrayTy GroupLists;
10317 while (It != AttachPtrMapDataPairs.end() &&
10318 (It->first == AttachPtrExpr ||
10319 AttachPtrComparator.areEqual(It->first, AttachPtrExpr))) {
10320 GroupLists.push_back(It->second);
10323 assert(!GroupLists.empty() &&
"GroupLists should not be empty");
10328 bool IsEligibleForTargetParamFlag =
10329 IsFirstGroup && NoDefaultMappingDoneForVD;
10331 GenerateInfoForComponentLists(GroupLists, IsEligibleForTargetParamFlag);
10332 IsFirstGroup =
false;
10339 void generateInfoForCaptureFromComponentLists(
10340 const ValueDecl *VD, ArrayRef<MapData> DeclComponentLists,
10341 MapCombinedInfoTy &CurComponentListInfo, StructRangeInfoTy &PartialStruct,
10342 AttachInfoTy &AttachInfo,
bool IsListEligibleForTargetParamFlag)
const {
10344 llvm::SmallDenseMap<
10351 for (
const MapData &L : DeclComponentLists) {
10354 ArrayRef<OpenMPMapModifierKind> MapModifiers;
10356 const ValueDecl *Mapper;
10357 const Expr *VarRef;
10358 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =
10361 for (
const MapData &L1 : ArrayRef(DeclComponentLists).slice(Count)) {
10363 std::tie(Components1, MapType, MapModifiers, IsImplicit, Mapper,
10365 auto CI = Components.rbegin();
10366 auto CE = Components.rend();
10367 auto SI = Components1.rbegin();
10368 auto SE = Components1.rend();
10369 for (; CI != CE && SI != SE; ++CI, ++SI) {
10370 if (CI->getAssociatedExpression()->getStmtClass() !=
10371 SI->getAssociatedExpression()->getStmtClass())
10374 if (CI->getAssociatedDeclaration() != SI->getAssociatedDeclaration())
10379 if (CI == CE || SI == SE) {
10381 if (CI == CE && SI == SE)
10383 const auto It = (SI == SE) ? CI : SI;
10390 (std::prev(It)->getAssociatedDeclaration() &&
10392 ->getAssociatedDeclaration()
10394 ->isPointerType()) ||
10395 (It->getAssociatedDeclaration() &&
10396 It->getAssociatedDeclaration()->getType()->isPointerType() &&
10397 std::next(It) != CE && std::next(It) != SE))
10399 const MapData &BaseData = CI == CE ? L : L1;
10401 SI == SE ? Components : Components1;
10402 OverlappedData[&BaseData].push_back(SubData);
10407 llvm::SmallVector<const FieldDecl *, 4> Layout;
10408 if (!OverlappedData.empty()) {
10411 while (BaseType != OrigType) {
10417 getPlainLayout(CRD, Layout,
false);
10423 for (
auto &Pair : OverlappedData) {
10430 auto CI = First.rbegin();
10431 auto CE = First.rend();
10432 auto SI = Second.rbegin();
10433 auto SE = Second.rend();
10434 for (; CI != CE && SI != SE; ++CI, ++SI) {
10435 if (CI->getAssociatedExpression()->getStmtClass() !=
10436 SI->getAssociatedExpression()->getStmtClass())
10439 if (CI->getAssociatedDeclaration() !=
10440 SI->getAssociatedDeclaration())
10445 if (CI == CE && SI == SE)
10449 if (CI == CE || SI == SE)
10454 if (FD1->getParent() == FD2->getParent())
10455 return FD1->getFieldIndex() < FD2->getFieldIndex();
10457 llvm::find_if(Layout, [FD1, FD2](
const FieldDecl *FD) {
10458 return FD == FD1 || FD == FD2;
10466 bool AddTargetParamFlag = IsListEligibleForTargetParamFlag;
10467 MapCombinedInfoTy StructBaseCombinedInfo;
10468 for (
const auto &Pair : OverlappedData) {
10469 const MapData &L = *Pair.getFirst();
10472 ArrayRef<OpenMPMapModifierKind> MapModifiers;
10474 const ValueDecl *Mapper;
10475 const Expr *VarRef;
10476 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =
10478 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
10479 OverlappedComponents = Pair.getSecond();
10480 generateInfoForComponentList(
10481 MapType, MapModifiers, {}, Components, CurComponentListInfo,
10482 StructBaseCombinedInfo, PartialStruct, AttachInfo, AddTargetParamFlag,
10483 IsImplicit,
false, Mapper,
10484 false, VD, VarRef, OverlappedComponents);
10485 AddTargetParamFlag =
false;
10488 for (
const MapData &L : DeclComponentLists) {
10491 ArrayRef<OpenMPMapModifierKind> MapModifiers;
10493 const ValueDecl *Mapper;
10494 const Expr *VarRef;
10495 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =
10497 auto It = OverlappedData.find(&L);
10498 if (It == OverlappedData.end())
10499 generateInfoForComponentList(
10500 MapType, MapModifiers, {}, Components, CurComponentListInfo,
10501 StructBaseCombinedInfo, PartialStruct, AttachInfo,
10502 AddTargetParamFlag, IsImplicit,
false,
10503 Mapper,
false, VD, VarRef,
10505 AddTargetParamFlag =
false;
10511 bool isEffectivelyFirstprivate(
const VarDecl *VD, QualType
Type)
const {
10513 auto I = FirstPrivateDecls.find(VD);
10514 if (I != FirstPrivateDecls.end() && !I->getSecond())
10518 if (DefaultmapFirstprivateKinds.count(OMPC_DEFAULTMAP_scalar)) {
10519 if (
Type->isScalarType())
10524 if (DefaultmapFirstprivateKinds.count(OMPC_DEFAULTMAP_pointer)) {
10525 if (
Type->isAnyPointerType())
10530 if (DefaultmapFirstprivateKinds.count(OMPC_DEFAULTMAP_aggregate)) {
10531 if (
Type->isAggregateType())
10536 return DefaultmapFirstprivateKinds.count(OMPC_DEFAULTMAP_all);
10541 void generateDefaultMapInfo(
const CapturedStmt::Capture &CI,
10542 const FieldDecl &RI, llvm::Value *CV,
10543 MapCombinedInfoTy &CombinedInfo)
const {
10544 bool IsImplicit =
true;
10547 CombinedInfo.Exprs.push_back(
nullptr);
10548 CombinedInfo.BasePointers.push_back(CV);
10549 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
10550 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
10551 CombinedInfo.Pointers.push_back(CV);
10553 CombinedInfo.Sizes.push_back(
10557 CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_TO |
10558 OpenMPOffloadMappingFlags::OMP_MAP_FROM);
10562 CombinedInfo.BasePointers.push_back(CV);
10563 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
10564 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
10565 CombinedInfo.Pointers.push_back(CV);
10566 bool IsFirstprivate =
10572 CombinedInfo.Types.push_back(
10573 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
10574 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
10576 }
else if (IsFirstprivate) {
10579 CombinedInfo.Types.push_back(
10580 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
10582 CombinedInfo.Sizes.push_back(llvm::Constant::getNullValue(CGF.
Int64Ty));
10586 CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_NONE);
10587 CombinedInfo.Sizes.push_back(llvm::Constant::getNullValue(CGF.
Int64Ty));
10589 auto I = FirstPrivateDecls.find(VD);
10590 if (I != FirstPrivateDecls.end())
10591 IsImplicit = I->getSecond();
10597 bool IsFirstprivate = isEffectivelyFirstprivate(VD, ElementType);
10599 CombinedInfo.BasePointers.push_back(CV);
10600 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
10601 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
10604 if (IsFirstprivate && ElementType->isAnyPointerType()) {
10606 CombinedInfo.Pointers.push_back(CV);
10608 CombinedInfo.Sizes.push_back(llvm::Constant::getNullValue(CGF.
Int64Ty));
10609 CombinedInfo.Types.push_back(
10610 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
10612 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
10617 CombinedInfo.Types.push_back(getMapModifiersForPrivateClauses(CI));
10618 CombinedInfo.Pointers.push_back(CV);
10620 auto I = FirstPrivateDecls.find(VD);
10621 if (I != FirstPrivateDecls.end())
10622 IsImplicit = I->getSecond();
10625 CombinedInfo.Types.back() |=
10626 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
10630 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
10632 CombinedInfo.HasAttachPtr.push_back(
false);
10634 CombinedInfo.Mappers.push_back(
nullptr);
10646 dyn_cast<MemberExpr>(OASE->getBase()->IgnoreParenImpCasts()))
10647 return ME->getMemberDecl();
10653static llvm::Constant *
10655 MappableExprsHandler::MappingExprInfo &MapExprs) {
10658 if (!MapExprs.getMapDecl() && !MapExprs.getMapExpr())
10659 return OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
10662 if (!MapExprs.getMapDecl() && MapExprs.getMapExpr()) {
10666 Loc = MapExprs.getMapExpr()->getExprLoc();
10668 Loc = MapExprs.getMapDecl()->getLocation();
10671 std::string ExprName;
10672 if (MapExprs.getMapExpr()) {
10674 llvm::raw_string_ostream OS(ExprName);
10675 MapExprs.getMapExpr()->printPretty(OS,
nullptr, P);
10677 ExprName = MapExprs.getMapDecl()->getNameAsString();
10686 return OMPBuilder.getOrCreateSrcLocStr(
FileName, ExprName, PLoc.
getLine(),
10693 CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
10695 bool IsNonContiguous =
false,
bool ForEndCall =
false) {
10698 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
10700 InsertPointTy CodeGenIP(CGF.
Builder.GetInsertPoint());
10702 auto DeviceAddrCB = [&](
unsigned int I, llvm::Value *NewDecl) {
10703 if (
const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
10708 auto CustomMapperCB = [&](
unsigned int I) {
10709 llvm::Function *MFunc =
nullptr;
10710 if (CombinedInfo.Mappers[I]) {
10711 Info.HasMapper =
true;
10717 cantFail(OMPBuilder.emitOffloadingArraysAndArgs(
10718 AllocaIP, CodeGenIP, Info, Info.RTArgs, CombinedInfo, CustomMapperCB,
10719 IsNonContiguous, ForEndCall, DeviceAddrCB));
10723static const OMPExecutableDirective *
10725 const auto *CS = D.getInnermostCapturedStmt();
10728 const Stmt *ChildStmt =
10731 if (
const auto *NestedDir =
10732 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
10734 switch (D.getDirectiveKind()) {
10740 if (DKind == OMPD_teams) {
10741 Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers(
10746 if (
const auto *NND =
10747 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
10748 DKind = NND->getDirectiveKind();
10754 case OMPD_target_teams:
10758 case OMPD_target_parallel:
10759 case OMPD_target_simd:
10760 case OMPD_target_loop:
10761 case OMPD_target_parallel_for:
10762 case OMPD_target_parallel_for_simd:
10764 case OMPD_target_teams_distribute:
10765 case OMPD_target_teams_distribute_simd:
10766 case OMPD_target_teams_distribute_parallel_for:
10767 case OMPD_target_teams_distribute_parallel_for_simd:
10768 case OMPD_parallel:
10770 case OMPD_parallel_for:
10771 case OMPD_parallel_master:
10772 case OMPD_parallel_sections:
10773 case OMPD_for_simd:
10774 case OMPD_parallel_for_simd:
10776 case OMPD_cancellation_point:
10777 case OMPD_ordered_standalone:
10778 case OMPD_ordered_blockassoc:
10779 case OMPD_threadprivate:
10780 case OMPD_allocate:
10785 case OMPD_sections:
10789 case OMPD_critical:
10790 case OMPD_taskyield:
10792 case OMPD_taskwait:
10793 case OMPD_taskgroup:
10799 case OMPD_target_data:
10800 case OMPD_target_exit_data:
10801 case OMPD_target_enter_data:
10802 case OMPD_distribute:
10803 case OMPD_distribute_simd:
10804 case OMPD_distribute_parallel_for:
10805 case OMPD_distribute_parallel_for_simd:
10806 case OMPD_teams_distribute:
10807 case OMPD_teams_distribute_simd:
10808 case OMPD_teams_distribute_parallel_for:
10809 case OMPD_teams_distribute_parallel_for_simd:
10810 case OMPD_target_update:
10811 case OMPD_declare_simd:
10812 case OMPD_declare_variant:
10813 case OMPD_begin_declare_variant:
10814 case OMPD_end_declare_variant:
10815 case OMPD_declare_target:
10816 case OMPD_end_declare_target:
10817 case OMPD_declare_reduction:
10818 case OMPD_declare_mapper:
10819 case OMPD_taskloop:
10820 case OMPD_taskloop_simd:
10821 case OMPD_master_taskloop:
10822 case OMPD_master_taskloop_simd:
10823 case OMPD_parallel_master_taskloop:
10824 case OMPD_parallel_master_taskloop_simd:
10825 case OMPD_requires:
10826 case OMPD_metadirective:
10829 llvm_unreachable(
"Unexpected directive.");
10889 if (
UDMMap.count(D) > 0)
10893 auto *MapperVarDecl =
10895 CharUnits ElementSize =
C.getTypeSizeInChars(Ty);
10896 llvm::Type *ElemTy =
CGM.getTypes().ConvertTypeForMem(Ty);
10899 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
10900 auto PrivatizeAndGenMapInfoCB =
10901 [&](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP, llvm::Value *PtrPHI,
10902 llvm::Value *BeginArg) -> llvm::OpenMPIRBuilder::MapInfosTy & {
10903 MapperCGF.
Builder.restoreIP(CodeGenIP);
10913 Scope.addPrivate(MapperVarDecl, PtrCurrent);
10914 (void)
Scope.Privatize();
10917 MappableExprsHandler MEHandler(*D, MapperCGF);
10918 MEHandler.generateAllInfoForMapper(CombinedInfo,
OMPBuilder);
10920 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
10923 if (
CGM.getCodeGenOpts().getDebugInfo() !=
10924 llvm::codegenoptions::NoDebugInfo) {
10925 CombinedInfo.Names.resize(CombinedInfo.Exprs.size());
10926 llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),
10930 return CombinedInfo;
10933 auto CustomMapperCB = [&](
unsigned I) {
10934 llvm::Function *MapperFunc =
nullptr;
10935 if (CombinedInfo.Mappers[I]) {
10939 assert(MapperFunc &&
"Expect a valid mapper function is available.");
10945 llvm::raw_svector_ostream Out(TyStr);
10946 CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(Ty, Out);
10947 std::string Name =
getName({
"omp_mapper", TyStr, D->
getName()});
10953 bool PropagatePresentToPointeeStorage =
CGM.getLangOpts().OpenMP >= 60;
10956 bool PropagatePresentToNonPointeeStorage =
false;
10957 llvm::Function *NewFn = cantFail(
OMPBuilder.emitUserDefinedMapper(
10958 PrivatizeAndGenMapInfoCB, ElemTy, Name, CustomMapperCB,
10959 false, PropagatePresentToPointeeStorage,
10960 PropagatePresentToNonPointeeStorage));
10961 UDMMap.try_emplace(D, NewFn);
10968 auto I =
UDMMap.find(D);
10972 return UDMMap.lookup(D);
10985 Kind != OMPD_target_teams_loop)
10988 return llvm::ConstantInt::get(CGF.
Int64Ty, 0);
10991 if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD))
10992 return NumIterations;
10993 return llvm::ConstantInt::get(CGF.
Int64Ty, 0);
11002 if (OffloadingMandatory) {
11003 CGF.
Builder.CreateUnreachable();
11005 if (RequiresOuterTask) {
11006 CapturedVars.clear();
11010 CapturedVars.end());
11011 Args.push_back(llvm::Constant::getNullValue(CGF.
Builder.getPtrTy()));
11018 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
11021 llvm::Value *DeviceID;
11022 if (
Device.getPointer()) {
11024 Device.getInt() == OMPC_DEVICE_device_num) &&
11025 "Expected device_num modifier.");
11030 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
11035static std::pair<llvm::Value *, OMPDynGroupprivateFallbackType>
11037 llvm::Value *DynGP = CGF.
Builder.getInt32(0);
11038 auto DynGPFallback = OMPDynGroupprivateFallbackType::Abort;
11042 llvm::Value *DynGPVal =
11046 auto FallbackModifier = DynGPClause->getDynGroupprivateFallbackModifier();
11047 switch (FallbackModifier) {
11048 case OMPC_DYN_GROUPPRIVATE_FALLBACK_abort:
11049 DynGPFallback = OMPDynGroupprivateFallbackType::Abort;
11051 case OMPC_DYN_GROUPPRIVATE_FALLBACK_null:
11052 DynGPFallback = OMPDynGroupprivateFallbackType::Null;
11054 case OMPC_DYN_GROUPPRIVATE_FALLBACK_default_mem:
11057 DynGPFallback = OMPDynGroupprivateFallbackType::DefaultMem;
11060 llvm_unreachable(
"Unknown fallback modifier for OpenMP dyn_groupprivate");
11062 }
else if (
auto *OMPXDynCGClause =
11065 llvm::Value *DynCGMemVal = CGF.
EmitScalarExpr(OMPXDynCGClause->getSize(),
11070 return {DynGP, DynGPFallback};
11076 llvm::OpenMPIRBuilder &OMPBuilder,
11078 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) {
11080 llvm::DenseMap<llvm::Value *, llvm::Value *> LambdaPointers;
11082 auto *CV = CapturedVars.begin();
11085 CI != CE; ++CI, ++RI, ++CV) {
11086 MappableExprsHandler::MapCombinedInfoTy CurInfo;
11091 CurInfo.Exprs.push_back(
nullptr);
11092 CurInfo.BasePointers.push_back(*CV);
11093 CurInfo.DevicePtrDecls.push_back(
nullptr);
11094 CurInfo.DevicePointers.push_back(
11095 MappableExprsHandler::DeviceInfoTy::None);
11096 CurInfo.Pointers.push_back(*CV);
11097 CurInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
11100 CurInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
11101 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM |
11102 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
11103 CurInfo.HasAttachPtr.push_back(
false);
11104 CurInfo.Mappers.push_back(
nullptr);
11109 bool HasEntryWithCVAsAttachPtr =
false;
11111 HasEntryWithCVAsAttachPtr =
11112 MEHandler.hasAttachEntryForCapturedVar(CapturedVD);
11115 MappableExprsHandler::MapDataArrayTy DeclComponentLists;
11118 StorageForImplicitlyAddedComponentLists;
11119 MEHandler.populateComponentListsForNonLambdaCaptureFromClauses(
11120 CapturedVD, DeclComponentLists,
11121 StorageForImplicitlyAddedComponentLists);
11132 bool HasEntryWithoutAttachPtr =
11133 llvm::any_of(DeclComponentLists, [&](
const auto &MapData) {
11135 Components = std::get<0>(MapData);
11136 return !MEHandler.getAttachPtrExpr(Components);
11141 if (DeclComponentLists.empty() ||
11142 (!HasEntryWithCVAsAttachPtr && !HasEntryWithoutAttachPtr))
11143 MEHandler.generateDefaultMapInfo(*CI, **RI, *CV, CurInfo);
11147 MEHandler.generateInfoForCaptureFromClauseInfo(
11148 DeclComponentLists, CI, *CV, CurInfo, OMPBuilder,
11149 CombinedInfo.BasePointers.size());
11154 MappedVarSet.insert(
nullptr);
11159 MEHandler.generateInfoForLambdaCaptures(CI->
getCapturedVar(), *CV,
11160 CurInfo, LambdaPointers);
11163 assert(!CurInfo.BasePointers.empty() &&
11164 "Non-existing map pointer for capture!");
11165 assert(CurInfo.BasePointers.size() == CurInfo.Pointers.size() &&
11166 CurInfo.BasePointers.size() == CurInfo.Sizes.size() &&
11167 CurInfo.BasePointers.size() == CurInfo.Types.size() &&
11168 CurInfo.BasePointers.size() == CurInfo.Mappers.size() &&
11169 "Inconsistent map information sizes!");
11172 CombinedInfo.append(CurInfo);
11175 MEHandler.adjustMemberOfForLambdaCaptures(
11176 OMPBuilder, LambdaPointers, CombinedInfo.BasePointers,
11177 CombinedInfo.Pointers, CombinedInfo.Types);
11181 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
11182 llvm::OpenMPIRBuilder &OMPBuilder,
11189 MEHandler.generateAllInfo(CombinedInfo, OMPBuilder, SkippedVarSet);
11191 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
11195 llvm::codegenoptions::NoDebugInfo) {
11196 CombinedInfo.Names.resize(CombinedInfo.Exprs.size());
11197 llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),
11205 llvm::OpenMPIRBuilder &OMPBuilder,
11206 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) {
11208 MappableExprsHandler MEHandler(D, CGF);
11209 llvm::DenseSet<CanonicalDeclPtr<const Decl>> MappedVarSet;
11212 MappedVarSet, CombinedInfo);
11213 genMapInfo(MEHandler, CGF, CombinedInfo, OMPBuilder, MappedVarSet);
11216template <
typename ClauseTy>
11221 const auto *
C = D.getSingleClause<ClauseTy>();
11222 assert(!
C->varlist_empty() &&
11223 "ompx_bare requires explicit num_teams and thread_limit");
11225 for (
auto *E :
C->varlist()) {
11237 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
11239 llvm::Value *&MapTypesArray, llvm::Value *&MapNamesArray,
11244 llvm::OpenMPIRBuilder &OMPBuilder = OMPRuntime->
getOMPBuilder();
11247 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
11249 genMapInfo(D, CGF, CS, CapturedVars, OMPBuilder, CombinedInfo);
11252 using OpenMPOffloadMappingFlags = llvm::omp::OpenMPOffloadMappingFlags;
11253 auto *NullPtr = llvm::Constant::getNullValue(CGF.
Builder.getPtrTy());
11254 CombinedInfo.BasePointers.push_back(NullPtr);
11255 CombinedInfo.Pointers.push_back(NullPtr);
11256 CombinedInfo.DevicePointers.push_back(
11257 llvm::OpenMPIRBuilder::DeviceInfoTy::None);
11258 CombinedInfo.Sizes.push_back(CGF.
Builder.getInt64(0));
11259 CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM |
11260 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
11261 CombinedInfo.HasAttachPtr.push_back(
false);
11262 if (!CombinedInfo.Names.empty())
11263 CombinedInfo.Names.push_back(NullPtr);
11264 CombinedInfo.Exprs.push_back(
nullptr);
11265 CombinedInfo.Mappers.push_back(
nullptr);
11266 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
11280 MapTypesArray = Info.RTArgs.MapTypesArray;
11281 MapNamesArray = Info.RTArgs.MapNamesArray;
11283 auto &&ThenGen = [&OMPRuntime, OutlinedFn, &D, &CapturedVars,
11284 RequiresOuterTask, &CS, OffloadingMandatory,
Device,
11285 OutlinedFnID, &InputInfo, &MapTypesArray, &MapNamesArray,
11287 bool IsReverseOffloading =
Device.getInt() == OMPC_DEVICE_ancestor;
11289 if (IsReverseOffloading) {
11295 RequiresOuterTask, CS, OffloadingMandatory, CGF);
11299 bool HasNoWait = D.hasClausesOfKind<OMPNowaitClause>();
11300 unsigned NumTargetItems = InputInfo.NumberOfTargetItems;
11302 llvm::Value *BasePointersArray =
11303 InputInfo.BasePointersArray.emitRawPointer(CGF);
11304 llvm::Value *PointersArray = InputInfo.PointersArray.emitRawPointer(CGF);
11305 llvm::Value *SizesArray = InputInfo.SizesArray.emitRawPointer(CGF);
11306 llvm::Value *MappersArray = InputInfo.MappersArray.emitRawPointer(CGF);
11308 auto &&EmitTargetCallFallbackCB =
11309 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
11310 OffloadingMandatory, &CGF](llvm::OpenMPIRBuilder::InsertPointTy IP)
11311 -> llvm::OpenMPIRBuilder::InsertPointTy {
11314 RequiresOuterTask, CS, OffloadingMandatory, CGF);
11327 NumThreads.push_back(
11333 llvm::Value *NumIterations =
11336 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
11339 llvm::OpenMPIRBuilder::TargetDataRTArgs RTArgs(
11340 BasePointersArray, PointersArray, SizesArray, MapTypesArray,
11341 nullptr , MappersArray, MapNamesArray);
11343 llvm::OpenMPIRBuilder::TargetKernelArgs Args(
11344 NumTargetItems, RTArgs, NumIterations, NumTeams, NumThreads,
11345 DynCGroupMem, HasNoWait, IsBare,
11346 IsBare, DynCGroupMemFallback);
11348 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
11350 CGF.
Builder, OutlinedFnID, EmitTargetCallFallbackCB, Args, DeviceID,
11352 CGF.
Builder.restoreIP(AfterIP);
11355 if (RequiresOuterTask)
11370 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
11373 RequiresOuterTask, CS, OffloadingMandatory, CGF);
11376 if (RequiresOuterTask) {
11386 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID,
const Expr *IfCond,
11387 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
11394 const bool OffloadingMandatory = !
CGM.getLangOpts().OpenMPIsTargetDevice &&
11395 CGM.getLangOpts().OpenMPOffloadMandatory;
11397 assert((OffloadingMandatory || OutlinedFn) &&
"Invalid outlined function!");
11399 const bool RequiresOuterTask =
11401 D.hasClausesOfKind<OMPNowaitClause>() ||
11402 D.hasClausesOfKind<OMPInReductionClause>() ||
11403 (
CGM.getLangOpts().OpenMP >= 51 &&
11407 const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target);
11415 llvm::Value *MapTypesArray =
nullptr;
11416 llvm::Value *MapNamesArray =
nullptr;
11418 auto &&TargetThenGen = [
this, OutlinedFn, &D, &CapturedVars,
11419 RequiresOuterTask, &CS, OffloadingMandatory,
Device,
11420 OutlinedFnID, &InputInfo, &MapTypesArray,
11424 RequiresOuterTask, CS, OffloadingMandatory,
11425 Device, OutlinedFnID, InputInfo, MapTypesArray,
11426 MapNamesArray, SizeEmitter, CGF,
CGM);
11429 auto &&TargetElseGen =
11430 [
this, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
11433 CS, OffloadingMandatory, CGF);
11440 if (OutlinedFnID) {
11442 emitIfClause(CGF, IfCond, TargetThenGen, TargetElseGen);
11454 StringRef ParentName) {
11461 if (
auto *E = dyn_cast<OMPExecutableDirective>(S);
11470 bool RequiresDeviceCodegen =
11475 if (RequiresDeviceCodegen) {
11483 if (!
OMPBuilder.OffloadInfoManager.hasTargetRegionEntryInfo(EntryInfo))
11486 switch (E.getDirectiveKind()) {
11491 case OMPD_target_parallel:
11495 case OMPD_target_teams:
11499 case OMPD_target_teams_distribute:
11503 case OMPD_target_teams_distribute_simd:
11507 case OMPD_target_parallel_for:
11511 case OMPD_target_parallel_for_simd:
11515 case OMPD_target_simd:
11516 case OMPD_target_loop:
11520 case OMPD_target_teams_distribute_parallel_for:
11525 case OMPD_target_teams_distribute_parallel_for_simd:
11531 case OMPD_target_teams_loop:
11535 case OMPD_target_parallel_loop:
11539 case OMPD_parallel:
11541 case OMPD_parallel_for:
11542 case OMPD_parallel_master:
11543 case OMPD_parallel_sections:
11544 case OMPD_for_simd:
11545 case OMPD_parallel_for_simd:
11547 case OMPD_cancellation_point:
11548 case OMPD_ordered_standalone:
11549 case OMPD_ordered_blockassoc:
11550 case OMPD_threadprivate:
11551 case OMPD_allocate:
11556 case OMPD_sections:
11560 case OMPD_critical:
11561 case OMPD_taskyield:
11563 case OMPD_taskwait:
11564 case OMPD_taskgroup:
11570 case OMPD_target_data:
11571 case OMPD_target_exit_data:
11572 case OMPD_target_enter_data:
11573 case OMPD_distribute:
11574 case OMPD_distribute_simd:
11575 case OMPD_distribute_parallel_for:
11576 case OMPD_distribute_parallel_for_simd:
11577 case OMPD_teams_distribute:
11578 case OMPD_teams_distribute_simd:
11579 case OMPD_teams_distribute_parallel_for:
11580 case OMPD_teams_distribute_parallel_for_simd:
11581 case OMPD_target_update:
11582 case OMPD_declare_simd:
11583 case OMPD_declare_variant:
11584 case OMPD_begin_declare_variant:
11585 case OMPD_end_declare_variant:
11586 case OMPD_declare_target:
11587 case OMPD_end_declare_target:
11588 case OMPD_declare_reduction:
11589 case OMPD_declare_mapper:
11590 case OMPD_taskloop:
11591 case OMPD_taskloop_simd:
11592 case OMPD_master_taskloop:
11593 case OMPD_master_taskloop_simd:
11594 case OMPD_parallel_master_taskloop:
11595 case OMPD_parallel_master_taskloop_simd:
11596 case OMPD_requires:
11597 case OMPD_metadirective:
11600 llvm_unreachable(
"Unknown target directive for OpenMP device codegen.");
11606 if (
const auto *MD = dyn_cast<OMPMetaDirective>(S)) {
11607 for (
const Stmt *
V : MD->variants())
11612 if (
const auto *E = dyn_cast<OMPExecutableDirective>(S)) {
11613 if (!E->hasAssociatedStmt() || !E->getAssociatedStmt())
11621 if (
const auto *E = dyn_cast<CXXDefaultArgExpr>(S)) {
11625 if (
const auto *E = dyn_cast<CXXDefaultInitExpr>(S)) {
11632 if (
const auto *E = dyn_cast<CXXConstructExpr>(S))
11633 Ctor = E->getConstructor();
11634 else if (
const auto *E = dyn_cast<CXXInheritedCtorInitExpr>(S))
11635 Ctor = E->getConstructor();
11641 if (
const auto *L = dyn_cast<LambdaExpr>(S)) {
11657 const auto *Def = dyn_cast_or_null<CXXConstructorDecl>(Ctor->
getDefinition());
11666 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
11667 OMPDeclareTargetDeclAttr::getDeviceType(VD);
11671 if (!IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_NoHost)
11674 if (IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_Host)
11682 if (!
CGM.getLangOpts().OpenMPIsTargetDevice) {
11683 if (
const auto *FD = dyn_cast<FunctionDecl>(GD.
getDecl()))
11685 CGM.getLangOpts().OpenMPIsTargetDevice))
11692 if (
const auto *FD = dyn_cast<FunctionDecl>(VD)) {
11694 if (
const auto *Ctor = dyn_cast<CXXConstructorDecl>(FD))
11699 CGM.getLangOpts().OpenMPIsTargetDevice))
11704 return !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD) &&
11709 if (!
CGM.getLangOpts().OpenMPIsTargetDevice)
11734 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
11735 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
11736 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
11737 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
11738 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
11747 llvm::Constant *
Addr) {
11748 if (
CGM.getLangOpts().OMPTargetTriples.empty() &&
11749 !
CGM.getLangOpts().OpenMPIsTargetDevice)
11752 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
11753 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
11757 if (Res && *Res != OMPDeclareTargetDeclAttr::MT_Link &&
11763 if (Res && *Res == OMPDeclareTargetDeclAttr::MT_Local)
11767 if (
CGM.getLangOpts().OpenMPIsTargetDevice) {
11770 StringRef VarName =
CGM.getMangledName(VD);
11776 auto AddrOfGlobal = [&VD,
this]() {
return CGM.GetAddrOfGlobal(VD); };
11777 auto LinkageForVariable = [&VD,
this]() {
11778 return CGM.getLLVMLinkageVarDefinition(VD);
11781 std::vector<llvm::GlobalVariable *> GeneratedRefs;
11788 CGM.getMangledName(VD), GeneratedRefs,
CGM.getLangOpts().OpenMPSimd,
11789 CGM.getLangOpts().OMPTargetTriples, AddrOfGlobal, LinkageForVariable,
11790 CGM.getTypes().ConvertTypeForMem(
11791 CGM.getContext().getPointerType(VD->
getType())),
11794 for (
auto *ref : GeneratedRefs)
11795 CGM.addCompilerUsedGlobal(ref);
11808 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
11809 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
11813 if (*Res == OMPDeclareTargetDeclAttr::MT_Local ||
11814 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
11815 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
11817 CGM.EmitGlobal(VD);
11819 assert((*Res == OMPDeclareTargetDeclAttr::MT_Link ||
11820 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
11821 *Res == OMPDeclareTargetDeclAttr::MT_Enter ||
11822 *Res == OMPDeclareTargetDeclAttr::MT_Local) &&
11824 "Expected link clause or to clause with unified memory.");
11825 (void)
CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
11833 " Expected target-based directive.");
11838 if (Clause->getClauseKind() == OMPC_unified_shared_memory) {
11840 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(
true);
11841 }
else if (
const auto *AC =
11842 dyn_cast<OMPAtomicDefaultMemOrderClause>(Clause)) {
11843 switch (AC->getAtomicDefaultMemOrderKind()) {
11844 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_acq_rel:
11847 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_seq_cst:
11850 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_relaxed:
11866 if (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())
11868 const auto *A = VD->
getAttr<OMPAllocateDeclAttr>();
11869 switch(A->getAllocatorType()) {
11870 case OMPAllocateDeclAttr::OMPNullMemAlloc:
11871 case OMPAllocateDeclAttr::OMPDefaultMemAlloc:
11873 case OMPAllocateDeclAttr::OMPLargeCapMemAlloc:
11874 case OMPAllocateDeclAttr::OMPCGroupMemAlloc:
11875 case OMPAllocateDeclAttr::OMPHighBWMemAlloc:
11876 case OMPAllocateDeclAttr::OMPLowLatMemAlloc:
11877 case OMPAllocateDeclAttr::OMPThreadMemAlloc:
11878 case OMPAllocateDeclAttr::OMPConstMemAlloc:
11879 case OMPAllocateDeclAttr::OMPPTeamMemAlloc:
11882 case OMPAllocateDeclAttr::OMPUserDefinedMemAlloc:
11883 llvm_unreachable(
"Expected predefined allocator for the variables with the "
11884 "static storage.");
11896 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
11897 SavedShouldMarkAsGlobal = CGM.getOpenMPRuntime().ShouldMarkAsGlobal;
11898 CGM.getOpenMPRuntime().ShouldMarkAsGlobal = false;
11903 if (CGM.getLangOpts().OpenMPIsTargetDevice)
11904 CGM.getOpenMPRuntime().ShouldMarkAsGlobal = SavedShouldMarkAsGlobal;
11914 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(D)) {
11916 if (
auto *F = dyn_cast_or_null<llvm::Function>(
11917 CGM.GetGlobalValue(
CGM.getMangledName(GD))))
11918 return !F->isDeclaration();
11930 llvm::Function *OutlinedFn,
11939 llvm::Value *Args[] = {
11941 CGF.
Builder.getInt32(CapturedVars.size()),
11944 RealArgs.append(std::begin(Args), std::end(Args));
11945 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
11947 llvm::FunctionCallee RTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
11948 CGM.getModule(), OMPRTL___kmpc_fork_teams);
11953 const Expr *NumTeams,
11954 const Expr *ThreadLimit,
11961 llvm::Value *NumTeamsVal =
11967 llvm::Value *ThreadLimitVal =
11974 llvm::Value *PushNumTeamsArgs[] = {RTLoc,
getThreadID(CGF, Loc), NumTeamsVal,
11977 CGM.getModule(), OMPRTL___kmpc_push_num_teams),
11982 const Expr *ThreadLimit,
11985 llvm::Value *ThreadLimitVal =
11992 llvm::Value *ThreadLimitArgs[] = {RTLoc,
getThreadID(CGF, Loc),
11995 CGM.getModule(), OMPRTL___kmpc_set_thread_limit),
12010 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
12012 llvm::Value *IfCondVal =
nullptr;
12017 llvm::Value *DeviceID =
nullptr;
12022 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
12026 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
12027 auto GenMapInfoCB =
12028 [&](InsertPointTy CodeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {
12029 CGF.
Builder.restoreIP(CodeGenIP);
12031 MappableExprsHandler MEHandler(D, CGF);
12032 MEHandler.generateAllInfo(CombinedInfo,
OMPBuilder);
12034 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
12037 if (
CGM.getCodeGenOpts().getDebugInfo() !=
12038 llvm::codegenoptions::NoDebugInfo) {
12039 CombinedInfo.Names.resize(CombinedInfo.Exprs.size());
12040 llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),
12044 return CombinedInfo;
12046 using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;
12047 auto BodyCB = [&](InsertPointTy CodeGenIP, BodyGenTy BodyGenType) {
12048 CGF.
Builder.restoreIP(CodeGenIP);
12049 switch (BodyGenType) {
12050 case BodyGenTy::Priv:
12054 case BodyGenTy::DupNoPriv:
12056 CodeGen.setAction(NoPrivAction);
12060 case BodyGenTy::NoPriv:
12062 CodeGen.setAction(NoPrivAction);
12067 return InsertPointTy(CGF.
Builder.GetInsertPoint());
12070 auto DeviceAddrCB = [&](
unsigned int I, llvm::Value *NewDecl) {
12071 if (
const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
12076 auto CustomMapperCB = [&](
unsigned int I) {
12077 llvm::Function *MFunc =
nullptr;
12078 if (CombinedInfo.Mappers[I]) {
12079 Info.HasMapper =
true;
12090 InsertPointTy CodeGenIP(CGF.
Builder.GetInsertPoint());
12091 llvm::OpenMPIRBuilder::LocationDescription OmpLoc(CGF.
Builder);
12092 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
12094 OmpLoc, AllocaIP, CodeGenIP, {}, DeviceID,
12095 IfCondVal, Info, GenMapInfoCB, CustomMapperCB,
12096 nullptr, BodyCB, DeviceAddrCB, RTLoc));
12097 CGF.
Builder.restoreIP(AfterIP);
12109 "Expecting either target enter, exit data, or update directives.");
12112 llvm::Value *MapTypesArray =
nullptr;
12113 llvm::Value *MapNamesArray =
nullptr;
12115 auto &&ThenGen = [
this, &D,
Device, &InputInfo, &MapTypesArray,
12118 llvm::Value *DeviceID =
nullptr;
12123 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
12127 llvm::Constant *PointerNum =
12134 {RTLoc, DeviceID, PointerNum,
12142 const bool HasNowait = D.hasClausesOfKind<OMPNowaitClause>();
12143 RuntimeFunction RTLFn;
12144 switch (D.getDirectiveKind()) {
12145 case OMPD_target_enter_data:
12146 RTLFn = HasNowait ? OMPRTL___tgt_target_data_begin_nowait_mapper
12147 : OMPRTL___tgt_target_data_begin_mapper;
12149 case OMPD_target_exit_data:
12150 RTLFn = HasNowait ? OMPRTL___tgt_target_data_end_nowait_mapper
12151 : OMPRTL___tgt_target_data_end_mapper;
12153 case OMPD_target_update:
12154 RTLFn = HasNowait ? OMPRTL___tgt_target_data_update_nowait_mapper
12155 : OMPRTL___tgt_target_data_update_mapper;
12157 case OMPD_parallel:
12159 case OMPD_parallel_for:
12160 case OMPD_parallel_master:
12161 case OMPD_parallel_sections:
12162 case OMPD_for_simd:
12163 case OMPD_parallel_for_simd:
12165 case OMPD_cancellation_point:
12166 case OMPD_ordered_standalone:
12167 case OMPD_ordered_blockassoc:
12168 case OMPD_threadprivate:
12169 case OMPD_allocate:
12174 case OMPD_sections:
12178 case OMPD_critical:
12179 case OMPD_taskyield:
12181 case OMPD_taskwait:
12182 case OMPD_taskgroup:
12188 case OMPD_target_data:
12189 case OMPD_distribute:
12190 case OMPD_distribute_simd:
12191 case OMPD_distribute_parallel_for:
12192 case OMPD_distribute_parallel_for_simd:
12193 case OMPD_teams_distribute:
12194 case OMPD_teams_distribute_simd:
12195 case OMPD_teams_distribute_parallel_for:
12196 case OMPD_teams_distribute_parallel_for_simd:
12197 case OMPD_declare_simd:
12198 case OMPD_declare_variant:
12199 case OMPD_begin_declare_variant:
12200 case OMPD_end_declare_variant:
12201 case OMPD_declare_target:
12202 case OMPD_end_declare_target:
12203 case OMPD_declare_reduction:
12204 case OMPD_declare_mapper:
12205 case OMPD_taskloop:
12206 case OMPD_taskloop_simd:
12207 case OMPD_master_taskloop:
12208 case OMPD_master_taskloop_simd:
12209 case OMPD_parallel_master_taskloop:
12210 case OMPD_parallel_master_taskloop_simd:
12212 case OMPD_target_simd:
12213 case OMPD_target_teams_distribute:
12214 case OMPD_target_teams_distribute_simd:
12215 case OMPD_target_teams_distribute_parallel_for:
12216 case OMPD_target_teams_distribute_parallel_for_simd:
12217 case OMPD_target_teams:
12218 case OMPD_target_parallel:
12219 case OMPD_target_parallel_for:
12220 case OMPD_target_parallel_for_simd:
12221 case OMPD_requires:
12222 case OMPD_metadirective:
12225 llvm_unreachable(
"Unexpected standalone target data directive.");
12229 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.
Int32Ty));
12230 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.
VoidPtrTy));
12231 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.
Int32Ty));
12232 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.
VoidPtrTy));
12235 OMPBuilder.getOrCreateRuntimeFunction(
CGM.getModule(), RTLFn),
12239 auto &&TargetThenGen = [
this, &ThenGen, &D, &InputInfo, &MapTypesArray,
12243 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
12245 MappableExprsHandler MEHandler(D, CGF);
12251 D.hasClausesOfKind<OMPNowaitClause>();
12257 CGM.getPointerAlign());
12262 MapTypesArray = Info.RTArgs.MapTypesArray;
12263 MapNamesArray = Info.RTArgs.MapNamesArray;
12264 if (RequiresOuterTask)
12309 unsigned Offset = 0;
12310 if (
const auto *MD = dyn_cast<CXXMethodDecl>(FD)) {
12311 if (ParamAttrs[Offset].Kind ==
12312 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector)
12313 CDT =
C.getPointerType(
C.getCanonicalTagType(MD->
getParent()));
12317 for (
unsigned I = 0, E = FD->
getNumParams(); I < E; ++I) {
12318 if (ParamAttrs[I + Offset].Kind ==
12319 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector) {
12331 return C.getTypeSize(CDT);
12342 llvm::OpenMPIRBuilder::DeclareSimdKindTy Kind) {
12348 if (Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::Uniform)
12351 if (Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal ||
12352 Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef)
12355 if ((Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear ||
12356 Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal) &&
12366 unsigned Size =
C.getTypeSize(QT);
12369 if (Size != 8 && Size != 16 && Size != 32 && Size != 64 && Size != 128)
12390 llvm::OpenMPIRBuilder::DeclareSimdKindTy Kind,
12395 return C.getTypeSize(PTy);
12398 return C.getTypeSize(QT);
12400 return C.getTypeSize(
C.getUIntPtrType());
12406static std::tuple<unsigned, unsigned, bool>
12413 bool OutputBecomesInput =
false;
12418 RetType, llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector,
C));
12420 OutputBecomesInput =
true;
12422 for (
unsigned I = 0, E = FD->
getNumParams(); I < E; ++I) {
12427 assert(!Sizes.empty() &&
"Unable to determine NDS and WDS.");
12430 assert(llvm::all_of(Sizes,
12431 [](
unsigned Size) {
12432 return Size == 8 || Size == 16 || Size == 32 ||
12433 Size == 64 || Size == 128;
12437 return std::make_tuple(*llvm::min_element(Sizes), *llvm::max_element(Sizes),
12438 OutputBecomesInput);
12441static llvm::OpenMPIRBuilder::DeclareSimdBranch
12444 case OMPDeclareSimdDeclAttr::BS_Undefined:
12445 return llvm::OpenMPIRBuilder::DeclareSimdBranch::Undefined;
12446 case OMPDeclareSimdDeclAttr::BS_Inbranch:
12447 return llvm::OpenMPIRBuilder::DeclareSimdBranch::Inbranch;
12448 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
12449 return llvm::OpenMPIRBuilder::DeclareSimdBranch::Notinbranch;
12451 llvm_unreachable(
"unexpected declare simd branch state");
12456 unsigned UserVLEN,
unsigned WDS,
char ISA) {
12458 if (UserVLEN == 1) {
12465 if (ISA ==
'n' && UserVLEN && !llvm::isPowerOf2_32(UserVLEN)) {
12471 if (ISA ==
's' && UserVLEN != 0 &&
12472 ((UserVLEN * WDS > 2048) || (UserVLEN * WDS % 128 != 0))) {
12481 llvm::Function *Fn) {
12486 llvm::DenseMap<const Decl *, unsigned> ParamPositions;
12488 ParamPositions.try_emplace(FD, 0);
12489 unsigned ParamPos = ParamPositions.size();
12491 ParamPositions.try_emplace(P->getCanonicalDecl(), ParamPos);
12496 ParamPositions.size());
12498 for (
const Expr *E :
Attr->uniforms()) {
12502 Pos = ParamPositions[FD];
12505 ->getCanonicalDecl();
12506 auto It = ParamPositions.find(PVD);
12507 assert(It != ParamPositions.end() &&
"Function parameter not found");
12510 ParamAttrs[Pos].Kind =
12511 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Uniform;
12514 auto *NI =
Attr->alignments_begin();
12515 for (
const Expr *E :
Attr->aligneds()) {
12520 Pos = ParamPositions[FD];
12524 ->getCanonicalDecl();
12525 auto It = ParamPositions.find(PVD);
12526 assert(It != ParamPositions.end() &&
"Function parameter not found");
12528 ParmTy = PVD->getType();
12530 ParamAttrs[Pos].Alignment =
12532 ? (*NI)->EvaluateKnownConstInt(
C)
12533 : llvm::APSInt::getUnsigned(
12534 C.toCharUnitsFromBits(
C.getOpenMPDefaultSimdAlign(ParmTy))
12539 auto *SI =
Attr->steps_begin();
12540 auto *MI =
Attr->modifiers_begin();
12541 for (
const Expr *E :
Attr->linears()) {
12544 bool IsReferenceType =
false;
12547 unsigned PtrRescalingFactor = 1;
12549 Pos = ParamPositions[FD];
12551 PtrRescalingFactor =
CGM.getContext()
12552 .getTypeSizeInChars(P->getPointeeType())
12556 ->getCanonicalDecl();
12557 auto It = ParamPositions.find(PVD);
12558 assert(It != ParamPositions.end() &&
"Function parameter not found");
12560 if (
auto *P = dyn_cast<PointerType>(PVD->getType()))
12561 PtrRescalingFactor =
CGM.getContext()
12562 .getTypeSizeInChars(P->getPointeeType())
12564 else if (PVD->getType()->isReferenceType()) {
12565 IsReferenceType =
true;
12566 PtrRescalingFactor =
12568 .getTypeSizeInChars(PVD->getType().getNonReferenceType())
12572 llvm::OpenMPIRBuilder::DeclareSimdAttrTy &ParamAttr = ParamAttrs[Pos];
12573 if (*MI == OMPC_LINEAR_ref)
12574 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef;
12575 else if (*MI == OMPC_LINEAR_uval)
12576 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal;
12577 else if (IsReferenceType)
12578 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal;
12580 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear;
12582 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(1);
12586 if (
const auto *DRE =
12588 if (
const auto *StridePVD =
12589 dyn_cast<ParmVarDecl>(DRE->getDecl())) {
12590 ParamAttr.HasVarStride =
true;
12591 auto It = ParamPositions.find(StridePVD->getCanonicalDecl());
12592 assert(It != ParamPositions.end() &&
12593 "Function parameter not found");
12594 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(It->second);
12598 ParamAttr.StrideOrArg =
Result.Val.getInt();
12604 if (!ParamAttr.HasVarStride &&
12606 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear ||
12608 llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef))
12609 ParamAttr.StrideOrArg = ParamAttr.StrideOrArg * PtrRescalingFactor;
12613 llvm::APSInt VLENVal;
12615 const Expr *VLENExpr =
Attr->getSimdlen();
12620 llvm::OpenMPIRBuilder::DeclareSimdBranch State =
12622 if (
CGM.getTriple().isX86()) {
12624 assert(NumElts &&
"Non-zero simdlen/cdtsize expected");
12625 OMPBuilder.emitX86DeclareSimdFunction(Fn, NumElts, VLENVal, ParamAttrs,
12627 }
else if (
CGM.getTriple().getArch() == llvm::Triple::aarch64) {
12628 unsigned VLEN = VLENVal.getExtValue();
12631 const unsigned NDS = std::get<0>(
Data);
12632 const unsigned WDS = std::get<1>(
Data);
12633 const bool OutputBecomesInput = std::get<2>(
Data);
12634 if (
CGM.getTarget().hasFeature(
"sve")) {
12637 Fn, VLEN, ParamAttrs, State,
's', NDS, OutputBecomesInput);
12638 }
else if (
CGM.getTarget().hasFeature(
"neon")) {
12641 Fn, VLEN, ParamAttrs, State,
'n', NDS, OutputBecomesInput);
12651class DoacrossCleanupTy final :
public EHScopeStack::Cleanup {
12653 static const int DoacrossFinArgs = 2;
12656 llvm::FunctionCallee RTLFn;
12657 llvm::Value *Args[DoacrossFinArgs];
12660 DoacrossCleanupTy(llvm::FunctionCallee RTLFn,
12663 assert(CallArgs.size() == DoacrossFinArgs);
12664 std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args));
12681 QualType Int64Ty =
C.getIntTypeForBitwidth(64,
true);
12689 RD =
C.buildImplicitRecord(
"kmp_dim");
12697 RD =
KmpDimTy->castAsRecordDecl();
12699 llvm::APInt Size(32, NumIterations.size());
12705 enum { LowerFD = 0, UpperFD, StrideFD };
12707 for (
unsigned I = 0, E = NumIterations.size(); I < E; ++I) {
12712 DimsLVal, *std::next(RD->
field_begin(), UpperFD));
12714 CGF.
EmitScalarExpr(NumIterations[I]), NumIterations[I]->getType(),
12715 Int64Ty, NumIterations[I]->getExprLoc());
12719 DimsLVal, *std::next(RD->
field_begin(), StrideFD));
12726 llvm::Value *Args[] = {
12729 llvm::ConstantInt::getSigned(
CGM.Int32Ty, NumIterations.size()),
12734 llvm::FunctionCallee RTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
12735 CGM.getModule(), OMPRTL___kmpc_doacross_init);
12737 llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = {
12739 llvm::FunctionCallee FiniRTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
12740 CGM.getModule(), OMPRTL___kmpc_doacross_fini);
12745template <
typename T>
12747 const T *
C, llvm::Value *ULoc,
12748 llvm::Value *ThreadID) {
12751 llvm::APInt Size(32,
C->getNumLoops());
12755 for (
unsigned I = 0, E =
C->getNumLoops(); I < E; ++I) {
12756 const Expr *CounterVal =
C->getLoopData(I);
12757 assert(CounterVal);
12764 llvm::Value *Args[] = {
12767 llvm::FunctionCallee RTLFn;
12769 OMPDoacrossKind<T> ODK;
12770 if (ODK.isSource(
C)) {
12772 OMPRTL___kmpc_doacross_post);
12774 assert(ODK.isSink(
C) &&
"Expect sink modifier.");
12776 OMPRTL___kmpc_doacross_wait);
12796 llvm::FunctionCallee Callee,
12798 assert(Loc.
isValid() &&
"Outlined function call location must be valid.");
12801 if (
auto *Fn = dyn_cast<llvm::Function>(Callee.getCallee())) {
12802 if (Fn->doesNotThrow()) {
12813 emitCall(CGF, Loc, OutlinedFn, Args);
12817 if (
const auto *FD = dyn_cast<FunctionDecl>(D))
12818 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(FD))
12824 const VarDecl *TargetParam)
const {
12831 const Expr *Allocator) {
12832 llvm::Value *AllocVal;
12842 AllocVal = llvm::Constant::getNullValue(
12852 if (!AllocateAlignment)
12855 return llvm::ConstantInt::get(
CGM.
SizeTy, AllocateAlignment->getQuantity());
12868 auto I = UntiedData.find(VD);
12869 if (I != UntiedData.end()) {
12870 UntiedAddr = I->second.first;
12871 UntiedRealAddr = I->second.second;
12875 if (CVD->
hasAttr<OMPAllocateDeclAttr>()) {
12884 Size = CGF.
Builder.CreateNUWAdd(
12886 Size = CGF.
Builder.CreateUDiv(Size,
CGM.getSize(Align));
12887 Size = CGF.
Builder.CreateNUWMul(Size,
CGM.getSize(Align));
12893 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
12894 const Expr *Allocator = AA->getAllocator();
12898 Args.push_back(ThreadID);
12900 Args.push_back(Alignment);
12901 Args.push_back(Size);
12902 Args.push_back(AllocVal);
12903 llvm::omp::RuntimeFunction FnID =
12904 Alignment ? OMPRTL___kmpc_aligned_alloc : OMPRTL___kmpc_alloc;
12906 OMPBuilder.getOrCreateRuntimeFunction(
CGM.getModule(), FnID), Args,
12908 llvm::FunctionCallee FiniRTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
12909 CGM.getModule(), OMPRTL___kmpc_free);
12917 class OMPAllocateCleanupTy final :
public EHScopeStack::Cleanup {
12918 llvm::FunctionCallee RTLFn;
12921 const Expr *AllocExpr;
12924 OMPAllocateCleanupTy(llvm::FunctionCallee RTLFn,
12926 const Expr *AllocExpr)
12927 : RTLFn(RTLFn), LocEncoding(LocEncoding),
Addr(
Addr),
12928 AllocExpr(AllocExpr) {}
12932 llvm::Value *Args[3];
12938 Args[2] = AllocVal;
12946 CGF.
EHStack.pushCleanup<OMPAllocateCleanupTy>(
12948 VDAddr, Allocator);
12949 if (UntiedRealAddr.
isValid())
12952 Region->emitUntiedSwitch(CGF);
12969 assert(CGM.getLangOpts().OpenMP &&
"Not in OpenMP mode.");
12973 CGM.getOpenMPRuntime().NontemporalDeclsStack.emplace_back();
12975 for (
const Stmt *Ref :
C->private_refs()) {
12976 const auto *SimpleRefExpr =
cast<Expr>(Ref)->IgnoreParenImpCasts();
12978 if (
const auto *DRE = dyn_cast<DeclRefExpr>(SimpleRefExpr)) {
12979 VD = DRE->getDecl();
12982 assert((ME->isImplicitCXXThis() ||
12984 "Expected member of current class.");
12985 VD = ME->getMemberDecl();
12995 CGM.getOpenMPRuntime().NontemporalDeclsStack.pop_back();
13001 std::pair<Address, Address>> &LocalVars)
13002 : CGM(CGF.CGM), NeedToPush(!LocalVars.empty()) {
13006 CGF.
CurFn, CGM.getOpenMPRuntime().UntiedLocalVarsStack.size());
13007 CGM.getOpenMPRuntime().UntiedLocalVarsStack.push_back(LocalVars);
13013 CGM.getOpenMPRuntime().UntiedLocalVarsStack.pop_back();
13017 assert(
CGM.getLangOpts().OpenMP &&
"Not in OpenMP mode.");
13019 return llvm::any_of(
13020 CGM.getOpenMPRuntime().NontemporalDeclsStack,
13024void CGOpenMPRuntime::LastprivateConditionalRAII::tryToDisableInnerAnalysis(
13028 llvm::DenseSet<CanonicalDeclPtr<const Decl>> NeedToCheckForLPCs;
13034 const CapturedStmt *CS = S.getCapturedStmt(CaptureRegions.front());
13041 for (
const auto *
C : S.getClausesOfKind<OMPPrivateClause>()) {
13042 for (
const Expr *Ref :
C->varlist()) {
13043 if (!Ref->getType()->isScalarType())
13045 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
13048 NeedToCheckForLPCs.insert(DRE->getDecl());
13051 for (
const auto *
C : S.getClausesOfKind<OMPFirstprivateClause>()) {
13052 for (
const Expr *Ref :
C->varlist()) {
13053 if (!Ref->getType()->isScalarType())
13055 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
13058 NeedToCheckForLPCs.insert(DRE->getDecl());
13061 for (
const auto *
C : S.getClausesOfKind<OMPLastprivateClause>()) {
13062 for (
const Expr *Ref :
C->varlist()) {
13063 if (!Ref->getType()->isScalarType())
13065 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
13068 NeedToCheckForLPCs.insert(DRE->getDecl());
13071 for (
const auto *
C : S.getClausesOfKind<OMPReductionClause>()) {
13072 for (
const Expr *Ref :
C->varlist()) {
13073 if (!Ref->getType()->isScalarType())
13075 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
13078 NeedToCheckForLPCs.insert(DRE->getDecl());
13081 for (
const auto *
C : S.getClausesOfKind<OMPLinearClause>()) {
13082 for (
const Expr *Ref :
C->varlist()) {
13083 if (!Ref->getType()->isScalarType())
13085 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
13088 NeedToCheckForLPCs.insert(DRE->getDecl());
13091 for (
const Decl *VD : NeedToCheckForLPCs) {
13093 llvm::reverse(
CGM.getOpenMPRuntime().LastprivateConditionalStack)) {
13094 if (
Data.DeclToUniqueName.count(VD) > 0) {
13095 if (!
Data.Disabled)
13096 NeedToAddForLPCsAsDisabled.insert(VD);
13103CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
13106 Action((CGM.getLangOpts().OpenMP >= 50 &&
13107 llvm::any_of(S.getClausesOfKind<OMPLastprivateClause>(),
13108 [](const OMPLastprivateClause *
C) {
13109 return C->getKind() ==
13110 OMPC_LASTPRIVATE_conditional;
13112 ? ActionToDo::PushAsLastprivateConditional
13113 : ActionToDo::DoNotPush) {
13114 assert(
CGM.getLangOpts().OpenMP &&
"Not in OpenMP mode.");
13115 if (
CGM.getLangOpts().OpenMP < 50 || Action == ActionToDo::DoNotPush)
13117 assert(Action == ActionToDo::PushAsLastprivateConditional &&
13118 "Expected a push action.");
13120 CGM.getOpenMPRuntime().LastprivateConditionalStack.emplace_back();
13121 for (
const auto *
C : S.getClausesOfKind<OMPLastprivateClause>()) {
13122 if (
C->getKind() != OMPC_LASTPRIVATE_conditional)
13125 for (
const Expr *Ref :
C->varlist()) {
13126 Data.DeclToUniqueName.insert(std::make_pair(
13131 Data.IVLVal = IVLVal;
13135CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
13137 :
CGM(CGF.
CGM), Action(ActionToDo::DoNotPush) {
13141 llvm::DenseSet<CanonicalDeclPtr<const Decl>> NeedToAddForLPCsAsDisabled;
13142 tryToDisableInnerAnalysis(S, NeedToAddForLPCsAsDisabled);
13143 if (!NeedToAddForLPCsAsDisabled.empty()) {
13144 Action = ActionToDo::DisableLastprivateConditional;
13145 LastprivateConditionalData &
Data =
13147 for (
const Decl *VD : NeedToAddForLPCsAsDisabled)
13148 Data.DeclToUniqueName.try_emplace(VD);
13150 Data.Disabled =
true;
13154CGOpenMPRuntime::LastprivateConditionalRAII
13157 return LastprivateConditionalRAII(CGF, S);
13161 if (CGM.getLangOpts().OpenMP < 50)
13163 if (Action == ActionToDo::DisableLastprivateConditional) {
13164 assert(CGM.getOpenMPRuntime().LastprivateConditionalStack.back().Disabled &&
13165 "Expected list of disabled private vars.");
13166 CGM.getOpenMPRuntime().LastprivateConditionalStack.pop_back();
13168 if (Action == ActionToDo::PushAsLastprivateConditional) {
13170 !CGM.getOpenMPRuntime().LastprivateConditionalStack.back().Disabled &&
13171 "Expected list of lastprivate conditional vars.");
13172 CGM.getOpenMPRuntime().LastprivateConditionalStack.pop_back();
13184 auto VI = I->getSecond().find(VD);
13185 if (VI == I->getSecond().end()) {
13186 RecordDecl *RD =
C.buildImplicitRecord(
"lasprivate.conditional");
13191 NewType =
C.getCanonicalTagType(RD);
13194 I->getSecond().try_emplace(VD, NewType, VDField, FiredField, BaseLVal);
13196 NewType = std::get<0>(VI->getSecond());
13197 VDField = std::get<1>(VI->getSecond());
13198 FiredField = std::get<2>(VI->getSecond());
13199 BaseLVal = std::get<3>(VI->getSecond());
13211class LastprivateConditionalRefChecker final
13214 const Expr *FoundE =
nullptr;
13215 const Decl *FoundD =
nullptr;
13216 StringRef UniqueDeclName;
13218 llvm::Function *FoundFn =
nullptr;
13224 llvm::reverse(LPM)) {
13225 auto It = D.DeclToUniqueName.find(E->
getDecl());
13226 if (It == D.DeclToUniqueName.end())
13232 UniqueDeclName = It->second;
13237 return FoundE == E;
13243 llvm::reverse(LPM)) {
13245 if (It == D.DeclToUniqueName.end())
13251 UniqueDeclName = It->second;
13256 return FoundE == E;
13258 bool VisitStmt(
const Stmt *S) {
13259 for (
const Stmt *Child : S->
children()) {
13262 if (
const auto *E = dyn_cast<Expr>(Child))
13270 explicit LastprivateConditionalRefChecker(
13271 ArrayRef<CGOpenMPRuntime::LastprivateConditionalData> LPM)
13273 std::tuple<const Expr *, const Decl *, StringRef, LValue, llvm::Function *>
13274 getFoundData()
const {
13275 return std::make_tuple(FoundE, FoundD, UniqueDeclName, IVLVal, FoundFn);
13282 StringRef UniqueDeclName,
13288 llvm::Constant *LastIV =
OMPBuilder.getOrCreateInternalVariable(
13289 LLIVTy,
getName({UniqueDeclName,
"iv"}));
13297 llvm::GlobalVariable *
Last =
OMPBuilder.getOrCreateInternalVariable(
13313 auto &&
CodeGen = [&LastIVLVal, &IVLVal, IVVal, &LVal, &LastLVal,
13319 llvm::Value *CmpRes;
13321 CmpRes = CGF.
Builder.CreateICmpSLE(LastIVVal, IVVal);
13324 "Loop iteration variable must be integer.");
13325 CmpRes = CGF.
Builder.CreateICmpULE(LastIVVal, IVVal);
13329 CGF.
Builder.CreateCondBr(CmpRes, ThenBB, ExitBB);
13350 "Aggregates are not supported in lastprivate conditional.");
13359 if (
CGM.getLangOpts().OpenMPSimd) {
13373 if (!Checker.Visit(LHS))
13375 const Expr *FoundE;
13376 const Decl *FoundD;
13377 StringRef UniqueDeclName;
13379 llvm::Function *FoundFn;
13380 std::tie(FoundE, FoundD, UniqueDeclName, IVLVal, FoundFn) =
13381 Checker.getFoundData();
13382 if (FoundFn != CGF.
CurFn) {
13387 "Lastprivate conditional is not found in outer region.");
13388 QualType StructTy = std::get<0>(It->getSecond());
13389 const FieldDecl* FiredDecl = std::get<2>(It->getSecond());
13400 FiredLVal, llvm::AtomicOrdering::Unordered,
13418 auto It = llvm::find_if(
13420 if (It == Range.end() || It->Fn != CGF.
CurFn)
13424 "Lastprivates must be registered already.");
13427 const CapturedStmt *CS = D.getCapturedStmt(CaptureRegions.back());
13428 for (
const auto &Pair : It->DeclToUniqueName) {
13429 const auto *VD =
cast<VarDecl>(Pair.first->getCanonicalDecl());
13432 auto I = LPCI->getSecond().find(Pair.first);
13433 assert(I != LPCI->getSecond().end() &&
13434 "Lastprivate must be rehistered already.");
13436 LValue BaseLVal = std::get<3>(I->getSecond());
13440 llvm::Value *
Cmp = CGF.
Builder.CreateIsNotNull(Res);
13444 CGF.
Builder.CreateCondBr(
Cmp, ThenBB, DoneBB);
13469 "Unknown lastprivate conditional variable.");
13470 StringRef UniqueName = It->second;
13471 llvm::GlobalVariable *GV =
CGM.getModule().getNamedGlobal(UniqueName);
13485 llvm_unreachable(
"Not supported in SIMD-only mode");
13492 llvm_unreachable(
"Not supported in SIMD-only mode");
13499 bool Tied,
unsigned &NumberOfParts) {
13500 llvm_unreachable(
"Not supported in SIMD-only mode");
13508 llvm_unreachable(
"Not supported in SIMD-only mode");
13514 const Expr *Hint) {
13515 llvm_unreachable(
"Not supported in SIMD-only mode");
13521 llvm_unreachable(
"Not supported in SIMD-only mode");
13527 const Expr *Filter) {
13528 llvm_unreachable(
"Not supported in SIMD-only mode");
13533 llvm_unreachable(
"Not supported in SIMD-only mode");
13539 llvm_unreachable(
"Not supported in SIMD-only mode");
13547 llvm_unreachable(
"Not supported in SIMD-only mode");
13554 llvm_unreachable(
"Not supported in SIMD-only mode");
13561 bool ForceSimpleCall) {
13562 llvm_unreachable(
"Not supported in SIMD-only mode");
13569 llvm_unreachable(
"Not supported in SIMD-only mode");
13574 llvm_unreachable(
"Not supported in SIMD-only mode");
13580 llvm_unreachable(
"Not supported in SIMD-only mode");
13586 llvm_unreachable(
"Not supported in SIMD-only mode");
13593 llvm_unreachable(
"Not supported in SIMD-only mode");
13599 llvm_unreachable(
"Not supported in SIMD-only mode");
13604 unsigned IVSize,
bool IVSigned,
13607 llvm_unreachable(
"Not supported in SIMD-only mode");
13615 llvm_unreachable(
"Not supported in SIMD-only mode");
13619 ProcBindKind ProcBind,
13621 llvm_unreachable(
"Not supported in SIMD-only mode");
13628 llvm_unreachable(
"Not supported in SIMD-only mode");
13634 llvm_unreachable(
"Not supported in SIMD-only mode");
13639 llvm_unreachable(
"Not supported in SIMD-only mode");
13645 llvm::AtomicOrdering AO) {
13646 llvm_unreachable(
"Not supported in SIMD-only mode");
13651 llvm::Function *TaskFunction,
13653 const Expr *IfCond,
13655 llvm_unreachable(
"Not supported in SIMD-only mode");
13662 llvm_unreachable(
"Not supported in SIMD-only mode");
13669 assert(Options.
SimpleReduction &&
"Only simple reduction is expected.");
13671 ReductionOps, Options);
13677 llvm_unreachable(
"Not supported in SIMD-only mode");
13682 bool IsWorksharingReduction) {
13683 llvm_unreachable(
"Not supported in SIMD-only mode");
13690 llvm_unreachable(
"Not supported in SIMD-only mode");
13695 llvm::Value *ReductionsPtr,
13697 llvm_unreachable(
"Not supported in SIMD-only mode");
13703 llvm_unreachable(
"Not supported in SIMD-only mode");
13709 llvm_unreachable(
"Not supported in SIMD-only mode");
13715 llvm_unreachable(
"Not supported in SIMD-only mode");
13720 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
13722 llvm_unreachable(
"Not supported in SIMD-only mode");
13727 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID,
const Expr *IfCond,
13728 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
13732 llvm_unreachable(
"Not supported in SIMD-only mode");
13736 llvm_unreachable(
"Not supported in SIMD-only mode");
13740 llvm_unreachable(
"Not supported in SIMD-only mode");
13750 llvm::Function *OutlinedFn,
13752 llvm_unreachable(
"Not supported in SIMD-only mode");
13756 const Expr *NumTeams,
13757 const Expr *ThreadLimit,
13759 llvm_unreachable(
"Not supported in SIMD-only mode");
13766 llvm_unreachable(
"Not supported in SIMD-only mode");
13772 llvm_unreachable(
"Not supported in SIMD-only mode");
13778 llvm_unreachable(
"Not supported in SIMD-only mode");
13783 llvm_unreachable(
"Not supported in SIMD-only mode");
13788 llvm_unreachable(
"Not supported in SIMD-only mode");
13793 const VarDecl *NativeParam)
const {
13794 llvm_unreachable(
"Not supported in SIMD-only mode");
13800 const VarDecl *TargetParam)
const {
13801 llvm_unreachable(
"Not supported in SIMD-only mode");
static llvm::Value * emitCopyprivateCopyFunction(CodeGenModule &CGM, llvm::Type *ArgsElemType, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps, SourceLocation Loc)
static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF, SourceLocation Loc, SmallString< 128 > &Buffer)
static void emitOffloadingArraysAndArgs(CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo, CGOpenMPRuntime::TargetDataInfo &Info, llvm::OpenMPIRBuilder &OMPBuilder, bool IsNonContiguous=false, bool ForEndCall=false)
Emit the arrays used to pass the captures and map information to the offloading runtime library.
static RecordDecl * createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy, ArrayRef< PrivateDataTy > Privates)
static void emitInitWithReductionInitializer(CodeGenFunction &CGF, const OMPDeclareReductionDecl *DRD, const Expr *InitOp, Address Private, Address Original, QualType Ty)
static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, Address OriginalBaseAddress, llvm::Value *Addr)
static void emitPrivatesInit(CodeGenFunction &CGF, const OMPExecutableDirective &D, Address KmpTaskSharedsPtr, LValue TDBase, const RecordDecl *KmpTaskTWithPrivatesQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef< PrivateDataTy > Privates, bool ForDup)
Emit initialization for private variables in task-based directives.
static void emitClauseForBareTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &Values)
static llvm::Value * emitDestructorsFunction(CodeGenModule &CGM, SourceLocation Loc, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy)
static void EmitOMPAggregateReduction(CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar, const VarDecl *RHSVar, const llvm::function_ref< void(CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *)> &RedOpGen, const Expr *XExpr=nullptr, const Expr *EExpr=nullptr, const Expr *UpExpr=nullptr)
Emit reduction operation for each element of array (required for array sections) LHS op = RHS.
static void emitTargetCallFallback(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, CodeGenFunction &CGF)
static llvm::Value * emitReduceInitFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Emits reduction initializer function:
static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion)
static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy, llvm::PointerUnion< unsigned *, LValue * > Pos, const OMPTaskDataTy::DependData &Data, Address DependenciesArray)
static llvm::Value * emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc, const OMPTaskDataTy &Data, QualType PrivatesQTy, ArrayRef< PrivateDataTy > Privates)
Emit a privates mapping function for correct handling of private and firstprivate variables.
static llvm::Value * emitReduceCombFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N, const Expr *ReductionOp, const Expr *LHS, const Expr *RHS, const Expr *PrivateRef)
Emits reduction combiner function:
static RecordDecl * createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef< PrivateDataTy > Privates)
static llvm::Value * getAllocatorVal(CodeGenFunction &CGF, const Expr *Allocator)
Return allocator value from expression, or return a null allocator (default when no allocator specifi...
static llvm::Function * emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy, QualType SharedsPtrTy, llvm::Function *TaskFunction, llvm::Value *TaskPrivatesMap)
Emit a proxy function which accepts kmp_task_t as the second argument.
static bool isAllocatableDecl(const VarDecl *VD)
static llvm::Value * getAlignmentValue(CodeGenModule &CGM, const VarDecl *VD)
Return the alignment from an allocate directive if present.
static void emitTargetCallKernelLaunch(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::Value *OutlinedFnID, CodeGenFunction::OMPTargetDataInfo &InputInfo, llvm::Value *&MapTypesArray, llvm::Value *&MapNamesArray, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter, CodeGenFunction &CGF, CodeGenModule &CGM)
static const OMPExecutableDirective * getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D)
Check for inner distribute directive.
static std::pair< llvm::Value *, llvm::Value * > getPointerAndSize(CodeGenFunction &CGF, const Expr *E)
static const VarDecl * getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE)
static bool getAArch64MTV(QualType QT, llvm::OpenMPIRBuilder::DeclareSimdKindTy Kind)
Maps To Vector (MTV), as defined in 4.1.1 of the AAVFABI (2021Q1).
static bool isTrivial(ASTContext &Ctx, const Expr *E)
Checks if the expression is constant or does not have non-trivial function calls.
static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind, bool Chunked, bool Ordered)
Map the OpenMP loop schedule to the runtime enumeration.
static void getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS, const Expr **E, int32_t &UpperBound, bool UpperBoundOnly, llvm::Value **CondVal)
Check for a num threads constant value (stored in DefaultVal), or expression (stored in E).
static llvm::Value * emitDeviceID(llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, CodeGenFunction &CGF)
static const OMPDeclareReductionDecl * getReductionInit(const Expr *ReductionOp)
Check if the combiner is a call to UDR combiner and if it is so return the UDR decl used for reductio...
static bool checkInitIsRequired(CodeGenFunction &CGF, ArrayRef< PrivateDataTy > Privates)
Check if duplication function is required for taskloops.
static bool validateAArch64Simdlen(CodeGenModule &CGM, SourceLocation SLoc, unsigned UserVLEN, unsigned WDS, char ISA)
static bool checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD, ArrayRef< PrivateDataTy > Privates)
Checks if destructor function is required to be generated.
static llvm::TargetRegionEntryInfo getEntryInfoFromPresumedLoc(CodeGenModule &CGM, llvm::OpenMPIRBuilder &OMPBuilder, SourceLocation BeginLoc, llvm::StringRef ParentName="")
static void genMapInfo(MappableExprsHandler &MEHandler, CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo, llvm::OpenMPIRBuilder &OMPBuilder, const llvm::DenseSet< CanonicalDeclPtr< const Decl > > &SkippedVarSet=llvm::DenseSet< CanonicalDeclPtr< const Decl > >())
static unsigned getAArch64LS(QualType QT, llvm::OpenMPIRBuilder::DeclareSimdKindTy Kind, ASTContext &C)
Computes the lane size (LS) of a return type or of an input parameter, as defined by LS(P) in 3....
static llvm::OpenMPIRBuilder::DeclareSimdBranch convertDeclareSimdBranch(OMPDeclareSimdDeclAttr::BranchStateTy State)
static void emitForStaticInitCall(CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId, llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2, const CGOpenMPRuntime::StaticRTInput &Values)
static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, LValue BaseLV)
static void getKmpAffinityType(ASTContext &C, QualType &KmpTaskAffinityInfoTy)
Builds kmp_depend_info, if it is not built yet, and builds flags type.
static llvm::Constant * emitMappingInformation(CodeGenFunction &CGF, llvm::OpenMPIRBuilder &OMPBuilder, MappableExprsHandler::MappingExprInfo &MapExprs)
Emit a string constant containing the names of the values mapped to the offloading runtime library.
static void getDependTypes(ASTContext &C, QualType &KmpDependInfoTy, QualType &FlagsTy)
Builds kmp_depend_info, if it is not built yet, and builds flags type.
static llvm::Value * emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc, const OMPExecutableDirective &D, QualType KmpTaskTWithPrivatesPtrQTy, const RecordDecl *KmpTaskTWithPrivatesQTyRD, const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef< PrivateDataTy > Privates, bool WithLastIter)
Emit task_dup function (for initialization of private/firstprivate/lastprivate vars and last_iter fla...
static std::pair< llvm::Value *, OMPDynGroupprivateFallbackType > emitDynCGroupMem(const OMPExecutableDirective &D, CodeGenFunction &CGF)
static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind convertDeviceClause(const VarDecl *VD)
static llvm::Value * emitReduceFiniFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Emits reduction finalizer function:
static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, QualType Type, bool EmitDeclareReductionInit, const Expr *Init, const OMPDeclareReductionDecl *DRD, Address SrcAddr=Address::invalid())
Emit initialization of arrays of complex types.
static bool getAArch64PBV(QualType QT, ASTContext &C)
Pass By Value (PBV), as defined in 3.1.2 of the AAVFABI.
static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM, const T *C, llvm::Value *ULoc, llvm::Value *ThreadID)
static RTLDependenceKindTy translateDependencyKind(OpenMPDependClauseKind K)
Translates internal dependency kind into the runtime kind.
static void emitTargetCallElse(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, CodeGenFunction &CGF)
static llvm::Function * emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty, const Expr *CombinerInitializer, const VarDecl *In, const VarDecl *Out, bool IsCombiner)
static void emitReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp)
Emit reduction combiner.
static std::tuple< unsigned, unsigned, bool > getNDSWDS(const FunctionDecl *FD, ArrayRef< llvm::OpenMPIRBuilder::DeclareSimdAttrTy > ParamAttrs)
static std::string generateUniqueName(CodeGenModule &CGM, llvm::StringRef Prefix, const Expr *Ref)
static llvm::Function * emitParallelOrTeamsOutlinedFunction(CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen)
static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array, unsigned Index, const VarDecl *Var)
Given an array of pointers to variables, project the address of a given variable.
static const VarDecl * getOriginalVarDecl(const ValueDecl *Decl)
For BindingDecls, returns the DecomposedDecl as the original VarDecl.
static void mergeThreadCountUpperBound(int32_t &UpperBound, int32_t Val)
Merge the thread count upper bound Val into UpperBound.
static FieldDecl * addFieldToRecordDecl(ASTContext &C, DeclContext *DC, QualType FieldTy)
static unsigned evaluateCDTSize(const FunctionDecl *FD, ArrayRef< llvm::OpenMPIRBuilder::DeclareSimdAttrTy > ParamAttrs)
static ValueDecl * getDeclFromThisExpr(const Expr *E)
static void genMapInfoForCaptures(MappableExprsHandler &MEHandler, CodeGenFunction &CGF, const CapturedStmt &CS, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, llvm::OpenMPIRBuilder &OMPBuilder, llvm::DenseSet< CanonicalDeclPtr< const Decl > > &MappedVarSet, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo)
static RecordDecl * createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpRoutineEntryPointerQTy)
static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2)
static mlir::omp::DeclareTargetCaptureClause convertCaptureClause(OMPDeclareTargetDeclAttr::MapTypeTy mapTy)
static bool isAssumedToBeNotEmitted(const ValueDecl *vd, bool isDevice)
Returns true if the declaration should be skipped based on its device_type attribute and the current ...
Result
Implement __builtin_bit_cast and related operations.
This file defines OpenMP AST classes for clauses.
Defines some OpenMP-specific enums and functions.
Defines the SourceManager interface.
This file defines OpenMP AST classes for executable directives and clauses.
__DEVICE__ int max(int __a, int __b)
This represents clause 'affinity' in the 'pragma omp task'-based directives.
ValueDecl * getAssociatedDeclaration() const
Expr * getAssociatedExpression() const
static std::pair< const Expr *, std::optional< size_t > > findAttachPtrExpr(MappableExprComponentListRef Components, OpenMPDirectiveKind CurDirKind)
Find the attach pointer expression from a list of mappable expression components.
static QualType getComponentExprElementType(const Expr *Exp)
Get the type of an element of a ComponentList Expr Exp.
ArrayRef< MappableComponent > MappableExprComponentListRef
This represents implicit clause 'depend' for the 'pragma omp task' directive.
This represents 'detach' clause in the 'pragma omp task' directive.
This represents 'device' clause in the 'pragma omp ...' directive.
This represents the 'doacross' clause for the 'pragma omp ordered' directive.
This represents 'dyn_groupprivate' clause in 'pragma omp target ...' and 'pragma omp teams ....
This is a common base class for loop directives ('omp simd', 'omp for', 'omp for simd' etc....
Expr * getLowerBoundVariable() const
Expr * getUpperBoundVariable() const
Expr * getStrideVariable() const
This represents clause 'map' in the 'pragma omp ...' directives.
This represents clause 'nontemporal' in the 'pragma omp ...' directives.
This represents 'num_teams' clause in the 'pragma omp ...' directive.
This represents 'thread_limit' clause in the 'pragma omp ...' directive.
This represents clause 'uses_allocators' in the 'pragma omp target'-based directives.
This represents 'ompx_attribute' clause in a directive that might generate an outlined function.
This represents 'ompx_bare' clause in the 'pragma omp target teams ...' directive.
This represents 'ompx_dyn_cgroup_mem' clause in the 'pragma omp target ...' directive.
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
SourceManager & getSourceManager()
const ConstantArrayType * getAsConstantArrayType(QualType T) const
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
const ASTRecordLayout & getASTRecordLayout(const RecordDecl *D) const
Get or compute information about the layout of the specified record (struct/union/class) D,...
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
const LangOptions & getLangOpts() const
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth,...
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
static bool hasSameType(QualType T1, QualType T2)
Determine whether the given types T1 and T2 are equivalent.
const VariableArrayType * getAsVariableArrayType(QualType T) const
QualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
unsigned getTypeAlign(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in bits.
CharUnits getSize() const
getSize - Get the record size in characters.
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
CharUnits getNonVirtualSize() const
getNonVirtualSize - Get the non-virtual size (in chars) of an object, which is the size of the object...
static QualType getBaseOriginalType(const Expr *Base)
Return original type of the base expression for array section.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Attr - This represents one attribute.
A binding in a decomposition declaration.
Expr * getBinding() const
Get the expression to which this declaration is bound.
DecompositionDecl * getDecomposedDecl() const
Get the decomposition declaration that this binding represents a decomposition of.
Represents a base class of a C++ class.
Represents a C++ constructor within a class.
bool isInheritingConstructor() const
Determine whether this is an implicit constructor synthesized to model a call to a constructor inheri...
Represents a C++ base or member initializer.
Represents a C++ destructor within a class.
Represents a static or instance method of a struct/union/class.
const CXXRecordDecl * getParent() const
Return the parent of this method declaration, which is the class in which this method is defined.
QualType getFunctionObjectParameterType() const
Represents a C++ struct/union/class.
bool isLambda() const
Determine whether this class describes a lambda function object.
void getCaptureFields(llvm::DenseMap< const ValueDecl *, FieldDecl * > &Captures, FieldDecl *&ThisCapture) const
For a closure type, retrieve the mapping from captured variables and this to the non-static data memb...
unsigned getNumBases() const
Retrieves the number of base classes of this class.
base_class_range vbases()
capture_const_range captures() const
CXXDestructorDecl * getDestructor() const
Returns the destructor decl for this class.
CanProxy< U > castAs() const
A wrapper class around a pointer that always points to its canonical declaration.
Describes the capture of either a variable, or 'this', or variable-length array type.
bool capturesVariableByCopy() const
Determine whether this capture handles a variable by copy.
VarDecl * getCapturedVar() const
Retrieve the declaration of the variable being captured.
bool capturesVariableArrayType() const
Determine whether this capture handles a variable-length array type.
bool capturesThis() const
Determine whether this capture handles the C++ 'this' pointer.
bool capturesVariable() const
Determine whether this capture handles a variable (by reference).
This captures a statement into a function.
const Capture * const_capture_iterator
capture_iterator capture_end() const
Retrieve an iterator pointing past the end of the sequence of captures.
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Stmt * getCapturedStmt()
Retrieve the statement being captured.
bool capturesVariable(const VarDecl *Var) const
True if this variable has been captured.
capture_iterator capture_begin()
Retrieve an iterator pointing to the first capture.
This is an opaque type for sizes expressed in character units.
bool isZero() const
Test whether the quantity equals zero.
CharUnits alignTo(CharUnits Align) const
Returns the next integer (mod 2**64) that is greater than or equal to this quantity and is a multiple...
llvm::Align getAsAlign() const
Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit bytes.
QuantityType getQuantity() const
Get the raw integer representation of this quantity.
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
static CharUnits fromQuantity(QuantityType Quantity)
Construct a CharUnits quantity from a raw integer type.
std::string SampleProfileFile
Name of the profile file to use with -fprofile-sample-use.
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
CharUnits getAlignment() const
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Address withPointer(llvm::Value *NewPointer, KnownNonNull_t IsKnownNonNull) const
Return address with different pointer, but same element type and alignment.
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
llvm::PointerType * getType() const
Return the type of the pointer value.
static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateDefaultArtificial(CodeGenFunction &CGF, SourceLocation TemporaryLocation)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF)
Set the IRBuilder to not attach debug locations.
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
Address CreateGEP(CodeGenFunction &CGF, Address Addr, llvm::Value *Index, const llvm::Twine &Name="")
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Address CreateConstArrayGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = [n x T]* ... produce name = getelementptr inbounds addr, i64 0, i64 index where i64 is a...
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
Address CreateConstGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = T* ... produce name = getelementptr inbounds addr, i64 index where i64 is actually the t...
Address CreateAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
CGFunctionInfo - Class to encapsulate the information about a function definition.
DisableAutoDeclareTargetRAII(CodeGenModule &CGM)
~DisableAutoDeclareTargetRAII()
~LastprivateConditionalRAII()
static LastprivateConditionalRAII disable(CodeGenFunction &CGF, const OMPExecutableDirective &S)
NontemporalDeclsRAII(CodeGenModule &CGM, const OMPLoopDirective &S)
Struct that keeps all the relevant information that should be kept throughout a 'target data' region.
llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap
Map between the a declaration of a capture and the corresponding new llvm address where the runtime r...
~UntiedTaskLocalDeclsRAII()
UntiedTaskLocalDeclsRAII(CodeGenFunction &CGF, const llvm::MapVector< CanonicalDeclPtr< const VarDecl >, std::pair< Address, Address > > &LocalVars)
virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc)
Emits address of the word in a memory where current thread id is stored.
llvm::StringSet ThreadPrivateWithDefinition
Set of threadprivate variables with the generated initializer.
CGOpenMPRuntime(CodeGenModule &CGM)
void emitUpdateDependObjectsClause(CodeGenFunction &CGF, LValue DepobjLVal, OpenMPDependClauseKind NewDepKind, SourceLocation Loc)
Updates the dependency kind in the specified depobj object.
virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
Emit task region for the task directive.
void createOffloadEntriesAndInfoMetadata()
Creates all the offload entries in the current compilation unit along with the associated metadata.
const Expr * getNumTeamsExprForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &MinTeamsVal, int32_t &MaxTeamsVal)
Emit the number of teams for a target directive.
virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc)
Returns address of the threadprivate variable for the current thread.
void emitDeferredTargetDecls() const
Emit deferred declare target variables marked for deferred emission.
virtual llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST)
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
bool markAsGlobalTarget(GlobalDecl GD)
Marks the declaration as already emitted for the device code and returns true, if it was marked alrea...
virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond, llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, const Expr *Message=nullptr)
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
llvm::SmallDenseSet< CanonicalDeclPtr< const Decl > > NontemporalDeclsSet
virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device)
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
virtual void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc, OpenMPNumThreadsClauseModifier Modifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, SourceLocation SeverityLoc=SourceLocation(), const Expr *Message=nullptr, SourceLocation MessageLoc=SourceLocation())
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32global_tid, kmp_int32 num_threads) ...
QualType SavedKmpTaskloopTQTy
Saved kmp_task_t for taskloop-based directive.
virtual void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps)
Emits a single region.
virtual bool emitTargetGlobal(GlobalDecl GD)
Emit the global GD if it is meaningful for the target.
void setLocThreadIdInsertPt(CodeGenFunction &CGF, bool AtCurrentPoint=false)
std::string getOutlinedHelperName(StringRef Name) const
Get the function name of an outlined region.
StringRef getTargetRegionParentName(const FunctionDecl *FD) const
Returns the parent name of the target regions in FD: its enclosing non-closure function if FD is a cl...
bool HasEmittedDeclareTargetRegion
Flag for keeping track of weather a device routine has been emitted.
llvm::Constant * getOrCreateThreadPrivateCache(const VarDecl *VD)
If the specified mangled name is not in the module, create and return threadprivate cache object.
virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal)
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
virtual void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc)
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
void emitCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee Callee, ArrayRef< llvm::Value * > Args={}) const
Emits Callee function call with arguments Args with location Loc.
virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF, const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const
Choose default schedule type and chunk value for the schedule clause.
virtual std::pair< llvm::Function *, llvm::Function * > getUserDefinedReduction(const OMPDeclareReductionDecl *D)
Get combiner/initializer for the specified user-defined reduction, if any.
virtual bool isGPU() const
Returns true if the current target is a GPU.
static const Stmt * getSingleCompoundChild(ASTContext &Ctx, const Stmt *Body)
Checks if the Body is the CompoundStmt and returns its child statement iff there is only one that is ...
virtual void emitDeclareTargetFunction(const FunctionDecl *FD, llvm::GlobalValue *GV)
Emit code for handling declare target functions in the runtime.
bool HasRequiresUnifiedSharedMemory
Flag for keeping track of weather a requires unified_shared_memory directive is present.
llvm::Value * emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc, unsigned Flags=0, bool EmitLoc=false)
Emits object of ident_t type with info for source location.
bool isLocalVarInUntiedTask(CodeGenFunction &CGF, const VarDecl *VD) const
Returns true if the variable is a local variable in untied task.
virtual void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars)
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
virtual void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancellation point' construct.
virtual llvm::Function * emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF=nullptr)
Emit a code for initialization of threadprivate variable.
FunctionUDMMapTy FunctionUDMMap
virtual ConstantAddress getAddrOfDeclareTargetVar(const VarDecl *VD)
Returns the address of the variable marked as declare target with link clause OR as declare target wi...
llvm::Function * getOrCreateUserDefinedMapperFunc(const OMPDeclareMapperDecl *D)
Get the function for the specified user-defined mapper.
OpenMPLocThreadIDMapTy OpenMPLocThreadIDMap
virtual void functionFinished(CodeGenFunction &CGF)
Cleans up references to the objects in finished function.
virtual llvm::Function * emitTeamsOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP teams directive D.
QualType KmpTaskTQTy
Type typedef struct kmp_task { void * shareds; /**< pointer to block of pointers to shared vars / k...
llvm::OpenMPIRBuilder OMPBuilder
An OpenMP-IR-Builder instance.
virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr * > NumIterations)
Emit initialization for doacross loop nesting support.
virtual void adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF, const OMPExecutableDirective &D) const
Adjust some parameters for the target-based directives, like addresses of the variables captured by r...
FunctionUDRMapTy FunctionUDRMap
virtual void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, CGOpenMPRuntime::TargetDataInfo &Info)
Emit the target data mapping code associated with D.
virtual unsigned getDefaultLocationReserved2Flags() const
Returns additional flags that can be stored in reserved_2 field of the default location.
virtual Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const
Gets the address of the native argument basing on the address of the target-specific parameter.
void emitUsesAllocatorsFini(CodeGenFunction &CGF, const Expr *Allocator)
Destroys user defined allocators specified in the uses_allocators clause.
QualType KmpTaskAffinityInfoTy
Type typedef struct kmp_task_affinity_info { kmp_intptr_t base_addr; size_t len; struct { bool flag1 ...
void emitPrivateReduction(CodeGenFunction &CGF, SourceLocation Loc, const Expr *Privates, const Expr *LHSExprs, const Expr *RHSExprs, const Expr *ReductionOps)
Emits code for private variable reduction.
llvm::Value * emitNumTeamsForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D)
virtual void emitTargetOutlinedFunctionHelper(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Helper to emit outlined function for 'target' directive.
void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName)
Start scanning from statement S and emit all target regions found along the way.
SmallVector< llvm::Value *, 4 > emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy, const OMPTaskDataTy::DependData &Data)
virtual llvm::Value * emitMessageClause(CodeGenFunction &CGF, const Expr *Message, SourceLocation Loc)
virtual void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc)
Emit a taskgroup region.
llvm::DenseMap< llvm::Function *, llvm::DenseMap< CanonicalDeclPtr< const Decl >, std::tuple< QualType, const FieldDecl *, const FieldDecl *, LValue > > > LastprivateConditionalToTypes
Maps local variables marked as lastprivate conditional to their internal types.
virtual bool emitTargetGlobalVariable(GlobalDecl GD)
Emit the global variable if it is a valid device global variable.
virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32global_tid, kmp_int32 num_teams,...
bool hasRequiresUnifiedSharedMemory() const
Return whether the unified_shared_memory has been specified.
virtual Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, QualType VarType, StringRef Name)
Creates artificial threadprivate variable with name Name and type VarType.
void emitUserDefinedMapper(const OMPDeclareMapperDecl *D, CodeGenFunction *CGF=nullptr)
Emit the function for the user defined mapper construct.
bool HasEmittedTargetRegion
Flag for keeping track of weather a target region has been emitted.
void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy, LValue PosLVal, const OMPTaskDataTy::DependData &Data, Address DependenciesArray)
std::string getReductionFuncName(StringRef Name) const
Get the function name of a reduction function.
virtual void processRequiresDirective(const OMPRequiresDecl *D)
Perform check on requires decl to ensure that target architecture supports unified addressing.
llvm::DenseSet< CanonicalDeclPtr< const Decl > > AlreadyEmittedTargetDecls
List of the emitted declarations.
virtual llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data)
Emit a code for initialization of task reduction clause.
llvm::Value * getThreadID(CodeGenFunction &CGF, SourceLocation Loc)
Gets thread id value for the current thread.
virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD, SourceLocation Loc)
Gets the address of the global copy used for lastprivate conditional update, if any.
llvm::MapVector< CanonicalDeclPtr< const VarDecl >, std::pair< Address, Address > > UntiedLocalVarsAddressesMap
virtual void emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, Expr *ME, bool IsFatal)
Emit __kmpc_error call for error directive extern void __kmpc_error(ident_t *loc, int severity,...
void clearLocThreadIdInsertPt(CodeGenFunction &CGF)
virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc)
Emits code for a taskyield directive.
std::string getName(ArrayRef< StringRef > Parts) const
Get the platform-specific name separator.
QualType KmpRoutineEntryPtrQTy
void computeMinAndMaxThreadsAndTeams(const OMPExecutableDirective &D, CodeGenFunction &CGF, llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs)
Helper to determine the min/max number of threads/teams for D.
virtual void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc, llvm::AtomicOrdering AO)
Emit flush of the variables specified in 'omp flush' directive.
virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPTaskDataTy &Data)
Emit code for 'taskwait' directive.
virtual void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc)
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32global_tid, int proc_bind) to generate...
void emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LValue IVLVal, StringRef UniqueDeclName, LValue LVal, SourceLocation Loc)
Emit update for lastprivate conditional data.
virtual void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
Emit task region for the taskloop directive.
virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false)
Emit an implicit/explicit barrier for OpenMP threads.
static unsigned getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind)
Returns default flags for the barriers depending on the directive, for which this barier is going to ...
virtual bool emitTargetFunctions(GlobalDecl GD)
Emit the target regions enclosed in GD function definition or the function itself in case it is a val...
TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const OMPTaskDataTy &Data)
Emit task region for the task directive.
llvm::Value * emitTargetNumIterationsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Return the trip count of loops associated with constructs / 'target teams distribute' and 'teams dist...
llvm::StringMap< llvm::AssertingVH< llvm::GlobalVariable >, llvm::BumpPtrAllocator > InternalVars
An ordered map of auto-generated variables to their unique names.
virtual void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values)
llvm::SmallVector< UntiedLocalVarsAddressesMap, 4 > UntiedLocalVarsStack
virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind)
Call the appropriate runtime routine to notify that we finished all the work with current loop.
virtual void emitThreadLimitClause(CodeGenFunction &CGF, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_set_thread_limit(ident_t *loc, kmp_int32global_tid, kmp_int32 thread_limit)...
void emitIfClause(CodeGenFunction &CGF, const Expr *Cond, const RegionCodeGenTy &ThenGen, const RegionCodeGenTy &ElseGen)
Emits code for OpenMP 'if' clause using specified CodeGen function.
Address emitDepobjDependClause(CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies, SourceLocation Loc)
Emits list of dependecies based on the provided data (array of dependence/expression pairs) for depob...
bool isNontemporalDecl(const ValueDecl *VD) const
Checks if the VD variable is marked as nontemporal declaration in current context.
virtual llvm::Function * emitParallelOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP parallel directive D.
const Expr * getNumThreadsExprForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &UpperBound, bool UpperBoundOnly, llvm::Value **CondExpr=nullptr, const Expr **ThreadLimitExpr=nullptr)
Check for a number of threads upper bound constant value (stored in UpperBound), or expression (retur...
virtual void registerVTableOffloadEntry(llvm::GlobalVariable *VTable, const VarDecl *VD)
Register VTable to OpenMP offload entry.
virtual llvm::Value * emitSeverityClause(OpenMPSeverityClauseKind Severity, SourceLocation Loc)
llvm::SmallVector< LastprivateConditionalData, 4 > LastprivateConditionalStack
Stack for list of addresses of declarations in current context marked as lastprivate conditional.
virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values)
Call the appropriate runtime routine to initialize it before start of loop.
virtual void emitDeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn)
Marks function Fn with properly mangled versions of vector functions.
llvm::AtomicOrdering getDefaultMemoryOrdering() const
Gets default memory ordering as specified in requires directive.
virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static non-chunked.
virtual void emitAndRegisterVTable(CodeGenModule &CGM, CXXRecordDecl *CXXRecord, const VarDecl *VD)
Emit and register VTable for the C++ class in OpenMP offload entry.
void scanConstructorForTargetRegions(const CXXConstructorDecl *Ctor, StringRef ParentName)
Scans the mem-initializers and the body of Ctor for target regions.
llvm::Value * getCriticalRegionLock(StringRef CriticalName)
Returns corresponding lock object for the specified critical region name.
virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancel' construct.
QualType SavedKmpTaskTQTy
Saved kmp_task_t for task directive.
virtual void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc)
Emits a master region.
virtual llvm::Function * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts)
Emits outlined function for the OpenMP task directive D.
llvm::DenseMap< llvm::Function *, unsigned > FunctionToUntiedTaskStackMap
Maps function to the position of the untied task locals stack.
void emitHostKernelEnvironment(const OMPExecutableDirective &D, CodeGenFunction &CGF)
Emit the '<kernel>_kernel_environment' global for a target region that is compiled for a non-GPU (hos...
void emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal, SourceLocation Loc)
Emits the code to destroy the dependency object provided in depobj directive.
virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Required to resolve existing problems in the runtime.
llvm::ArrayType * KmpCriticalNameTy
Type kmp_critical_name, originally defined as typedef kmp_int32 kmp_critical_name[8];.
virtual void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C)
Emit code for doacross ordered directive with 'depend' clause.
llvm::DenseMap< const OMPDeclareMapperDecl *, llvm::Function * > UDMMap
Map from the user-defined mapper declaration to its corresponding functions.
virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF, const Expr *LHS)
Checks if the provided LVal is lastprivate conditional and emits the code to update the value of the ...
std::pair< llvm::Value *, LValue > getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal, SourceLocation Loc)
Returns the number of the elements and the address of the depobj dependency array.
llvm::SmallDenseSet< const VarDecl * > DeferredGlobalVariables
List of variables that can become declare target implicitly and, thus, must be emitted.
void emitUsesAllocatorsInit(CodeGenFunction &CGF, const Expr *Allocator, const Expr *AllocatorTraits)
Initializes user defined allocators specified in the uses_allocators clauses.
virtual void registerVTable(const OMPExecutableDirective &D)
Emit code for registering vtable by scanning through map clause in OpenMP target region.
llvm::Type * KmpRoutineEntryPtrTy
Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *);.
llvm::Type * getIdentTyPointerTy()
Returns pointer to ident_t type.
void emitSingleReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp, const Expr *PrivateRef, const DeclRefExpr *LHS, const DeclRefExpr *RHS)
Emits single reduction combiner.
llvm::OpenMPIRBuilder & getOMPBuilder()
virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Emit outilined function for 'target' directive.
virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr)
Emits a critical region.
virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned)
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
virtual void emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, ArrayRef< llvm::Value * > Args={}) const
Emits call of the outlined function with the provided arguments, translating these arguments to corre...
llvm::Value * emitNumThreadsForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D)
Emit an expression that denotes the number of threads a target region shall use.
void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor, llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc)
Emits initialization code for the threadprivate variables.
virtual void emitUserDefinedReduction(CodeGenFunction *CGF, const OMPDeclareReductionDecl *D)
Emit code for the specified user defined reduction construct.
virtual void checkAndEmitSharedLastprivateConditional(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::DenseSet< CanonicalDeclPtr< const VarDecl > > &IgnoredDecls)
Checks if the lastprivate conditional was updated in inner region and writes the value.
QualType KmpDimTy
struct kmp_dim { // loop bounds info casted to kmp_int64 kmp_int64 lo; // lower kmp_int64 up; // uppe...
llvm::SmallPtrSet< const CXXConstructorDecl *, 8 > ScannedImplicitCtors
Implicitly-defined constructors already scanned for target regions; only lambdas can hold such region...
virtual void emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool HasCancel=false)
Emit code for the directive that does not require outlining.
virtual void registerTargetGlobalVariable(const VarDecl *VD, llvm::Constant *Addr)
Checks if the provided global decl GD is a declare target variable and registers it when emitting cod...
virtual void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D)
Emits OpenMP-specific function prolog.
void emitKmpRoutineEntryT(QualType KmpInt32Ty)
Build type kmp_routine_entry_t (if not built yet).
virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static chunked.
virtual void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Emit the target offloading code associated with D.
virtual bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS)
Checks if the variable has associated OMPAllocateDeclAttr attribute with the predefined allocator and...
llvm::AtomicOrdering RequiresAtomicOrdering
Atomic ordering from the omp requires directive.
virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps, ReductionOptionsTy Options)
Emit a code for reduction clause.
std::pair< llvm::Value *, Address > emitDependClause(CodeGenFunction &CGF, ArrayRef< OMPTaskDataTy::DependData > Dependencies, SourceLocation Loc)
Emits list of dependecies based on the provided data (array of dependence/expression pairs).
llvm::StringMap< llvm::WeakTrackingVH > EmittedNonTargetVariables
List of the global variables with their addresses that should not be emitted for the target.
virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const
Check if the specified ScheduleKind is dynamic.
Address emitLastprivateConditionalInit(CodeGenFunction &CGF, const VarDecl *VD)
Create specialized alloca to handle lastprivate conditionals.
virtual void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads)
Emit an ordered region.
virtual Address getAddressOfLocalVariable(CodeGenFunction &CGF, const VarDecl *VD)
Gets the OpenMP-specific address of the local variable.
virtual void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, bool IsWorksharingReduction)
Emits the following code for reduction clause with task modifier:
virtual void emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter=nullptr)
Emits a masked region.
QualType KmpDependInfoTy
Type typedef struct kmp_depend_info { kmp_intptr_t base_addr; size_t len; struct { bool in:1; bool ou...
llvm::Function * emitReductionFunction(StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps)
Emits reduction function.
virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues)
Call the appropriate runtime routine to initialize it before start of loop.
Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal) override
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr) override
Emits a critical region.
void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values) override
void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) override
Call the appropriate runtime routine to initialize it before start of loop.
bool emitTargetGlobalVariable(GlobalDecl GD) override
Emit the global variable if it is a valid device global variable.
llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST) override
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
llvm::Function * emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF=nullptr) override
Emit a code for initialization of threadprivate variable.
void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device) override
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
llvm::Function * emitTeamsOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) override
Emits outlined function for the specified OpenMP teams directive D.
void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond, llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, const Expr *Message=nullptr) override
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps, ReductionOptionsTy Options) override
Emit a code for reduction clause.
void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc, llvm::AtomicOrdering AO) override
Emit flush of the variables specified in 'omp flush' directive.
void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C) override
Emit code for doacross ordered directive with 'depend' clause.
void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc) override
Emits a masked region.
Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, QualType VarType, StringRef Name) override
Creates artificial threadprivate variable with name Name and type VarType.
Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc) override
Returns address of the threadprivate variable for the current thread.
void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps) override
Emits a single region.
void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N) override
Required to resolve existing problems in the runtime.
llvm::Function * emitParallelOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) override
Emits outlined function for the specified OpenMP parallel directive D.
void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion) override
Emit code for 'cancellation point' construct.
void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false) override
Emit an implicit/explicit barrier for OpenMP threads.
Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const override
Gets the address of the native argument basing on the address of the target-specific parameter.
void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars) override
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned) override
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
bool emitTargetGlobal(GlobalDecl GD) override
Emit the global GD if it is meaningful for the target.
void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, bool IsWorksharingReduction) override
Emits the following code for reduction clause with task modifier:
void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads) override
Emit an ordered region.
void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind) override
Call the appropriate runtime routine to notify that we finished all the work with current loop.
llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data) override
Emit a code for initialization of task reduction clause.
void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc) override
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32global_tid, int proc_bind) to generate...
void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) override
Emit outilined function for 'target' directive.
void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc) override
Emits a master region.
void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc) override
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32global_tid, kmp_int32 num_teams,...
void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc) override
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
const VarDecl * translateParameter(const FieldDecl *FD, const VarDecl *NativeParam) const override
Translates the native parameter of outlined function if this is required for target.
void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc, OpenMPNumThreadsClauseModifier Modifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, SourceLocation SeverityLoc=SourceLocation(), const Expr *Message=nullptr, SourceLocation MessageLoc=SourceLocation()) override
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32global_tid, kmp_int32 num_threads) ...
void emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter=nullptr) override
Emits a masked region.
void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data) override
Emit task region for the task directive.
void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter) override
Emit the target offloading code associated with D.
bool emitTargetFunctions(GlobalDecl GD) override
Emit the target regions enclosed in GD function definition or the function itself in case it is a val...
void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr * > NumIterations) override
Emit initialization for doacross loop nesting support.
void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion) override
Emit code for 'cancel' construct.
void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPTaskDataTy &Data) override
Emit code for 'taskwait' directive.
void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc) override
Emit a taskgroup region.
void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, CGOpenMPRuntime::TargetDataInfo &Info) override
Emit the target data mapping code associated with D.
void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues) override
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
llvm::Function * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts) override
Emits outlined function for the OpenMP task directive D.
void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data) override
Emit task region for the taskloop directive.
unsigned getNonVirtualBaseLLVMFieldNo(const CXXRecordDecl *RD) const
llvm::StructType * getLLVMType() const
Return the "complete object" LLVM type associated with this record.
llvm::StructType * getBaseSubobjectLLVMType() const
Return the "base subobject" LLVM type associated with this record.
unsigned getLLVMFieldNo(const FieldDecl *FD) const
Return llvm::StructType element number that corresponds to the field FD.
unsigned getVirtualBaseIndex(const CXXRecordDecl *base) const
Return the LLVM field index corresponding to the given virtual base.
API for captured statement code generation.
virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S)
Emit the captured statement body.
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
RAII for correct setting/restoring of CapturedStmtInfo.
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
bool Privatize()
Privatizes local variables previously registered as private.
bool addPrivate(const ValueDecl *LocalVD, Address Addr)
Registers LocalVD variable as a private with Addr as the address of the corresponding private variabl...
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited.
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
LValue EmitLoadOfReferenceLValue(LValue RefLVal)
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount, Stmt::Likelihood LH=Stmt::LH_None, const Expr *ConditionalOp=nullptr, const VarDecl *ConditionalDecl=nullptr)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
void emitDestroy(Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray)
emitDestroy - Immediately perform the destruction of the given object.
JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target)
The given basic block lies in the current EH scope, but may be a target of a potentially scope-crossi...
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
void EmitNullInitialization(Address DestPtr, QualType Ty)
EmitNullInitialization - Generate code to set a value of the given type to null, If the type contains...
Address LoadCXXThisAddress()
CGCapturedStmtInfo * CapturedStmtInfo
ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc)
EmitLoadOfComplex - Load a complex number from the specified l-value.
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
void eraseLocalDecl(const Decl *D)
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
llvm::Function * GenerateOpenMPCapturedStmtFunctionAggregate(const CapturedStmt &S, const OMPExecutableDirective &D)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
const LangOptions & getLangOpts() const
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
EmitAutoVarAlloca - Emit the alloca and debug information for a local variable.
void pushDestroy(QualType::DestructionKind dtorKind, Address addr, QualType type)
pushDestroy - Push the standard destructor for the given type as at least a normal cleanup.
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPLoopDirective &S)
Emit device code for the target simd and target loop directives.
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Load a pointer with type PtrTy stored at address Ptr.
DeclMapTy::iterator localDeclMapEnd()
void EmitBranchThroughCleanup(JumpDest Dest)
EmitBranchThroughCleanup - Emit a branch from the current insert block through the normal cleanup han...
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
Destroyer * getDestroyer(QualType::DestructionKind destructionKind)
llvm::AssertingVH< llvm::Instruction > AllocaInsertPt
AllocaInsertPoint - This is an instruction in the entry block before which we prefer to insert alloca...
void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy)
Emit an aggregate assignment.
JumpDest ReturnBlock
ReturnBlock - Unified return block.
void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy, AggValueSlot::Overlap_t MayOverlap, bool isVolatile=false)
EmitAggregateCopy - Emit an aggregate copy.
LValue EmitLValueForField(LValue Base, const FieldDecl *Field, bool IsInBounds=true)
RawAddress CreateDefaultAlignTempAlloca(llvm::Type *Ty, const Twine &Name="tmp")
CreateDefaultAlignedTempAlloca - This creates an alloca with the default ABI alignment of the given L...
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value * > &CapturedVars)
DeclMapTy::iterator findLocalDecl(const Decl *D)
Accessors for LocalDeclMap.
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
std::pair< DeclMapTy::iterator, bool > insertLocalDecl(const Decl *D, Address Addr)
LValue EmitArraySectionExpr(const ArraySectionExpr *E, bool IsLowerBound=true)
LValue EmitOMPSharedLValue(const Expr *E)
Emits the lvalue for the expression with possibly captured variable.
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
llvm::Value * emitArrayLength(const ArrayType *arrayType, QualType &baseType, Address &addr)
emitArrayLength - Compute the length of an array, even if it's a VLA, and drill down to the base elem...
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
CGDebugInfo * getDebugInfo()
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
LValue MakeRawAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment, AlignmentSource Source=AlignmentSource::Type)
Same as MakeAddrLValue above except that the pointer is known to be unsigned.
LValue EmitLValueForFieldInitialization(LValue Base, const FieldDecl *Field)
EmitLValueForFieldInitialization - Like EmitLValueForField, except that if the Field is a reference,...
RawAddress CreateMemTempWithoutCast(QualType T, const Twine &Name="tmp")
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen without...
VlaSizePair getVLASize(const VariableArrayType *vla)
Returns an LLVM value that corresponds to the size, in non-variably-sized elements,...
llvm::CallInst * EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
ASTContext & getContext() const
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
void EmitAutoVarCleanups(const AutoVarEmission &emission)
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy)
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
bool needsEHCleanup(QualType::DestructionKind kind)
Determines whether an EH cleanup is required to destroy a type with the given destruction kind.
llvm::DenseMap< const ValueDecl *, FieldDecl * > LambdaCaptureFields
llvm::CallInst * EmitRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
llvm::Type * ConvertTypeForMem(QualType T)
static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S)
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
CodeGenTypes & getTypes() const
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Emit device code for the target teams distribute parallel for simd directive.
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block,...
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S, const OMPExecutableDirective &D)
RawAddress CreateMemTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
llvm::Value * EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr, ArrayRef< llvm::Value * > IdxList, bool SignedIndices, bool IsSubtraction, SourceLocation Loc, const Twine &Name="")
Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to detect undefined behavior whe...
static void EmitOMPTargetParallelGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelGenericLoopDirective &S)
Emit device code for the target parallel loop directive.
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
static bool IsWrappedCXXThis(const Expr *E)
Check if E is a C++ "this" pointer wrapped in value-preserving casts.
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant,...
static void EmitOMPTargetTeamsGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsGenericLoopDirective &S)
Emit device code for the target teams loop directive.
LValue EmitMemberExpr(const MemberExpr *E)
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler's counter for the given statement by StepV.
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
void EmitVariablyModifiedType(QualType Ty)
EmitVLASize - Capture all the sizes for the VLA expressions in the given variably-modified type and s...
bool isTrivialInitializer(const Expr *Init)
Determine whether the given initializer is trivial in the sense that it requires no code to be genera...
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit)
EmitExprAsInit - Emits the code necessary to initialize a location in memory with the given initializ...
LValue MakeNaturalAlignRawAddrLValue(llvm::Value *V, QualType T)
This class organizes the cross-function state that is used while generating LLVM code.
void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
llvm::Module & getModule() const
const IntrusiveRefCntPtr< llvm::vfs::FileSystem > & getFileSystem() const
DiagnosticsEngine & getDiags() const
const LangOptions & getLangOpts() const
CharUnits getNaturalTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, bool forPointeeType=false)
CodeGenTypes & getTypes()
const llvm::DataLayout & getDataLayout() const
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
TBAAAccessInfo getTBAAInfoForSubobject(LValue Base, QualType AccessType)
getTBAAInfoForSubobject - Get TBAA information for an access with a given base lvalue.
ASTContext & getContext() const
const CodeGenOptions & getCodeGenOpts() const
StringRef getMangledName(GlobalDecl GD)
std::optional< CharUnits > getOMPAllocateAlignment(const VarDecl *VD)
Return the alignment specified in an allocate directive, if present.
llvm::Constant * EmitNullConstant(QualType T)
Return the result of value-initializing the given type, i.e.
llvm::Type * ConvertType(QualType T)
ConvertType - Convert type T into a llvm::Type.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
const CGRecordLayout & getCGRecordLayout(const RecordDecl *)
getCGRecordLayout - Return record layout info for the given record decl.
llvm::GlobalVariable * GetAddrOfVTable(const CXXRecordDecl *RD)
GetAddrOfVTable - Get the address of the VTable for the given record decl.
A specialization of Address that requires the address to be an LLVM Constant.
static ConstantAddress invalid()
bool requiresLandingPad() const
void pushTerminate()
Push a terminate handler on the stack.
void popTerminate()
Pops a terminate handler off the stack.
FunctionArgList - Type for representing both the decl and type of parameters to a function.
LValue - This represents an lvalue references.
CharUnits getAlignment() const
llvm::Value * getPointer(CodeGenFunction &CGF) const
const Qualifiers & getQuals() const
Address getAddress() const
LValueBaseInfo getBaseInfo() const
TBAAAccessInfo getTBAAInfo() const
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
virtual void Enter(CodeGenFunction &CGF)
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
static RValue get(llvm::Value *V)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
An abstract representation of an aligned address.
llvm::Type * getElementType() const
Return the type of the values stored in this address.
llvm::Value * getPointer() const
static RawAddress invalid()
Class intended to support codegen of all kind of the reduction clauses.
LValue getSharedLValue(unsigned N) const
Returns LValue for the reduction item.
const Expr * getRefExpr(unsigned N) const
Returns the base declaration of the reduction item.
LValue getOrigLValue(unsigned N) const
Returns LValue for the original reduction item.
bool needCleanups(unsigned N)
Returns true if the private copy requires cleanups.
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
const VarDecl * getBaseDecl(unsigned N) const
Returns the base declaration of the reduction item.
QualType getPrivateType(unsigned N) const
Return the type of the private item.
bool usesReductionInitializer(unsigned N) const
Returns true if the initialization of the reduction item uses initializer from declare reduction cons...
void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N)
Emits lvalue for the shared and original reduction item.
void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, llvm::function_ref< bool(CodeGenFunction &)> DefaultInit)
Performs initialization of the private copy for the reduction item.
std::pair< llvm::Value *, llvm::Value * > getSizes(unsigned N) const
Returns the size of the reduction item (in chars and total number of elements in the item),...
ReductionCodeGen(ArrayRef< const Expr * > Shareds, ArrayRef< const Expr * > Origs, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > ReductionOps)
void emitCleanups(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Emits cleanup code for the reduction item.
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations.
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
void operator()(CodeGenFunction &CGF) const
void setAction(PrePostActionTy &Action) const
ConstStmtVisitor - This class implements a simple visitor for Stmt subclasses.
DeclContext - This is used only as base class of specific decl types that can act as declaration cont...
void addDecl(Decl *D)
Add the declaration D into this context.
A reference to a declared variable, function, enum, etc.
Decl - This represents one declaration (or definition), e.g.
ASTContext & getASTContext() const LLVM_READONLY
virtual Stmt * getBody() const
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
bool isTemplated() const
Determine whether this declaration is a templated entity (whether it is.
Decl * getNonClosureContext()
Find the innermost non-closure ancestor of this declaration, walking up through blocks,...
llvm::iterator_range< specific_attr_iterator< T > > specific_attrs() const
SourceLocation getLocation() const
DeclContext * getDeclContext()
virtual Decl * getCanonicalDecl()
Retrieves the "canonical" declaration of the given declaration.
SourceLocation getBeginLoc() const LLVM_READONLY
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
This represents one expression.
bool isIntegerConstantExpr(const ASTContext &Ctx) const
Expr * IgnoreParenNoopCasts(const ASTContext &Ctx) LLVM_READONLY
Skip past any parentheses and casts which do not change the value (including ptr->int casts of the sa...
@ SE_AllowSideEffects
Allow any unmodeled side effect.
@ SE_AllowUndefinedBehavior
Allow UB that we can give a value, but not arbitrary unmodeled side effects.
Expr * IgnoreParenCasts() LLVM_READONLY
Skip past any parentheses and casts which might surround this expression until reaching a fixed point...
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
bool isEvaluatable(const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects) const
isEvaluatable - Call EvaluateAsRValue to see if this expression can be constant folded without side-e...
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
std::optional< llvm::APSInt > getIntegerConstantExpr(const ASTContext &Ctx, bool AllowRelaxedEval=false) const
isIntegerConstantExpr - Return the value if this expression is a valid integer constant expression.
bool EvaluateAsBooleanCondition(bool &Result, const ASTContext &Ctx, bool InConstantContext=false) const
EvaluateAsBooleanCondition - Return true if this is a constant which we can fold and convert to a boo...
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
static bool isSameComparisonOperand(const Expr *E1, const Expr *E2)
Checks that the two Expr's will refer to the same value as a comparison operand.
bool hasNonTrivialCall(const ASTContext &Ctx) const
Determine whether this expression involves a call to any function that is not trivial.
Represents a member of a struct/union/class.
unsigned getFieldIndex() const
Returns the index of this field within its record, as appropriate for passing to ASTRecordLayout::get...
const RecordDecl * getParent() const
Returns the parent of this field declaration, which is the struct in which this field is defined.
static FieldDecl * Create(const ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, const IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, Expr *BW, bool Mutable, InClassInitStyle InitStyle)
Represents a function declaration or definition.
const ParmVarDecl * getParamDecl(unsigned i) const
QualType getReturnType() const
ArrayRef< ParmVarDecl * > parameters() const
FunctionDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
FunctionDecl * getMostRecentDecl()
Returns the most recent (re)declaration of this declaration.
FunctionDecl * getDefinition()
Get the definition for this declaration.
bool isUserProvided() const
True if this method is user-declared and was not deleted or defaulted on its first declaration.
unsigned getNumParams() const
Return the number of parameters this function must have based on its FunctionType.
bool hasBody(const FunctionDecl *&Definition) const
Returns true if the function has a body.
FunctionDecl * getPreviousDecl()
Return the previous declaration of this declaration or NULL if this is the first declaration.
GlobalDecl - represents a global declaration.
const Decl * getDecl() const
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, const IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
static IntegerLiteral * Create(const ASTContext &C, const llvm::APInt &V, QualType type, SourceLocation l)
Returns a new integer literal with value 'V' and type 'type'.
An lvalue reference type, per C++11 [dcl.ref].
MemberExpr - [C99 6.5.2.3] Structure and Union Members.
ValueDecl * getMemberDecl() const
Retrieve the member declaration to which this expression refers.
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
bool isExternallyVisible() const
const Stmt * getPreInitStmt() const
Get pre-initialization statement for the clause.
This is a basic class for representing single OpenMP clause.
ArrayRef< OMPClause * > clauses() const
This represents 'pragma omp declare mapper ...' directive.
Expr * getMapperVarRef()
Get the variable declared in the mapper.
This represents 'pragma omp declare reduction ...' directive.
Expr * getInitializer()
Get initializer expression (if specified) of the declare reduction construct.
Expr * getInitPriv()
Get Priv variable of the initializer.
Expr * getCombinerOut()
Get Out variable of the combiner.
Expr * getCombinerIn()
Get In variable of the combiner.
Expr * getCombiner()
Get combiner expression of the declare reduction construct.
Expr * getInitOrig()
Get Orig variable of the initializer.
OMPDeclareReductionInitKind getInitializerKind() const
Get initializer kind.
This represents 'if' clause in the 'pragma omp ...' directive.
Expr * getCondition() const
Returns condition.
OMPIteratorHelperData & getHelper(unsigned I)
Fetches helper data for the specified iteration space.
unsigned numOfIterators() const
Returns number of iterator definitions.
This represents 'num_threads' clause in the 'pragma omp ...' directive.
This represents 'pragma omp requires...' directive.
clauselist_range clauselists()
This represents 'threadset' clause in the 'pragma omp task ...' directive.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class.
Represents a parameter to a function.
PointerType - C99 6.7.5.1 - Pointer Declarators.
Represents an unpacked "presumed" location which can be presented to the user.
unsigned getColumn() const
Return the presumed column number of this location.
const char * getFilename() const
Return the presumed filename of this location.
unsigned getLine() const
Return the presumed line number of this location.
A (possibly-)qualified type.
void addRestrict()
Add the restrict qualifier to this QualType.
QualType withRestrict() const
bool isNull() const
Return true if this QualType doesn't point to a type yet.
const Type * getTypePtr() const
Retrieves a pointer to the underlying (unqualified) type.
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
QualType getCanonicalType() const
DestructionKind isDestructedType() const
Returns a nonzero value if objects of this type require non-trivial work to clean up after.
Represents a struct/union/class.
field_iterator field_end() const
field_range fields() const
virtual void completeDefinition()
Note that the definition of this type is now complete.
field_iterator field_begin() const
Scope - A scope is a transient data structure that is used while parsing the program.
Encodes a location in the source.
static SourceLocation getFromRawEncoding(UIntTy Encoding)
Turn a raw encoding of a SourceLocation object into a real SourceLocation.
bool isValid() const
Return true if this is a valid SourceLocation object.
UIntTy getRawEncoding() const
When a SourceLocation itself cannot be used, this returns an (opaque) 32-bit integer encoding for it.
This class handles loading and caching of source files into memory.
PresumedLoc getPresumedLoc(SourceLocation Loc, bool UseLineDirectives=true) const
Returns the "presumed" location of a SourceLocation specifies.
Stmt - This represents one statement.
StmtClass getStmtClass() const
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top,...
SourceLocation getBeginLoc() const LLVM_READONLY
void startDefinition()
Starts the definition of this tag declaration.
The base class of the type hierarchy.
const Type * getPointeeOrArrayElementType() const
If this is a pointer type, return the pointee type.
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
RecordDecl * getAsRecordDecl() const
Retrieves the RecordDecl this type refers to.
bool isPointerType() const
CanQualType getCanonicalTypeUnqualified() const
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
const T * castAs() const
Member-template castAs<specific type>.
bool isReferenceType() const
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
bool isLValueReferenceType() const
bool isAggregateType() const
Determines whether the type is a C++ aggregate type or C aggregate or union type.
RecordDecl * castAsRecordDecl() const
QualType getCanonicalTypeInternal() const
const Type * getBaseElementTypeUnsafe() const
Get the base element type of this type, potentially discarding type qualifiers.
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type.
bool isFloatingType() const
bool isUnsignedIntegerType() const
Return true if this is an integer type that is unsigned, according to C99 6.2.5p6 [which returns true...
bool isAnyPointerType() const
const T * getAs() const
Member-template getAs<specific type>'.
bool isRecordType() const
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Represents a variable declaration or definition.
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
VarDecl * getDefinition(ASTContext &)
Get the real (not just tentative) definition for this declaration.
const Expr * getInit() const
bool hasExternalStorage() const
Returns true if a variable has extern or private_extern storage.
@ DeclarationOnly
This declaration is only a declaration.
DefinitionKind hasDefinition(ASTContext &) const
Check whether this variable is defined in this translation unit.
bool isLocalVarDeclOrParm() const
Similar to isLocalVarDecl but also includes parameters.
const Expr * getAnyInitializer() const
Get the initializer for this variable, no matter which declaration it is attached to.
Represents a C array with a specified size that is not an integer-constant-expression.
Expr * getSizeExpr() const
specific_attr_iterator - Iterates over a subrange of an AttrVec, only providing attributes that are o...
const clang::Expr * ExprTy
@ LLVM_MARK_AS_BITMASK_ENUM
bool isEmptyFieldForLayout(const ASTContext &Ctx, const FieldDecl *FD)
Return true iff the field is "empty", that is, either a zero-width bit-field or an isEmptyRecordForLa...
bool isEmptyRecordForLayout(const ASTContext &Ctx, QualType T)
Return true iff a structure contains only empty base classes (per isEmptyRecordForLayout) and fields ...
@ Type
The l-value was considered opaque, so the alignment was determined from a type.
@ Decl
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
ComparisonResult
Indicates the result of a tentative comparison.
@ Address
A pointer to a ValueDecl.
Top level wrappers for InstallAPI frontend operations.
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
CanQual< Type > CanQualType
Represents a canonical, potentially-qualified type.
bool needsTaskBasedThreadLimit(OpenMPDirectiveKind DKind)
Checks if the specified target directive, combined or not, needs task based thread_limit.
@ Match
This is not an overload because the signature exactly matches an existing declaration.
@ Ctor_Complete
Complete object ctor.
Privates[]
This class represents the 'transparent' clause in the 'pragma omp task' directive.
bool isa(CodeGen::Address addr)
if(T->getSizeExpr()) TRY_TO(TraverseStmt(const_cast< Expr * >(T -> getSizeExpr())))
bool isOpenMPTargetDataManagementDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target data offload directive.
static bool classof(const OMPClause *T)
@ Conditional
A conditional (?:) operator.
@ ICIS_NoInit
No in-class initializer.
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
@ LCK_ByRef
Capturing by reference.
LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE()
@ Private
'private' clause, allowed on 'parallel', 'serial', 'loop', 'parallel loop', and 'serial loop' constru...
@ Reduction
'reduction' clause, allowed on Parallel, Serial, Loop, and the combined constructs.
@ Present
'present' clause, allowed on Compute and Combined constructs, plus 'data' and 'declare'.
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
@ OMPC_SCHEDULE_MODIFIER_last
@ OMPC_SCHEDULE_MODIFIER_unknown
nullptr
This class represents a compute construct, representing a 'Kind' of ‘parallel’, 'serial',...
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
OpenMPDistScheduleClauseKind
OpenMP attributes for 'dist_schedule' clause.
bool isOpenMPTaskingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of tasking directives - task or any taskloop-based dire...
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
@ OMPC_DYN_GROUPPRIVATE_FALLBACK_unknown
@ Result
The result type of a method or function.
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
const FunctionProtoType * T
OpenMPDependClauseKind
OpenMP attributes for 'depend' clause.
@ Dtor_Complete
Complete object dtor.
@ Union
The "union" keyword.
bool isOpenMPTargetMapEnteringDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a map-entering target directive.
bool isOpenMPLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a directive with an associated loop construct.
OpenMPSeverityClauseKind
OpenMP attributes for 'severity' clause.
LangAS
Defines the address space values used by the address space qualifier of QualType.
llvm::omp::Directive OpenMPDirectiveKind
OpenMP directives.
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
@ VK_PRValue
A pr-value expression (in the C++11 taxonomy) produces a temporary value.
@ VK_LValue
An l-value expression is a reference to an object with independent storage.
for(const auto &A :T->param_types())
void getOpenMPCaptureRegions(llvm::SmallVectorImpl< OpenMPDirectiveKind > &CaptureRegions, OpenMPDirectiveKind DKind)
Return the captured regions of an OpenMP directive.
OpenMPNumThreadsClauseModifier
@ OMPC_ATOMIC_DEFAULT_MEM_ORDER_unknown
U cast(CodeGen::Address addr)
OpenMPMapModifierKind
OpenMP modifier kind for 'map' clause.
@ OMPC_MAP_MODIFIER_unknown
@ Other
Other implicit parameter.
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
OpenMPThreadsetKind
OpenMP modifiers for 'threadset' clause.
OpenMPMapClauseKind
OpenMP mapping kind for 'map' clause.
Diagnostic wrappers for TextAPI types for error reporting.
__builtin_bitreverse32 uint32_t __builtin_riscv_aaddu_u32 __builtin_riscv_asubu_u32 int32_t
__builtin_riscv_nclipru_u32 int64_t
Data for list of allocators.
Expr * AllocatorTraits
Allocator traits.
Expr * Allocator
Allocator.
Maps the expression for the lastprivate variable to the global copy used to store new value because o...
llvm::SmallVector< bool, 8 > IsPrivateVarReduction
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
llvm::BasicBlock * getBlock() const
unsigned NumberOfTargetItems
Address BasePointersArray
llvm::PointerType * VoidPtrTy
llvm::IntegerType * Int64Ty
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::CallingConv::ID getRuntimeCC() const
llvm::IntegerType * SizeTy
llvm::PointerType * VoidPtrPtrTy
llvm::IntegerType * Int32Ty
llvm::IntegerType * IntPtrTy
llvm::IntegerType * IntTy
int
CharUnits getPointerAlign() const
OpenMPDependClauseKind DepKind
const Expr * IteratorExpr
SmallVector< const Expr *, 4 > DepExprs
EvalResult is a struct with detailed info about an evaluated expression.
Extra information about a function prototype.
Expr * CounterUpdate
Updater for the internal counter: ++CounterVD;.
Scheduling data for loop-based OpenMP directives.
bool UseFusedDistChunkSchedule
Request the fused distr_static_chunk + static_chunkone runtime schedule in for_static_init.
OpenMPScheduleClauseModifier M2
OpenMPScheduleClauseModifier M1
OpenMPScheduleClauseKind Schedule
Describes how types, statements, expressions, and declarations should be printed.