30#include "llvm/ADT/ArrayRef.h"
31#include "llvm/ADT/SmallVector.h"
32#include "llvm/ADT/StringExtras.h"
33#include "llvm/Bitcode/BitcodeReader.h"
34#include "llvm/IR/Constants.h"
35#include "llvm/IR/DerivedTypes.h"
36#include "llvm/IR/GlobalValue.h"
37#include "llvm/IR/InstrTypes.h"
38#include "llvm/IR/Value.h"
39#include "llvm/Support/AtomicOrdering.h"
40#include "llvm/Support/raw_ostream.h"
48using namespace llvm::omp;
55 enum CGOpenMPRegionKind {
58 ParallelOutlinedRegion,
68 CGOpenMPRegionInfo(
const CapturedStmt &CS,
69 const CGOpenMPRegionKind RegionKind,
72 : CGCapturedStmtInfo(CS,
CR_OpenMP), RegionKind(RegionKind),
73 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
75 CGOpenMPRegionInfo(
const CGOpenMPRegionKind RegionKind,
78 : CGCapturedStmtInfo(
CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
79 Kind(Kind), HasCancel(HasCancel) {}
83 virtual const VarDecl *getThreadIDVariable()
const = 0;
86 void EmitBody(CodeGenFunction &CGF,
const Stmt *S)
override;
90 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
92 virtual void emitUntiedSwitch(CodeGenFunction & ) {}
94 CGOpenMPRegionKind getRegionKind()
const {
return RegionKind; }
98 bool hasCancel()
const {
return HasCancel; }
100 static bool classof(
const CGCapturedStmtInfo *Info) {
104 ~CGOpenMPRegionInfo()
override =
default;
107 CGOpenMPRegionKind RegionKind;
108 RegionCodeGenTy CodeGen;
114class CGOpenMPOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
116 CGOpenMPOutlinedRegionInfo(
const CapturedStmt &CS,
const VarDecl *ThreadIDVar,
117 const RegionCodeGenTy &CodeGen,
119 StringRef HelperName)
120 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen,
Kind,
122 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
123 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
128 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
131 StringRef getHelperName()
const override {
return HelperName; }
133 static bool classof(
const CGCapturedStmtInfo *Info) {
134 return CGOpenMPRegionInfo::classof(Info) &&
136 ParallelOutlinedRegion;
142 const VarDecl *ThreadIDVar;
143 StringRef HelperName;
147class CGOpenMPTaskOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
149 class UntiedTaskActionTy final :
public PrePostActionTy {
151 const VarDecl *PartIDVar;
152 const RegionCodeGenTy UntiedCodeGen;
153 llvm::SwitchInst *UntiedSwitch =
nullptr;
156 UntiedTaskActionTy(
bool Tied,
const VarDecl *PartIDVar,
157 const RegionCodeGenTy &UntiedCodeGen)
158 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
159 void Enter(CodeGenFunction &CGF)
override {
164 PartIDVar->
getType()->castAs<PointerType>());
168 UntiedSwitch = CGF.
Builder.CreateSwitch(Res, DoneBB);
172 UntiedSwitch->addCase(CGF.
Builder.getInt32(0),
174 emitUntiedSwitch(CGF);
177 void emitUntiedSwitch(CodeGenFunction &CGF)
const {
181 PartIDVar->
getType()->castAs<PointerType>());
185 CodeGenFunction::JumpDest CurPoint =
189 UntiedSwitch->addCase(CGF.
Builder.getInt32(UntiedSwitch->getNumCases()),
195 unsigned getNumberOfParts()
const {
return UntiedSwitch->getNumCases(); }
197 CGOpenMPTaskOutlinedRegionInfo(
const CapturedStmt &CS,
198 const VarDecl *ThreadIDVar,
199 const RegionCodeGenTy &CodeGen,
201 const UntiedTaskActionTy &Action)
202 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen,
Kind, HasCancel),
203 ThreadIDVar(ThreadIDVar), Action(Action) {
204 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
209 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
212 LValue getThreadIDVariableLValue(CodeGenFunction &CGF)
override;
215 StringRef getHelperName()
const override {
return ".omp_outlined."; }
217 void emitUntiedSwitch(CodeGenFunction &CGF)
override {
218 Action.emitUntiedSwitch(CGF);
221 static bool classof(
const CGCapturedStmtInfo *Info) {
222 return CGOpenMPRegionInfo::classof(Info) &&
230 const VarDecl *ThreadIDVar;
232 const UntiedTaskActionTy &Action;
237class CGOpenMPInlinedRegionInfo :
public CGOpenMPRegionInfo {
239 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
240 const RegionCodeGenTy &CodeGen,
242 : CGOpenMPRegionInfo(InlinedRegion, CodeGen,
Kind, HasCancel),
244 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
247 llvm::Value *getContextValue()
const override {
249 return OuterRegionInfo->getContextValue();
250 llvm_unreachable(
"No context value for inlined OpenMP region");
253 void setContextValue(llvm::Value *
V)
override {
254 if (OuterRegionInfo) {
255 OuterRegionInfo->setContextValue(
V);
258 llvm_unreachable(
"No context value for inlined OpenMP region");
262 const FieldDecl *lookup(
const VarDecl *VD)
const override {
264 return OuterRegionInfo->lookup(VD);
270 FieldDecl *getThisFieldDecl()
const override {
272 return OuterRegionInfo->getThisFieldDecl();
278 const VarDecl *getThreadIDVariable()
const override {
280 return OuterRegionInfo->getThreadIDVariable();
285 LValue getThreadIDVariableLValue(CodeGenFunction &CGF)
override {
287 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
288 llvm_unreachable(
"No LValue for inlined OpenMP construct");
292 StringRef getHelperName()
const override {
293 if (
auto *OuterRegionInfo = getOldCSI())
294 return OuterRegionInfo->getHelperName();
295 llvm_unreachable(
"No helper name for inlined OpenMP construct");
298 void emitUntiedSwitch(CodeGenFunction &CGF)
override {
300 OuterRegionInfo->emitUntiedSwitch(CGF);
303 CodeGenFunction::CGCapturedStmtInfo *getOldCSI()
const {
return OldCSI; }
305 static bool classof(
const CGCapturedStmtInfo *Info) {
306 return CGOpenMPRegionInfo::classof(Info) &&
310 ~CGOpenMPInlinedRegionInfo()
override =
default;
314 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
315 CGOpenMPRegionInfo *OuterRegionInfo;
323class CGOpenMPTargetRegionInfo final :
public CGOpenMPRegionInfo {
325 CGOpenMPTargetRegionInfo(
const CapturedStmt &CS,
326 const RegionCodeGenTy &CodeGen, StringRef HelperName)
327 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
329 HelperName(HelperName) {}
333 const VarDecl *getThreadIDVariable()
const override {
return nullptr; }
336 StringRef getHelperName()
const override {
return HelperName; }
338 static bool classof(
const CGCapturedStmtInfo *Info) {
339 return CGOpenMPRegionInfo::classof(Info) &&
344 StringRef HelperName;
348 llvm_unreachable(
"No codegen for expressions");
352class CGOpenMPInnerExprInfo final :
public CGOpenMPInlinedRegionInfo {
354 CGOpenMPInnerExprInfo(CodeGenFunction &CGF,
const CapturedStmt &CS)
355 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
363 if (!C.capturesVariable() && !C.capturesVariableByCopy())
366 const VarDecl *VD = C.getCapturedVar();
367 if (VD->isLocalVarDeclOrParm())
370 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
372 VD->getType().getNonReferenceType(), VK_LValue,
374 PrivScope.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress());
376 (
void)PrivScope.Privatize();
380 const FieldDecl *lookup(
const VarDecl *VD)
const override {
381 if (
const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
387 void EmitBody(CodeGenFunction &CGF,
const Stmt *S)
override {
388 llvm_unreachable(
"No body for expressions");
393 const VarDecl *getThreadIDVariable()
const override {
394 llvm_unreachable(
"No thread id for expressions");
398 StringRef getHelperName()
const override {
399 llvm_unreachable(
"No helper name for expressions");
402 static bool classof(
const CGCapturedStmtInfo *Info) {
return false; }
406 CodeGenFunction::OMPPrivateScope PrivScope;
410class InlinedOpenMPRegionRAII {
411 CodeGenFunction &CGF;
412 llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
413 FieldDecl *LambdaThisCaptureField =
nullptr;
414 const CodeGen::CGBlockInfo *BlockInfo =
nullptr;
415 bool NoInheritance =
false;
422 InlinedOpenMPRegionRAII(CodeGenFunction &CGF,
const RegionCodeGenTy &CodeGen,
424 bool NoInheritance =
true)
425 : CGF(CGF), NoInheritance(NoInheritance) {
427 CGF.CapturedStmtInfo =
new CGOpenMPInlinedRegionInfo(
428 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
430 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
431 LambdaThisCaptureField = CGF.LambdaThisCaptureField;
432 CGF.LambdaThisCaptureField =
nullptr;
433 BlockInfo = CGF.BlockInfo;
434 CGF.BlockInfo =
nullptr;
438 ~InlinedOpenMPRegionRAII() {
442 delete CGF.CapturedStmtInfo;
443 CGF.CapturedStmtInfo = OldCSI;
445 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
446 CGF.LambdaThisCaptureField = LambdaThisCaptureField;
447 CGF.BlockInfo = BlockInfo;
455enum OpenMPLocationFlags :
unsigned {
457 OMP_IDENT_IMD = 0x01,
459 OMP_IDENT_KMPC = 0x02,
461 OMP_ATOMIC_REDUCE = 0x10,
463 OMP_IDENT_BARRIER_EXPL = 0x20,
465 OMP_IDENT_BARRIER_IMPL = 0x40,
467 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
469 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
471 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
473 OMP_IDENT_WORK_LOOP = 0x200,
475 OMP_IDENT_WORK_SECTIONS = 0x400,
477 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
507enum IdentFieldIndex {
509 IdentField_Reserved_1,
513 IdentField_Reserved_2,
515 IdentField_Reserved_3,
524enum OpenMPSchedType {
527 OMP_sch_static_chunked = 33,
529 OMP_sch_dynamic_chunked = 35,
530 OMP_sch_guided_chunked = 36,
531 OMP_sch_runtime = 37,
534 OMP_sch_static_balanced_chunked = 45,
537 OMP_ord_static_chunked = 65,
539 OMP_ord_dynamic_chunked = 67,
540 OMP_ord_guided_chunked = 68,
541 OMP_ord_runtime = 69,
543 OMP_sch_default = OMP_sch_static,
545 OMP_dist_sch_static_chunked = 91,
546 OMP_dist_sch_static = 92,
549 OMP_sch_modifier_monotonic = (1 << 29),
551 OMP_sch_modifier_nonmonotonic = (1 << 30),
556class CleanupTy final :
public EHScopeStack::Cleanup {
557 PrePostActionTy *Action;
560 explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
561 void Emit(CodeGenFunction &CGF, Flags )
override {
574 Callback(CodeGen, CGF, *PrePostAction);
577 Callback(CodeGen, CGF, Action);
585 if (
const auto *CE = dyn_cast<CallExpr>(ReductionOp))
586 if (
const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
587 if (
const auto *DRE =
588 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
589 if (
const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
600 std::pair<llvm::Function *, llvm::Function *>
Reduction =
620 auto *GV =
new llvm::GlobalVariable(
622 llvm::GlobalValue::PrivateLinkage,
Init, Name);
663 llvm::Value *NumElements = CGF.
emitArrayLength(ArrayTy, ElementTy, DestAddr);
667 llvm::Value *SrcBegin =
nullptr;
669 SrcBegin = SrcAddr.emitRawPointer(CGF);
672 llvm::Value *DestEnd =
677 llvm::Value *IsEmpty =
678 CGF.
Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arrayinit.isempty");
679 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
682 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
687 llvm::PHINode *SrcElementPHI =
nullptr;
690 SrcElementPHI = CGF.
Builder.CreatePHI(SrcBegin->getType(), 2,
691 "omp.arraycpy.srcElementPast");
692 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
694 Address(SrcElementPHI, SrcAddr.getElementType(),
695 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
697 llvm::PHINode *DestElementPHI = CGF.
Builder.CreatePHI(
698 DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
699 DestElementPHI->addIncoming(DestBegin, EntryBB);
707 if (EmitDeclareReductionInit) {
709 SrcElementCurrent, ElementTy);
717 llvm::Value *SrcElementNext = CGF.
Builder.CreateConstGEP1_32(
718 SrcAddr.getElementType(), SrcElementPHI, 1,
719 "omp.arraycpy.dest.element");
720 SrcElementPHI->addIncoming(SrcElementNext, CGF.
Builder.GetInsertBlock());
724 llvm::Value *DestElementNext = CGF.
Builder.CreateConstGEP1_32(
726 "omp.arraycpy.dest.element");
729 CGF.
Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
730 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
731 DestElementPHI->addIncoming(DestElementNext, CGF.
Builder.GetInsertBlock());
743 if (
const auto *OASE = dyn_cast<ArraySectionExpr>(E))
748void ReductionCodeGen::emitAggregateInitialization(
750 const OMPDeclareReductionDecl *DRD) {
754 const auto *PrivateVD =
756 bool EmitDeclareReductionInit =
759 EmitDeclareReductionInit,
760 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
761 : PrivateVD->getInit(),
769 ClausesData.reserve(Shareds.size());
770 SharedAddresses.reserve(Shareds.size());
771 Sizes.reserve(Shareds.size());
772 BaseDecls.reserve(Shareds.size());
773 const auto *IOrig = Origs.begin();
774 const auto *IPriv = Privates.begin();
775 const auto *IRed = ReductionOps.begin();
776 for (
const Expr *Ref : Shareds) {
777 ClausesData.emplace_back(Ref, *IOrig, *IPriv, *IRed);
778 std::advance(IOrig, 1);
779 std::advance(IPriv, 1);
780 std::advance(IRed, 1);
785 assert(SharedAddresses.size() == N && OrigAddresses.size() == N &&
786 "Number of generated lvalues must be exactly N.");
787 LValue First = emitSharedLValue(CGF, ClausesData[N].Shared);
788 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Shared);
789 SharedAddresses.emplace_back(
First, Second);
790 if (ClausesData[N].Shared == ClausesData[N].Ref) {
791 OrigAddresses.emplace_back(
First, Second);
793 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
794 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
795 OrigAddresses.emplace_back(
First, Second);
804 CGF.
getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()),
809 llvm::Value *SizeInChars;
810 auto *ElemType = OrigAddresses[N].first.getAddress().getElementType();
811 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
812 if (AsArraySection) {
813 Size = CGF.
Builder.CreatePtrDiff(ElemType,
814 OrigAddresses[N].second.getPointer(CGF),
815 OrigAddresses[N].first.getPointer(CGF));
816 Size = CGF.
Builder.CreateNUWAdd(
817 Size, llvm::ConstantInt::get(Size->getType(), 1));
818 SizeInChars = CGF.
Builder.CreateNUWMul(Size, ElemSizeOf);
821 CGF.
getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType());
822 Size = CGF.
Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
824 Sizes.emplace_back(SizeInChars, Size);
837 assert(!Size && !Sizes[N].second &&
838 "Size should be nullptr for non-variably modified reduction "
853 assert(SharedAddresses.size() > N &&
"No variable was generated");
854 const auto *PrivateVD =
860 (void)DefaultInit(CGF);
861 emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD);
862 }
else if (DRD && (DRD->
getInitializer() || !PrivateVD->hasInit())) {
863 (void)DefaultInit(CGF);
864 QualType SharedType = SharedAddresses[N].first.getType();
866 PrivateAddr, SharedAddr, SharedType);
867 }
else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
870 PrivateVD->
getType().getQualifiers(),
888 CGF.
pushDestroy(DTorKind, PrivateAddr, PrivateType);
907 BaseLV.getType(), BaseLV.getBaseInfo(),
941 const VarDecl *OrigVD =
nullptr;
942 if (
const auto *OASE = dyn_cast<ArraySectionExpr>(Ref)) {
943 const Expr *
Base = OASE->getBase()->IgnoreParenImpCasts();
944 while (
const auto *TempOASE = dyn_cast<ArraySectionExpr>(
Base))
945 Base = TempOASE->getBase()->IgnoreParenImpCasts();
946 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
947 Base = TempASE->getBase()->IgnoreParenImpCasts();
950 }
else if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
951 const Expr *
Base = ASE->getBase()->IgnoreParenImpCasts();
952 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
953 Base = TempASE->getBase()->IgnoreParenImpCasts();
964 BaseDecls.emplace_back(OrigVD);
967 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
969 Address SharedAddr = SharedAddresses[N].first.getAddress();
970 llvm::Value *Adjustment = CGF.
Builder.CreatePtrDiff(
973 llvm::Value *PrivatePointer =
979 SharedAddresses[N].first.getType(),
982 BaseDecls.emplace_back(
996 getThreadIDVariable()->
getType()->castAs<PointerType>());
1014LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1017 getThreadIDVariable()->
getType(),
1035 llvm::OpenMPIRBuilderConfig Config(
1036 CGM.getLangOpts().OpenMPIsTargetDevice,
isGPU(),
1037 CGM.getLangOpts().OpenMPOffloadMandatory,
1040 Config.setDefaultTargetAS(
1046 CGM.getLangOpts().OpenMPIsTargetDevice
1047 ?
CGM.getLangOpts().OMPHostIRFile
1052 if (
CGM.getLangOpts().OpenMPForceUSM) {
1054 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(
true);
1062 if (!
Data.getValue().pointsToAliveValue())
1064 auto *GV = dyn_cast<llvm::GlobalVariable>(
Data.getValue());
1067 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1069 GV->eraseFromParent();
1074 return OMPBuilder.createPlatformSpecificName(Parts);
1077static llvm::Function *
1079 const Expr *CombinerInitializer,
const VarDecl *In,
1080 const VarDecl *Out,
bool IsCombiner) {
1083 QualType PtrTy =
C.getPointerType(Ty).withRestrict();
1089 Args.push_back(&OmpOutParm);
1090 Args.push_back(&OmpInParm);
1095 {IsCombiner ?
"omp_combiner" :
"omp_initializer",
""});
1096 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1100 Fn->removeFnAttr(llvm::Attribute::NoInline);
1101 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1102 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1108 Out->getLocation());
1118 (void)
Scope.Privatize();
1119 if (!IsCombiner && Out->hasInit() &&
1122 Out->getType().getQualifiers(),
1125 if (CombinerInitializer)
1127 Scope.ForceCleanup();
1156std::pair<llvm::Function *, llvm::Function *>
1168struct PushAndPopStackRAII {
1169 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder,
CodeGenFunction &CGF,
1170 bool HasCancel, llvm::omp::Directive Kind)
1171 : OMPBuilder(OMPBuilder) {
1187 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1188 assert(IP.getBlock()->end() == IP.getPoint() &&
1189 "Clang CG should cause non-terminated block!");
1190 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1195 return llvm::Error::success();
1200 llvm::OpenMPIRBuilder::FinalizationInfo FI({FiniCB, Kind, HasCancel});
1201 OMPBuilder->pushFinalizationCB(std::move(FI));
1203 ~PushAndPopStackRAII() {
1205 OMPBuilder->popFinalizationCB();
1207 llvm::OpenMPIRBuilder *OMPBuilder;
1216 "thread id variable must be of type kmp_int32 *");
1218 bool HasCancel =
false;
1219 if (
const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1220 HasCancel = OPD->hasCancel();
1221 else if (
const auto *OPD = dyn_cast<OMPTargetParallelDirective>(&D))
1222 HasCancel = OPD->hasCancel();
1223 else if (
const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1224 HasCancel = OPSD->hasCancel();
1225 else if (
const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1226 HasCancel = OPFD->hasCancel();
1227 else if (
const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1228 HasCancel = OPFD->hasCancel();
1229 else if (
const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1230 HasCancel = OPFD->hasCancel();
1231 else if (
const auto *OPFD =
1232 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1233 HasCancel = OPFD->hasCancel();
1234 else if (
const auto *OPFD =
1235 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1236 HasCancel = OPFD->hasCancel();
1241 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind);
1242 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar,
CodeGen, InnermostKind,
1243 HasCancel, OutlinedHelperName);
1249 std::string Suffix =
getName({
"omp_outlined"});
1250 return (Name + Suffix).str();
1258 std::string Suffix =
getName({
"omp",
"reduction",
"reduction_func"});
1259 return (Name + Suffix).str();
1266 const CapturedStmt *CS = D.getCapturedStmt(OMPD_parallel);
1276 const CapturedStmt *CS = D.getCapturedStmt(OMPD_teams);
1286 bool Tied,
unsigned &NumberOfParts) {
1289 llvm::Value *ThreadID =
getThreadID(CGF, D.getBeginLoc());
1291 llvm::Value *TaskArgs[] = {
1293 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1296 CGF.EmitRuntimeCall(
OMPBuilder.getOrCreateRuntimeFunction(
1297 CGM.getModule(), OMPRTL___kmpc_omp_task),
1300 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1304 "thread id variable must be of type kmp_int32 for tasks");
1309 bool HasCancel =
false;
1310 if (
const auto *TD = dyn_cast<OMPTaskDirective>(&D))
1311 HasCancel = TD->hasCancel();
1312 else if (
const auto *TD = dyn_cast<OMPTaskLoopDirective>(&D))
1313 HasCancel = TD->hasCancel();
1314 else if (
const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(&D))
1315 HasCancel = TD->hasCancel();
1316 else if (
const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(&D))
1317 HasCancel = TD->hasCancel();
1320 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar,
CodeGen,
1321 InnermostKind, HasCancel, Action);
1323 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1325 NumberOfParts = Action.getNumberOfParts();
1330 bool AtCurrentPoint) {
1332 assert(!Elem.ServiceInsertPt &&
"Insert point is set already.");
1334 llvm::Value *Undef = llvm::UndefValue::get(CGF.
Int32Ty);
1335 if (AtCurrentPoint) {
1336 Elem.ServiceInsertPt =
new llvm::BitCastInst(Undef, CGF.
Int32Ty,
"svcpt",
1337 CGF.
Builder.GetInsertBlock());
1339 Elem.ServiceInsertPt =
new llvm::BitCastInst(Undef, CGF.
Int32Ty,
"svcpt");
1340 Elem.ServiceInsertPt->insertAfter(CGF.
AllocaInsertPt->getIterator());
1346 if (Elem.ServiceInsertPt) {
1347 llvm::Instruction *Ptr = Elem.ServiceInsertPt;
1348 Elem.ServiceInsertPt =
nullptr;
1349 Ptr->eraseFromParent();
1356 llvm::raw_svector_ostream OS(Buffer);
1365 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.
CurFuncDecl))
1366 OS << FD->getQualifiedNameAsString();
1373 unsigned Flags,
bool EmitLoc) {
1374 uint32_t SrcLocStrSize;
1375 llvm::Constant *SrcLocStr;
1376 if ((!EmitLoc &&
CGM.getCodeGenOpts().getDebugInfo() ==
1377 llvm::codegenoptions::NoDebugInfo) ||
1379 SrcLocStr =
OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1381 std::string FunctionName;
1383 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.
CurFuncDecl))
1384 FunctionName = FD->getQualifiedNameAsString();
1397 SrcLocStr, SrcLocStrSize, llvm::omp::IdentFlag(Flags), Reserved2Flags);
1402 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1405 if (
CGM.getLangOpts().OpenMPIRBuilder) {
1408 uint32_t SrcLocStrSize;
1409 auto *SrcLocStr =
OMPBuilder.getOrCreateSrcLocStr(
1412 OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize));
1415 llvm::Value *ThreadID =
nullptr;
1420 ThreadID = I->second.ThreadID;
1421 if (ThreadID !=
nullptr)
1425 if (
auto *OMPRegionInfo =
1427 if (OMPRegionInfo->getThreadIDVariable()) {
1429 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1433 CGF.
Builder.GetInsertBlock() == TopBlock ||
1438 CGF.
Builder.GetInsertBlock()) {
1442 if (CGF.
Builder.GetInsertBlock() == TopBlock)
1454 if (!Elem.ServiceInsertPt)
1456 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1457 CGF.
Builder.SetInsertPoint(Elem.ServiceInsertPt);
1461 OMPRTL___kmpc_global_thread_num),
1464 Elem.ThreadID =
Call;
1469 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1475 for (
const auto *D : I->second)
1480 for (
const auto *D : I->second)
1492static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind
1494 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
1495 OMPDeclareTargetDeclAttr::getDeviceType(VD);
1497 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1499 switch ((
int)*DevTy) {
1500 case OMPDeclareTargetDeclAttr::DT_Host:
1501 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseHost;
1503 case OMPDeclareTargetDeclAttr::DT_NoHost:
1504 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNoHost;
1506 case OMPDeclareTargetDeclAttr::DT_Any:
1507 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny;
1510 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1515static llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind
1517 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> MapType =
1518 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1520 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1521 switch ((
int)*MapType) {
1522 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_To:
1523 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
1525 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Enter:
1526 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter;
1528 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Link:
1529 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
1532 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1541 auto FileInfoCallBack = [&]() {
1546 PLoc =
SM.getPresumedLoc(BeginLoc,
false);
1551 return OMPBuilder.getTargetEntryUniqueInfo(FileInfoCallBack,
1556 auto AddrOfGlobal = [&VD,
this]() {
return CGM.GetAddrOfGlobal(VD); };
1558 auto LinkageForVariable = [&VD,
this]() {
1559 return CGM.getLLVMLinkageVarDefinition(VD);
1562 std::vector<llvm::GlobalVariable *> GeneratedRefs;
1564 llvm::Type *LlvmPtrTy =
CGM.getTypes().ConvertTypeForMem(
1565 CGM.getContext().getPointerType(VD->
getType()));
1566 llvm::Constant *addr =
OMPBuilder.getAddrOfDeclareTargetVar(
1572 CGM.getMangledName(VD), GeneratedRefs,
CGM.getLangOpts().OpenMPSimd,
1573 CGM.getLangOpts().OMPTargetTriples, LlvmPtrTy, AddrOfGlobal,
1574 LinkageForVariable);
1583 assert(!
CGM.getLangOpts().OpenMPUseTLS ||
1584 !
CGM.getContext().getTargetInfo().isTLSSupported());
1586 std::string Suffix =
getName({
"cache",
""});
1587 return OMPBuilder.getOrCreateInternalVariable(
1588 CGM.Int8PtrPtrTy, Twine(
CGM.getMangledName(VD)).concat(Suffix).str());
1595 if (
CGM.getLangOpts().OpenMPUseTLS &&
1596 CGM.getContext().getTargetInfo().isTLSSupported())
1600 llvm::Value *Args[] = {
1603 CGM.getSize(
CGM.GetTargetTypeStoreSize(VarTy)),
1608 CGM.getModule(), OMPRTL___kmpc_threadprivate_cached),
1620 CGM.getModule(), OMPRTL___kmpc_global_thread_num),
1624 llvm::Value *Args[] = {
1627 Ctor, CopyCtor, Dtor};
1630 CGM.getModule(), OMPRTL___kmpc_threadprivate_register),
1637 if (
CGM.getLangOpts().OpenMPUseTLS &&
1638 CGM.getContext().getTargetInfo().isTLSSupported())
1645 llvm::Value *Ctor =
nullptr, *CopyCtor =
nullptr, *Dtor =
nullptr;
1647 if (
CGM.getLangOpts().CPlusPlus && PerformInit) {
1653 nullptr,
CGM.getContext().VoidPtrTy,
1655 Args.push_back(&Dst);
1657 const auto &FI =
CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1658 CGM.getContext().VoidPtrTy, Args);
1659 llvm::FunctionType *FTy =
CGM.getTypes().GetFunctionType(FI);
1660 std::string Name =
getName({
"__kmpc_global_ctor_",
""});
1661 llvm::Function *Fn =
1662 CGM.CreateGlobalInitOrCleanUpFunction(FTy, Name, FI, Loc);
1685 nullptr,
CGM.getContext().VoidPtrTy,
1687 Args.push_back(&Dst);
1689 const auto &FI =
CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1690 CGM.getContext().VoidTy, Args);
1691 llvm::FunctionType *FTy =
CGM.getTypes().GetFunctionType(FI);
1692 std::string Name =
getName({
"__kmpc_global_dtor_",
""});
1693 llvm::Function *Fn =
1694 CGM.CreateGlobalInitOrCleanUpFunction(FTy, Name, FI, Loc);
1717 CopyCtor = llvm::Constant::getNullValue(
CGM.DefaultPtrTy);
1718 if (Ctor ==
nullptr) {
1719 Ctor = llvm::Constant::getNullValue(
CGM.DefaultPtrTy);
1721 if (Dtor ==
nullptr) {
1722 Dtor = llvm::Constant::getNullValue(
CGM.DefaultPtrTy);
1725 auto *InitFunctionTy =
1726 llvm::FunctionType::get(
CGM.VoidTy,
false);
1727 std::string Name =
getName({
"__omp_threadprivate_init_",
""});
1728 llvm::Function *InitFunction =
CGM.CreateGlobalInitOrCleanUpFunction(
1729 InitFunctionTy, Name,
CGM.getTypes().arrangeNullaryFunction());
1733 CGM.getTypes().arrangeNullaryFunction(), ArgList,
1737 return InitFunction;
1745 llvm::GlobalValue *GV) {
1746 std::optional<OMPDeclareTargetDeclAttr *> ActiveAttr =
1747 OMPDeclareTargetDeclAttr::getActiveAttr(FD);
1750 if (!ActiveAttr || !(*ActiveAttr)->getIndirect())
1757 OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(Name, EntryInfo);
1763 llvm::GlobalValue *
Addr = GV;
1764 if (
CGM.getLangOpts().OpenMPIsTargetDevice) {
1765 llvm::PointerType *FnPtrTy = llvm::PointerType::get(
1766 CGM.getLLVMContext(),
1767 CGM.getModule().getDataLayout().getProgramAddressSpace());
1768 Addr =
new llvm::GlobalVariable(
1769 CGM.getModule(), FnPtrTy,
1770 true, llvm::GlobalValue::ExternalLinkage, GV, Name,
1771 nullptr, llvm::GlobalValue::NotThreadLocal,
1772 CGM.getModule().getDataLayout().getDefaultGlobalsAddressSpace());
1773 Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1776 OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(
1777 Name,
Addr,
CGM.GetTargetTypeStoreSize(
CGM.VoidPtrTy).getQuantity(),
1778 llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect,
1779 llvm::GlobalValue::WeakODRLinkage);
1785 std::string Suffix =
getName({
"artificial",
""});
1787 llvm::GlobalVariable *GAddr =
OMPBuilder.getOrCreateInternalVariable(
1788 VarLVType, Twine(Name).concat(Suffix).str());
1789 if (
CGM.getLangOpts().OpenMP &&
CGM.getLangOpts().OpenMPUseTLS &&
1790 CGM.getTarget().isTLSSupported()) {
1791 GAddr->setThreadLocal(
true);
1792 return Address(GAddr, GAddr->getValueType(),
1793 CGM.getContext().getTypeAlignInChars(VarType));
1795 std::string CacheSuffix =
getName({
"cache",
""});
1796 llvm::Value *Args[] = {
1804 Twine(Name).concat(Suffix).concat(CacheSuffix).str())};
1809 CGM.getModule(), OMPRTL___kmpc_threadprivate_cached),
1812 VarLVType,
CGM.getContext().getTypeAlignInChars(VarType));
1862 auto &M =
CGM.getModule();
1863 auto &&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc,
1866 llvm::Value *Args[] = {
1868 CGF.
Builder.getInt32(CapturedVars.size()),
1871 RealArgs.append(std::begin(Args), std::end(Args));
1872 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
1874 llvm::FunctionCallee RTLFn =
1875 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_fork_call);
1878 auto &&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc, Loc,
1884 llvm::Value *Args[] = {RTLoc, ThreadID};
1886 M, OMPRTL___kmpc_serialized_parallel),
1893 ".bound.zero.addr");
1898 OutlinedFnArgs.push_back(ZeroAddrBound.
getPointer());
1899 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
1907 OutlinedFn->removeFnAttr(llvm::Attribute::AlwaysInline);
1908 OutlinedFn->addFnAttr(llvm::Attribute::NoInline);
1914 M, OMPRTL___kmpc_end_serialized_parallel),
1933 if (
auto *OMPRegionInfo =
1935 if (OMPRegionInfo->getThreadIDVariable())
1936 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
1945 return ThreadIDTemp;
1949 std::string Prefix = Twine(
"gomp_critical_user_", CriticalName).str();
1950 std::string Name =
getName({Prefix,
"var"});
1957 llvm::FunctionCallee EnterCallee;
1959 llvm::FunctionCallee ExitCallee;
1962 llvm::BasicBlock *ContBlock =
nullptr;
1965 CommonActionTy(llvm::FunctionCallee EnterCallee,
1967 llvm::FunctionCallee ExitCallee,
1969 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
1974 llvm::Value *CallBool = CGF.
Builder.CreateIsNotNull(EnterRes);
1978 CGF.
Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
1982 void Done(CodeGenFunction &CGF) {
1987 void Exit(CodeGenFunction &CGF)
override {
1994 StringRef CriticalName,
2008 EnterArgs.push_back(CGF.
Builder.CreateIntCast(
2011 CommonActionTy Action(
2014 Hint ? OMPRTL___kmpc_critical_with_hint : OMPRTL___kmpc_critical),
2017 OMPRTL___kmpc_end_critical),
2034 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2035 CGM.getModule(), OMPRTL___kmpc_master),
2038 CGM.getModule(), OMPRTL___kmpc_end_master),
2056 llvm::Value *FilterVal = Filter
2058 : llvm::ConstantInt::get(
CGM.Int32Ty, 0);
2063 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2064 CGM.getModule(), OMPRTL___kmpc_masked),
2067 CGM.getModule(), OMPRTL___kmpc_end_masked),
2083 llvm::Value *Args[] = {
2085 llvm::ConstantInt::get(
CGM.IntTy, 0,
true)};
2087 CGM.getModule(), OMPRTL___kmpc_omp_taskyield),
2091 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
2092 Region->emitUntiedSwitch(CGF);
2105 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2106 CGM.getModule(), OMPRTL___kmpc_taskgroup),
2109 CGM.getModule(), OMPRTL___kmpc_end_taskgroup),
2118 unsigned Index,
const VarDecl *Var) {
2139 Args.push_back(&LHSArg);
2140 Args.push_back(&RHSArg);
2146 llvm::GlobalValue::InternalLinkage, Name,
2149 Fn->setDoesNotRecurse();
2166 for (
unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
2167 const auto *DestVar =
2171 const auto *SrcVar =
2177 CGF.
EmitOMPCopy(
Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
2192 assert(CopyprivateVars.size() == SrcExprs.size() &&
2193 CopyprivateVars.size() == DstExprs.size() &&
2194 CopyprivateVars.size() == AssignmentOps.size());
2206 if (!CopyprivateVars.empty()) {
2209 C.getIntTypeForBitwidth(32, 1);
2210 DidIt = CGF.
CreateMemTemp(KmpInt32Ty,
".omp.copyprivate.did_it");
2215 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2216 CGM.getModule(), OMPRTL___kmpc_single),
2219 CGM.getModule(), OMPRTL___kmpc_end_single),
2232 llvm::APInt ArraySize(32, CopyprivateVars.size());
2233 QualType CopyprivateArrayTy =
C.getConstantArrayType(
2238 CGF.
CreateMemTemp(CopyprivateArrayTy,
".omp.copyprivate.cpr_list");
2239 for (
unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
2251 SrcExprs, DstExprs, AssignmentOps, Loc);
2252 llvm::Value *BufSize = CGF.
getTypeSize(CopyprivateArrayTy);
2256 llvm::Value *Args[] = {
2260 CL.emitRawPointer(CGF),
2265 CGM.getModule(), OMPRTL___kmpc_copyprivate),
2281 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
2282 CGM.getModule(), OMPRTL___kmpc_ordered),
2285 CGM.getModule(), OMPRTL___kmpc_end_ordered),
2296 if (Kind == OMPD_for)
2297 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
2298 else if (Kind == OMPD_sections)
2299 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
2300 else if (Kind == OMPD_single)
2301 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
2302 else if (Kind == OMPD_barrier)
2303 Flags = OMP_IDENT_BARRIER_EXPL;
2305 Flags = OMP_IDENT_BARRIER_IMPL;
2317 ScheduleKind = OMPC_SCHEDULE_static;
2319 llvm::APInt ChunkSize(32, 1);
2329 bool ForceSimpleCall) {
2331 auto *OMPRegionInfo =
2334 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
2337 CGF.
Builder.restoreIP(AfterIP);
2350 if (OMPRegionInfo) {
2351 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
2354 OMPRTL___kmpc_cancel_barrier),
2363 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
2375 CGM.getModule(), OMPRTL___kmpc_barrier),
2380 Expr *ME,
bool IsFatal) {
2382 : llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
2385 llvm::Value *Args[] = {
2387 llvm::ConstantInt::get(
CGM.Int32Ty, IsFatal ? 2 : 1),
2388 CGF.
Builder.CreatePointerCast(MVL,
CGM.Int8PtrTy)};
2390 CGM.getModule(), OMPRTL___kmpc_error),
2396 bool Chunked,
bool Ordered) {
2397 switch (ScheduleKind) {
2398 case OMPC_SCHEDULE_static:
2399 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
2400 : (Ordered ? OMP_ord_static : OMP_sch_static);
2401 case OMPC_SCHEDULE_dynamic:
2402 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
2403 case OMPC_SCHEDULE_guided:
2404 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
2405 case OMPC_SCHEDULE_runtime:
2406 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
2407 case OMPC_SCHEDULE_auto:
2408 return Ordered ? OMP_ord_auto : OMP_sch_auto;
2410 assert(!Chunked &&
"chunk was specified but schedule kind not known");
2411 return Ordered ? OMP_ord_static : OMP_sch_static;
2413 llvm_unreachable(
"Unexpected runtime schedule");
2417static OpenMPSchedType
2420 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
2424 bool Chunked)
const {
2425 OpenMPSchedType Schedule =
2427 return Schedule == OMP_sch_static;
2433 return Schedule == OMP_dist_sch_static;
2437 bool Chunked)
const {
2438 OpenMPSchedType Schedule =
2440 return Schedule == OMP_sch_static_chunked;
2446 return Schedule == OMP_dist_sch_static_chunked;
2450 OpenMPSchedType Schedule =
2452 assert(Schedule != OMP_sch_static_chunked &&
"cannot be chunked here");
2453 return Schedule != OMP_sch_static;
2461 case OMPC_SCHEDULE_MODIFIER_monotonic:
2462 Modifier = OMP_sch_modifier_monotonic;
2464 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2465 Modifier = OMP_sch_modifier_nonmonotonic;
2467 case OMPC_SCHEDULE_MODIFIER_simd:
2468 if (Schedule == OMP_sch_static_chunked)
2469 Schedule = OMP_sch_static_balanced_chunked;
2476 case OMPC_SCHEDULE_MODIFIER_monotonic:
2477 Modifier = OMP_sch_modifier_monotonic;
2479 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2480 Modifier = OMP_sch_modifier_nonmonotonic;
2482 case OMPC_SCHEDULE_MODIFIER_simd:
2483 if (Schedule == OMP_sch_static_chunked)
2484 Schedule = OMP_sch_static_balanced_chunked;
2496 if (CGM.
getLangOpts().OpenMP >= 50 && Modifier == 0) {
2497 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
2498 Schedule == OMP_sch_static_balanced_chunked ||
2499 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||
2500 Schedule == OMP_dist_sch_static_chunked ||
2501 Schedule == OMP_dist_sch_static))
2502 Modifier = OMP_sch_modifier_nonmonotonic;
2504 return Schedule | Modifier;
2514 ScheduleKind.
Schedule, DispatchValues.
Chunk !=
nullptr, Ordered);
2516 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
2517 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&
2518 Schedule != OMP_sch_static_balanced_chunked));
2525 llvm::Value *Chunk = DispatchValues.
Chunk ? DispatchValues.
Chunk
2526 : CGF.
Builder.getIntN(IVSize, 1);
2527 llvm::Value *Args[] = {
2531 CGM, Schedule, ScheduleKind.
M1, ScheduleKind.
M2)),
2534 CGF.
Builder.getIntN(IVSize, 1),
2551 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
2552 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
2559 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||
2560 Schedule == OMP_sch_static_balanced_chunked ||
2561 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||
2562 Schedule == OMP_dist_sch_static ||
2563 Schedule == OMP_dist_sch_static_chunked);
2570 llvm::Value *Chunk = Values.
Chunk;
2571 if (Chunk ==
nullptr) {
2572 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
2573 Schedule == OMP_dist_sch_static) &&
2574 "expected static non-chunked schedule");
2578 assert((Schedule == OMP_sch_static_chunked ||
2579 Schedule == OMP_sch_static_balanced_chunked ||
2580 Schedule == OMP_ord_static_chunked ||
2581 Schedule == OMP_dist_sch_static_chunked) &&
2582 "expected static chunked schedule");
2584 llvm::Value *Args[] = {
2607 "Expected loop-based or sections-based directive.");
2610 ? OMP_IDENT_WORK_LOOP
2611 : OMP_IDENT_WORK_SECTIONS);
2613 llvm::FunctionCallee StaticInitFunction =
2618 ScheduleNum, ScheduleKind.
M1, ScheduleKind.
M2, Values);
2625 OpenMPSchedType ScheduleNum =
2627 llvm::Value *UpdatedLocation =
2630 llvm::FunctionCallee StaticInitFunction;
2631 bool isGPUDistribute =
2632 CGM.getLangOpts().OpenMPIsTargetDevice &&
CGM.getTriple().isGPU();
2633 StaticInitFunction =
OMPBuilder.createForStaticInitFunction(
2644 assert((DKind == OMPD_distribute || DKind == OMPD_for ||
2645 DKind == OMPD_sections) &&
2646 "Expected distribute, for, or sections directive kind");
2650 llvm::Value *Args[] = {
2653 (DKind == OMPD_target_teams_loop)
2654 ? OMP_IDENT_WORK_DISTRIBUTE
2656 ? OMP_IDENT_WORK_LOOP
2657 : OMP_IDENT_WORK_SECTIONS),
2661 CGM.getLangOpts().OpenMPIsTargetDevice &&
CGM.getTriple().isGPU())
2664 CGM.getModule(), OMPRTL___kmpc_distribute_static_fini),
2668 CGM.getModule(), OMPRTL___kmpc_for_static_fini),
2693 llvm::Value *Args[] = {
2701 OMPBuilder.createDispatchNextFunction(IVSize, IVSigned), Args);
2708 const Expr *Message,
2711 return llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
2720 return llvm::ConstantInt::get(
CGM.Int32Ty,
2721 Severity == OMPC_SEVERITY_warning ? 1 : 2);
2737 RuntimeFunction FnID = OMPRTL___kmpc_push_num_threads;
2738 if (Modifier == OMPC_NUMTHREADS_strict) {
2739 FnID = OMPRTL___kmpc_push_num_threads_strict;
2744 OMPBuilder.getOrCreateRuntimeFunction(
CGM.getModule(), FnID), Args);
2748 ProcBindKind ProcBind,
2752 assert(ProcBind != OMP_PROC_BIND_unknown &&
"Unsupported proc_bind value.");
2754 llvm::Value *Args[] = {
2756 llvm::ConstantInt::get(
CGM.IntTy,
unsigned(ProcBind),
true)};
2758 CGM.getModule(), OMPRTL___kmpc_push_proc_bind),
2771 CGM.getModule(), OMPRTL___kmpc_flush),
2778enum KmpTaskTFields {
2805 if (
CGM.getLangOpts().OpenMPSimd ||
OMPBuilder.OffloadInfoManager.empty())
2808 llvm::OpenMPIRBuilder::EmitMetadataErrorReportFunctionTy &&ErrorReportFn =
2809 [
this](llvm::OpenMPIRBuilder::EmitMetadataErrorKind Kind,
2810 const llvm::TargetRegionEntryInfo &EntryInfo) ->
void {
2812 if (Kind != llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR) {
2813 for (
auto I =
CGM.getContext().getSourceManager().fileinfo_begin(),
2814 E =
CGM.getContext().getSourceManager().fileinfo_end();
2816 if (I->getFirst().getUniqueID().getDevice() == EntryInfo.DeviceID &&
2817 I->getFirst().getUniqueID().getFile() == EntryInfo.FileID) {
2818 Loc =
CGM.getContext().getSourceManager().translateFileLineCol(
2819 I->getFirst(), EntryInfo.Line, 1);
2825 case llvm::OpenMPIRBuilder::EMIT_MD_TARGET_REGION_ERROR: {
2826 unsigned DiagID =
CGM.getDiags().getCustomDiagID(
2828 "%0 is incorrect: either the "
2829 "address or the ID is invalid.");
2830 CGM.getDiags().Report(Loc, DiagID) << EntryInfo.ParentName;
2832 case llvm::OpenMPIRBuilder::EMIT_MD_DECLARE_TARGET_ERROR: {
2833 unsigned DiagID =
CGM.getDiags().getCustomDiagID(
2835 "variable %0 is incorrect: the "
2836 "address is invalid.");
2837 CGM.getDiags().Report(Loc, DiagID) << EntryInfo.ParentName;
2839 case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR: {
2840 unsigned DiagID =
CGM.getDiags().getCustomDiagID(
2842 "Offloading entry for declare target variable is incorrect: the "
2843 "address is invalid.");
2844 CGM.getDiags().Report(DiagID);
2849 OMPBuilder.createOffloadEntriesAndInfoMetadata(ErrorReportFn);
2856 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty,
C.VoidPtrTy};
2859 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
2865struct PrivateHelpersTy {
2866 PrivateHelpersTy(
const Expr *OriginalRef,
const VarDecl *Original,
2868 : OriginalRef(OriginalRef), Original(Original), PrivateCopy(PrivateCopy),
2869 PrivateElemInit(PrivateElemInit) {}
2870 PrivateHelpersTy(
const VarDecl *Original) : Original(Original) {}
2871 const Expr *OriginalRef =
nullptr;
2872 const VarDecl *Original =
nullptr;
2873 const VarDecl *PrivateCopy =
nullptr;
2874 const VarDecl *PrivateElemInit =
nullptr;
2875 bool isLocalPrivate()
const {
2876 return !OriginalRef && !PrivateCopy && !PrivateElemInit;
2879typedef std::pair<CharUnits , PrivateHelpersTy> PrivateDataTy;
2884 if (!CVD->
hasAttr<OMPAllocateDeclAttr>())
2886 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
2888 return !(AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc &&
2889 !AA->getAllocator());
2894 if (!Privates.empty()) {
2899 RecordDecl *RD =
C.buildImplicitRecord(
".kmp_privates.t");
2901 for (
const auto &Pair : Privates) {
2902 const VarDecl *VD = Pair.second.Original;
2906 if (Pair.second.isLocalPrivate()) {
2929 QualType KmpRoutineEntryPointerQTy) {
2949 CanQualType KmpCmplrdataTy =
C.getCanonicalTagType(UD);
2950 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_task_t");
2980 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_task_t_with_privates");
3000static llvm::Function *
3003 QualType KmpTaskTWithPrivatesPtrQTy,
3005 QualType SharedsPtrTy, llvm::Function *TaskFunction,
3006 llvm::Value *TaskPrivatesMap) {
3014 Args.push_back(&GtidArg);
3015 Args.push_back(&TaskTypeArg);
3016 const auto &TaskEntryFnInfo =
3018 llvm::FunctionType *TaskEntryTy =
3021 auto *TaskEntry = llvm::Function::Create(
3022 TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.
getModule());
3024 TaskEntry->setDoesNotRecurse();
3039 const auto *KmpTaskTWithPrivatesQTyRD =
3044 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
3046 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF);
3048 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
3054 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
3055 llvm::Value *PrivatesParam;
3056 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
3059 PrivatesLVal.getPointer(CGF), CGF.
VoidPtrTy);
3061 PrivatesParam = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
3064 llvm::Value *CommonArgs[] = {
3065 GtidParam, PartidParam, PrivatesParam, TaskPrivatesMap,
3071 std::end(CommonArgs));
3073 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
3076 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
3079 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
3082 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
3085 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
3088 CallArgs.push_back(LBParam);
3089 CallArgs.push_back(UBParam);
3090 CallArgs.push_back(StParam);
3091 CallArgs.push_back(LIParam);
3092 CallArgs.push_back(RParam);
3094 CallArgs.push_back(SharedsParam);
3107 QualType KmpTaskTWithPrivatesPtrQTy,
3108 QualType KmpTaskTWithPrivatesQTy) {
3116 Args.push_back(&GtidArg);
3117 Args.push_back(&TaskTypeArg);
3118 const auto &DestructorFnInfo =
3120 llvm::FunctionType *DestructorFnTy =
3124 auto *DestructorFn =
3125 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
3129 DestructorFn->setDoesNotRecurse();
3137 const auto *KmpTaskTWithPrivatesQTyRD =
3139 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
3141 for (
const auto *Field : FI->getType()->castAsRecordDecl()->fields()) {
3143 Field->getType().isDestructedType()) {
3145 CGF.
pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType());
3149 return DestructorFn;
3169 C,
nullptr, Loc,
nullptr,
3170 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
3172 Args.push_back(&TaskPrivatesArg);
3173 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>,
unsigned> PrivateVarsPos;
3174 unsigned Counter = 1;
3175 for (
const Expr *E :
Data.PrivateVars) {
3177 C,
nullptr, Loc,
nullptr,
3178 C.getPointerType(
C.getPointerType(E->
getType()))
3183 PrivateVarsPos[VD] = Counter;
3186 for (
const Expr *E :
Data.FirstprivateVars) {
3188 C,
nullptr, Loc,
nullptr,
3189 C.getPointerType(
C.getPointerType(E->
getType()))
3194 PrivateVarsPos[VD] = Counter;
3197 for (
const Expr *E :
Data.LastprivateVars) {
3199 C,
nullptr, Loc,
nullptr,
3200 C.getPointerType(
C.getPointerType(E->
getType()))
3205 PrivateVarsPos[VD] = Counter;
3211 Ty =
C.getPointerType(Ty);
3213 Ty =
C.getPointerType(Ty);
3215 C,
nullptr, Loc,
nullptr,
3216 C.getPointerType(
C.getPointerType(Ty)).withConst().withRestrict(),
3218 PrivateVarsPos[VD] = Counter;
3221 const auto &TaskPrivatesMapFnInfo =
3223 llvm::FunctionType *TaskPrivatesMapTy =
3227 auto *TaskPrivatesMap = llvm::Function::Create(
3228 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
3231 TaskPrivatesMapFnInfo);
3233 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
3234 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
3235 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
3239 TaskPrivatesMapFnInfo, Args, Loc, Loc);
3247 for (
const FieldDecl *Field : PrivatesQTyRD->fields()) {
3249 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
3253 RefLVal.getAddress(), RefLVal.getType()->castAs<
PointerType>());
3258 return TaskPrivatesMap;
3264 Address KmpTaskSharedsPtr, LValue TDBase,
3270 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
3284 if ((!IsTargetTask && !
Data.FirstprivateVars.empty() && ForDup) ||
3285 (IsTargetTask && KmpTaskSharedsPtr.
isValid())) {
3292 FI = FI->getType()->castAsRecordDecl()->field_begin();
3293 for (
const PrivateDataTy &Pair : Privates) {
3295 if (Pair.second.isLocalPrivate()) {
3299 const VarDecl *VD = Pair.second.PrivateCopy;
3304 if (
const VarDecl *Elem = Pair.second.PrivateElemInit) {
3305 const VarDecl *OriginalVD = Pair.second.Original;
3308 LValue SharedRefLValue;
3311 if (IsTargetTask && !SharedField) {
3315 ->getNumParams() == 0 &&
3318 ->getDeclContext()) &&
3319 "Expected artificial target data variable.");
3322 }
else if (ForDup) {
3325 SharedRefLValue.getAddress().withAlignment(
3326 C.getDeclAlign(OriginalVD)),
3328 SharedRefLValue.getTBAAInfo());
3330 Pair.second.Original->getCanonicalDecl()) > 0 ||
3332 SharedRefLValue = CGF.
EmitLValue(Pair.second.OriginalRef);
3335 InlinedOpenMPRegionRAII Region(
3338 SharedRefLValue = CGF.
EmitLValue(Pair.second.OriginalRef);
3349 PrivateLValue.getAddress(), SharedRefLValue.getAddress(),
Type,
3350 [&CGF, Elem,
Init, &CapturesInfo](
Address DestElement,
3353 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3354 InitScope.addPrivate(Elem, SrcElement);
3355 (void)InitScope.Privatize();
3357 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
3358 CGF, &CapturesInfo);
3359 CGF.EmitAnyExprToMem(Init, DestElement,
3360 Init->getType().getQualifiers(),
3366 InitScope.addPrivate(Elem, SharedRefLValue.getAddress());
3367 (void)InitScope.Privatize();
3383 bool InitRequired =
false;
3384 for (
const PrivateDataTy &Pair : Privates) {
3385 if (Pair.second.isLocalPrivate())
3387 const VarDecl *VD = Pair.second.PrivateCopy;
3389 InitRequired = InitRequired || (isa_and_nonnull<CXXConstructExpr>(
Init) &&
3394 return InitRequired;
3411 QualType KmpTaskTWithPrivatesPtrQTy,
3419 KmpTaskTWithPrivatesPtrQTy,
3422 KmpTaskTWithPrivatesPtrQTy,
3426 Args.push_back(&DstArg);
3427 Args.push_back(&SrcArg);
3428 Args.push_back(&LastprivArg);
3429 const auto &TaskDupFnInfo =
3433 auto *TaskDup = llvm::Function::Create(
3434 TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.
getModule());
3436 TaskDup->setDoesNotRecurse();
3446 auto LIFI = std::next(KmpTaskTQTyRD->
field_begin(), KmpTaskTLastIter);
3448 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
3456 assert(!Privates.empty());
3458 if (!
Data.FirstprivateVars.empty()) {
3463 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
3471 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
3472 SharedsTy, SharedsPtrTy,
Data, Privates,
true);
3482 for (
const PrivateDataTy &P : Privates) {
3483 if (P.second.isLocalPrivate())
3485 QualType Ty = P.second.Original->getType().getNonReferenceType();
3494class OMPIteratorGeneratorScope final
3496 CodeGenFunction &CGF;
3497 const OMPIteratorExpr *E =
nullptr;
3498 SmallVector<CodeGenFunction::JumpDest, 4> ContDests;
3499 SmallVector<CodeGenFunction::JumpDest, 4> ExitDests;
3500 OMPIteratorGeneratorScope() =
delete;
3501 OMPIteratorGeneratorScope(OMPIteratorGeneratorScope &) =
delete;
3504 OMPIteratorGeneratorScope(CodeGenFunction &CGF,
const OMPIteratorExpr *E)
3505 : CodeGenFunction::OMPPrivateScope(CGF), CGF(CGF), E(E) {
3508 SmallVector<llvm::Value *, 4> Uppers;
3510 Uppers.push_back(CGF.EmitScalarExpr(E->getHelper(I).Upper));
3511 const auto *VD = cast<VarDecl>(E->getIteratorDecl(I));
3512 addPrivate(VD, CGF.CreateMemTemp(VD->getType(), VD->getName()));
3513 const OMPIteratorHelperData &HelperData = E->getHelper(I);
3515 HelperData.CounterVD,
3516 CGF.CreateMemTemp(HelperData.CounterVD->getType(),
"counter.addr"));
3521 const OMPIteratorHelperData &HelperData = E->getHelper(I);
3523 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(HelperData.CounterVD),
3524 HelperData.CounterVD->getType());
3526 CGF.EmitStoreOfScalar(
3527 llvm::ConstantInt::get(CLVal.getAddress().getElementType(), 0),
3529 CodeGenFunction::JumpDest &ContDest =
3530 ContDests.emplace_back(CGF.getJumpDestInCurrentScope(
"iter.cont"));
3531 CodeGenFunction::JumpDest &ExitDest =
3532 ExitDests.emplace_back(CGF.getJumpDestInCurrentScope(
"iter.exit"));
3534 llvm::Value *N = Uppers[I];
3537 CGF.EmitBlock(ContDest.getBlock());
3539 CGF.EmitLoadOfScalar(CLVal, HelperData.CounterVD->getLocation());
3541 HelperData.CounterVD->getType()->isSignedIntegerOrEnumerationType()
3542 ? CGF.Builder.CreateICmpSLT(CVal, N)
3543 : CGF.Builder.CreateICmpULT(CVal, N);
3544 llvm::BasicBlock *BodyBB = CGF.createBasicBlock(
"iter.body");
3545 CGF.Builder.CreateCondBr(Cmp, BodyBB, ExitDest.getBlock());
3547 CGF.EmitBlock(BodyBB);
3549 CGF.EmitIgnoredExpr(HelperData.Update);
3552 ~OMPIteratorGeneratorScope() {
3557 const OMPIteratorHelperData &HelperData = E->
getHelper(I - 1);
3562 CGF.
EmitBlock(ExitDests[I - 1].getBlock(), I == 1);
3568static std::pair<llvm::Value *, llvm::Value *>
3570 const auto *OASE = dyn_cast<OMPArrayShapingExpr>(E);
3573 const Expr *
Base = OASE->getBase();
3578 llvm::Value *SizeVal;
3581 SizeVal = CGF.
getTypeSize(OASE->getBase()->getType()->getPointeeType());
3582 for (
const Expr *SE : OASE->getDimensions()) {
3586 SizeVal = CGF.
Builder.CreateNUWMul(SizeVal, Sz);
3588 }
else if (
const auto *ASE =
3591 Address UpAddrAddress = UpAddrLVal.getAddress();
3592 llvm::Value *UpAddr = CGF.
Builder.CreateConstGEP1_32(
3596 llvm::Value *UpIntPtr = CGF.
Builder.CreatePtrToInt(UpAddr, CGF.
SizeTy);
3597 SizeVal = CGF.
Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
3601 return std::make_pair(
Addr, SizeVal);
3606 QualType FlagsTy =
C.getIntTypeForBitwidth(32,
false);
3607 if (KmpTaskAffinityInfoTy.
isNull()) {
3609 C.buildImplicitRecord(
"kmp_task_affinity_info_t");
3615 KmpTaskAffinityInfoTy =
C.getCanonicalTagType(KmpAffinityInfoRD);
3622 llvm::Function *TaskFunction,
QualType SharedsTy,
3627 const auto *I =
Data.PrivateCopies.begin();
3628 for (
const Expr *E :
Data.PrivateVars) {
3630 Privates.emplace_back(
3636 I =
Data.FirstprivateCopies.begin();
3637 const auto *IElemInitRef =
Data.FirstprivateInits.begin();
3638 for (
const Expr *E :
Data.FirstprivateVars) {
3640 Privates.emplace_back(
3648 I =
Data.LastprivateCopies.begin();
3649 for (
const Expr *E :
Data.LastprivateVars) {
3651 Privates.emplace_back(
3659 Privates.emplace_back(
CGM.getPointerAlign(), PrivateHelpersTy(VD));
3661 Privates.emplace_back(
C.getDeclAlign(VD), PrivateHelpersTy(VD));
3663 llvm::stable_sort(Privates,
3664 [](
const PrivateDataTy &L,
const PrivateDataTy &R) {
3665 return L.first > R.first;
3667 QualType KmpInt32Ty =
C.getIntTypeForBitwidth(32, 1);
3678 assert((D.getDirectiveKind() == OMPD_task ||
3681 "Expected taskloop, task or target directive");
3688 const auto *KmpTaskTQTyRD =
KmpTaskTQTy->castAsRecordDecl();
3690 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
3693 C.getCanonicalTagType(KmpTaskTWithPrivatesQTyRD);
3694 QualType KmpTaskTWithPrivatesPtrQTy =
3695 C.getPointerType(KmpTaskTWithPrivatesQTy);
3696 llvm::Type *KmpTaskTWithPrivatesPtrTy = CGF.
Builder.getPtrTy(0);
3697 llvm::Value *KmpTaskTWithPrivatesTySize =
3699 QualType SharedsPtrTy =
C.getPointerType(SharedsTy);
3702 llvm::Value *TaskPrivatesMap =
nullptr;
3703 llvm::Type *TaskPrivatesMapTy =
3704 std::next(TaskFunction->arg_begin(), 3)->getType();
3705 if (!Privates.empty()) {
3706 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
3710 TaskPrivatesMap, TaskPrivatesMapTy);
3712 TaskPrivatesMap = llvm::ConstantPointerNull::get(
3718 CGM, Loc, D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
3719 KmpTaskTWithPrivatesQTy,
KmpTaskTQTy, SharedsPtrTy, TaskFunction,
3731 DestructorsFlag = 0x8,
3732 PriorityFlag = 0x20,
3733 DetachableFlag = 0x40,
3735 unsigned Flags =
Data.Tied ? TiedFlag : 0;
3736 bool NeedsCleanup =
false;
3737 if (!Privates.empty()) {
3741 Flags = Flags | DestructorsFlag;
3743 if (
Data.Priority.getInt())
3744 Flags = Flags | PriorityFlag;
3746 Flags = Flags | DetachableFlag;
3747 llvm::Value *TaskFlags =
3748 Data.Final.getPointer()
3749 ? CGF.
Builder.CreateSelect(
Data.Final.getPointer(),
3750 CGF.
Builder.getInt32(FinalFlag),
3752 : CGF.
Builder.getInt32(
Data.Final.getInt() ? FinalFlag : 0);
3753 TaskFlags = CGF.
Builder.CreateOr(TaskFlags, CGF.
Builder.getInt32(Flags));
3754 llvm::Value *SharedsSize =
CGM.getSize(
C.getTypeSizeInChars(SharedsTy));
3756 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
3759 llvm::Value *NewTask;
3766 llvm::Value *DeviceID;
3771 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
3772 AllocArgs.push_back(DeviceID);
3775 CGM.getModule(), OMPRTL___kmpc_omp_target_task_alloc),
3780 CGM.getModule(), OMPRTL___kmpc_omp_task_alloc),
3793 llvm::Value *Tid =
getThreadID(CGF, DC->getBeginLoc());
3794 Tid = CGF.
Builder.CreateIntCast(Tid, CGF.
IntTy,
false);
3797 CGM.getModule(), OMPRTL___kmpc_task_allow_completion_event),
3798 {Loc, Tid, NewTask});
3809 llvm::Value *NumOfElements =
nullptr;
3810 unsigned NumAffinities = 0;
3812 if (
const Expr *Modifier =
C->getModifier()) {
3814 for (
unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
3818 NumOfElements ? CGF.
Builder.CreateNUWMul(NumOfElements, Sz) : Sz;
3821 NumAffinities +=
C->varlist_size();
3826 enum RTLAffinityInfoFieldsTy { BaseAddr, Len, Flags };
3828 QualType KmpTaskAffinityInfoArrayTy;
3829 if (NumOfElements) {
3830 NumOfElements = CGF.
Builder.CreateNUWAdd(
3831 llvm::ConstantInt::get(CGF.
SizeTy, NumAffinities), NumOfElements);
3834 C.getIntTypeForBitwidth(
C.getTypeSize(
C.getSizeType()), 0),
3838 KmpTaskAffinityInfoArrayTy =
C.getVariableArrayType(
3846 NumOfElements = CGF.
Builder.CreateIntCast(NumOfElements, CGF.
Int32Ty,
3849 KmpTaskAffinityInfoArrayTy =
C.getConstantArrayType(
3851 llvm::APInt(
C.getTypeSize(
C.getSizeType()), NumAffinities),
nullptr,
3854 CGF.
CreateMemTemp(KmpTaskAffinityInfoArrayTy,
".affs.arr.addr");
3856 NumOfElements = llvm::ConstantInt::get(
CGM.Int32Ty, NumAffinities,
3863 bool HasIterator =
false;
3865 if (
C->getModifier()) {
3869 for (
const Expr *E :
C->varlist()) {
3878 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
3883 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
3897 const Expr *Modifier =
C->getModifier();
3900 OMPIteratorGeneratorScope IteratorScope(
3902 for (
const Expr *E :
C->varlist()) {
3912 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
3917 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
3919 Idx = CGF.
Builder.CreateNUWAdd(
3920 Idx, llvm::ConstantInt::get(Idx->getType(), 1));
3935 CGM.getModule(), OMPRTL___kmpc_omp_reg_task_with_affinity),
3936 {LocRef, GTid, NewTask, NumOfElements, AffinListPtr});
3938 llvm::Value *NewTaskNewTaskTTy =
3940 NewTask, KmpTaskTWithPrivatesPtrTy);
3942 KmpTaskTWithPrivatesQTy);
3953 *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds)),
3955 CGF.
Int8Ty,
CGM.getNaturalTypeAlignment(SharedsTy));
3962 if (!Privates.empty()) {
3964 SharedsTy, SharedsPtrTy,
Data, Privates,
3969 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
3970 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy,
Data, Privates,
3971 !
Data.LastprivateVars.empty());
3975 enum { Priority = 0, Destructors = 1 };
3977 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
3978 const auto *KmpCmplrdataUD = (*FI)->getType()->castAsRecordDecl();
3979 assert(KmpCmplrdataUD->isUnion());
3982 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
3983 KmpTaskTWithPrivatesQTy);
3986 Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
3992 if (
Data.Priority.getInt()) {
3994 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
3996 Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority));
3999 Result.NewTask = NewTask;
4000 Result.TaskEntry = TaskEntry;
4001 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
4003 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
4009 RTLDependenceKindTy DepKind;
4011 case OMPC_DEPEND_in:
4012 DepKind = RTLDependenceKindTy::DepIn;
4015 case OMPC_DEPEND_out:
4016 case OMPC_DEPEND_inout:
4017 DepKind = RTLDependenceKindTy::DepInOut;
4019 case OMPC_DEPEND_mutexinoutset:
4020 DepKind = RTLDependenceKindTy::DepMutexInOutSet;
4022 case OMPC_DEPEND_inoutset:
4023 DepKind = RTLDependenceKindTy::DepInOutSet;
4025 case OMPC_DEPEND_outallmemory:
4026 DepKind = RTLDependenceKindTy::DepOmpAllMem;
4028 case OMPC_DEPEND_source:
4029 case OMPC_DEPEND_sink:
4030 case OMPC_DEPEND_depobj:
4031 case OMPC_DEPEND_inoutallmemory:
4033 llvm_unreachable(
"Unknown task dependence type");
4041 FlagsTy =
C.getIntTypeForBitwidth(
C.getTypeSize(
C.BoolTy),
false);
4042 if (KmpDependInfoTy.
isNull()) {
4043 RecordDecl *KmpDependInfoRD =
C.buildImplicitRecord(
"kmp_depend_info");
4049 KmpDependInfoTy =
C.getCanonicalTagType(KmpDependInfoRD);
4053std::pair<llvm::Value *, LValue>
4066 CGF,
Base.getAddress(),
4067 llvm::ConstantInt::get(CGF.
IntPtrTy, -1,
true));
4073 *std::next(KmpDependInfoRD->field_begin(),
4074 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4076 return std::make_pair(NumDeps,
Base);
4080 llvm::PointerUnion<unsigned *, LValue *> Pos,
4090 OMPIteratorGeneratorScope IteratorScope(
4091 CGF, cast_or_null<OMPIteratorExpr>(
4092 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4094 for (
const Expr *E :
Data.DepExprs) {
4104 Size = llvm::ConstantInt::get(CGF.
SizeTy, 0);
4107 if (
unsigned *P = dyn_cast<unsigned *>(Pos)) {
4111 assert(E &&
"Expected a non-null expression");
4120 *std::next(KmpDependInfoRD->field_begin(),
4121 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4125 Base, *std::next(KmpDependInfoRD->field_begin(),
4126 static_cast<unsigned int>(RTLDependInfoFields::Len)));
4132 *std::next(KmpDependInfoRD->field_begin(),
4133 static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4135 llvm::ConstantInt::get(LLVMFlagsTy,
static_cast<unsigned int>(DepKind)),
4137 if (
unsigned *P = dyn_cast<unsigned *>(Pos)) {
4142 Idx = CGF.
Builder.CreateNUWAdd(Idx,
4143 llvm::ConstantInt::get(Idx->getType(), 1));
4152 assert(
Data.DepKind == OMPC_DEPEND_depobj &&
4153 "Expected depobj dependency kind.");
4158 OMPIteratorGeneratorScope IteratorScope(
4159 CGF, cast_or_null<OMPIteratorExpr>(
4160 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4162 for (
const Expr *E :
Data.DepExprs) {
4163 llvm::Value *NumDeps;
4166 std::tie(NumDeps,
Base) =
4170 C.getUIntPtrType());
4174 llvm::Value *Add = CGF.
Builder.CreateNUWAdd(PrevVal, NumDeps);
4176 SizeLVals.push_back(NumLVal);
4179 for (
unsigned I = 0, E = SizeLVals.size(); I < E; ++I) {
4182 Sizes.push_back(Size);
4192 assert(
Data.DepKind == OMPC_DEPEND_depobj &&
4193 "Expected depobj dependency kind.");
4196 OMPIteratorGeneratorScope IteratorScope(
4197 CGF, cast_or_null<OMPIteratorExpr>(
4198 Data.IteratorExpr ?
Data.IteratorExpr->IgnoreParenImpCasts()
4200 for (
const Expr *E :
Data.DepExprs) {
4201 llvm::Value *NumDeps;
4204 std::tie(NumDeps,
Base) =
4208 llvm::Value *Size = CGF.
Builder.CreateNUWMul(
4217 llvm::Value *Add = CGF.
Builder.CreateNUWAdd(Pos, NumDeps);
4233 llvm::Value *NumOfElements =
nullptr;
4234 unsigned NumDependencies = std::accumulate(
4235 Dependencies.begin(), Dependencies.end(), 0,
4237 return D.DepKind == OMPC_DEPEND_depobj
4239 : (V + (D.IteratorExpr ? 0 : D.DepExprs.size()));
4243 bool HasDepobjDeps =
false;
4244 bool HasRegularWithIterators =
false;
4245 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4246 llvm::Value *NumOfRegularWithIterators =
4247 llvm::ConstantInt::get(CGF.
IntPtrTy, 0);
4251 if (D.
DepKind == OMPC_DEPEND_depobj) {
4254 for (llvm::Value *Size : Sizes) {
4255 NumOfDepobjElements =
4256 CGF.
Builder.CreateNUWAdd(NumOfDepobjElements, Size);
4258 HasDepobjDeps =
true;
4263 if (
const auto *IE = cast_or_null<OMPIteratorExpr>(D.
IteratorExpr)) {
4264 llvm::Value *ClauseIteratorSpace =
4265 llvm::ConstantInt::get(CGF.
IntPtrTy, 1);
4269 ClauseIteratorSpace = CGF.
Builder.CreateNUWMul(Sz, ClauseIteratorSpace);
4271 llvm::Value *NumClauseDeps = CGF.
Builder.CreateNUWMul(
4272 ClauseIteratorSpace,
4274 NumOfRegularWithIterators =
4275 CGF.
Builder.CreateNUWAdd(NumOfRegularWithIterators, NumClauseDeps);
4276 HasRegularWithIterators =
true;
4282 if (HasDepobjDeps || HasRegularWithIterators) {
4283 NumOfElements = llvm::ConstantInt::get(
CGM.IntPtrTy, NumDependencies,
4285 if (HasDepobjDeps) {
4287 CGF.
Builder.CreateNUWAdd(NumOfDepobjElements, NumOfElements);
4289 if (HasRegularWithIterators) {
4291 CGF.
Builder.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements);
4294 Loc,
C.getIntTypeForBitwidth(64, 0),
4298 KmpDependInfoArrayTy =
4307 NumOfElements = CGF.
Builder.CreateIntCast(NumOfElements, CGF.
Int32Ty,
4310 KmpDependInfoArrayTy =
C.getConstantArrayType(
4316 NumOfElements = llvm::ConstantInt::get(
CGM.Int32Ty, NumDependencies,
4321 if (Dep.DepKind == OMPC_DEPEND_depobj || Dep.IteratorExpr)
4327 CGF.
CreateMemTemp(
C.getSizeType(),
"dep.counter.addr"),
C.getSizeType());
4330 if (Dep.DepKind == OMPC_DEPEND_depobj || !Dep.IteratorExpr)
4335 if (HasDepobjDeps) {
4337 if (Dep.DepKind != OMPC_DEPEND_depobj)
4344 return std::make_pair(NumOfElements, DependenciesArray);
4355 unsigned NumDependencies = Dependencies.
DepExprs.size();
4365 llvm::Value *NumDepsVal;
4367 if (
const auto *IE =
4368 cast_or_null<OMPIteratorExpr>(Dependencies.
IteratorExpr)) {
4369 NumDepsVal = llvm::ConstantInt::get(CGF.
SizeTy, 1);
4373 NumDepsVal = CGF.
Builder.CreateNUWMul(NumDepsVal, Sz);
4375 Size = CGF.
Builder.CreateNUWAdd(llvm::ConstantInt::get(CGF.
SizeTy, 1),
4379 llvm::Value *RecSize =
CGM.getSize(SizeInBytes);
4380 Size = CGF.
Builder.CreateNUWMul(Size, RecSize);
4384 QualType KmpDependInfoArrayTy =
C.getConstantArrayType(
4387 CharUnits Sz =
C.getTypeSizeInChars(KmpDependInfoArrayTy);
4389 NumDepsVal = llvm::ConstantInt::get(CGF.
IntPtrTy, NumDependencies);
4394 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4395 llvm::Value *Args[] = {ThreadID, Size, Allocator};
4399 CGM.getModule(), OMPRTL___kmpc_alloc),
4400 Args,
".dep.arr.addr");
4404 DependenciesArray =
Address(
Addr, KmpDependInfoLlvmTy, Align);
4410 *std::next(KmpDependInfoRD->field_begin(),
4411 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4413 llvm::PointerUnion<unsigned *, LValue *> Pos;
4430 return DependenciesArray;
4445 Addr.getElementType(),
Addr.emitRawPointer(CGF),
4446 llvm::ConstantInt::get(CGF.
IntPtrTy, -1,
true));
4451 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4452 llvm::Value *Args[] = {ThreadID, DepObjAddr, Allocator};
4456 CGM.getModule(), OMPRTL___kmpc_free),
4468 llvm::Value *NumDeps;
4479 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
4481 llvm::PHINode *ElementPHI =
4486 Base.getTBAAInfo());
4490 Base, *std::next(KmpDependInfoRD->field_begin(),
4491 static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4493 llvm::ConstantInt::get(LLVMFlagsTy,
static_cast<unsigned int>(DepKind)),
4497 llvm::Value *ElementNext =
4500 ElementPHI->addIncoming(ElementNext, CGF.
Builder.GetInsertBlock());
4501 llvm::Value *IsEmpty =
4502 CGF.
Builder.CreateICmpEQ(ElementNext, End,
"omp.isempty");
4503 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
4510 llvm::Function *TaskFunction,
4519 llvm::Value *NewTask =
Result.NewTask;
4520 llvm::Function *TaskEntry =
Result.TaskEntry;
4521 llvm::Value *NewTaskNewTaskTTy =
Result.NewTaskNewTaskTTy;
4526 llvm::Value *NumOfElements;
4527 std::tie(NumOfElements, DependenciesArray) =
4538 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
4539 llvm::Value *DepTaskArgs[7];
4540 if (!
Data.Dependences.empty()) {
4541 DepTaskArgs[0] = UpLoc;
4542 DepTaskArgs[1] = ThreadID;
4543 DepTaskArgs[2] = NewTask;
4544 DepTaskArgs[3] = NumOfElements;
4546 DepTaskArgs[5] = CGF.
Builder.getInt32(0);
4547 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4549 auto &&ThenCodeGen = [
this, &
Data, TDBase, KmpTaskTQTyRD, &TaskArgs,
4552 auto PartIdFI = std::next(KmpTaskTQTyRD->
field_begin(), KmpTaskTPartId);
4556 if (!
Data.Dependences.empty()) {
4559 CGM.getModule(), OMPRTL___kmpc_omp_task_with_deps),
4563 CGM.getModule(), OMPRTL___kmpc_omp_task),
4569 Region->emitUntiedSwitch(CGF);
4572 llvm::Value *DepWaitTaskArgs[7];
4573 if (!
Data.Dependences.empty()) {
4574 DepWaitTaskArgs[0] = UpLoc;
4575 DepWaitTaskArgs[1] = ThreadID;
4576 DepWaitTaskArgs[2] = NumOfElements;
4578 DepWaitTaskArgs[4] = CGF.
Builder.getInt32(0);
4579 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4580 DepWaitTaskArgs[6] =
4581 llvm::ConstantInt::get(CGF.
Int32Ty,
Data.HasNowaitClause);
4583 auto &M =
CGM.getModule();
4584 auto &&ElseCodeGen = [
this, &M, &TaskArgs, ThreadID, NewTaskNewTaskTTy,
4585 TaskEntry, &
Data, &DepWaitTaskArgs,
4592 if (!
Data.Dependences.empty())
4594 M, OMPRTL___kmpc_omp_taskwait_deps_51),
4597 auto &&
CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
4600 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
4601 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
4610 CommonActionTy Action(
OMPBuilder.getOrCreateRuntimeFunction(
4611 M, OMPRTL___kmpc_omp_task_begin_if0),
4614 M, OMPRTL___kmpc_omp_task_complete_if0),
4630 llvm::Function *TaskFunction,
4650 IfVal = llvm::ConstantInt::getSigned(CGF.
IntTy, 1);
4655 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
4662 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
4669 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
4677 *std::next(
Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
4678 if (
Data.Reductions) {
4684 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
4693 llvm::ConstantInt::getSigned(
4695 llvm::ConstantInt::getSigned(
4697 ?
Data.Schedule.getInt() ? NumTasks : Grainsize
4699 Data.Schedule.getPointer()
4702 : llvm::ConstantInt::get(CGF.
Int64Ty, 0)};
4703 if (
Data.HasModifier)
4704 TaskArgs.push_back(llvm::ConstantInt::get(CGF.
Int32Ty, 1));
4706 TaskArgs.push_back(
Result.TaskDupFn
4709 : llvm::ConstantPointerNull::get(CGF.
VoidPtrTy));
4711 CGM.getModule(),
Data.HasModifier
4712 ? OMPRTL___kmpc_taskloop_5
4713 : OMPRTL___kmpc_taskloop),
4730 const Expr *,
const Expr *)> &RedOpGen,
4731 const Expr *XExpr =
nullptr,
const Expr *EExpr =
nullptr,
4732 const Expr *UpExpr =
nullptr) {
4740 llvm::Value *NumElements = CGF.
emitArrayLength(ArrayTy, ElementTy, LHSAddr);
4745 llvm::Value *LHSEnd =
4750 llvm::Value *IsEmpty =
4751 CGF.
Builder.CreateICmpEQ(LHSBegin, LHSEnd,
"omp.arraycpy.isempty");
4752 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
4755 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
4760 llvm::PHINode *RHSElementPHI = CGF.
Builder.CreatePHI(
4761 RHSBegin->getType(), 2,
"omp.arraycpy.srcElementPast");
4762 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
4767 llvm::PHINode *LHSElementPHI = CGF.
Builder.CreatePHI(
4768 LHSBegin->getType(), 2,
"omp.arraycpy.destElementPast");
4769 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
4776 Scope.addPrivate(LHSVar, LHSElementCurrent);
4777 Scope.addPrivate(RHSVar, RHSElementCurrent);
4779 RedOpGen(CGF, XExpr, EExpr, UpExpr);
4780 Scope.ForceCleanup();
4783 llvm::Value *LHSElementNext = CGF.
Builder.CreateConstGEP1_32(
4785 "omp.arraycpy.dest.element");
4786 llvm::Value *RHSElementNext = CGF.
Builder.CreateConstGEP1_32(
4788 "omp.arraycpy.src.element");
4791 CGF.
Builder.CreateICmpEQ(LHSElementNext, LHSEnd,
"omp.arraycpy.done");
4792 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
4793 LHSElementPHI->addIncoming(LHSElementNext, CGF.
Builder.GetInsertBlock());
4794 RHSElementPHI->addIncoming(RHSElementNext, CGF.
Builder.GetInsertBlock());
4804 const Expr *ReductionOp) {
4805 if (
const auto *CE = dyn_cast<CallExpr>(ReductionOp))
4806 if (
const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
4807 if (
const auto *DRE =
4808 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
4809 if (
const auto *DRD =
4810 dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
4811 std::pair<llvm::Function *, llvm::Function *>
Reduction =
4822 StringRef ReducerName,
SourceLocation Loc, llvm::Type *ArgsElemType,
4833 Args.push_back(&LHSArg);
4834 Args.push_back(&RHSArg);
4836 CGM.getTypes().arrangeBuiltinFunctionDeclaration(
C.VoidTy, Args);
4838 auto *Fn = llvm::Function::Create(
CGM.getTypes().GetFunctionType(CGFI),
4839 llvm::GlobalValue::InternalLinkage, Name,
4842 Fn->setDoesNotRecurse();
4861 const auto *IPriv = Privates.begin();
4863 for (
unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
4864 const auto *RHSVar =
4867 const auto *LHSVar =
4870 QualType PrivTy = (*IPriv)->getType();
4885 IPriv = Privates.begin();
4886 const auto *ILHS = LHSExprs.begin();
4887 const auto *IRHS = RHSExprs.begin();
4888 for (
const Expr *E : ReductionOps) {
4889 if ((*IPriv)->getType()->isArrayType()) {
4894 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
4896 emitReductionCombiner(CGF, E);
4906 Scope.ForceCleanup();
4912 const Expr *ReductionOp,
4913 const Expr *PrivateRef,
4921 CGF, PrivateRef->
getType(), LHSVar, RHSVar,
4923 emitReductionCombiner(CGF, ReductionOp);
4932 llvm::StringRef Prefix,
const Expr *Ref);
4936 const Expr *LHSExprs,
const Expr *RHSExprs,
const Expr *ReductionOps) {
4963 std::string ReductionVarNameStr;
4965 ReductionVarNameStr =
4968 ReductionVarNameStr =
"unnamed_priv_var";
4971 std::string SharedName =
4972 CGM.getOpenMPRuntime().getName({
"internal_pivate_", ReductionVarNameStr});
4973 llvm::GlobalVariable *SharedVar =
OMPBuilder.getOrCreateInternalVariable(
4974 LLVMType,
".omp.reduction." + SharedName);
4976 SharedVar->setAlignment(
4984 llvm::Value *BarrierArgs[] = {BarrierLoc, ThreadId};
4989 llvm::Value *IsWorker = CGF.
Builder.CreateICmpEQ(
4990 ThreadId, llvm::ConstantInt::get(ThreadId->getType(), 0));
4991 CGF.
Builder.CreateCondBr(IsWorker, InitBB, InitEndBB);
4995 auto EmitSharedInit = [&]() {
4998 std::pair<llvm::Function *, llvm::Function *> FnPair =
5000 llvm::Function *InitializerFn = FnPair.second;
5001 if (InitializerFn) {
5002 if (
const auto *CE =
5003 dyn_cast<CallExpr>(UDRInitExpr->IgnoreParenImpCasts())) {
5010 LocalScope.addPrivate(OutVD, SharedResult);
5012 (void)LocalScope.Privatize();
5013 if (
const auto *OVE = dyn_cast<OpaqueValueExpr>(
5014 CE->getCallee()->IgnoreParenImpCasts())) {
5040 if (
const auto *DRE = dyn_cast<DeclRefExpr>(Privates)) {
5041 if (
const auto *VD = dyn_cast<VarDecl>(DRE->getDecl())) {
5052 CGF.
Builder.CreateBr(InitEndBB);
5056 CGM.getModule(), OMPRTL___kmpc_barrier),
5059 const Expr *ReductionOp = ReductionOps;
5064 auto EmitCriticalReduction = [&](
auto ReductionGen) {
5065 std::string CriticalName =
getName({
"reduction_critical"});
5073 std::pair<llvm::Function *, llvm::Function *> FnPair =
5076 if (
const auto *CE = dyn_cast<CallExpr>(ReductionOp)) {
5088 (void)LocalScope.Privatize();
5093 EmitCriticalReduction(ReductionGen);
5098 if (
const auto *Cleanup = dyn_cast<ExprWithCleanups>(ReductionClauseExpr))
5101 const Expr *AssignRHS =
nullptr;
5102 if (
const auto *BinOp = dyn_cast<BinaryOperator>(ReductionClauseExpr)) {
5103 if (BinOp->getOpcode() == BO_Assign)
5104 AssignRHS = BinOp->getRHS();
5105 }
else if (
const auto *OpCall =
5106 dyn_cast<CXXOperatorCallExpr>(ReductionClauseExpr)) {
5107 if (OpCall->getOperator() == OO_Equal)
5108 AssignRHS = OpCall->getArg(1);
5112 "Private Variable Reduction : Invalid ReductionOp expression");
5117 const auto *OmpOutDRE =
5119 const auto *OmpInDRE =
5122 OmpOutDRE && OmpInDRE &&
5123 "Private Variable Reduction : LHSExpr/RHSExpr must be DeclRefExprs");
5127 LocalScope.addPrivate(OmpOutVD, SharedLV.
getAddress());
5128 LocalScope.addPrivate(OmpInVD, LHSLV.
getAddress());
5129 (void)LocalScope.Privatize();
5133 EmitCriticalReduction(ReductionGen);
5137 CGM.getModule(), OMPRTL___kmpc_barrier),
5143 llvm::Value *FinalResultVal =
nullptr;
5147 FinalResultAddr = SharedResult;
5161 CGM.getModule(), OMPRTL___kmpc_barrier),
5172 EmitCriticalReduction(OriginalListCombiner);
5224 if (SimpleReduction) {
5226 const auto *IPriv = OrgPrivates.begin();
5227 const auto *ILHS = OrgLHSExprs.begin();
5228 const auto *IRHS = OrgRHSExprs.begin();
5229 for (
const Expr *E : OrgReductionOps) {
5242 FilteredRHSExprs, FilteredReductionOps;
5243 for (
unsigned I : llvm::seq<unsigned>(
5244 std::min(OrgReductionOps.size(), OrgLHSExprs.size()))) {
5246 FilteredPrivates.emplace_back(OrgPrivates[I]);
5247 FilteredLHSExprs.emplace_back(OrgLHSExprs[I]);
5248 FilteredRHSExprs.emplace_back(OrgRHSExprs[I]);
5249 FilteredReductionOps.emplace_back(OrgReductionOps[I]);
5261 auto Size = RHSExprs.size();
5262 for (
const Expr *E : Privates) {
5267 llvm::APInt ArraySize(32, Size);
5268 QualType ReductionArrayTy =
C.getConstantArrayType(
5272 CGF.
CreateMemTemp(ReductionArrayTy,
".omp.reduction.red_list");
5273 const auto *IPriv = Privates.begin();
5275 for (
unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
5281 if ((*IPriv)->getType()->isVariablyModifiedType()) {
5285 llvm::Value *Size = CGF.
Builder.CreateIntCast(
5298 Privates, LHSExprs, RHSExprs, ReductionOps);
5301 std::string Name =
getName({
"reduction"});
5308 llvm::Value *ReductionArrayTySize = CGF.
getTypeSize(ReductionArrayTy);
5311 llvm::Value *Args[] = {
5314 CGF.
Builder.getInt32(RHSExprs.size()),
5315 ReductionArrayTySize,
5323 WithNowait ? OMPRTL___kmpc_reduce_nowait : OMPRTL___kmpc_reduce),
5327 llvm::BasicBlock *DefaultBB = CGF.
createBasicBlock(
".omp.reduction.default");
5328 llvm::SwitchInst *SwInst =
5329 CGF.
Builder.CreateSwitch(Res, DefaultBB, 2);
5338 SwInst->addCase(CGF.
Builder.getInt32(1), Case1BB);
5342 llvm::Value *EndArgs[] = {
5347 auto &&
CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps](
5350 const auto *IPriv = Privates.begin();
5351 const auto *ILHS = LHSExprs.begin();
5352 const auto *IRHS = RHSExprs.begin();
5353 for (
const Expr *E : ReductionOps) {
5362 CommonActionTy Action(
5365 CGM.getModule(), WithNowait ? OMPRTL___kmpc_end_reduce_nowait
5366 : OMPRTL___kmpc_end_reduce),
5379 SwInst->addCase(CGF.
Builder.getInt32(2), Case2BB);
5382 auto &&AtomicCodeGen = [Loc, Privates, LHSExprs, RHSExprs, ReductionOps](
5384 const auto *ILHS = LHSExprs.begin();
5385 const auto *IRHS = RHSExprs.begin();
5386 const auto *IPriv = Privates.begin();
5387 for (
const Expr *E : ReductionOps) {
5388 const Expr *XExpr =
nullptr;
5389 const Expr *EExpr =
nullptr;
5390 const Expr *UpExpr =
nullptr;
5392 if (
const auto *BO = dyn_cast<BinaryOperator>(E)) {
5393 if (BO->getOpcode() == BO_Assign) {
5394 XExpr = BO->getLHS();
5395 UpExpr = BO->getRHS();
5399 const Expr *RHSExpr = UpExpr;
5402 if (
const auto *ACO = dyn_cast<AbstractConditionalOperator>(
5406 RHSExpr = ACO->getCond();
5408 if (
const auto *BORHS =
5410 EExpr = BORHS->getRHS();
5411 BO = BORHS->getOpcode();
5416 auto &&AtomicRedGen = [BO, VD,
5418 const Expr *EExpr,
const Expr *UpExpr) {
5419 LValue X = CGF.EmitLValue(XExpr);
5422 E = CGF.EmitAnyExpr(EExpr);
5423 CGF.EmitOMPAtomicSimpleUpdateExpr(
5425 llvm::AtomicOrdering::Monotonic, Loc,
5426 [&CGF, UpExpr, VD, Loc](
RValue XRValue) {
5428 Address LHSTemp = CGF.CreateMemTemp(VD->getType());
5429 CGF.emitOMPSimpleStore(
5430 CGF.MakeAddrLValue(LHSTemp, VD->
getType()), XRValue,
5431 VD->getType().getNonReferenceType(), Loc);
5434 return CGF.EmitAnyExpr(UpExpr);
5437 if ((*IPriv)->getType()->isArrayType()) {
5439 const auto *RHSVar =
5442 AtomicRedGen, XExpr, EExpr, UpExpr);
5445 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5452 std::string Name = RT.
getName({
"atomic_reduction"});
5461 if ((*IPriv)->getType()->isArrayType()) {
5462 const auto *LHSVar =
5464 const auto *RHSVar =
5469 CritRedGen(CGF,
nullptr,
nullptr,
nullptr);
5480 llvm::Value *EndArgs[] = {
5485 CommonActionTy Action(
nullptr, {},
5487 CGM.getModule(), OMPRTL___kmpc_end_reduce),
5497 assert(OrgLHSExprs.size() == OrgPrivates.size() &&
5498 "PrivateVarReduction: Privates size mismatch");
5499 assert(OrgLHSExprs.size() == OrgReductionOps.size() &&
5500 "PrivateVarReduction: ReductionOps size mismatch");
5501 for (
unsigned I : llvm::seq<unsigned>(
5502 std::min(OrgReductionOps.size(), OrgLHSExprs.size()))) {
5505 OrgRHSExprs[I], OrgReductionOps[I]);
5514 llvm::raw_svector_ostream Out(Buffer);
5522 Out << Prefix << Name <<
"_"
5524 return std::string(Out.str());
5546 Args.emplace_back(&Param);
5547 Args.emplace_back(&ParamOrig);
5548 const auto &FnInfo =
5552 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5555 Fn->setDoesNotRecurse();
5562 llvm::Value *Size =
nullptr;
5605 const Expr *ReductionOp,
5607 const Expr *PrivateRef) {
5616 Args.emplace_back(&ParamInOut);
5617 Args.emplace_back(&ParamIn);
5618 const auto &FnInfo =
5622 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5625 Fn->setDoesNotRecurse();
5628 llvm::Value *Size =
nullptr;
5649 C.getPointerType(LHSVD->getType())->castAs<
PointerType>()));
5656 C.getPointerType(RHSVD->getType())->castAs<
PointerType>()));
5685 Args.emplace_back(&Param);
5686 const auto &FnInfo =
5690 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5693 Fn->setDoesNotRecurse();
5698 llvm::Value *Size =
nullptr;
5733 RecordDecl *RD =
C.buildImplicitRecord(
"kmp_taskred_input_t");
5742 C, RD,
C.getIntTypeForBitwidth(32,
false));
5745 unsigned Size =
Data.ReductionVars.size();
5746 llvm::APInt ArraySize(64, Size);
5748 C.getConstantArrayType(RDType, ArraySize,
nullptr,
5753 Data.ReductionCopies,
Data.ReductionOps);
5754 for (
unsigned Cnt = 0; Cnt < Size; ++Cnt) {
5756 llvm::Value *Idxs[] = {llvm::ConstantInt::get(
CGM.SizeTy, 0),
5757 llvm::ConstantInt::get(
CGM.SizeTy, Cnt)};
5773 llvm::Value *SizeValInChars;
5774 llvm::Value *SizeVal;
5775 std::tie(SizeValInChars, SizeVal) = RCG.
getSizes(Cnt);
5781 bool DelayedCreation = !!SizeVal;
5782 SizeValInChars = CGF.
Builder.CreateIntCast(SizeValInChars,
CGM.SizeTy,
5793 llvm::Value *FiniAddr =
5794 Fini ? Fini : llvm::ConstantPointerNull::get(
CGM.VoidPtrTy);
5799 CGM, Loc, RCG, Cnt,
Data.ReductionOps[Cnt], LHSExprs[Cnt],
5800 RHSExprs[Cnt],
Data.ReductionCopies[Cnt]);
5804 if (DelayedCreation) {
5806 llvm::ConstantInt::get(
CGM.Int32Ty, 1,
true),
5811 if (
Data.IsReductionWithTaskMod) {
5817 llvm::Value *Args[] = {
5819 llvm::ConstantInt::get(
CGM.IntTy,
Data.IsWorksharingReduction ? 1 : 0,
5821 llvm::ConstantInt::get(
CGM.IntTy, Size,
true),
5826 CGM.getModule(), OMPRTL___kmpc_taskred_modifier_init),
5830 llvm::Value *Args[] = {
5833 llvm::ConstantInt::get(
CGM.IntTy, Size,
true),
5837 CGM.getModule(), OMPRTL___kmpc_taskred_init),
5843 bool IsWorksharingReduction) {
5849 llvm::Value *Args[] = {IdentTLoc, GTid,
5850 llvm::ConstantInt::get(
CGM.IntTy,
5851 IsWorksharingReduction ? 1 : 0,
5855 CGM.getModule(), OMPRTL___kmpc_task_reduction_modifier_fini),
5867 llvm::Value *SizeVal = CGF.
Builder.CreateIntCast(Sizes.second,
CGM.SizeTy,
5870 CGF,
CGM.getContext().getSizeType(),
5878 llvm::Value *ReductionsPtr,
5891 CGM.getModule(), OMPRTL___kmpc_task_reduction_get_th_data),
5907 auto &M =
CGM.getModule();
5909 llvm::Value *NumOfElements;
5910 std::tie(NumOfElements, DependenciesArray) =
5912 if (!
Data.Dependences.empty()) {
5913 llvm::Value *DepWaitTaskArgs[7];
5914 DepWaitTaskArgs[0] = UpLoc;
5915 DepWaitTaskArgs[1] = ThreadID;
5916 DepWaitTaskArgs[2] = NumOfElements;
5918 DepWaitTaskArgs[4] = CGF.
Builder.getInt32(0);
5919 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
5920 DepWaitTaskArgs[6] =
5921 llvm::ConstantInt::get(CGF.
Int32Ty,
Data.HasNowaitClause);
5930 M, OMPRTL___kmpc_omp_taskwait_deps_51),
5937 llvm::Value *Args[] = {UpLoc, ThreadID};
5940 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_omp_taskwait),
5945 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
5946 Region->emitUntiedSwitch(CGF);
5955 InlinedOpenMPRegionRAII Region(CGF,
CodeGen, InnerKind, HasCancel,
5956 InnerKind != OMPD_critical &&
5957 InnerKind != OMPD_master &&
5958 InnerKind != OMPD_masked);
5973 RTCancelKind CancelKind = CancelNoreq;
5974 if (CancelRegion == OMPD_parallel)
5975 CancelKind = CancelParallel;
5976 else if (CancelRegion == OMPD_for)
5977 CancelKind = CancelLoop;
5978 else if (CancelRegion == OMPD_sections)
5979 CancelKind = CancelSections;
5981 assert(CancelRegion == OMPD_taskgroup);
5982 CancelKind = CancelTaskgroup;
5994 if (
auto *OMPRegionInfo =
5998 if (CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) {
5999 llvm::Value *Args[] = {
6005 CGM.getModule(), OMPRTL___kmpc_cancellationpoint),
6014 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
6016 if (CancelRegion == OMPD_parallel)
6034 auto &M =
CGM.getModule();
6035 if (
auto *OMPRegionInfo =
6037 auto &&ThenGen = [
this, &M, Loc, CancelRegion,
6040 llvm::Value *Args[] = {
6044 llvm::Value *
Result = CGF.EmitRuntimeCall(
6045 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_cancel), Args);
6050 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
".cancel.exit");
6051 llvm::BasicBlock *ContBB = CGF.createBasicBlock(
".cancel.continue");
6052 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(
Result);
6053 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
6054 CGF.EmitBlock(ExitBB);
6055 if (CancelRegion == OMPD_parallel)
6059 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
6060 CGF.EmitBranchThroughCleanup(CancelDest);
6061 CGF.EmitBlock(ContBB,
true);
6079 OMPUsesAllocatorsActionTy(
6080 ArrayRef<std::pair<const Expr *, const Expr *>> Allocators)
6081 : Allocators(Allocators) {}
6085 for (
const auto &AllocatorData : Allocators) {
6087 CGF, AllocatorData.first, AllocatorData.second);
6090 void Exit(CodeGenFunction &CGF)
override {
6093 for (
const auto &AllocatorData : Allocators) {
6095 AllocatorData.first);
6103 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6105 assert(!ParentName.empty() &&
"Invalid target entry parent name!");
6109 for (
unsigned I = 0, E =
C->getNumberOfAllocators(); I < E; ++I) {
6116 OMPUsesAllocatorsActionTy UsesAllocatorAction(Allocators);
6117 CodeGen.setAction(UsesAllocatorAction);
6123 const Expr *Allocator,
6124 const Expr *AllocatorTraits) {
6126 ThreadId = CGF.
Builder.CreateIntCast(ThreadId, CGF.
IntTy,
true);
6128 llvm::Value *MemSpaceHandle = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
6129 llvm::Value *NumTraits = llvm::ConstantInt::get(
6133 .getLimitedValue());
6140 llvm::Value *Traits =
Addr.emitRawPointer(CGF);
6142 llvm::Value *AllocatorVal =
6144 CGM.getModule(), OMPRTL___kmpc_init_allocator),
6145 {ThreadId, MemSpaceHandle, NumTraits, Traits});
6157 const Expr *Allocator) {
6159 ThreadId = CGF.
Builder.CreateIntCast(ThreadId, CGF.
IntTy,
true);
6161 llvm::Value *AllocatorVal =
6168 OMPRTL___kmpc_destroy_allocator),
6169 {ThreadId, AllocatorVal});
6174 llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs) {
6175 assert(Attrs.MaxTeams.size() == 1 && Attrs.MaxThreads.size() == 1 &&
6176 "invalid default attrs structure");
6177 int32_t &MaxTeamsVal = Attrs.MaxTeams.front();
6178 int32_t &MaxThreadsVal = Attrs.MaxThreads.front();
6185 for (
auto *A :
C->getAttrs()) {
6186 int32_t AttrMinThreadsVal = 1, AttrMaxThreadsVal = -1;
6187 int32_t AttrMinBlocksVal = 1, AttrMaxBlocksVal = -1;
6188 if (
auto *
Attr = dyn_cast<CUDALaunchBoundsAttr>(A))
6189 CGM.handleCUDALaunchBoundsAttr(
nullptr,
Attr, &AttrMaxThreadsVal,
6190 &AttrMinBlocksVal, &AttrMaxBlocksVal);
6191 else if (
auto *
Attr = dyn_cast<AMDGPUFlatWorkGroupSizeAttr>(A))
6192 CGM.handleAMDGPUFlatWorkGroupSizeAttr(
6193 nullptr,
Attr,
nullptr, &AttrMinThreadsVal,
6194 &AttrMaxThreadsVal);
6198 Attrs.MinThreads = std::max(Attrs.MinThreads, AttrMinThreadsVal);
6199 if (AttrMaxThreadsVal > 0)
6200 MaxThreadsVal = MaxThreadsVal > 0
6201 ? std::min(MaxThreadsVal, AttrMaxThreadsVal)
6202 : AttrMaxThreadsVal;
6203 Attrs.MinTeams = std::max(Attrs.MinTeams, AttrMinBlocksVal);
6204 if (AttrMaxBlocksVal > 0)
6205 MaxTeamsVal = MaxTeamsVal > 0 ? std::min(MaxTeamsVal, AttrMaxBlocksVal)
6213 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6216 llvm::TargetRegionEntryInfo EntryInfo =
6220 llvm::OpenMPIRBuilder::FunctionGenCallback &&GenerateOutlinedFunction =
6221 [&CGF, &D, &
CodeGen](StringRef EntryFnName) {
6222 const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target);
6224 CGOpenMPTargetRegionInfo CGInfo(CS,
CodeGen, EntryFnName);
6229 cantFail(
OMPBuilder.emitTargetRegionFunction(
6230 EntryInfo, GenerateOutlinedFunction, IsOffloadEntry, OutlinedFn,
6236 CGM.getTargetCodeGenInfo().setTargetAttributes(
nullptr, OutlinedFn,
CGM);
6239 for (
auto *A :
C->getAttrs()) {
6240 if (
auto *
Attr = dyn_cast<AMDGPUWavesPerEUAttr>(A))
6241 CGM.handleAMDGPUWavesPerEUAttr(OutlinedFn,
Attr);
6259 while (
const auto *
C = dyn_cast_or_null<CompoundStmt>(Child)) {
6261 for (
const Stmt *S :
C->body()) {
6262 if (
const auto *E = dyn_cast<Expr>(S)) {
6271 if (
const auto *DS = dyn_cast<DeclStmt>(S)) {
6272 if (llvm::all_of(DS->decls(), [](
const Decl *D) {
6273 if (isa<EmptyDecl>(D) || isa<DeclContext>(D) ||
6274 isa<TypeDecl>(D) || isa<PragmaCommentDecl>(D) ||
6275 isa<PragmaDetectMismatchDecl>(D) || isa<UsingDecl>(D) ||
6276 isa<UsingDirectiveDecl>(D) ||
6277 isa<OMPDeclareReductionDecl>(D) ||
6278 isa<OMPThreadPrivateDecl>(D) || isa<OMPAllocateDecl>(D))
6280 const auto *VD = dyn_cast<VarDecl>(D);
6283 return VD->hasGlobalStorage() || !VD->isUsed();
6293 Child = Child->IgnoreContainers();
6300 int32_t &MaxTeamsVal) {
6304 "Expected target-based executable directive.");
6305 switch (DirectiveKind) {
6307 const auto *CS = D.getInnermostCapturedStmt();
6310 const Stmt *ChildStmt =
6312 if (
const auto *NestedDir =
6313 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
6322 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();
6325 MinTeamsVal = MaxTeamsVal = 0;
6328 MinTeamsVal = MaxTeamsVal = 1;
6332 MinTeamsVal = MaxTeamsVal = -1;
6335 case OMPD_target_teams_loop:
6336 case OMPD_target_teams:
6337 case OMPD_target_teams_distribute:
6338 case OMPD_target_teams_distribute_simd:
6339 case OMPD_target_teams_distribute_parallel_for:
6340 case OMPD_target_teams_distribute_parallel_for_simd: {
6342 const Expr *NumTeams =
6346 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();
6349 MinTeamsVal = MaxTeamsVal = 0;
6352 case OMPD_target_parallel:
6353 case OMPD_target_parallel_for:
6354 case OMPD_target_parallel_for_simd:
6355 case OMPD_target_parallel_loop:
6356 case OMPD_target_simd:
6357 MinTeamsVal = MaxTeamsVal = 1;
6361 case OMPD_parallel_for:
6362 case OMPD_parallel_loop:
6363 case OMPD_parallel_master:
6364 case OMPD_parallel_sections:
6366 case OMPD_parallel_for_simd:
6368 case OMPD_cancellation_point:
6370 case OMPD_threadprivate:
6381 case OMPD_taskyield:
6384 case OMPD_taskgroup:
6390 case OMPD_target_data:
6391 case OMPD_target_exit_data:
6392 case OMPD_target_enter_data:
6393 case OMPD_distribute:
6394 case OMPD_distribute_simd:
6395 case OMPD_distribute_parallel_for:
6396 case OMPD_distribute_parallel_for_simd:
6397 case OMPD_teams_distribute:
6398 case OMPD_teams_distribute_simd:
6399 case OMPD_teams_distribute_parallel_for:
6400 case OMPD_teams_distribute_parallel_for_simd:
6401 case OMPD_target_update:
6402 case OMPD_declare_simd:
6403 case OMPD_declare_variant:
6404 case OMPD_begin_declare_variant:
6405 case OMPD_end_declare_variant:
6406 case OMPD_declare_target:
6407 case OMPD_end_declare_target:
6408 case OMPD_declare_reduction:
6409 case OMPD_declare_mapper:
6411 case OMPD_taskloop_simd:
6412 case OMPD_master_taskloop:
6413 case OMPD_master_taskloop_simd:
6414 case OMPD_parallel_master_taskloop:
6415 case OMPD_parallel_master_taskloop_simd:
6417 case OMPD_metadirective:
6423 llvm_unreachable(
"Unexpected directive kind.");
6429 "Clauses associated with the teams directive expected to be emitted "
6430 "only for the host!");
6432 int32_t MinNT = -1, MaxNT = -1;
6433 const Expr *NumTeams =
6435 if (NumTeams !=
nullptr) {
6438 switch (DirectiveKind) {
6440 const auto *CS = D.getInnermostCapturedStmt();
6441 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6445 return Bld.CreateIntCast(NumTeamsVal, CGF.
Int32Ty,
6448 case OMPD_target_teams:
6449 case OMPD_target_teams_distribute:
6450 case OMPD_target_teams_distribute_simd:
6451 case OMPD_target_teams_distribute_parallel_for:
6452 case OMPD_target_teams_distribute_parallel_for_simd: {
6456 return Bld.CreateIntCast(NumTeamsVal, CGF.
Int32Ty,
6464 assert(MinNT == MaxNT &&
"Num threads ranges require handling here.");
6465 return llvm::ConstantInt::get(CGF.
Int32Ty, MinNT);
6473 const Expr **E, int32_t &UpperBound,
6474 bool UpperBoundOnly, llvm::Value **CondVal) {
6477 const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child);
6484 if (CondVal && Dir->hasClausesOfKind<
OMPIfClause>()) {
6485 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6488 for (
const auto *
C : Dir->getClausesOfKind<
OMPIfClause>()) {
6489 if (
C->getNameModifier() == OMPD_unknown ||
6490 C->getNameModifier() == OMPD_parallel) {
6505 if (
const auto *PreInit =
6507 for (
const auto *I : PreInit->decls()) {
6508 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6524 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6526 const auto *NumThreadsClause =
6528 const Expr *NTExpr = NumThreadsClause->getNumThreads();
6529 if (NTExpr->isIntegerConstantExpr(CGF.
getContext()))
6530 if (
auto Constant = NTExpr->getIntegerConstantExpr(CGF.
getContext()))
6533 ? Constant->getZExtValue()
6534 : std::min(UpperBound,
6535 static_cast<int32_t
>(Constant->getZExtValue()));
6538 if (UpperBound == -1)
6543 if (
const auto *PreInit =
6544 cast_or_null<DeclStmt>(NumThreadsClause->getPreInitStmt())) {
6545 for (
const auto *I : PreInit->decls()) {
6546 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6565 bool UpperBoundOnly, llvm::Value **CondVal,
const Expr **ThreadLimitExpr) {
6566 assert((!CGF.
getLangOpts().OpenMPIsTargetDevice || UpperBoundOnly) &&
6567 "Clauses associated with the teams directive expected to be emitted "
6568 "only for the host!");
6571 "Expected target-based executable directive.");
6573 const Expr *NT =
nullptr;
6574 const Expr **NTPtr = UpperBoundOnly ?
nullptr : &NT;
6576 auto CheckForConstExpr = [&](
const Expr *E,
const Expr **EPtr) {
6579 UpperBound = UpperBound ? Constant->getZExtValue()
6580 : std::min(UpperBound,
6581 int32_t(Constant->getZExtValue()));
6585 if (UpperBound == -1)
6591 auto ReturnSequential = [&]() {
6596 switch (DirectiveKind) {
6599 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6605 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6607 ThreadLimitClause = TLC;
6608 if (ThreadLimitExpr) {
6609 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6613 ThreadLimitClause->getThreadLimit().front()->getSourceRange());
6614 if (
const auto *PreInit =
6615 cast_or_null<DeclStmt>(ThreadLimitClause->getPreInitStmt())) {
6616 for (
const auto *I : PreInit->decls()) {
6617 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6629 if (ThreadLimitClause)
6630 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
6632 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6635 CS = Dir->getInnermostCapturedStmt();
6638 Dir = dyn_cast_or_null<OMPExecutableDirective>(Child);
6641 CS = Dir->getInnermostCapturedStmt();
6642 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6644 return ReturnSequential();
6648 case OMPD_target_teams: {
6652 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
6656 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6659 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6660 if (Dir->getDirectiveKind() == OMPD_distribute) {
6661 CS = Dir->getInnermostCapturedStmt();
6662 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
6667 case OMPD_target_teams_distribute:
6671 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
6674 getNumThreads(CGF, D.getInnermostCapturedStmt(), NTPtr, UpperBound,
6675 UpperBoundOnly, CondVal);
6677 case OMPD_target_teams_loop:
6678 case OMPD_target_parallel_loop:
6679 case OMPD_target_parallel:
6680 case OMPD_target_parallel_for:
6681 case OMPD_target_parallel_for_simd:
6682 case OMPD_target_teams_distribute_parallel_for:
6683 case OMPD_target_teams_distribute_parallel_for_simd: {
6684 if (CondVal && D.hasClausesOfKind<
OMPIfClause>()) {
6686 for (
const auto *
C : D.getClausesOfKind<
OMPIfClause>()) {
6687 if (
C->getNameModifier() == OMPD_unknown ||
6688 C->getNameModifier() == OMPD_parallel) {
6698 return ReturnSequential();
6708 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
6714 CheckForConstExpr(NumThreadsClause->getNumThreads(),
nullptr);
6715 return NumThreadsClause->getNumThreads();
6719 case OMPD_target_teams_distribute_simd:
6720 case OMPD_target_simd:
6721 return ReturnSequential();
6725 llvm_unreachable(
"Unsupported directive kind.");
6730 llvm::Value *NumThreadsVal =
nullptr;
6731 llvm::Value *CondVal =
nullptr;
6732 llvm::Value *ThreadLimitVal =
nullptr;
6733 const Expr *ThreadLimitExpr =
nullptr;
6734 int32_t UpperBound = -1;
6737 CGF, D, UpperBound,
false, &CondVal,
6741 if (ThreadLimitExpr) {
6744 ThreadLimitVal = CGF.
Builder.CreateIntCast(ThreadLimitVal, CGF.
Int32Ty,
6749 if (UpperBound == 1) {
6750 NumThreadsVal = CGF.
Builder.getInt32(UpperBound);
6753 NumThreadsVal = CGF.
Builder.CreateIntCast(NumThreadsVal, CGF.
Int32Ty,
6755 }
else if (ThreadLimitVal) {
6758 NumThreadsVal = ThreadLimitVal;
6759 ThreadLimitVal =
nullptr;
6762 assert(!ThreadLimitVal &&
"Default not applicable with thread limit value");
6763 NumThreadsVal = CGF.
Builder.getInt32(0);
6770 NumThreadsVal = CGF.
Builder.CreateSelect(CondVal, NumThreadsVal,
6776 if (ThreadLimitVal) {
6777 NumThreadsVal = CGF.
Builder.CreateSelect(
6778 CGF.
Builder.CreateICmpULT(ThreadLimitVal, NumThreadsVal),
6779 ThreadLimitVal, NumThreadsVal);
6782 return NumThreadsVal;
6792class MappableExprsHandler {
6798 struct AttachPtrExprComparator {
6799 const MappableExprsHandler &Handler;
6801 mutable llvm::DenseMap<std::pair<const Expr *, const Expr *>,
bool>
6802 CachedEqualityComparisons;
6804 AttachPtrExprComparator(
const MappableExprsHandler &H) : Handler(H) {}
6805 AttachPtrExprComparator() =
delete;
6808 bool operator()(
const Expr *LHS,
const Expr *RHS)
const {
6813 const auto ItLHS = Handler.AttachPtrComponentDepthMap.find(LHS);
6814 const auto ItRHS = Handler.AttachPtrComponentDepthMap.find(RHS);
6816 std::optional<size_t> DepthLHS =
6817 (ItLHS != Handler.AttachPtrComponentDepthMap.end()) ? ItLHS->second
6819 std::optional<size_t> DepthRHS =
6820 (ItRHS != Handler.AttachPtrComponentDepthMap.end()) ? ItRHS->second
6824 if (!DepthLHS.has_value() && !DepthRHS.has_value()) {
6826 if (areEqual(LHS, RHS))
6829 return wasComputedBefore(LHS, RHS);
6831 if (!DepthLHS.has_value())
6833 if (!DepthRHS.has_value())
6837 if (DepthLHS.value() != DepthRHS.value())
6838 return DepthLHS.value() < DepthRHS.value();
6841 if (areEqual(LHS, RHS))
6844 return wasComputedBefore(LHS, RHS);
6850 bool areEqual(
const Expr *LHS,
const Expr *RHS)
const {
6852 const auto CachedResultIt = CachedEqualityComparisons.find({LHS, RHS});
6853 if (CachedResultIt != CachedEqualityComparisons.end())
6854 return CachedResultIt->second;
6868 bool wasComputedBefore(
const Expr *LHS,
const Expr *RHS)
const {
6869 const size_t &OrderLHS = Handler.AttachPtrComputationOrderMap.at(LHS);
6870 const size_t &OrderRHS = Handler.AttachPtrComputationOrderMap.at(RHS);
6872 return OrderLHS < OrderRHS;
6881 bool areSemanticallyEqual(
const Expr *LHS,
const Expr *RHS)
const {
6903 if (
const auto *LD = dyn_cast<DeclRefExpr>(LHS)) {
6904 const auto *RD = dyn_cast<DeclRefExpr>(RHS);
6907 return LD->getDecl()->getCanonicalDecl() ==
6908 RD->getDecl()->getCanonicalDecl();
6912 if (
const auto *LA = dyn_cast<ArraySubscriptExpr>(LHS)) {
6913 const auto *RA = dyn_cast<ArraySubscriptExpr>(RHS);
6916 return areSemanticallyEqual(LA->getBase(), RA->getBase()) &&
6917 areSemanticallyEqual(LA->getIdx(), RA->getIdx());
6921 if (
const auto *LM = dyn_cast<MemberExpr>(LHS)) {
6922 const auto *RM = dyn_cast<MemberExpr>(RHS);
6925 if (LM->getMemberDecl()->getCanonicalDecl() !=
6926 RM->getMemberDecl()->getCanonicalDecl())
6928 return areSemanticallyEqual(LM->getBase(), RM->getBase());
6932 if (
const auto *LU = dyn_cast<UnaryOperator>(LHS)) {
6933 const auto *RU = dyn_cast<UnaryOperator>(RHS);
6936 if (LU->getOpcode() != RU->getOpcode())
6938 return areSemanticallyEqual(LU->getSubExpr(), RU->getSubExpr());
6942 if (
const auto *LB = dyn_cast<BinaryOperator>(LHS)) {
6943 const auto *RB = dyn_cast<BinaryOperator>(RHS);
6946 if (LB->getOpcode() != RB->getOpcode())
6948 return areSemanticallyEqual(LB->getLHS(), RB->getLHS()) &&
6949 areSemanticallyEqual(LB->getRHS(), RB->getRHS());
6955 if (
const auto *LAS = dyn_cast<ArraySectionExpr>(LHS)) {
6956 const auto *RAS = dyn_cast<ArraySectionExpr>(RHS);
6959 return areSemanticallyEqual(LAS->getBase(), RAS->getBase()) &&
6960 areSemanticallyEqual(LAS->getLowerBound(),
6961 RAS->getLowerBound()) &&
6962 areSemanticallyEqual(LAS->getLength(), RAS->getLength());
6966 if (
const auto *LC = dyn_cast<CastExpr>(LHS)) {
6967 const auto *RC = dyn_cast<CastExpr>(RHS);
6970 if (LC->getCastKind() != RC->getCastKind())
6972 return areSemanticallyEqual(LC->getSubExpr(), RC->getSubExpr());
6980 if (
const auto *LI = dyn_cast<IntegerLiteral>(LHS)) {
6981 const auto *RI = dyn_cast<IntegerLiteral>(RHS);
6984 return LI->getValue() == RI->getValue();
6988 if (
const auto *LC = dyn_cast<CharacterLiteral>(LHS)) {
6989 const auto *RC = dyn_cast<CharacterLiteral>(RHS);
6992 return LC->getValue() == RC->getValue();
6996 if (
const auto *LF = dyn_cast<FloatingLiteral>(LHS)) {
6997 const auto *RF = dyn_cast<FloatingLiteral>(RHS);
7001 return LF->getValue().bitwiseIsEqual(RF->getValue());
7005 if (
const auto *LS = dyn_cast<StringLiteral>(LHS)) {
7006 const auto *RS = dyn_cast<StringLiteral>(RHS);
7009 return LS->getString() == RS->getString();
7017 if (
const auto *LB = dyn_cast<CXXBoolLiteralExpr>(LHS)) {
7018 const auto *RB = dyn_cast<CXXBoolLiteralExpr>(RHS);
7021 return LB->getValue() == RB->getValue();
7030 static unsigned getFlagMemberOffset() {
7031 unsigned Offset = 0;
7032 for (uint64_t Remain =
7033 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
7034 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
7035 !(Remain & 1); Remain = Remain >> 1)
7042 class MappingExprInfo {
7044 const ValueDecl *MapDecl =
nullptr;
7047 const Expr *MapExpr =
nullptr;
7050 MappingExprInfo(
const ValueDecl *MapDecl,
const Expr *MapExpr =
nullptr)
7051 : MapDecl(MapDecl), MapExpr(MapExpr) {}
7053 const ValueDecl *getMapDecl()
const {
return MapDecl; }
7054 const Expr *getMapExpr()
const {
return MapExpr; }
7057 using DeviceInfoTy = llvm::OpenMPIRBuilder::DeviceInfoTy;
7058 using MapBaseValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
7059 using MapValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
7060 using MapFlagsArrayTy = llvm::OpenMPIRBuilder::MapFlagsArrayTy;
7061 using MapDimArrayTy = llvm::OpenMPIRBuilder::MapDimArrayTy;
7062 using MapNonContiguousArrayTy =
7063 llvm::OpenMPIRBuilder::MapNonContiguousArrayTy;
7064 using MapExprsArrayTy = SmallVector<MappingExprInfo, 4>;
7065 using MapValueDeclsArrayTy = SmallVector<const ValueDecl *, 4>;
7069 bool ,
const ValueDecl *,
const Expr *>;
7070 using MapDataArrayTy = SmallVector<MapData, 4>;
7075 struct MapCombinedInfoTy : llvm::OpenMPIRBuilder::MapInfosTy {
7076 MapExprsArrayTy Exprs;
7077 MapValueDeclsArrayTy Mappers;
7078 MapValueDeclsArrayTy DevicePtrDecls;
7081 void append(MapCombinedInfoTy &CurInfo) {
7082 Exprs.append(CurInfo.Exprs.begin(), CurInfo.Exprs.end());
7083 DevicePtrDecls.append(CurInfo.DevicePtrDecls.begin(),
7084 CurInfo.DevicePtrDecls.end());
7085 Mappers.append(CurInfo.Mappers.begin(), CurInfo.Mappers.end());
7086 llvm::OpenMPIRBuilder::MapInfosTy::append(CurInfo);
7094 struct StructRangeInfoTy {
7095 MapCombinedInfoTy PreliminaryMapData;
7096 std::pair<
unsigned , Address > LowestElem = {
7098 std::pair<
unsigned , Address > HighestElem = {
7102 bool IsArraySection =
false;
7103 bool HasCompleteRecord =
false;
7108 struct AttachInfoTy {
7111 const ValueDecl *AttachPtrDecl =
nullptr;
7112 const Expr *AttachMapExpr =
nullptr;
7114 bool isValid()
const {
7121 bool hasAttachEntryForCapturedVar(
const ValueDecl *VD)
const {
7122 for (
const auto &AttachEntry : AttachPtrExprMap) {
7123 if (AttachEntry.second) {
7126 if (
const auto *DRE = dyn_cast<DeclRefExpr>(AttachEntry.second))
7127 if (DRE->getDecl() == VD)
7135 const Expr *getAttachPtrExpr(
7138 const auto It = AttachPtrExprMap.find(Components);
7139 if (It != AttachPtrExprMap.end())
7150 ArrayRef<OpenMPMapModifierKind> MapModifiers;
7151 ArrayRef<OpenMPMotionModifierKind> MotionModifiers;
7152 bool ReturnDevicePointer =
false;
7153 bool IsImplicit =
false;
7154 const ValueDecl *Mapper =
nullptr;
7155 const Expr *VarRef =
nullptr;
7156 bool ForDeviceAddr =
false;
7158 MapInfo() =
default;
7162 ArrayRef<OpenMPMapModifierKind> MapModifiers,
7163 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
7164 bool ReturnDevicePointer,
bool IsImplicit,
7165 const ValueDecl *Mapper =
nullptr,
const Expr *VarRef =
nullptr,
7166 bool ForDeviceAddr =
false)
7167 : Components(Components), MapType(MapType), MapModifiers(MapModifiers),
7168 MotionModifiers(MotionModifiers),
7169 ReturnDevicePointer(ReturnDevicePointer), IsImplicit(IsImplicit),
7170 Mapper(Mapper), VarRef(VarRef), ForDeviceAddr(ForDeviceAddr) {}
7176 struct DeferredDevicePtrEntryTy {
7177 const Expr *IE =
nullptr;
7178 const ValueDecl *VD =
nullptr;
7179 bool ForDeviceAddr =
false;
7181 DeferredDevicePtrEntryTy(
const Expr *IE,
const ValueDecl *VD,
7183 : IE(IE), VD(VD), ForDeviceAddr(ForDeviceAddr) {}
7188 llvm::PointerUnion<
const OMPExecutableDirective *,
7189 const OMPDeclareMapperDecl *>
7193 CodeGenFunction &CGF;
7198 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>,
bool> FirstPrivateDecls;
7204 SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>>
7211 SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>>
7215 llvm::DenseMap<const ValueDecl *, const OMPMapClause *> LambdasMap;
7230 llvm::DenseMap<const Expr *, std::optional<size_t>>
7231 AttachPtrComponentDepthMap = {{
nullptr, std::nullopt}};
7235 llvm::DenseMap<const Expr *, size_t> AttachPtrComputationOrderMap = {
7240 AttachPtrExprComparator AttachPtrComparator;
7242 llvm::Value *getExprTypeSize(
const Expr *E)
const {
7246 if (
const auto *OAE = dyn_cast<OMPArrayShapingExpr>(E)) {
7248 CGF.
getTypeSize(OAE->getBase()->getType()->getPointeeType());
7249 for (
const Expr *SE : OAE->getDimensions()) {
7260 if (
const auto *RefTy = ExprTy->
getAs<ReferenceType>())
7266 if (
const auto *OAE = dyn_cast<ArraySectionExpr>(E)) {
7268 OAE->getBase()->IgnoreParenImpCasts())
7274 if (!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
7275 !OAE->getLowerBound())
7278 llvm::Value *ElemSize;
7279 if (
const auto *PTy = BaseTy->
getAs<PointerType>()) {
7280 ElemSize = CGF.
getTypeSize(PTy->getPointeeType().getCanonicalType());
7283 assert(ATy &&
"Expecting array type if not a pointer type.");
7284 ElemSize = CGF.
getTypeSize(ATy->getElementType().getCanonicalType());
7289 if (!OAE->getLength() && OAE->getColonLocFirst().isInvalid())
7292 if (
const Expr *LenExpr = OAE->getLength()) {
7296 LenExpr->getExprLoc());
7297 return CGF.
Builder.CreateNUWMul(LengthVal, ElemSize);
7299 assert(!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
7300 OAE->getLowerBound() &&
"expected array_section[lb:].");
7306 OAE->getLowerBound()->getExprLoc());
7307 LBVal = CGF.
Builder.CreateNUWMul(LBVal, ElemSize);
7308 llvm::Value *Cmp = CGF.
Builder.CreateICmpUGT(LengthVal, LBVal);
7309 llvm::Value *TrueVal = CGF.
Builder.CreateNUWSub(LengthVal, LBVal);
7310 LengthVal = CGF.
Builder.CreateSelect(
7311 Cmp, TrueVal, llvm::ConstantInt::get(CGF.
SizeTy, 0));
7321 OpenMPOffloadMappingFlags getMapTypeBits(
7323 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
bool IsImplicit,
7324 bool AddPtrFlag,
bool AddIsTargetParamFlag,
bool IsNonContiguous)
const {
7325 OpenMPOffloadMappingFlags Bits =
7326 IsImplicit ? OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT
7327 : OpenMPOffloadMappingFlags::OMP_MAP_NONE;
7329 case OMPC_MAP_alloc:
7330 case OMPC_MAP_release:
7337 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO;
7340 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_FROM;
7342 case OMPC_MAP_tofrom:
7343 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO |
7344 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
7346 case OMPC_MAP_delete:
7347 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_DELETE;
7350 llvm_unreachable(
"Unexpected map type!");
7353 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
7354 if (AddIsTargetParamFlag)
7355 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
7356 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_always))
7357 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS;
7358 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_close))
7359 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_CLOSE;
7360 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_present) ||
7361 llvm::is_contained(MotionModifiers, OMPC_MOTION_MODIFIER_present))
7362 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
7363 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_ompx_hold))
7364 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
7365 if (IsNonContiguous)
7366 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG;
7372 bool isFinalArraySectionExpression(
const Expr *E)
const {
7373 const auto *OASE = dyn_cast<ArraySectionExpr>(E);
7380 if (OASE->getColonLocFirst().isInvalid())
7383 const Expr *Length = OASE->getLength();
7390 OASE->getBase()->IgnoreParenImpCasts())
7392 if (
const auto *ATy = dyn_cast<ConstantArrayType>(BaseQTy.
getTypePtr()))
7393 return ATy->getSExtSize() != 1;
7405 llvm::APSInt ConstLength =
Result.Val.getInt();
7406 return ConstLength.getSExtValue() != 1;
7413 class CopyOverlappedEntryGaps {
7414 CodeGenFunction &CGF;
7415 MapCombinedInfoTy &CombinedInfo;
7416 OpenMPOffloadMappingFlags Flags = OpenMPOffloadMappingFlags::OMP_MAP_NONE;
7417 const ValueDecl *MapDecl =
nullptr;
7418 const Expr *MapExpr =
nullptr;
7420 bool IsNonContiguous =
false;
7424 const RecordDecl *LastParent =
nullptr;
7426 unsigned LastIndex = -1u;
7430 CopyOverlappedEntryGaps(CodeGenFunction &CGF,
7431 MapCombinedInfoTy &CombinedInfo,
7432 OpenMPOffloadMappingFlags Flags,
7433 const ValueDecl *MapDecl,
const Expr *MapExpr,
7434 Address BP, Address LB,
bool IsNonContiguous,
7436 : CGF(CGF), CombinedInfo(CombinedInfo), Flags(Flags), MapDecl(MapDecl),
7437 MapExpr(MapExpr), BP(BP), IsNonContiguous(IsNonContiguous),
7438 DimSize(DimSize), LB(LB) {}
7441 const OMPClauseMappableExprCommon::MappableComponent &MC,
7442 const FieldDecl *FD,
7443 llvm::function_ref<LValue(CodeGenFunction &,
const MemberExpr *)>
7444 EmitMemberExprBase) {
7454 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
7466 copyUntilField(FD, ComponentLB);
7469 if (((int64_t)FieldOffset - (int64_t)Cursor) > 0)
7470 copyUntilField(FD, ComponentLB);
7472 Cursor = FieldOffset + FieldSize;
7477 void copyUntilField(
const FieldDecl *FD, Address ComponentLB) {
7481 CGF.
Builder.CreatePtrDiff(CGF.
Int8Ty, ComponentLBPtr, LBPtr);
7482 copySizedChunk(LBPtr, Size);
7485 void copyUntilEnd(Address HB) {
7487 const ASTRecordLayout &RL =
7496 copySizedChunk(LBPtr, Size);
7499 void copySizedChunk(llvm::Value *Base, llvm::Value *Size) {
7500 CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
7502 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
7503 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
7504 CombinedInfo.Pointers.push_back(Base);
7505 CombinedInfo.Sizes.push_back(
7507 CombinedInfo.Types.push_back(Flags);
7508 CombinedInfo.Mappers.push_back(
nullptr);
7509 CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize : 1);
7518 void generateInfoForComponentList(
7520 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
7522 MapCombinedInfoTy &CombinedInfo,
7523 MapCombinedInfoTy &StructBaseCombinedInfo,
7524 StructRangeInfoTy &PartialStruct,
bool IsFirstComponentList,
7525 bool IsImplicit,
bool GenerateAllInfoForClauses,
7526 const ValueDecl *Mapper =
nullptr,
bool ForDeviceAddr =
false,
7527 const ValueDecl *BaseDecl =
nullptr,
const Expr *MapExpr =
nullptr,
7528 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
7529 OverlappedElements = {},
7530 bool AreBothBasePtrAndPteeMapped =
false)
const {
7712 bool IsCaptureFirstInfo = IsFirstComponentList;
7716 bool RequiresReference =
false;
7719 auto CI = Components.rbegin();
7720 auto CE = Components.rend();
7725 bool IsExpressionFirstInfo =
true;
7726 bool FirstPointerInComplexData =
false;
7728 const Expr *AssocExpr = I->getAssociatedExpression();
7729 const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr);
7730 const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr);
7731 const auto *OAShE = dyn_cast<OMPArrayShapingExpr>(AssocExpr);
7733 if (AreBothBasePtrAndPteeMapped && std::next(I) == CE)
7739 }
else if ((AE &&
isa<CXXThisExpr>(AE->getBase()->IgnoreParenImpCasts())) ||
7753 if (
const auto *VD =
7754 dyn_cast_or_null<VarDecl>(I->getAssociatedDeclaration())) {
7755 if (std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
7756 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) {
7757 if ((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
7758 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
7759 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
7761 RequiresReference =
true;
7771 I->getAssociatedDeclaration()->
getType().getNonReferenceType();
7776 const auto *VD = dyn_cast<VarDecl>(I->getAssociatedDeclaration());
7777 if (!AreBothBasePtrAndPteeMapped &&
7779 !VD || VD->hasLocalStorage()))
7782 FirstPointerInComplexData =
true;
7801 bool ShouldBeMemberOf =
false;
7810 const MemberExpr *EncounteredME =
nullptr;
7822 bool IsNonContiguous =
7823 CombinedInfo.NonContigInfo.IsNonContiguous ||
7824 any_of(Components, [&](
const auto &Component) {
7826 dyn_cast<ArraySectionExpr>(Component.getAssociatedExpression());
7830 const Expr *StrideExpr = OASE->getStride();
7834 const auto Constant =
7839 return !Constant->isOne();
7842 bool IsPrevMemberReference =
false;
7844 bool IsPartialMapped =
7845 !PartialStruct.PreliminaryMapData.BasePointers.empty();
7852 bool IsMappingWholeStruct =
true;
7853 if (!GenerateAllInfoForClauses) {
7854 IsMappingWholeStruct =
false;
7856 for (
auto TempI = I; TempI != CE; ++TempI) {
7857 const MemberExpr *PossibleME =
7858 dyn_cast<MemberExpr>(TempI->getAssociatedExpression());
7860 IsMappingWholeStruct =
false;
7866 for (; I != CE; ++I) {
7868 if (!EncounteredME) {
7869 EncounteredME = dyn_cast<MemberExpr>(I->getAssociatedExpression());
7872 if (EncounteredME) {
7873 ShouldBeMemberOf =
true;
7876 if (FirstPointerInComplexData) {
7877 QualType Ty = std::prev(I)
7878 ->getAssociatedDeclaration()
7880 .getNonReferenceType();
7882 FirstPointerInComplexData =
false;
7887 auto Next = std::next(I);
7897 bool IsFinalArraySection =
7899 isFinalArraySectionExpression(I->getAssociatedExpression());
7903 const ValueDecl *MapDecl = (I->getAssociatedDeclaration())
7904 ? I->getAssociatedDeclaration()
7906 MapExpr = (I->getAssociatedExpression()) ? I->getAssociatedExpression()
7913 dyn_cast<ArraySectionExpr>(I->getAssociatedExpression());
7915 dyn_cast<OMPArrayShapingExpr>(I->getAssociatedExpression());
7916 const auto *UO = dyn_cast<UnaryOperator>(I->getAssociatedExpression());
7917 const auto *BO = dyn_cast<BinaryOperator>(I->getAssociatedExpression());
7923 I->getAssociatedExpression()->getType()->isAnyPointerType();
7924 bool IsMemberReference =
isa<MemberExpr>(I->getAssociatedExpression()) &&
7927 bool IsNonDerefPointer = IsPointer &&
7928 !(UO && UO->getOpcode() != UO_Deref) && !BO &&
7934 if (
Next == CE || IsMemberReference || IsNonDerefPointer ||
7935 IsFinalArraySection) {
7938 assert((
Next == CE ||
7945 "Unexpected expression");
7949 auto &&EmitMemberExprBase = [](CodeGenFunction &CGF,
7950 const MemberExpr *E) {
7951 const Expr *BaseExpr = E->getBase();
7956 LValueBaseInfo BaseInfo;
7957 TBAAAccessInfo TBAAInfo;
7971 OAShE->getBase()->getType()->getPointeeType()),
7973 OAShE->getBase()->getType()));
7974 }
else if (IsMemberReference) {
7976 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
7991 bool IsMemberPointerOrAddr =
7993 (((IsPointer || ForDeviceAddr) &&
7994 I->getAssociatedExpression() == EncounteredME) ||
7995 (IsPrevMemberReference && !IsPointer) ||
7996 (IsMemberReference &&
Next != CE &&
7997 !
Next->getAssociatedExpression()->getType()->isPointerType()));
7998 if (!OverlappedElements.empty() &&
Next == CE) {
8000 assert(!PartialStruct.Base.isValid() &&
"The base element is set.");
8001 assert(!IsPointer &&
8002 "Unexpected base element with the pointer type.");
8005 PartialStruct.LowestElem = {0, LowestElem};
8007 I->getAssociatedExpression()->getType());
8012 PartialStruct.HighestElem = {
8013 std::numeric_limits<
decltype(
8014 PartialStruct.HighestElem.first)>
::max(),
8016 PartialStruct.Base = BP;
8017 PartialStruct.LB = LB;
8019 PartialStruct.PreliminaryMapData.BasePointers.empty() &&
8020 "Overlapped elements must be used only once for the variable.");
8021 std::swap(PartialStruct.PreliminaryMapData, CombinedInfo);
8023 OpenMPOffloadMappingFlags Flags =
8024 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
8025 getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,
8027 false, IsNonContiguous);
8028 CopyOverlappedEntryGaps CopyGaps(CGF, CombinedInfo, Flags, MapDecl,
8029 MapExpr, BP, LB, IsNonContiguous,
8033 Component : OverlappedElements) {
8034 for (
const OMPClauseMappableExprCommon::MappableComponent &MC :
8037 if (
const auto *FD = dyn_cast<FieldDecl>(VD)) {
8038 CopyGaps.processField(MC, FD, EmitMemberExprBase);
8043 CopyGaps.copyUntilEnd(HB);
8046 llvm::Value *
Size = getExprTypeSize(I->getAssociatedExpression());
8053 if ((!IsMemberPointerOrAddr && !IsPartialMapped) ||
8055 if (!IsMappingWholeStruct) {
8056 CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
8058 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
8059 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8061 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
8063 CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize
8066 StructBaseCombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
8067 StructBaseCombinedInfo.BasePointers.push_back(
8069 StructBaseCombinedInfo.DevicePtrDecls.push_back(
nullptr);
8070 StructBaseCombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8071 StructBaseCombinedInfo.Pointers.push_back(LB.
emitRawPointer(CGF));
8072 StructBaseCombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
8074 StructBaseCombinedInfo.NonContigInfo.Dims.push_back(
8075 IsNonContiguous ? DimSize : 1);
8079 bool HasMapper = Mapper &&
Next == CE;
8080 if (!IsMappingWholeStruct)
8081 CombinedInfo.Mappers.push_back(HasMapper ? Mapper :
nullptr);
8083 StructBaseCombinedInfo.Mappers.push_back(HasMapper ? Mapper
8090 OpenMPOffloadMappingFlags Flags =
8091 getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,
8092 !IsExpressionFirstInfo || RequiresReference ||
8093 FirstPointerInComplexData || IsMemberReference,
8094 AreBothBasePtrAndPteeMapped ||
8095 (IsCaptureFirstInfo && !RequiresReference),
8098 if (!IsExpressionFirstInfo || IsMemberReference) {
8101 if (IsPointer || (IsMemberReference &&
Next != CE))
8102 Flags &= ~(OpenMPOffloadMappingFlags::OMP_MAP_TO |
8103 OpenMPOffloadMappingFlags::OMP_MAP_FROM |
8104 OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS |
8105 OpenMPOffloadMappingFlags::OMP_MAP_DELETE |
8106 OpenMPOffloadMappingFlags::OMP_MAP_CLOSE);
8108 if (ShouldBeMemberOf) {
8111 Flags |= OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;
8114 ShouldBeMemberOf =
false;
8118 if (!IsMappingWholeStruct)
8119 CombinedInfo.Types.push_back(Flags);
8121 StructBaseCombinedInfo.Types.push_back(Flags);
8127 if (EncounteredME) {
8132 if (!PartialStruct.Base.isValid()) {
8133 PartialStruct.LowestElem = {FieldIndex, LowestElem};
8134 if (IsFinalArraySection && OASE) {
8138 PartialStruct.HighestElem = {FieldIndex, HB};
8140 PartialStruct.HighestElem = {FieldIndex, LowestElem};
8142 PartialStruct.Base = BP;
8143 PartialStruct.LB = BP;
8144 }
else if (FieldIndex < PartialStruct.LowestElem.first) {
8145 PartialStruct.LowestElem = {FieldIndex, LowestElem};
8146 }
else if (FieldIndex > PartialStruct.HighestElem.first) {
8147 if (IsFinalArraySection && OASE) {
8151 PartialStruct.HighestElem = {FieldIndex, HB};
8153 PartialStruct.HighestElem = {FieldIndex, LowestElem};
8159 if (IsFinalArraySection || IsNonContiguous)
8160 PartialStruct.IsArraySection =
true;
8163 if (IsFinalArraySection)
8168 BP = IsMemberReference ? LowestElem : LB;
8169 if (!IsPartialMapped)
8170 IsExpressionFirstInfo =
false;
8171 IsCaptureFirstInfo =
false;
8172 FirstPointerInComplexData =
false;
8173 IsPrevMemberReference = IsMemberReference;
8174 }
else if (FirstPointerInComplexData) {
8175 QualType Ty = Components.rbegin()
8176 ->getAssociatedDeclaration()
8178 .getNonReferenceType();
8180 FirstPointerInComplexData =
false;
8186 PartialStruct.HasCompleteRecord =
true;
8188 if (!IsNonContiguous)
8191 const ASTContext &Context = CGF.
getContext();
8195 MapValuesArrayTy CurOffsets = {llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, 0)};
8196 MapValuesArrayTy CurCounts = {llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, 1)};
8197 MapValuesArrayTy CurStrides;
8198 MapValuesArrayTy DimSizes{llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, 1)};
8204 for (
const OMPClauseMappableExprCommon::MappableComponent &Component :
8206 const Expr *AssocExpr = Component.getAssociatedExpression();
8207 const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr);
8217 assert((VAT || CAT || &Component == &*Components.begin()) &&
8218 "Should be either ConstantArray or VariableArray if not the "
8222 if (CurStrides.empty()) {
8223 const Type *ElementType =
nullptr;
8225 ElementType = CAT->getElementType().getTypePtr();
8227 ElementType = VAT->getElementType().getTypePtr();
8229 assert(&Component == &*Components.begin() &&
8230 "Only expect pointer (non CAT or VAT) when this is the "
8238 if (&Component != &*Components.begin())
8242 CurStrides.push_back(
8243 llvm::ConstantInt::get(CGF.
Int64Ty, ElementTypeSize));
8248 if (DimSizes.size() < Components.size() - 1) {
8251 llvm::ConstantInt::get(CGF.
Int64Ty, CAT->getZExtSize()));
8253 DimSizes.push_back(CGF.
Builder.CreateIntCast(
8260 auto *DI = DimSizes.begin() + 1;
8262 llvm::Value *DimProd =
8263 llvm::ConstantInt::get(CGF.
CGM.
Int64Ty, ElementTypeSize);
8272 for (
const OMPClauseMappableExprCommon::MappableComponent &Component :
8274 const Expr *AssocExpr = Component.getAssociatedExpression();
8276 if (
const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr)) {
8277 llvm::Value *Offset = CGF.
Builder.CreateIntCast(
8280 CurOffsets.push_back(Offset);
8281 CurCounts.push_back(llvm::ConstantInt::get(CGF.
Int64Ty, 1));
8282 CurStrides.push_back(CurStrides.back());
8286 const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr);
8292 const Expr *OffsetExpr = OASE->getLowerBound();
8293 llvm::Value *Offset =
nullptr;
8296 Offset = llvm::ConstantInt::get(CGF.
Int64Ty, 0);
8302 CurOffsets.push_back(Offset);
8305 const Expr *CountExpr = OASE->getLength();
8306 llvm::Value *Count =
nullptr;
8312 if (!OASE->getColonLocFirst().isValid() &&
8313 !OASE->getColonLocSecond().isValid()) {
8314 Count = llvm::ConstantInt::get(CGF.
Int64Ty, 1);
8320 const Expr *StrideExpr = OASE->getStride();
8321 llvm::Value *Stride =
8327 Count = CGF.
Builder.CreateUDiv(
8328 CGF.
Builder.CreateNUWSub(*DI, Offset), Stride);
8330 Count = CGF.
Builder.CreateNUWSub(*DI, Offset);
8336 CurCounts.push_back(Count);
8345 const Expr *StrideExpr = OASE->getStride();
8346 llvm::Value *Stride =
8351 DimProd = CGF.
Builder.CreateNUWMul(DimProd, *(DI - 1));
8353 CurStrides.push_back(CGF.
Builder.CreateNUWMul(DimProd, Stride));
8355 CurStrides.push_back(DimProd);
8356 if (DI != DimSizes.end())
8360 CombinedInfo.NonContigInfo.Offsets.push_back(CurOffsets);
8361 CombinedInfo.NonContigInfo.Counts.push_back(CurCounts);
8362 CombinedInfo.NonContigInfo.Strides.push_back(CurStrides);
8368 OpenMPOffloadMappingFlags
8369 getMapModifiersForPrivateClauses(
const CapturedStmt::Capture &Cap)
const {
8377 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
8378 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
8379 return OpenMPOffloadMappingFlags::OMP_MAP_PRIVATE |
8380 OpenMPOffloadMappingFlags::OMP_MAP_TO;
8383 if (I != LambdasMap.end())
8385 return getMapTypeBits(
8386 I->getSecond()->getMapType(), I->getSecond()->getMapTypeModifiers(),
8387 {}, I->getSecond()->isImplicit(),
8391 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
8392 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
8395 void getPlainLayout(
const CXXRecordDecl *RD,
8396 llvm::SmallVectorImpl<const FieldDecl *> &Layout,
8397 bool AsBase)
const {
8400 llvm::StructType *St =
8403 unsigned NumElements = St->getNumElements();
8405 llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>, 4>
8406 RecordLayout(NumElements);
8409 for (
const auto &I : RD->
bases()) {
8413 QualType BaseTy = I.getType();
8424 RecordLayout[FieldIndex] =
Base;
8427 for (
const auto &I : RD->
vbases()) {
8428 QualType BaseTy = I.getType();
8435 if (RecordLayout[FieldIndex])
8437 RecordLayout[FieldIndex] =
Base;
8440 assert(!RD->
isUnion() &&
"Unexpected union.");
8441 for (
const auto *Field : RD->
fields()) {
8444 if (!
Field->isBitField() &&
8447 RecordLayout[FieldIndex] =
Field;
8450 for (
const llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>
8451 &
Data : RecordLayout) {
8454 if (
const auto *Base = dyn_cast<const CXXRecordDecl *>(
Data))
8455 getPlainLayout(Base, Layout,
true);
8462 static Address getAttachPtrAddr(
const Expr *PointerExpr,
8463 CodeGenFunction &CGF) {
8464 assert(PointerExpr &&
"Cannot get addr from null attach-ptr expr");
8467 if (
auto *DRE = dyn_cast<DeclRefExpr>(PointerExpr)) {
8470 }
else if (
auto *OASE = dyn_cast<ArraySectionExpr>(PointerExpr)) {
8473 }
else if (
auto *ASE = dyn_cast<ArraySubscriptExpr>(PointerExpr)) {
8475 }
else if (
auto *ME = dyn_cast<MemberExpr>(PointerExpr)) {
8477 }
else if (
auto *UO = dyn_cast<UnaryOperator>(PointerExpr)) {
8478 assert(UO->getOpcode() == UO_Deref &&
8479 "Unexpected unary-operator on attach-ptr-expr");
8482 assert(AttachPtrAddr.
isValid() &&
8483 "Failed to get address for attach pointer expression");
8484 return AttachPtrAddr;
8491 static std::pair<Address, Address>
8492 getAttachPtrAddrAndPteeBaseAddr(
const Expr *AttachPtrExpr,
8493 CodeGenFunction &CGF) {
8498 Address AttachPtrAddr = getAttachPtrAddr(AttachPtrExpr, CGF);
8499 assert(AttachPtrAddr.
isValid() &&
"Invalid attach pointer addr");
8501 QualType AttachPtrType =
8506 AttachPtrAddr, AttachPtrType->
castAs<PointerType>());
8507 assert(AttachPteeBaseAddr.
isValid() &&
"Invalid attach pointee base addr");
8509 return {AttachPtrAddr, AttachPteeBaseAddr};
8515 shouldEmitAttachEntry(
const Expr *PointerExpr,
const ValueDecl *MapBaseDecl,
8516 CodeGenFunction &CGF,
8517 llvm::PointerUnion<
const OMPExecutableDirective *,
8518 const OMPDeclareMapperDecl *>
8528 ->getDirectiveKind());
8537 void collectAttachPtrExprInfo(
8539 llvm::PointerUnion<
const OMPExecutableDirective *,
8540 const OMPDeclareMapperDecl *>
8545 ? OMPD_declare_mapper
8548 const auto &[AttachPtrExpr, Depth] =
8552 AttachPtrComputationOrderMap.try_emplace(
8553 AttachPtrExpr, AttachPtrComputationOrderMap.size());
8554 AttachPtrComponentDepthMap.try_emplace(AttachPtrExpr, Depth);
8555 AttachPtrExprMap.try_emplace(Components, AttachPtrExpr);
8563 void generateAllInfoForClauses(
8564 ArrayRef<const OMPClause *> Clauses, MapCombinedInfoTy &CombinedInfo,
8565 llvm::OpenMPIRBuilder &OMPBuilder,
8566 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkipVarSet =
8567 llvm::DenseSet<CanonicalDeclPtr<const Decl>>())
const {
8572 llvm::MapVector<CanonicalDeclPtr<const Decl>,
8573 SmallVector<SmallVector<MapInfo, 8>, 4>>
8579 [&Info, &SkipVarSet](
8580 const ValueDecl *D, MapKind
Kind,
8583 ArrayRef<OpenMPMapModifierKind> MapModifiers,
8584 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
8585 bool ReturnDevicePointer,
bool IsImplicit,
const ValueDecl *Mapper,
8586 const Expr *VarRef =
nullptr,
bool ForDeviceAddr =
false) {
8587 if (SkipVarSet.contains(D))
8589 auto It = Info.try_emplace(D, Total).first;
8590 It->second[
Kind].emplace_back(
8591 L, MapType, MapModifiers, MotionModifiers, ReturnDevicePointer,
8592 IsImplicit, Mapper, VarRef, ForDeviceAddr);
8595 for (
const auto *
Cl : Clauses) {
8596 const auto *
C = dyn_cast<OMPMapClause>(
Cl);
8600 if (llvm::is_contained(
C->getMapTypeModifiers(),
8601 OMPC_MAP_MODIFIER_present))
8603 else if (
C->getMapType() == OMPC_MAP_alloc)
8605 const auto *EI =
C->getVarRefs().begin();
8606 for (
const auto L :
C->component_lists()) {
8607 const Expr *E = (
C->getMapLoc().isValid()) ? *EI :
nullptr;
8608 InfoGen(std::get<0>(L), Kind, std::get<1>(L),
C->getMapType(),
8609 C->getMapTypeModifiers(), {},
8610 false,
C->isImplicit(), std::get<2>(L),
8615 for (
const auto *
Cl : Clauses) {
8616 const auto *
C = dyn_cast<OMPToClause>(
Cl);
8620 if (llvm::is_contained(
C->getMotionModifiers(),
8621 OMPC_MOTION_MODIFIER_present))
8623 const auto *EI =
C->getVarRefs().begin();
8624 for (
const auto L :
C->component_lists()) {
8625 InfoGen(std::get<0>(L), Kind, std::get<1>(L), OMPC_MAP_to, {},
8626 C->getMotionModifiers(),
false,
8627 C->isImplicit(), std::get<2>(L), *EI);
8631 for (
const auto *
Cl : Clauses) {
8632 const auto *
C = dyn_cast<OMPFromClause>(
Cl);
8636 if (llvm::is_contained(
C->getMotionModifiers(),
8637 OMPC_MOTION_MODIFIER_present))
8639 const auto *EI =
C->getVarRefs().begin();
8640 for (
const auto L :
C->component_lists()) {
8641 InfoGen(std::get<0>(L), Kind, std::get<1>(L), OMPC_MAP_from, {},
8642 C->getMotionModifiers(),
8643 false,
C->isImplicit(), std::get<2>(L),
8656 llvm::MapVector<CanonicalDeclPtr<const Decl>,
8657 SmallVector<DeferredDevicePtrEntryTy, 4>>
8659 MapCombinedInfoTy UseDeviceDataCombinedInfo;
8661 auto &&UseDeviceDataCombinedInfoGen =
8662 [&UseDeviceDataCombinedInfo](
const ValueDecl *VD, llvm::Value *Ptr,
8663 CodeGenFunction &CGF,
bool IsDevAddr) {
8664 UseDeviceDataCombinedInfo.Exprs.push_back(VD);
8665 UseDeviceDataCombinedInfo.BasePointers.emplace_back(Ptr);
8666 UseDeviceDataCombinedInfo.DevicePtrDecls.emplace_back(VD);
8667 UseDeviceDataCombinedInfo.DevicePointers.emplace_back(
8668 IsDevAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer);
8669 UseDeviceDataCombinedInfo.Pointers.push_back(Ptr);
8670 UseDeviceDataCombinedInfo.Sizes.push_back(
8671 llvm::Constant::getNullValue(CGF.Int64Ty));
8672 UseDeviceDataCombinedInfo.Types.push_back(
8673 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM);
8674 UseDeviceDataCombinedInfo.Mappers.push_back(
nullptr);
8678 [&DeferredInfo, &UseDeviceDataCombinedInfoGen,
8679 &InfoGen](CodeGenFunction &CGF,
const Expr *IE,
const ValueDecl *VD,
8682 bool IsImplicit,
bool IsDevAddr) {
8695 false, IsImplicit,
nullptr,
nullptr,
8697 DeferredInfo[
nullptr].emplace_back(IE, VD, IsDevAddr);
8701 if (IE->isGLValue())
8702 Ptr = CGF.EmitLValue(IE).getPointer(CGF);
8704 Ptr = CGF.EmitScalarExpr(IE);
8706 Ptr = CGF.EmitLoadOfScalar(CGF.EmitLValue(IE), IE->getExprLoc());
8708 UseDeviceDataCombinedInfoGen(VD, Ptr, CGF, IsDevAddr);
8712 auto &&IsMapInfoExist = [&Info](CodeGenFunction &CGF,
const ValueDecl *VD,
8713 const Expr *IE,
bool IsDevAddr) ->
bool {
8721 if (It != Info.end()) {
8723 for (
auto &
Data : It->second) {
8724 auto *CI = llvm::find_if(
Data, [VD](
const MapInfo &MI) {
8725 return MI.Components.back().getAssociatedDeclaration() == VD;
8733 if (CI !=
Data.end()) {
8735 CI->ForDeviceAddr = IsDevAddr;
8736 CI->ReturnDevicePointer =
true;
8740 auto PrevCI = std::next(CI->Components.rbegin());
8741 const auto *VarD = dyn_cast<VarDecl>(VD);
8744 !VD->getType().getNonReferenceType()->isPointerType() ||
8745 PrevCI == CI->Components.rend() ||
8747 VarD->hasLocalStorage()) {
8748 CI->ForDeviceAddr = IsDevAddr;
8749 CI->ReturnDevicePointer =
true;
8767 for (
const auto *
Cl : Clauses) {
8768 const auto *
C = dyn_cast<OMPUseDevicePtrClause>(
Cl);
8771 for (
const auto L :
C->component_lists()) {
8774 assert(!Components.empty() &&
8775 "Not expecting empty list of components!");
8776 const ValueDecl *VD = Components.back().getAssociatedDeclaration();
8778 const Expr *IE = Components.back().getAssociatedExpression();
8779 if (IsMapInfoExist(CGF, VD, IE,
false))
8781 MapInfoGen(CGF, IE, VD, Components,
C->isImplicit(),
8786 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
8787 for (
const auto *
Cl : Clauses) {
8788 const auto *
C = dyn_cast<OMPUseDeviceAddrClause>(
Cl);
8791 for (
const auto L :
C->component_lists()) {
8794 assert(!std::get<1>(L).empty() &&
8795 "Not expecting empty list of components!");
8796 const ValueDecl *VD = std::get<1>(L).back().getAssociatedDeclaration();
8797 if (!Processed.insert(VD).second)
8800 const Expr *IE = std::get<1>(L).back().getAssociatedExpression();
8801 if (IsMapInfoExist(CGF, VD, IE,
true))
8803 MapInfoGen(CGF, IE, VD, Components,
C->isImplicit(),
8808 for (
const auto &
Data : Info) {
8809 StructRangeInfoTy PartialStruct;
8811 MapCombinedInfoTy CurInfo;
8813 MapCombinedInfoTy StructBaseCurInfo;
8815 const ValueDecl *VD = cast_or_null<ValueDecl>(D);
8816 bool HasMapBasePtr =
false;
8817 bool HasMapArraySec =
false;
8819 for (
const auto &M :
Data.second) {
8820 HasMapBasePtr = any_of(M, [](
const MapInfo &L) {
8821 return isa_and_present<DeclRefExpr>(L.VarRef);
8823 HasMapArraySec = any_of(M, [](
const MapInfo &L) {
8824 return isa_and_present<ArraySectionExpr, ArraySubscriptExpr>(
8827 if (HasMapBasePtr && HasMapArraySec)
8831 for (
const auto &M :
Data.second) {
8832 for (
const MapInfo &L : M) {
8833 assert(!L.Components.empty() &&
8834 "Not expecting declaration with no component lists.");
8837 unsigned CurrentBasePointersIdx = CurInfo.BasePointers.size();
8838 unsigned StructBasePointersIdx =
8839 StructBaseCurInfo.BasePointers.size();
8840 CurInfo.NonContigInfo.IsNonContiguous =
8841 L.Components.back().isNonContiguous();
8842 generateInfoForComponentList(
8843 L.MapType, L.MapModifiers, L.MotionModifiers, L.Components,
8844 CurInfo, StructBaseCurInfo, PartialStruct,
8845 false, L.IsImplicit,
8846 true, L.Mapper, L.ForDeviceAddr, VD,
8848 HasMapBasePtr && HasMapArraySec);
8852 if (L.ReturnDevicePointer) {
8856 assert((CurrentBasePointersIdx < CurInfo.BasePointers.size() ||
8857 StructBasePointersIdx <
8858 StructBaseCurInfo.BasePointers.size()) &&
8859 "Unexpected number of mapped base pointers.");
8862 const ValueDecl *RelevantVD =
8863 L.Components.back().getAssociatedDeclaration();
8864 assert(RelevantVD &&
8865 "No relevant declaration related with device pointer??");
8872 if (StructBasePointersIdx < StructBaseCurInfo.BasePointers.size()) {
8873 StructBaseCurInfo.DevicePtrDecls[StructBasePointersIdx] =
8875 StructBaseCurInfo.DevicePointers[StructBasePointersIdx] =
8876 L.ForDeviceAddr ? DeviceInfoTy::Address
8877 : DeviceInfoTy::Pointer;
8878 StructBaseCurInfo.Types[StructBasePointersIdx] |=
8879 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
8881 CurInfo.DevicePtrDecls[CurrentBasePointersIdx] = RelevantVD;
8882 CurInfo.DevicePointers[CurrentBasePointersIdx] =
8883 L.ForDeviceAddr ? DeviceInfoTy::Address
8884 : DeviceInfoTy::Pointer;
8885 CurInfo.Types[CurrentBasePointersIdx] |=
8886 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
8894 auto CI = DeferredInfo.find(
Data.first);
8895 if (CI != DeferredInfo.end()) {
8896 for (
const DeferredDevicePtrEntryTy &L : CI->second) {
8897 llvm::Value *BasePtr;
8899 if (L.ForDeviceAddr) {
8900 if (L.IE->isGLValue())
8908 CurInfo.Types.push_back(
8909 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM |
8910 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
8914 L.IE->getExprLoc());
8918 CurInfo.Types.push_back(
8919 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
8920 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM |
8921 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
8923 CurInfo.Exprs.push_back(L.VD);
8924 CurInfo.BasePointers.emplace_back(BasePtr);
8925 CurInfo.DevicePtrDecls.emplace_back(L.VD);
8926 CurInfo.DevicePointers.emplace_back(
8927 L.ForDeviceAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer);
8928 CurInfo.Pointers.push_back(Ptr);
8929 CurInfo.Sizes.push_back(
8930 llvm::Constant::getNullValue(this->CGF.
Int64Ty));
8931 CurInfo.Mappers.push_back(
nullptr);
8937 MapCombinedInfoTy UnionCurInfo;
8938 UnionCurInfo.append(StructBaseCurInfo);
8939 UnionCurInfo.append(CurInfo);
8943 if (PartialStruct.Base.isValid()) {
8944 UnionCurInfo.NonContigInfo.Dims.push_back(0);
8946 emitCombinedEntry(CombinedInfo, UnionCurInfo.Types, PartialStruct,
8947 !VD, OMPBuilder, VD);
8951 CombinedInfo.append(UnionCurInfo);
8954 CombinedInfo.append(UseDeviceDataCombinedInfo);
8958 MappableExprsHandler(
const OMPExecutableDirective &Dir, CodeGenFunction &CGF)
8959 : CurDir(&Dir), CGF(CGF), AttachPtrComparator(*this) {
8961 for (
const auto *
C : Dir.getClausesOfKind<OMPFirstprivateClause>())
8962 for (
const auto *D :
C->varlist())
8963 FirstPrivateDecls.try_emplace(
8966 for (
const auto *
C : Dir.getClausesOfKind<OMPUsesAllocatorsClause>()) {
8967 for (unsigned I = 0, E = C->getNumberOfAllocators(); I < E; ++I) {
8968 OMPUsesAllocatorsClause::Data D = C->getAllocatorData(I);
8969 if (const auto *DRE = dyn_cast_or_null<DeclRefExpr>(D.AllocatorTraits))
8970 FirstPrivateDecls.try_emplace(cast<VarDecl>(DRE->getDecl()),
8972 else if (const auto *VD = dyn_cast<VarDecl>(
8973 cast<DeclRefExpr>(D.Allocator->IgnoreParenImpCasts())
8975 FirstPrivateDecls.try_emplace(VD, true);
8979 for (
const auto *
C : Dir.getClausesOfKind<OMPIsDevicePtrClause>())
8980 for (
auto L :
C->component_lists())
8981 DevPointersMap[std::get<0>(L)].push_back(std::get<1>(L));
8983 for (
const auto *
C : Dir.getClausesOfKind<OMPHasDeviceAddrClause>())
8984 for (
auto L :
C->component_lists())
8985 HasDevAddrsMap[std::get<0>(L)].push_back(std::get<1>(L));
8987 for (
const auto *
C : Dir.getClausesOfKind<OMPMapClause>()) {
8988 if (C->getMapType() != OMPC_MAP_to)
8990 for (auto L : C->component_lists()) {
8991 const ValueDecl *VD = std::get<0>(L);
8992 const auto *RD = VD ? VD->getType()
8994 .getNonReferenceType()
8995 ->getAsCXXRecordDecl()
8997 if (RD && RD->isLambda())
8998 LambdasMap.try_emplace(std::get<0>(L), C);
9004 MappableExprsHandler(
const OMPDeclareMapperDecl &Dir,
CodeGenFunction &CGF)
9005 : CurDir(&Dir), CGF(CGF), AttachPtrComparator(*this) {}
9010 void emitCombinedEntry(MapCombinedInfoTy &CombinedInfo,
9011 MapFlagsArrayTy &CurTypes,
9012 const StructRangeInfoTy &PartialStruct,
bool IsMapThis,
9013 llvm::OpenMPIRBuilder &OMPBuilder,
9014 const ValueDecl *VD =
nullptr,
9015 unsigned OffsetForMemberOfFlag = 0,
9016 bool NotTargetParams =
true)
const {
9017 if (CurTypes.size() == 1 &&
9018 ((CurTypes.back() & OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) !=
9019 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) &&
9020 !PartialStruct.IsArraySection)
9022 Address LBAddr = PartialStruct.LowestElem.second;
9023 Address HBAddr = PartialStruct.HighestElem.second;
9024 if (PartialStruct.HasCompleteRecord) {
9025 LBAddr = PartialStruct.LB;
9026 HBAddr = PartialStruct.LB;
9028 CombinedInfo.Exprs.push_back(VD);
9030 CombinedInfo.BasePointers.push_back(PartialStruct.Base.emitRawPointer(CGF));
9031 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
9032 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
9035 const CXXMethodDecl *MD =
9037 const CXXRecordDecl *RD = MD ? MD->
getParent() :
nullptr;
9038 bool HasBaseClass = RD && IsMapThis ? RD->
getNumBases() > 0 :
false;
9048 CombinedInfo.Pointers.push_back(PartialStruct.Base.emitRawPointer(CGF));
9053 CombinedInfo.Sizes.push_back(Size);
9055 CombinedInfo.Pointers.push_back(LB);
9058 llvm::Value *HAddr = CGF.
Builder.CreateConstGEP1_32(
9062 llvm::Value *Diff = CGF.
Builder.CreatePtrDiff(CGF.
Int8Ty, CHAddr, CLAddr);
9065 CombinedInfo.Sizes.push_back(Size);
9067 CombinedInfo.Mappers.push_back(
nullptr);
9069 CombinedInfo.Types.push_back(
9070 NotTargetParams ? OpenMPOffloadMappingFlags::OMP_MAP_NONE
9071 : !PartialStruct.PreliminaryMapData.BasePointers.empty()
9072 ? OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ
9073 : OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
9076 if (CurTypes.end() !=
9077 llvm::find_if(CurTypes, [](OpenMPOffloadMappingFlags
Type) {
9078 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
9079 Type & OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
9081 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
9083 (*CurTypes.begin()) &= ~OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
9090 if (CurTypes.end() !=
9091 llvm::find_if(CurTypes, [](OpenMPOffloadMappingFlags
Type) {
9092 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
9093 Type & OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD);
9095 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
9096 for (
auto &M : CurTypes)
9097 M |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
9103 OpenMPOffloadMappingFlags MemberOfFlag = OMPBuilder.getMemberOfFlag(
9104 OffsetForMemberOfFlag + CombinedInfo.BasePointers.size() - 1);
9105 for (
auto &M : CurTypes)
9106 OMPBuilder.setCorrectMemberOfFlag(M, MemberOfFlag);
9114 void generateAllInfo(
9115 MapCombinedInfoTy &CombinedInfo, llvm::OpenMPIRBuilder &OMPBuilder,
9116 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkipVarSet =
9117 llvm::DenseSet<CanonicalDeclPtr<const Decl>>())
const {
9119 "Expect a executable directive");
9121 generateAllInfoForClauses(CurExecDir->clauses(), CombinedInfo, OMPBuilder,
9128 void generateAllInfoForMapper(MapCombinedInfoTy &CombinedInfo,
9129 llvm::OpenMPIRBuilder &OMPBuilder)
const {
9131 "Expect a declare mapper directive");
9133 generateAllInfoForClauses(CurMapperDir->clauses(), CombinedInfo,
9138 void generateInfoForLambdaCaptures(
9139 const ValueDecl *VD, llvm::Value *Arg, MapCombinedInfoTy &CombinedInfo,
9140 llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers)
const {
9148 llvm::DenseMap<const ValueDecl *, FieldDecl *> Captures;
9149 FieldDecl *ThisCapture =
nullptr;
9155 LambdaPointers.try_emplace(ThisLVal.getPointer(CGF),
9156 VDLVal.getPointer(CGF));
9157 CombinedInfo.Exprs.push_back(VD);
9158 CombinedInfo.BasePointers.push_back(ThisLVal.getPointer(CGF));
9159 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
9160 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
9161 CombinedInfo.Pointers.push_back(ThisLValVal.getPointer(CGF));
9162 CombinedInfo.Sizes.push_back(
9165 CombinedInfo.Types.push_back(
9166 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
9167 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9168 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
9169 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
9170 CombinedInfo.Mappers.push_back(
nullptr);
9172 for (
const LambdaCapture &LC : RD->
captures()) {
9173 if (!LC.capturesVariable())
9178 auto It = Captures.find(VD);
9179 assert(It != Captures.end() &&
"Found lambda capture without field.");
9183 LambdaPointers.try_emplace(VarLVal.getPointer(CGF),
9184 VDLVal.getPointer(CGF));
9185 CombinedInfo.Exprs.push_back(VD);
9186 CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF));
9187 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
9188 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
9189 CombinedInfo.Pointers.push_back(VarLValVal.getPointer(CGF));
9190 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
9196 LambdaPointers.try_emplace(VarLVal.getPointer(CGF),
9197 VDLVal.getPointer(CGF));
9198 CombinedInfo.Exprs.push_back(VD);
9199 CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF));
9200 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
9201 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
9202 CombinedInfo.Pointers.push_back(VarRVal.
getScalarVal());
9203 CombinedInfo.Sizes.push_back(llvm::ConstantInt::get(CGF.
Int64Ty, 0));
9205 CombinedInfo.Types.push_back(
9206 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
9207 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9208 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
9209 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
9210 CombinedInfo.Mappers.push_back(
nullptr);
9215 void adjustMemberOfForLambdaCaptures(
9216 llvm::OpenMPIRBuilder &OMPBuilder,
9217 const llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers,
9218 MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
9219 MapFlagsArrayTy &Types)
const {
9220 for (
unsigned I = 0, E = Types.size(); I < E; ++I) {
9222 if (Types[I] != (OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
9223 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9224 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
9225 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT))
9227 llvm::Value *BasePtr = LambdaPointers.lookup(BasePointers[I]);
9228 assert(BasePtr &&
"Unable to find base lambda address.");
9230 for (
unsigned J = I; J > 0; --J) {
9231 unsigned Idx = J - 1;
9232 if (Pointers[Idx] != BasePtr)
9237 assert(TgtIdx != -1 &&
"Unable to find parent lambda.");
9241 OpenMPOffloadMappingFlags MemberOfFlag =
9242 OMPBuilder.getMemberOfFlag(TgtIdx);
9243 OMPBuilder.setCorrectMemberOfFlag(Types[I], MemberOfFlag);
9250 void generateInfoForCaptureFromClauseInfo(
9251 const CapturedStmt::Capture *Cap, llvm::Value *Arg,
9252 MapCombinedInfoTy &CurCaptureVarInfo, llvm::OpenMPIRBuilder &OMPBuilder,
9253 unsigned OffsetForMemberOfFlag)
const {
9255 "Not expecting to generate map info for a variable array type!");
9264 if (LambdasMap.count(VD))
9270 if (VD && (DevPointersMap.count(VD) || HasDevAddrsMap.count(VD))) {
9271 CurCaptureVarInfo.Exprs.push_back(VD);
9272 CurCaptureVarInfo.BasePointers.emplace_back(Arg);
9273 CurCaptureVarInfo.DevicePtrDecls.emplace_back(VD);
9274 CurCaptureVarInfo.DevicePointers.emplace_back(DeviceInfoTy::Pointer);
9275 CurCaptureVarInfo.Pointers.push_back(Arg);
9276 CurCaptureVarInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
9279 CurCaptureVarInfo.Types.push_back(
9280 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9281 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
9282 CurCaptureVarInfo.Mappers.push_back(
nullptr);
9286 MapDataArrayTy DeclComponentLists;
9290 auto It = DevPointersMap.find(VD);
9291 if (It != DevPointersMap.end())
9292 for (
const auto &MCL : It->second)
9293 DeclComponentLists.emplace_back(MCL, OMPC_MAP_to,
Unknown,
9296 auto I = HasDevAddrsMap.find(VD);
9297 if (I != HasDevAddrsMap.end())
9298 for (
const auto &MCL : I->second)
9299 DeclComponentLists.emplace_back(MCL, OMPC_MAP_tofrom,
Unknown,
9303 "Expect a executable directive");
9305 bool HasMapBasePtr =
false;
9306 bool HasMapArraySec =
false;
9307 for (
const auto *
C : CurExecDir->getClausesOfKind<OMPMapClause>()) {
9308 const auto *EI =
C->getVarRefs().begin();
9309 for (
const auto L :
C->decl_component_lists(VD)) {
9310 const ValueDecl *VDecl, *Mapper;
9312 const Expr *E = (
C->getMapLoc().isValid()) ? *EI :
nullptr;
9314 std::tie(VDecl, Components, Mapper) = L;
9315 assert(VDecl == VD &&
"We got information for the wrong declaration??");
9316 assert(!Components.empty() &&
9317 "Not expecting declaration with no component lists.");
9319 HasMapBasePtr =
true;
9322 HasMapArraySec =
true;
9323 DeclComponentLists.emplace_back(Components,
C->getMapType(),
9324 C->getMapTypeModifiers(),
9325 C->isImplicit(), Mapper, E);
9329 llvm::stable_sort(DeclComponentLists, [](
const MapData &LHS,
9330 const MapData &RHS) {
9331 ArrayRef<OpenMPMapModifierKind> MapModifiers = std::get<2>(LHS);
9334 llvm::is_contained(MapModifiers, clang::OMPC_MAP_MODIFIER_present);
9335 bool HasAllocs = MapType == OMPC_MAP_alloc;
9336 MapModifiers = std::get<2>(RHS);
9337 MapType = std::get<1>(LHS);
9339 llvm::is_contained(MapModifiers, clang::OMPC_MAP_MODIFIER_present);
9340 bool HasAllocsR = MapType == OMPC_MAP_alloc;
9341 return (HasPresent && !HasPresentR) || (HasAllocs && !HasAllocsR);
9344 auto GenerateInfoForComponentLists =
9345 [&](ArrayRef<MapData> DeclComponentLists,
9346 bool IsEligibleForTargetParamFlag) {
9347 MapCombinedInfoTy CurInfoForComponentLists;
9348 StructRangeInfoTy PartialStruct;
9350 if (DeclComponentLists.empty())
9353 generateInfoForCaptureFromComponentLists(
9354 VD, DeclComponentLists, CurInfoForComponentLists, PartialStruct,
9355 IsEligibleForTargetParamFlag,
9356 HasMapBasePtr && HasMapArraySec);
9361 if (PartialStruct.Base.isValid()) {
9362 CurCaptureVarInfo.append(PartialStruct.PreliminaryMapData);
9364 CurCaptureVarInfo, CurInfoForComponentLists.Types,
9365 PartialStruct, Cap->
capturesThis(), OMPBuilder,
nullptr,
9366 OffsetForMemberOfFlag,
9367 !IsEligibleForTargetParamFlag);
9371 if (CurInfoForComponentLists.BasePointers.empty())
9374 CurCaptureVarInfo.append(CurInfoForComponentLists);
9377 GenerateInfoForComponentLists(DeclComponentLists,
9384 void generateInfoForCaptureFromComponentLists(
9385 const ValueDecl *VD, ArrayRef<MapData> DeclComponentLists,
9386 MapCombinedInfoTy &CurComponentListInfo, StructRangeInfoTy &PartialStruct,
9387 bool IsListEligibleForTargetParamFlag,
9388 bool AreBothBasePtrAndPteeMapped =
false)
const {
9390 llvm::SmallDenseMap<
9397 for (
const MapData &L : DeclComponentLists) {
9400 ArrayRef<OpenMPMapModifierKind> MapModifiers;
9402 const ValueDecl *Mapper;
9404 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =
9407 for (
const MapData &L1 : ArrayRef(DeclComponentLists).slice(Count)) {
9409 std::tie(Components1, MapType, MapModifiers, IsImplicit, Mapper,
9411 auto CI = Components.rbegin();
9412 auto CE = Components.rend();
9413 auto SI = Components1.rbegin();
9414 auto SE = Components1.rend();
9415 for (; CI != CE && SI != SE; ++CI, ++SI) {
9416 if (CI->getAssociatedExpression()->getStmtClass() !=
9417 SI->getAssociatedExpression()->getStmtClass())
9420 if (CI->getAssociatedDeclaration() != SI->getAssociatedDeclaration())
9425 if (CI == CE || SI == SE) {
9427 if (CI == CE && SI == SE)
9429 const auto It = (SI == SE) ? CI : SI;
9436 (std::prev(It)->getAssociatedDeclaration() &&
9438 ->getAssociatedDeclaration()
9440 ->isPointerType()) ||
9441 (It->getAssociatedDeclaration() &&
9442 It->getAssociatedDeclaration()->getType()->isPointerType() &&
9443 std::next(It) != CE && std::next(It) != SE))
9445 const MapData &BaseData = CI == CE ? L : L1;
9447 SI == SE ? Components : Components1;
9448 OverlappedData[&BaseData].push_back(SubData);
9453 llvm::SmallVector<const FieldDecl *, 4> Layout;
9454 if (!OverlappedData.empty()) {
9457 while (BaseType != OrigType) {
9463 getPlainLayout(CRD, Layout,
false);
9469 for (
auto &Pair : OverlappedData) {
9476 auto CI = First.rbegin();
9477 auto CE = First.rend();
9478 auto SI = Second.rbegin();
9479 auto SE = Second.rend();
9480 for (; CI != CE && SI != SE; ++CI, ++SI) {
9481 if (CI->getAssociatedExpression()->getStmtClass() !=
9482 SI->getAssociatedExpression()->getStmtClass())
9485 if (CI->getAssociatedDeclaration() !=
9486 SI->getAssociatedDeclaration())
9491 if (CI == CE && SI == SE)
9495 if (CI == CE || SI == SE)
9500 if (FD1->getParent() == FD2->getParent())
9501 return FD1->getFieldIndex() < FD2->getFieldIndex();
9503 llvm::find_if(Layout, [FD1, FD2](
const FieldDecl *FD) {
9504 return FD == FD1 || FD == FD2;
9512 bool AddTargetParamFlag = IsListEligibleForTargetParamFlag;
9513 MapCombinedInfoTy StructBaseCombinedInfo;
9514 for (
const auto &Pair : OverlappedData) {
9515 const MapData &L = *Pair.getFirst();
9518 ArrayRef<OpenMPMapModifierKind> MapModifiers;
9520 const ValueDecl *Mapper;
9522 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =
9524 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
9525 OverlappedComponents = Pair.getSecond();
9526 generateInfoForComponentList(
9527 MapType, MapModifiers, {}, Components, CurComponentListInfo,
9528 StructBaseCombinedInfo, PartialStruct, AddTargetParamFlag, IsImplicit,
9530 false, VD, VarRef, OverlappedComponents);
9531 AddTargetParamFlag =
false;
9534 for (
const MapData &L : DeclComponentLists) {
9537 ArrayRef<OpenMPMapModifierKind> MapModifiers;
9539 const ValueDecl *Mapper;
9541 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =
9543 auto It = OverlappedData.find(&L);
9544 if (It == OverlappedData.end())
9545 generateInfoForComponentList(
9546 MapType, MapModifiers, {}, Components, CurComponentListInfo,
9547 StructBaseCombinedInfo, PartialStruct, AddTargetParamFlag,
9548 IsImplicit,
false, Mapper,
9550 {}, AreBothBasePtrAndPteeMapped);
9551 AddTargetParamFlag =
false;
9557 void generateDefaultMapInfo(
const CapturedStmt::Capture &CI,
9558 const FieldDecl &RI, llvm::Value *CV,
9559 MapCombinedInfoTy &CombinedInfo)
const {
9560 bool IsImplicit =
true;
9563 CombinedInfo.Exprs.push_back(
nullptr);
9564 CombinedInfo.BasePointers.push_back(CV);
9565 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
9566 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
9567 CombinedInfo.Pointers.push_back(CV);
9569 CombinedInfo.Sizes.push_back(
9573 CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_TO |
9574 OpenMPOffloadMappingFlags::OMP_MAP_FROM);
9578 CombinedInfo.BasePointers.push_back(CV);
9579 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
9580 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
9581 CombinedInfo.Pointers.push_back(CV);
9585 CombinedInfo.Types.push_back(
9586 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
9587 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
9592 CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_NONE);
9593 CombinedInfo.Sizes.push_back(llvm::Constant::getNullValue(CGF.
Int64Ty));
9595 auto I = FirstPrivateDecls.find(VD);
9596 if (I != FirstPrivateDecls.end())
9597 IsImplicit = I->getSecond();
9602 CombinedInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
9607 CombinedInfo.Types.push_back(getMapModifiersForPrivateClauses(CI));
9609 auto I = FirstPrivateDecls.find(VD);
9611 CombinedInfo.BasePointers.push_back(CV);
9612 CombinedInfo.DevicePtrDecls.push_back(
nullptr);
9613 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
9620 CombinedInfo.Pointers.push_back(CV);
9622 if (I != FirstPrivateDecls.end())
9623 IsImplicit = I->getSecond();
9626 CombinedInfo.Types.back() |=
9627 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
9631 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
9634 CombinedInfo.Mappers.push_back(
nullptr);
9646 dyn_cast<MemberExpr>(OASE->getBase()->IgnoreParenImpCasts()))
9647 return ME->getMemberDecl();
9653static llvm::Constant *
9655 MappableExprsHandler::MappingExprInfo &MapExprs) {
9657 uint32_t SrcLocStrSize;
9658 if (!MapExprs.getMapDecl() && !MapExprs.getMapExpr())
9659 return OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
9662 if (!MapExprs.getMapDecl() && MapExprs.getMapExpr()) {
9666 Loc = MapExprs.getMapExpr()->getExprLoc();
9668 Loc = MapExprs.getMapDecl()->getLocation();
9671 std::string ExprName;
9672 if (MapExprs.getMapExpr()) {
9674 llvm::raw_string_ostream OS(ExprName);
9675 MapExprs.getMapExpr()->printPretty(OS,
nullptr, P);
9677 ExprName = MapExprs.getMapDecl()->getNameAsString();
9686 return OMPBuilder.getOrCreateSrcLocStr(
FileName, ExprName, PLoc.
getLine(),
9693 CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
9695 bool IsNonContiguous =
false,
bool ForEndCall =
false) {
9698 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
9701 InsertPointTy CodeGenIP(CGF.
Builder.GetInsertBlock(),
9702 CGF.
Builder.GetInsertPoint());
9704 auto DeviceAddrCB = [&](
unsigned int I, llvm::Value *NewDecl) {
9705 if (
const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
9710 auto CustomMapperCB = [&](
unsigned int I) {
9711 llvm::Function *MFunc =
nullptr;
9712 if (CombinedInfo.Mappers[I]) {
9713 Info.HasMapper =
true;
9719 cantFail(OMPBuilder.emitOffloadingArraysAndArgs(
9720 AllocaIP, CodeGenIP, Info, Info.RTArgs, CombinedInfo, CustomMapperCB,
9721 IsNonContiguous, ForEndCall, DeviceAddrCB));
9725static const OMPExecutableDirective *
9727 const auto *CS = D.getInnermostCapturedStmt();
9730 const Stmt *ChildStmt =
9733 if (
const auto *NestedDir =
9734 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
9736 switch (D.getDirectiveKind()) {
9742 if (DKind == OMPD_teams) {
9743 Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers(
9748 if (
const auto *NND =
9749 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
9750 DKind = NND->getDirectiveKind();
9756 case OMPD_target_teams:
9760 case OMPD_target_parallel:
9761 case OMPD_target_simd:
9762 case OMPD_target_parallel_for:
9763 case OMPD_target_parallel_for_simd:
9765 case OMPD_target_teams_distribute:
9766 case OMPD_target_teams_distribute_simd:
9767 case OMPD_target_teams_distribute_parallel_for:
9768 case OMPD_target_teams_distribute_parallel_for_simd:
9771 case OMPD_parallel_for:
9772 case OMPD_parallel_master:
9773 case OMPD_parallel_sections:
9775 case OMPD_parallel_for_simd:
9777 case OMPD_cancellation_point:
9779 case OMPD_threadprivate:
9790 case OMPD_taskyield:
9793 case OMPD_taskgroup:
9799 case OMPD_target_data:
9800 case OMPD_target_exit_data:
9801 case OMPD_target_enter_data:
9802 case OMPD_distribute:
9803 case OMPD_distribute_simd:
9804 case OMPD_distribute_parallel_for:
9805 case OMPD_distribute_parallel_for_simd:
9806 case OMPD_teams_distribute:
9807 case OMPD_teams_distribute_simd:
9808 case OMPD_teams_distribute_parallel_for:
9809 case OMPD_teams_distribute_parallel_for_simd:
9810 case OMPD_target_update:
9811 case OMPD_declare_simd:
9812 case OMPD_declare_variant:
9813 case OMPD_begin_declare_variant:
9814 case OMPD_end_declare_variant:
9815 case OMPD_declare_target:
9816 case OMPD_end_declare_target:
9817 case OMPD_declare_reduction:
9818 case OMPD_declare_mapper:
9820 case OMPD_taskloop_simd:
9821 case OMPD_master_taskloop:
9822 case OMPD_master_taskloop_simd:
9823 case OMPD_parallel_master_taskloop:
9824 case OMPD_parallel_master_taskloop_simd:
9826 case OMPD_metadirective:
9829 llvm_unreachable(
"Unexpected directive.");
9874 auto *MapperVarDecl =
9876 CharUnits ElementSize =
C.getTypeSizeInChars(Ty);
9877 llvm::Type *ElemTy =
CGM.getTypes().ConvertTypeForMem(Ty);
9880 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
9881 auto PrivatizeAndGenMapInfoCB =
9882 [&](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP, llvm::Value *PtrPHI,
9883 llvm::Value *BeginArg) -> llvm::OpenMPIRBuilder::MapInfosTy & {
9884 MapperCGF.
Builder.restoreIP(CodeGenIP);
9894 Scope.addPrivate(MapperVarDecl, PtrCurrent);
9895 (void)
Scope.Privatize();
9898 MappableExprsHandler MEHandler(*D, MapperCGF);
9899 MEHandler.generateAllInfoForMapper(CombinedInfo,
OMPBuilder);
9901 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
9904 if (
CGM.getCodeGenOpts().getDebugInfo() !=
9905 llvm::codegenoptions::NoDebugInfo) {
9906 CombinedInfo.Names.resize(CombinedInfo.Exprs.size());
9907 llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),
9911 return CombinedInfo;
9914 auto CustomMapperCB = [&](
unsigned I) {
9915 llvm::Function *MapperFunc =
nullptr;
9916 if (CombinedInfo.Mappers[I]) {
9920 assert(MapperFunc &&
"Expect a valid mapper function is available.");
9926 llvm::raw_svector_ostream Out(TyStr);
9927 CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(Ty, Out);
9930 llvm::Function *NewFn = cantFail(
OMPBuilder.emitUserDefinedMapper(
9931 PrivatizeAndGenMapInfoCB, ElemTy, Name, CustomMapperCB));
9932 UDMMap.try_emplace(D, NewFn);
9956 Kind != OMPD_target_teams_loop)
9959 return llvm::ConstantInt::get(CGF.
Int64Ty, 0);
9962 if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD))
9963 return NumIterations;
9964 return llvm::ConstantInt::get(CGF.
Int64Ty, 0);
9973 if (OffloadingMandatory) {
9974 CGF.
Builder.CreateUnreachable();
9976 if (RequiresOuterTask) {
9977 CapturedVars.clear();
9986 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
9989 llvm::Value *DeviceID;
9990 if (
Device.getPointer()) {
9992 Device.getInt() == OMPC_DEVICE_device_num) &&
9993 "Expected device_num modifier.");
9998 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
10005 llvm::Value *DynCGroupMem = CGF.
Builder.getInt32(0);
10010 DynMemClause->getSize(),
true);
10011 DynCGroupMem = CGF.
Builder.CreateIntCast(DynCGroupMemVal, CGF.
Int32Ty,
10014 return DynCGroupMem;
10019 llvm::OpenMPIRBuilder &OMPBuilder,
10021 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) {
10023 llvm::DenseMap<llvm::Value *, llvm::Value *> LambdaPointers;
10025 auto *CV = CapturedVars.begin();
10028 CI != CE; ++CI, ++RI, ++CV) {
10029 MappableExprsHandler::MapCombinedInfoTy CurInfo;
10034 CurInfo.Exprs.push_back(
nullptr);
10035 CurInfo.BasePointers.push_back(*CV);
10036 CurInfo.DevicePtrDecls.push_back(
nullptr);
10037 CurInfo.DevicePointers.push_back(
10038 MappableExprsHandler::DeviceInfoTy::None);
10039 CurInfo.Pointers.push_back(*CV);
10040 CurInfo.Sizes.push_back(CGF.
Builder.CreateIntCast(
10043 CurInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
10044 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM |
10045 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
10046 CurInfo.Mappers.push_back(
nullptr);
10050 MEHandler.generateInfoForCaptureFromClauseInfo(
10051 CI, *CV, CurInfo, OMPBuilder,
10052 CombinedInfo.BasePointers.size());
10057 MappedVarSet.insert(
nullptr);
10059 if (CurInfo.BasePointers.empty())
10060 MEHandler.generateDefaultMapInfo(*CI, **RI, *CV, CurInfo);
10065 MEHandler.generateInfoForLambdaCaptures(CI->
getCapturedVar(), *CV,
10066 CurInfo, LambdaPointers);
10069 assert(!CurInfo.BasePointers.empty() &&
10070 "Non-existing map pointer for capture!");
10071 assert(CurInfo.BasePointers.size() == CurInfo.Pointers.size() &&
10072 CurInfo.BasePointers.size() == CurInfo.Sizes.size() &&
10073 CurInfo.BasePointers.size() == CurInfo.Types.size() &&
10074 CurInfo.BasePointers.size() == CurInfo.Mappers.size() &&
10075 "Inconsistent map information sizes!");
10078 CombinedInfo.append(CurInfo);
10081 MEHandler.adjustMemberOfForLambdaCaptures(
10082 OMPBuilder, LambdaPointers, CombinedInfo.BasePointers,
10083 CombinedInfo.Pointers, CombinedInfo.Types);
10087 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
10088 llvm::OpenMPIRBuilder &OMPBuilder,
10095 MEHandler.generateAllInfo(CombinedInfo, OMPBuilder, SkippedVarSet);
10097 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
10101 llvm::codegenoptions::NoDebugInfo) {
10102 CombinedInfo.Names.resize(CombinedInfo.Exprs.size());
10103 llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),
10111 llvm::OpenMPIRBuilder &OMPBuilder,
10112 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) {
10114 MappableExprsHandler MEHandler(D, CGF);
10115 llvm::DenseSet<CanonicalDeclPtr<const Decl>> MappedVarSet;
10118 MappedVarSet, CombinedInfo);
10119 genMapInfo(MEHandler, CGF, CombinedInfo, OMPBuilder, MappedVarSet);
10122template <
typename ClauseTy>
10127 const auto *
C = D.getSingleClause<ClauseTy>();
10128 assert(!
C->varlist_empty() &&
10129 "ompx_bare requires explicit num_teams and thread_limit");
10131 for (
auto *E :
C->varlist()) {
10143 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
10145 llvm::Value *&MapTypesArray, llvm::Value *&MapNamesArray,
10150 llvm::OpenMPIRBuilder &OMPBuilder = OMPRuntime->
getOMPBuilder();
10153 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
10155 genMapInfo(D, CGF, CS, CapturedVars, OMPBuilder, CombinedInfo);
10169 MapTypesArray = Info.RTArgs.MapTypesArray;
10170 MapNamesArray = Info.RTArgs.MapNamesArray;
10172 auto &&ThenGen = [&OMPRuntime, OutlinedFn, &D, &CapturedVars,
10173 RequiresOuterTask, &CS, OffloadingMandatory,
Device,
10174 OutlinedFnID, &InputInfo, &MapTypesArray, &MapNamesArray,
10176 bool IsReverseOffloading =
Device.getInt() == OMPC_DEVICE_ancestor;
10178 if (IsReverseOffloading) {
10184 RequiresOuterTask, CS, OffloadingMandatory, CGF);
10189 unsigned NumTargetItems = InputInfo.NumberOfTargetItems;
10191 llvm::Value *BasePointersArray =
10192 InputInfo.BasePointersArray.emitRawPointer(CGF);
10193 llvm::Value *PointersArray = InputInfo.PointersArray.emitRawPointer(CGF);
10194 llvm::Value *SizesArray = InputInfo.SizesArray.emitRawPointer(CGF);
10195 llvm::Value *MappersArray = InputInfo.MappersArray.emitRawPointer(CGF);
10197 auto &&EmitTargetCallFallbackCB =
10198 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
10199 OffloadingMandatory, &CGF](llvm::OpenMPIRBuilder::InsertPointTy IP)
10200 -> llvm::OpenMPIRBuilder::InsertPointTy {
10203 RequiresOuterTask, CS, OffloadingMandatory, CGF);
10216 NumThreads.push_back(
10222 llvm::Value *NumIterations =
10225 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
10228 llvm::OpenMPIRBuilder::TargetDataRTArgs RTArgs(
10229 BasePointersArray, PointersArray, SizesArray, MapTypesArray,
10230 nullptr , MappersArray, MapNamesArray);
10232 llvm::OpenMPIRBuilder::TargetKernelArgs Args(
10233 NumTargetItems, RTArgs, NumIterations, NumTeams, NumThreads,
10234 DynCGGroupMem, HasNoWait);
10236 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
10238 CGF.
Builder, OutlinedFnID, EmitTargetCallFallbackCB, Args, DeviceID,
10240 CGF.
Builder.restoreIP(AfterIP);
10243 if (RequiresOuterTask)
10258 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
10261 RequiresOuterTask, CS, OffloadingMandatory, CGF);
10264 if (RequiresOuterTask) {
10274 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID,
const Expr *IfCond,
10275 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
10282 const bool OffloadingMandatory = !
CGM.getLangOpts().OpenMPIsTargetDevice &&
10283 CGM.getLangOpts().OpenMPOffloadMandatory;
10285 assert((OffloadingMandatory || OutlinedFn) &&
"Invalid outlined function!");
10287 const bool RequiresOuterTask =
10291 (
CGM.getLangOpts().OpenMP >= 51 &&
10295 const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target);
10303 llvm::Value *MapTypesArray =
nullptr;
10304 llvm::Value *MapNamesArray =
nullptr;
10306 auto &&TargetThenGen = [
this, OutlinedFn, &D, &CapturedVars,
10307 RequiresOuterTask, &CS, OffloadingMandatory,
Device,
10308 OutlinedFnID, &InputInfo, &MapTypesArray,
10312 RequiresOuterTask, CS, OffloadingMandatory,
10313 Device, OutlinedFnID, InputInfo, MapTypesArray,
10314 MapNamesArray, SizeEmitter, CGF,
CGM);
10317 auto &&TargetElseGen =
10318 [
this, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
10321 CS, OffloadingMandatory, CGF);
10328 if (OutlinedFnID) {
10330 emitIfClause(CGF, IfCond, TargetThenGen, TargetElseGen);
10342 StringRef ParentName) {
10347 bool RequiresDeviceCodegen =
10352 if (RequiresDeviceCodegen) {
10360 if (!
OMPBuilder.OffloadInfoManager.hasTargetRegionEntryInfo(EntryInfo))
10363 switch (E.getDirectiveKind()) {
10368 case OMPD_target_parallel:
10372 case OMPD_target_teams:
10376 case OMPD_target_teams_distribute:
10380 case OMPD_target_teams_distribute_simd:
10384 case OMPD_target_parallel_for:
10388 case OMPD_target_parallel_for_simd:
10392 case OMPD_target_simd:
10396 case OMPD_target_teams_distribute_parallel_for:
10401 case OMPD_target_teams_distribute_parallel_for_simd:
10407 case OMPD_target_teams_loop:
10411 case OMPD_target_parallel_loop:
10415 case OMPD_parallel:
10417 case OMPD_parallel_for:
10418 case OMPD_parallel_master:
10419 case OMPD_parallel_sections:
10420 case OMPD_for_simd:
10421 case OMPD_parallel_for_simd:
10423 case OMPD_cancellation_point:
10425 case OMPD_threadprivate:
10426 case OMPD_allocate:
10431 case OMPD_sections:
10435 case OMPD_critical:
10436 case OMPD_taskyield:
10438 case OMPD_taskwait:
10439 case OMPD_taskgroup:
10445 case OMPD_target_data:
10446 case OMPD_target_exit_data:
10447 case OMPD_target_enter_data:
10448 case OMPD_distribute:
10449 case OMPD_distribute_simd:
10450 case OMPD_distribute_parallel_for:
10451 case OMPD_distribute_parallel_for_simd:
10452 case OMPD_teams_distribute:
10453 case OMPD_teams_distribute_simd:
10454 case OMPD_teams_distribute_parallel_for:
10455 case OMPD_teams_distribute_parallel_for_simd:
10456 case OMPD_target_update:
10457 case OMPD_declare_simd:
10458 case OMPD_declare_variant:
10459 case OMPD_begin_declare_variant:
10460 case OMPD_end_declare_variant:
10461 case OMPD_declare_target:
10462 case OMPD_end_declare_target:
10463 case OMPD_declare_reduction:
10464 case OMPD_declare_mapper:
10465 case OMPD_taskloop:
10466 case OMPD_taskloop_simd:
10467 case OMPD_master_taskloop:
10468 case OMPD_master_taskloop_simd:
10469 case OMPD_parallel_master_taskloop:
10470 case OMPD_parallel_master_taskloop_simd:
10471 case OMPD_requires:
10472 case OMPD_metadirective:
10475 llvm_unreachable(
"Unknown target directive for OpenMP device codegen.");
10480 if (
const auto *E = dyn_cast<OMPExecutableDirective>(S)) {
10481 if (!E->hasAssociatedStmt() || !E->getAssociatedStmt())
10489 if (
const auto *L = dyn_cast<LambdaExpr>(S))
10498 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
10499 OMPDeclareTargetDeclAttr::getDeviceType(VD);
10503 if (!IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_NoHost)
10506 if (IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_Host)
10514 if (!
CGM.getLangOpts().OpenMPIsTargetDevice) {
10515 if (
const auto *FD = dyn_cast<FunctionDecl>(GD.
getDecl()))
10517 CGM.getLangOpts().OpenMPIsTargetDevice))
10524 if (
const auto *FD = dyn_cast<FunctionDecl>(VD)) {
10525 StringRef Name =
CGM.getMangledName(GD);
10528 CGM.getLangOpts().OpenMPIsTargetDevice))
10533 return !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD) &&
10539 CGM.getLangOpts().OpenMPIsTargetDevice))
10542 if (!
CGM.getLangOpts().OpenMPIsTargetDevice)
10551 StringRef ParentName =
10556 StringRef ParentName =
10563 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
10564 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(
10566 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
10567 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
10568 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
10577 llvm::Constant *
Addr) {
10578 if (
CGM.getLangOpts().OMPTargetTriples.empty() &&
10579 !
CGM.getLangOpts().OpenMPIsTargetDevice)
10582 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
10583 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
10587 if (Res && *Res != OMPDeclareTargetDeclAttr::MT_Link &&
10592 if (
CGM.getLangOpts().OpenMPIsTargetDevice) {
10595 StringRef VarName =
CGM.getMangledName(VD);
10601 auto AddrOfGlobal = [&VD,
this]() {
return CGM.GetAddrOfGlobal(VD); };
10602 auto LinkageForVariable = [&VD,
this]() {
10603 return CGM.getLLVMLinkageVarDefinition(VD);
10606 std::vector<llvm::GlobalVariable *> GeneratedRefs;
10613 CGM.getMangledName(VD), GeneratedRefs,
CGM.getLangOpts().OpenMPSimd,
10614 CGM.getLangOpts().OMPTargetTriples, AddrOfGlobal, LinkageForVariable,
10615 CGM.getTypes().ConvertTypeForMem(
10616 CGM.getContext().getPointerType(VD->
getType())),
10619 for (
auto *ref : GeneratedRefs)
10620 CGM.addCompilerUsedGlobal(ref);
10633 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
10634 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
10637 if ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
10638 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
10640 CGM.EmitGlobal(VD);
10642 assert((*Res == OMPDeclareTargetDeclAttr::MT_Link ||
10643 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
10644 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
10646 "Expected link clause or to clause with unified memory.");
10647 (void)
CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
10655 " Expected target-based directive.");
10660 if (Clause->getClauseKind() == OMPC_unified_shared_memory) {
10662 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(
true);
10663 }
else if (
const auto *AC =
10664 dyn_cast<OMPAtomicDefaultMemOrderClause>(Clause)) {
10665 switch (AC->getAtomicDefaultMemOrderKind()) {
10666 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_acq_rel:
10669 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_seq_cst:
10672 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_relaxed:
10688 if (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())
10690 const auto *A = VD->
getAttr<OMPAllocateDeclAttr>();
10691 switch(A->getAllocatorType()) {
10692 case OMPAllocateDeclAttr::OMPNullMemAlloc:
10693 case OMPAllocateDeclAttr::OMPDefaultMemAlloc:
10695 case OMPAllocateDeclAttr::OMPLargeCapMemAlloc:
10696 case OMPAllocateDeclAttr::OMPCGroupMemAlloc:
10697 case OMPAllocateDeclAttr::OMPHighBWMemAlloc:
10698 case OMPAllocateDeclAttr::OMPLowLatMemAlloc:
10699 case OMPAllocateDeclAttr::OMPThreadMemAlloc:
10700 case OMPAllocateDeclAttr::OMPConstMemAlloc:
10701 case OMPAllocateDeclAttr::OMPPTeamMemAlloc:
10704 case OMPAllocateDeclAttr::OMPUserDefinedMemAlloc:
10705 llvm_unreachable(
"Expected predefined allocator for the variables with the "
10706 "static storage.");
10718 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
10719 SavedShouldMarkAsGlobal = CGM.getOpenMPRuntime().ShouldMarkAsGlobal;
10720 CGM.getOpenMPRuntime().ShouldMarkAsGlobal = false;
10725 if (CGM.getLangOpts().OpenMPIsTargetDevice)
10726 CGM.getOpenMPRuntime().ShouldMarkAsGlobal = SavedShouldMarkAsGlobal;
10736 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(D)) {
10738 if (
auto *F = dyn_cast_or_null<llvm::Function>(
10739 CGM.GetGlobalValue(
CGM.getMangledName(GD))))
10740 return !F->isDeclaration();
10752 llvm::Function *OutlinedFn,
10761 llvm::Value *Args[] = {
10763 CGF.
Builder.getInt32(CapturedVars.size()),
10766 RealArgs.append(std::begin(Args), std::end(Args));
10767 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
10769 llvm::FunctionCallee RTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
10770 CGM.getModule(), OMPRTL___kmpc_fork_teams);
10775 const Expr *NumTeams,
10776 const Expr *ThreadLimit,
10783 llvm::Value *NumTeamsVal =
10789 llvm::Value *ThreadLimitVal =
10796 llvm::Value *PushNumTeamsArgs[] = {RTLoc,
getThreadID(CGF, Loc), NumTeamsVal,
10799 CGM.getModule(), OMPRTL___kmpc_push_num_teams),
10804 const Expr *ThreadLimit,
10807 llvm::Value *ThreadLimitVal =
10814 llvm::Value *ThreadLimitArgs[] = {RTLoc,
getThreadID(CGF, Loc),
10817 CGM.getModule(), OMPRTL___kmpc_set_thread_limit),
10832 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
10834 llvm::Value *IfCondVal =
nullptr;
10839 llvm::Value *DeviceID =
nullptr;
10844 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
10848 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
10849 auto GenMapInfoCB =
10850 [&](InsertPointTy CodeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {
10851 CGF.
Builder.restoreIP(CodeGenIP);
10853 MappableExprsHandler MEHandler(D, CGF);
10854 MEHandler.generateAllInfo(CombinedInfo,
OMPBuilder);
10856 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
10859 if (
CGM.getCodeGenOpts().getDebugInfo() !=
10860 llvm::codegenoptions::NoDebugInfo) {
10861 CombinedInfo.Names.resize(CombinedInfo.Exprs.size());
10862 llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),
10866 return CombinedInfo;
10868 using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;
10869 auto BodyCB = [&](InsertPointTy CodeGenIP, BodyGenTy BodyGenType) {
10870 CGF.
Builder.restoreIP(CodeGenIP);
10871 switch (BodyGenType) {
10872 case BodyGenTy::Priv:
10876 case BodyGenTy::DupNoPriv:
10878 CodeGen.setAction(NoPrivAction);
10882 case BodyGenTy::NoPriv:
10884 CodeGen.setAction(NoPrivAction);
10889 return InsertPointTy(CGF.
Builder.GetInsertBlock(),
10890 CGF.
Builder.GetInsertPoint());
10893 auto DeviceAddrCB = [&](
unsigned int I, llvm::Value *NewDecl) {
10894 if (
const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
10899 auto CustomMapperCB = [&](
unsigned int I) {
10900 llvm::Function *MFunc =
nullptr;
10901 if (CombinedInfo.Mappers[I]) {
10902 Info.HasMapper =
true;
10914 InsertPointTy CodeGenIP(CGF.
Builder.GetInsertBlock(),
10915 CGF.
Builder.GetInsertPoint());
10916 llvm::OpenMPIRBuilder::LocationDescription OmpLoc(CodeGenIP);
10917 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
10919 OmpLoc, AllocaIP, CodeGenIP, DeviceID, IfCondVal, Info, GenMapInfoCB,
10921 nullptr, BodyCB, DeviceAddrCB, RTLoc));
10922 CGF.
Builder.restoreIP(AfterIP);
10934 "Expecting either target enter, exit data, or update directives.");
10937 llvm::Value *MapTypesArray =
nullptr;
10938 llvm::Value *MapNamesArray =
nullptr;
10940 auto &&ThenGen = [
this, &D,
Device, &InputInfo, &MapTypesArray,
10943 llvm::Value *DeviceID =
nullptr;
10948 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
10952 llvm::Constant *PointerNum =
10959 {RTLoc, DeviceID, PointerNum,
10968 RuntimeFunction RTLFn;
10969 switch (D.getDirectiveKind()) {
10970 case OMPD_target_enter_data:
10971 RTLFn = HasNowait ? OMPRTL___tgt_target_data_begin_nowait_mapper
10972 : OMPRTL___tgt_target_data_begin_mapper;
10974 case OMPD_target_exit_data:
10975 RTLFn = HasNowait ? OMPRTL___tgt_target_data_end_nowait_mapper
10976 : OMPRTL___tgt_target_data_end_mapper;
10978 case OMPD_target_update:
10979 RTLFn = HasNowait ? OMPRTL___tgt_target_data_update_nowait_mapper
10980 : OMPRTL___tgt_target_data_update_mapper;
10982 case OMPD_parallel:
10984 case OMPD_parallel_for:
10985 case OMPD_parallel_master:
10986 case OMPD_parallel_sections:
10987 case OMPD_for_simd:
10988 case OMPD_parallel_for_simd:
10990 case OMPD_cancellation_point:
10992 case OMPD_threadprivate:
10993 case OMPD_allocate:
10998 case OMPD_sections:
11002 case OMPD_critical:
11003 case OMPD_taskyield:
11005 case OMPD_taskwait:
11006 case OMPD_taskgroup:
11012 case OMPD_target_data:
11013 case OMPD_distribute:
11014 case OMPD_distribute_simd:
11015 case OMPD_distribute_parallel_for:
11016 case OMPD_distribute_parallel_for_simd:
11017 case OMPD_teams_distribute:
11018 case OMPD_teams_distribute_simd:
11019 case OMPD_teams_distribute_parallel_for:
11020 case OMPD_teams_distribute_parallel_for_simd:
11021 case OMPD_declare_simd:
11022 case OMPD_declare_variant:
11023 case OMPD_begin_declare_variant:
11024 case OMPD_end_declare_variant:
11025 case OMPD_declare_target:
11026 case OMPD_end_declare_target:
11027 case OMPD_declare_reduction:
11028 case OMPD_declare_mapper:
11029 case OMPD_taskloop:
11030 case OMPD_taskloop_simd:
11031 case OMPD_master_taskloop:
11032 case OMPD_master_taskloop_simd:
11033 case OMPD_parallel_master_taskloop:
11034 case OMPD_parallel_master_taskloop_simd:
11036 case OMPD_target_simd:
11037 case OMPD_target_teams_distribute:
11038 case OMPD_target_teams_distribute_simd:
11039 case OMPD_target_teams_distribute_parallel_for:
11040 case OMPD_target_teams_distribute_parallel_for_simd:
11041 case OMPD_target_teams:
11042 case OMPD_target_parallel:
11043 case OMPD_target_parallel_for:
11044 case OMPD_target_parallel_for_simd:
11045 case OMPD_requires:
11046 case OMPD_metadirective:
11049 llvm_unreachable(
"Unexpected standalone target data directive.");
11053 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.
Int32Ty));
11054 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.
VoidPtrTy));
11055 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.
Int32Ty));
11056 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.
VoidPtrTy));
11059 OMPBuilder.getOrCreateRuntimeFunction(
CGM.getModule(), RTLFn),
11063 auto &&TargetThenGen = [
this, &ThenGen, &D, &InputInfo, &MapTypesArray,
11067 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
11069 MappableExprsHandler MEHandler(D, CGF);
11081 CGM.getPointerAlign());
11086 MapTypesArray = Info.RTArgs.MapTypesArray;
11087 MapNamesArray = Info.RTArgs.MapNamesArray;
11088 if (RequiresOuterTask)
11114struct ParamAttrTy {
11115 ParamKindTy Kind =
Vector;
11116 llvm::APSInt StrideOrArg;
11117 llvm::APSInt Alignment;
11118 bool HasVarStride =
false;
11151 unsigned Offset = 0;
11152 if (
const auto *MD = dyn_cast<CXXMethodDecl>(FD)) {
11153 if (ParamAttrs[Offset].Kind ==
Vector)
11154 CDT =
C.getPointerType(
C.getCanonicalTagType(MD->
getParent()));
11158 for (
unsigned I = 0, E = FD->
getNumParams(); I < E; ++I) {
11159 if (ParamAttrs[I + Offset].Kind ==
Vector) {
11171 return C.getTypeSize(CDT);
11179 llvm::raw_svector_ostream Out(Buffer);
11180 for (
const auto &ParamAttr : ParamAttrs) {
11181 switch (ParamAttr.Kind) {
11201 if (ParamAttr.HasVarStride)
11202 Out <<
"s" << ParamAttr.StrideOrArg;
11203 else if (ParamAttr.Kind == Linear || ParamAttr.Kind == LinearRef ||
11204 ParamAttr.Kind == LinearUVal || ParamAttr.Kind == LinearVal) {
11207 if (ParamAttr.StrideOrArg < 0)
11208 Out <<
'n' << -ParamAttr.StrideOrArg;
11209 else if (ParamAttr.StrideOrArg != 1)
11210 Out << ParamAttr.StrideOrArg;
11213 if (!!ParamAttr.Alignment)
11214 Out <<
'a' << ParamAttr.Alignment;
11217 return std::string(Out.str());
11222 const llvm::APSInt &VLENVal,
11224 OMPDeclareSimdDeclAttr::BranchStateTy State) {
11227 unsigned VecRegSize;
11229 ISADataTy ISAData[] = {
11245 case OMPDeclareSimdDeclAttr::BS_Undefined:
11246 Masked.push_back(
'N');
11247 Masked.push_back(
'M');
11249 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
11250 Masked.push_back(
'N');
11252 case OMPDeclareSimdDeclAttr::BS_Inbranch:
11253 Masked.push_back(
'M');
11256 for (
char Mask : Masked) {
11257 for (
const ISADataTy &
Data : ISAData) {
11259 llvm::raw_svector_ostream Out(Buffer);
11260 Out <<
"_ZGV" <<
Data.ISA << Mask;
11263 assert(NumElts &&
"Non-zero simdlen/cdtsize expected");
11264 Out << llvm::APSInt::getUnsigned(
Data.VecRegSize / NumElts);
11269 Out <<
'_' << Fn->getName();
11270 Fn->addFnAttr(Out.str());
11288 if (Kind == ParamKindTy::Uniform)
11291 if (Kind == ParamKindTy::LinearUVal || Kind == ParamKindTy::LinearRef)
11294 if ((Kind == ParamKindTy::Linear || Kind == ParamKindTy::LinearVal) &&
11304 unsigned Size =
C.getTypeSize(QT);
11307 if (Size != 8 && Size != 16 && Size != 32 && Size != 64 && Size != 128)
11331 return C.getTypeSize(PTy);
11334 return C.getTypeSize(QT);
11336 return C.getTypeSize(
C.getUIntPtrType());
11342static std::tuple<unsigned, unsigned, bool>
11348 bool OutputBecomesInput =
false;
11352 Sizes.push_back(
getAArch64LS(RetType, ParamKindTy::Vector,
C));
11354 OutputBecomesInput =
true;
11356 for (
unsigned I = 0, E = FD->
getNumParams(); I < E; ++I) {
11361 assert(!Sizes.empty() &&
"Unable to determine NDS and WDS.");
11364 assert(llvm::all_of(Sizes,
11365 [](
unsigned Size) {
11366 return Size == 8 || Size == 16 || Size == 32 ||
11367 Size == 64 || Size == 128;
11371 return std::make_tuple(*llvm::min_element(Sizes), *llvm::max_element(Sizes),
11372 OutputBecomesInput);
11378template <
typename T>
11380 char ISA, StringRef ParSeq,
11381 StringRef MangledName,
bool OutputBecomesInput,
11382 llvm::Function *Fn) {
11384 llvm::raw_svector_ostream Out(Buffer);
11385 Out << Prefix << ISA << LMask << VLEN;
11386 if (OutputBecomesInput)
11388 Out << ParSeq <<
"_" << MangledName;
11389 Fn->addFnAttr(Out.str());
11395 StringRef Prefix,
char ISA,
11396 StringRef ParSeq, StringRef MangledName,
11397 bool OutputBecomesInput,
11398 llvm::Function *Fn) {
11402 OutputBecomesInput, Fn);
11404 OutputBecomesInput, Fn);
11408 OutputBecomesInput, Fn);
11410 OutputBecomesInput, Fn);
11414 OutputBecomesInput, Fn);
11416 OutputBecomesInput, Fn);
11421 OutputBecomesInput, Fn);
11424 llvm_unreachable(
"Scalar type is too wide.");
11432 OMPDeclareSimdDeclAttr::BranchStateTy State, StringRef MangledName,
11433 char ISA,
unsigned VecRegSize, llvm::Function *Fn,
SourceLocation SLoc) {
11437 const unsigned NDS = std::get<0>(
Data);
11438 const unsigned WDS = std::get<1>(
Data);
11439 const bool OutputBecomesInput = std::get<2>(
Data);
11443 if (UserVLEN == 1) {
11446 "The clause simdlen(1) has no effect when targeting aarch64.");
11453 if (ISA ==
'n' && UserVLEN && !llvm::isPowerOf2_32(UserVLEN)) {
11456 "power of 2 when targeting Advanced SIMD.");
11463 if (ISA ==
's' && UserVLEN != 0) {
11464 if ((UserVLEN * WDS > 2048) || (UserVLEN * WDS % 128 != 0)) {
11467 "lanes in the architectural constraints "
11468 "for SVE (min is 128-bit, max is "
11469 "2048-bit, by steps of 128-bit)");
11477 StringRef Prefix =
"_ZGV";
11483 OutputBecomesInput, Fn);
11485 assert(ISA ==
'n' &&
"Expected ISA either 's' or 'n'.");
11489 case OMPDeclareSimdDeclAttr::BS_Undefined:
11491 OutputBecomesInput, Fn);
11493 OutputBecomesInput, Fn);
11495 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
11497 OutputBecomesInput, Fn);
11499 case OMPDeclareSimdDeclAttr::BS_Inbranch:
11501 OutputBecomesInput, Fn);
11511 OutputBecomesInput, Fn);
11513 assert(ISA ==
'n' &&
"Expected ISA either 's' or 'n'.");
11518 case OMPDeclareSimdDeclAttr::BS_Undefined:
11520 OutputBecomesInput, Fn);
11522 OutputBecomesInput, Fn);
11524 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
11526 OutputBecomesInput, Fn);
11528 case OMPDeclareSimdDeclAttr::BS_Inbranch:
11530 OutputBecomesInput, Fn);
11538 llvm::Function *Fn) {
11543 llvm::DenseMap<const Decl *, unsigned> ParamPositions;
11545 ParamPositions.try_emplace(FD, 0);
11546 unsigned ParamPos = ParamPositions.size();
11548 ParamPositions.try_emplace(P->getCanonicalDecl(), ParamPos);
11554 for (
const Expr *E :
Attr->uniforms()) {
11558 Pos = ParamPositions[FD];
11561 ->getCanonicalDecl();
11562 auto It = ParamPositions.find(PVD);
11563 assert(It != ParamPositions.end() &&
"Function parameter not found");
11566 ParamAttrs[Pos].Kind = Uniform;
11569 auto *NI =
Attr->alignments_begin();
11570 for (
const Expr *E :
Attr->aligneds()) {
11575 Pos = ParamPositions[FD];
11579 ->getCanonicalDecl();
11580 auto It = ParamPositions.find(PVD);
11581 assert(It != ParamPositions.end() &&
"Function parameter not found");
11583 ParmTy = PVD->getType();
11585 ParamAttrs[Pos].Alignment =
11587 ? (*NI)->EvaluateKnownConstInt(
C)
11588 : llvm::APSInt::getUnsigned(
11589 C.toCharUnitsFromBits(
C.getOpenMPDefaultSimdAlign(ParmTy))
11594 auto *SI =
Attr->steps_begin();
11595 auto *MI =
Attr->modifiers_begin();
11596 for (
const Expr *E :
Attr->linears()) {
11599 bool IsReferenceType =
false;
11602 unsigned PtrRescalingFactor = 1;
11604 Pos = ParamPositions[FD];
11606 PtrRescalingFactor =
CGM.getContext()
11607 .getTypeSizeInChars(P->getPointeeType())
11611 ->getCanonicalDecl();
11612 auto It = ParamPositions.find(PVD);
11613 assert(It != ParamPositions.end() &&
"Function parameter not found");
11615 if (
auto *P = dyn_cast<PointerType>(PVD->getType()))
11616 PtrRescalingFactor =
CGM.getContext()
11617 .getTypeSizeInChars(P->getPointeeType())
11619 else if (PVD->getType()->isReferenceType()) {
11620 IsReferenceType =
true;
11621 PtrRescalingFactor =
11623 .getTypeSizeInChars(PVD->getType().getNonReferenceType())
11627 ParamAttrTy &ParamAttr = ParamAttrs[Pos];
11628 if (*MI == OMPC_LINEAR_ref)
11629 ParamAttr.Kind = LinearRef;
11630 else if (*MI == OMPC_LINEAR_uval)
11631 ParamAttr.Kind = LinearUVal;
11632 else if (IsReferenceType)
11633 ParamAttr.Kind = LinearVal;
11635 ParamAttr.Kind = Linear;
11637 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(1);
11641 if (
const auto *DRE =
11643 if (
const auto *StridePVD =
11644 dyn_cast<ParmVarDecl>(DRE->getDecl())) {
11645 ParamAttr.HasVarStride =
true;
11646 auto It = ParamPositions.find(StridePVD->getCanonicalDecl());
11647 assert(It != ParamPositions.end() &&
11648 "Function parameter not found");
11649 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(It->second);
11653 ParamAttr.StrideOrArg =
Result.Val.getInt();
11659 if (!ParamAttr.HasVarStride &&
11660 (ParamAttr.Kind == Linear || ParamAttr.Kind == LinearRef))
11661 ParamAttr.StrideOrArg = ParamAttr.StrideOrArg * PtrRescalingFactor;
11665 llvm::APSInt VLENVal;
11667 const Expr *VLENExpr =
Attr->getSimdlen();
11672 OMPDeclareSimdDeclAttr::BranchStateTy State =
Attr->getBranchState();
11673 if (
CGM.getTriple().isX86()) {
11675 }
else if (
CGM.getTriple().getArch() == llvm::Triple::aarch64) {
11676 unsigned VLEN = VLENVal.getExtValue();
11677 StringRef MangledName = Fn->getName();
11678 if (
CGM.getTarget().hasFeature(
"sve"))
11680 MangledName,
's', 128, Fn, ExprLoc);
11681 else if (
CGM.getTarget().hasFeature(
"neon"))
11683 MangledName,
'n', 128, Fn, ExprLoc);
11692class DoacrossCleanupTy final :
public EHScopeStack::Cleanup {
11694 static const int DoacrossFinArgs = 2;
11697 llvm::FunctionCallee RTLFn;
11698 llvm::Value *Args[DoacrossFinArgs];
11701 DoacrossCleanupTy(llvm::FunctionCallee RTLFn,
11704 assert(CallArgs.size() == DoacrossFinArgs);
11705 std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args));
11722 QualType Int64Ty =
C.getIntTypeForBitwidth(64,
true);
11730 RD =
C.buildImplicitRecord(
"kmp_dim");
11738 RD =
KmpDimTy->castAsRecordDecl();
11740 llvm::APInt Size(32, NumIterations.size());
11746 enum { LowerFD = 0, UpperFD, StrideFD };
11748 for (
unsigned I = 0, E = NumIterations.size(); I < E; ++I) {
11753 DimsLVal, *std::next(RD->
field_begin(), UpperFD));
11755 CGF.
EmitScalarExpr(NumIterations[I]), NumIterations[I]->getType(),
11756 Int64Ty, NumIterations[I]->getExprLoc());
11760 DimsLVal, *std::next(RD->
field_begin(), StrideFD));
11767 llvm::Value *Args[] = {
11770 llvm::ConstantInt::getSigned(
CGM.Int32Ty, NumIterations.size()),
11775 llvm::FunctionCallee RTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
11776 CGM.getModule(), OMPRTL___kmpc_doacross_init);
11778 llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = {
11780 llvm::FunctionCallee FiniRTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
11781 CGM.getModule(), OMPRTL___kmpc_doacross_fini);
11786template <
typename T>
11788 const T *
C, llvm::Value *ULoc,
11789 llvm::Value *ThreadID) {
11792 llvm::APInt Size(32,
C->getNumLoops());
11796 for (
unsigned I = 0, E =
C->getNumLoops(); I < E; ++I) {
11797 const Expr *CounterVal =
C->getLoopData(I);
11798 assert(CounterVal);
11805 llvm::Value *Args[] = {
11808 llvm::FunctionCallee RTLFn;
11810 OMPDoacrossKind<T> ODK;
11811 if (ODK.isSource(
C)) {
11813 OMPRTL___kmpc_doacross_post);
11815 assert(ODK.isSink(
C) &&
"Expect sink modifier.");
11817 OMPRTL___kmpc_doacross_wait);
11837 llvm::FunctionCallee Callee,
11839 assert(Loc.
isValid() &&
"Outlined function call location must be valid.");
11842 if (
auto *Fn = dyn_cast<llvm::Function>(Callee.getCallee())) {
11843 if (Fn->doesNotThrow()) {
11854 emitCall(CGF, Loc, OutlinedFn, Args);
11858 if (
const auto *FD = dyn_cast<FunctionDecl>(D))
11859 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(FD))
11865 const VarDecl *TargetParam)
const {
11872 const Expr *Allocator) {
11873 llvm::Value *AllocVal;
11883 AllocVal = llvm::Constant::getNullValue(
11893 if (!AllocateAlignment)
11896 return llvm::ConstantInt::get(
CGM.
SizeTy, AllocateAlignment->getQuantity());
11909 auto I = UntiedData.find(VD);
11910 if (I != UntiedData.end()) {
11911 UntiedAddr = I->second.first;
11912 UntiedRealAddr = I->second.second;
11916 if (CVD->
hasAttr<OMPAllocateDeclAttr>()) {
11925 Size = CGF.
Builder.CreateNUWAdd(
11927 Size = CGF.
Builder.CreateUDiv(Size,
CGM.getSize(Align));
11928 Size = CGF.
Builder.CreateNUWMul(Size,
CGM.getSize(Align));
11934 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
11935 const Expr *Allocator = AA->getAllocator();
11939 Args.push_back(ThreadID);
11941 Args.push_back(Alignment);
11942 Args.push_back(Size);
11943 Args.push_back(AllocVal);
11944 llvm::omp::RuntimeFunction FnID =
11945 Alignment ? OMPRTL___kmpc_aligned_alloc : OMPRTL___kmpc_alloc;
11947 OMPBuilder.getOrCreateRuntimeFunction(
CGM.getModule(), FnID), Args,
11949 llvm::FunctionCallee FiniRTLFn =
OMPBuilder.getOrCreateRuntimeFunction(
11950 CGM.getModule(), OMPRTL___kmpc_free);
11958 class OMPAllocateCleanupTy final :
public EHScopeStack::Cleanup {
11959 llvm::FunctionCallee RTLFn;
11962 const Expr *AllocExpr;
11965 OMPAllocateCleanupTy(llvm::FunctionCallee RTLFn,
11967 const Expr *AllocExpr)
11968 : RTLFn(RTLFn), LocEncoding(LocEncoding),
Addr(
Addr),
11969 AllocExpr(AllocExpr) {}
11973 llvm::Value *Args[3];
11979 Args[2] = AllocVal;
11987 CGF.
EHStack.pushCleanup<OMPAllocateCleanupTy>(
11989 VDAddr, Allocator);
11990 if (UntiedRealAddr.
isValid())
11993 Region->emitUntiedSwitch(CGF);
12010 assert(CGM.getLangOpts().OpenMP &&
"Not in OpenMP mode.");
12014 CGM.getOpenMPRuntime().NontemporalDeclsStack.emplace_back();
12016 for (
const Stmt *Ref :
C->private_refs()) {
12017 const auto *SimpleRefExpr =
cast<Expr>(Ref)->IgnoreParenImpCasts();
12019 if (
const auto *DRE = dyn_cast<DeclRefExpr>(SimpleRefExpr)) {
12020 VD = DRE->getDecl();
12023 assert((ME->isImplicitCXXThis() ||
12025 "Expected member of current class.");
12026 VD = ME->getMemberDecl();
12036 CGM.getOpenMPRuntime().NontemporalDeclsStack.pop_back();
12042 std::pair<Address, Address>> &LocalVars)
12043 : CGM(CGF.CGM), NeedToPush(!LocalVars.empty()) {
12047 CGF.
CurFn, CGM.getOpenMPRuntime().UntiedLocalVarsStack.size());
12048 CGM.getOpenMPRuntime().UntiedLocalVarsStack.push_back(LocalVars);
12054 CGM.getOpenMPRuntime().UntiedLocalVarsStack.pop_back();
12058 assert(
CGM.getLangOpts().OpenMP &&
"Not in OpenMP mode.");
12060 return llvm::any_of(
12061 CGM.getOpenMPRuntime().NontemporalDeclsStack,
12065void CGOpenMPRuntime::LastprivateConditionalRAII::tryToDisableInnerAnalysis(
12069 llvm::DenseSet<CanonicalDeclPtr<const Decl>> NeedToCheckForLPCs;
12075 const CapturedStmt *CS = S.getCapturedStmt(CaptureRegions.front());
12083 for (
const Expr *Ref :
C->varlist()) {
12084 if (!Ref->getType()->isScalarType())
12086 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
12089 NeedToCheckForLPCs.insert(DRE->getDecl());
12092 for (
const auto *
C : S.getClausesOfKind<OMPFirstprivateClause>()) {
12093 for (
const Expr *Ref :
C->varlist()) {
12094 if (!Ref->getType()->isScalarType())
12096 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
12099 NeedToCheckForLPCs.insert(DRE->getDecl());
12102 for (
const auto *
C : S.getClausesOfKind<OMPLastprivateClause>()) {
12103 for (
const Expr *Ref :
C->varlist()) {
12104 if (!Ref->getType()->isScalarType())
12106 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
12109 NeedToCheckForLPCs.insert(DRE->getDecl());
12112 for (
const auto *
C : S.getClausesOfKind<OMPReductionClause>()) {
12113 for (
const Expr *Ref :
C->varlist()) {
12114 if (!Ref->getType()->isScalarType())
12116 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
12119 NeedToCheckForLPCs.insert(DRE->getDecl());
12122 for (
const auto *
C : S.getClausesOfKind<OMPLinearClause>()) {
12123 for (
const Expr *Ref :
C->varlist()) {
12124 if (!Ref->getType()->isScalarType())
12126 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
12129 NeedToCheckForLPCs.insert(DRE->getDecl());
12132 for (
const Decl *VD : NeedToCheckForLPCs) {
12134 llvm::reverse(
CGM.getOpenMPRuntime().LastprivateConditionalStack)) {
12135 if (
Data.DeclToUniqueName.count(VD) > 0) {
12136 if (!
Data.Disabled)
12137 NeedToAddForLPCsAsDisabled.insert(VD);
12144CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
12147 Action((CGM.getLangOpts().OpenMP >= 50 &&
12150 return C->getKind() ==
12151 OMPC_LASTPRIVATE_conditional;
12153 ? ActionToDo::PushAsLastprivateConditional
12154 : ActionToDo::DoNotPush) {
12155 assert(
CGM.getLangOpts().OpenMP &&
"Not in OpenMP mode.");
12156 if (
CGM.getLangOpts().OpenMP < 50 || Action == ActionToDo::DoNotPush)
12158 assert(Action == ActionToDo::PushAsLastprivateConditional &&
12159 "Expected a push action.");
12161 CGM.getOpenMPRuntime().LastprivateConditionalStack.emplace_back();
12163 if (
C->getKind() != OMPC_LASTPRIVATE_conditional)
12166 for (
const Expr *Ref :
C->varlist()) {
12167 Data.DeclToUniqueName.insert(std::make_pair(
12172 Data.IVLVal = IVLVal;
12176CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
12178 :
CGM(CGF.
CGM), Action(ActionToDo::DoNotPush) {
12182 llvm::DenseSet<CanonicalDeclPtr<const Decl>> NeedToAddForLPCsAsDisabled;
12183 tryToDisableInnerAnalysis(S, NeedToAddForLPCsAsDisabled);
12184 if (!NeedToAddForLPCsAsDisabled.empty()) {
12185 Action = ActionToDo::DisableLastprivateConditional;
12186 LastprivateConditionalData &
Data =
12188 for (
const Decl *VD : NeedToAddForLPCsAsDisabled)
12189 Data.DeclToUniqueName.try_emplace(VD);
12191 Data.Disabled =
true;
12195CGOpenMPRuntime::LastprivateConditionalRAII
12198 return LastprivateConditionalRAII(CGF, S);
12202 if (CGM.getLangOpts().OpenMP < 50)
12204 if (Action == ActionToDo::DisableLastprivateConditional) {
12205 assert(CGM.getOpenMPRuntime().LastprivateConditionalStack.back().Disabled &&
12206 "Expected list of disabled private vars.");
12207 CGM.getOpenMPRuntime().LastprivateConditionalStack.pop_back();
12209 if (Action == ActionToDo::PushAsLastprivateConditional) {
12211 !CGM.getOpenMPRuntime().LastprivateConditionalStack.back().Disabled &&
12212 "Expected list of lastprivate conditional vars.");
12213 CGM.getOpenMPRuntime().LastprivateConditionalStack.pop_back();
12225 auto VI = I->getSecond().find(VD);
12226 if (VI == I->getSecond().end()) {
12227 RecordDecl *RD =
C.buildImplicitRecord(
"lasprivate.conditional");
12232 NewType =
C.getCanonicalTagType(RD);
12235 I->getSecond().try_emplace(VD, NewType, VDField, FiredField, BaseLVal);
12237 NewType = std::get<0>(VI->getSecond());
12238 VDField = std::get<1>(VI->getSecond());
12239 FiredField = std::get<2>(VI->getSecond());
12240 BaseLVal = std::get<3>(VI->getSecond());
12252class LastprivateConditionalRefChecker final
12255 const Expr *FoundE =
nullptr;
12256 const Decl *FoundD =
nullptr;
12257 StringRef UniqueDeclName;
12259 llvm::Function *FoundFn =
nullptr;
12265 llvm::reverse(LPM)) {
12266 auto It = D.DeclToUniqueName.find(E->
getDecl());
12267 if (It == D.DeclToUniqueName.end())
12273 UniqueDeclName = It->second;
12278 return FoundE == E;
12284 llvm::reverse(LPM)) {
12286 if (It == D.DeclToUniqueName.end())
12292 UniqueDeclName = It->second;
12297 return FoundE == E;
12299 bool VisitStmt(
const Stmt *S) {
12300 for (
const Stmt *Child : S->
children()) {
12303 if (
const auto *E = dyn_cast<Expr>(Child))
12311 explicit LastprivateConditionalRefChecker(
12312 ArrayRef<CGOpenMPRuntime::LastprivateConditionalData> LPM)
12314 std::tuple<const Expr *, const Decl *, StringRef, LValue, llvm::Function *>
12315 getFoundData()
const {
12316 return std::make_tuple(FoundE, FoundD, UniqueDeclName, IVLVal, FoundFn);
12323 StringRef UniqueDeclName,
12329 llvm::Constant *LastIV =
OMPBuilder.getOrCreateInternalVariable(
12330 LLIVTy,
getName({UniqueDeclName,
"iv"}));
12338 llvm::GlobalVariable *
Last =
OMPBuilder.getOrCreateInternalVariable(
12354 auto &&
CodeGen = [&LastIVLVal, &IVLVal, IVVal, &LVal, &LastLVal,
12360 llvm::Value *CmpRes;
12362 CmpRes = CGF.
Builder.CreateICmpSLE(LastIVVal, IVVal);
12365 "Loop iteration variable must be integer.");
12366 CmpRes = CGF.
Builder.CreateICmpULE(LastIVVal, IVVal);
12370 CGF.
Builder.CreateCondBr(CmpRes, ThenBB, ExitBB);
12391 "Aggregates are not supported in lastprivate conditional.");
12400 if (
CGM.getLangOpts().OpenMPSimd) {
12414 if (!Checker.Visit(LHS))
12416 const Expr *FoundE;
12417 const Decl *FoundD;
12418 StringRef UniqueDeclName;
12420 llvm::Function *FoundFn;
12421 std::tie(FoundE, FoundD, UniqueDeclName, IVLVal, FoundFn) =
12422 Checker.getFoundData();
12423 if (FoundFn != CGF.
CurFn) {
12428 "Lastprivate conditional is not found in outer region.");
12429 QualType StructTy = std::get<0>(It->getSecond());
12430 const FieldDecl* FiredDecl = std::get<2>(It->getSecond());
12441 FiredLVal, llvm::AtomicOrdering::Unordered,
12459 auto It = llvm::find_if(
12461 if (It == Range.end() || It->Fn != CGF.
CurFn)
12465 "Lastprivates must be registered already.");
12468 const CapturedStmt *CS = D.getCapturedStmt(CaptureRegions.back());
12469 for (
const auto &Pair : It->DeclToUniqueName) {
12470 const auto *VD =
cast<VarDecl>(Pair.first->getCanonicalDecl());
12473 auto I = LPCI->getSecond().find(Pair.first);
12474 assert(I != LPCI->getSecond().end() &&
12475 "Lastprivate must be rehistered already.");
12477 LValue BaseLVal = std::get<3>(I->getSecond());
12481 llvm::Value *Cmp = CGF.
Builder.CreateIsNotNull(Res);
12485 CGF.
Builder.CreateCondBr(Cmp, ThenBB, DoneBB);
12510 "Unknown lastprivate conditional variable.");
12511 StringRef UniqueName = It->second;
12512 llvm::GlobalVariable *GV =
CGM.getModule().getNamedGlobal(UniqueName);
12526 llvm_unreachable(
"Not supported in SIMD-only mode");
12533 llvm_unreachable(
"Not supported in SIMD-only mode");
12540 bool Tied,
unsigned &NumberOfParts) {
12541 llvm_unreachable(
"Not supported in SIMD-only mode");
12549 llvm_unreachable(
"Not supported in SIMD-only mode");
12555 const Expr *Hint) {
12556 llvm_unreachable(
"Not supported in SIMD-only mode");
12562 llvm_unreachable(
"Not supported in SIMD-only mode");
12568 const Expr *Filter) {
12569 llvm_unreachable(
"Not supported in SIMD-only mode");
12574 llvm_unreachable(
"Not supported in SIMD-only mode");
12580 llvm_unreachable(
"Not supported in SIMD-only mode");
12588 llvm_unreachable(
"Not supported in SIMD-only mode");
12595 llvm_unreachable(
"Not supported in SIMD-only mode");
12602 bool ForceSimpleCall) {
12603 llvm_unreachable(
"Not supported in SIMD-only mode");
12610 llvm_unreachable(
"Not supported in SIMD-only mode");
12615 llvm_unreachable(
"Not supported in SIMD-only mode");
12621 llvm_unreachable(
"Not supported in SIMD-only mode");
12627 llvm_unreachable(
"Not supported in SIMD-only mode");
12634 llvm_unreachable(
"Not supported in SIMD-only mode");
12640 llvm_unreachable(
"Not supported in SIMD-only mode");
12645 unsigned IVSize,
bool IVSigned,
12648 llvm_unreachable(
"Not supported in SIMD-only mode");
12656 llvm_unreachable(
"Not supported in SIMD-only mode");
12660 ProcBindKind ProcBind,
12662 llvm_unreachable(
"Not supported in SIMD-only mode");
12669 llvm_unreachable(
"Not supported in SIMD-only mode");
12675 llvm_unreachable(
"Not supported in SIMD-only mode");
12680 llvm_unreachable(
"Not supported in SIMD-only mode");
12686 llvm::AtomicOrdering AO) {
12687 llvm_unreachable(
"Not supported in SIMD-only mode");
12692 llvm::Function *TaskFunction,
12694 const Expr *IfCond,
12696 llvm_unreachable(
"Not supported in SIMD-only mode");
12703 llvm_unreachable(
"Not supported in SIMD-only mode");
12710 assert(Options.
SimpleReduction &&
"Only simple reduction is expected.");
12712 ReductionOps, Options);
12718 llvm_unreachable(
"Not supported in SIMD-only mode");
12723 bool IsWorksharingReduction) {
12724 llvm_unreachable(
"Not supported in SIMD-only mode");
12731 llvm_unreachable(
"Not supported in SIMD-only mode");
12736 llvm::Value *ReductionsPtr,
12738 llvm_unreachable(
"Not supported in SIMD-only mode");
12744 llvm_unreachable(
"Not supported in SIMD-only mode");
12750 llvm_unreachable(
"Not supported in SIMD-only mode");
12756 llvm_unreachable(
"Not supported in SIMD-only mode");
12761 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
12763 llvm_unreachable(
"Not supported in SIMD-only mode");
12768 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID,
const Expr *IfCond,
12769 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device,
12773 llvm_unreachable(
"Not supported in SIMD-only mode");
12777 llvm_unreachable(
"Not supported in SIMD-only mode");
12781 llvm_unreachable(
"Not supported in SIMD-only mode");
12791 llvm::Function *OutlinedFn,
12793 llvm_unreachable(
"Not supported in SIMD-only mode");
12797 const Expr *NumTeams,
12798 const Expr *ThreadLimit,
12800 llvm_unreachable(
"Not supported in SIMD-only mode");
12807 llvm_unreachable(
"Not supported in SIMD-only mode");
12813 llvm_unreachable(
"Not supported in SIMD-only mode");
12819 llvm_unreachable(
"Not supported in SIMD-only mode");
12824 llvm_unreachable(
"Not supported in SIMD-only mode");
12829 llvm_unreachable(
"Not supported in SIMD-only mode");
12834 const VarDecl *NativeParam)
const {
12835 llvm_unreachable(
"Not supported in SIMD-only mode");
12841 const VarDecl *TargetParam)
const {
12842 llvm_unreachable(
"Not supported in SIMD-only mode");
static llvm::Value * emitCopyprivateCopyFunction(CodeGenModule &CGM, llvm::Type *ArgsElemType, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps, SourceLocation Loc)
static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF, SourceLocation Loc, SmallString< 128 > &Buffer)
static void emitOffloadingArraysAndArgs(CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo, CGOpenMPRuntime::TargetDataInfo &Info, llvm::OpenMPIRBuilder &OMPBuilder, bool IsNonContiguous=false, bool ForEndCall=false)
Emit the arrays used to pass the captures and map information to the offloading runtime library.
static RecordDecl * createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy, ArrayRef< PrivateDataTy > Privates)
static void emitInitWithReductionInitializer(CodeGenFunction &CGF, const OMPDeclareReductionDecl *DRD, const Expr *InitOp, Address Private, Address Original, QualType Ty)
static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, Address OriginalBaseAddress, llvm::Value *Addr)
static void emitPrivatesInit(CodeGenFunction &CGF, const OMPExecutableDirective &D, Address KmpTaskSharedsPtr, LValue TDBase, const RecordDecl *KmpTaskTWithPrivatesQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef< PrivateDataTy > Privates, bool ForDup)
Emit initialization for private variables in task-based directives.
static void emitClauseForBareTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &Values)
static llvm::Value * emitDestructorsFunction(CodeGenModule &CGM, SourceLocation Loc, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy)
static unsigned evaluateCDTSize(const FunctionDecl *FD, ArrayRef< ParamAttrTy > ParamAttrs)
static void EmitOMPAggregateReduction(CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar, const VarDecl *RHSVar, const llvm::function_ref< void(CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *)> &RedOpGen, const Expr *XExpr=nullptr, const Expr *EExpr=nullptr, const Expr *UpExpr=nullptr)
Emit reduction operation for each element of array (required for array sections) LHS op = RHS.
static void emitTargetCallFallback(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, CodeGenFunction &CGF)
static llvm::Value * emitReduceInitFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Emits reduction initializer function:
static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion)
static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy, llvm::PointerUnion< unsigned *, LValue * > Pos, const OMPTaskDataTy::DependData &Data, Address DependenciesArray)
static llvm::Value * emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc, const OMPTaskDataTy &Data, QualType PrivatesQTy, ArrayRef< PrivateDataTy > Privates)
Emit a privates mapping function for correct handling of private and firstprivate variables.
static llvm::Value * emitReduceCombFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N, const Expr *ReductionOp, const Expr *LHS, const Expr *RHS, const Expr *PrivateRef)
Emits reduction combiner function:
static RecordDecl * createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef< PrivateDataTy > Privates)
static llvm::Value * getAllocatorVal(CodeGenFunction &CGF, const Expr *Allocator)
Return allocator value from expression, or return a null allocator (default when no allocator specifi...
static llvm::Function * emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy, QualType SharedsPtrTy, llvm::Function *TaskFunction, llvm::Value *TaskPrivatesMap)
Emit a proxy function which accepts kmp_task_t as the second argument.
static void addAArch64VectorName(T VLEN, StringRef LMask, StringRef Prefix, char ISA, StringRef ParSeq, StringRef MangledName, bool OutputBecomesInput, llvm::Function *Fn)
static bool isAllocatableDecl(const VarDecl *VD)
static llvm::Value * getAlignmentValue(CodeGenModule &CGM, const VarDecl *VD)
Return the alignment from an allocate directive if present.
static void emitTargetCallKernelLaunch(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::Value *OutlinedFnID, CodeGenFunction::OMPTargetDataInfo &InputInfo, llvm::Value *&MapTypesArray, llvm::Value *&MapNamesArray, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter, CodeGenFunction &CGF, CodeGenModule &CGM)
static llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind convertCaptureClause(const VarDecl *VD)
static std::tuple< unsigned, unsigned, bool > getNDSWDS(const FunctionDecl *FD, ArrayRef< ParamAttrTy > ParamAttrs)
static const OMPExecutableDirective * getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D)
Check for inner distribute directive.
static std::pair< llvm::Value *, llvm::Value * > getPointerAndSize(CodeGenFunction &CGF, const Expr *E)
static const VarDecl * getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE)
static bool isTrivial(ASTContext &Ctx, const Expr *E)
Checks if the expression is constant or does not have non-trivial function calls.
static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind, bool Chunked, bool Ordered)
Map the OpenMP loop schedule to the runtime enumeration.
static void getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS, const Expr **E, int32_t &UpperBound, bool UpperBoundOnly, llvm::Value **CondVal)
Check for a num threads constant value (stored in DefaultVal), or expression (stored in E).
static llvm::Value * emitDeviceID(llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, CodeGenFunction &CGF)
static const OMPDeclareReductionDecl * getReductionInit(const Expr *ReductionOp)
Check if the combiner is a call to UDR combiner and if it is so return the UDR decl used for reductio...
static bool checkInitIsRequired(CodeGenFunction &CGF, ArrayRef< PrivateDataTy > Privates)
Check if duplication function is required for taskloops.
static bool checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD, ArrayRef< PrivateDataTy > Privates)
Checks if destructor function is required to be generated.
static llvm::TargetRegionEntryInfo getEntryInfoFromPresumedLoc(CodeGenModule &CGM, llvm::OpenMPIRBuilder &OMPBuilder, SourceLocation BeginLoc, llvm::StringRef ParentName="")
static void genMapInfo(MappableExprsHandler &MEHandler, CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo, llvm::OpenMPIRBuilder &OMPBuilder, const llvm::DenseSet< CanonicalDeclPtr< const Decl > > &SkippedVarSet=llvm::DenseSet< CanonicalDeclPtr< const Decl > >())
static void emitForStaticInitCall(CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId, llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2, const CGOpenMPRuntime::StaticRTInput &Values)
static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, LValue BaseLV)
static void getKmpAffinityType(ASTContext &C, QualType &KmpTaskAffinityInfoTy)
Builds kmp_depend_info, if it is not built yet, and builds flags type.
static llvm::Constant * emitMappingInformation(CodeGenFunction &CGF, llvm::OpenMPIRBuilder &OMPBuilder, MappableExprsHandler::MappingExprInfo &MapExprs)
Emit a string constant containing the names of the values mapped to the offloading runtime library.
static void getDependTypes(ASTContext &C, QualType &KmpDependInfoTy, QualType &FlagsTy)
Builds kmp_depend_info, if it is not built yet, and builds flags type.
static llvm::Value * emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc, const OMPExecutableDirective &D, QualType KmpTaskTWithPrivatesPtrQTy, const RecordDecl *KmpTaskTWithPrivatesQTyRD, const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef< PrivateDataTy > Privates, bool WithLastIter)
Emit task_dup function (for initialization of private/firstprivate/lastprivate vars and last_iter fla...
static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind convertDeviceClause(const VarDecl *VD)
static llvm::Value * emitReduceFiniFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Emits reduction finalizer function:
static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, QualType Type, bool EmitDeclareReductionInit, const Expr *Init, const OMPDeclareReductionDecl *DRD, Address SrcAddr=Address::invalid())
Emit initialization of arrays of complex types.
static bool getAArch64PBV(QualType QT, ASTContext &C)
Pass By Value (PBV), as defined in 3.1.2 of the AAVFABI.
static unsigned getAArch64LS(QualType QT, ParamKindTy Kind, ASTContext &C)
Computes the lane size (LS) of a return type or of an input parameter, as defined by LS(P) in 3....
static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM, const T *C, llvm::Value *ULoc, llvm::Value *ThreadID)
static RTLDependenceKindTy translateDependencyKind(OpenMPDependClauseKind K)
Translates internal dependency kind into the runtime kind.
static void emitTargetCallElse(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, CodeGenFunction &CGF)
static llvm::Function * emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty, const Expr *CombinerInitializer, const VarDecl *In, const VarDecl *Out, bool IsCombiner)
static void emitX86DeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn, const llvm::APSInt &VLENVal, ArrayRef< ParamAttrTy > ParamAttrs, OMPDeclareSimdDeclAttr::BranchStateTy State)
static void emitReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp)
Emit reduction combiner.
static std::string mangleVectorParameters(ArrayRef< ParamAttrTy > ParamAttrs)
Mangle the parameter part of the vector function name according to their OpenMP classification.
static std::string generateUniqueName(CodeGenModule &CGM, llvm::StringRef Prefix, const Expr *Ref)
static llvm::Function * emitParallelOrTeamsOutlinedFunction(CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen)
static void emitAArch64DeclareSimdFunction(CodeGenModule &CGM, const FunctionDecl *FD, unsigned UserVLEN, ArrayRef< ParamAttrTy > ParamAttrs, OMPDeclareSimdDeclAttr::BranchStateTy State, StringRef MangledName, char ISA, unsigned VecRegSize, llvm::Function *Fn, SourceLocation SLoc)
Emit vector function attributes for AArch64, as defined in the AAVFABI.
static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array, unsigned Index, const VarDecl *Var)
Given an array of pointers to variables, project the address of a given variable.
static llvm::Value * emitDynCGGroupMem(const OMPExecutableDirective &D, CodeGenFunction &CGF)
static bool isAssumedToBeNotEmitted(const ValueDecl *VD, bool IsDevice)
static void addAArch64AdvSIMDNDSNames(unsigned NDS, StringRef Mask, StringRef Prefix, char ISA, StringRef ParSeq, StringRef MangledName, bool OutputBecomesInput, llvm::Function *Fn)
static FieldDecl * addFieldToRecordDecl(ASTContext &C, DeclContext *DC, QualType FieldTy)
static ValueDecl * getDeclFromThisExpr(const Expr *E)
static void genMapInfoForCaptures(MappableExprsHandler &MEHandler, CodeGenFunction &CGF, const CapturedStmt &CS, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, llvm::OpenMPIRBuilder &OMPBuilder, llvm::DenseSet< CanonicalDeclPtr< const Decl > > &MappedVarSet, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo)
static RecordDecl * createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpRoutineEntryPointerQTy)
static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2)
static bool getAArch64MTV(QualType QT, ParamKindTy Kind)
Maps To Vector (MTV), as defined in 4.1.1 of the AAVFABI (2021Q1).
@ LLVM_MARK_AS_BITMASK_ENUM
This file defines OpenMP AST classes for clauses.
Defines some OpenMP-specific enums and functions.
Defines the SourceManager interface.
This file defines OpenMP AST classes for executable directives and clauses.
__DEVICE__ int max(int __a, int __b)
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
SourceManager & getSourceManager()
const ConstantArrayType * getAsConstantArrayType(QualType T) const
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
const ASTRecordLayout & getASTRecordLayout(const RecordDecl *D) const
Get or compute information about the layout of the specified record (struct/union/class) D,...
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
const LangOptions & getLangOpts() const
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth,...
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
static bool hasSameType(QualType T1, QualType T2)
Determine whether the given types T1 and T2 are equivalent.
const VariableArrayType * getAsVariableArrayType(QualType T) const
QualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
unsigned getTypeAlign(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in bits.
CharUnits getSize() const
getSize - Get the record size in characters.
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
CharUnits getNonVirtualSize() const
getNonVirtualSize - Get the non-virtual size (in chars) of an object, which is the size of the object...
static QualType getBaseOriginalType(const Expr *Base)
Return original type of the base expression for array section.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Attr - This represents one attribute.
Represents a C++ constructor within a class.
Represents a C++ destructor within a class.
const CXXRecordDecl * getParent() const
Return the parent of this method declaration, which is the class in which this method is defined.
QualType getFunctionObjectParameterType() const
bool isLambda() const
Determine whether this class describes a lambda function object.
void getCaptureFields(llvm::DenseMap< const ValueDecl *, FieldDecl * > &Captures, FieldDecl *&ThisCapture) const
For a closure type, retrieve the mapping from captured variables and this to the non-static data memb...
unsigned getNumBases() const
Retrieves the number of base classes of this class.
base_class_range vbases()
capture_const_range captures() const
CXXDestructorDecl * getDestructor() const
Returns the destructor decl for this class.
CanProxy< U > castAs() const
A wrapper class around a pointer that always points to its canonical declaration.
Describes the capture of either a variable, or 'this', or variable-length array type.
bool capturesVariableByCopy() const
Determine whether this capture handles a variable by copy.
VarDecl * getCapturedVar() const
Retrieve the declaration of the variable being captured.
bool capturesVariableArrayType() const
Determine whether this capture handles a variable-length array type.
bool capturesThis() const
Determine whether this capture handles the C++ 'this' pointer.
bool capturesVariable() const
Determine whether this capture handles a variable (by reference).
This captures a statement into a function.
const Capture * const_capture_iterator
capture_iterator capture_end() const
Retrieve an iterator pointing past the end of the sequence of captures.
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Stmt * getCapturedStmt()
Retrieve the statement being captured.
bool capturesVariable(const VarDecl *Var) const
True if this variable has been captured.
capture_iterator capture_begin()
Retrieve an iterator pointing to the first capture.
CharUnits - This is an opaque type for sizes expressed in character units.
bool isZero() const
isZero - Test whether the quantity equals zero.
llvm::Align getAsAlign() const
getAsAlign - Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit b...
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
CharUnits alignTo(const CharUnits &Align) const
alignTo - Returns the next integer (mod 2**64) that is greater than or equal to this quantity and is ...
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
CharUnits getAlignment() const
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Address withPointer(llvm::Value *NewPointer, KnownNonNull_t IsKnownNonNull) const
Return address with different pointer, but same element type and alignment.
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
llvm::PointerType * getType() const
Return the type of the pointer value.
static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateDefaultArtificial(CodeGenFunction &CGF, SourceLocation TemporaryLocation)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF)
Set the IRBuilder to not attach debug locations.
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
Address CreateGEP(CodeGenFunction &CGF, Address Addr, llvm::Value *Index, const llvm::Twine &Name="")
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Address CreateConstArrayGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = [n x T]* ... produce name = getelementptr inbounds addr, i64 0, i64 index where i64 is a...
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
Address CreateConstGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = T* ... produce name = getelementptr inbounds addr, i64 index where i64 is actually the t...
CGFunctionInfo - Class to encapsulate the information about a function definition.
DisableAutoDeclareTargetRAII(CodeGenModule &CGM)
~DisableAutoDeclareTargetRAII()
~LastprivateConditionalRAII()
static LastprivateConditionalRAII disable(CodeGenFunction &CGF, const OMPExecutableDirective &S)
NontemporalDeclsRAII(CodeGenModule &CGM, const OMPLoopDirective &S)
Struct that keeps all the relevant information that should be kept throughout a 'target data' region.
llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap
Map between the a declaration of a capture and the corresponding new llvm address where the runtime r...
~UntiedTaskLocalDeclsRAII()
UntiedTaskLocalDeclsRAII(CodeGenFunction &CGF, const llvm::MapVector< CanonicalDeclPtr< const VarDecl >, std::pair< Address, Address > > &LocalVars)
virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc)
Emits address of the word in a memory where current thread id is stored.
llvm::StringSet ThreadPrivateWithDefinition
Set of threadprivate variables with the generated initializer.
CGOpenMPRuntime(CodeGenModule &CGM)
virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
Emit task region for the task directive.
void createOffloadEntriesAndInfoMetadata()
Creates all the offload entries in the current compilation unit along with the associated metadata.
const Expr * getNumTeamsExprForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &MinTeamsVal, int32_t &MaxTeamsVal)
Emit the number of teams for a target directive.
virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc)
Returns address of the threadprivate variable for the current thread.
void emitDeferredTargetDecls() const
Emit deferred declare target variables marked for deferred emission.
virtual llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST)
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
bool markAsGlobalTarget(GlobalDecl GD)
Marks the declaration as already emitted for the device code and returns true, if it was marked alrea...
virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond, llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, const Expr *Message=nullptr)
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
llvm::SmallDenseSet< CanonicalDeclPtr< const Decl > > NontemporalDeclsSet
virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device)
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
virtual void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc, OpenMPNumThreadsClauseModifier Modifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, SourceLocation SeverityLoc=SourceLocation(), const Expr *Message=nullptr, SourceLocation MessageLoc=SourceLocation())
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32global_tid, kmp_int32 num_threads) ...
QualType SavedKmpTaskloopTQTy
Saved kmp_task_t for taskloop-based directive.
virtual void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps)
Emits a single region.
virtual bool emitTargetGlobal(GlobalDecl GD)
Emit the global GD if it is meaningful for the target.
void setLocThreadIdInsertPt(CodeGenFunction &CGF, bool AtCurrentPoint=false)
std::string getOutlinedHelperName(StringRef Name) const
Get the function name of an outlined region.
bool HasEmittedDeclareTargetRegion
Flag for keeping track of weather a device routine has been emitted.
llvm::Constant * getOrCreateThreadPrivateCache(const VarDecl *VD)
If the specified mangled name is not in the module, create and return threadprivate cache object.
virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal)
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
virtual void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc)
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
void emitCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee Callee, ArrayRef< llvm::Value * > Args={}) const
Emits Callee function call with arguments Args with location Loc.
virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF, const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const
Choose default schedule type and chunk value for the schedule clause.
virtual std::pair< llvm::Function *, llvm::Function * > getUserDefinedReduction(const OMPDeclareReductionDecl *D)
Get combiner/initializer for the specified user-defined reduction, if any.
virtual bool isGPU() const
Returns true if the current target is a GPU.
static const Stmt * getSingleCompoundChild(ASTContext &Ctx, const Stmt *Body)
Checks if the Body is the CompoundStmt and returns its child statement iff there is only one that is ...
virtual void emitDeclareTargetFunction(const FunctionDecl *FD, llvm::GlobalValue *GV)
Emit code for handling declare target functions in the runtime.
bool HasRequiresUnifiedSharedMemory
Flag for keeping track of weather a requires unified_shared_memory directive is present.
llvm::Value * emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc, unsigned Flags=0, bool EmitLoc=false)
Emits object of ident_t type with info for source location.
bool isLocalVarInUntiedTask(CodeGenFunction &CGF, const VarDecl *VD) const
Returns true if the variable is a local variable in untied task.
virtual void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars)
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
virtual void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancellation point' construct.
virtual llvm::Function * emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF=nullptr)
Emit a code for initialization of threadprivate variable.
FunctionUDMMapTy FunctionUDMMap
virtual ConstantAddress getAddrOfDeclareTargetVar(const VarDecl *VD)
Returns the address of the variable marked as declare target with link clause OR as declare target wi...
llvm::Function * getOrCreateUserDefinedMapperFunc(const OMPDeclareMapperDecl *D)
Get the function for the specified user-defined mapper.
OpenMPLocThreadIDMapTy OpenMPLocThreadIDMap
virtual void functionFinished(CodeGenFunction &CGF)
Cleans up references to the objects in finished function.
virtual llvm::Function * emitTeamsOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP teams directive D.
QualType KmpTaskTQTy
Type typedef struct kmp_task { void * shareds; /**< pointer to block of pointers to shared vars / k...
llvm::OpenMPIRBuilder OMPBuilder
An OpenMP-IR-Builder instance.
virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr * > NumIterations)
Emit initialization for doacross loop nesting support.
virtual void adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF, const OMPExecutableDirective &D) const
Adjust some parameters for the target-based directives, like addresses of the variables captured by r...
FunctionUDRMapTy FunctionUDRMap
virtual void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, CGOpenMPRuntime::TargetDataInfo &Info)
Emit the target data mapping code associated with D.
virtual unsigned getDefaultLocationReserved2Flags() const
Returns additional flags that can be stored in reserved_2 field of the default location.
virtual Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const
Gets the address of the native argument basing on the address of the target-specific parameter.
void emitUsesAllocatorsFini(CodeGenFunction &CGF, const Expr *Allocator)
Destroys user defined allocators specified in the uses_allocators clause.
QualType KmpTaskAffinityInfoTy
Type typedef struct kmp_task_affinity_info { kmp_intptr_t base_addr; size_t len; struct { bool flag1 ...
void emitPrivateReduction(CodeGenFunction &CGF, SourceLocation Loc, const Expr *Privates, const Expr *LHSExprs, const Expr *RHSExprs, const Expr *ReductionOps)
Emits code for private variable reduction.
llvm::Value * emitNumTeamsForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D)
virtual void emitTargetOutlinedFunctionHelper(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Helper to emit outlined function for 'target' directive.
void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName)
Start scanning from statement S and emit all target regions found along the way.
SmallVector< llvm::Value *, 4 > emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy, const OMPTaskDataTy::DependData &Data)
virtual llvm::Value * emitMessageClause(CodeGenFunction &CGF, const Expr *Message, SourceLocation Loc)
virtual void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc)
Emit a taskgroup region.
llvm::DenseMap< llvm::Function *, llvm::DenseMap< CanonicalDeclPtr< const Decl >, std::tuple< QualType, const FieldDecl *, const FieldDecl *, LValue > > > LastprivateConditionalToTypes
Maps local variables marked as lastprivate conditional to their internal types.
virtual bool emitTargetGlobalVariable(GlobalDecl GD)
Emit the global variable if it is a valid device global variable.
virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32global_tid, kmp_int32 num_teams,...
bool hasRequiresUnifiedSharedMemory() const
Return whether the unified_shared_memory has been specified.
virtual Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, QualType VarType, StringRef Name)
Creates artificial threadprivate variable with name Name and type VarType.
void emitUserDefinedMapper(const OMPDeclareMapperDecl *D, CodeGenFunction *CGF=nullptr)
Emit the function for the user defined mapper construct.
bool HasEmittedTargetRegion
Flag for keeping track of weather a target region has been emitted.
void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy, LValue PosLVal, const OMPTaskDataTy::DependData &Data, Address DependenciesArray)
std::string getReductionFuncName(StringRef Name) const
Get the function name of a reduction function.
virtual void processRequiresDirective(const OMPRequiresDecl *D)
Perform check on requires decl to ensure that target architecture supports unified addressing.
llvm::DenseSet< CanonicalDeclPtr< const Decl > > AlreadyEmittedTargetDecls
List of the emitted declarations.
virtual llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data)
Emit a code for initialization of task reduction clause.
llvm::Value * getThreadID(CodeGenFunction &CGF, SourceLocation Loc)
Gets thread id value for the current thread.
void emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, OpenMPDependClauseKind NewDepKind, SourceLocation Loc)
Updates the dependency kind in the specified depobj object.
virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD, SourceLocation Loc)
Gets the address of the global copy used for lastprivate conditional update, if any.
llvm::MapVector< CanonicalDeclPtr< const VarDecl >, std::pair< Address, Address > > UntiedLocalVarsAddressesMap
virtual void emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, Expr *ME, bool IsFatal)
Emit __kmpc_error call for error directive extern void __kmpc_error(ident_t *loc, int severity,...
void clearLocThreadIdInsertPt(CodeGenFunction &CGF)
virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc)
Emits code for a taskyield directive.
std::string getName(ArrayRef< StringRef > Parts) const
Get the platform-specific name separator.
QualType KmpRoutineEntryPtrQTy
void computeMinAndMaxThreadsAndTeams(const OMPExecutableDirective &D, CodeGenFunction &CGF, llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs)
Helper to determine the min/max number of threads/teams for D.
virtual void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc, llvm::AtomicOrdering AO)
Emit flush of the variables specified in 'omp flush' directive.
virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPTaskDataTy &Data)
Emit code for 'taskwait' directive.
virtual void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc)
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32global_tid, int proc_bind) to generate...
void emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LValue IVLVal, StringRef UniqueDeclName, LValue LVal, SourceLocation Loc)
Emit update for lastprivate conditional data.
virtual void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
Emit task region for the taskloop directive.
virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false)
Emit an implicit/explicit barrier for OpenMP threads.
static unsigned getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind)
Returns default flags for the barriers depending on the directive, for which this barier is going to ...
virtual bool emitTargetFunctions(GlobalDecl GD)
Emit the target regions enclosed in GD function definition or the function itself in case it is a val...
TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const OMPTaskDataTy &Data)
Emit task region for the task directive.
llvm::Value * emitTargetNumIterationsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Return the trip count of loops associated with constructs / 'target teams distribute' and 'teams dist...
llvm::StringMap< llvm::AssertingVH< llvm::GlobalVariable >, llvm::BumpPtrAllocator > InternalVars
An ordered map of auto-generated variables to their unique names.
virtual void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values)
llvm::SmallVector< UntiedLocalVarsAddressesMap, 4 > UntiedLocalVarsStack
virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind)
Call the appropriate runtime routine to notify that we finished all the work with current loop.
virtual void emitThreadLimitClause(CodeGenFunction &CGF, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_set_thread_limit(ident_t *loc, kmp_int32global_tid, kmp_int32 thread_limit)...
void emitIfClause(CodeGenFunction &CGF, const Expr *Cond, const RegionCodeGenTy &ThenGen, const RegionCodeGenTy &ElseGen)
Emits code for OpenMP 'if' clause using specified CodeGen function.
Address emitDepobjDependClause(CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies, SourceLocation Loc)
Emits list of dependecies based on the provided data (array of dependence/expression pairs) for depob...
bool isNontemporalDecl(const ValueDecl *VD) const
Checks if the VD variable is marked as nontemporal declaration in current context.
virtual llvm::Function * emitParallelOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP parallel directive D.
const Expr * getNumThreadsExprForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &UpperBound, bool UpperBoundOnly, llvm::Value **CondExpr=nullptr, const Expr **ThreadLimitExpr=nullptr)
Check for a number of threads upper bound constant value (stored in UpperBound), or expression (retur...
virtual llvm::Value * emitSeverityClause(OpenMPSeverityClauseKind Severity, SourceLocation Loc)
llvm::SmallVector< LastprivateConditionalData, 4 > LastprivateConditionalStack
Stack for list of addresses of declarations in current context marked as lastprivate conditional.
virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values)
Call the appropriate runtime routine to initialize it before start of loop.
virtual void emitDeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn)
Marks function Fn with properly mangled versions of vector functions.
llvm::AtomicOrdering getDefaultMemoryOrdering() const
Gets default memory ordering as specified in requires directive.
virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static non-chunked.
llvm::Value * getCriticalRegionLock(StringRef CriticalName)
Returns corresponding lock object for the specified critical region name.
virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancel' construct.
QualType SavedKmpTaskTQTy
Saved kmp_task_t for task directive.
virtual void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc)
Emits a master region.
virtual llvm::Function * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts)
Emits outlined function for the OpenMP task directive D.
llvm::DenseMap< llvm::Function *, unsigned > FunctionToUntiedTaskStackMap
Maps function to the position of the untied task locals stack.
void emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal, SourceLocation Loc)
Emits the code to destroy the dependency object provided in depobj directive.
virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Required to resolve existing problems in the runtime.
llvm::ArrayType * KmpCriticalNameTy
Type kmp_critical_name, originally defined as typedef kmp_int32 kmp_critical_name[8];.
virtual void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C)
Emit code for doacross ordered directive with 'depend' clause.
llvm::DenseMap< const OMPDeclareMapperDecl *, llvm::Function * > UDMMap
Map from the user-defined mapper declaration to its corresponding functions.
virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF, const Expr *LHS)
Checks if the provided LVal is lastprivate conditional and emits the code to update the value of the ...
std::pair< llvm::Value *, LValue > getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal, SourceLocation Loc)
Returns the number of the elements and the address of the depobj dependency array.
llvm::SmallDenseSet< const VarDecl * > DeferredGlobalVariables
List of variables that can become declare target implicitly and, thus, must be emitted.
void emitUsesAllocatorsInit(CodeGenFunction &CGF, const Expr *Allocator, const Expr *AllocatorTraits)
Initializes user defined allocators specified in the uses_allocators clauses.
llvm::Type * KmpRoutineEntryPtrTy
Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *);.
llvm::Type * getIdentTyPointerTy()
Returns pointer to ident_t type.
void emitSingleReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp, const Expr *PrivateRef, const DeclRefExpr *LHS, const DeclRefExpr *RHS)
Emits single reduction combiner.
llvm::OpenMPIRBuilder & getOMPBuilder()
virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Emit outilined function for 'target' directive.
virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr)
Emits a critical region.
virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned)
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
virtual void emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, ArrayRef< llvm::Value * > Args={}) const
Emits call of the outlined function with the provided arguments, translating these arguments to corre...
llvm::Value * emitNumThreadsForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D)
Emit an expression that denotes the number of threads a target region shall use.
void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor, llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc)
Emits initialization code for the threadprivate variables.
virtual void emitUserDefinedReduction(CodeGenFunction *CGF, const OMPDeclareReductionDecl *D)
Emit code for the specified user defined reduction construct.
virtual void checkAndEmitSharedLastprivateConditional(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::DenseSet< CanonicalDeclPtr< const VarDecl > > &IgnoredDecls)
Checks if the lastprivate conditional was updated in inner region and writes the value.
QualType KmpDimTy
struct kmp_dim { // loop bounds info casted to kmp_int64 kmp_int64 lo; // lower kmp_int64 up; // uppe...
virtual void emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool HasCancel=false)
Emit code for the directive that does not require outlining.
virtual void registerTargetGlobalVariable(const VarDecl *VD, llvm::Constant *Addr)
Checks if the provided global decl GD is a declare target variable and registers it when emitting cod...
virtual void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D)
Emits OpenMP-specific function prolog.
void emitKmpRoutineEntryT(QualType KmpInt32Ty)
Build type kmp_routine_entry_t (if not built yet).
virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static chunked.
virtual void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Emit the target offloading code associated with D.
virtual bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS)
Checks if the variable has associated OMPAllocateDeclAttr attribute with the predefined allocator and...
llvm::AtomicOrdering RequiresAtomicOrdering
Atomic ordering from the omp requires directive.
virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps, ReductionOptionsTy Options)
Emit a code for reduction clause.
std::pair< llvm::Value *, Address > emitDependClause(CodeGenFunction &CGF, ArrayRef< OMPTaskDataTy::DependData > Dependencies, SourceLocation Loc)
Emits list of dependecies based on the provided data (array of dependence/expression pairs).
llvm::StringMap< llvm::WeakTrackingVH > EmittedNonTargetVariables
List of the global variables with their addresses that should not be emitted for the target.
virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const
Check if the specified ScheduleKind is dynamic.
Address emitLastprivateConditionalInit(CodeGenFunction &CGF, const VarDecl *VD)
Create specialized alloca to handle lastprivate conditionals.
virtual void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads)
Emit an ordered region.
virtual Address getAddressOfLocalVariable(CodeGenFunction &CGF, const VarDecl *VD)
Gets the OpenMP-specific address of the local variable.
virtual void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, bool IsWorksharingReduction)
Emits the following code for reduction clause with task modifier:
virtual void emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter=nullptr)
Emits a masked region.
QualType KmpDependInfoTy
Type typedef struct kmp_depend_info { kmp_intptr_t base_addr; size_t len; struct { bool in:1; bool ou...
llvm::Function * emitReductionFunction(StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps)
Emits reduction function.
virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues)
Call the appropriate runtime routine to initialize it before start of loop.
Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal) override
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr) override
Emits a critical region.
void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values) override
void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) override
Call the appropriate runtime routine to initialize it before start of loop.
bool emitTargetGlobalVariable(GlobalDecl GD) override
Emit the global variable if it is a valid device global variable.
llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST) override
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
llvm::Function * emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF=nullptr) override
Emit a code for initialization of threadprivate variable.
void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device) override
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
llvm::Function * emitTeamsOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) override
Emits outlined function for the specified OpenMP teams directive D.
void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond, llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, const Expr *Message=nullptr) override
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps, ReductionOptionsTy Options) override
Emit a code for reduction clause.
void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc, llvm::AtomicOrdering AO) override
Emit flush of the variables specified in 'omp flush' directive.
void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C) override
Emit code for doacross ordered directive with 'depend' clause.
void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc) override
Emits a masked region.
Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, QualType VarType, StringRef Name) override
Creates artificial threadprivate variable with name Name and type VarType.
Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc) override
Returns address of the threadprivate variable for the current thread.
void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps) override
Emits a single region.
void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N) override
Required to resolve existing problems in the runtime.
llvm::Function * emitParallelOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) override
Emits outlined function for the specified OpenMP parallel directive D.
void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion) override
Emit code for 'cancellation point' construct.
void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false) override
Emit an implicit/explicit barrier for OpenMP threads.
Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const override
Gets the address of the native argument basing on the address of the target-specific parameter.
void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars) override
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned) override
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
bool emitTargetGlobal(GlobalDecl GD) override
Emit the global GD if it is meaningful for the target.
void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, bool IsWorksharingReduction) override
Emits the following code for reduction clause with task modifier:
void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads) override
Emit an ordered region.
void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind) override
Call the appropriate runtime routine to notify that we finished all the work with current loop.
llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data) override
Emit a code for initialization of task reduction clause.
void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc) override
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32global_tid, int proc_bind) to generate...
void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) override
Emit outilined function for 'target' directive.
void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc) override
Emits a master region.
void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc) override
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32global_tid, kmp_int32 num_teams,...
void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc) override
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
const VarDecl * translateParameter(const FieldDecl *FD, const VarDecl *NativeParam) const override
Translates the native parameter of outlined function if this is required for target.
void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc, OpenMPNumThreadsClauseModifier Modifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, SourceLocation SeverityLoc=SourceLocation(), const Expr *Message=nullptr, SourceLocation MessageLoc=SourceLocation()) override
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32global_tid, kmp_int32 num_threads) ...
void emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter=nullptr) override
Emits a masked region.
void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data) override
Emit task region for the task directive.
void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter) override
Emit the target offloading code associated with D.
bool emitTargetFunctions(GlobalDecl GD) override
Emit the target regions enclosed in GD function definition or the function itself in case it is a val...
void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr * > NumIterations) override
Emit initialization for doacross loop nesting support.
void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion) override
Emit code for 'cancel' construct.
void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPTaskDataTy &Data) override
Emit code for 'taskwait' directive.
void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc) override
Emit a taskgroup region.
void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, CGOpenMPRuntime::TargetDataInfo &Info) override
Emit the target data mapping code associated with D.
void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues) override
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
llvm::Function * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts) override
Emits outlined function for the OpenMP task directive D.
void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data) override
Emit task region for the taskloop directive.
unsigned getNonVirtualBaseLLVMFieldNo(const CXXRecordDecl *RD) const
llvm::StructType * getLLVMType() const
Return the "complete object" LLVM type associated with this record.
llvm::StructType * getBaseSubobjectLLVMType() const
Return the "base subobject" LLVM type associated with this record.
unsigned getLLVMFieldNo(const FieldDecl *FD) const
Return llvm::StructType element number that corresponds to the field FD.
unsigned getVirtualBaseIndex(const CXXRecordDecl *base) const
Return the LLVM field index corresponding to the given virtual base.
API for captured statement code generation.
virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S)
Emit the captured statement body.
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
RAII for correct setting/restoring of CapturedStmtInfo.
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
bool Privatize()
Privatizes local variables previously registered as private.
bool addPrivate(const VarDecl *LocalVD, Address Addr)
Registers LocalVD variable as a private with Addr as the address of the corresponding private variabl...
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited.
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
LValue EmitLoadOfReferenceLValue(LValue RefLVal)
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount, Stmt::Likelihood LH=Stmt::LH_None, const Expr *ConditionalOp=nullptr, const VarDecl *ConditionalDecl=nullptr)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
void emitDestroy(Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray)
emitDestroy - Immediately perform the destruction of the given object.
JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target)
The given basic block lies in the current EH scope, but may be a target of a potentially scope-crossi...
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
void EmitNullInitialization(Address DestPtr, QualType Ty)
EmitNullInitialization - Generate code to set a value of the given type to null, If the type contains...
Address LoadCXXThisAddress()
CGCapturedStmtInfo * CapturedStmtInfo
ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc)
EmitLoadOfComplex - Load a complex number from the specified l-value.
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
const LangOptions & getLangOpts() const
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
EmitAutoVarAlloca - Emit the alloca and debug information for a local variable.
void pushDestroy(QualType::DestructionKind dtorKind, Address addr, QualType type)
pushDestroy - Push the standard destructor for the given type as at least a normal cleanup.
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Load a pointer with type PtrTy stored at address Ptr.
void EmitBranchThroughCleanup(JumpDest Dest)
EmitBranchThroughCleanup - Emit a branch from the current insert block through the normal cleanup han...
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
Destroyer * getDestroyer(QualType::DestructionKind destructionKind)
llvm::AssertingVH< llvm::Instruction > AllocaInsertPt
AllocaInsertPoint - This is an instruction in the entry block before which we prefer to insert alloca...
void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy)
Emit an aggregate assignment.
JumpDest ReturnBlock
ReturnBlock - Unified return block.
void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy, AggValueSlot::Overlap_t MayOverlap, bool isVolatile=false)
EmitAggregateCopy - Emit an aggregate copy.
LValue EmitLValueForField(LValue Base, const FieldDecl *Field, bool IsInBounds=true)
RawAddress CreateDefaultAlignTempAlloca(llvm::Type *Ty, const Twine &Name="tmp")
CreateDefaultAlignedTempAlloca - This creates an alloca with the default ABI alignment of the given L...
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value * > &CapturedVars)
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
LValue EmitArraySectionExpr(const ArraySectionExpr *E, bool IsLowerBound=true)
LValue EmitOMPSharedLValue(const Expr *E)
Emits the lvalue for the expression with possibly captured variable.
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
llvm::Value * emitArrayLength(const ArrayType *arrayType, QualType &baseType, Address &addr)
emitArrayLength - Compute the length of an array, even if it's a VLA, and drill down to the base elem...
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
CGDebugInfo * getDebugInfo()
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
LValue MakeRawAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment, AlignmentSource Source=AlignmentSource::Type)
Same as MakeAddrLValue above except that the pointer is known to be unsigned.
LValue EmitLValueForFieldInitialization(LValue Base, const FieldDecl *Field)
EmitLValueForFieldInitialization - Like EmitLValueForField, except that if the Field is a reference,...
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler's counter for the given statement by StepV.
VlaSizePair getVLASize(const VariableArrayType *vla)
Returns an LLVM value that corresponds to the size, in non-variably-sized elements,...
llvm::CallInst * EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
ASTContext & getContext() const
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
void EmitAutoVarCleanups(const AutoVarEmission &emission)
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy)
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
bool needsEHCleanup(QualType::DestructionKind kind)
Determines whether an EH cleanup is required to destroy a type with the given destruction kind.
llvm::DenseMap< const ValueDecl *, FieldDecl * > LambdaCaptureFields
llvm::CallInst * EmitRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
llvm::Type * ConvertTypeForMem(QualType T)
static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S)
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
CodeGenTypes & getTypes() const
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Emit device code for the target teams distribute parallel for simd directive.
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block,...
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S, const OMPExecutableDirective &D)
RawAddress CreateMemTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
llvm::Value * EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr, ArrayRef< llvm::Value * > IdxList, bool SignedIndices, bool IsSubtraction, SourceLocation Loc, const Twine &Name="")
Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to detect undefined behavior whe...
static void EmitOMPTargetParallelGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelGenericLoopDirective &S)
Emit device code for the target parallel loop directive.
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
static bool IsWrappedCXXThis(const Expr *E)
Check if E is a C++ "this" pointer wrapped in value-preserving casts.
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S)
Emit device code for the target simd directive.
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant,...
static void EmitOMPTargetTeamsGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsGenericLoopDirective &S)
Emit device code for the target teams loop directive.
LValue EmitMemberExpr(const MemberExpr *E)
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
void EmitVariablyModifiedType(QualType Ty)
EmitVLASize - Capture all the sizes for the VLA expressions in the given variably-modified type and s...
bool isTrivialInitializer(const Expr *Init)
Determine whether the given initializer is trivial in the sense that it requires no code to be genera...
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit)
EmitExprAsInit - Emits the code necessary to initialize a location in memory with the given initializ...
LValue MakeNaturalAlignRawAddrLValue(llvm::Value *V, QualType T)
This class organizes the cross-function state that is used while generating LLVM code.
void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
llvm::Module & getModule() const
const IntrusiveRefCntPtr< llvm::vfs::FileSystem > & getFileSystem() const
DiagnosticsEngine & getDiags() const
const LangOptions & getLangOpts() const
CharUnits getNaturalTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, bool forPointeeType=false)
CodeGenTypes & getTypes()
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
TBAAAccessInfo getTBAAInfoForSubobject(LValue Base, QualType AccessType)
getTBAAInfoForSubobject - Get TBAA information for an access with a given base lvalue.
ASTContext & getContext() const
const CodeGenOptions & getCodeGenOpts() const
StringRef getMangledName(GlobalDecl GD)
std::optional< CharUnits > getOMPAllocateAlignment(const VarDecl *VD)
Return the alignment specified in an allocate directive, if present.
llvm::Constant * EmitNullConstant(QualType T)
Return the result of value-initializing the given type, i.e.
llvm::Type * ConvertType(QualType T)
ConvertType - Convert type T into a llvm::Type.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
const CGRecordLayout & getCGRecordLayout(const RecordDecl *)
getCGRecordLayout - Return record layout info for the given record decl.
A specialization of Address that requires the address to be an LLVM Constant.
static ConstantAddress invalid()
bool requiresLandingPad() const
void pushTerminate()
Push a terminate handler on the stack.
void popTerminate()
Pops a terminate handler off the stack.
FunctionArgList - Type for representing both the decl and type of parameters to a function.
LValue - This represents an lvalue references.
CharUnits getAlignment() const
llvm::Value * getPointer(CodeGenFunction &CGF) const
const Qualifiers & getQuals() const
Address getAddress() const
LValueBaseInfo getBaseInfo() const
TBAAAccessInfo getTBAAInfo() const
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
virtual void Enter(CodeGenFunction &CGF)
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
static RValue get(llvm::Value *V)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
An abstract representation of an aligned address.
llvm::Type * getElementType() const
Return the type of the values stored in this address.
llvm::Value * getPointer() const
static RawAddress invalid()
Class intended to support codegen of all kind of the reduction clauses.
LValue getSharedLValue(unsigned N) const
Returns LValue for the reduction item.
const Expr * getRefExpr(unsigned N) const
Returns the base declaration of the reduction item.
LValue getOrigLValue(unsigned N) const
Returns LValue for the original reduction item.
bool needCleanups(unsigned N)
Returns true if the private copy requires cleanups.
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
const VarDecl * getBaseDecl(unsigned N) const
Returns the base declaration of the reduction item.
QualType getPrivateType(unsigned N) const
Return the type of the private item.
bool usesReductionInitializer(unsigned N) const
Returns true if the initialization of the reduction item uses initializer from declare reduction cons...
void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N)
Emits lvalue for the shared and original reduction item.
void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, llvm::function_ref< bool(CodeGenFunction &)> DefaultInit)
Performs initialization of the private copy for the reduction item.
std::pair< llvm::Value *, llvm::Value * > getSizes(unsigned N) const
Returns the size of the reduction item (in chars and total number of elements in the item),...
ReductionCodeGen(ArrayRef< const Expr * > Shareds, ArrayRef< const Expr * > Origs, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > ReductionOps)
void emitCleanups(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Emits cleanup code for the reduction item.
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations.
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
void operator()(CodeGenFunction &CGF) const
void setAction(PrePostActionTy &Action) const
ConstStmtVisitor - This class implements a simple visitor for Stmt subclasses.
DeclContext - This is used only as base class of specific decl types that can act as declaration cont...
void addDecl(Decl *D)
Add the declaration D into this context.
A reference to a declared variable, function, enum, etc.
Decl - This represents one declaration (or definition), e.g.
ASTContext & getASTContext() const LLVM_READONLY
virtual Stmt * getBody() const
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
llvm::iterator_range< specific_attr_iterator< T > > specific_attrs() const
SourceLocation getLocation() const
DeclContext * getDeclContext()
virtual Decl * getCanonicalDecl()
Retrieves the "canonical" declaration of the given declaration.
SourceLocation getBeginLoc() const LLVM_READONLY
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
unsigned getCustomDiagID(Level L, const char(&FormatString)[N])
Return an ID for a diagnostic with the specified format string and level.
This represents one expression.
bool isIntegerConstantExpr(const ASTContext &Ctx) const
Expr * IgnoreParenNoopCasts(const ASTContext &Ctx) LLVM_READONLY
Skip past any parentheses and casts which do not change the value (including ptr->int casts of the sa...
@ SE_AllowSideEffects
Allow any unmodeled side effect.
@ SE_AllowUndefinedBehavior
Allow UB that we can give a value, but not arbitrary unmodeled side effects.
Expr * IgnoreParenCasts() LLVM_READONLY
Skip past any parentheses and casts which might surround this expression until reaching a fixed point...
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
bool isEvaluatable(const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects) const
isEvaluatable - Call EvaluateAsRValue to see if this expression can be constant folded without side-e...
std::optional< llvm::APSInt > getIntegerConstantExpr(const ASTContext &Ctx) const
isIntegerConstantExpr - Return the value if this expression is a valid integer constant expression.
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
bool EvaluateAsBooleanCondition(bool &Result, const ASTContext &Ctx, bool InConstantContext=false) const
EvaluateAsBooleanCondition - Return true if this is a constant which we can fold and convert to a boo...
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
static bool isSameComparisonOperand(const Expr *E1, const Expr *E2)
Checks that the two Expr's will refer to the same value as a comparison operand.
bool hasNonTrivialCall(const ASTContext &Ctx) const
Determine whether this expression involves a call to any function that is not trivial.
Represents a member of a struct/union/class.
unsigned getFieldIndex() const
Returns the index of this field within its record, as appropriate for passing to ASTRecordLayout::get...
const RecordDecl * getParent() const
Returns the parent of this field declaration, which is the struct in which this field is defined.
static FieldDecl * Create(const ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, const IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, Expr *BW, bool Mutable, InClassInitStyle InitStyle)
Represents a function declaration or definition.
const ParmVarDecl * getParamDecl(unsigned i) const
QualType getReturnType() const
ArrayRef< ParmVarDecl * > parameters() const
FunctionDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
FunctionDecl * getMostRecentDecl()
Returns the most recent (re)declaration of this declaration.
unsigned getNumParams() const
Return the number of parameters this function must have based on its FunctionType.
FunctionDecl * getPreviousDecl()
Return the previous declaration of this declaration or NULL if this is the first declaration.
GlobalDecl - represents a global declaration.
const Decl * getDecl() const
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
static IntegerLiteral * Create(const ASTContext &C, const llvm::APInt &V, QualType type, SourceLocation l)
Returns a new integer literal with value 'V' and type 'type'.
MemberExpr - [C99 6.5.2.3] Structure and Union Members.
ValueDecl * getMemberDecl() const
Retrieve the member declaration to which this expression refers.
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
bool isExternallyVisible() const
This represents clause 'affinity' in the 'pragma omp task'-based directives.
Expr * getAssociatedExpression() const
ValueDecl * getAssociatedDeclaration() const
ArrayRef< MappableComponent > MappableExprComponentListRef
static std::pair< const Expr *, std::optional< size_t > > findAttachPtrExpr(MappableExprComponentListRef Components, OpenMPDirectiveKind CurDirKind)
Find the attach pointer expression from a list of mappable expression components.
static QualType getComponentExprElementType(const Expr *Exp)
Get the type of an element of a ComponentList Expr Exp.
const Stmt * getPreInitStmt() const
Get pre-initialization statement for the clause.
This is a basic class for representing single OpenMP clause.
This represents 'pragma omp declare mapper ...' directive.
Expr * getMapperVarRef()
Get the variable declared in the mapper.
This represents 'pragma omp declare reduction ...' directive.
Expr * getInitializer()
Get initializer expression (if specified) of the declare reduction construct.
Expr * getInitPriv()
Get Priv variable of the initializer.
Expr * getCombinerOut()
Get Out variable of the combiner.
Expr * getCombinerIn()
Get In variable of the combiner.
Expr * getCombiner()
Get combiner expression of the declare reduction construct.
Expr * getInitOrig()
Get Orig variable of the initializer.
OMPDeclareReductionInitKind getInitializerKind() const
Get initializer kind.
This represents implicit clause 'depend' for the 'pragma omp task' directive.
This represents 'detach' clause in the 'pragma omp task' directive.
This represents 'device' clause in the 'pragma omp ...' directive.
This represents the 'doacross' clause for the 'pragma omp ordered' directive.
This represents 'if' clause in the 'pragma omp ...' directive.
Expr * getCondition() const
Returns condition.
This represents clause 'in_reduction' in the 'pragma omp task' directives.
OMPIteratorHelperData & getHelper(unsigned I)
Fetches helper data for the specified iteration space.
unsigned numOfIterators() const
Returns number of iterator definitions.
This represents clause 'lastprivate' in the 'pragma omp ...' directives.
This represents clause 'nontemporal' in the 'pragma omp ...' directives.
This represents 'nowait' clause in the 'pragma omp ...' directive.
This represents 'num_teams' clause in the 'pragma omp ...' directive.
This represents 'num_threads' clause in the 'pragma omp ...' directive.
This represents 'ordered' clause in the 'pragma omp ...' directive.
This represents clause 'private' in the 'pragma omp ...' directives.
This represents 'pragma omp requires...' directive.
clauselist_range clauselists()
This represents 'thread_limit' clause in the 'pragma omp ...' directive.
This represents clause 'uses_allocators' in the 'pragma omp target'-based directives.
This represents 'ompx_attribute' clause in a directive that might generate an outlined function.
This represents 'ompx_bare' clause in the 'pragma omp target teams ...' directive.
This represents 'ompx_dyn_cgroup_mem' clause in the 'pragma omp target ...' directive.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class.
Represents a parameter to a function.
PointerType - C99 6.7.5.1 - Pointer Declarators.
Represents an unpacked "presumed" location which can be presented to the user.
unsigned getColumn() const
Return the presumed column number of this location.
const char * getFilename() const
Return the presumed filename of this location.
unsigned getLine() const
Return the presumed line number of this location.
A (possibly-)qualified type.
void addRestrict()
Add the restrict qualifier to this QualType.
QualType withRestrict() const
bool isNull() const
Return true if this QualType doesn't point to a type yet.
const Type * getTypePtr() const
Retrieves a pointer to the underlying (unqualified) type.
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
QualType getCanonicalType() const
DestructionKind isDestructedType() const
Returns a nonzero value if objects of this type require non-trivial work to clean up after.
Represents a struct/union/class.
field_iterator field_end() const
field_range fields() const
virtual void completeDefinition()
Note that the definition of this type is now complete.
field_iterator field_begin() const
Scope - A scope is a transient data structure that is used while parsing the program.
Encodes a location in the source.
static SourceLocation getFromRawEncoding(UIntTy Encoding)
Turn a raw encoding of a SourceLocation object into a real SourceLocation.
bool isValid() const
Return true if this is a valid SourceLocation object.
UIntTy getRawEncoding() const
When a SourceLocation itself cannot be used, this returns an (opaque) 32-bit integer encoding for it.
This class handles loading and caching of source files into memory.
PresumedLoc getPresumedLoc(SourceLocation Loc, bool UseLineDirectives=true) const
Returns the "presumed" location of a SourceLocation specifies.
Stmt - This represents one statement.
StmtClass getStmtClass() const
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top,...
SourceLocation getBeginLoc() const LLVM_READONLY
void startDefinition()
Starts the definition of this tag declaration.
The base class of the type hierarchy.
const Type * getPointeeOrArrayElementType() const
If this is a pointer type, return the pointee type.
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
RecordDecl * getAsRecordDecl() const
Retrieves the RecordDecl this type refers to.
bool isPointerType() const
CanQualType getCanonicalTypeUnqualified() const
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
const T * castAs() const
Member-template castAs<specific type>.
bool isReferenceType() const
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
bool isLValueReferenceType() const
bool isAggregateType() const
Determines whether the type is a C++ aggregate type or C aggregate or union type.
RecordDecl * castAsRecordDecl() const
QualType getCanonicalTypeInternal() const
const Type * getBaseElementTypeUnsafe() const
Get the base element type of this type, potentially discarding type qualifiers.
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type.
bool isFloatingType() const
bool isUnsignedIntegerType() const
Return true if this is an integer type that is unsigned, according to C99 6.2.5p6 [which returns true...
bool isAnyPointerType() const
const T * getAs() const
Member-template getAs<specific type>'.
bool isRecordType() const
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Represents a variable declaration or definition.
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
VarDecl * getDefinition(ASTContext &)
Get the real (not just tentative) definition for this declaration.
const Expr * getInit() const
bool hasExternalStorage() const
Returns true if a variable has extern or private_extern storage.
@ DeclarationOnly
This declaration is only a declaration.
DefinitionKind hasDefinition(ASTContext &) const
Check whether this variable is defined in this translation unit.
bool isLocalVarDeclOrParm() const
Similar to isLocalVarDecl but also includes parameters.
const Expr * getAnyInitializer() const
Get the initializer for this variable, no matter which declaration it is attached to.
Represents a C array with a specified size that is not an integer-constant-expression.
Expr * getSizeExpr() const
specific_attr_iterator - Iterates over a subrange of an AttrVec, only providing attributes that are o...
bool isEmptyRecordForLayout(const ASTContext &Context, QualType T)
isEmptyRecordForLayout - Return true iff a structure contains only empty base classes (per isEmptyRec...
@ Type
The l-value was considered opaque, so the alignment was determined from a type.
@ Decl
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
bool isEmptyFieldForLayout(const ASTContext &Context, const FieldDecl *FD)
isEmptyFieldForLayout - Return true iff the field is "empty", that is, either a zero-width bit-field ...
ComparisonResult
Indicates the result of a tentative comparison.
The JSON file list parser is used to communicate input to InstallAPI.
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
CanQual< Type > CanQualType
Represents a canonical, potentially-qualified type.
bool needsTaskBasedThreadLimit(OpenMPDirectiveKind DKind)
Checks if the specified target directive, combined or not, needs task based thread_limit.
@ Ctor_Complete
Complete object ctor.
bool isa(CodeGen::Address addr)
if(T->getSizeExpr()) TRY_TO(TraverseStmt(const_cast< Expr * >(T -> getSizeExpr())))
bool isOpenMPTargetDataManagementDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target data offload directive.
@ Conditional
A conditional (?:) operator.
@ ICIS_NoInit
No in-class initializer.
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
@ LCK_ByRef
Capturing by reference.
LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE()
@ Private
'private' clause, allowed on 'parallel', 'serial', 'loop', 'parallel loop', and 'serial loop' constru...
@ Vector
'vector' clause, allowed on 'loop', Combined, and 'routine' directives.
@ Reduction
'reduction' clause, allowed on Parallel, Serial, Loop, and the combined constructs.
@ Present
'present' clause, allowed on Compute and Combined constructs, plus 'data' and 'declare'.
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
@ OMPC_SCHEDULE_MODIFIER_last
@ OMPC_SCHEDULE_MODIFIER_unknown
nullptr
This class represents a compute construct, representing a 'Kind' of ‘parallel’, 'serial',...
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
OpenMPDistScheduleClauseKind
OpenMP attributes for 'dist_schedule' clause.
static bool classof(const Stmt *T)
bool isOpenMPTaskingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of tasking directives - task, taskloop,...
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
@ Result
The result type of a method or function.
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
const FunctionProtoType * T
OpenMPDependClauseKind
OpenMP attributes for 'depend' clause.
@ Dtor_Complete
Complete object dtor.
@ Union
The "union" keyword.
bool isOpenMPTargetMapEnteringDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a map-entering target directive.
@ Type
The name was classified as a type.
bool isOpenMPLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a directive with an associated loop construct.
OpenMPSeverityClauseKind
OpenMP attributes for 'severity' clause.
LangAS
Defines the address space values used by the address space qualifier of QualType.
llvm::omp::Directive OpenMPDirectiveKind
OpenMP directives.
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
@ VK_PRValue
A pr-value expression (in the C++11 taxonomy) produces a temporary value.
@ VK_LValue
An l-value expression is a reference to an object with independent storage.
for(const auto &A :T->param_types())
void getOpenMPCaptureRegions(llvm::SmallVectorImpl< OpenMPDirectiveKind > &CaptureRegions, OpenMPDirectiveKind DKind)
Return the captured regions of an OpenMP directive.
OpenMPNumThreadsClauseModifier
@ OMPC_ATOMIC_DEFAULT_MEM_ORDER_unknown
U cast(CodeGen::Address addr)
OpenMPMapModifierKind
OpenMP modifier kind for 'map' clause.
@ OMPC_MAP_MODIFIER_unknown
@ Other
Other implicit parameter.
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
OpenMPMapClauseKind
OpenMP mapping kind for 'map' clause.
Diagnostic wrappers for TextAPI types for error reporting.
Maps the expression for the lastprivate variable to the global copy used to store new value because o...
llvm::SmallVector< bool, 8 > IsPrivateVarReduction
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
llvm::BasicBlock * getBlock() const
unsigned NumberOfTargetItems
Address BasePointersArray
llvm::PointerType * VoidPtrTy
llvm::IntegerType * Int64Ty
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::CallingConv::ID getRuntimeCC() const
llvm::IntegerType * SizeTy
llvm::PointerType * VoidPtrPtrTy
llvm::IntegerType * Int32Ty
llvm::IntegerType * IntPtrTy
llvm::IntegerType * IntTy
int
CharUnits getPointerAlign() const
OpenMPDependClauseKind DepKind
const Expr * IteratorExpr
SmallVector< const Expr *, 4 > DepExprs
EvalResult is a struct with detailed info about an evaluated expression.
Extra information about a function prototype.
Expr * CounterUpdate
Updater for the internal counter: ++CounterVD;.
Data for list of allocators.
Expr * AllocatorTraits
Allocator traits.
Expr * Allocator
Allocator.
Scheduling data for loop-based OpenMP directives.
OpenMPScheduleClauseModifier M2
OpenMPScheduleClauseModifier M1
OpenMPScheduleClauseKind Schedule
Describes how types, statements, expressions, and declarations should be printed.