27#include "llvm/ADT/SmallSet.h"
28#include "llvm/BinaryFormat/Dwarf.h"
29#include "llvm/Frontend/OpenMP/OMPConstants.h"
30#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
31#include "llvm/IR/Constants.h"
32#include "llvm/IR/DebugInfoMetadata.h"
33#include "llvm/IR/Instructions.h"
34#include "llvm/IR/IntrinsicInst.h"
35#include "llvm/IR/Metadata.h"
36#include "llvm/Support/AtomicOrdering.h"
39using namespace CodeGen;
40using namespace llvm::omp;
47class OMPLexicalScope :
public CodeGenFunction::LexicalScope {
49 for (
const auto *
C : S.clauses()) {
51 if (
const auto *PreInit =
52 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
53 for (
const auto *I : PreInit->decls()) {
54 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
57 CodeGenFunction::AutoVarEmission Emission =
66 CodeGenFunction::OMPPrivateScope InlinedShareds;
72 cast<BlockDecl>(CGF.
CurCodeDecl)->capturesVariable(VD));
78 const std::optional<OpenMPDirectiveKind> CapturedRegion = std::nullopt,
79 const bool EmitPreInitStmt =
true)
83 emitPreInitStmt(CGF, S);
86 assert(S.hasAssociatedStmt() &&
87 "Expected associated statement for inlined directive.");
88 const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
90 if (
C.capturesVariable() ||
C.capturesVariableByCopy()) {
91 auto *VD =
C.getCapturedVar();
93 "Canonical decl must be captured.");
97 InlinedShareds.isGlobalVarCaptured(VD)),
102 (void)InlinedShareds.Privatize();
108class OMPParallelScope final :
public OMPLexicalScope {
118 : OMPLexicalScope(CGF, S,
std::nullopt,
119 EmitPreInitStmt(S)) {}
124class OMPTeamsScope final :
public OMPLexicalScope {
133 : OMPLexicalScope(CGF, S,
std::nullopt,
134 EmitPreInitStmt(S)) {}
139class OMPLoopScope :
public CodeGenFunction::RunCleanupsScope {
142 CodeGenFunction::OMPMapVars PreCondVars;
143 if (
auto *LD = dyn_cast<OMPLoopDirective>(&S)) {
145 for (
const auto *E : LD->counters()) {
146 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
148 (void)PreCondVars.setVarAddr(
153 for (
const Expr *IRef :
C->varlists()) {
155 cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
156 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
157 QualType OrigVDTy = OrigVD->getType().getNonReferenceType();
158 (void)PreCondVars.setVarAddr(
167 (void)PreCondVars.apply(CGF);
170 LD->getInnermostCapturedStmt()->getCapturedStmt(),
171 true, LD->getLoopsNumber(),
172 [&CGF](
unsigned Cnt,
const Stmt *CurStmt) {
173 if (
const auto *CXXFor = dyn_cast<CXXForRangeStmt>(CurStmt)) {
174 if (
const Stmt *
Init = CXXFor->getInit())
176 CGF.
EmitStmt(CXXFor->getRangeStmt());
181 PreInits = cast_or_null<DeclStmt>(LD->getPreInits());
182 }
else if (
const auto *
Tile = dyn_cast<OMPTileDirective>(&S)) {
183 PreInits = cast_or_null<DeclStmt>(
Tile->getPreInits());
184 }
else if (
const auto *Unroll = dyn_cast<OMPUnrollDirective>(&S)) {
185 PreInits = cast_or_null<DeclStmt>(Unroll->getPreInits());
187 llvm_unreachable(
"Unknown loop-based directive kind.");
190 for (
const auto *I : PreInits->
decls())
193 PreCondVars.restore(CGF);
199 emitPreInitStmt(CGF, S);
203class OMPSimdLexicalScope :
public CodeGenFunction::LexicalScope {
204 CodeGenFunction::OMPPrivateScope InlinedShareds;
210 cast<BlockDecl>(CGF.
CurCodeDecl)->capturesVariable(VD));
216 InlinedShareds(CGF) {
217 for (
const auto *
C : S.clauses()) {
219 if (
const auto *PreInit =
220 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
221 for (
const auto *I : PreInit->decls()) {
222 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
225 CodeGenFunction::AutoVarEmission Emission =
231 }
else if (
const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(
C)) {
232 for (
const Expr *E : UDP->varlists()) {
233 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
234 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
237 }
else if (
const auto *UDP = dyn_cast<OMPUseDeviceAddrClause>(
C)) {
238 for (
const Expr *E : UDP->varlists()) {
240 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
247 if (
const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
248 if (
const Expr *E = TG->getReductionRef())
249 CGF.
EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
255 if (
C->getModifier() != OMPC_REDUCTION_inscan)
257 for (
const Expr *E :
C->copy_array_temps())
258 CopyArrayTemps.insert(cast<DeclRefExpr>(E)->getDecl());
260 const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
263 if (
C.capturesVariable() ||
C.capturesVariableByCopy()) {
264 auto *VD =
C.getCapturedVar();
265 if (CopyArrayTemps.contains(VD))
268 "Canonical decl must be captured.");
270 isCapturedVar(CGF, VD) ||
272 InlinedShareds.isGlobalVarCaptured(VD)),
280 (void)InlinedShareds.Privatize();
290LValue CodeGenFunction::EmitOMPSharedLValue(
const Expr *E) {
291 if (
const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
292 if (
const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
293 OrigVD = OrigVD->getCanonicalDecl();
308 llvm::Value *
Size =
nullptr;
309 auto SizeInChars =
C.getTypeSizeInChars(Ty);
310 if (SizeInChars.isZero()) {
316 Size ?
Builder.CreateNUWMul(Size, VlaSize.NumElts) : VlaSize.NumElts;
318 SizeInChars =
C.getTypeSizeInChars(Ty);
319 if (SizeInChars.isZero())
320 return llvm::ConstantInt::get(
SizeTy, 0);
328 const RecordDecl *RD = S.getCapturedRecordDecl();
330 auto CurCap = S.captures().begin();
332 E = S.capture_init_end();
333 I != E; ++I, ++CurField, ++CurCap) {
334 if (CurField->hasCapturedVLAType()) {
337 CapturedVars.push_back(Val);
338 }
else if (CurCap->capturesThis()) {
339 CapturedVars.push_back(CXXThisValue);
340 }
else if (CurCap->capturesVariableByCopy()) {
345 if (!CurField->getType()->isAnyPointerType()) {
349 Twine(CurCap->getCapturedVar()->getName(),
".casted"));
364 CapturedVars.push_back(CV);
366 assert(CurCap->capturesVariable() &&
"Expected capture by reference.");
387 return C.getLValueReferenceType(
393 if (
const auto *VLA = dyn_cast<VariableArrayType>(A))
395 if (!A->isVariablyModifiedType())
396 return C.getCanonicalType(T);
398 return C.getCanonicalParamType(T);
403struct FunctionOptions {
408 const bool UIntPtrCastRequired =
true;
411 const bool RegisterCastedArgsOnly =
false;
413 const StringRef FunctionName;
416 explicit FunctionOptions(
const CapturedStmt *S,
bool UIntPtrCastRequired,
417 bool RegisterCastedArgsOnly, StringRef FunctionName,
419 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
420 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
421 FunctionName(FunctionName), Loc(Loc) {}
427 llvm::MapVector<
const Decl *, std::pair<const VarDecl *, Address>>
429 llvm::DenseMap<
const Decl *, std::pair<const Expr *, llvm::Value *>>
431 llvm::Value *&CXXThisValue,
const FunctionOptions &FO) {
433 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
434 assert(CD->
hasBody() &&
"missing CapturedDecl body");
436 CXXThisValue =
nullptr;
446 auto I = FO.S->captures().begin();
448 if (!FO.UIntPtrCastRequired) {
468 if (FO.UIntPtrCastRequired &&
470 I->capturesVariableArrayType()))
473 if (I->capturesVariable() || I->capturesVariableByCopy()) {
474 CapVar = I->getCapturedVar();
476 }
else if (I->capturesThis()) {
479 assert(I->capturesVariableArrayType());
489 }
else if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
491 Ctx, DebugFunctionDecl,
492 CapVar ? CapVar->
getBeginLoc() : FD->getBeginLoc(),
493 CapVar ? CapVar->
getLocation() : FD->getLocation(), II, ArgType,
499 Args.emplace_back(Arg);
501 TargetArgs.emplace_back(
502 FO.UIntPtrCastRequired
519 llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
523 F->setDoesNotThrow();
524 F->setDoesNotRecurse();
528 F->removeFnAttr(llvm::Attribute::NoInline);
529 F->addFnAttr(llvm::Attribute::AlwaysInline);
534 FO.UIntPtrCastRequired ? FO.Loc : FO.S->getBeginLoc(),
535 FO.UIntPtrCastRequired ? FO.Loc
538 I = FO.S->captures().begin();
542 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
550 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
551 const VarDecl *CurVD = I->getCapturedVar();
552 if (!FO.RegisterCastedArgsOnly)
553 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
561 if (FD->hasCapturedVLAType()) {
562 if (FO.UIntPtrCastRequired) {
565 Args[Cnt]->getName(), ArgLVal),
570 VLASizes.try_emplace(Args[Cnt], VAT->
getSizeExpr(), ExprArg);
571 }
else if (I->capturesVariable()) {
572 const VarDecl *Var = I->getCapturedVar();
582 if (!FO.RegisterCastedArgsOnly) {
586 }
else if (I->capturesVariableByCopy()) {
587 assert(!FD->getType()->isAnyPointerType() &&
588 "Not expecting a captured pointer.");
589 const VarDecl *Var = I->getCapturedVar();
590 LocalAddrs.insert({Args[Cnt],
591 {Var, FO.UIntPtrCastRequired
593 CGF, I->getLocation(), FD->getType(),
594 Args[Cnt]->getName(), ArgLVal)
598 assert(I->capturesThis());
600 LocalAddrs.insert({Args[Cnt], {
nullptr, ArgLVal.
getAddress(CGF)}});
614 "CapturedStmtInfo should be set when generating the captured function");
617 bool NeedWrapperFunction =
620 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
621 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
623 llvm::raw_svector_ostream Out(Buffer);
625 if (NeedWrapperFunction)
627 FunctionOptions FO(&S, !NeedWrapperFunction,
false,
630 VLASizes, CXXThisValue, FO);
631 CodeGenFunction::OMPPrivateScope LocalScope(*
this);
632 for (
const auto &LocalAddrPair : LocalAddrs) {
633 if (LocalAddrPair.second.first) {
634 LocalScope.addPrivate(LocalAddrPair.second.first,
635 LocalAddrPair.second.second);
638 (void)LocalScope.Privatize();
639 for (
const auto &VLASizePair : VLASizes)
640 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
643 (void)LocalScope.ForceCleanup();
645 if (!NeedWrapperFunction)
648 FunctionOptions WrapperFO(&S,
true,
656 llvm::Function *WrapperF =
658 WrapperCGF.CXXThisValue, WrapperFO);
660 auto *PI = F->arg_begin();
661 for (
const auto *Arg : Args) {
663 auto I = LocalAddrs.find(Arg);
664 if (I != LocalAddrs.end()) {
665 LValue LV = WrapperCGF.MakeAddrLValue(
667 I->second.first ? I->second.first->getType() : Arg->getType(),
671 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
673 auto EI = VLASizes.find(Arg);
674 if (EI != VLASizes.end()) {
678 WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
680 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
683 CallArgs.emplace_back(WrapperCGF.EmitFromMemory(
CallArg, Arg->
getType()));
687 WrapperCGF.FinishFunction();
702 llvm::Value *NumElements =
emitArrayLength(ArrayTy, ElementTy, DestAddr);
706 llvm::Value *DestBegin = DestAddr.
getPointer();
709 DestBegin, NumElements);
714 llvm::Value *IsEmpty =
715 Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arraycpy.isempty");
716 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
719 llvm::BasicBlock *EntryBB =
Builder.GetInsertBlock();
724 llvm::PHINode *SrcElementPHI =
725 Builder.CreatePHI(SrcBegin->getType(), 2,
"omp.arraycpy.srcElementPast");
726 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
731 llvm::PHINode *DestElementPHI =
Builder.CreatePHI(
732 DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
733 DestElementPHI->addIncoming(DestBegin, EntryBB);
739 CopyGen(DestElementCurrent, SrcElementCurrent);
742 llvm::Value *DestElementNext =
744 1,
"omp.arraycpy.dest.element");
745 llvm::Value *SrcElementNext =
747 1,
"omp.arraycpy.src.element");
750 Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
751 Builder.CreateCondBr(Done, DoneBB, BodyBB);
752 DestElementPHI->addIncoming(DestElementNext,
Builder.GetInsertBlock());
753 SrcElementPHI->addIncoming(SrcElementNext,
Builder.GetInsertBlock());
763 const auto *BO = dyn_cast<BinaryOperator>(
Copy);
764 if (BO && BO->getOpcode() == BO_Assign) {
773 DestAddr, SrcAddr, OriginalType,
778 CodeGenFunction::OMPPrivateScope Remap(*
this);
779 Remap.addPrivate(DestVD, DestElement);
780 Remap.addPrivate(SrcVD, SrcElement);
781 (void)Remap.Privatize();
787 CodeGenFunction::OMPPrivateScope Remap(*
this);
788 Remap.addPrivate(SrcVD, SrcAddr);
789 Remap.addPrivate(DestVD, DestAddr);
790 (void)Remap.Privatize();
797 OMPPrivateScope &PrivateScope) {
800 bool DeviceConstTarget =
803 bool FirstprivateIsLastprivate =
false;
804 llvm::DenseMap<const VarDecl *, OpenMPLastprivateModifier> Lastprivates;
806 for (
const auto *D :
C->varlists())
807 Lastprivates.try_emplace(
816 bool MustEmitFirstprivateCopy =
817 CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
819 const auto *IRef =
C->varlist_begin();
820 const auto *InitsRef =
C->inits().begin();
821 for (
const Expr *IInit :
C->private_copies()) {
822 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
823 bool ThisFirstprivateIsLastprivate =
824 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
826 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
827 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
829 (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())) {
830 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
837 if (DeviceConstTarget && OrigVD->getType().isConstant(
getContext()) &&
839 (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())) {
840 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
845 FirstprivateIsLastprivate =
846 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
847 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
849 cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
858 if (CE && !CE.isReference()) {
864 if (CE && CE.isReference()) {
865 OriginalLVal = CE.getReferenceLValue(*
this, &DRE);
867 assert(!CE &&
"Expected non-constant firstprivate.");
886 Emission.getAllocatedAddress(), OriginalLVal.
getAddress(*
this),
891 RunCleanupsScope InitScope(*this);
893 setAddrOfLocalVar(VDInit, SrcElement);
894 EmitAnyExprToMem(Init, DestElement,
895 Init->getType().getQualifiers(),
897 LocalDeclMap.erase(VDInit);
902 PrivateScope.addPrivate(OrigVD, Emission.getAllocatedAddress());
908 setAddrOfLocalVar(VDInit, OriginalAddr);
910 LocalDeclMap.erase(VDInit);
912 if (ThisFirstprivateIsLastprivate &&
913 Lastprivates[OrigVD->getCanonicalDecl()] ==
914 OMPC_LASTPRIVATE_conditional) {
919 (*IRef)->getExprLoc());
924 LocalDeclMap.erase(VD);
925 setAddrOfLocalVar(VD, VDAddr);
927 IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);
929 assert(IsRegistered &&
930 "firstprivate var already registered as private");
938 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
943 CodeGenFunction::OMPPrivateScope &PrivateScope) {
948 auto IRef =
C->varlist_begin();
949 for (
const Expr *IInit :
C->private_copies()) {
950 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
951 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
952 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
957 assert(IsRegistered &&
"private var already registered as private");
974 llvm::BasicBlock *CopyBegin =
nullptr, *CopyEnd =
nullptr;
976 auto IRef =
C->varlist_begin();
977 auto ISrcRef =
C->source_exprs().begin();
978 auto IDestRef =
C->destination_exprs().begin();
979 for (
const Expr *AssignOp :
C->assignment_ops()) {
980 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
988 getContext().getTargetInfo().isTLSSupported()) {
990 "Copyin threadprivates should have been captured!");
994 LocalDeclMap.erase(VD);
1004 if (CopiedVars.size() == 1) {
1010 auto *MasterAddrInt =
1012 auto *PrivateAddrInt =
1015 Builder.CreateICmpNE(MasterAddrInt, PrivateAddrInt), CopyBegin,
1020 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1021 const auto *DestVD =
1022 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1042 bool HasAtLeastOneLastprivate =
false;
1045 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1046 for (
const Expr *
C : LoopDirective->counters()) {
1053 HasAtLeastOneLastprivate =
true;
1057 const auto *IRef =
C->varlist_begin();
1058 const auto *IDestRef =
C->destination_exprs().begin();
1059 for (
const Expr *IInit :
C->private_copies()) {
1062 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1065 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
1066 const auto *DestVD =
1067 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1072 PrivateScope.addPrivate(DestVD,
EmitLValue(&DRE).getAddress(*
this));
1076 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
1077 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
1079 if (
C->getKind() == OMPC_LASTPRIVATE_conditional) {
1082 setAddrOfLocalVar(VD, VDAddr);
1088 bool IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);
1089 assert(IsRegistered &&
1090 "lastprivate var already registered as private");
1098 return HasAtLeastOneLastprivate;
1103 llvm::Value *IsLastIterCond) {
1112 llvm::BasicBlock *ThenBB =
nullptr;
1113 llvm::BasicBlock *DoneBB =
nullptr;
1114 if (IsLastIterCond) {
1120 return C->getKind() == OMPC_LASTPRIVATE_conditional;
1129 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1133 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1134 if (
const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1135 auto IC = LoopDirective->counters().begin();
1136 for (
const Expr *F : LoopDirective->finals()) {
1138 cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1140 AlreadyEmittedVars.insert(D);
1142 LoopCountersAndUpdates[D] = F;
1147 auto IRef =
C->varlist_begin();
1148 auto ISrcRef =
C->source_exprs().begin();
1149 auto IDestRef =
C->destination_exprs().begin();
1150 for (
const Expr *AssignOp :
C->assignment_ops()) {
1151 const auto *PrivateVD =
1152 cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1154 const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1155 if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1159 if (
const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1162 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1163 const auto *DestVD =
1164 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1167 if (
const auto *RefTy = PrivateVD->getType()->getAs<
ReferenceType>())
1173 if (
C->getKind() == OMPC_LASTPRIVATE_conditional)
1175 *
this,
MakeAddrLValue(PrivateAddr, (*IRef)->getType()), PrivateVD,
1176 (*IRef)->getExprLoc());
1179 EmitOMPCopy(
Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1185 if (
const Expr *PostUpdate =
C->getPostUpdateExpr())
1194 CodeGenFunction::OMPPrivateScope &PrivateScope,
bool ForInscan) {
1206 if (ForInscan != (
C->getModifier() == OMPC_REDUCTION_inscan))
1208 Shareds.append(
C->varlist_begin(),
C->varlist_end());
1209 Privates.append(
C->privates().begin(),
C->privates().end());
1210 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
1211 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
1212 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
1213 if (
C->getModifier() == OMPC_REDUCTION_task) {
1214 Data.ReductionVars.append(
C->privates().begin(),
C->privates().end());
1215 Data.ReductionOrigs.append(
C->varlist_begin(),
C->varlist_end());
1216 Data.ReductionCopies.append(
C->privates().begin(),
C->privates().end());
1217 Data.ReductionOps.append(
C->reduction_ops().begin(),
1218 C->reduction_ops().end());
1219 TaskLHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
1220 TaskRHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
1225 auto *ILHS = LHSs.begin();
1226 auto *IRHS = RHSs.begin();
1227 auto *IPriv = Privates.begin();
1228 for (
const Expr *IRef : Shareds) {
1229 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1231 RedCG.emitSharedOrigLValue(*
this, Count);
1232 RedCG.emitAggregateType(*
this, Count);
1234 RedCG.emitInitialization(*
this, Count, Emission.getAllocatedAddress(),
1235 RedCG.getSharedLValue(Count).getAddress(*
this),
1237 CGF.EmitAutoVarInit(Emission);
1241 Address BaseAddr = RedCG.adjustPrivateAddress(
1242 *
this, Count, Emission.getAllocatedAddress());
1244 PrivateScope.addPrivate(RedCG.getBaseDecl(Count), BaseAddr);
1245 assert(IsRegistered &&
"private var already registered as private");
1249 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1250 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1252 bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1256 PrivateScope.addPrivate(LHSVD,
1257 RedCG.getSharedLValue(Count).getAddress(*
this));
1260 isa<ArraySubscriptExpr>(IRef)) {
1263 PrivateScope.addPrivate(LHSVD,
1264 RedCG.getSharedLValue(Count).getAddress(*
this));
1265 PrivateScope.addPrivate(RHSVD,
1271 Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress(*
this);
1278 PrivateScope.addPrivate(LHSVD, OriginalAddr);
1279 PrivateScope.addPrivate(
1289 if (!
Data.ReductionVars.empty()) {
1290 Data.IsReductionWithTaskMod =
true;
1291 Data.IsWorksharingReduction =
1295 const Expr *TaskRedRef =
nullptr;
1298 TaskRedRef = cast<OMPParallelDirective>(D).getTaskReductionRefExpr();
1301 TaskRedRef = cast<OMPForDirective>(D).getTaskReductionRefExpr();
1304 TaskRedRef = cast<OMPSectionsDirective>(D).getTaskReductionRefExpr();
1306 case OMPD_parallel_for:
1307 TaskRedRef = cast<OMPParallelForDirective>(D).getTaskReductionRefExpr();
1309 case OMPD_parallel_master:
1311 cast<OMPParallelMasterDirective>(D).getTaskReductionRefExpr();
1313 case OMPD_parallel_sections:
1315 cast<OMPParallelSectionsDirective>(D).getTaskReductionRefExpr();
1317 case OMPD_target_parallel:
1319 cast<OMPTargetParallelDirective>(D).getTaskReductionRefExpr();
1321 case OMPD_target_parallel_for:
1323 cast<OMPTargetParallelForDirective>(D).getTaskReductionRefExpr();
1325 case OMPD_distribute_parallel_for:
1327 cast<OMPDistributeParallelForDirective>(D).getTaskReductionRefExpr();
1329 case OMPD_teams_distribute_parallel_for:
1330 TaskRedRef = cast<OMPTeamsDistributeParallelForDirective>(D)
1331 .getTaskReductionRefExpr();
1333 case OMPD_target_teams_distribute_parallel_for:
1334 TaskRedRef = cast<OMPTargetTeamsDistributeParallelForDirective>(D)
1335 .getTaskReductionRefExpr();
1343 case OMPD_parallel_for_simd:
1345 case OMPD_taskyield:
1349 case OMPD_taskgroup:
1357 case OMPD_cancellation_point:
1359 case OMPD_target_data:
1360 case OMPD_target_enter_data:
1361 case OMPD_target_exit_data:
1363 case OMPD_taskloop_simd:
1364 case OMPD_master_taskloop:
1365 case OMPD_master_taskloop_simd:
1366 case OMPD_parallel_master_taskloop:
1367 case OMPD_parallel_master_taskloop_simd:
1368 case OMPD_distribute:
1369 case OMPD_target_update:
1370 case OMPD_distribute_parallel_for_simd:
1371 case OMPD_distribute_simd:
1372 case OMPD_target_parallel_for_simd:
1373 case OMPD_target_simd:
1374 case OMPD_teams_distribute:
1375 case OMPD_teams_distribute_simd:
1376 case OMPD_teams_distribute_parallel_for_simd:
1377 case OMPD_target_teams:
1378 case OMPD_target_teams_distribute:
1379 case OMPD_target_teams_distribute_parallel_for_simd:
1380 case OMPD_target_teams_distribute_simd:
1381 case OMPD_declare_target:
1382 case OMPD_end_declare_target:
1383 case OMPD_threadprivate:
1385 case OMPD_declare_reduction:
1386 case OMPD_declare_mapper:
1387 case OMPD_declare_simd:
1389 case OMPD_declare_variant:
1390 case OMPD_begin_declare_variant:
1391 case OMPD_end_declare_variant:
1394 llvm_unreachable(
"Enexpected directive with task reductions.");
1397 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(TaskRedRef)->getDecl());
1400 false, TaskRedRef->
getType());
1412 bool HasAtLeastOneReduction =
false;
1413 bool IsReductionWithTaskMod =
false;
1416 if (
C->getModifier() == OMPC_REDUCTION_inscan)
1418 HasAtLeastOneReduction =
true;
1419 Privates.append(
C->privates().begin(),
C->privates().end());
1420 LHSExprs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
1421 RHSExprs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
1422 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
1423 IsReductionWithTaskMod =
1424 IsReductionWithTaskMod ||
C->getModifier() == OMPC_REDUCTION_task;
1426 if (HasAtLeastOneReduction) {
1427 if (IsReductionWithTaskMod) {
1434 ReductionKind == OMPD_simd;
1435 bool SimpleReduction = ReductionKind == OMPD_simd;
1439 *
this, D.
getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1440 {WithNowait, SimpleReduction, ReductionKind});
1449 llvm::BasicBlock *DoneBB =
nullptr;
1451 if (
const Expr *PostUpdate =
C->getPostUpdateExpr()) {
1453 if (llvm::Value *Cond = CondGen(CGF)) {
1458 CGF.
Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1476 CodeGenBoundParametersTy;
1486 for (
const Expr *Ref :
C->varlists()) {
1487 if (!Ref->getType()->isScalarType())
1489 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1492 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1497 for (
const Expr *Ref :
C->varlists()) {
1498 if (!Ref->getType()->isScalarType())
1500 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1503 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1508 for (
const Expr *Ref :
C->varlists()) {
1509 if (!Ref->getType()->isScalarType())
1511 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1514 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1523 for (
const Expr *Ref :
C->varlists()) {
1524 if (!Ref->getType()->isScalarType())
1526 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1529 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1533 CGF, S, PrivateDecls);
1539 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1540 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1541 llvm::Value *NumThreads =
nullptr;
1542 llvm::Function *OutlinedFn =
1547 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1548 NumThreads = CGF.
EmitScalarExpr(NumThreadsClause->getNumThreads(),
1551 CGF, NumThreads, NumThreadsClause->getBeginLoc());
1554 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1556 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1558 const Expr *IfCond =
nullptr;
1559 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
1560 if (
C->getNameModifier() == OMPD_unknown ||
1561 C->getNameModifier() == OMPD_parallel) {
1562 IfCond =
C->getCondition();
1567 OMPParallelScope
Scope(CGF, S);
1573 CodeGenBoundParameters(CGF, S, CapturedVars);
1576 CapturedVars, IfCond, NumThreads);
1581 if (!CVD->
hasAttr<OMPAllocateDeclAttr>())
1583 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
1585 return !((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||
1586 AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&
1587 !AA->getAllocator());
1602 CGF, S.getBeginLoc(), OMPD_unknown,
false,
1622 Size = CGF.
Builder.CreateNUWAdd(
1631 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
1632 assert(AA->getAllocator() &&
1633 "Expected allocator expression for non-default allocator.");
1637 if (Allocator->getType()->isIntegerTy())
1639 else if (Allocator->getType()->isPointerTy())
1643 llvm::Value *Addr = OMPBuilder.createOMPAlloc(
1646 llvm::CallInst *FreeCI =
1647 OMPBuilder.createOMPFree(CGF.
Builder, Addr, Allocator);
1671 std::string Suffix = getNameWithSeparators({
"cache",
""});
1674 llvm::CallInst *ThreadPrivateCacheCall =
1675 OMPBuilder.createCachedThreadPrivate(CGF.
Builder,
Data, Size, CacheName);
1683 llvm::raw_svector_ostream OS(Buffer);
1684 StringRef Sep = FirstSeparator;
1685 for (StringRef Part : Parts) {
1689 return OS.str().str();
1697 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
Builder,
false,
1698 "." + RegionName +
".after");
1714 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
Builder,
false,
1715 "." + RegionName +
".after");
1730 llvm::Value *IfCond =
nullptr;
1735 llvm::Value *NumThreads =
nullptr;
1740 ProcBindKind ProcBind = OMP_PROC_BIND_default;
1742 ProcBind = ProcBindClause->getProcBindKind();
1744 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1748 auto FiniCB = [
this](InsertPointTy IP) {
1756 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
1757 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
1765 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1768 auto BodyGenCB = [&,
this](InsertPointTy AllocaIP,
1769 InsertPointTy CodeGenIP) {
1771 *
this, ParallelRegionBodyStmt, AllocaIP, CodeGenIP,
"parallel");
1774 CGCapturedStmtInfo CGSI(*CS,
CR_OpenMP);
1775 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*
this, &CGSI);
1776 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
1779 OMPBuilder.createParallel(
Builder, AllocaIP, BodyGenCB, PrivCB, FiniCB,
1780 IfCond, NumThreads, ProcBind, S.hasCancel()));
1787 OMPPrivateScope PrivateScope(CGF);
1792 (void)PrivateScope.Privatize();
1793 CGF.
EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1814class OMPTransformDirectiveScopeRAII {
1815 OMPLoopScope *
Scope =
nullptr;
1816 CodeGenFunction::CGCapturedStmtInfo *CGSI =
nullptr;
1817 CodeGenFunction::CGCapturedStmtRAII *CapInfoRAII =
nullptr;
1819 OMPTransformDirectiveScopeRAII(
const OMPTransformDirectiveScopeRAII &) =
1821 OMPTransformDirectiveScopeRAII &
1822 operator=(
const OMPTransformDirectiveScopeRAII &) =
delete;
1826 if (
const auto *Dir = dyn_cast<OMPLoopBasedDirective>(S)) {
1827 Scope =
new OMPLoopScope(CGF, *Dir);
1828 CGSI =
new CodeGenFunction::CGCapturedStmtInfo(
CR_OpenMP);
1829 CapInfoRAII =
new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI);
1832 ~OMPTransformDirectiveScopeRAII() {
1843 int MaxLevel,
int Level = 0) {
1844 assert(Level < MaxLevel &&
"Too deep lookup during loop body codegen.");
1845 const Stmt *SimplifiedS = S->IgnoreContainers();
1846 if (
const auto *CS = dyn_cast<CompoundStmt>(SimplifiedS)) {
1849 "LLVM IR generation of compound statement ('{}')");
1852 CodeGenFunction::LexicalScope
Scope(CGF, S->getSourceRange());
1853 for (
const Stmt *CurStmt : CS->body())
1854 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level);
1857 if (SimplifiedS == NextLoop) {
1858 if (
auto *Dir = dyn_cast<OMPLoopTransformationDirective>(SimplifiedS))
1859 SimplifiedS = Dir->getTransformedStmt();
1860 if (
const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(SimplifiedS))
1861 SimplifiedS = CanonLoop->getLoopStmt();
1862 if (
const auto *For = dyn_cast<ForStmt>(SimplifiedS)) {
1865 assert(isa<CXXForRangeStmt>(SimplifiedS) &&
1866 "Expected canonical for loop or range-based for loop.");
1867 const auto *CXXFor = cast<CXXForRangeStmt>(SimplifiedS);
1868 CGF.
EmitStmt(CXXFor->getLoopVarStmt());
1869 S = CXXFor->getBody();
1871 if (Level + 1 < MaxLevel) {
1874 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1);
1883 RunCleanupsScope BodyScope(*
this);
1892 for (
const Expr *UE :
C->updates())
1899 BreakContinueStack.push_back(BreakContinue(
LoopExit, Continue));
1911 OMPPrivateScope InscanScope(*
this);
1913 bool IsInscanRegion = InscanScope.Privatize();
1914 if (IsInscanRegion) {
1946 BreakContinueStack.pop_back();
1957 std::unique_ptr<CodeGenFunction::CGCapturedStmtInfo> CSI =
1958 std::make_unique<CodeGenFunction::CGCapturedStmtInfo>(*S);
1959 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, CSI.get());
1966static llvm::CallInst *
1971 EffectiveArgs.reserve(Args.size() + 1);
1972 llvm::append_range(EffectiveArgs, Args);
1973 EffectiveArgs.push_back(Cap.second);
1978llvm::CanonicalLoopInfo *
1980 assert(Depth == 1 &&
"Nested loops with OpenMPIRBuilder not yet implemented");
2006 const Stmt *SyntacticalLoop = S->getLoopStmt();
2013 LexicalScope ForScope(*
this, S->getSourceRange());
2017 const Stmt *BodyStmt;
2018 if (
const auto *For = dyn_cast<ForStmt>(SyntacticalLoop)) {
2019 if (
const Stmt *InitStmt = For->getInit())
2021 BodyStmt = For->getBody();
2022 }
else if (
const auto *RangeFor =
2023 dyn_cast<CXXForRangeStmt>(SyntacticalLoop)) {
2024 if (
const DeclStmt *RangeStmt = RangeFor->getRangeStmt())
2026 if (
const DeclStmt *BeginStmt = RangeFor->getBeginStmt())
2028 if (
const DeclStmt *EndStmt = RangeFor->getEndStmt())
2030 if (
const DeclStmt *LoopVarStmt = RangeFor->getLoopVarStmt())
2032 BodyStmt = RangeFor->getBody();
2034 llvm_unreachable(
"Expected for-stmt or range-based for-stmt");
2037 const CapturedStmt *DistanceFunc = S->getDistanceFunc();
2054 auto BodyGen = [&,
this](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP,
2055 llvm::Value *IndVar) {
2060 const DeclRefExpr *LoopVarRef = S->getLoopVarRef();
2066 RunCleanupsScope BodyScope(*
this);
2069 llvm::CanonicalLoopInfo *CL =
2070 OMPBuilder.createCanonicalLoop(
Builder, BodyGen, DistVal);
2073 Builder.restoreIP(CL->getAfterIP());
2074 ForScope.ForceCleanup();
2082 const Expr *IncExpr,
2093 const auto &OMPED = cast<OMPExecutableDirective>(S);
2094 const CapturedStmt *ICS = OMPED.getInnermostCapturedStmt();
2108 llvm::BasicBlock *ExitBlock =
LoopExit.getBlock();
2109 if (RequiresCleanup)
2116 if (ExitBlock !=
LoopExit.getBlock()) {
2126 BreakContinueStack.push_back(BreakContinue(
LoopExit, Continue));
2134 BreakContinueStack.pop_back();
2145 bool HasLinears =
false;
2149 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
Init)->getDecl());
2150 if (
const auto *Ref =
2153 const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
2169 if (
const auto *CS = cast_or_null<BinaryOperator>(
C->getCalcStep()))
2170 if (
const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
2184 llvm::BasicBlock *DoneBB =
nullptr;
2187 auto IC =
C->varlist_begin();
2188 for (
const Expr *F :
C->finals()) {
2190 if (llvm::Value *Cond = CondGen(*
this)) {
2195 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2199 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
2204 CodeGenFunction::OMPPrivateScope VarScope(*
this);
2205 VarScope.addPrivate(OrigVD, OrigAddr);
2206 (void)VarScope.Privatize();
2210 if (
const Expr *PostUpdate =
C->getPostUpdateExpr())
2222 llvm::APInt ClauseAlignment(64, 0);
2223 if (
const Expr *AlignmentExpr = Clause->getAlignment()) {
2226 ClauseAlignment = AlignmentCI->getValue();
2228 for (
const Expr *E : Clause->varlists()) {
2229 llvm::APInt Alignment(ClauseAlignment);
2230 if (Alignment == 0) {
2240 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2241 "alignment is not power of 2");
2242 if (Alignment != 0) {
2256 auto I = S.private_counters().begin();
2257 for (
const Expr *E : S.counters()) {
2258 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2259 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
2263 LocalDeclMap.erase(PrivateVD);
2264 (void)LoopScope.addPrivate(VD, VarEmission.getAllocatedAddress());
2272 (void)LoopScope.addPrivate(PrivateVD, VarEmission.getAllocatedAddress());
2278 if (!
C->getNumForLoops())
2280 for (
unsigned I = S.getLoopsNumber(), E =
C->getLoopNumIterations().size();
2282 const auto *DRE = cast<DeclRefExpr>(
C->getLoopCounter(I));
2283 const auto *VD = cast<VarDecl>(DRE->getDecl());
2286 if (DRE->refersToEnclosingVariableOrCapture()) {
2287 (void)LoopScope.addPrivate(
2295 const Expr *Cond, llvm::BasicBlock *TrueBlock,
2296 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
2300 CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
2302 (void)PreCondScope.Privatize();
2304 for (
const Expr *I : S.inits()) {
2310 CodeGenFunction::OMPMapVars PreCondVars;
2311 for (
const Expr *E : S.dependent_counters()) {
2314 assert(!E->getType().getNonReferenceType()->isRecordType() &&
2315 "dependent counter must not be an iterator.");
2316 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2319 (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr);
2321 (void)PreCondVars.apply(CGF);
2322 for (
const Expr *E : S.dependent_inits()) {
2329 PreCondVars.restore(CGF);
2333 const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
2338 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
2339 for (
const Expr *
C : LoopDirective->counters()) {
2345 auto CurPrivate =
C->privates().begin();
2346 for (
const Expr *E :
C->varlists()) {
2347 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2348 const auto *PrivateVD =
2349 cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
2355 assert(IsRegistered &&
"linear var already registered as private");
2373 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
2382 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
2397 if (
C->getKind() == OMPC_ORDER_concurrent)
2404 return C->getModifier() == OMPC_REDUCTION_inscan;
2415 llvm::BasicBlock *DoneBB =
nullptr;
2419 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
2420 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
2421 const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
2423 OrigVD->hasGlobalStorage() || CED) {
2425 if (llvm::Value *Cond = CondGen(*
this)) {
2430 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2444 OMPPrivateScope VarScope(*
this);
2445 VarScope.addPrivate(OrigVD, OrigAddr);
2446 (void)VarScope.Privatize();
2458 CodeGenFunction::JumpDest
LoopExit) {
2466 auto VDecl = cast<VarDecl>(Helper->
getDecl());
2474 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](
CodeGenFunction &CGF,
2477 CodeGenFunction::OMPLocalDeclMapRAII
Scope(CGF);
2483 CodeGenFunction::OMPLocalDeclMapRAII
Scope(CGF);
2488 const Expr *IfCond =
nullptr;
2490 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
2492 (
C->getNameModifier() == OMPD_unknown ||
2493 C->getNameModifier() == OMPD_simd)) {
2494 IfCond =
C->getCondition();
2511 "Expected simd directive");
2512 OMPLoopScope PreInitScope(CGF, S);
2529 llvm::BasicBlock *ContBlock =
nullptr;
2536 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
2543 const Expr *IVExpr = S.getIterationVariable();
2544 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
2551 if (
const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2552 CGF.
EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2560 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2566 CGF, S, CGF.
EmitLValue(S.getIterationVariable()));
2568 (void)LoopScope.Privatize();
2579 S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),
2581 emitOMPLoopBodyWithStopPoint(CGF, S,
2582 CodeGenFunction::JumpDest());
2588 if (HasLastprivateClause)
2593 LoopScope.restoreMap();
2607 if (!(isa<OMPSimdlenClause>(
C) || isa<OMPSafelenClause>(
C) ||
2608 isa<OMPOrderClause>(
C) || isa<OMPAlignedClause>(
C)))
2615 if (
const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(S.getRawStmt())) {
2616 if (
const Stmt *SyntacticalLoop = CanonLoop->getLoopStmt()) {
2617 for (
const Stmt *SubStmt : SyntacticalLoop->
children()) {
2620 if (
const CompoundStmt *CS = dyn_cast<CompoundStmt>(SubStmt)) {
2624 if (isa<OMPOrderedDirective>(CSSubStmt)) {
2634static llvm::MapVector<llvm::Value *, llvm::Value *>
2636 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars;
2638 llvm::APInt ClauseAlignment(64, 0);
2639 if (
const Expr *AlignmentExpr = Clause->getAlignment()) {
2642 ClauseAlignment = AlignmentCI->getValue();
2644 for (
const Expr *E : Clause->varlists()) {
2645 llvm::APInt Alignment(ClauseAlignment);
2646 if (Alignment == 0) {
2653 E->getType()->getPointeeType()))
2656 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2657 "alignment is not power of 2");
2659 AlignedVars[PtrValue] = CGF.
Builder.getInt64(Alignment.getSExtValue());
2666 bool UseOMPIRBuilder =
2668 if (UseOMPIRBuilder) {
2669 auto &&CodeGenIRBuilder = [
this, &S, UseOMPIRBuilder](
CodeGenFunction &CGF,
2672 if (UseOMPIRBuilder) {
2673 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars =
2676 const Stmt *Inner = S.getRawStmt();
2677 llvm::CanonicalLoopInfo *CLI =
2680 llvm::OpenMPIRBuilder &OMPBuilder =
2683 llvm::ConstantInt *Simdlen =
nullptr;
2688 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
2691 llvm::ConstantInt *Safelen =
nullptr;
2696 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
2699 llvm::omp::OrderKind Order = llvm::omp::OrderKind::OMP_ORDER_unknown;
2701 if (
C->getKind() == OpenMPOrderClauseKind ::OMPC_ORDER_concurrent) {
2702 Order = llvm::omp::OrderKind::OMP_ORDER_concurrent;
2707 OMPBuilder.applySimd(CLI, AlignedVars,
2708 nullptr, Order, Simdlen, Safelen);
2715 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
2722 ParentLoopDirectiveForScanRegion ScanRegion(*
this, S);
2730 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
2739 OMPTransformDirectiveScopeRAII TileScope(*
this, &S);
2746 if (UseOMPIRBuilder) {
2748 const Stmt *Inner = S.getRawStmt();
2759 llvm::CanonicalLoopInfo *UnrolledCLI =
nullptr;
2763 OMPBuilder.unrollLoopFull(DL, CLI);
2766 if (
Expr *FactorExpr = PartialClause->getFactor()) {
2767 Factor = FactorExpr->EvaluateKnownConstInt(
getContext()).getZExtValue();
2768 assert(Factor >= 1 &&
"Only positive factors are valid");
2770 OMPBuilder.unrollLoopPartial(DL, CLI, Factor,
2771 NeedsUnrolledCLI ? &UnrolledCLI :
nullptr);
2773 OMPBuilder.unrollLoopHeuristic(DL, CLI);
2776 assert((!NeedsUnrolledCLI || UnrolledCLI) &&
2777 "NeedsUnrolledCLI implies UnrolledCLI to be set");
2794 if (
Expr *FactorExpr = PartialClause->getFactor()) {
2796 FactorExpr->EvaluateKnownConstInt(
getContext()).getZExtValue();
2797 assert(Factor >= 1 &&
"Only positive factors are valid");
2805void CodeGenFunction::EmitOMPOuterLoop(
2807 CodeGenFunction::OMPPrivateScope &LoopScope,
2808 const CodeGenFunction::OMPLoopArguments &LoopArgs,
2813 const Expr *IVExpr = S.getIterationVariable();
2827 llvm::Value *BoolCondVal =
nullptr;
2828 if (!DynamicOrOrdered) {
2839 RT.
emitForNext(*
this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
2840 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
2845 llvm::BasicBlock *ExitBlock =
LoopExit.getBlock();
2846 if (LoopScope.requiresCleanups())
2850 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
2851 if (ExitBlock !=
LoopExit.getBlock()) {
2859 if (DynamicOrOrdered)
2864 BreakContinueStack.push_back(BreakContinue(
LoopExit, Continue));
2872 CGF.LoopStack.setParallel(!IsMonotonic);
2873 if (const auto *C = S.getSingleClause<OMPOrderClause>())
2874 if (C->getKind() == OMPC_ORDER_concurrent)
2875 CGF.LoopStack.setParallel(true);
2877 CGF.EmitOMPSimdInit(S);
2880 [&S, &LoopArgs,
LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered,
2888 CGF.EmitOMPInnerLoop(
2889 S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
2891 CodeGenLoop(CGF, S, LoopExit);
2894 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
2899 BreakContinueStack.pop_back();
2900 if (!DynamicOrOrdered) {
2913 auto &&CodeGen = [DynamicOrOrdered, &S, &LoopArgs](
CodeGenFunction &CGF) {
2914 if (!DynamicOrOrdered)
2915 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2918 OMPCancelStack.emitExit(*
this, S.getDirectiveKind(), CodeGen);
2921void CodeGenFunction::EmitOMPForOuterLoop(
2924 const OMPLoopArguments &LoopArgs,
2932 LoopArgs.Chunk !=
nullptr)) &&
2933 "static non-chunked schedule does not need outer loop");
2985 const Expr *IVExpr = S.getIterationVariable();
2989 if (DynamicOrOrdered) {
2990 const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
2991 CGDispatchBounds(*
this, S, LoopArgs.LB, LoopArgs.UB);
2992 llvm::Value *LBVal = DispatchBounds.first;
2993 llvm::Value *UBVal = DispatchBounds.second;
2997 IVSigned, Ordered, DipatchRTInputValues);
3000 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
3001 LoopArgs.ST, LoopArgs.Chunk);
3003 ScheduleKind, StaticInit);
3007 const unsigned IVSize,
3008 const bool IVSigned) {
3015 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
3016 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
3017 OuterLoopArgs.IncExpr = S.getInc();
3018 OuterLoopArgs.Init = S.getInit();
3019 OuterLoopArgs.Cond = S.getCond();
3020 OuterLoopArgs.NextLB = S.getNextLowerBound();
3021 OuterLoopArgs.NextUB = S.getNextUpperBound();
3022 OuterLoopArgs.DKind = LoopArgs.DKind;
3023 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
3028 const unsigned IVSize,
const bool IVSigned) {}
3030void CodeGenFunction::EmitOMPDistributeOuterLoop(
3032 OMPPrivateScope &LoopScope,
const OMPLoopArguments &LoopArgs,
3042 const Expr *IVExpr = S.getIterationVariable();
3047 IVSize, IVSigned,
false, LoopArgs.IL, LoopArgs.LB,
3048 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
3055 IncExpr = S.getDistInc();
3057 IncExpr = S.getInc();
3062 OMPLoopArguments OuterLoopArgs;
3063 OuterLoopArgs.LB = LoopArgs.LB;
3064 OuterLoopArgs.UB = LoopArgs.UB;
3065 OuterLoopArgs.ST = LoopArgs.ST;
3066 OuterLoopArgs.IL = LoopArgs.IL;
3067 OuterLoopArgs.Chunk = LoopArgs.Chunk;
3069 ? S.getCombinedEnsureUpperBound()
3070 : S.getEnsureUpperBound();
3071 OuterLoopArgs.IncExpr = IncExpr;
3073 ? S.getCombinedInit()
3076 ? S.getCombinedCond()
3079 ? S.getCombinedNextLowerBound()
3080 : S.getNextLowerBound();
3082 ? S.getCombinedNextUpperBound()
3083 : S.getNextUpperBound();
3084 OuterLoopArgs.DKind = OMPD_distribute;
3086 EmitOMPOuterLoop(
false,
false, S,
3087 LoopScope, OuterLoopArgs, CodeGenLoopContent,
3091static std::pair<LValue, LValue>
3134static std::pair<llvm::Value *, llvm::Value *>
3145 llvm::Value *LBVal =
3147 llvm::Value *UBVal =
3149 return {LBVal, UBVal};
3155 const auto &Dir = cast<OMPLoopDirective>(S);
3157 CGF.
EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
3158 llvm::Value *LBCast =
3161 CapturedVars.push_back(LBCast);
3163 CGF.
EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
3165 llvm::Value *UBCast =
3168 CapturedVars.push_back(UBCast);
3174 CodeGenFunction::JumpDest
LoopExit) {
3178 bool HasCancel =
false;
3180 if (
const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
3181 HasCancel = D->hasCancel();
3182 else if (
const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
3183 HasCancel = D->hasCancel();
3184 else if (
const auto *D =
3185 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
3186 HasCancel = D->hasCancel();
3188 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
3198 CGInlinedWorksharingLoop,
3208 OMPLexicalScope
Scope(*
this, S, OMPD_parallel);
3218 OMPLexicalScope
Scope(*
this, S, OMPD_parallel);
3227 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
3238 llvm::Constant *Addr;
3241 S, ParentName, Fn, Addr,
true, CodeGen);
3242 assert(Fn && Addr &&
"Target device function emission failed.");
3254struct ScheduleKindModifiersTy {
3270 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3271 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3277 if (
const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3285 bool HasLastprivateClause;
3288 OMPLoopScope PreInitScope(*
this, S);
3293 llvm::BasicBlock *ContBlock =
nullptr;
3300 emitPreCond(*
this, S, S.getPreCond(), ThenBlock, ContBlock,
3306 RunCleanupsScope DoacrossCleanupScope(*
this);
3307 bool Ordered =
false;
3309 if (OrderedClause->getNumForLoops())
3320 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*
this, S);
3321 LValue LB = Bounds.first;
3322 LValue UB = Bounds.second;
3330 OMPPrivateScope LoopScope(*
this);
3336 *
this, S.getBeginLoc(), OMPD_unknown,
false,
3341 *
this, S,
EmitLValue(S.getIterationVariable()));
3346 (void)LoopScope.Privatize();
3351 const Expr *ChunkExpr =
nullptr;
3354 ScheduleKind.
Schedule =
C->getScheduleKind();
3355 ScheduleKind.
M1 =
C->getFirstScheduleModifier();
3356 ScheduleKind.
M2 =
C->getSecondScheduleModifier();
3357 ChunkExpr =
C->getChunkSize();
3361 *
this, S, ScheduleKind.
Schedule, ChunkExpr);
3363 bool HasChunkSizeOne =
false;
3364 llvm::Value *Chunk =
nullptr;
3368 S.getIterationVariable()->getType(),
3372 llvm::APSInt EvaluatedChunk =
Result.Val.getInt();
3373 HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
3382 bool StaticChunkedOne =
3384 Chunk !=
nullptr) &&
3389 (ScheduleKind.
Schedule == OMPC_SCHEDULE_static &&
3390 !(ScheduleKind.
M1 == OMPC_SCHEDULE_MODIFIER_nonmonotonic ||
3391 ScheduleKind.
M2 == OMPC_SCHEDULE_MODIFIER_nonmonotonic)) ||
3392 ScheduleKind.
M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
3393 ScheduleKind.
M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
3395 Chunk !=
nullptr) ||
3396 StaticChunkedOne) &&
3404 CGF.EmitOMPSimdInit(S);
3406 if (C->getKind() == OMPC_ORDER_concurrent)
3407 CGF.LoopStack.setParallel(true);
3410 [IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk,
3419 IVSize, IVSigned, Ordered, IL.getAddress(CGF),
3420 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF),
3421 StaticChunkedOne ? Chunk :
nullptr);
3422 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
3423 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind,
3426 if (!StaticChunkedOne)
3427 CGF.EmitIgnoredExpr(S.getEnsureUpperBound());
3429 CGF.EmitIgnoredExpr(S.getInit());
3443 CGF.EmitOMPInnerLoop(
3444 S, LoopScope.requiresCleanups(),
3445 StaticChunkedOne ? S.getCombinedParForInDistCond()
3447 StaticChunkedOne ? S.getDistInc() : S.getInc(),
3449 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit);
3456 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
3459 OMPCancelStack.emitExit(*
this, S.getDirectiveKind(), CodeGen);
3463 OMPLoopArguments LoopArguments(
3464 LB.getAddress(*
this), UB.getAddress(*
this), ST.getAddress(*
this),
3465 IL.getAddress(*
this), Chunk, EUB);
3466 LoopArguments.DKind = OMPD_for;
3467 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
3468 LoopArguments, CGDispatchBounds);
3472 return CGF.
Builder.CreateIsNotNull(
3478 ? OMPD_parallel_for_simd
3483 return CGF.
Builder.CreateIsNotNull(
3487 if (HasLastprivateClause)
3491 LoopScope.restoreMap();
3493 return CGF.
Builder.CreateIsNotNull(
3497 DoacrossCleanupScope.ForceCleanup();
3504 return HasLastprivateClause;
3510static std::pair<LValue, LValue>
3512 const auto &LS = cast<OMPLoopDirective>(S);
3524static std::pair<llvm::Value *, llvm::Value *>
3527 const auto &LS = cast<OMPLoopDirective>(S);
3528 const Expr *IVExpr = LS.getIterationVariable();
3530 llvm::Value *LBVal = CGF.
Builder.getIntN(IVSize, 0);
3532 return {LBVal, UBVal};
3544 llvm::function_ref<llvm::Value *(
CodeGenFunction &)> NumIteratorsGen) {
3545 llvm::Value *OMPScanNumIterations = CGF.
Builder.CreateIntCast(
3546 NumIteratorsGen(CGF), CGF.
SizeTy,
false);
3552 assert(
C->getModifier() == OMPC_REDUCTION_inscan &&
3553 "Only inscan reductions are expected.");
3554 Shareds.append(
C->varlist_begin(),
C->varlist_end());
3555 Privates.append(
C->privates().begin(),
C->privates().end());
3556 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
3557 CopyArrayTemps.append(
C->copy_array_temps().begin(),
3558 C->copy_array_temps().end());
3566 auto *ITA = CopyArrayTemps.begin();
3567 for (
const Expr *IRef : Privates) {
3568 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
3571 if (PrivateVD->getType()->isVariablyModifiedType()) {
3575 CodeGenFunction::OpaqueValueMapping DimMapping(
3577 cast<OpaqueValueExpr>(
3578 cast<VariableArrayType>((*ITA)->getType()->getAsArrayTypeUnsafe())
3582 CGF.
EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(*ITA)->getDecl()));
3596 llvm::function_ref<llvm::Value *(
CodeGenFunction &)> NumIteratorsGen) {
3597 llvm::Value *OMPScanNumIterations = CGF.
Builder.CreateIntCast(
3598 NumIteratorsGen(CGF), CGF.
SizeTy,
false);
3606 assert(
C->getModifier() == OMPC_REDUCTION_inscan &&
3607 "Only inscan reductions are expected.");
3608 Shareds.append(
C->varlist_begin(),
C->varlist_end());
3609 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
3610 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
3611 Privates.append(
C->privates().begin(),
C->privates().end());
3612 CopyOps.append(
C->copy_ops().begin(),
C->copy_ops().end());
3613 CopyArrayElems.append(
C->copy_array_elems().begin(),
3614 C->copy_array_elems().end());
3618 llvm::Value *OMPLast = CGF.
Builder.CreateNSWSub(
3619 OMPScanNumIterations,
3620 llvm::ConstantInt::get(CGF.
SizeTy, 1,
false));
3621 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
3622 const Expr *PrivateExpr = Privates[I];
3623 const Expr *OrigExpr = Shareds[I];
3624 const Expr *CopyArrayElem = CopyArrayElems[I];
3625 CodeGenFunction::OpaqueValueMapping IdxMapping(
3627 cast<OpaqueValueExpr>(
3628 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3634 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
3635 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
3664 llvm::Value *OMPScanNumIterations = CGF.
Builder.CreateIntCast(
3665 NumIteratorsGen(CGF), CGF.
SizeTy,
false);
3672 assert(
C->getModifier() == OMPC_REDUCTION_inscan &&
3673 "Only inscan reductions are expected.");
3674 Privates.append(
C->privates().begin(),
C->privates().end());
3675 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
3676 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
3677 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
3678 CopyArrayElems.append(
C->copy_array_elems().begin(),
3679 C->copy_array_elems().end());
3681 CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S);
3690 CodeGenFunction::OMPLocalDeclMapRAII
Scope(CGF);
3694 auto &&CodeGen = [&S, OMPScanNumIterations, &LHSs, &RHSs, &CopyArrayElems,
3701 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock();
3702 llvm::BasicBlock *LoopBB = CGF.createBasicBlock(
"omp.outer.log.scan.body");
3703 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
"omp.outer.log.scan.exit");
3705 CGF.CGM.getIntrinsic(llvm::Intrinsic::log2, CGF.DoubleTy);
3707 CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy);
3708 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg);
3709 F = CGF.CGM.getIntrinsic(llvm::Intrinsic::ceil, CGF.DoubleTy);
3710 LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal);
3711 LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy);
3712 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub(
3713 OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1));
3715 CGF.EmitBlock(LoopBB);
3716 auto *Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2);
3718 auto *Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3719 Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB);
3720 Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB);
3723 llvm::BasicBlock *InnerLoopBB =
3724 CGF.createBasicBlock(
"omp.inner.log.scan.body");
3725 llvm::BasicBlock *InnerExitBB =
3726 CGF.createBasicBlock(
"omp.inner.log.scan.exit");
3727 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K);
3728 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3729 CGF.EmitBlock(InnerLoopBB);
3730 auto *IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3731 IVal->addIncoming(NMin1, LoopBB);
3733 CodeGenFunction::OMPPrivateScope PrivScope(CGF);
3734 auto *ILHS = LHSs.begin();
3735 auto *IRHS = RHSs.begin();
3736 for (
const Expr *CopyArrayElem : CopyArrayElems) {
3737 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
3738 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
3741 CodeGenFunction::OpaqueValueMapping IdxMapping(
3743 cast<OpaqueValueExpr>(
3744 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3746 LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF);
3748 PrivScope.addPrivate(LHSVD, LHSAddr);
3751 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K);
3752 CodeGenFunction::OpaqueValueMapping IdxMapping(
3754 cast<OpaqueValueExpr>(
3755 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3757 RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF);
3759 PrivScope.addPrivate(RHSVD, RHSAddr);
3763 PrivScope.Privatize();
3764 CGF.CGM.getOpenMPRuntime().emitReduction(
3765 CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
3766 {true, true, OMPD_unknown});
3768 llvm::Value *NextIVal =
3769 CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1));
3770 IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock());
3771 CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K);
3772 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3773 CGF.EmitBlock(InnerExitBB);
3775 CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1));
3776 Counter->addIncoming(Next, CGF.Builder.GetInsertBlock());
3778 llvm::Value *NextPow2K =
3779 CGF.Builder.CreateShl(Pow2K, 1,
"",
true);
3780 Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock());
3781 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(Next, LogVal);
3782 CGF.Builder.CreateCondBr(Cmp, LoopBB, ExitBB);
3784 CGF.EmitBlock(ExitBB);
3787 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());
3788 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
3789 CGF, S.getBeginLoc(), OMPD_unknown,
false,
3796 CGF.OMPFirstScanLoop =
false;
3803 bool HasLastprivates;
3806 return C->getModifier() == OMPC_REDUCTION_inscan;
3809 CodeGenFunction::OMPLocalDeclMapRAII
Scope(CGF);
3810 OMPLoopScope LoopScope(CGF, S);
3814 CodeGenFunction::OMPCancelStackRAII CancelRegion(
3815 CGF, S.getDirectiveKind(), HasCancel);
3823 const auto &&SecondGen = [&S, HasCancel,
3825 CodeGenFunction::OMPCancelStackRAII CancelRegion(
3826 CGF, S.getDirectiveKind(), HasCancel);
3837 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
3843 return HasLastprivates;
3850 if (isa<OMPNowaitClause>(
C))
3853 if (
auto *SC = dyn_cast<OMPScheduleClause>(
C)) {
3858 switch (SC->getScheduleKind()) {
3859 case OMPC_SCHEDULE_auto:
3860 case OMPC_SCHEDULE_dynamic:
3861 case OMPC_SCHEDULE_runtime:
3862 case OMPC_SCHEDULE_guided:
3863 case OMPC_SCHEDULE_static:
3876static llvm::omp::ScheduleKind
3878 switch (ScheduleClauseKind) {
3880 return llvm::omp::OMP_SCHEDULE_Default;
3881 case OMPC_SCHEDULE_auto:
3882 return llvm::omp::OMP_SCHEDULE_Auto;
3883 case OMPC_SCHEDULE_dynamic:
3884 return llvm::omp::OMP_SCHEDULE_Dynamic;
3885 case OMPC_SCHEDULE_guided:
3886 return llvm::omp::OMP_SCHEDULE_Guided;
3887 case OMPC_SCHEDULE_runtime:
3888 return llvm::omp::OMP_SCHEDULE_Runtime;
3889 case OMPC_SCHEDULE_static:
3890 return llvm::omp::OMP_SCHEDULE_Static;
3892 llvm_unreachable(
"Unhandled schedule kind");
3896 bool HasLastprivates =
false;
3897 bool UseOMPIRBuilder =
3899 auto &&CodeGen = [
this, &S, &HasLastprivates,
3902 if (UseOMPIRBuilder) {
3905 llvm::omp::ScheduleKind SchedKind = llvm::omp::OMP_SCHEDULE_Default;
3906 llvm::Value *ChunkSize =
nullptr;
3910 if (
const Expr *ChunkSizeExpr = SchedClause->getChunkSize())
3915 const Stmt *Inner = S.getRawStmt();
3916 llvm::CanonicalLoopInfo *CLI =
3919 llvm::OpenMPIRBuilder &OMPBuilder =
3921 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
3923 OMPBuilder.applyWorkshareLoop(
3924 Builder.getCurrentDebugLocation(), CLI, AllocaIP, NeedsBarrier,
3925 SchedKind, ChunkSize,
false,
3936 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
3941 if (!UseOMPIRBuilder) {
3951 bool HasLastprivates =
false;
3959 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
3972 llvm::Value *
Init =
nullptr) {
3980 const Stmt *
CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
3982 bool HasLastprivates =
false;
3987 C.getIntTypeForBitwidth(32, 1);
3990 CGF.Builder.getInt32(0));
3991 llvm::ConstantInt *GlobalUBVal = CS !=
nullptr
3992 ? CGF.Builder.getInt32(CS->size() - 1)
3993 : CGF.Builder.getInt32(0);
3997 CGF.Builder.getInt32(1));
3999 CGF.Builder.getInt32(0));
4003 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
4005 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
4026 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
".omp.sections.exit");
4028 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
4029 ExitBB, CS ==
nullptr ? 1 : CS->size());
4031 unsigned CaseNumber = 0;
4033 auto CaseBB = CGF.createBasicBlock(
".omp.sections.case");
4034 CGF.EmitBlock(CaseBB);
4035 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
4036 CGF.EmitStmt(SubStmt);
4037 CGF.EmitBranch(ExitBB);
4041 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(
".omp.sections.case");
4042 CGF.EmitBlock(CaseBB);
4043 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
4045 CGF.EmitBranch(ExitBB);
4047 CGF.EmitBlock(ExitBB,
true);
4050 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
4051 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
4055 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
4056 CGF, S.getBeginLoc(), OMPD_unknown,
false,
4059 CGF.EmitOMPPrivateClause(S, LoopScope);
4061 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4062 CGF.EmitOMPReductionClauseInit(S, LoopScope);
4063 (void)LoopScope.Privatize();
4065 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4069 ScheduleKind.
Schedule = OMPC_SCHEDULE_static;
4073 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
4074 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit);
4076 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
4077 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
4078 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
4079 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
4081 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
4083 CGF.EmitOMPInnerLoop(S,
false, Cond, Inc, BodyGen,
4087 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
4090 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
4091 CGF.EmitOMPReductionClauseFinal(S, OMPD_parallel);
4094 return CGF.
Builder.CreateIsNotNull(
4099 if (HasLastprivates)
4106 bool HasCancel =
false;
4107 if (
auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
4108 HasCancel = OSD->hasCancel();
4109 else if (
auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
4110 HasCancel = OPSD->hasCancel();
4111 OMPCancelStackRAII CancelRegion(*
this, S.getDirectiveKind(), HasCancel);
4128 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4129 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4131 auto FiniCB = [
this](InsertPointTy IP) {
4135 const CapturedStmt *ICS = S.getInnermostCapturedStmt();
4136 const Stmt *
CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4141 auto SectionCB = [
this, SubStmt](InsertPointTy AllocaIP,
4142 InsertPointTy CodeGenIP) {
4144 *
this, SubStmt, AllocaIP, CodeGenIP,
"section");
4146 SectionCBVector.push_back(SectionCB);
4149 auto SectionCB = [
this,
CapturedStmt](InsertPointTy AllocaIP,
4150 InsertPointTy CodeGenIP) {
4154 SectionCBVector.push_back(SectionCB);
4161 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
4162 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
4170 CGCapturedStmtInfo CGSI(*ICS,
CR_OpenMP);
4171 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*
this, &CGSI);
4172 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
4174 Builder.restoreIP(OMPBuilder.createSections(
4175 Builder, AllocaIP, SectionCBVector, PrivCB, FiniCB, S.hasCancel(),
4182 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
4197 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4199 const Stmt *SectionRegionBodyStmt = S.getAssociatedStmt();
4200 auto FiniCB = [
this](InsertPointTy IP) {
4204 auto BodyGenCB = [SectionRegionBodyStmt,
this](InsertPointTy AllocaIP,
4205 InsertPointTy CodeGenIP) {
4207 *
this, SectionRegionBodyStmt, AllocaIP, CodeGenIP,
"section");
4210 LexicalScope
Scope(*
this, S.getSourceRange());
4212 Builder.restoreIP(OMPBuilder.createSection(
Builder, BodyGenCB, FiniCB));
4216 LexicalScope
Scope(*
this, S.getSourceRange());
4231 CopyprivateVars.append(
C->varlists().begin(),
C->varlists().end());
4232 DestExprs.append(
C->destination_exprs().begin(),
4233 C->destination_exprs().end());
4234 SrcExprs.append(
C->source_exprs().begin(),
C->source_exprs().end());
4235 AssignmentOps.append(
C->assignment_ops().begin(),
4236 C->assignment_ops().end());
4245 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4250 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
4252 CopyprivateVars, DestExprs,
4253 SrcExprs, AssignmentOps);
4257 if (!S.getSingleClause<
OMPNowaitClause>() && CopyprivateVars.empty()) {
4259 *
this, S.getBeginLoc(),
4277 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4279 const Stmt *MasterRegionBodyStmt = S.getAssociatedStmt();
4281 auto FiniCB = [
this](InsertPointTy IP) {
4285 auto BodyGenCB = [MasterRegionBodyStmt,
this](InsertPointTy AllocaIP,
4286 InsertPointTy CodeGenIP) {
4288 *
this, MasterRegionBodyStmt, AllocaIP, CodeGenIP,
"master");
4291 LexicalScope
Scope(*
this, S.getSourceRange());
4293 Builder.restoreIP(OMPBuilder.createMaster(
Builder, BodyGenCB, FiniCB));
4297 LexicalScope
Scope(*
this, S.getSourceRange());
4307 Expr *Filter =
nullptr;
4309 Filter = FilterClause->getThreadID();
4317 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4319 const Stmt *MaskedRegionBodyStmt = S.getAssociatedStmt();
4322 Filter = FilterClause->getThreadID();
4323 llvm::Value *FilterVal =
Filter
4327 auto FiniCB = [
this](InsertPointTy IP) {
4331 auto BodyGenCB = [MaskedRegionBodyStmt,
this](InsertPointTy AllocaIP,
4332 InsertPointTy CodeGenIP) {
4334 *
this, MaskedRegionBodyStmt, AllocaIP, CodeGenIP,
"masked");
4337 LexicalScope
Scope(*
this, S.getSourceRange());
4340 OMPBuilder.createMasked(
Builder, BodyGenCB, FiniCB, FilterVal));
4344 LexicalScope
Scope(*
this, S.getSourceRange());
4352 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4354 const Stmt *CriticalRegionBodyStmt = S.getAssociatedStmt();
4355 const Expr *Hint =
nullptr;
4356 if (
const auto *HintClause = S.getSingleClause<
OMPHintClause>())
4357 Hint = HintClause->getHint();
4362 llvm::Value *HintInst =
nullptr;
4367 auto FiniCB = [
this](InsertPointTy IP) {
4371 auto BodyGenCB = [CriticalRegionBodyStmt,
this](InsertPointTy AllocaIP,
4372 InsertPointTy CodeGenIP) {
4374 *
this, CriticalRegionBodyStmt, AllocaIP, CodeGenIP,
"critical");
4377 LexicalScope
Scope(*
this, S.getSourceRange());
4379 Builder.restoreIP(OMPBuilder.createCritical(
4380 Builder, BodyGenCB, FiniCB, S.getDirectiveName().getAsString(),
4388 CGF.
EmitStmt(S.getAssociatedStmt());
4390 const Expr *Hint =
nullptr;
4391 if (
const auto *HintClause = S.getSingleClause<
OMPHintClause>())
4392 Hint = HintClause->getHint();
4393 LexicalScope
Scope(*
this, S.getSourceRange());
4396 S.getDirectiveName().getAsString(),
4397 CodeGen, S.getBeginLoc(), Hint);
4411 CodeGenFunction::OMPLocalDeclMapRAII
Scope(CGF);
4413 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
4414 OMPLoopScope LoopScope(CGF, S);
4419 return C->getModifier() == OMPC_REDUCTION_inscan;
4445 CodeGenFunction::OMPLocalDeclMapRAII
Scope(CGF);
4447 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
4448 OMPLoopScope LoopScope(CGF, S);
4453 return C->getModifier() == OMPC_REDUCTION_inscan;
4474 OMPPrivateScope PrivateScope(CGF);
4479 (void)PrivateScope.Privatize();
4501 OMPPrivateScope PrivateScope(CGF);
4506 (void)PrivateScope.Privatize();
4529 CGF.EmitSections(S);
4543class CheckVarsEscapingUntiedTaskDeclContext final
4548 explicit CheckVarsEscapingUntiedTaskDeclContext() =
default;
4549 virtual ~CheckVarsEscapingUntiedTaskDeclContext() =
default;
4550 void VisitDeclStmt(
const DeclStmt *S) {
4554 for (
const Decl *D : S->decls()) {
4555 if (
const auto *VD = dyn_cast_or_null<VarDecl>(D))
4557 PrivateDecls.push_back(VD);
4563 void VisitBlockExpr(
const BlockExpr *) {}
4564 void VisitStmt(
const Stmt *S) {
4567 for (
const Stmt *Child : S->children())
4581 bool OmpAllMemory =
false;
4584 return C->getDependencyKind() == OMPC_DEPEND_outallmemory ||
4585 C->getDependencyKind() == OMPC_DEPEND_inoutallmemory;
4587 OmpAllMemory =
true;
4592 Data.Dependences.emplace_back(OMPC_DEPEND_outallmemory,
4601 if (Kind == OMPC_DEPEND_outallmemory || Kind == OMPC_DEPEND_inoutallmemory)
4603 if (OmpAllMemory && (Kind == OMPC_DEPEND_out || Kind == OMPC_DEPEND_inout))
4606 Data.Dependences.emplace_back(
C->getDependencyKind(),
C->getModifier());
4607 DD.
DepExprs.append(
C->varlist_begin(),
C->varlist_end());
4616 const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
4618 auto PartId = std::next(I);
4619 auto TaskT = std::next(I, 4);
4624 const Expr *Cond = Clause->getCondition();
4627 Data.Final.setInt(CondConstant);
4632 Data.Final.setInt(
false);
4636 const Expr *Prio = Clause->getPriority();
4637 Data.Priority.setInt(
true);
4648 auto IRef =
C->varlist_begin();
4649 for (
const Expr *IInit :
C->private_copies()) {
4650 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4651 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4652 Data.PrivateVars.push_back(*IRef);
4653 Data.PrivateCopies.push_back(IInit);
4658 EmittedAsPrivate.clear();
4661 auto IRef =
C->varlist_begin();
4662 auto IElemInitRef =
C->inits().begin();
4663 for (
const Expr *IInit :
C->private_copies()) {
4664 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4665 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4666 Data.FirstprivateVars.push_back(*IRef);
4667 Data.FirstprivateCopies.push_back(IInit);
4668 Data.FirstprivateInits.push_back(*IElemInitRef);
4675 llvm::MapVector<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
4677 auto IRef =
C->varlist_begin();
4678 auto ID =
C->destination_exprs().begin();
4679 for (
const Expr *IInit :
C->private_copies()) {
4680 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4681 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4682 Data.LastprivateVars.push_back(*IRef);
4683 Data.LastprivateCopies.push_back(IInit);
4685 LastprivateDstsOrigs.insert(
4686 std::make_pair(cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
4687 cast<DeclRefExpr>(*IRef)));
4695 Data.ReductionVars.append(
C->varlist_begin(),
C->varlist_end());
4696 Data.ReductionOrigs.append(
C->varlist_begin(),
C->varlist_end());
4697 Data.ReductionCopies.append(
C->privates().begin(),
C->privates().end());
4698 Data.ReductionOps.append(
C->reduction_ops().begin(),
4699 C->reduction_ops().end());
4700 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
4701 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
4704 *
this, S.getBeginLoc(), LHSs, RHSs,
Data);
4709 CheckVarsEscapingUntiedTaskDeclContext Checker;
4710 Checker.Visit(S.getInnermostCapturedStmt()->getCapturedStmt());
4711 Data.PrivateLocals.append(Checker.getPrivateDecls().begin(),
4712 Checker.getPrivateDecls().end());
4714 auto &&CodeGen = [&
Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
4717 llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
4718 std::pair<Address, Address>>
4721 OMPPrivateScope
Scope(CGF);
4723 if (
auto *DI = CGF.getDebugInfo()) {
4724 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields =
4725 CGF.CapturedStmtInfo->getCaptureFields();
4726 llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue();
4727 if (CaptureFields.size() && ContextValue) {
4728 unsigned CharWidth = CGF.getContext().getCharWidth();
4742 for (
auto It = CaptureFields.begin(); It != CaptureFields.end(); ++It) {
4743 const VarDecl *SharedVar = It->first;
4746 CGF.getContext().getASTRecordLayout(CaptureRecord);
4749 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
4750 (void)DI->EmitDeclareOfAutoVariable(SharedVar, ContextValue,
4751 CGF.Builder,
false);
4752 llvm::Instruction &
Last = CGF.Builder.GetInsertBlock()->back();
4755 if (
auto DDI = dyn_cast<llvm::DbgVariableIntrinsic>(&
Last)) {
4759 Ops.push_back(llvm::dwarf::DW_OP_plus_uconst);
4760 Ops.push_back(Offset);
4762 Ops.push_back(llvm::dwarf::DW_OP_deref);
4763 auto &Ctx = DDI->getContext();
4764 llvm::DIExpression *DIExpr = llvm::DIExpression::get(Ctx, Ops);
4765 Last.setOperand(2, llvm::MetadataAsValue::get(Ctx, DIExpr));
4771 if (!
Data.PrivateVars.empty() || !
Data.FirstprivateVars.empty() ||
4772 !
Data.LastprivateVars.empty() || !
Data.PrivateLocals.empty()) {
4773 enum { PrivatesParam = 2, CopyFnParam = 3 };
4774 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
4775 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
4776 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
4777 CS->getCapturedDecl()->getParam(PrivatesParam)));
4782 CallArgs.push_back(PrivatesPtr);
4783 ParamTypes.push_back(PrivatesPtr->getType());
4784 for (
const Expr *E :
Data.PrivateVars) {
4785 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4786 Address PrivatePtr = CGF.CreateMemTemp(
4787 CGF.getContext().getPointerType(E->
getType()),
".priv.ptr.addr");
4788 PrivatePtrs.emplace_back(VD, PrivatePtr);
4790 ParamTypes.push_back(PrivatePtr.
getType());
4792 for (
const Expr *E :
Data.FirstprivateVars) {
4793 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4795 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->
getType()),
4796 ".firstpriv.ptr.addr");
4797 PrivatePtrs.emplace_back(VD, PrivatePtr);
4798 FirstprivatePtrs.emplace_back(VD, PrivatePtr);
4800 ParamTypes.push_back(PrivatePtr.
getType());
4802 for (
const Expr *E :
Data.LastprivateVars) {
4803 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4805 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->
getType()),
4806 ".lastpriv.ptr.addr");
4807 PrivatePtrs.emplace_back(VD, PrivatePtr);
4809 ParamTypes.push_back(PrivatePtr.
getType());
4814 Ty = CGF.getContext().getPointerType(Ty);
4816 Ty = CGF.getContext().getPointerType(Ty);
4817 Address PrivatePtr = CGF.CreateMemTemp(
4818 CGF.getContext().getPointerType(Ty),
".local.ptr.addr");
4819 auto Result = UntiedLocalVars.insert(
4822 if (
Result.second ==
false)
4823 *
Result.first = std::make_pair(
4826 ParamTypes.push_back(PrivatePtr.
getType());
4828 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
4830 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
4831 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
4832 for (
const auto &Pair : LastprivateDstsOrigs) {
4833 const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
4836 CGF.CapturedStmtInfo->lookup(OrigVD) !=
nullptr,
4838 Pair.second->getExprLoc());
4839 Scope.addPrivate(Pair.first, CGF.EmitLValue(&DRE).getAddress(CGF));
4841 for (
const auto &Pair : PrivatePtrs) {
4843 CGF.Builder.CreateLoad(Pair.second),
4844 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
4845 CGF.getContext().getDeclAlign(Pair.first));
4846 Scope.addPrivate(Pair.first, Replacement);
4847 if (
auto *DI = CGF.getDebugInfo())
4848 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
4849 (void)DI->EmitDeclareOfAutoVariable(
4850 Pair.first, Pair.second.getPointer(), CGF.Builder,
4855 for (
auto &Pair : UntiedLocalVars) {
4856 QualType VDType = Pair.first->getType().getNonReferenceType();
4857 if (Pair.first->getType()->isLValueReferenceType())
4858 VDType = CGF.getContext().getPointerType(VDType);
4860 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
4863 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(VDType)),
4864 CGF.getPointerAlign());
4865 Pair.second.first = Replacement;
4866 Ptr = CGF.Builder.CreateLoad(Replacement);
4867 Replacement =
Address(Ptr, CGF.ConvertTypeForMem(VDType),
4868 CGF.getContext().getDeclAlign(Pair.first));
4869 Pair.second.second = Replacement;
4871 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
4872 Address Replacement(Ptr, CGF.ConvertTypeForMem(VDType),
4873 CGF.getContext().getDeclAlign(Pair.first));
4874 Pair.second.first = Replacement;
4878 if (
Data.Reductions) {
4879 OMPPrivateScope FirstprivateScope(CGF);
4880 for (
const auto &Pair : FirstprivatePtrs) {
4882 CGF.Builder.CreateLoad(Pair.second),
4883 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
4884 CGF.getContext().getDeclAlign(Pair.first));
4885 FirstprivateScope.addPrivate(Pair.first, Replacement);
4887 (void)FirstprivateScope.Privatize();
4888 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
4890 Data.ReductionCopies,
Data.ReductionOps);
4891 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
4892 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
4893 for (
unsigned Cnt = 0, E =
Data.ReductionVars.size(); Cnt < E; ++Cnt) {
4894 RedCG.emitSharedOrigLValue(CGF, Cnt);
4895 RedCG.emitAggregateType(CGF, Cnt);
4899 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
4901 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
4902 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
4904 Address(CGF.EmitScalarConversion(
4905 Replacement.getPointer(), CGF.getContext().VoidPtrTy,
4906 CGF.getContext().getPointerType(
4907 Data.ReductionCopies[Cnt]->getType()),
4908 Data.ReductionCopies[Cnt]->getExprLoc()),
4909 CGF.ConvertTypeForMem(
Data.ReductionCopies[Cnt]->getType()),
4910 Replacement.getAlignment());
4911 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
4912 Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
4916 (void)
Scope.Privatize();
4922 auto IPriv =
C->privates().begin();
4923 auto IRed =
C->reduction_ops().begin();
4924 auto ITD =
C->taskgroup_descriptors().begin();
4925 for (
const Expr *Ref :
C->varlists()) {
4926 InRedVars.emplace_back(Ref);
4927 InRedPrivs.emplace_back(*IPriv);
4928 InRedOps.emplace_back(*IRed);
4929 TaskgroupDescriptors.emplace_back(*ITD);
4930 std::advance(IPriv, 1);
4931 std::advance(IRed, 1);
4932 std::advance(ITD, 1);
4937 OMPPrivateScope InRedScope(CGF);
4938 if (!InRedVars.empty()) {
4940 for (
unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
4941 RedCG.emitSharedOrigLValue(CGF, Cnt);
4942 RedCG.emitAggregateType(CGF, Cnt);
4948 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
4950 llvm::Value *ReductionsPtr;
4951 if (
const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
4952 ReductionsPtr = CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr),
4953 TRExpr->getExprLoc());
4955 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4957 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
4958 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
4960 CGF.EmitScalarConversion(
4961 Replacement.getPointer(), CGF.getContext().VoidPtrTy,
4962 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
4963 InRedPrivs[Cnt]->getExprLoc()),
4964 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),
4965 Replacement.getAlignment());
4966 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
4967 InRedScope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
4970 (void)InRedScope.Privatize();
4978 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen,
Data.Tied,
4979 Data.NumberOfParts);
4980 OMPLexicalScope
Scope(*
this, S, std::nullopt,
4983 TaskGen(*
this, OutlinedFn,
Data);
5000 QualType ElemType =
C.getBaseElementType(Ty);
5010 Data.FirstprivateVars.emplace_back(OrigRef);
5011 Data.FirstprivateCopies.emplace_back(PrivateRef);
5012 Data.FirstprivateInits.emplace_back(InitRef);
5018 OMPTargetDataInfo &InputInfo) {
5024 auto PartId = std::next(I);
5025 auto TaskT = std::next(I, 4);
5028 Data.Final.setInt(
false);
5031 auto IRef =
C->varlist_begin();
5032 auto IElemInitRef =
C->inits().begin();
5033 for (
auto *IInit :
C->private_copies()) {
5034 Data.FirstprivateVars.push_back(*IRef);
5035 Data.FirstprivateCopies.push_back(IInit);
5036 Data.FirstprivateInits.push_back(*IElemInitRef);
5044 Data.ReductionVars.append(
C->varlist_begin(),
C->varlist_end());
5045 Data.ReductionOrigs.append(
C->varlist_begin(),
C->varlist_end());
5046 Data.ReductionCopies.append(
C->privates().begin(),
C->privates().end());
5047 Data.ReductionOps.append(
C->reduction_ops().begin(),
5048 C->reduction_ops().end());
5049 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
5050 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
5052 OMPPrivateScope TargetScope(*
this);
5057 if (InputInfo.NumberOfTargetItems > 0) {
5060 llvm::APInt ArrSize(32, InputInfo.NumberOfTargetItems);
5065 getContext(),
Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5067 getContext(),
Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5074 TargetScope.addPrivate(BPVD, InputInfo.BasePointersArray);
5075 TargetScope.addPrivate(PVD, InputInfo.PointersArray);
5076 TargetScope.addPrivate(SVD, InputInfo.SizesArray);
5079 if (!isa_and_nonnull<llvm::ConstantPointerNull>(
5080 InputInfo.MappersArray.getPointer())) {
5082 getContext(),
Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5083 TargetScope.addPrivate(MVD, InputInfo.MappersArray);
5086 (void)TargetScope.Privatize();
5088 auto &&CodeGen = [&
Data, &S, CS, &BodyGen, BPVD, PVD, SVD, MVD,
5091 OMPPrivateScope
Scope(CGF);
5092 if (!
Data.FirstprivateVars.empty()) {
5093 enum { PrivatesParam = 2, CopyFnParam = 3 };
5094 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
5095 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
5096 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
5097 CS->getCapturedDecl()->getParam(PrivatesParam)));
5102 CallArgs.push_back(PrivatesPtr);
5103 ParamTypes.push_back(PrivatesPtr->getType());
5104 for (
const Expr *E :
Data.FirstprivateVars) {
5105 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5107 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->
getType()),
5108 ".firstpriv.ptr.addr");
5109 PrivatePtrs.emplace_back(VD, PrivatePtr);
5111 ParamTypes.push_back(PrivatePtr.
getType());
5113 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
5115 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
5116 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
5117 for (
const auto &Pair : PrivatePtrs) {
5119 CGF.Builder.CreateLoad(Pair.second),
5120 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
5121 CGF.getContext().getDeclAlign(Pair.first));
5122 Scope.addPrivate(Pair.first, Replacement);
5125 CGF.processInReduction(S,
Data, CGF, CS,
Scope);
5126 if (InputInfo.NumberOfTargetItems > 0) {
5127 InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
5128 CGF.GetAddrOfLocalVar(BPVD), 0);
5129 InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
5130 CGF.GetAddrOfLocalVar(PVD), 0);
5131 InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
5132 CGF.GetAddrOfLocalVar(SVD), 0);
5135 InputInfo.MappersArray = CGF.Builder.CreateConstArrayGEP(
5136 CGF.GetAddrOfLocalVar(MVD), 0);
5140 OMPLexicalScope LexScope(CGF, S, OMPD_task,
false);
5142 if (CGF.CGM.getLangOpts().OpenMP >= 51 &&
5147 CGF.CGM.getOpenMPRuntime().emitThreadLimitClause(
5148 CGF, TL->getThreadLimit(), S.getBeginLoc());
5153 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen,
true,
5154 Data.NumberOfParts);
5155 llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<
OMPNowaitClause>() ? 1 : 0);
5160 SharedsTy, CapturedStruct, &IfCond,
Data);
5167 OMPPrivateScope &
Scope) {
5168 if (
Data.Reductions) {
5170 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
5172 Data.ReductionCopies,
Data.ReductionOps);
5175 for (
unsigned Cnt = 0, E =
Data.ReductionVars.size(); Cnt < E; ++Cnt) {
5176 RedCG.emitSharedOrigLValue(CGF, Cnt);
5177 RedCG.emitAggregateType(CGF, Cnt);
5184 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
5189 Data.ReductionCopies[Cnt]->getType()),
5190 Data.ReductionCopies[Cnt]->getExprLoc()),
5192 Replacement.getAlignment());
5193 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
5194 Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
5197 (void)
Scope.Privatize();
5203 auto IPriv =
C->privates().begin();
5204 auto IRed =
C->reduction_ops().begin();
5205 auto ITD =
C->taskgroup_descriptors().begin();
5206 for (
const Expr *Ref :
C->varlists()) {
5207 InRedVars.emplace_back(Ref);
5208 InRedPrivs.emplace_back(*IPriv);
5209 InRedOps.emplace_back(*IRed);
5210 TaskgroupDescriptors.emplace_back(*ITD);
5211 std::advance(IPriv, 1);
5212 std::advance(IRed, 1);
5213 std::advance(ITD, 1);
5216 OMPPrivateScope InRedScope(CGF);
5217 if (!InRedVars.empty()) {
5219 for (
unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
5220 RedCG.emitSharedOrigLValue(CGF, Cnt);
5221 RedCG.emitAggregateType(CGF, Cnt);
5227 llvm::Value *ReductionsPtr;
5228 if (
const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
5232 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
5235 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
5240 InRedPrivs[Cnt]->getExprLoc()),
5242 Replacement.getAlignment());
5243 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
5244 InRedScope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
5247 (void)InRedScope.Privatize();
5255 const Expr *IfCond =
nullptr;
5256 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
5257 if (
C->getNameModifier() == OMPD_unknown ||
5258 C->getNameModifier() == OMPD_task) {
5259 IfCond =
C->getCondition();
5270 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
5273 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn,
5274 SharedsTy, CapturedStruct, IfCond,
5291 bool IsFatal =
false;
5315 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
5318 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5322 auto BodyGenCB = [&,
this](InsertPointTy AllocaIP,
5323 InsertPointTy CodeGenIP) {
5325 EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
5327 CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;
5330 Builder.restoreIP(OMPBuilder.createTaskgroup(
Builder, AllocaIP, BodyGenCB));
5335 if (
const Expr *E = S.getReductionRef()) {
5340 Data.ReductionVars.append(
C->varlist_begin(),
C->varlist_end());
5341 Data.ReductionOrigs.append(
C->varlist_begin(),
C->varlist_end());
5342 Data.ReductionCopies.append(
C->privates().begin(),
C->privates().end());
5343 Data.ReductionOps.append(
C->reduction_ops().begin(),
5344 C->reduction_ops().end());
5345 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
5346 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
5348 llvm::Value *ReductionDesc =
5351 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5356 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
5363 ? llvm::AtomicOrdering::NotAtomic
5364 : llvm::AtomicOrdering::AcquireRelease;
5368 if (
const auto *FlushClause = S.getSingleClause<
OMPFlushClause>())
5370 FlushClause->varlist_end());
5371 return std::nullopt;
5373 S.getBeginLoc(), AO);
5382 Dependencies.DepExprs.append(DC->varlist_begin(), DC->varlist_end());
5384 *
this, Dependencies, DC->getBeginLoc());
5394 *
this, DOLVal, UC->getDependencyKind(), UC->getBeginLoc());
5413 if (
C->getModifier() != OMPC_REDUCTION_inscan)
5415 Shareds.append(
C->varlist_begin(),
C->varlist_end());
5416 Privates.append(
C->privates().begin(),
C->privates().end());
5417 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
5418 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
5419 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
5420 CopyOps.append(
C->copy_ops().begin(),
C->copy_ops().end());
5421 CopyArrayTemps.append(
C->copy_array_temps().begin(),
5422 C->copy_array_temps().end());
5423 CopyArrayElems.append(
C->copy_array_elems().begin(),
5424 C->copy_array_elems().end());
5468 : BreakContinueStack.back().ContinueBlock.getBlock());
5473 LexicalScope
Scope(*
this, S.getSourceRange());
5479 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5480 const Expr *PrivateExpr = Privates[I];
5481 const Expr *TempExpr = CopyArrayTemps[I];
5483 *cast<VarDecl>(cast<DeclRefExpr>(TempExpr)->getDecl()));
5488 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5489 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
5494 *
this, ParentDir.
getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
5495 {true, true, OMPD_simd});
5496 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5497 const Expr *PrivateExpr = Privates[I];
5504 const Expr *TempExpr = CopyArrayTemps[I];
5510 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5511 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
5517 ? BreakContinueStack.back().ContinueBlock.getBlock()
5523 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
5528 const auto *IVExpr = cast<OMPLoopDirective>(ParentDir)
5529 .getIterationVariable()
5534 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5535 const Expr *PrivateExpr = Privates[I];
5536 const Expr *OrigExpr = Shareds[I];
5537 const Expr *CopyArrayElem = CopyArrayElems[I];
5538 OpaqueValueMapping IdxMapping(
5540 cast<OpaqueValueExpr>(
5541 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
5547 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5548 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
5552 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
5555 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
5560 const auto *IVExpr = cast<OMPLoopDirective>(ParentDir)
5561 .getIterationVariable()
5566 llvm::BasicBlock *ExclusiveExitBB =
nullptr;
5570 llvm::Value *Cmp =
Builder.CreateIsNull(IdxVal);
5571 Builder.CreateCondBr(Cmp, ExclusiveExitBB, ContBB);
5574 IdxVal =
Builder.CreateNUWSub(IdxVal, llvm::ConstantInt::get(
SizeTy, 1));
5576 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5577 const Expr *PrivateExpr = Privates[I];
5578 const Expr *OrigExpr = Shareds[I];
5579 const Expr *CopyArrayElem = CopyArrayElems[I];
5580 OpaqueValueMapping IdxMapping(
5582 cast<OpaqueValueExpr>(
5583 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
5589 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5590 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
5606 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
5607 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
5613 if (
const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
5621 bool HasLastprivateClause =
false;
5624 OMPLoopScope PreInitScope(*
this, S);
5629 llvm::BasicBlock *ContBlock =
nullptr;
5636 emitPreCond(*
this, S, S.getPreCond(), ThenBlock, ContBlock,
5648 *
this, cast<DeclRefExpr>(
5650 ? S.getCombinedLowerBoundVariable()
5651 : S.getLowerBoundVariable())));
5653 *
this, cast<DeclRefExpr>(
5655 ? S.getCombinedUpperBoundVariable()
5656 : S.getUpperBoundVariable())));
5662 OMPPrivateScope LoopScope(*
this);
5668 *
this, S.getBeginLoc(), OMPD_unknown,
false,
5678 (void)LoopScope.Privatize();
5683 llvm::Value *Chunk =
nullptr;
5686 ScheduleKind =
C->getDistScheduleKind();
5687 if (
const Expr *Ch =
C->getChunkSize()) {
5690 S.getIterationVariable()->getType(),
5696 *
this, S, ScheduleKind, Chunk);
5709 bool StaticChunked =
5713 Chunk !=
nullptr) ||
5716 IVSize, IVSigned,
false, IL.
getAddress(*
this),
5718 StaticChunked ? Chunk :
nullptr);
5725 ? S.getCombinedEnsureUpperBound()
5726 : S.getEnsureUpperBound());
5729 ? S.getCombinedInit()
5734 ? S.getCombinedCond()
5738 Cond = S.getCombinedDistCond();
5770 [&S, &LoopScope, Cond, IncExpr,
LoopExit, &CodeGenLoop,
5772 CGF.EmitOMPInnerLoop(
5773 S, LoopScope.requiresCleanups(), Cond, IncExpr,
5775 CodeGenLoop(CGF, S, LoopExit);
5778 if (StaticChunked) {
5779 CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());
5780 CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());
5781 CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());
5782 CGF.EmitIgnoredExpr(S.getCombinedInit());
5792 const OMPLoopArguments LoopArguments = {
5795 EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
5800 return CGF.
Builder.CreateIsNotNull(
5811 return CGF.
Builder.CreateIsNotNull(
5816 if (HasLastprivateClause) {
5836 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
5844 CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;
5847 Fn->setDoesNotRecurse();
5851template <
typename T>
5853 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP,
5854 llvm::OpenMPIRBuilder &OMPBuilder) {
5856 unsigned NumLoops =
C->getNumLoops();
5860 for (
unsigned I = 0; I < NumLoops; I++) {
5861 const Expr *CounterVal =
C->getLoopData(I);
5866 StoreValues.emplace_back(StoreValue);
5868 OMPDoacrossKind<T> ODK;
5869 bool IsDependSource = ODK.isSource(
C);
5871 OMPBuilder.createOrderedDepend(CGF.
Builder, AllocaIP, NumLoops,
5872 StoreValues,
".cnt.addr", IsDependSource));
5878 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5883 assert(!S.hasAssociatedStmt() &&
"No associated statement must be in "
5884 "ordered depend|doacross construct.");
5896 auto FiniCB = [
this](InsertPointTy IP) {
5900 auto BodyGenCB = [&S,
C,
this](InsertPointTy AllocaIP,
5901 InsertPointTy CodeGenIP) {
5906 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
5907 Builder,
false,
".ordered.after");
5910 llvm::Function *OutlinedFn =
5912 assert(S.getBeginLoc().isValid() &&
5913 "Outlined function call location must be valid.");
5916 OutlinedFn, CapturedVars);
5923 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
5925 OMPBuilder.createOrderedThreadsSimd(
Builder, BodyGenCB, FiniCB, !
C));
5931 assert(!S.hasAssociatedStmt() &&
5932 "No associated statement must be in ordered depend construct.");
5938 assert(!S.hasAssociatedStmt() &&
5939 "No associated statement must be in ordered doacross construct.");
5950 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
5951 llvm::Function *OutlinedFn =
5954 OutlinedFn, CapturedVars);
5960 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
5968 "DestType must have scalar evaluation kind.");
5969 assert(!Val.
isAggregate() &&
"Must be a scalar or complex.");
5980 "DestType must have complex evaluation kind.");
5989 ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
5991 assert(Val.
isComplex() &&
"Must be a scalar or complex.");
5996 Val.
getComplexVal().first, SrcElementType, DestElementType, Loc);
5998 Val.
getComplexVal().second, SrcElementType, DestElementType, Loc);
6012 llvm::AtomicOrdering AO,
LValue LVal,
6017 LVal, Loc, llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO),
6026 *
this, RVal, RValTy, LVal.
getType(), Loc)),
6035 llvm_unreachable(
"Must be a scalar or complex.");
6043 assert(
V->isLValue() &&
"V of 'omp atomic read' is not lvalue");
6044 assert(
X->isLValue() &&
"X of 'omp atomic read' is not lvalue");
6053 case llvm::AtomicOrdering::Acquire:
6054 case llvm::AtomicOrdering::AcquireRelease:
6055 case llvm::AtomicOrdering::SequentiallyConsistent:
6057 llvm::AtomicOrdering::Acquire);
6059 case llvm::AtomicOrdering::Monotonic:
6060 case llvm::AtomicOrdering::Release:
6062 case llvm::AtomicOrdering::NotAtomic:
6063 case llvm::AtomicOrdering::Unordered:
6064 llvm_unreachable(
"Unexpected ordering.");
6071 llvm::AtomicOrdering AO,
const Expr *
X,
6074 assert(
X->isLValue() &&
"X of 'omp atomic write' is not lvalue");
6082 case llvm::AtomicOrdering::Release:
6083 case llvm::AtomicOrdering::AcquireRelease:
6084 case llvm::AtomicOrdering::SequentiallyConsistent:
6086 llvm::AtomicOrdering::Release);
6088 case llvm::AtomicOrdering::Acquire:
6089 case llvm::AtomicOrdering::Monotonic:
6091 case llvm::AtomicOrdering::NotAtomic:
6092 case llvm::AtomicOrdering::Unordered:
6093 llvm_unreachable(
"Unexpected ordering.");
6100 llvm::AtomicOrdering AO,
6101 bool IsXLHSInRHSPart) {
6106 if (BO == BO_Comma || !
Update.isScalar() || !
X.isSimple() ||
6107 (!isa<llvm::ConstantInt>(
Update.getScalarVal()) &&
6108 (
Update.getScalarVal()->getType() !=
6109 X.getAddress(CGF).getElementType())) ||
6112 return std::make_pair(
false,
RValue::get(
nullptr));
6115 if (T->isIntegerTy())
6118 if (T->isFloatingPointTy() && (BO == BO_Add || BO == BO_Sub))
6124 if (!CheckAtomicSupport(
Update.getScalarVal()->getType(), BO) ||
6125 !CheckAtomicSupport(
X.getAddress(CGF).getElementType(), BO))
6126 return std::make_pair(
false,
RValue::get(
nullptr));
6128 bool IsInteger =
X.getAddress(CGF).getElementType()->isIntegerTy();
6129 llvm::AtomicRMWInst::BinOp RMWOp;
6132 RMWOp = IsInteger ? llvm::AtomicRMWInst::Add : llvm::AtomicRMWInst::FAdd;
6135 if (!IsXLHSInRHSPart)
6136 return std::make_pair(
false,
RValue::get(
nullptr));
6137 RMWOp = IsInteger ? llvm::AtomicRMWInst::Sub : llvm::AtomicRMWInst::FSub;
6140 RMWOp = llvm::AtomicRMWInst::And;
6143 RMWOp = llvm::AtomicRMWInst::Or;
6146 RMWOp = llvm::AtomicRMWInst::Xor;
6150 RMWOp =
X.getType()->hasSignedIntegerRepresentation()
6151 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
6152 : llvm::AtomicRMWInst::Max)
6153 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
6154 : llvm::AtomicRMWInst::UMax);
6156 RMWOp = IsXLHSInRHSPart ? llvm::AtomicRMWInst::FMin
6157 : llvm::AtomicRMWInst::FMax;
6161 RMWOp =
X.getType()->hasSignedIntegerRepresentation()
6162 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
6163 : llvm::AtomicRMWInst::Min)
6164 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
6165 : llvm::AtomicRMWInst::UMin);
6167 RMWOp = IsXLHSInRHSPart ? llvm::AtomicRMWInst::FMax
6168 : llvm::AtomicRMWInst::FMin;
6171 RMWOp = llvm::AtomicRMWInst::Xchg;
6180 return std::make_pair(
false,
RValue::get(
nullptr));
6199 llvm_unreachable(
"Unsupported atomic update operation");
6201 llvm::Value *UpdateVal =
Update.getScalarVal();
6202 if (
auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
6204 UpdateVal = CGF.
Builder.CreateIntCast(
6205 IC,
X.getAddress(CGF).getElementType(),
6206 X.getType()->hasSignedIntegerRepresentation());
6208 UpdateVal = CGF.
Builder.CreateCast(llvm::Instruction::CastOps::UIToFP, IC,
6209 X.getAddress(CGF).getElementType());
6228 if (
X.isGlobalReg()) {
6241 llvm::AtomicOrdering AO,
const Expr *
X,
6245 "Update expr in 'atomic update' must be a binary operator.");
6253 assert(
X->isLValue() &&
"X of 'omp atomic update' is not lvalue");
6256 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
6257 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
6260 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](
RValue XRValue) {
6261 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
6262 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
6266 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
6273 case llvm::AtomicOrdering::Release:
6274 case llvm::AtomicOrdering::AcquireRelease:
6275 case llvm::AtomicOrdering::SequentiallyConsistent:
6277 llvm::AtomicOrdering::Release);
6279 case llvm::AtomicOrdering::Acquire:
6280 case llvm::AtomicOrdering::Monotonic:
6282 case llvm::AtomicOrdering::NotAtomic:
6283 case llvm::AtomicOrdering::Unordered:
6284 llvm_unreachable(
"Unexpected ordering.");
6302 llvm_unreachable(
"Must be a scalar or complex.");
6306 llvm::AtomicOrdering AO,
6307 bool IsPostfixUpdate,
const Expr *
V,
6309 const Expr *UE,
bool IsXLHSInRHSPart,
6311 assert(
X->isLValue() &&
"X of 'omp atomic capture' is not lvalue");
6312 assert(
V->isLValue() &&
"V of 'omp atomic capture' is not lvalue");
6321 "Update expr in 'atomic capture' must be a binary operator.");
6329 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
6330 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
6332 NewVValType = XRValExpr->
getType();
6334 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
6335 IsPostfixUpdate](
RValue XRValue) {
6336 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
6337 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
6339 NewVVal = IsPostfixUpdate ? XRValue : Res;
6343 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
6347 if (IsPostfixUpdate) {
6349 NewVVal = Res.second;
6353 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
6354 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
6360 NewVValType =
X->getType().getNonReferenceType();
6362 X->getType().getNonReferenceType(), Loc);
6363 auto &&Gen = [&NewVVal, ExprRValue](
RValue XRValue) {
6369 XLValue, ExprRValue, BO_Assign,
false, AO,
6374 NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
6390 case llvm::AtomicOrdering::Release:
6392 llvm::AtomicOrdering::Release);
6394 case llvm::AtomicOrdering::Acquire:
6396 llvm::AtomicOrdering::Acquire);
6398 case llvm::AtomicOrdering::AcquireRelease:
6399 case llvm::AtomicOrdering::SequentiallyConsistent:
6401 CGF, std::nullopt, Loc, llvm::AtomicOrdering::AcquireRelease);
6403 case llvm::AtomicOrdering::Monotonic:
6405 case llvm::AtomicOrdering::NotAtomic:
6406 case llvm::AtomicOrdering::Unordered:
6407 llvm_unreachable(
"Unexpected ordering.");
6413 CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO,
6415 const Expr *CE,
bool IsXBinopExpr,
bool IsPostfixUpdate,
bool IsFailOnly,
6417 llvm::OpenMPIRBuilder &OMPBuilder =
6420 OMPAtomicCompareOp Op;
6421 assert(isa<BinaryOperator>(CE) &&
"CE is not a BinaryOperator");
6422 switch (cast<BinaryOperator>(CE)->getOpcode()) {
6424 Op = OMPAtomicCompareOp::EQ;
6427 Op = OMPAtomicCompareOp::MIN;
6430 Op = OMPAtomicCompareOp::MAX;
6433 llvm_unreachable(
"unsupported atomic compare binary operator");
6439 auto EmitRValueWithCastIfNeeded = [&CGF, Loc](
const Expr *
X,
const Expr *E) {
6444 if (NewE->
getType() ==
X->getType())
6449 llvm::Value *EVal = EmitRValueWithCastIfNeeded(
X, E);
6450 llvm::Value *DVal = D ? EmitRValueWithCastIfNeeded(
X, D) :
nullptr;
6451 if (
auto *CI = dyn_cast<llvm::ConstantInt>(EVal))
6452 EVal = CGF.
Builder.CreateIntCast(
6456 if (
auto *CI = dyn_cast<llvm::ConstantInt>(DVal))
6457 DVal = CGF.
Builder.CreateIntCast(
6461 llvm::OpenMPIRBuilder::AtomicOpValue XOpVal{
6463 X->getType()->hasSignedIntegerRepresentation(),
6464 X->getType().isVolatileQualified()};
6465 llvm::OpenMPIRBuilder::AtomicOpValue VOpVal, ROpVal;
6470 V->getType()->hasSignedIntegerRepresentation(),
6471 V->getType().isVolatileQualified()};
6481 if (FailAO == llvm::AtomicOrdering::NotAtomic) {
6484 CGF.
Builder.restoreIP(OMPBuilder.createAtomicCompare(
6485 CGF.
Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
6486 IsPostfixUpdate, IsFailOnly));
6488 CGF.
Builder.restoreIP(OMPBuilder.createAtomicCompare(
6489 CGF.
Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
6490 IsPostfixUpdate, IsFailOnly, FailAO));
6494 llvm::AtomicOrdering AO,
6495 llvm::AtomicOrdering FailAO,
bool IsPostfixUpdate,
6498 const Expr *CE,
bool IsXLHSInRHSPart,
6513 IsXLHSInRHSPart, Loc);
6515 case OMPC_compare: {
6517 IsXLHSInRHSPart, IsPostfixUpdate, IsFailOnly, Loc);
6521 llvm_unreachable(
"Clause is not allowed in 'omp atomic'.");
6526 llvm::AtomicOrdering AO = llvm::AtomicOrdering::Monotonic;
6528 llvm::AtomicOrdering FailAO = llvm::AtomicOrdering::NotAtomic;
6529 bool MemOrderingSpecified =
false;
6531 AO = llvm::AtomicOrdering::SequentiallyConsistent;
6532 MemOrderingSpecified =
true;
6534 AO = llvm::AtomicOrdering::AcquireRelease;
6535 MemOrderingSpecified =
true;
6537 AO = llvm::AtomicOrdering::Acquire;
6538 MemOrderingSpecified =
true;
6540 AO = llvm::AtomicOrdering::Release;
6541 MemOrderingSpecified =
true;
6543 AO = llvm::AtomicOrdering::Monotonic;
6544 MemOrderingSpecified =
true;
6546 llvm::SmallSet<OpenMPClauseKind, 2> KindsEncountered;
6555 if (K == OMPC_seq_cst || K == OMPC_acq_rel || K == OMPC_acquire ||
6556 K == OMPC_release || K == OMPC_relaxed || K == OMPC_hint)
6559 KindsEncountered.insert(K);
6564 if (KindsEncountered.contains(OMPC_compare) &&
6565 KindsEncountered.contains(OMPC_capture))
6566 Kind = OMPC_compare;
6567 if (!MemOrderingSpecified) {
6568 llvm::AtomicOrdering DefaultOrder =
6570 if (DefaultOrder == llvm::AtomicOrdering::Monotonic ||
6571 DefaultOrder == llvm::AtomicOrdering::SequentiallyConsistent ||
6572 (DefaultOrder == llvm::AtomicOrdering::AcquireRelease &&
6573 Kind == OMPC_capture)) {
6575 }
else if (DefaultOrder == llvm::AtomicOrdering::AcquireRelease) {
6576 if (Kind == OMPC_unknown || Kind == OMPC_update || Kind == OMPC_write) {
6577 AO = llvm::AtomicOrdering::Release;
6578 }
else if (Kind == OMPC_read) {
6579 assert(Kind == OMPC_read &&
"Unexpected atomic kind.");
6580 AO = llvm::AtomicOrdering::Acquire;
6585 if (KindsEncountered.contains(OMPC_compare) &&
6586 KindsEncountered.contains(OMPC_fail)) {
6587 Kind = OMPC_compare;
6588 const auto *FailClause = S.getSingleClause<
OMPFailClause>();
6591 if (FailParameter == llvm::omp::OMPC_relaxed)
6592 FailAO = llvm::AtomicOrdering::Monotonic;
6593 else if (FailParameter == llvm::omp::OMPC_acquire)
6594 FailAO = llvm::AtomicOrdering::Acquire;
6595 else if (FailParameter == llvm::omp::OMPC_seq_cst)
6596 FailAO = llvm::AtomicOrdering::SequentiallyConsistent;
6600 LexicalScope
Scope(*
this, S.getSourceRange());
6603 S.getV(), S.getR(), S.getExpr(), S.getUpdateExpr(),
6604 S.getD(), S.getCondExpr(), S.isXLHSInRHSPart(),
6605 S.isFailOnly(), S.getBeginLoc());
6616 OMPLexicalScope
Scope(CGF, S, OMPD_target);
6619 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
6625 llvm::Function *Fn =
nullptr;
6626 llvm::Constant *FnID =
nullptr;
6628 const Expr *IfCond =
nullptr;
6630 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
6631 if (
C->getNameModifier() == OMPD_unknown ||
6632 C->getNameModifier() == OMPD_target) {
6633 IfCond =
C->getCondition();
6639 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device(
6642 Device.setPointerAndInt(
C->getDevice(),
C->getModifier());
6647 bool IsOffloadEntry =
true;
6651 IsOffloadEntry =
false;
6654 IsOffloadEntry =
false;
6656 if (
CGM.
getLangOpts().OpenMPOffloadMandatory && !IsOffloadEntry) {
6659 "No offloading entry generated while offloading is mandatory.");
6663 assert(CGF.
CurFuncDecl &&
"No parent declaration for target region!");
6664 StringRef ParentName;
6667 if (
const auto *D = dyn_cast<CXXConstructorDecl>(CGF.
CurFuncDecl))
6669 else if (
const auto *D = dyn_cast<CXXDestructorDecl>(CGF.
CurFuncDecl))
6677 IsOffloadEntry, CodeGen);
6678 OMPLexicalScope
Scope(CGF, S, OMPD_task);
6679 auto &&SizeEmitter =
6682 if (IsOffloadEntry) {
6683 OMPLoopScope(CGF, D);
6686 NumIterations = CGF.
Builder.CreateIntCast(NumIterations, CGF.
Int64Ty,
6688 return NumIterations;
6699 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
6702 (void)PrivateScope.Privatize();
6706 CGF.
EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
6711 StringRef ParentName,
6717 llvm::Constant *Addr;
6720 S, ParentName, Fn, Addr,
true, CodeGen);
6721 assert(Fn && Addr &&
"Target device function emission failed.");
6735 const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
6736 llvm::Function *OutlinedFn =
6744 const Expr *NumTeams = NT ? NT->getNumTeams() :
nullptr;
6745 const Expr *ThreadLimit = TL ? TL->getThreadLimit() :
nullptr;
6751 OMPTeamsScope
Scope(CGF, S);
6762 OMPPrivateScope PrivateScope(CGF);
6766 (void)PrivateScope.Privatize();
6767 CGF.
EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
6777 auto *CS = S.getCapturedStmt(OMPD_teams);
6782 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
6786 (void)PrivateScope.Privatize();
6804 llvm::Constant *Addr;
6807 S, ParentName, Fn, Addr,
true, CodeGen);
6808 assert(Fn && Addr &&
"Target device function emission failed.");
6831 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
6833 (void)PrivateScope.Privatize();
6850 llvm::Constant *Addr;
6853 S, ParentName, Fn, Addr,
true, CodeGen);
6854 assert(Fn && Addr &&
"Target device function emission failed.");
6877 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
6879 (void)PrivateScope.Privatize();
6896 llvm::Constant *Addr;
6899 S, ParentName, Fn, Addr,
true, CodeGen);
6900 assert(Fn && Addr &&
"Target device function emission failed.");
6922 OMPPrivateScope PrivateScope(CGF);
6923 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
6924 (void)PrivateScope.Privatize();
6925 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
6927 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
6944 OMPPrivateScope PrivateScope(CGF);
6945 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
6946 (void)PrivateScope.Privatize();
6947 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
6949 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
6967 OMPPrivateScope PrivateScope(CGF);
6968 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
6969 (void)PrivateScope.Privatize();
6970 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
6972 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
6990 OMPPrivateScope PrivateScope(CGF);
6991 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
6992 (void)PrivateScope.Privatize();
6993 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
6994 CGF, OMPD_distribute, CodeGenDistribute,
false);
6995 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
7005 llvm::Value *
Device =
nullptr;
7006 llvm::Value *NumDependences =
nullptr;
7007 llvm::Value *DependenceList =
nullptr;
7015 if (!
Data.Dependences.empty()) {
7017 std::tie(NumDependences, DependenciesArray) =
7020 DependenceList = DependenciesArray.
getPointer();
7027 "OMPNowaitClause clause is used separately in OMPInteropDirective.");
7030 if (!ItOMPInitClause.empty()) {
7033 llvm::Value *InteropvarPtr =
7035 llvm::omp::OMPInteropType InteropType =
7036 llvm::omp::OMPInteropType::Unknown;
7037 if (
C->getIsTarget()) {
7038 InteropType = llvm::omp::OMPInteropType::Target;
7040 assert(
C->getIsTargetSync() &&
7041 "Expected interop-type target/targetsync");
7042 InteropType = llvm::omp::OMPInteropType::TargetSync;
7044 OMPBuilder.createOMPInteropInit(
Builder, InteropvarPtr, InteropType,
7045 Device, NumDependences, DependenceList,
7046 Data.HasNowaitClause);
7050 if (!ItOMPDestroyClause.empty()) {
7053 llvm::Value *InteropvarPtr =
7055 OMPBuilder.createOMPInteropDestroy(
Builder, InteropvarPtr,
Device,
7056 NumDependences, DependenceList,
7057 Data.HasNowaitClause);
7060 auto ItOMPUseClause = S.getClausesOfKind<
OMPUseClause>();
7061 if (!ItOMPUseClause.empty()) {
7064 llvm::Value *InteropvarPtr =
7066 OMPBuilder.createOMPInteropUse(
Builder, InteropvarPtr,
Device,
7067 NumDependences, DependenceList,
7068 Data.HasNowaitClause);
7086 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7088 (void)PrivateScope.Privatize();
7090 CGF, OMPD_distribute, CodeGenDistribute,
false);
7109 llvm::Constant *Addr;
7112 S, ParentName, Fn, Addr,
true, CodeGen);
7113 assert(Fn && Addr &&
"Target device function emission failed.");
7138 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7140 (void)PrivateScope.Privatize();
7142 CGF, OMPD_distribute, CodeGenDistribute,
false);
7161 llvm::Constant *Addr;
7164 S, ParentName, Fn, Addr,
true, CodeGen);
7165 assert(Fn && Addr &&
"Target device function emission failed.");
7179 S.getCancelRegion());
7183 const Expr *IfCond =
nullptr;
7184 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
7185 if (
C->getNameModifier() == OMPD_unknown ||
7186 C->getNameModifier() == OMPD_cancel) {
7187 IfCond =
C->getCondition();
7195 if (S.getCancelRegion() == OMPD_parallel ||
7196 S.getCancelRegion() == OMPD_sections ||
7197 S.getCancelRegion() == OMPD_section) {
7198 llvm::Value *IfCondition =
nullptr;
7203 OMPBuilder.createCancel(
Builder, IfCondition, S.getCancelRegion()));
7208 S.getCancelRegion());
7211CodeGenFunction::JumpDest
7213 if (Kind == OMPD_parallel || Kind == OMPD_task ||
7214 Kind == OMPD_target_parallel || Kind == OMPD_taskloop ||
7215 Kind == OMPD_master_taskloop || Kind == OMPD_parallel_master_taskloop)
7217 assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
7218 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
7219 Kind == OMPD_distribute_parallel_for ||
7220 Kind == OMPD_target_parallel_for ||
7221 Kind == OMPD_teams_distribute_parallel_for ||
7222 Kind == OMPD_target_teams_distribute_parallel_for);
7223 return OMPCancelStack.getExitBlock();
7228 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
7229 CaptureDeviceAddrMap) {
7230 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
7231 for (
const Expr *OrigVarIt :
C.varlists()) {
7232 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(OrigVarIt)->getDecl());
7233 if (!Processed.insert(OrigVD).second)
7240 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
7243 const auto *ME = cast<MemberExpr>(OED->getInit());
7244 assert(isa<CXXThisExpr>(ME->getBase()->IgnoreImpCasts()) &&
7245 "Base should be the current struct!");
7246 MatchingVD = ME->getMemberDecl();
7251 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
7252 if (InitAddrIt == CaptureDeviceAddrMap.end())
7258 bool IsRegistered = PrivateScope.addPrivate(
7260 Address(InitAddrIt->second, Ty,
7262 assert(IsRegistered &&
"firstprivate var already registered as private");
7270 while (
const auto *OASE = dyn_cast<OMPArraySectionExpr>(
Base))
7271 Base = OASE->getBase()->IgnoreParenImpCasts();
7272 while (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(
Base))
7273 Base = ASE->getBase()->IgnoreParenImpCasts();
7274 return cast<VarDecl>(cast<DeclRefExpr>(
Base)->getDecl());
7279 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
7280 CaptureDeviceAddrMap) {
7281 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
7282 for (
const Expr *Ref :
C.varlists()) {
7284 if (!Processed.insert(OrigVD).second)
7290 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
7293 const auto *ME = cast<MemberExpr>(OED->getInit());
7294 assert(isa<CXXThisExpr>(ME->getBase()) &&
7295 "Base should be the current struct!");
7296 MatchingVD = ME->getMemberDecl();
7301 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
7302 if (InitAddrIt == CaptureDeviceAddrMap.end())
7308 Address(InitAddrIt->second, Ty,
7312 if (isa<DeclRefExpr>(Ref->IgnoreParenImpCasts()) ||
7321 (void)PrivateScope.addPrivate(OrigVD, PrivAddr);
7334 bool PrivatizeDevicePointers =
false;
7336 bool &PrivatizeDevicePointers;
7339 explicit DevicePointerPrivActionTy(
bool &PrivatizeDevicePointers)
7340 : PrivatizeDevicePointers(PrivatizeDevicePointers) {}
7342 PrivatizeDevicePointers =
true;
7345 DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
7349 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
7355 PrivatizeDevicePointers =
false;
7361 if (PrivatizeDevicePointers) {
7362 OMPPrivateScope PrivateScope(CGF);
7366 Info.CaptureDeviceAddrMap);
7369 Info.CaptureDeviceAddrMap);
7370 (void)PrivateScope.Privatize();
7375 std::optional<OpenMPDirectiveKind> CaptureRegion;
7379 for (
const Expr *E :
C->varlists()) {
7380 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
7381 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
7385 for (
const Expr *E :
C->varlists()) {
7387 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
7391 CaptureRegion = OMPD_unknown;
7394 OMPLexicalScope
Scope(CGF, S, CaptureRegion);
7401 PrivRCG.setAction(Action);
7406 OMPLexicalScope
Scope(CGF, S);
7421 const Expr *IfCond =
nullptr;
7423 IfCond =
C->getCondition();
7431 RCG.setAction(PrivAction);
7446 const Expr *IfCond =
nullptr;
7448 IfCond =
C->getCondition();
7455 OMPLexicalScope
Scope(*
this, S, OMPD_task);
7467 const Expr *IfCond =
nullptr;
7469 IfCond =
C->getCondition();
7476 OMPLexicalScope
Scope(*
this, S, OMPD_task);
7484 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
7488 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7492 (void)PrivateScope.Privatize();
7512 llvm::Constant *Addr;
7515 S, ParentName, Fn, Addr,
true, CodeGen);
7516 assert(Fn && Addr &&
"Target device function emission failed.");
7535 CodeGenFunction::OMPCancelStackRAII CancelRegion(
7536 CGF, OMPD_target_parallel_for, S.hasCancel());
7552 llvm::Constant *Addr;
7555 S, ParentName, Fn, Addr,
true, CodeGen);
7556 assert(Fn && Addr &&
"Target device function emission failed.");
7591 llvm::Constant *Addr;
7594 S, ParentName, Fn, Addr,
true, CodeGen);
7595 assert(Fn && Addr &&
"Target device function emission failed.");
7609 CodeGenFunction::OMPPrivateScope &Privates) {
7610 const auto *VDecl = cast<VarDecl>(Helper->
getDecl());
7617 const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);
7620 OMPLexicalScope
Scope(*
this, S, OMPD_taskloop,
false);
7624 const Expr *IfCond =
nullptr;
7625 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
7626 if (
C->getNameModifier() == OMPD_unknown ||
7627 C->getNameModifier() == OMPD_taskloop) {
7628 IfCond =
C->getCondition();
7641 Data.Schedule.setInt(
false);
7645 Data.Schedule.setInt(
true);
7660 llvm::BasicBlock *ContBlock =
nullptr;
7661 OMPLoopScope PreInitScope(CGF, S);
7662 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
7666 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock(
"taskloop.if.then");
7667 ContBlock = CGF.createBasicBlock(
"taskloop.if.end");
7668 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
7669 CGF.getProfileCount(&S));
7670 CGF.EmitBlock(ThenBlock);
7671 CGF.incrementProfileCounter(&S);
7674 (void)CGF.EmitOMPLinearClauseInit(S);
7676 OMPPrivateScope LoopScope(CGF);
7678 enum { LowerBound = 5, UpperBound, Stride, LastIter };
7680 auto *LBP = std::next(I, LowerBound);
7681 auto *UBP = std::next(I, UpperBound);
7682 auto *STP = std::next(I, Stride);
7683 auto *LIP = std::next(I, LastIter);
7684 mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
7686 mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
7688 mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
7689 mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
7691 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
7692 CGF.EmitOMPLinearClause(S, LoopScope);
7693 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
7694 (void)LoopScope.Privatize();
7696 const Expr *IVExpr = S.getIterationVariable();
7697 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
7698 CGF.EmitVarDecl(*IVDecl);
7699 CGF.EmitIgnoredExpr(S.getInit());
7704 if (
const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
7705 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
7707 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
7711 OMPLexicalScope
Scope(CGF, S, OMPD_taskloop,
false);
7719 CGF.EmitOMPInnerLoop(
7720 S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),
7722 emitOMPLoopBodyWithStopPoint(CGF, S,
7723 CodeGenFunction::JumpDest());
7730 CGF.EmitBranch(ContBlock);
7731 CGF.EmitBlock(ContBlock,
true);
7734 if (HasLastprivateClause) {
7735 CGF.EmitOMPLastprivateClauseFinal(
7737 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
7738 CGF.GetAddrOfLocalVar(*LIP),
false,
7739 (*LIP)->getType(), S.getBeginLoc())));
7741 LoopScope.restoreMap();
7743 return CGF.
Builder.CreateIsNotNull(
7745 (*LIP)->
getType(), S.getBeginLoc()));
7748 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
7751 auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
7753 OMPLoopScope PreInitScope(CGF, S);
7754 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
7755 OutlinedFn, SharedsTy,
7756 CapturedStruct, IfCond,
Data);
7758 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
7786 OMPLexicalScope
Scope(*
this, S);
7798 OMPLexicalScope
Scope(*
this, S, std::nullopt,
false);
7810 OMPLexicalScope
Scope(*
this, S);
7822 OMPLexicalScope
Scope(CGF, S, OMPD_parallel,
false);
7840 OMPLexicalScope
Scope(CGF, S, OMPD_parallel,
false);
7859 const Expr *IfCond =
nullptr;
7861 IfCond =
C->getCondition();
7868 OMPLexicalScope
Scope(*
this, S, OMPD_task);
7879 const auto *ForS = dyn_cast<ForStmt>(CS);
7880 if (ForS && !isa<DeclStmt>(ForS->getInit())) {
7881 OMPPrivateScope LoopScope(CGF);
7883 (void)LoopScope.Privatize();
7885 LoopScope.restoreMap();
7890 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
7927 OMPPrivateScope PrivateScope(CGF);
7928 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7929 (void)PrivateScope.Privatize();
7930 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
7932 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
7955 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7957 (void)PrivateScope.Privatize();
7959 CGF, OMPD_distribute, CodeGenDistribute,
false);
7987 llvm::Constant *Addr;
7990 S, ParentName, Fn, Addr,
true, CodeGen);
7991 assert(Fn && Addr &&
7992 "Target device function emission failed for 'target teams loop'.");
8002 CodeGenFunction::OMPCancelStackRAII CancelRegion(
8003 CGF, OMPD_target_parallel_loop,
false);
8019 llvm::Constant *Addr;
8022 S, ParentName, Fn, Addr,
true, CodeGen);
8023 assert(Fn && Addr &&
"Target device function emission failed.");
8038 if (
const auto *SD = dyn_cast<OMPScanDirective>(&D)) {
8045 OMPPrivateScope GlobalsScope(CGF);
8049 for (
const Expr *Ref :
C->varlists()) {
8050 const auto *DRE = cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
8053 const auto *VD = dyn_cast<VarDecl>(DRE->getDecl());
8056 if (!CGF.LocalDeclMap.count(VD)) {
8058 GlobalsScope.addPrivate(VD, GlobLVal.
getAddress(CGF));
8064 (void)GlobalsScope.Privatize();
8065 ParentLoopDirectiveForScanRegion ScanRegion(CGF, D);
8068 if (
const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
8069 for (
const Expr *E : LD->counters()) {
8070 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
8073 GlobalsScope.addPrivate(VD, GlobLVal.
getAddress(CGF));
8075 if (isa<OMPCapturedExprDecl>(VD)) {
8077 if (!CGF.LocalDeclMap.count(VD))
8082 if (!
C->getNumForLoops())
8084 for (
unsigned I = LD->getLoopsNumber(),
8085 E =
C->getLoopNumIterations().size();
8087 if (
const auto *VD = dyn_cast<OMPCapturedExprDecl>(
8088 cast<DeclRefExpr>(
C->getLoopCounter(I))->getDecl())) {
8090 if (!CGF.LocalDeclMap.count(VD))
8096 (void)GlobalsScope.Privatize();
8110 OMPSimdLexicalScope
Scope(*
this, D);
Defines the clang::ASTContext interface.
static bool isAllocatableDecl(const VarDecl *VD)
static const VarDecl * getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE)
static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, PrePostActionTy &Action)
static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, PrePostActionTy &Action)
static llvm::Function * emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S, SourceLocation Loc)
static const VarDecl * getBaseDecl(const Expr *Ref)
static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *V, SourceLocation Loc)
static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, bool IsPostfixUpdate, const Expr *V, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
static void emitScanBasedDirective(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen, llvm::function_ref< void(CodeGenFunction &)> FirstGen, llvm::function_ref< void(CodeGenFunction &)> SecondGen)
Emits the code for the directive with inscan reductions.
static bool isSupportedByOpenMPIRBuilder(const OMPSimdDirective &S)
static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO, LValue LVal, RValue RVal)
static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, QualType DstType, StringRef Name, LValue AddrLV)
static void emitDistributeParallelForDistributeInnerBoundParams(CodeGenFunction &CGF, const OMPExecutableDirective &S, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars)
static void emitScanBasedDirectiveFinals(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen)
Copies final inscan reductions values to the original variables.
static void checkForLastprivateConditionalUpdate(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static std::pair< LValue, LValue > emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
The following two functions generate expressions for the loop lower and upper bounds in case of stati...
static void emitTargetParallelForRegion(CodeGenFunction &CGF, const OMPTargetParallelForDirective &S, PrePostActionTy &Action)
static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper)
Emit a helper variable and return corresponding lvalue.
static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
static llvm::Value * convertToScalarValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
static std::pair< bool, RValue > emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, RValue Update, BinaryOperatorKind BO, llvm::AtomicOrdering AO, bool IsXLHSInRHSPart)
static std::pair< LValue, LValue > emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitTargetParallelRegion(CodeGenFunction &CGF, const OMPTargetParallelDirective &S, PrePostActionTy &Action)
static std::pair< llvm::Value *, llvm::Value * > emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
When dealing with dispatch schedules (e.g.
static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitRestoreIP(CodeGenFunction &CGF, const T *C, llvm::OpenMPIRBuilder::InsertPointTy AllocaIP, llvm::OpenMPIRBuilder &OMPBuilder)
static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, const RegionCodeGenTy &CodeGen)
static void emitSimdlenSafelenClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
static void emitAlignedClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters)
static bool emitWorksharingDirective(CodeGenFunction &CGF, const OMPLoopDirective &S, bool HasCancel)
static void emitTargetTeamsGenericLoopRegion(CodeGenFunction &CGF, const OMPTargetTeamsGenericLoopDirective &S, PrePostActionTy &Action)
static void emitPostUpdateForReductionClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc, const unsigned IVSize, const bool IVSigned)
static RValue emitSimpleAtomicLoad(CodeGenFunction &CGF, llvm::AtomicOrdering AO, LValue LVal, SourceLocation Loc)
static std::pair< llvm::Value *, llvm::Value * > emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
if the 'for' loop has a dispatch schedule (e.g.
static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *UE, const Expr *D, const Expr *CE, bool IsXLHSInRHSPart, bool IsFailOnly, SourceLocation Loc)
static CodeGenFunction::ComplexPairTy convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static ImplicitParamDecl * createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data, QualType Ty, CapturedDecl *CD, SourceLocation Loc)
static EmittedClosureTy emitCapturedStmtFunc(CodeGenFunction &ParentCGF, const CapturedStmt *S)
Emit a captured statement and return the function as well as its captured closure context.
static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
static llvm::MapVector< llvm::Value *, llvm::Value * > GetAlignedMapping(const OMPSimdDirective &S, CodeGenFunction &CGF)
static void emitOMPCopyinClause(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitTargetTeamsDistributeParallelForRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, PrePostActionTy &Action)
static llvm::CallInst * emitCapturedStmtCall(CodeGenFunction &ParentCGF, EmittedClosureTy Cap, llvm::ArrayRef< llvm::Value * > Args)
Emit a call to a previously captured closure.
static void emitMasked(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop, int MaxLevel, int Level=0)
static void emitEmptyBoundParameters(CodeGenFunction &, const OMPExecutableDirective &, llvm::SmallVectorImpl< llvm::Value * > &)
static void emitTargetParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetParallelForSimdDirective &S, PrePostActionTy &Action)
static void emitOMPAtomicCompareExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO, const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *D, const Expr *CE, bool IsXBinopExpr, bool IsPostfixUpdate, bool IsFailOnly, SourceLocation Loc)
std::pair< llvm::Function *, llvm::Value * > EmittedClosureTy
static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDirective &S)
static void buildDependences(const OMPExecutableDirective &S, OMPTaskDataTy &Data)
static RValue convertToType(CodeGenFunction &CGF, RValue Value, QualType SourceType, QualType ResType, SourceLocation Loc)
static void emitScanBasedDirectiveDecls(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen)
Emits internal temp array declarations for the directive with inscan reductions.
static void emitTargetTeamsDistributeParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForSimdDirective &S, PrePostActionTy &Action)
static void emitTargetTeamsDistributeSimdRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeSimdDirective &S)
static llvm::omp::ScheduleKind convertClauseKindToSchedKind(OpenMPScheduleClauseKind ScheduleClauseKind)
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
static void emitTargetParallelGenericLoopRegion(CodeGenFunction &CGF, const OMPTargetParallelGenericLoopDirective &S, PrePostActionTy &Action)
static QualType getCanonicalParamType(ASTContext &C, QualType T)
static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S, const RegionCodeGenTy &SimdInitGen, const RegionCodeGenTy &BodyCodeGen)
static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, const Twine &Name, llvm::Value *Init=nullptr)
static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *E, SourceLocation Loc)
static llvm::Function * emitOutlinedFunctionPrologue(CodeGenFunction &CGF, FunctionArgList &Args, llvm::MapVector< const Decl *, std::pair< const VarDecl *, Address > > &LocalAddrs, llvm::DenseMap< const Decl *, std::pair< const Expr *, llvm::Value * > > &VLASizes, llvm::Value *&CXXThisValue, const FunctionOptions &FO)
static void emitInnerParallelForWhenCombined(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
static void emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeDirective &S)
This file defines OpenMP nodes for declarative directives.
static const Decl * getCanonicalDecl(const Decl *D)
This file defines OpenMP AST classes for clauses.
Defines some OpenMP-specific enums and functions.
Defines the PrettyStackTraceEntry class, which is used to make crashes give more contextual informati...
This file defines OpenMP AST classes for executable directives and clauses.
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
SourceManager & getSourceManager()
TranslationUnitDecl * getTranslationUnitDecl() const
QualType getRecordType(const RecordDecl *Decl) const
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
QualType getUIntPtrType() const
Return a type compatible with "uintptr_t" (C99 7.18.1.4), as defined by the target.
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth,...
TypeSourceInfo * getTrivialTypeSourceInfo(QualType T, SourceLocation Loc=SourceLocation()) const
Allocate a TypeSourceInfo where all locations have been initialized to a given location,...
unsigned getOpenMPDefaultSimdAlign(QualType T) const
Get default simd alignment of the specified complete type in bits.
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
QualType getFunctionType(QualType ResultTy, ArrayRef< QualType > Args, const FunctionProtoType::ExtProtoInfo &EPI) const
Return a normal function type with a typed argument list.
const TargetInfo & getTargetInfo() const
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
ASTRecordLayout - This class contains layout information for one RecordDecl, which is a struct/union/...
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Represents an attribute applied to a statement.
ArrayRef< const Attr * > getAttrs() const
A builtin binary operation expression such as "x + y" or "x <= y".
static BinaryOperator * Create(const ASTContext &C, Expr *lhs, Expr *rhs, Opcode opc, QualType ResTy, ExprValueKind VK, ExprObjectKind OK, SourceLocation opLoc, FPOptionsOverride FPFeatures)
BlockExpr - Adaptor class for mixing a BlockDecl with expressions.
Represents the body of a CapturedStmt, and serves as its DeclContext.
unsigned getContextParamPosition() const
static CapturedDecl * Create(ASTContext &C, DeclContext *DC, unsigned NumParams)
param_iterator param_end() const
Retrieve an iterator one past the last parameter decl.
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
Stmt * getBody() const override
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
ImplicitParamDecl * getParam(unsigned i) const
This captures a statement into a function.
CapturedDecl * getCapturedDecl()
Retrieve the outlined function declaration.
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Stmt * getCapturedStmt()
Retrieve the statement being captured.
Expr *const * const_capture_init_iterator
Const iterator that walks over the capture initialization arguments.
CharUnits - This is an opaque type for sizes expressed in character units.
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
CharUnits alignTo(const CharUnits &Align) const
alignTo - Returns the next integer (mod 2**64) that is greater than or equal to this quantity and is ...
bool hasReducedDebugInfo() const
Check if type and variable info should be emitted.
CharUnits getAlignment() const
Return the alignment of this pointer.
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Address withAlignment(CharUnits NewAlignment) const
Return address with different alignment, but same pointer and element type.
llvm::Value * getPointer() const
llvm::PointerType * getType() const
Return the type of the pointer value.
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored.
static ApplyDebugLocation CreateDefaultArtificial(CodeGenFunction &CGF, SourceLocation TemporaryLocation)
Apply TemporaryLocation if it is valid.
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
llvm::AtomicRMWInst * CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Ordering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
CGFunctionInfo - Class to encapsulate the information about a function definition.
Manages list of lastprivate conditional decls for the specified directive.
static LastprivateConditionalRAII disable(CodeGenFunction &CGF, const OMPExecutableDirective &S)
Manages list of nontemporal decls for the specified directive.
Struct that keeps all the relevant information that should be kept throughout a 'target data' region.
Manages list of nontemporal decls for the specified directive.
virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
Emit task region for the task directive.
virtual llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST)
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device)
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
virtual void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc)
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)...
virtual void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps)
Emits a single region.
virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal)
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF, const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const
Choose default schedule type and chunk value for the schedule clause.
virtual void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars)
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
virtual void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancellation point' construct.
virtual const VarDecl * translateParameter(const FieldDecl *FD, const VarDecl *NativeParam) const
Translates the native parameter of outlined function if this is required for target.
virtual llvm::Function * emitTeamsOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP teams directive D.
virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr * > NumIterations)
Emit initialization for doacross loop nesting support.
virtual void adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF, const OMPExecutableDirective &D) const
Adjust some parameters for the target-based directives, like addresses of the variables captured by r...
virtual void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, CGOpenMPRuntime::TargetDataInfo &Info)
Emit the target data mapping code associated with D.
virtual Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const
Gets the address of the native argument basing on the address of the target-specific parameter.
virtual void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc)
Emit a taskgroup region.
virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams,...
virtual llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data)
Emit a code for initialization of task reduction clause.
void emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, OpenMPDependClauseKind NewDepKind, SourceLocation Loc)
Updates the dependency kind in the specified depobj object.
virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD, SourceLocation Loc)
Gets the address of the global copy used for lastprivate conditional update, if any.
virtual void emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, Expr *ME, bool IsFatal)
Emit __kmpc_error call for error directive extern void __kmpc_error(ident_t *loc, int severity,...
virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc)
Emits code for a taskyield directive.
virtual void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc, llvm::AtomicOrdering AO)
Emit flush of the variables specified in 'omp flush' directive.
virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPTaskDataTy &Data)
Emit code for 'taskwait' directive.
virtual void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc)
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, int proc_bind) to generat...
virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false)
Emit an implicit/explicit barrier for OpenMP threads.
virtual void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values)
virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind)
Call the appropriate runtime routine to notify that we finished all the work with current loop.
void emitIfClause(CodeGenFunction &CGF, const Expr *Cond, const RegionCodeGenTy &ThenGen, const RegionCodeGenTy &ElseGen)
Emits code for OpenMP 'if' clause using specified CodeGen function.
Address emitDepobjDependClause(CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies, SourceLocation Loc)
Emits list of dependecies based on the provided data (array of dependence/expression pairs) for depob...
virtual llvm::Function * emitParallelOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP parallel directive D.
virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values)
Call the appropriate runtime routine to initialize it before start of loop.
llvm::AtomicOrdering getDefaultMemoryOrdering() const
Gets default memory ordering as specified in requires directive.
virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static non-chunked.
virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancel' construct.
virtual void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc)
Emits a master region.
virtual llvm::Function * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts)
Emits outlined function for the OpenMP task directive D.
void emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal, SourceLocation Loc)
Emits the code to destroy the dependency object provided in depobj directive.
virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Required to resolve existing problems in the runtime.
virtual void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C)
Emit code for doacross ordered directive with 'depend' clause.
virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF, const Expr *LHS)
Checks if the provided LVal is lastprivate conditional and emits the code to update the value of the ...
virtual void getDefaultDistScheduleAndChunk(CodeGenFunction &CGF, const OMPLoopDirective &S, OpenMPDistScheduleClauseKind &ScheduleKind, llvm::Value *&Chunk) const
Choose default schedule type and chunk value for the dist_schedule clause.
llvm::OpenMPIRBuilder & getOMPBuilder()
virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Emit outilined function for 'target' directive.
virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr)
Emits a critical region.
virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned)
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
virtual void checkAndEmitSharedLastprivateConditional(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::DenseSet< CanonicalDeclPtr< const VarDecl > > &IgnoredDecls)
Checks if the lastprivate conditional was updated in inner region and writes the value.
virtual void emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool HasCancel=false)
Emit code for the directive that does not require outlining.
virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond, llvm::Value *NumThreads)
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static chunked.
virtual void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Emit the target offloading code associated with D.
virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps, ReductionOptionsTy Options)
Emit a code for reduction clause.
std::pair< llvm::Value *, Address > emitDependClause(CodeGenFunction &CGF, ArrayRef< OMPTaskDataTy::DependData > Dependencies, SourceLocation Loc)
Emits list of dependecies based on the provided data (array of dependence/expression pairs).
virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const
Check if the specified ScheduleKind is dynamic.
Address emitLastprivateConditionalInit(CodeGenFunction &CGF, const VarDecl *VD)
Create specialized alloca to handle lastprivate conditionals.
virtual void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads)
Emit an ordered region.
virtual void emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, ArrayRef< llvm::Value * > Args=std::nullopt) const
Emits call of the outlined function with the provided arguments, translating these arguments to corre...
virtual void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, bool IsWorksharingReduction)
Emits the following code for reduction clause with task modifier:
virtual void emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter=nullptr)
Emits a masked region.
virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues)
Call the appropriate runtime routine to initialize it before start of loop.
virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S)
Emit the captured statement body.
virtual StringRef getHelperName() const
Get the name of the capture helper.
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
RAII for preserving necessary info during inlined region body codegen.
Cleanup action for allocate support.
RAII for preserving necessary info during Outlined region body codegen.
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion, const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen, OMPTaskDataTy &Data)
void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S)
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
void EmitOMPParallelGenericLoopDirective(const OMPLoopDirective &S)
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount, Stmt::Likelihood LH=Stmt::LH_None, const Expr *ConditionalOp=nullptr)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target)
The given basic block lies in the current EH scope, but may be a target of a potentially scope-crossi...
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
void EmitOMPOrderedDirective(const OMPOrderedDirective &S)
void EmitOMPTargetDirective(const OMPTargetDirective &S)
bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
void EmitOMPReductionClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope, bool ForInscan=false)
Emit initial code for reduction variables.
void EmitOMPAtomicDirective(const OMPAtomicDirective &S)
void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S)
void EmitOMPParallelMasterTaskLoopDirective(const OMPParallelMasterTaskLoopDirective &S)
void EmitAutoVarDecl(const VarDecl &D)
EmitAutoVarDecl - Emit an auto variable declaration.
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
static bool hasScalarEvaluationKind(QualType T)
llvm::function_ref< std::pair< llvm::Value *, llvm::Value * >(CodeGenFunction &, const OMPExecutableDirective &S, Address LB, Address UB)> CodeGenDispatchBoundsTy
CGCapturedStmtInfo * CapturedStmtInfo
void EmitDecl(const Decl &D)
EmitDecl - Emit a declaration.
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Load a pointer with type PtrTy stored at address Ptr.
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
llvm::Value * emitArrayLength(const ArrayType *arrayType, QualType &baseType, Address &addr)
emitArrayLength - Compute the length of an array, even if it's a VLA, and drill down to the base elem...
VlaSizePair getVLASize(const VariableArrayType *vla)
Returns an LLVM value that corresponds to the size, in non-variably-sized elements,...
const OMPExecutableDirective * OMPParentLoopDirectiveForScan
Parent loop-based directive for scan directive.
void EmitOMPTeamsDistributeParallelForDirective(const OMPTeamsDistributeParallelForDirective &S)
void EmitOMPTaskDirective(const OMPTaskDirective &S)
void EmitOMPScanDirective(const OMPScanDirective &S)
void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S)
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
void EmitOMPMasterTaskLoopSimdDirective(const OMPMasterTaskLoopSimdDirective &S)
void EmitOMPUseDevicePtrClause(const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap)
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
void EmitOMPDistributeLoop(const OMPLoopDirective &S, const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr)
Emit code for the distribute loop-based directive.
void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S)
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO, const llvm::function_ref< RValue(RValue)> &UpdateOp, bool IsVolatile)
const LangOptions & getLangOpts() const
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S)
Emit device code for the target simd directive.
void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S)
void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S, OMPPrivateScope &LoopScope)
Emit initial code for loop counters of loop-based directives.
void EmitOMPDistributeParallelForDirective(const OMPDistributeParallelForDirective &S)
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D, bool NoFinals, llvm::Value *IsLastIterCond=nullptr)
Emit final copying of lastprivate values to original variables at the end of the worksharing or simd ...
void EmitOMPTeamsGenericLoopDirective(const OMPTeamsGenericLoopDirective &S)
void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S)
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit)
EmitExprAsInit - Emits the code necessary to initialize a location in memory with the given initializ...
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
void EmitOMPParallelDirective(const OMPParallelDirective &S)
void EmitOMPSimdFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
llvm::Type * ConvertTypeForMem(QualType T)
void EmitOMPDistributeParallelForSimdDirective(const OMPDistributeParallelForSimdDirective &S)
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
void EmitOMPForSimdDirective(const OMPForSimdDirective &S)
llvm::AssertingVH< llvm::Instruction > AllocaInsertPt
AllocaInsertPoint - This is an instruction in the entry block before which we prefer to insert alloca...
void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy)
Emit an aggregate assignment.
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value * > &CapturedVars)
CodeGenFunction * ParentCGF
JumpDest ReturnBlock
ReturnBlock - Unified return block.
const llvm::function_ref< void(CodeGenFunction &, llvm::Function *, const OMPTaskDataTy &)> TaskGenTy
static void EmitOMPTargetTeamsGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsGenericLoopDirective &S)
Emit device code for the target teams loop directive.
void EmitOMPFlushDirective(const OMPFlushDirective &S)
llvm::BasicBlock * OMPBeforeScanBlock
void EmitOMPCancelDirective(const OMPCancelDirective &S)
void EmitOMPGenericLoopDirective(const OMPGenericLoopDirective &S)
void EmitOMPTargetTeamsDistributeDirective(const OMPTargetTeamsDistributeDirective &S)
ConstantEmission tryEmitAsConstant(DeclRefExpr *refExpr)
void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S)
static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S)
void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy, SourceLocation Loc)
void EmitOMPInnerLoop(const OMPExecutableDirective &S, bool RequiresCleanup, const Expr *LoopCond, const Expr *IncExpr, const llvm::function_ref< void(CodeGenFunction &)> BodyGen, const llvm::function_ref< void(CodeGenFunction &)> PostIncGen)
Emit inner loop of the worksharing/simd construct.
llvm::DebugLoc SourceLocToDebugLoc(SourceLocation Location)
Converts Location to a DebugLoc, if debug information is enabled.
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
void EmitOMPDepobjDirective(const OMPDepobjDirective &S)
void EmitOMPSingleDirective(const OMPSingleDirective &S)
void EmitOMPTargetTeamsGenericLoopDirective(const OMPTargetTeamsGenericLoopDirective &S)
llvm::CanonicalLoopInfo * EmitOMPCollapsedCanonicalLoopNest(const Stmt *S, int Depth)
Emit the Stmt S and return its topmost canonical loop, if any.
void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D)
Emit simple code for OpenMP directives in Simd-only mode.
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
EmitAnyExpr - Emit code to compute the specified expression which can have any type.
void EmitOMPParallelForDirective(const OMPParallelForDirective &S)
void EmitOMPTeamsDirective(const OMPTeamsDirective &S)
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
void EmitOMPUnrollDirective(const OMPUnrollDirective &S)
void EmitOMPParallelMasterTaskLoopSimdDirective(const OMPParallelMasterTaskLoopSimdDirective &S)
void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S)
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
llvm::Value * EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified complex type to the specified destination type,...
bool isTrivialInitializer(const Expr *Init)
Determine whether the given initializer is trivial in the sense that it requires no code to be genera...
CGDebugInfo * getDebugInfo()
void EmitOMPTargetTeamsDistributeParallelForSimdDirective(const OMPTargetTeamsDistributeParallelForSimdDirective &S)
void EmitOMPTeamsDistributeParallelForSimdDirective(const OMPTeamsDistributeParallelForSimdDirective &S)
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block,...
llvm::Function * GenerateCapturedStmtFunction(const CapturedStmt &S)
void EmitOMPInteropDirective(const OMPInteropDirective &S)
void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind)
Emit final update of reduction values to original variables at the end of the directive.
llvm::BasicBlock * OMPScanDispatch
llvm::function_ref< std::pair< LValue, LValue >(CodeGenFunction &, const OMPExecutableDirective &S)> CodeGenLoopBoundsTy
void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty, SourceLocation Loc, SourceLocation AssumptionLoc, llvm::Value *Alignment, llvm::Value *OffsetValue=nullptr)
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S)
void EmitOMPParallelMaskedDirective(const OMPParallelMaskedDirective &S)
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
void EmitOMPSimdInit(const OMPLoopDirective &D)
Helpers for the OpenMP loop directives.
int ExpectedOMPLoopDepth
Number of nested loop to be consumed by the last surrounding loop-associated directive.
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S)
ASTContext & getContext() const
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
SmallVector< llvm::CanonicalLoopInfo *, 4 > OMPLoopNestStack
List of recently emitted OMPCanonicalLoops.
bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB, const CodeGenLoopBoundsTy &CodeGenLoopBounds, const CodeGenDispatchBoundsTy &CGDispatchBounds)
Emit code for the worksharing loop-based directive.
void EmitOMPLinearClause(const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope)
Emit initial code for linear clauses.
void EmitBranchThroughCleanup(JumpDest Dest)
EmitBranchThroughCleanup - Emit a branch from the current insert block through the normal cleanup han...
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant,...
void EmitAutoVarCleanups(const AutoVarEmission &emission)
void EmitOMPTileDirective(const OMPTileDirective &S)
bool EmitOMPLinearClauseInit(const OMPLoopDirective &D)
Emit initial code for linear variables.
llvm::DenseMap< const ValueDecl *, FieldDecl * > LambdaCaptureFields
void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit)
Helper for the OpenMP loop directives.
void EmitOMPLinearClauseFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
Emit final code for linear clauses.
void EmitOMPSectionsDirective(const OMPSectionsDirective &S)
llvm::BasicBlock * OMPScanExitBlock
static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Emit device code for the target teams distribute parallel for simd directive.
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
void EmitOMPUseDeviceAddrClause(const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap)
void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S)
void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S)
void EmitOMPSimdDirective(const OMPSimdDirective &S)
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
void EmitOMPCriticalDirective(const OMPCriticalDirective &S)
void EmitOMPForDirective(const OMPForDirective &S)
void EmitOMPMetaDirective(const OMPMetaDirective &S)
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S)
LValue InitCapturedStruct(const CapturedStmt &S)
void EmitOMPParallelMasterDirective(const OMPParallelMasterDirective &S)
void EmitOMPTargetTeamsDistributeSimdDirective(const OMPTargetTeamsDistributeSimdDirective &S)
void processInReduction(const OMPExecutableDirective &S, OMPTaskDataTy &Data, CodeGenFunction &CGF, const CapturedStmt *CS, OMPPrivateScope &Scope)
void EmitOMPMasterDirective(const OMPMasterDirective &S)
void EmitOMPMasterTaskLoopDirective(const OMPMasterTaskLoopDirective &S)
void EmitOMPTargetParallelGenericLoopDirective(const OMPTargetParallelGenericLoopDirective &S)
bool EmitOMPCopyinClause(const OMPExecutableDirective &D)
Emit code for copyin clause in D directive.
void EmitOMPMaskedDirective(const OMPMaskedDirective &S)
uint64_t getProfileCount(const Stmt *S)
Get the profiler's count for the given statement.
void EmitOMPPrivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S, SourceLocation Loc)
void EmitOMPTargetParallelForSimdDirective(const OMPTargetParallelForSimdDirective &S)
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
Address GenerateCapturedStmtArgument(const CapturedStmt &S)
static void EmitOMPTargetParallelGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelGenericLoopDirective &S)
Emit device code for the target parallel loop directive.
llvm::function_ref< void(CodeGenFunction &, SourceLocation, const unsigned, const bool)> CodeGenOrderedTy
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
llvm::BasicBlock * OMPAfterScanBlock
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S)
void EmitOMPErrorDirective(const OMPErrorDirective &S)
void EmitOMPSectionDirective(const OMPSectionDirective &S)
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
void EmitOMPBarrierDirective(const OMPBarrierDirective &S)
void EmitStopPoint(const Stmt *S)
EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
void EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S)
void EmitStmt(const Stmt *S, ArrayRef< const Attr * > Attrs=std::nullopt)
EmitStmt - Emit the code for the statement.
void EnsureInsertPoint()
EnsureInsertPoint - Ensure that an insertion point is defined so that emitted IR has a place to go.
llvm::LLVMContext & getLLVMContext()
void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
void EmitOMPTargetTeamsDistributeParallelForDirective(const OMPTargetTeamsDistributeParallelForDirective &S)
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler's counter for the given statement by StepV.
bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for lastprivate variables.
void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S)
llvm::function_ref< void(CodeGenFunction &, const OMPLoopDirective &, JumpDest)> CodeGenLoopTy
void EmitOMPCanonicalLoop(const OMPCanonicalLoop *S)
Emit an OMPCanonicalLoop using the OpenMPIRBuilder.
void EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S)
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
void EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S)
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
This class organizes the cross-function state that is used while generating LLVM code.
void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
llvm::Module & getModule() const
CharUnits GetTargetTypeStoreSize(llvm::Type *Ty) const
Return the store size, in character units, of the given LLVM type.
DiagnosticsEngine & getDiags() const
const LangOptions & getLangOpts() const
CharUnits getNaturalTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, bool forPointeeType=false)
CodeGenTypes & getTypes()
const llvm::DataLayout & getDataLayout() const
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
llvm::Constant * GetAddrOfGlobal(GlobalDecl GD, ForDefinition_t IsForDefinition=NotForDefinition)
ASTContext & getContext() const
const CodeGenOptions & getCodeGenOpts() const
StringRef getMangledName(GlobalDecl GD)
llvm::Constant * getStaticLocalDeclAddress(const VarDecl *D)
llvm::ConstantInt * getSize(CharUnits numChars)
Emit the given number of characters as a value of type size_t.
void assignRegionCounters(GlobalDecl GD, llvm::Function *Fn)
Assign counters to regions and configure them for PGO of a given function.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
llvm::Type * ConvertTypeForMem(QualType T, bool ForBitField=false)
ConvertTypeForMem - Convert type T into a llvm::Type.
FunctionArgList - Type for representing both the decl and type of parameters to a function.
LValue - This represents an lvalue references.
Address getAddress(CodeGenFunction &CGF) const
llvm::Value * getPointer(CodeGenFunction &CGF) const
void setAddress(Address address)
void setUnrollCount(unsigned C)
Set the unroll count for the next loop pushed.
void setVectorizeWidth(unsigned W)
Set the vectorize width for the next loop pushed.
void setParallel(bool Enable=true)
Set the next pushed loop as parallel.
void setUnrollState(const LoopAttributes::LVEnableState &State)
Set the next pushed loop unroll state.
void pop()
End the current loop.
void push(llvm::BasicBlock *Header, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
Begin a new structured loop.
void setVectorizeEnable(bool Enable=true)
Set the next pushed loop 'vectorize.enable'.
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
virtual void Enter(CodeGenFunction &CGF)
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
static RValue get(llvm::Value *V)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
Class intended to support codegen of all kind of the reduction clauses.
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N)
Emits lvalue for the shared and original reduction item.
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
Complex values, per C99 6.2.5p11.
CompoundStmt - This represents a group of statements like { stmt stmt }.
ConstStmtVisitor - This class implements a simple visitor for Stmt subclasses.
DeclContext * getParent()
getParent - Returns the containing DeclContext.
A reference to a declared variable, function, enum, etc.
static DeclRefExpr * Create(const ASTContext &Context, NestedNameSpecifierLoc QualifierLoc, SourceLocation TemplateKWLoc, ValueDecl *D, bool RefersToEnclosingVariableOrCapture, SourceLocation NameLoc, QualType T, ExprValueKind VK, NamedDecl *FoundD=nullptr, const TemplateArgumentListInfo *TemplateArgs=nullptr, NonOdrUseReason NOUR=NOUR_None)
DeclStmt - Adaptor class for mixing declarations with statements and expressions.
Decl - This represents one declaration (or definition), e.g.
SourceLocation getBodyRBrace() const
getBodyRBrace - Gets the right brace of the body, if a body exists.
virtual bool hasBody() const
Returns true if this Decl represents a declaration for a body of code, such as a function or method d...
SourceLocation getLocation() const
The name of a declaration.
SourceLocation getBeginLoc() const LLVM_READONLY
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
unsigned getCustomDiagID(Level L, const char(&FormatString)[N])
Return an ID for a diagnostic with the specified format string and level.
This represents one expression.
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer,...
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
Expr * IgnoreImplicitAsWritten() LLVM_READONLY
Skip past any implicit AST nodes which might surround this expression until reaching a fixed point.
Expr * IgnoreImpCasts() LLVM_READONLY
Skip past any implicit casts which might surround this expression until reaching a fixed point.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Represents difference between two FPOptions values.
Represents a member of a struct/union/class.
Represents a function declaration or definition.
static FunctionDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation NLoc, DeclarationName N, QualType T, TypeSourceInfo *TInfo, StorageClass SC, bool UsesFPIntrin=false, bool isInlineSpecified=false, bool hasWrittenPrototype=true, ConstexprSpecKind ConstexprKind=ConstexprSpecKind::Unspecified, Expr *TrailingRequiresClause=nullptr)
GlobalDecl - represents a global declaration.
One of these records is kept for each identifier that is lexed.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
static ImplicitCastExpr * Create(const ASTContext &Context, QualType T, CastKind Kind, Expr *Operand, const CXXCastPath *BasePath, ExprValueKind Cat, FPOptionsOverride FPO)
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
A C++ lambda expression, which produces a function object (of unspecified type) that can be invoked l...
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
Represents a point when we exit a loop.
IdentifierInfo * getIdentifier() const
Get the identifier that names this declaration, if there is one.
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
A C++ nested-name-specifier augmented with source location information.
This represents 'acq_rel' clause in the '#pragma omp atomic|flush' directives.
This represents 'acquire' clause in the '#pragma omp atomic|flush' directives.
This represents clause 'aligned' in the '#pragma omp ...' directives.
This represents '#pragma omp atomic' directive.
This represents '#pragma omp barrier' directive.
This represents '#pragma omp cancel' directive.
This represents '#pragma omp cancellation point' directive.
Representation of an OpenMP canonical loop.
static OMPClauseWithPreInit * get(OMPClause *C)
This is a basic class for representing single OpenMP clause.
This represents clause 'copyin' in the '#pragma omp ...' directives.
This represents clause 'copyprivate' in the '#pragma omp ...' directives.
This represents '#pragma omp critical' directive.
This represents implicit clause 'depend' for the '#pragma omp task' directive.
This represents implicit clause 'depobj' for the '#pragma omp depobj' directive.
This represents '#pragma omp depobj' directive.
This represents 'destroy' clause in the '#pragma omp depobj' directive or the '#pragma omp interop' d...
This represents 'device' clause in the '#pragma omp ...' directive.
This represents 'dist_schedule' clause in the '#pragma omp ...' directive.
This represents '#pragma omp distribute' directive.
This represents '#pragma omp distribute parallel for' composite directive.
This represents '#pragma omp distribute parallel for simd' composite directive.
This represents '#pragma omp distribute simd' composite directive.
This represents the 'doacross' clause for the '#pragma omp ordered' directive.
This represents '#pragma omp error' directive.
This is a basic class for representing single OpenMP executable directive.
CapturedStmt * getInnermostCapturedStmt()
Get innermost captured statement for the construct.
bool hasAssociatedStmt() const
Returns true if directive has associated statement.
OpenMPDirectiveKind getDirectiveKind() const
const Stmt * getAssociatedStmt() const
Returns statement associated with the directive.
SourceLocation getBeginLoc() const
Returns starting location of directive kind.
ArrayRef< OMPClause * > clauses() const
SourceLocation getEndLoc() const
Returns ending location of directive.
static const SpecificClause * getSingleClause(ArrayRef< OMPClause * > Clauses)
Gets a single clause of the specified kind associated with the current directive iff there is only on...
static llvm::iterator_range< specific_clause_iterator< SpecificClause > > getClausesOfKind(ArrayRef< OMPClause * > Clauses)
This represents 'fail' clause in the '#pragma omp atomic' directive.
OpenMPClauseKind getFailParameter() const
Gets the parameter (type memory-order-clause) in Fail clause.
This represents 'filter' clause in the '#pragma omp ...' directive.
This represents 'final' clause in the '#pragma omp ...' directive.
This represents clause 'firstprivate' in the '#pragma omp ...' directives.
This represents implicit clause 'flush' for the '#pragma omp flush' directive.
This represents '#pragma omp flush' directive.
This represents '#pragma omp for' directive.
This represents '#pragma omp for simd' directive.
Representation of the 'full' clause of the '#pragma omp unroll' directive.
This represents '#pragma omp loop' directive.
This represents 'grainsize' clause in the '#pragma omp ...' directive.
This represents 'hint' clause in the '#pragma omp ...' directive.
This represents 'if' clause in the '#pragma omp ...' directive.
This represents clause 'in_reduction' in the '#pragma omp task' directives.
This represents clause 'inclusive' in the '#pragma omp scan' directive.
This represents the 'init' clause in '#pragma omp ...' directives.
This represents '#pragma omp interop' directive.
This represents clause 'lastprivate' in the '#pragma omp ...' directives.
This represents clause 'linear' in the '#pragma omp ...' directives.
The base class for all loop-based directives, including loop transformation directives.
static Stmt * tryToFindNextInnerLoop(Stmt *CurStmt, bool TryImperfectlyNestedLoops)
Try to find the next loop sub-statement in the specified statement CurStmt.
static bool doForAllLoops(Stmt *CurStmt, bool TryImperfectlyNestedLoops, unsigned NumLoops, llvm::function_ref< bool(unsigned, Stmt *)> Callback, llvm::function_ref< void(OMPLoopTransformationDirective *)> OnTransformationCallback)
Calls the specified callback function for all the loops in CurStmt, from the outermost to the innermo...
unsigned getLoopsNumber() const
Get number of collapsed loops.
This is a common base class for loop directives ('omp simd', 'omp for', 'omp for simd' etc....
ArrayRef< Expr * > finals_conditions()
Expr * getPrevUpperBoundVariable() const
ArrayRef< Expr * > counters()
Expr * getNumIterations() const
ArrayRef< Expr * > finals()
Expr * getPrevLowerBoundVariable() const
Expr * getIterationVariable() const
Expr * getUpperBoundVariable() const
ArrayRef< Expr * > updates()
Expr * getLowerBoundVariable() const
ArrayRef< Expr * > private_counters()
This represents '#pragma omp masked' directive.
This represents '#pragma omp master' directive.
This represents '#pragma omp master taskloop' directive.
This represents '#pragma omp master taskloop simd' directive.
This represents 'message' clause in the '#pragma omp error' directive.
Expr * getMessageString() const
Returns message string of the clause.
This represents 'nogroup' clause in the '#pragma omp ...' directive.
This represents 'nowait' clause in the '#pragma omp ...' directive.
This represents 'num_tasks' clause in the '#pragma omp ...' directive.
This represents 'num_teams' clause in the '#pragma omp ...' directive.
This represents 'num_threads' clause in the '#pragma omp ...' directive.
This represents 'order' clause in the '#pragma omp ...' directive.
This represents 'ordered' clause in the '#pragma omp ...' directive.
This represents '#pragma omp ordered' directive.
This represents '#pragma omp parallel' directive.
This represents '#pragma omp parallel for' directive.
This represents '#pragma omp parallel for simd' directive.
This represents '#pragma omp parallel masked' directive.
This represents '#pragma omp parallel master' directive.
This represents '#pragma omp parallel master taskloop' directive.
This represents '#pragma omp parallel master taskloop simd' directive.
This represents '#pragma omp parallel sections' directive.
Representation of the 'partial' clause of the '#pragma omp unroll' directive.
This represents 'priority' clause in the '#pragma omp ...' directive.
This represents clause 'private' in the '#pragma omp ...' directives.
This represents 'proc_bind' clause in the '#pragma omp ...' directive.
This represents clause 'reduction' in the '#pragma omp ...' directives.
This represents 'relaxed' clause in the '#pragma omp atomic' directives.
This represents 'release' clause in the '#pragma omp atomic|flush' directives.
This represents 'simd' clause in the '#pragma omp ...' directive.
This represents 'safelen' clause in the '#pragma omp ...' directive.
This represents '#pragma omp scan' directive.
This represents 'schedule' clause in the '#pragma omp ...' directive.
This represents '#pragma omp section' directive.
This represents '#pragma omp sections' directive.
This represents 'seq_cst' clause in the '#pragma omp atomic' directive.
This represents 'severity' clause in the '#pragma omp error' directive.
OpenMPSeverityClauseKind getSeverityKind() const
Returns kind of the clause.
This represents '#pragma omp simd' directive.
This represents 'simdlen' clause in the '#pragma omp ...' directive.
This represents '#pragma omp single' directive.
This represents '#pragma omp target data' directive.
This represents '#pragma omp target' directive.
This represents '#pragma omp target enter data' directive.
This represents '#pragma omp target exit data' directive.
This represents '#pragma omp target parallel' directive.
This represents '#pragma omp target parallel for' directive.
This represents '#pragma omp target parallel for simd' directive.
This represents '#pragma omp target parallel loop' directive.
This represents '#pragma omp target simd' directive.
This represents '#pragma omp target teams' directive.
This represents '#pragma omp target teams distribute' combined directive.
This represents '#pragma omp target teams distribute parallel for' combined directive.
This represents '#pragma omp target teams distribute parallel for simd' combined directive.
This represents '#pragma omp target teams distribute simd' combined directive.
This represents '#pragma omp target teams loop' directive.
This represents '#pragma omp target update' directive.
This represents '#pragma omp task' directive.
This represents '#pragma omp taskloop' directive.
This represents '#pragma omp taskloop simd' directive.
This represents clause 'task_reduction' in the '#pragma omp taskgroup' directives.
This represents '#pragma omp taskgroup' directive.
This represents '#pragma omp taskwait' directive.
This represents '#pragma omp taskyield' directive.
This represents '#pragma omp teams' directive.
This represents '#pragma omp teams distribute' directive.
This represents '#pragma omp teams distribute parallel for' composite directive.
This represents '#pragma omp teams distribute parallel for simd' composite directive.
This represents '#pragma omp teams distribute simd' combined directive.
This represents '#pragma omp teams loop' directive.
This represents 'thread_limit' clause in the '#pragma omp ...' directive.
This represents the '#pragma omp tile' loop transformation directive.
This represents the '#pragma omp unroll' loop transformation directive.
This represents 'untied' clause in the '#pragma omp ...' directive.
This represents 'update' clause in the '#pragma omp atomic' directive.
This represents the 'use' clause in '#pragma omp ...' directives.
This represents clause 'use_device_addr' in the '#pragma omp ...' directives.
This represents clause 'use_device_ptr' in the '#pragma omp ...' directives.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class.
static ParmVarDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, StorageClass S, Expr *DefArg)
PointerType - C99 6.7.5.1 - Pointer Declarators.
If a crash happens while one of these objects are live, the message is printed out along with the spe...
A (possibly-)qualified type.
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Represents a struct/union/class.
field_range fields() const
field_iterator field_begin() const
Base for LValueReferenceType and RValueReferenceType.
Scope - A scope is a transient data structure that is used while parsing the program.
Encodes a location in the source.
A trivial tuple used to represent a source range.
SourceLocation getEnd() const
SourceLocation getBegin() const
Stmt - This represents one statement.
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top,...
SourceLocation getBeginLoc() const LLVM_READONLY
SwitchStmt - This represents a 'switch' stmt.
virtual bool hasBuiltinAtomic(uint64_t AtomicSizeInBits, uint64_t AlignmentInBits) const
Returns true if the given target supports lock-free atomic operations at the specified width and alig...
bool isTLSSupported() const
Whether the target supports thread-local storage.
The base class of the type hierarchy.
bool isPointerType() const
const T * castAs() const
Member-template castAs<specific type>.
bool isReferenceType() const
bool isScalarType() const
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
bool isLValueReferenceType() const
bool isAnyComplexType() const
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g....
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type.
bool isAnyPointerType() const
UnaryOperator - This represents the unary-expression's (except sizeof and alignof),...
static UnaryOperator * Create(const ASTContext &C, Expr *input, Opcode opc, QualType type, ExprValueKind VK, ExprObjectKind OK, SourceLocation l, bool CanOverflow, FPOptionsOverride FPFeatures)
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Represents a variable declaration or definition.
TLSKind getTLSKind() const
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
@ CInit
C-style initialization with assignment.
bool hasGlobalStorage() const
Returns true for all variables that do not have local storage.
bool isStaticLocal() const
Returns true if a variable with function scope is a static local variable.
const Expr * getInit() const
bool hasLocalStorage() const
Returns true if a variable with function scope is a non-static local variable.
@ TLS_None
Not a TLS variable.
Represents a C array with a specified size that is not an integer-constant-expression.
Expr * getSizeExpr() const
@ Decl
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
bool Inc(InterpState &S, CodePtr OpPC)
1) Pops a pointer from the stack 2) Load the value from the pointer 3) Writes the value increased by ...
The JSON file list parser is used to communicate input to InstallAPI.
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
@ Tile
'tile' clause, allowed on 'loop' and Combined constructs.
@ Device
'device' clause, allowed on the 'update' construct.
bool needsTaskBasedThreadLimit(OpenMPDirectiveKind DKind)
Checks if the specified target directive, combined or not, needs task based thread_limit.
@ Ctor_Complete
Complete object ctor.
llvm::omp::Directive OpenMPDirectiveKind
OpenMP directives.
@ OK_Ordinary
An ordinary object is located at an address in memory.
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
@ OMPC_SCHEDULE_MODIFIER_unknown
llvm::omp::Clause OpenMPClauseKind
OpenMP clauses.
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
OpenMPDistScheduleClauseKind
OpenMP attributes for 'dist_schedule' clause.
@ OMPC_DIST_SCHEDULE_unknown
bool isOpenMPTaskingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of tasking directives - task, taskloop,...
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
@ Result
The result type of a method or function.
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
OpenMPDependClauseKind
OpenMP attributes for 'depend' clause.
@ Dtor_Complete
Complete object dtor.
bool isOpenMPLoopBoundSharingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of the composite or combined directives that need loop ...
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
@ VK_PRValue
A pr-value expression (in the C++11 taxonomy) produces a temporary value.
@ VK_LValue
An l-value expression is a reference to an object with independent storage.
void getOpenMPCaptureRegions(llvm::SmallVectorImpl< OpenMPDirectiveKind > &CaptureRegions, OpenMPDirectiveKind DKind)
Return the captured regions of an OpenMP directive.
if(T->getSizeExpr()) TRY_TO(TraverseStmt(T -> getSizeExpr()))
@ ThreadPrivateVar
Parameter for Thread private variable.
@ Other
Other implicit parameter.
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
YAML serialization mapping.
static Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc)
Returns address of the threadprivate variable for the current thread.
llvm::OpenMPIRBuilder::InsertPointTy InsertPointTy
static void EmitOMPOutlinedRegionBody(CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Twine RegionName)
Emit the body of an OMP region that will be outlined in OpenMPIRBuilder::finalize().
static Address getAddressOfLocalVariable(CodeGenFunction &CGF, const VarDecl *VD)
Gets the OpenMP-specific address of the local variable /p VD.
static void EmitCaptureStmt(CodeGenFunction &CGF, InsertPointTy CodeGenIP, llvm::BasicBlock &FiniBB, llvm::Function *Fn, ArrayRef< llvm::Value * > Args)
static std::string getNameWithSeparators(ArrayRef< StringRef > Parts, StringRef FirstSeparator=".", StringRef Separator=".")
Get the platform-specific name separator.
static void FinalizeOMPRegion(CodeGenFunction &CGF, InsertPointTy IP)
Emit the Finalization for an OMP region.
static void EmitOMPInlinedRegionBody(CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Twine RegionName)
Emit the body of an OMP region.
llvm::PointerType * VoidPtrTy
llvm::IntegerType * Int64Ty
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::IntegerType * SizeTy
llvm::IntegerType * Int32Ty
llvm::IntegerType * IntPtrTy
llvm::PointerType * Int8PtrTy
SmallVector< const Expr *, 4 > DepExprs
EvalResult is a struct with detailed info about an evaluated expression.
Extra information about a function prototype.
Scheduling data for loop-based OpenMP directives.
OpenMPScheduleClauseModifier M2
OpenMPScheduleClauseModifier M1
OpenMPScheduleClauseKind Schedule