27#include "llvm/ADT/SmallSet.h"
28#include "llvm/BinaryFormat/Dwarf.h"
29#include "llvm/Frontend/OpenMP/OMPConstants.h"
30#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
31#include "llvm/IR/Constants.h"
32#include "llvm/IR/DebugInfoMetadata.h"
33#include "llvm/IR/Instructions.h"
34#include "llvm/IR/IntrinsicInst.h"
35#include "llvm/IR/Metadata.h"
36#include "llvm/Support/AtomicOrdering.h"
39using namespace CodeGen;
40using namespace llvm::omp;
47class OMPLexicalScope :
public CodeGenFunction::LexicalScope {
49 for (
const auto *
C : S.clauses()) {
51 if (
const auto *PreInit =
52 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
53 for (
const auto *I : PreInit->decls()) {
54 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
57 CodeGenFunction::AutoVarEmission Emission =
66 CodeGenFunction::OMPPrivateScope InlinedShareds;
72 cast<BlockDecl>(CGF.
CurCodeDecl)->capturesVariable(VD));
78 const std::optional<OpenMPDirectiveKind> CapturedRegion = std::nullopt,
79 const bool EmitPreInitStmt =
true)
83 emitPreInitStmt(CGF, S);
86 assert(S.hasAssociatedStmt() &&
87 "Expected associated statement for inlined directive.");
88 const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
90 if (
C.capturesVariable() ||
C.capturesVariableByCopy()) {
91 auto *VD =
C.getCapturedVar();
93 "Canonical decl must be captured.");
97 InlinedShareds.isGlobalVarCaptured(VD)),
102 (void)InlinedShareds.Privatize();
108class OMPParallelScope final :
public OMPLexicalScope {
118 : OMPLexicalScope(CGF, S,
std::nullopt,
119 EmitPreInitStmt(S)) {}
124class OMPTeamsScope final :
public OMPLexicalScope {
133 : OMPLexicalScope(CGF, S,
std::nullopt,
134 EmitPreInitStmt(S)) {}
139class OMPLoopScope :
public CodeGenFunction::RunCleanupsScope {
142 CodeGenFunction::OMPMapVars PreCondVars;
143 if (
auto *LD = dyn_cast<OMPLoopDirective>(&S)) {
145 for (
const auto *E : LD->counters()) {
146 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
148 (void)PreCondVars.setVarAddr(
153 for (
const Expr *IRef :
C->varlists()) {
155 cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
156 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
157 QualType OrigVDTy = OrigVD->getType().getNonReferenceType();
158 (void)PreCondVars.setVarAddr(
167 (void)PreCondVars.apply(CGF);
170 LD->getInnermostCapturedStmt()->getCapturedStmt(),
171 true, LD->getLoopsNumber(),
172 [&CGF](
unsigned Cnt,
const Stmt *CurStmt) {
173 if (
const auto *CXXFor = dyn_cast<CXXForRangeStmt>(CurStmt)) {
174 if (
const Stmt *Init = CXXFor->getInit())
176 CGF.
EmitStmt(CXXFor->getRangeStmt());
181 PreInits = cast_or_null<DeclStmt>(LD->getPreInits());
182 }
else if (
const auto *Tile = dyn_cast<OMPTileDirective>(&S)) {
183 PreInits = cast_or_null<DeclStmt>(Tile->getPreInits());
184 }
else if (
const auto *Unroll = dyn_cast<OMPUnrollDirective>(&S)) {
185 PreInits = cast_or_null<DeclStmt>(Unroll->getPreInits());
187 llvm_unreachable(
"Unknown loop-based directive kind.");
190 for (
const auto *I : PreInits->
decls())
193 PreCondVars.restore(CGF);
199 emitPreInitStmt(CGF, S);
203class OMPSimdLexicalScope :
public CodeGenFunction::LexicalScope {
204 CodeGenFunction::OMPPrivateScope InlinedShareds;
210 cast<BlockDecl>(CGF.
CurCodeDecl)->capturesVariable(VD));
216 InlinedShareds(CGF) {
217 for (
const auto *
C : S.clauses()) {
219 if (
const auto *PreInit =
220 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
221 for (
const auto *I : PreInit->decls()) {
222 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
225 CodeGenFunction::AutoVarEmission Emission =
231 }
else if (
const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(
C)) {
232 for (
const Expr *E : UDP->varlists()) {
233 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
234 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
237 }
else if (
const auto *UDP = dyn_cast<OMPUseDeviceAddrClause>(
C)) {
238 for (
const Expr *E : UDP->varlists()) {
240 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
247 if (
const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
248 if (
const Expr *E = TG->getReductionRef())
249 CGF.
EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
255 if (
C->getModifier() != OMPC_REDUCTION_inscan)
257 for (
const Expr *E :
C->copy_array_temps())
258 CopyArrayTemps.insert(cast<DeclRefExpr>(E)->getDecl());
260 const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
263 if (
C.capturesVariable() ||
C.capturesVariableByCopy()) {
264 auto *VD =
C.getCapturedVar();
265 if (CopyArrayTemps.contains(VD))
268 "Canonical decl must be captured.");
270 isCapturedVar(CGF, VD) ||
272 InlinedShareds.isGlobalVarCaptured(VD)),
280 (void)InlinedShareds.Privatize();
290LValue CodeGenFunction::EmitOMPSharedLValue(
const Expr *E) {
291 if (
const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
292 if (
const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
293 OrigVD = OrigVD->getCanonicalDecl();
308 llvm::Value *
Size =
nullptr;
309 auto SizeInChars =
C.getTypeSizeInChars(Ty);
310 if (SizeInChars.isZero()) {
316 Size ?
Builder.CreateNUWMul(Size, VlaSize.NumElts) : VlaSize.NumElts;
318 SizeInChars =
C.getTypeSizeInChars(Ty);
319 if (SizeInChars.isZero())
320 return llvm::ConstantInt::get(
SizeTy, 0);
328 const RecordDecl *RD = S.getCapturedRecordDecl();
330 auto CurCap = S.captures().begin();
332 E = S.capture_init_end();
333 I != E; ++I, ++CurField, ++CurCap) {
334 if (CurField->hasCapturedVLAType()) {
337 CapturedVars.push_back(Val);
338 }
else if (CurCap->capturesThis()) {
339 CapturedVars.push_back(CXXThisValue);
340 }
else if (CurCap->capturesVariableByCopy()) {
345 if (!CurField->getType()->isAnyPointerType()) {
349 Twine(CurCap->getCapturedVar()->getName(),
".casted"));
364 CapturedVars.push_back(CV);
366 assert(CurCap->capturesVariable() &&
"Expected capture by reference.");
387 return C.getLValueReferenceType(
393 if (
const auto *VLA = dyn_cast<VariableArrayType>(A))
395 if (!A->isVariablyModifiedType())
396 return C.getCanonicalType(T);
398 return C.getCanonicalParamType(T);
403struct FunctionOptions {
408 const bool UIntPtrCastRequired =
true;
411 const bool RegisterCastedArgsOnly =
false;
413 const StringRef FunctionName;
416 explicit FunctionOptions(
const CapturedStmt *S,
bool UIntPtrCastRequired,
417 bool RegisterCastedArgsOnly, StringRef FunctionName,
419 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
420 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
421 FunctionName(FunctionName), Loc(Loc) {}
427 llvm::MapVector<
const Decl *, std::pair<const VarDecl *, Address>>
429 llvm::DenseMap<
const Decl *, std::pair<const Expr *, llvm::Value *>>
431 llvm::Value *&CXXThisValue,
const FunctionOptions &FO) {
433 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
434 assert(CD->
hasBody() &&
"missing CapturedDecl body");
436 CXXThisValue =
nullptr;
446 auto I = FO.S->captures().begin();
448 if (!FO.UIntPtrCastRequired) {
468 if (FO.UIntPtrCastRequired &&
470 I->capturesVariableArrayType()))
473 if (I->capturesVariable() || I->capturesVariableByCopy()) {
474 CapVar = I->getCapturedVar();
476 }
else if (I->capturesThis()) {
479 assert(I->capturesVariableArrayType());
489 }
else if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
491 Ctx, DebugFunctionDecl,
492 CapVar ? CapVar->
getBeginLoc() : FD->getBeginLoc(),
493 CapVar ? CapVar->
getLocation() : FD->getLocation(), II, ArgType,
499 Args.emplace_back(Arg);
501 TargetArgs.emplace_back(
502 FO.UIntPtrCastRequired
519 llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
523 F->setDoesNotThrow();
524 F->setDoesNotRecurse();
528 F->removeFnAttr(llvm::Attribute::NoInline);
529 F->addFnAttr(llvm::Attribute::AlwaysInline);
534 FO.UIntPtrCastRequired ? FO.Loc : FO.S->getBeginLoc(),
535 FO.UIntPtrCastRequired ? FO.Loc
538 I = FO.S->captures().begin();
542 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
550 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
551 const VarDecl *CurVD = I->getCapturedVar();
552 if (!FO.RegisterCastedArgsOnly)
553 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
561 if (FD->hasCapturedVLAType()) {
562 if (FO.UIntPtrCastRequired) {
565 Args[Cnt]->getName(), ArgLVal),
570 VLASizes.try_emplace(Args[Cnt], VAT->
getSizeExpr(), ExprArg);
571 }
else if (I->capturesVariable()) {
572 const VarDecl *Var = I->getCapturedVar();
582 if (!FO.RegisterCastedArgsOnly) {
586 }
else if (I->capturesVariableByCopy()) {
587 assert(!FD->getType()->isAnyPointerType() &&
588 "Not expecting a captured pointer.");
589 const VarDecl *Var = I->getCapturedVar();
590 LocalAddrs.insert({Args[Cnt],
591 {Var, FO.UIntPtrCastRequired
593 CGF, I->getLocation(), FD->getType(),
594 Args[Cnt]->getName(), ArgLVal)
598 assert(I->capturesThis());
600 LocalAddrs.insert({Args[Cnt], {
nullptr, ArgLVal.
getAddress(CGF)}});
614 "CapturedStmtInfo should be set when generating the captured function");
617 bool NeedWrapperFunction =
620 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
621 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
623 llvm::raw_svector_ostream Out(Buffer);
625 if (NeedWrapperFunction)
627 FunctionOptions FO(&S, !NeedWrapperFunction,
false,
630 VLASizes, CXXThisValue, FO);
631 CodeGenFunction::OMPPrivateScope LocalScope(*
this);
632 for (
const auto &LocalAddrPair : LocalAddrs) {
633 if (LocalAddrPair.second.first) {
634 LocalScope.addPrivate(LocalAddrPair.second.first,
635 LocalAddrPair.second.second);
638 (void)LocalScope.Privatize();
639 for (
const auto &VLASizePair : VLASizes)
640 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
643 (void)LocalScope.ForceCleanup();
645 if (!NeedWrapperFunction)
648 FunctionOptions WrapperFO(&S,
true,
656 llvm::Function *WrapperF =
658 WrapperCGF.CXXThisValue, WrapperFO);
660 auto *PI = F->arg_begin();
661 for (
const auto *Arg : Args) {
663 auto I = LocalAddrs.find(Arg);
664 if (I != LocalAddrs.end()) {
665 LValue LV = WrapperCGF.MakeAddrLValue(
667 I->second.first ? I->second.first->getType() : Arg->getType(),
670 LV.
setAddress(WrapperCGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
675 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
677 auto EI = VLASizes.find(Arg);
678 if (EI != VLASizes.end()) {
682 WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
684 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
687 CallArgs.emplace_back(WrapperCGF.EmitFromMemory(
CallArg, Arg->
getType()));
691 WrapperCGF.FinishFunction();
706 llvm::Value *NumElements =
emitArrayLength(ArrayTy, ElementTy, DestAddr);
710 llvm::Value *DestBegin = DestAddr.
getPointer();
713 DestBegin, NumElements);
718 llvm::Value *IsEmpty =
719 Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arraycpy.isempty");
720 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
723 llvm::BasicBlock *EntryBB =
Builder.GetInsertBlock();
728 llvm::PHINode *SrcElementPHI =
729 Builder.CreatePHI(SrcBegin->getType(), 2,
"omp.arraycpy.srcElementPast");
730 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
735 llvm::PHINode *DestElementPHI =
Builder.CreatePHI(
736 DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
737 DestElementPHI->addIncoming(DestBegin, EntryBB);
743 CopyGen(DestElementCurrent, SrcElementCurrent);
746 llvm::Value *DestElementNext =
748 1,
"omp.arraycpy.dest.element");
749 llvm::Value *SrcElementNext =
751 1,
"omp.arraycpy.src.element");
754 Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
755 Builder.CreateCondBr(Done, DoneBB, BodyBB);
756 DestElementPHI->addIncoming(DestElementNext,
Builder.GetInsertBlock());
757 SrcElementPHI->addIncoming(SrcElementNext,
Builder.GetInsertBlock());
767 const auto *BO = dyn_cast<BinaryOperator>(
Copy);
768 if (BO && BO->getOpcode() == BO_Assign) {
777 DestAddr, SrcAddr, OriginalType,
782 CodeGenFunction::OMPPrivateScope Remap(*
this);
783 Remap.addPrivate(DestVD, DestElement);
784 Remap.addPrivate(SrcVD, SrcElement);
785 (void)Remap.Privatize();
791 CodeGenFunction::OMPPrivateScope Remap(*
this);
792 Remap.addPrivate(SrcVD, SrcAddr);
793 Remap.addPrivate(DestVD, DestAddr);
794 (void)Remap.Privatize();
801 OMPPrivateScope &PrivateScope) {
804 bool DeviceConstTarget =
807 bool FirstprivateIsLastprivate =
false;
808 llvm::DenseMap<const VarDecl *, OpenMPLastprivateModifier> Lastprivates;
810 for (
const auto *D :
C->varlists())
811 Lastprivates.try_emplace(
820 bool MustEmitFirstprivateCopy =
821 CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
823 const auto *IRef =
C->varlist_begin();
824 const auto *InitsRef =
C->inits().begin();
825 for (
const Expr *IInit :
C->private_copies()) {
826 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
827 bool ThisFirstprivateIsLastprivate =
828 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
830 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
831 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
833 (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())) {
834 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
841 if (DeviceConstTarget && OrigVD->getType().isConstant(
getContext()) &&
843 (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())) {
844 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
849 FirstprivateIsLastprivate =
850 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
851 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
853 cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
862 if (CE && !CE.isReference()) {
868 if (CE && CE.isReference()) {
869 OriginalLVal = CE.getReferenceLValue(*
this, &DRE);
871 assert(!CE &&
"Expected non-constant firstprivate.");
890 Emission.getAllocatedAddress(), OriginalLVal.
getAddress(*
this),
895 RunCleanupsScope InitScope(*this);
897 setAddrOfLocalVar(VDInit, SrcElement);
898 EmitAnyExprToMem(Init, DestElement,
899 Init->getType().getQualifiers(),
901 LocalDeclMap.erase(VDInit);
906 PrivateScope.addPrivate(OrigVD, Emission.getAllocatedAddress());
912 setAddrOfLocalVar(VDInit, OriginalAddr);
914 LocalDeclMap.erase(VDInit);
916 if (ThisFirstprivateIsLastprivate &&
917 Lastprivates[OrigVD->getCanonicalDecl()] ==
918 OMPC_LASTPRIVATE_conditional) {
923 (*IRef)->getExprLoc());
928 LocalDeclMap.erase(VD);
929 setAddrOfLocalVar(VD, VDAddr);
931 IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);
933 assert(IsRegistered &&
934 "firstprivate var already registered as private");
942 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
947 CodeGenFunction::OMPPrivateScope &PrivateScope) {
952 auto IRef =
C->varlist_begin();
953 for (
const Expr *IInit :
C->private_copies()) {
954 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
955 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
956 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
961 assert(IsRegistered &&
"private var already registered as private");
978 llvm::BasicBlock *CopyBegin =
nullptr, *CopyEnd =
nullptr;
980 auto IRef =
C->varlist_begin();
981 auto ISrcRef =
C->source_exprs().begin();
982 auto IDestRef =
C->destination_exprs().begin();
983 for (
const Expr *AssignOp :
C->assignment_ops()) {
984 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
992 getContext().getTargetInfo().isTLSSupported()) {
994 "Copyin threadprivates should have been captured!");
998 LocalDeclMap.erase(VD);
1008 if (CopiedVars.size() == 1) {
1014 auto *MasterAddrInt =
1016 auto *PrivateAddrInt =
1019 Builder.CreateICmpNE(MasterAddrInt, PrivateAddrInt), CopyBegin,
1024 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1025 const auto *DestVD =
1026 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1046 bool HasAtLeastOneLastprivate =
false;
1049 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1050 for (
const Expr *
C : LoopDirective->counters()) {
1057 HasAtLeastOneLastprivate =
true;
1061 const auto *IRef =
C->varlist_begin();
1062 const auto *IDestRef =
C->destination_exprs().begin();
1063 for (
const Expr *IInit :
C->private_copies()) {
1066 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1069 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
1070 const auto *DestVD =
1071 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1076 PrivateScope.addPrivate(DestVD,
EmitLValue(&DRE).getAddress(*
this));
1080 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
1081 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
1083 if (
C->getKind() == OMPC_LASTPRIVATE_conditional) {
1086 setAddrOfLocalVar(VD, VDAddr);
1092 bool IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);
1093 assert(IsRegistered &&
1094 "lastprivate var already registered as private");
1102 return HasAtLeastOneLastprivate;
1107 llvm::Value *IsLastIterCond) {
1116 llvm::BasicBlock *ThenBB =
nullptr;
1117 llvm::BasicBlock *DoneBB =
nullptr;
1118 if (IsLastIterCond) {
1124 return C->getKind() == OMPC_LASTPRIVATE_conditional;
1133 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1137 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1138 if (
const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1139 auto IC = LoopDirective->counters().begin();
1140 for (
const Expr *F : LoopDirective->finals()) {
1142 cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1144 AlreadyEmittedVars.insert(D);
1146 LoopCountersAndUpdates[D] = F;
1151 auto IRef =
C->varlist_begin();
1152 auto ISrcRef =
C->source_exprs().begin();
1153 auto IDestRef =
C->destination_exprs().begin();
1154 for (
const Expr *AssignOp :
C->assignment_ops()) {
1155 const auto *PrivateVD =
1156 cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1158 const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1159 if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1163 if (
const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1166 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1167 const auto *DestVD =
1168 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1171 if (
const auto *RefTy = PrivateVD->getType()->getAs<
ReferenceType>())
1177 if (
C->getKind() == OMPC_LASTPRIVATE_conditional)
1179 *
this,
MakeAddrLValue(PrivateAddr, (*IRef)->getType()), PrivateVD,
1180 (*IRef)->getExprLoc());
1183 EmitOMPCopy(
Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1189 if (
const Expr *PostUpdate =
C->getPostUpdateExpr())
1198 CodeGenFunction::OMPPrivateScope &PrivateScope,
bool ForInscan) {
1210 if (ForInscan != (
C->getModifier() == OMPC_REDUCTION_inscan))
1212 Shareds.append(
C->varlist_begin(),
C->varlist_end());
1213 Privates.append(
C->privates().begin(),
C->privates().end());
1214 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
1215 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
1216 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
1217 if (
C->getModifier() == OMPC_REDUCTION_task) {
1218 Data.ReductionVars.append(
C->privates().begin(),
C->privates().end());
1219 Data.ReductionOrigs.append(
C->varlist_begin(),
C->varlist_end());
1220 Data.ReductionCopies.append(
C->privates().begin(),
C->privates().end());
1221 Data.ReductionOps.append(
C->reduction_ops().begin(),
1222 C->reduction_ops().end());
1223 TaskLHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
1224 TaskRHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
1229 auto *ILHS = LHSs.begin();
1230 auto *IRHS = RHSs.begin();
1231 auto *IPriv = Privates.begin();
1232 for (
const Expr *IRef : Shareds) {
1233 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1235 RedCG.emitSharedOrigLValue(*
this, Count);
1236 RedCG.emitAggregateType(*
this, Count);
1238 RedCG.emitInitialization(*
this, Count, Emission.getAllocatedAddress(),
1239 RedCG.getSharedLValue(Count).getAddress(*
this),
1241 CGF.EmitAutoVarInit(Emission);
1245 Address BaseAddr = RedCG.adjustPrivateAddress(
1246 *
this, Count, Emission.getAllocatedAddress());
1248 PrivateScope.addPrivate(RedCG.getBaseDecl(Count), BaseAddr);
1249 assert(IsRegistered &&
"private var already registered as private");
1253 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1254 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1256 bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1260 PrivateScope.addPrivate(LHSVD,
1261 RedCG.getSharedLValue(Count).getAddress(*
this));
1264 isa<ArraySubscriptExpr>(IRef)) {
1267 PrivateScope.addPrivate(LHSVD,
1268 RedCG.getSharedLValue(Count).getAddress(*
this));
1269 PrivateScope.addPrivate(RHSVD,
1275 Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress(*
this);
1282 PrivateScope.addPrivate(LHSVD, OriginalAddr);
1283 PrivateScope.addPrivate(
1293 if (!
Data.ReductionVars.empty()) {
1294 Data.IsReductionWithTaskMod =
true;
1295 Data.IsWorksharingReduction =
1299 const Expr *TaskRedRef =
nullptr;
1302 TaskRedRef = cast<OMPParallelDirective>(D).getTaskReductionRefExpr();
1305 TaskRedRef = cast<OMPForDirective>(D).getTaskReductionRefExpr();
1308 TaskRedRef = cast<OMPSectionsDirective>(D).getTaskReductionRefExpr();
1310 case OMPD_parallel_for:
1311 TaskRedRef = cast<OMPParallelForDirective>(D).getTaskReductionRefExpr();
1313 case OMPD_parallel_master:
1315 cast<OMPParallelMasterDirective>(D).getTaskReductionRefExpr();
1317 case OMPD_parallel_sections:
1319 cast<OMPParallelSectionsDirective>(D).getTaskReductionRefExpr();
1321 case OMPD_target_parallel:
1323 cast<OMPTargetParallelDirective>(D).getTaskReductionRefExpr();
1325 case OMPD_target_parallel_for:
1327 cast<OMPTargetParallelForDirective>(D).getTaskReductionRefExpr();
1329 case OMPD_distribute_parallel_for:
1331 cast<OMPDistributeParallelForDirective>(D).getTaskReductionRefExpr();
1333 case OMPD_teams_distribute_parallel_for:
1334 TaskRedRef = cast<OMPTeamsDistributeParallelForDirective>(D)
1335 .getTaskReductionRefExpr();
1337 case OMPD_target_teams_distribute_parallel_for:
1338 TaskRedRef = cast<OMPTargetTeamsDistributeParallelForDirective>(D)
1339 .getTaskReductionRefExpr();
1347 case OMPD_parallel_for_simd:
1349 case OMPD_taskyield:
1353 case OMPD_taskgroup:
1361 case OMPD_cancellation_point:
1363 case OMPD_target_data:
1364 case OMPD_target_enter_data:
1365 case OMPD_target_exit_data:
1367 case OMPD_taskloop_simd:
1368 case OMPD_master_taskloop:
1369 case OMPD_master_taskloop_simd:
1370 case OMPD_parallel_master_taskloop:
1371 case OMPD_parallel_master_taskloop_simd:
1372 case OMPD_distribute:
1373 case OMPD_target_update:
1374 case OMPD_distribute_parallel_for_simd:
1375 case OMPD_distribute_simd:
1376 case OMPD_target_parallel_for_simd:
1377 case OMPD_target_simd:
1378 case OMPD_teams_distribute:
1379 case OMPD_teams_distribute_simd:
1380 case OMPD_teams_distribute_parallel_for_simd:
1381 case OMPD_target_teams:
1382 case OMPD_target_teams_distribute:
1383 case OMPD_target_teams_distribute_parallel_for_simd:
1384 case OMPD_target_teams_distribute_simd:
1385 case OMPD_declare_target:
1386 case OMPD_end_declare_target:
1387 case OMPD_threadprivate:
1389 case OMPD_declare_reduction:
1390 case OMPD_declare_mapper:
1391 case OMPD_declare_simd:
1393 case OMPD_declare_variant:
1394 case OMPD_begin_declare_variant:
1395 case OMPD_end_declare_variant:
1398 llvm_unreachable(
"Enexpected directive with task reductions.");
1401 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(TaskRedRef)->getDecl());
1404 false, TaskRedRef->
getType());
1416 bool HasAtLeastOneReduction =
false;
1417 bool IsReductionWithTaskMod =
false;
1420 if (
C->getModifier() == OMPC_REDUCTION_inscan)
1422 HasAtLeastOneReduction =
true;
1423 Privates.append(
C->privates().begin(),
C->privates().end());
1424 LHSExprs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
1425 RHSExprs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
1426 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
1427 IsReductionWithTaskMod =
1428 IsReductionWithTaskMod ||
C->getModifier() == OMPC_REDUCTION_task;
1430 if (HasAtLeastOneReduction) {
1431 if (IsReductionWithTaskMod) {
1438 ReductionKind == OMPD_simd;
1439 bool SimpleReduction = ReductionKind == OMPD_simd;
1443 *
this, D.
getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1444 {WithNowait, SimpleReduction, ReductionKind});
1453 llvm::BasicBlock *DoneBB =
nullptr;
1455 if (
const Expr *PostUpdate =
C->getPostUpdateExpr()) {
1457 if (llvm::Value *Cond = CondGen(CGF)) {
1462 CGF.
Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1480 CodeGenBoundParametersTy;
1490 for (
const Expr *Ref :
C->varlists()) {
1491 if (!Ref->getType()->isScalarType())
1493 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1496 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1501 for (
const Expr *Ref :
C->varlists()) {
1502 if (!Ref->getType()->isScalarType())
1504 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1507 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1512 for (
const Expr *Ref :
C->varlists()) {
1513 if (!Ref->getType()->isScalarType())
1515 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1518 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1527 for (
const Expr *Ref :
C->varlists()) {
1528 if (!Ref->getType()->isScalarType())
1530 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1533 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1537 CGF, S, PrivateDecls);
1543 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1544 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1545 llvm::Value *NumThreads =
nullptr;
1546 llvm::Function *OutlinedFn =
1551 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1552 NumThreads = CGF.
EmitScalarExpr(NumThreadsClause->getNumThreads(),
1555 CGF, NumThreads, NumThreadsClause->getBeginLoc());
1558 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1560 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1562 const Expr *IfCond =
nullptr;
1563 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
1564 if (
C->getNameModifier() == OMPD_unknown ||
1565 C->getNameModifier() == OMPD_parallel) {
1566 IfCond =
C->getCondition();
1571 OMPParallelScope
Scope(CGF, S);
1577 CodeGenBoundParameters(CGF, S, CapturedVars);
1580 CapturedVars, IfCond, NumThreads);
1585 if (!CVD->
hasAttr<OMPAllocateDeclAttr>())
1587 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
1589 return !((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||
1590 AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&
1591 !AA->getAllocator());
1606 CGF, S.getBeginLoc(), OMPD_unknown,
false,
1626 Size = CGF.
Builder.CreateNUWAdd(
1635 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
1636 assert(AA->getAllocator() &&
1637 "Expected allocator expression for non-default allocator.");
1641 if (Allocator->getType()->isIntegerTy())
1643 else if (Allocator->getType()->isPointerTy())
1647 llvm::Value *Addr = OMPBuilder.createOMPAlloc(
1650 llvm::CallInst *FreeCI =
1651 OMPBuilder.createOMPFree(CGF.
Builder, Addr, Allocator);
1675 std::string Suffix = getNameWithSeparators({
"cache",
""});
1678 llvm::CallInst *ThreadPrivateCacheCall =
1679 OMPBuilder.createCachedThreadPrivate(CGF.
Builder,
Data, Size, CacheName);
1687 llvm::raw_svector_ostream OS(Buffer);
1688 StringRef Sep = FirstSeparator;
1689 for (StringRef Part : Parts) {
1693 return OS.str().str();
1701 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
Builder,
false,
1702 "." + RegionName +
".after");
1718 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
Builder,
false,
1719 "." + RegionName +
".after");
1734 llvm::Value *IfCond =
nullptr;
1739 llvm::Value *NumThreads =
nullptr;
1744 ProcBindKind ProcBind = OMP_PROC_BIND_default;
1746 ProcBind = ProcBindClause->getProcBindKind();
1748 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1752 auto FiniCB = [
this](InsertPointTy IP) {
1760 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
1761 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
1769 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1772 auto BodyGenCB = [&,
this](InsertPointTy AllocaIP,
1773 InsertPointTy CodeGenIP) {
1775 *
this, ParallelRegionBodyStmt, AllocaIP, CodeGenIP,
"parallel");
1778 CGCapturedStmtInfo CGSI(*CS,
CR_OpenMP);
1779 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*
this, &CGSI);
1780 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
1783 OMPBuilder.createParallel(
Builder, AllocaIP, BodyGenCB, PrivCB, FiniCB,
1784 IfCond, NumThreads, ProcBind, S.hasCancel()));
1791 OMPPrivateScope PrivateScope(CGF);
1796 (void)PrivateScope.Privatize();
1797 CGF.
EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1818class OMPTransformDirectiveScopeRAII {
1819 OMPLoopScope *
Scope =
nullptr;
1820 CodeGenFunction::CGCapturedStmtInfo *CGSI =
nullptr;
1821 CodeGenFunction::CGCapturedStmtRAII *CapInfoRAII =
nullptr;
1823 OMPTransformDirectiveScopeRAII(
const OMPTransformDirectiveScopeRAII &) =
1825 OMPTransformDirectiveScopeRAII &
1826 operator=(
const OMPTransformDirectiveScopeRAII &) =
delete;
1830 if (
const auto *Dir = dyn_cast<OMPLoopBasedDirective>(S)) {
1831 Scope =
new OMPLoopScope(CGF, *Dir);
1832 CGSI =
new CodeGenFunction::CGCapturedStmtInfo(
CR_OpenMP);
1833 CapInfoRAII =
new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI);
1836 ~OMPTransformDirectiveScopeRAII() {
1847 int MaxLevel,
int Level = 0) {
1848 assert(Level < MaxLevel &&
"Too deep lookup during loop body codegen.");
1849 const Stmt *SimplifiedS = S->IgnoreContainers();
1850 if (
const auto *CS = dyn_cast<CompoundStmt>(SimplifiedS)) {
1853 "LLVM IR generation of compound statement ('{}')");
1856 CodeGenFunction::LexicalScope
Scope(CGF, S->getSourceRange());
1857 for (
const Stmt *CurStmt : CS->body())
1858 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level);
1861 if (SimplifiedS == NextLoop) {
1862 if (
auto *Dir = dyn_cast<OMPLoopTransformationDirective>(SimplifiedS))
1863 SimplifiedS = Dir->getTransformedStmt();
1864 if (
const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(SimplifiedS))
1865 SimplifiedS = CanonLoop->getLoopStmt();
1866 if (
const auto *For = dyn_cast<ForStmt>(SimplifiedS)) {
1869 assert(isa<CXXForRangeStmt>(SimplifiedS) &&
1870 "Expected canonical for loop or range-based for loop.");
1871 const auto *CXXFor = cast<CXXForRangeStmt>(SimplifiedS);
1872 CGF.
EmitStmt(CXXFor->getLoopVarStmt());
1873 S = CXXFor->getBody();
1875 if (Level + 1 < MaxLevel) {
1878 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1);
1887 RunCleanupsScope BodyScope(*
this);
1896 for (
const Expr *UE :
C->updates())
1903 BreakContinueStack.push_back(BreakContinue(
LoopExit, Continue));
1915 OMPPrivateScope InscanScope(*
this);
1917 bool IsInscanRegion = InscanScope.Privatize();
1918 if (IsInscanRegion) {
1950 BreakContinueStack.pop_back();
1961 std::unique_ptr<CodeGenFunction::CGCapturedStmtInfo> CSI =
1962 std::make_unique<CodeGenFunction::CGCapturedStmtInfo>(*S);
1963 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, CSI.get());
1970static llvm::CallInst *
1975 EffectiveArgs.reserve(Args.size() + 1);
1976 llvm::append_range(EffectiveArgs, Args);
1977 EffectiveArgs.push_back(Cap.second);
1982llvm::CanonicalLoopInfo *
1984 assert(Depth == 1 &&
"Nested loops with OpenMPIRBuilder not yet implemented");
2010 const Stmt *SyntacticalLoop = S->getLoopStmt();
2017 LexicalScope ForScope(*
this, S->getSourceRange());
2021 const Stmt *BodyStmt;
2022 if (
const auto *For = dyn_cast<ForStmt>(SyntacticalLoop)) {
2023 if (
const Stmt *InitStmt = For->getInit())
2025 BodyStmt = For->getBody();
2026 }
else if (
const auto *RangeFor =
2027 dyn_cast<CXXForRangeStmt>(SyntacticalLoop)) {
2028 if (
const DeclStmt *RangeStmt = RangeFor->getRangeStmt())
2030 if (
const DeclStmt *BeginStmt = RangeFor->getBeginStmt())
2032 if (
const DeclStmt *EndStmt = RangeFor->getEndStmt())
2034 if (
const DeclStmt *LoopVarStmt = RangeFor->getLoopVarStmt())
2036 BodyStmt = RangeFor->getBody();
2038 llvm_unreachable(
"Expected for-stmt or range-based for-stmt");
2041 const CapturedStmt *DistanceFunc = S->getDistanceFunc();
2058 auto BodyGen = [&,
this](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP,
2059 llvm::Value *IndVar) {
2064 const DeclRefExpr *LoopVarRef = S->getLoopVarRef();
2070 RunCleanupsScope BodyScope(*
this);
2073 llvm::CanonicalLoopInfo *CL =
2074 OMPBuilder.createCanonicalLoop(
Builder, BodyGen, DistVal);
2077 Builder.restoreIP(CL->getAfterIP());
2078 ForScope.ForceCleanup();
2086 const Expr *IncExpr,
2097 const auto &OMPED = cast<OMPExecutableDirective>(S);
2098 const CapturedStmt *ICS = OMPED.getInnermostCapturedStmt();
2112 llvm::BasicBlock *ExitBlock =
LoopExit.getBlock();
2113 if (RequiresCleanup)
2120 if (ExitBlock !=
LoopExit.getBlock()) {
2130 BreakContinueStack.push_back(BreakContinue(
LoopExit, Continue));
2138 BreakContinueStack.pop_back();
2149 bool HasLinears =
false;
2151 for (
const Expr *Init :
C->inits()) {
2153 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
2154 if (
const auto *Ref =
2157 const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
2173 if (
const auto *CS = cast_or_null<BinaryOperator>(
C->getCalcStep()))
2174 if (
const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
2188 llvm::BasicBlock *DoneBB =
nullptr;
2191 auto IC =
C->varlist_begin();
2192 for (
const Expr *F :
C->finals()) {
2194 if (llvm::Value *Cond = CondGen(*
this)) {
2199 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2203 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
2208 CodeGenFunction::OMPPrivateScope VarScope(*
this);
2209 VarScope.addPrivate(OrigVD, OrigAddr);
2210 (void)VarScope.Privatize();
2214 if (
const Expr *PostUpdate =
C->getPostUpdateExpr())
2226 llvm::APInt ClauseAlignment(64, 0);
2227 if (
const Expr *AlignmentExpr = Clause->getAlignment()) {
2230 ClauseAlignment = AlignmentCI->getValue();
2232 for (
const Expr *E : Clause->varlists()) {
2233 llvm::APInt Alignment(ClauseAlignment);
2234 if (Alignment == 0) {
2244 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2245 "alignment is not power of 2");
2246 if (Alignment != 0) {
2260 auto I = S.private_counters().begin();
2261 for (
const Expr *E : S.counters()) {
2262 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2263 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
2267 LocalDeclMap.erase(PrivateVD);
2268 (void)LoopScope.addPrivate(VD, VarEmission.getAllocatedAddress());
2276 (void)LoopScope.addPrivate(PrivateVD, VarEmission.getAllocatedAddress());
2282 if (!
C->getNumForLoops())
2284 for (
unsigned I = S.getLoopsNumber(), E =
C->getLoopNumIterations().size();
2286 const auto *DRE = cast<DeclRefExpr>(
C->getLoopCounter(I));
2287 const auto *VD = cast<VarDecl>(DRE->getDecl());
2290 if (DRE->refersToEnclosingVariableOrCapture()) {
2291 (void)LoopScope.addPrivate(
2299 const Expr *Cond, llvm::BasicBlock *TrueBlock,
2300 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
2304 CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
2306 (void)PreCondScope.Privatize();
2308 for (
const Expr *I : S.inits()) {
2314 CodeGenFunction::OMPMapVars PreCondVars;
2315 for (
const Expr *E : S.dependent_counters()) {
2318 assert(!E->getType().getNonReferenceType()->isRecordType() &&
2319 "dependent counter must not be an iterator.");
2320 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2323 (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr);
2325 (void)PreCondVars.apply(CGF);
2326 for (
const Expr *E : S.dependent_inits()) {
2333 PreCondVars.restore(CGF);
2337 const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
2342 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
2343 for (
const Expr *
C : LoopDirective->counters()) {
2349 auto CurPrivate =
C->privates().begin();
2350 for (
const Expr *E :
C->varlists()) {
2351 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2352 const auto *PrivateVD =
2353 cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
2359 assert(IsRegistered &&
"linear var already registered as private");
2377 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
2386 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
2401 if (
C->getKind() == OMPC_ORDER_concurrent)
2408 return C->getModifier() == OMPC_REDUCTION_inscan;
2419 llvm::BasicBlock *DoneBB =
nullptr;
2423 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
2424 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
2425 const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
2427 OrigVD->hasGlobalStorage() || CED) {
2429 if (llvm::Value *Cond = CondGen(*
this)) {
2434 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2448 OMPPrivateScope VarScope(*
this);
2449 VarScope.addPrivate(OrigVD, OrigAddr);
2450 (void)VarScope.Privatize();
2462 CodeGenFunction::JumpDest
LoopExit) {
2470 auto VDecl = cast<VarDecl>(Helper->
getDecl());
2478 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](
CodeGenFunction &CGF,
2481 CodeGenFunction::OMPLocalDeclMapRAII
Scope(CGF);
2487 CodeGenFunction::OMPLocalDeclMapRAII
Scope(CGF);
2492 const Expr *IfCond =
nullptr;
2494 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
2496 (
C->getNameModifier() == OMPD_unknown ||
2497 C->getNameModifier() == OMPD_simd)) {
2498 IfCond =
C->getCondition();
2515 "Expected simd directive");
2516 OMPLoopScope PreInitScope(CGF, S);
2533 llvm::BasicBlock *ContBlock =
nullptr;
2540 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
2547 const Expr *IVExpr = S.getIterationVariable();
2548 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
2555 if (
const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2556 CGF.
EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2564 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2570 CGF, S, CGF.
EmitLValue(S.getIterationVariable()));
2572 (void)LoopScope.Privatize();
2583 S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),
2585 emitOMPLoopBodyWithStopPoint(CGF, S,
2586 CodeGenFunction::JumpDest());
2592 if (HasLastprivateClause)
2597 LoopScope.restoreMap();
2611 if (!(isa<OMPSimdlenClause>(
C) || isa<OMPSafelenClause>(
C) ||
2612 isa<OMPOrderClause>(
C) || isa<OMPAlignedClause>(
C)))
2619 if (
const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(S.getRawStmt())) {
2620 if (
const Stmt *SyntacticalLoop = CanonLoop->getLoopStmt()) {
2621 for (
const Stmt *SubStmt : SyntacticalLoop->
children()) {
2624 if (
const CompoundStmt *CS = dyn_cast<CompoundStmt>(SubStmt)) {
2628 if (isa<OMPOrderedDirective>(CSSubStmt)) {
2638static llvm::MapVector<llvm::Value *, llvm::Value *>
2640 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars;
2642 llvm::APInt ClauseAlignment(64, 0);
2643 if (
const Expr *AlignmentExpr = Clause->getAlignment()) {
2646 ClauseAlignment = AlignmentCI->getValue();
2648 for (
const Expr *E : Clause->varlists()) {
2649 llvm::APInt Alignment(ClauseAlignment);
2650 if (Alignment == 0) {
2657 E->getType()->getPointeeType()))
2660 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2661 "alignment is not power of 2");
2663 AlignedVars[PtrValue] = CGF.
Builder.getInt64(Alignment.getSExtValue());
2670 bool UseOMPIRBuilder =
2672 if (UseOMPIRBuilder) {
2673 auto &&CodeGenIRBuilder = [
this, &S, UseOMPIRBuilder](
CodeGenFunction &CGF,
2676 if (UseOMPIRBuilder) {
2677 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars =
2680 const Stmt *Inner = S.getRawStmt();
2681 llvm::CanonicalLoopInfo *CLI =
2684 llvm::OpenMPIRBuilder &OMPBuilder =
2687 llvm::ConstantInt *Simdlen =
nullptr;
2692 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
2695 llvm::ConstantInt *Safelen =
nullptr;
2700 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
2703 llvm::omp::OrderKind Order = llvm::omp::OrderKind::OMP_ORDER_unknown;
2705 if (
C->getKind() == OpenMPOrderClauseKind ::OMPC_ORDER_concurrent) {
2706 Order = llvm::omp::OrderKind::OMP_ORDER_concurrent;
2711 OMPBuilder.applySimd(CLI, AlignedVars,
2712 nullptr, Order, Simdlen, Safelen);
2719 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
2726 ParentLoopDirectiveForScanRegion ScanRegion(*
this, S);
2734 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
2743 OMPTransformDirectiveScopeRAII TileScope(*
this, &S);
2750 if (UseOMPIRBuilder) {
2752 const Stmt *Inner = S.getRawStmt();
2763 llvm::CanonicalLoopInfo *UnrolledCLI =
nullptr;
2767 OMPBuilder.unrollLoopFull(DL, CLI);
2770 if (
Expr *FactorExpr = PartialClause->getFactor()) {
2771 Factor = FactorExpr->EvaluateKnownConstInt(
getContext()).getZExtValue();
2772 assert(Factor >= 1 &&
"Only positive factors are valid");
2774 OMPBuilder.unrollLoopPartial(DL, CLI, Factor,
2775 NeedsUnrolledCLI ? &UnrolledCLI :
nullptr);
2777 OMPBuilder.unrollLoopHeuristic(DL, CLI);
2780 assert((!NeedsUnrolledCLI || UnrolledCLI) &&
2781 "NeedsUnrolledCLI implies UnrolledCLI to be set");
2798 if (
Expr *FactorExpr = PartialClause->getFactor()) {
2800 FactorExpr->EvaluateKnownConstInt(
getContext()).getZExtValue();
2801 assert(Factor >= 1 &&
"Only positive factors are valid");
2809void CodeGenFunction::EmitOMPOuterLoop(
2811 CodeGenFunction::OMPPrivateScope &LoopScope,
2812 const CodeGenFunction::OMPLoopArguments &LoopArgs,
2817 const Expr *IVExpr = S.getIterationVariable();
2831 llvm::Value *BoolCondVal =
nullptr;
2832 if (!DynamicOrOrdered) {
2843 RT.
emitForNext(*
this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
2844 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
2849 llvm::BasicBlock *ExitBlock =
LoopExit.getBlock();
2850 if (LoopScope.requiresCleanups())
2854 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
2855 if (ExitBlock !=
LoopExit.getBlock()) {
2863 if (DynamicOrOrdered)
2868 BreakContinueStack.push_back(BreakContinue(
LoopExit, Continue));
2876 CGF.LoopStack.setParallel(!IsMonotonic);
2877 if (const auto *C = S.getSingleClause<OMPOrderClause>())
2878 if (C->getKind() == OMPC_ORDER_concurrent)
2879 CGF.LoopStack.setParallel(true);
2881 CGF.EmitOMPSimdInit(S);
2884 [&S, &LoopArgs,
LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered,
2892 CGF.EmitOMPInnerLoop(
2893 S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
2895 CodeGenLoop(CGF, S, LoopExit);
2898 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
2903 BreakContinueStack.pop_back();
2904 if (!DynamicOrOrdered) {
2918 if (!DynamicOrOrdered)
2919 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2920 S.getDirectiveKind());
2922 OMPCancelStack.emitExit(*
this, S.getDirectiveKind(), CodeGen);
2925void CodeGenFunction::EmitOMPForOuterLoop(
2928 const OMPLoopArguments &LoopArgs,
2936 LoopArgs.Chunk !=
nullptr)) &&
2937 "static non-chunked schedule does not need outer loop");
2989 const Expr *IVExpr = S.getIterationVariable();
2993 if (DynamicOrOrdered) {
2994 const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
2995 CGDispatchBounds(*
this, S, LoopArgs.LB, LoopArgs.UB);
2996 llvm::Value *LBVal = DispatchBounds.first;
2997 llvm::Value *UBVal = DispatchBounds.second;
3001 IVSigned, Ordered, DipatchRTInputValues);
3004 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
3005 LoopArgs.ST, LoopArgs.Chunk);
3007 ScheduleKind, StaticInit);
3011 const unsigned IVSize,
3012 const bool IVSigned) {
3019 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
3020 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
3021 OuterLoopArgs.IncExpr = S.getInc();
3022 OuterLoopArgs.Init = S.getInit();
3023 OuterLoopArgs.Cond = S.getCond();
3024 OuterLoopArgs.NextLB = S.getNextLowerBound();
3025 OuterLoopArgs.NextUB = S.getNextUpperBound();
3026 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
3031 const unsigned IVSize,
const bool IVSigned) {}
3033void CodeGenFunction::EmitOMPDistributeOuterLoop(
3035 OMPPrivateScope &LoopScope,
const OMPLoopArguments &LoopArgs,
3045 const Expr *IVExpr = S.getIterationVariable();
3050 IVSize, IVSigned,
false, LoopArgs.IL, LoopArgs.LB,
3051 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
3058 IncExpr = S.getDistInc();
3060 IncExpr = S.getInc();
3065 OMPLoopArguments OuterLoopArgs;
3066 OuterLoopArgs.LB = LoopArgs.LB;
3067 OuterLoopArgs.UB = LoopArgs.UB;
3068 OuterLoopArgs.ST = LoopArgs.ST;
3069 OuterLoopArgs.IL = LoopArgs.IL;
3070 OuterLoopArgs.Chunk = LoopArgs.Chunk;
3072 ? S.getCombinedEnsureUpperBound()
3073 : S.getEnsureUpperBound();
3074 OuterLoopArgs.IncExpr = IncExpr;
3076 ? S.getCombinedInit()
3079 ? S.getCombinedCond()
3082 ? S.getCombinedNextLowerBound()
3083 : S.getNextLowerBound();
3085 ? S.getCombinedNextUpperBound()
3086 : S.getNextUpperBound();
3088 EmitOMPOuterLoop(
false,
false, S,
3089 LoopScope, OuterLoopArgs, CodeGenLoopContent,
3093static std::pair<LValue, LValue>
3136static std::pair<llvm::Value *, llvm::Value *>
3147 llvm::Value *LBVal =
3149 llvm::Value *UBVal =
3151 return {LBVal, UBVal};
3157 const auto &Dir = cast<OMPLoopDirective>(S);
3159 CGF.
EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
3160 llvm::Value *LBCast =
3163 CapturedVars.push_back(LBCast);
3165 CGF.
EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
3167 llvm::Value *UBCast =
3170 CapturedVars.push_back(UBCast);
3176 CodeGenFunction::JumpDest
LoopExit) {
3180 bool HasCancel =
false;
3182 if (
const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
3183 HasCancel = D->hasCancel();
3184 else if (
const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
3185 HasCancel = D->hasCancel();
3186 else if (
const auto *D =
3187 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
3188 HasCancel = D->hasCancel();
3190 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
3200 CGInlinedWorksharingLoop,
3210 OMPLexicalScope
Scope(*
this, S, OMPD_parallel);
3220 OMPLexicalScope
Scope(*
this, S, OMPD_parallel);
3229 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
3240 llvm::Constant *Addr;
3243 S, ParentName, Fn, Addr,
true, CodeGen);
3244 assert(Fn && Addr &&
"Target device function emission failed.");
3256struct ScheduleKindModifiersTy {
3272 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3273 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3279 if (
const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3287 bool HasLastprivateClause;
3290 OMPLoopScope PreInitScope(*
this, S);
3295 llvm::BasicBlock *ContBlock =
nullptr;
3302 emitPreCond(*
this, S, S.getPreCond(), ThenBlock, ContBlock,
3308 RunCleanupsScope DoacrossCleanupScope(*
this);
3309 bool Ordered =
false;
3311 if (OrderedClause->getNumForLoops())
3322 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*
this, S);
3323 LValue LB = Bounds.first;
3324 LValue UB = Bounds.second;
3332 OMPPrivateScope LoopScope(*
this);
3338 *
this, S.getBeginLoc(), OMPD_unknown,
false,
3343 *
this, S,
EmitLValue(S.getIterationVariable()));
3348 (void)LoopScope.Privatize();
3353 const Expr *ChunkExpr =
nullptr;
3356 ScheduleKind.
Schedule =
C->getScheduleKind();
3357 ScheduleKind.
M1 =
C->getFirstScheduleModifier();
3358 ScheduleKind.
M2 =
C->getSecondScheduleModifier();
3359 ChunkExpr =
C->getChunkSize();
3363 *
this, S, ScheduleKind.
Schedule, ChunkExpr);
3365 bool HasChunkSizeOne =
false;
3366 llvm::Value *Chunk =
nullptr;
3370 S.getIterationVariable()->getType(),
3374 llvm::APSInt EvaluatedChunk =
Result.Val.getInt();
3375 HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
3384 bool StaticChunkedOne =
3386 Chunk !=
nullptr) &&
3391 (ScheduleKind.
Schedule == OMPC_SCHEDULE_static &&
3392 !(ScheduleKind.
M1 == OMPC_SCHEDULE_MODIFIER_nonmonotonic ||
3393 ScheduleKind.
M2 == OMPC_SCHEDULE_MODIFIER_nonmonotonic)) ||
3394 ScheduleKind.
M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
3395 ScheduleKind.
M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
3397 Chunk !=
nullptr) ||
3398 StaticChunkedOne) &&
3406 CGF.EmitOMPSimdInit(S);
3408 if (C->getKind() == OMPC_ORDER_concurrent)
3409 CGF.LoopStack.setParallel(true);
3412 [IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk,
3421 IVSize, IVSigned, Ordered, IL.getAddress(CGF),
3422 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF),
3423 StaticChunkedOne ? Chunk :
nullptr);
3424 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
3425 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind,
3428 if (!StaticChunkedOne)
3429 CGF.EmitIgnoredExpr(S.getEnsureUpperBound());
3431 CGF.EmitIgnoredExpr(S.getInit());
3445 CGF.EmitOMPInnerLoop(
3446 S, LoopScope.requiresCleanups(),
3447 StaticChunkedOne ? S.getCombinedParForInDistCond()
3449 StaticChunkedOne ? S.getDistInc() : S.getInc(),
3451 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit);
3458 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
3459 S.getDirectiveKind());
3461 OMPCancelStack.emitExit(*
this, S.getDirectiveKind(), CodeGen);
3465 const OMPLoopArguments LoopArguments(
3466 LB.getAddress(*
this), UB.getAddress(*
this), ST.getAddress(*
this),
3467 IL.getAddress(*
this), Chunk, EUB);
3468 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
3469 LoopArguments, CGDispatchBounds);
3473 return CGF.
Builder.CreateIsNotNull(
3479 ? OMPD_parallel_for_simd
3484 return CGF.
Builder.CreateIsNotNull(
3488 if (HasLastprivateClause)
3492 LoopScope.restoreMap();
3494 return CGF.
Builder.CreateIsNotNull(
3498 DoacrossCleanupScope.ForceCleanup();
3505 return HasLastprivateClause;
3511static std::pair<LValue, LValue>
3513 const auto &LS = cast<OMPLoopDirective>(S);
3525static std::pair<llvm::Value *, llvm::Value *>
3528 const auto &LS = cast<OMPLoopDirective>(S);
3529 const Expr *IVExpr = LS.getIterationVariable();
3531 llvm::Value *LBVal = CGF.
Builder.getIntN(IVSize, 0);
3533 return {LBVal, UBVal};
3545 llvm::function_ref<llvm::Value *(
CodeGenFunction &)> NumIteratorsGen) {
3546 llvm::Value *OMPScanNumIterations = CGF.
Builder.CreateIntCast(
3547 NumIteratorsGen(CGF), CGF.
SizeTy,
false);
3553 assert(
C->getModifier() == OMPC_REDUCTION_inscan &&
3554 "Only inscan reductions are expected.");
3555 Shareds.append(
C->varlist_begin(),
C->varlist_end());
3556 Privates.append(
C->privates().begin(),
C->privates().end());
3557 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
3558 CopyArrayTemps.append(
C->copy_array_temps().begin(),
3559 C->copy_array_temps().end());
3567 auto *ITA = CopyArrayTemps.begin();
3568 for (
const Expr *IRef : Privates) {
3569 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
3572 if (PrivateVD->getType()->isVariablyModifiedType()) {
3576 CodeGenFunction::OpaqueValueMapping DimMapping(
3578 cast<OpaqueValueExpr>(
3579 cast<VariableArrayType>((*ITA)->getType()->getAsArrayTypeUnsafe())
3583 CGF.
EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(*ITA)->getDecl()));
3597 llvm::function_ref<llvm::Value *(
CodeGenFunction &)> NumIteratorsGen) {
3598 llvm::Value *OMPScanNumIterations = CGF.
Builder.CreateIntCast(
3599 NumIteratorsGen(CGF), CGF.
SizeTy,
false);
3607 assert(
C->getModifier() == OMPC_REDUCTION_inscan &&
3608 "Only inscan reductions are expected.");
3609 Shareds.append(
C->varlist_begin(),
C->varlist_end());
3610 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
3611 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
3612 Privates.append(
C->privates().begin(),
C->privates().end());
3613 CopyOps.append(
C->copy_ops().begin(),
C->copy_ops().end());
3614 CopyArrayElems.append(
C->copy_array_elems().begin(),
3615 C->copy_array_elems().end());
3619 llvm::Value *OMPLast = CGF.
Builder.CreateNSWSub(
3620 OMPScanNumIterations,
3621 llvm::ConstantInt::get(CGF.
SizeTy, 1,
false));
3622 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
3623 const Expr *PrivateExpr = Privates[I];
3624 const Expr *OrigExpr = Shareds[I];
3625 const Expr *CopyArrayElem = CopyArrayElems[I];
3626 CodeGenFunction::OpaqueValueMapping IdxMapping(
3628 cast<OpaqueValueExpr>(
3629 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3635 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
3636 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
3665 llvm::Value *OMPScanNumIterations = CGF.
Builder.CreateIntCast(
3666 NumIteratorsGen(CGF), CGF.
SizeTy,
false);
3673 assert(
C->getModifier() == OMPC_REDUCTION_inscan &&
3674 "Only inscan reductions are expected.");
3675 Privates.append(
C->privates().begin(),
C->privates().end());
3676 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
3677 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
3678 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
3679 CopyArrayElems.append(
C->copy_array_elems().begin(),
3680 C->copy_array_elems().end());
3682 CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S);
3691 CodeGenFunction::OMPLocalDeclMapRAII
Scope(CGF);
3695 auto &&CodeGen = [&S, OMPScanNumIterations, &LHSs, &RHSs, &CopyArrayElems,
3702 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock();
3703 llvm::BasicBlock *LoopBB = CGF.createBasicBlock(
"omp.outer.log.scan.body");
3704 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
"omp.outer.log.scan.exit");
3706 CGF.CGM.getIntrinsic(llvm::Intrinsic::log2, CGF.DoubleTy);
3708 CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy);
3709 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg);
3710 F = CGF.CGM.getIntrinsic(llvm::Intrinsic::ceil, CGF.DoubleTy);
3711 LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal);
3712 LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy);
3713 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub(
3714 OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1));
3716 CGF.EmitBlock(LoopBB);
3717 auto *Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2);
3719 auto *Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3720 Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB);
3721 Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB);
3724 llvm::BasicBlock *InnerLoopBB =
3725 CGF.createBasicBlock(
"omp.inner.log.scan.body");
3726 llvm::BasicBlock *InnerExitBB =
3727 CGF.createBasicBlock(
"omp.inner.log.scan.exit");
3728 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K);
3729 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3730 CGF.EmitBlock(InnerLoopBB);
3731 auto *IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3732 IVal->addIncoming(NMin1, LoopBB);
3734 CodeGenFunction::OMPPrivateScope PrivScope(CGF);
3735 auto *ILHS = LHSs.begin();
3736 auto *IRHS = RHSs.begin();
3737 for (
const Expr *CopyArrayElem : CopyArrayElems) {
3738 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
3739 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
3742 CodeGenFunction::OpaqueValueMapping IdxMapping(
3744 cast<OpaqueValueExpr>(
3745 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3747 LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF);
3749 PrivScope.addPrivate(LHSVD, LHSAddr);
3752 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K);
3753 CodeGenFunction::OpaqueValueMapping IdxMapping(
3755 cast<OpaqueValueExpr>(
3756 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3758 RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF);
3760 PrivScope.addPrivate(RHSVD, RHSAddr);
3764 PrivScope.Privatize();
3765 CGF.CGM.getOpenMPRuntime().emitReduction(
3766 CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
3767 {true, true, OMPD_unknown});
3769 llvm::Value *NextIVal =
3770 CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1));
3771 IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock());
3772 CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K);
3773 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3774 CGF.EmitBlock(InnerExitBB);
3776 CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1));
3777 Counter->addIncoming(Next, CGF.Builder.GetInsertBlock());
3779 llvm::Value *NextPow2K =
3780 CGF.Builder.CreateShl(Pow2K, 1,
"",
true);
3781 Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock());
3782 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(Next, LogVal);
3783 CGF.Builder.CreateCondBr(Cmp, LoopBB, ExitBB);
3785 CGF.EmitBlock(ExitBB);
3788 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());
3789 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
3790 CGF, S.getBeginLoc(), OMPD_unknown,
false,
3797 CGF.OMPFirstScanLoop =
false;
3804 bool HasLastprivates;
3807 return C->getModifier() == OMPC_REDUCTION_inscan;
3810 CodeGenFunction::OMPLocalDeclMapRAII
Scope(CGF);
3811 OMPLoopScope LoopScope(CGF, S);
3815 CodeGenFunction::OMPCancelStackRAII CancelRegion(
3816 CGF, S.getDirectiveKind(), HasCancel);
3824 const auto &&SecondGen = [&S, HasCancel,
3826 CodeGenFunction::OMPCancelStackRAII CancelRegion(
3827 CGF, S.getDirectiveKind(), HasCancel);
3838 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
3844 return HasLastprivates;
3851 if (isa<OMPNowaitClause>(
C))
3854 if (
auto *SC = dyn_cast<OMPScheduleClause>(
C)) {
3859 switch (SC->getScheduleKind()) {
3860 case OMPC_SCHEDULE_auto:
3861 case OMPC_SCHEDULE_dynamic:
3862 case OMPC_SCHEDULE_runtime:
3863 case OMPC_SCHEDULE_guided:
3864 case OMPC_SCHEDULE_static:
3877static llvm::omp::ScheduleKind
3879 switch (ScheduleClauseKind) {
3881 return llvm::omp::OMP_SCHEDULE_Default;
3882 case OMPC_SCHEDULE_auto:
3883 return llvm::omp::OMP_SCHEDULE_Auto;
3884 case OMPC_SCHEDULE_dynamic:
3885 return llvm::omp::OMP_SCHEDULE_Dynamic;
3886 case OMPC_SCHEDULE_guided:
3887 return llvm::omp::OMP_SCHEDULE_Guided;
3888 case OMPC_SCHEDULE_runtime:
3889 return llvm::omp::OMP_SCHEDULE_Runtime;
3890 case OMPC_SCHEDULE_static:
3891 return llvm::omp::OMP_SCHEDULE_Static;
3893 llvm_unreachable(
"Unhandled schedule kind");
3897 bool HasLastprivates =
false;
3898 bool UseOMPIRBuilder =
3900 auto &&CodeGen = [
this, &S, &HasLastprivates,
3903 if (UseOMPIRBuilder) {
3906 llvm::omp::ScheduleKind SchedKind = llvm::omp::OMP_SCHEDULE_Default;
3907 llvm::Value *ChunkSize =
nullptr;
3911 if (
const Expr *ChunkSizeExpr = SchedClause->getChunkSize())
3916 const Stmt *Inner = S.getRawStmt();
3917 llvm::CanonicalLoopInfo *CLI =
3920 llvm::OpenMPIRBuilder &OMPBuilder =
3922 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
3924 OMPBuilder.applyWorkshareLoop(
3925 Builder.getCurrentDebugLocation(), CLI, AllocaIP, NeedsBarrier,
3926 SchedKind, ChunkSize,
false,
3937 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
3942 if (!UseOMPIRBuilder) {
3952 bool HasLastprivates =
false;
3960 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
3973 llvm::Value *Init =
nullptr) {
3981 const Stmt *
CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
3983 bool HasLastprivates =
false;
3988 C.getIntTypeForBitwidth(32, 1);
3991 CGF.Builder.getInt32(0));
3992 llvm::ConstantInt *GlobalUBVal = CS !=
nullptr
3993 ? CGF.Builder.getInt32(CS->size() - 1)
3994 : CGF.Builder.getInt32(0);
3998 CGF.Builder.getInt32(1));
4000 CGF.Builder.getInt32(0));
4004 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
4006 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
4027 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
".omp.sections.exit");
4029 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
4030 ExitBB, CS ==
nullptr ? 1 : CS->size());
4032 unsigned CaseNumber = 0;
4034 auto CaseBB = CGF.createBasicBlock(
".omp.sections.case");
4035 CGF.EmitBlock(CaseBB);
4036 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
4037 CGF.EmitStmt(SubStmt);
4038 CGF.EmitBranch(ExitBB);
4042 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(
".omp.sections.case");
4043 CGF.EmitBlock(CaseBB);
4044 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
4046 CGF.EmitBranch(ExitBB);
4048 CGF.EmitBlock(ExitBB,
true);
4051 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
4052 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
4056 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
4057 CGF, S.getBeginLoc(), OMPD_unknown,
false,
4060 CGF.EmitOMPPrivateClause(S, LoopScope);
4062 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4063 CGF.EmitOMPReductionClauseInit(S, LoopScope);
4064 (void)LoopScope.Privatize();
4066 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4070 ScheduleKind.
Schedule = OMPC_SCHEDULE_static;
4074 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
4075 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit);
4077 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
4078 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
4079 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
4080 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
4082 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
4084 CGF.EmitOMPInnerLoop(S,
false, Cond, Inc, BodyGen,
4088 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
4089 S.getDirectiveKind());
4091 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
4092 CGF.EmitOMPReductionClauseFinal(S, OMPD_parallel);
4095 return CGF.
Builder.CreateIsNotNull(
4100 if (HasLastprivates)
4107 bool HasCancel =
false;
4108 if (
auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
4109 HasCancel = OSD->hasCancel();
4110 else if (
auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
4111 HasCancel = OPSD->hasCancel();
4112 OMPCancelStackRAII CancelRegion(*
this, S.getDirectiveKind(), HasCancel);
4129 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4130 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4132 auto FiniCB = [
this](InsertPointTy IP) {
4136 const CapturedStmt *ICS = S.getInnermostCapturedStmt();
4137 const Stmt *
CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4142 auto SectionCB = [
this, SubStmt](InsertPointTy AllocaIP,
4143 InsertPointTy CodeGenIP) {
4145 *
this, SubStmt, AllocaIP, CodeGenIP,
"section");
4147 SectionCBVector.push_back(SectionCB);
4150 auto SectionCB = [
this,
CapturedStmt](InsertPointTy AllocaIP,
4151 InsertPointTy CodeGenIP) {
4155 SectionCBVector.push_back(SectionCB);
4162 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
4163 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
4171 CGCapturedStmtInfo CGSI(*ICS,
CR_OpenMP);
4172 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*
this, &CGSI);
4173 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
4175 Builder.restoreIP(OMPBuilder.createSections(
4176 Builder, AllocaIP, SectionCBVector, PrivCB, FiniCB, S.hasCancel(),
4183 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
4198 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4200 const Stmt *SectionRegionBodyStmt = S.getAssociatedStmt();
4201 auto FiniCB = [
this](InsertPointTy IP) {
4205 auto BodyGenCB = [SectionRegionBodyStmt,
this](InsertPointTy AllocaIP,
4206 InsertPointTy CodeGenIP) {
4208 *
this, SectionRegionBodyStmt, AllocaIP, CodeGenIP,
"section");
4211 LexicalScope
Scope(*
this, S.getSourceRange());
4213 Builder.restoreIP(OMPBuilder.createSection(
Builder, BodyGenCB, FiniCB));
4217 LexicalScope
Scope(*
this, S.getSourceRange());
4232 CopyprivateVars.append(
C->varlists().begin(),
C->varlists().end());
4233 DestExprs.append(
C->destination_exprs().begin(),
4234 C->destination_exprs().end());
4235 SrcExprs.append(
C->source_exprs().begin(),
C->source_exprs().end());
4236 AssignmentOps.append(
C->assignment_ops().begin(),
4237 C->assignment_ops().end());
4242 OMPPrivateScope SingleScope(CGF);
4245 (void)SingleScope.Privatize();
4246 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4251 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
4253 CopyprivateVars, DestExprs,
4254 SrcExprs, AssignmentOps);
4258 if (!S.getSingleClause<
OMPNowaitClause>() && CopyprivateVars.empty()) {
4260 *
this, S.getBeginLoc(),
4278 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4280 const Stmt *MasterRegionBodyStmt = S.getAssociatedStmt();
4282 auto FiniCB = [
this](InsertPointTy IP) {
4286 auto BodyGenCB = [MasterRegionBodyStmt,
this](InsertPointTy AllocaIP,
4287 InsertPointTy CodeGenIP) {
4289 *
this, MasterRegionBodyStmt, AllocaIP, CodeGenIP,
"master");
4292 LexicalScope
Scope(*
this, S.getSourceRange());
4294 Builder.restoreIP(OMPBuilder.createMaster(
Builder, BodyGenCB, FiniCB));
4298 LexicalScope
Scope(*
this, S.getSourceRange());
4308 Expr *Filter =
nullptr;
4310 Filter = FilterClause->getThreadID();
4318 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4320 const Stmt *MaskedRegionBodyStmt = S.getAssociatedStmt();
4323 Filter = FilterClause->getThreadID();
4324 llvm::Value *FilterVal =
Filter
4328 auto FiniCB = [
this](InsertPointTy IP) {
4332 auto BodyGenCB = [MaskedRegionBodyStmt,
this](InsertPointTy AllocaIP,
4333 InsertPointTy CodeGenIP) {
4335 *
this, MaskedRegionBodyStmt, AllocaIP, CodeGenIP,
"masked");
4338 LexicalScope
Scope(*
this, S.getSourceRange());
4341 OMPBuilder.createMasked(
Builder, BodyGenCB, FiniCB, FilterVal));
4345 LexicalScope
Scope(*
this, S.getSourceRange());
4353 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4355 const Stmt *CriticalRegionBodyStmt = S.getAssociatedStmt();
4356 const Expr *Hint =
nullptr;
4357 if (
const auto *HintClause = S.getSingleClause<
OMPHintClause>())
4358 Hint = HintClause->getHint();
4363 llvm::Value *HintInst =
nullptr;
4368 auto FiniCB = [
this](InsertPointTy IP) {
4372 auto BodyGenCB = [CriticalRegionBodyStmt,
this](InsertPointTy AllocaIP,
4373 InsertPointTy CodeGenIP) {
4375 *
this, CriticalRegionBodyStmt, AllocaIP, CodeGenIP,
"critical");
4378 LexicalScope
Scope(*
this, S.getSourceRange());
4380 Builder.restoreIP(OMPBuilder.createCritical(
4381 Builder, BodyGenCB, FiniCB, S.getDirectiveName().getAsString(),
4389 CGF.
EmitStmt(S.getAssociatedStmt());
4391 const Expr *Hint =
nullptr;
4392 if (
const auto *HintClause = S.getSingleClause<
OMPHintClause>())
4393 Hint = HintClause->getHint();
4394 LexicalScope
Scope(*
this, S.getSourceRange());
4397 S.getDirectiveName().getAsString(),
4398 CodeGen, S.getBeginLoc(), Hint);
4412 CodeGenFunction::OMPLocalDeclMapRAII
Scope(CGF);
4414 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
4415 OMPLoopScope LoopScope(CGF, S);
4420 return C->getModifier() == OMPC_REDUCTION_inscan;
4446 CodeGenFunction::OMPLocalDeclMapRAII
Scope(CGF);
4448 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
4449 OMPLoopScope LoopScope(CGF, S);
4454 return C->getModifier() == OMPC_REDUCTION_inscan;
4475 OMPPrivateScope PrivateScope(CGF);
4480 (void)PrivateScope.Privatize();
4502 OMPPrivateScope PrivateScope(CGF);
4507 (void)PrivateScope.Privatize();
4530 CGF.EmitSections(S);
4544class CheckVarsEscapingUntiedTaskDeclContext final
4549 explicit CheckVarsEscapingUntiedTaskDeclContext() =
default;
4550 virtual ~CheckVarsEscapingUntiedTaskDeclContext() =
default;
4551 void VisitDeclStmt(
const DeclStmt *S) {
4555 for (
const Decl *D : S->decls()) {
4556 if (
const auto *VD = dyn_cast_or_null<VarDecl>(D))
4558 PrivateDecls.push_back(VD);
4564 void VisitBlockExpr(
const BlockExpr *) {}
4565 void VisitStmt(
const Stmt *S) {
4568 for (
const Stmt *Child : S->children())
4582 bool OmpAllMemory =
false;
4585 return C->getDependencyKind() == OMPC_DEPEND_outallmemory ||
4586 C->getDependencyKind() == OMPC_DEPEND_inoutallmemory;
4588 OmpAllMemory =
true;
4593 Data.Dependences.emplace_back(OMPC_DEPEND_outallmemory,
4602 if (Kind == OMPC_DEPEND_outallmemory || Kind == OMPC_DEPEND_inoutallmemory)
4604 if (OmpAllMemory && (Kind == OMPC_DEPEND_out || Kind == OMPC_DEPEND_inout))
4607 Data.Dependences.emplace_back(
C->getDependencyKind(),
C->getModifier());
4608 DD.
DepExprs.append(
C->varlist_begin(),
C->varlist_end());
4617 const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
4619 auto PartId = std::next(I);
4620 auto TaskT = std::next(I, 4);
4625 const Expr *Cond = Clause->getCondition();
4628 Data.Final.setInt(CondConstant);
4633 Data.Final.setInt(
false);
4637 const Expr *Prio = Clause->getPriority();
4638 Data.Priority.setInt(
true);
4649 auto IRef =
C->varlist_begin();
4650 for (
const Expr *IInit :
C->private_copies()) {
4651 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4652 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4653 Data.PrivateVars.push_back(*IRef);
4654 Data.PrivateCopies.push_back(IInit);
4659 EmittedAsPrivate.clear();
4662 auto IRef =
C->varlist_begin();
4663 auto IElemInitRef =
C->inits().begin();
4664 for (
const Expr *IInit :
C->private_copies()) {
4665 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4666 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4667 Data.FirstprivateVars.push_back(*IRef);
4668 Data.FirstprivateCopies.push_back(IInit);
4669 Data.FirstprivateInits.push_back(*IElemInitRef);
4676 llvm::MapVector<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
4678 auto IRef =
C->varlist_begin();
4679 auto ID =
C->destination_exprs().begin();
4680 for (
const Expr *IInit :
C->private_copies()) {
4681 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4682 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4683 Data.LastprivateVars.push_back(*IRef);
4684 Data.LastprivateCopies.push_back(IInit);
4686 LastprivateDstsOrigs.insert(
4687 std::make_pair(cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
4688 cast<DeclRefExpr>(*IRef)));
4696 Data.ReductionVars.append(
C->varlist_begin(),
C->varlist_end());
4697 Data.ReductionOrigs.append(
C->varlist_begin(),
C->varlist_end());
4698 Data.ReductionCopies.append(
C->privates().begin(),
C->privates().end());
4699 Data.ReductionOps.append(
C->reduction_ops().begin(),
4700 C->reduction_ops().end());
4701 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
4702 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
4705 *
this, S.getBeginLoc(), LHSs, RHSs,
Data);
4710 CheckVarsEscapingUntiedTaskDeclContext Checker;
4711 Checker.Visit(S.getInnermostCapturedStmt()->getCapturedStmt());
4712 Data.PrivateLocals.append(Checker.getPrivateDecls().begin(),
4713 Checker.getPrivateDecls().end());
4715 auto &&CodeGen = [&
Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
4718 llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
4719 std::pair<Address, Address>>
4722 OMPPrivateScope
Scope(CGF);
4724 if (
auto *DI = CGF.getDebugInfo()) {
4725 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields =
4726 CGF.CapturedStmtInfo->getCaptureFields();
4727 llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue();
4728 if (CaptureFields.size() && ContextValue) {
4729 unsigned CharWidth = CGF.getContext().getCharWidth();
4743 for (
auto It = CaptureFields.begin(); It != CaptureFields.end(); ++It) {
4744 const VarDecl *SharedVar = It->first;
4747 CGF.getContext().getASTRecordLayout(CaptureRecord);
4750 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
4751 (void)DI->EmitDeclareOfAutoVariable(SharedVar, ContextValue,
4752 CGF.Builder,
false);
4753 llvm::Instruction &
Last = CGF.Builder.GetInsertBlock()->back();
4756 if (
auto DDI = dyn_cast<llvm::DbgVariableIntrinsic>(&
Last)) {
4760 Ops.push_back(llvm::dwarf::DW_OP_plus_uconst);
4763 Ops.push_back(llvm::dwarf::DW_OP_deref);
4764 auto &Ctx = DDI->getContext();
4765 llvm::DIExpression *DIExpr = llvm::DIExpression::get(Ctx, Ops);
4766 Last.setOperand(2, llvm::MetadataAsValue::get(Ctx, DIExpr));
4772 if (!
Data.PrivateVars.empty() || !
Data.FirstprivateVars.empty() ||
4773 !
Data.LastprivateVars.empty() || !
Data.PrivateLocals.empty()) {
4774 enum { PrivatesParam = 2, CopyFnParam = 3 };
4775 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
4776 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
4777 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
4778 CS->getCapturedDecl()->getParam(PrivatesParam)));
4783 CallArgs.push_back(PrivatesPtr);
4784 ParamTypes.push_back(PrivatesPtr->getType());
4785 for (
const Expr *E :
Data.PrivateVars) {
4786 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4787 Address PrivatePtr = CGF.CreateMemTemp(
4788 CGF.getContext().getPointerType(E->
getType()),
".priv.ptr.addr");
4789 PrivatePtrs.emplace_back(VD, PrivatePtr);
4791 ParamTypes.push_back(PrivatePtr.
getType());
4793 for (
const Expr *E :
Data.FirstprivateVars) {
4794 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4796 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->
getType()),
4797 ".firstpriv.ptr.addr");
4798 PrivatePtrs.emplace_back(VD, PrivatePtr);
4799 FirstprivatePtrs.emplace_back(VD, PrivatePtr);
4801 ParamTypes.push_back(PrivatePtr.
getType());
4803 for (
const Expr *E :
Data.LastprivateVars) {
4804 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4806 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->
getType()),
4807 ".lastpriv.ptr.addr");
4808 PrivatePtrs.emplace_back(VD, PrivatePtr);
4810 ParamTypes.push_back(PrivatePtr.
getType());
4815 Ty = CGF.getContext().getPointerType(Ty);
4817 Ty = CGF.getContext().getPointerType(Ty);
4818 Address PrivatePtr = CGF.CreateMemTemp(
4819 CGF.getContext().getPointerType(Ty),
".local.ptr.addr");
4820 auto Result = UntiedLocalVars.insert(
4823 if (
Result.second ==
false)
4824 *
Result.first = std::make_pair(
4827 ParamTypes.push_back(PrivatePtr.
getType());
4829 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
4831 CopyFn = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4832 CopyFn, CopyFnTy->getPointerTo());
4833 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
4834 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
4835 for (
const auto &Pair : LastprivateDstsOrigs) {
4836 const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
4839 CGF.CapturedStmtInfo->lookup(OrigVD) !=
nullptr,
4841 Pair.second->getExprLoc());
4842 Scope.addPrivate(Pair.first, CGF.EmitLValue(&DRE).getAddress(CGF));
4844 for (
const auto &Pair : PrivatePtrs) {
4846 CGF.Builder.CreateLoad(Pair.second),
4847 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
4848 CGF.getContext().getDeclAlign(Pair.first));
4849 Scope.addPrivate(Pair.first, Replacement);
4850 if (
auto *DI = CGF.getDebugInfo())
4851 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
4852 (void)DI->EmitDeclareOfAutoVariable(
4853 Pair.first, Pair.second.getPointer(), CGF.Builder,
4858 for (
auto &Pair : UntiedLocalVars) {
4859 QualType VDType = Pair.first->getType().getNonReferenceType();
4860 if (Pair.first->getType()->isLValueReferenceType())
4861 VDType = CGF.getContext().getPointerType(VDType);
4863 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
4866 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(VDType)),