27#include "llvm/ADT/SmallSet.h"
28#include "llvm/BinaryFormat/Dwarf.h"
29#include "llvm/Frontend/OpenMP/OMPConstants.h"
30#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
31#include "llvm/IR/Constants.h"
32#include "llvm/IR/DebugInfoMetadata.h"
33#include "llvm/IR/Instructions.h"
34#include "llvm/IR/IntrinsicInst.h"
35#include "llvm/IR/Metadata.h"
36#include "llvm/Support/AtomicOrdering.h"
39using namespace CodeGen;
40using namespace llvm::omp;
47class OMPLexicalScope :
public CodeGenFunction::LexicalScope {
49 for (
const auto *
C : S.clauses()) {
51 if (
const auto *PreInit =
52 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
53 for (
const auto *I : PreInit->decls()) {
54 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
57 CodeGenFunction::AutoVarEmission Emission =
66 CodeGenFunction::OMPPrivateScope InlinedShareds;
72 cast<BlockDecl>(CGF.
CurCodeDecl)->capturesVariable(VD));
78 const std::optional<OpenMPDirectiveKind> CapturedRegion = std::nullopt,
79 const bool EmitPreInitStmt =
true)
83 emitPreInitStmt(CGF, S);
86 assert(S.hasAssociatedStmt() &&
87 "Expected associated statement for inlined directive.");
88 const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
90 if (
C.capturesVariable() ||
C.capturesVariableByCopy()) {
91 auto *VD =
C.getCapturedVar();
93 "Canonical decl must be captured.");
97 InlinedShareds.isGlobalVarCaptured(VD)),
102 (void)InlinedShareds.Privatize();
108class OMPParallelScope final :
public OMPLexicalScope {
118 : OMPLexicalScope(CGF, S,
std::nullopt,
119 EmitPreInitStmt(S)) {}
124class OMPTeamsScope final :
public OMPLexicalScope {
133 : OMPLexicalScope(CGF, S,
std::nullopt,
134 EmitPreInitStmt(S)) {}
139class OMPLoopScope :
public CodeGenFunction::RunCleanupsScope {
142 CodeGenFunction::OMPMapVars PreCondVars;
143 if (
auto *LD = dyn_cast<OMPLoopDirective>(&S)) {
145 for (
const auto *E : LD->counters()) {
146 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
148 (void)PreCondVars.setVarAddr(
153 for (
const Expr *IRef :
C->varlists()) {
155 cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
156 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
157 QualType OrigVDTy = OrigVD->getType().getNonReferenceType();
158 (void)PreCondVars.setVarAddr(
167 (void)PreCondVars.apply(CGF);
170 LD->getInnermostCapturedStmt()->getCapturedStmt(),
171 true, LD->getLoopsNumber(),
172 [&CGF](
unsigned Cnt,
const Stmt *CurStmt) {
173 if (
const auto *CXXFor = dyn_cast<CXXForRangeStmt>(CurStmt)) {
174 if (
const Stmt *Init = CXXFor->getInit())
176 CGF.
EmitStmt(CXXFor->getRangeStmt());
181 PreInits = cast_or_null<DeclStmt>(LD->getPreInits());
182 }
else if (
const auto *Tile = dyn_cast<OMPTileDirective>(&S)) {
183 PreInits = cast_or_null<DeclStmt>(Tile->getPreInits());
184 }
else if (
const auto *Unroll = dyn_cast<OMPUnrollDirective>(&S)) {
185 PreInits = cast_or_null<DeclStmt>(Unroll->getPreInits());
187 llvm_unreachable(
"Unknown loop-based directive kind.");
190 for (
const auto *I : PreInits->
decls())
193 PreCondVars.restore(CGF);
199 emitPreInitStmt(CGF, S);
203class OMPSimdLexicalScope :
public CodeGenFunction::LexicalScope {
204 CodeGenFunction::OMPPrivateScope InlinedShareds;
210 cast<BlockDecl>(CGF.
CurCodeDecl)->capturesVariable(VD));
216 InlinedShareds(CGF) {
217 for (
const auto *
C : S.clauses()) {
219 if (
const auto *PreInit =
220 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
221 for (
const auto *I : PreInit->decls()) {
222 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
225 CodeGenFunction::AutoVarEmission Emission =
231 }
else if (
const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(
C)) {
232 for (
const Expr *E : UDP->varlists()) {
233 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
234 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
237 }
else if (
const auto *UDP = dyn_cast<OMPUseDeviceAddrClause>(
C)) {
238 for (
const Expr *E : UDP->varlists()) {
240 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
247 if (
const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
248 if (
const Expr *E = TG->getReductionRef())
249 CGF.
EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
255 if (
C->getModifier() != OMPC_REDUCTION_inscan)
257 for (
const Expr *E :
C->copy_array_temps())
258 CopyArrayTemps.insert(cast<DeclRefExpr>(E)->getDecl());
260 const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
263 if (
C.capturesVariable() ||
C.capturesVariableByCopy()) {
264 auto *VD =
C.getCapturedVar();
265 if (CopyArrayTemps.contains(VD))
268 "Canonical decl must be captured.");
270 isCapturedVar(CGF, VD) ||
272 InlinedShareds.isGlobalVarCaptured(VD)),
280 (void)InlinedShareds.Privatize();
290LValue CodeGenFunction::EmitOMPSharedLValue(
const Expr *E) {
291 if (
const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
292 if (
const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
293 OrigVD = OrigVD->getCanonicalDecl();
308 llvm::Value *
Size =
nullptr;
309 auto SizeInChars =
C.getTypeSizeInChars(Ty);
310 if (SizeInChars.isZero()) {
316 Size ?
Builder.CreateNUWMul(Size, VlaSize.NumElts) : VlaSize.NumElts;
318 SizeInChars =
C.getTypeSizeInChars(Ty);
319 if (SizeInChars.isZero())
320 return llvm::ConstantInt::get(
SizeTy, 0);
328 const RecordDecl *RD = S.getCapturedRecordDecl();
330 auto CurCap = S.captures().begin();
332 E = S.capture_init_end();
333 I != E; ++I, ++CurField, ++CurCap) {
334 if (CurField->hasCapturedVLAType()) {
337 CapturedVars.push_back(Val);
338 }
else if (CurCap->capturesThis()) {
339 CapturedVars.push_back(CXXThisValue);
340 }
else if (CurCap->capturesVariableByCopy()) {
345 if (!CurField->getType()->isAnyPointerType()) {
349 Twine(CurCap->getCapturedVar()->getName(),
".casted"));
364 CapturedVars.push_back(CV);
366 assert(CurCap->capturesVariable() &&
"Expected capture by reference.");
387 return C.getLValueReferenceType(
393 if (
const auto *VLA = dyn_cast<VariableArrayType>(A))
395 if (!A->isVariablyModifiedType())
396 return C.getCanonicalType(T);
398 return C.getCanonicalParamType(T);
403struct FunctionOptions {
408 const bool UIntPtrCastRequired =
true;
411 const bool RegisterCastedArgsOnly =
false;
413 const StringRef FunctionName;
416 explicit FunctionOptions(
const CapturedStmt *S,
bool UIntPtrCastRequired,
417 bool RegisterCastedArgsOnly, StringRef FunctionName,
419 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
420 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
421 FunctionName(FunctionName), Loc(Loc) {}
427 llvm::MapVector<
const Decl *, std::pair<const VarDecl *, Address>>
429 llvm::DenseMap<
const Decl *, std::pair<const Expr *, llvm::Value *>>
431 llvm::Value *&CXXThisValue,
const FunctionOptions &FO) {
433 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
434 assert(CD->
hasBody() &&
"missing CapturedDecl body");
436 CXXThisValue =
nullptr;
446 auto I = FO.S->captures().begin();
448 if (!FO.UIntPtrCastRequired) {
468 if (FO.UIntPtrCastRequired &&
470 I->capturesVariableArrayType()))
473 if (I->capturesVariable() || I->capturesVariableByCopy()) {
474 CapVar = I->getCapturedVar();
476 }
else if (I->capturesThis()) {
479 assert(I->capturesVariableArrayType());
489 }
else if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
491 Ctx, DebugFunctionDecl,
492 CapVar ? CapVar->
getBeginLoc() : FD->getBeginLoc(),
493 CapVar ? CapVar->
getLocation() : FD->getLocation(), II, ArgType,
499 Args.emplace_back(Arg);
501 TargetArgs.emplace_back(
502 FO.UIntPtrCastRequired
519 llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
523 F->setDoesNotThrow();
524 F->setDoesNotRecurse();
528 F->removeFnAttr(llvm::Attribute::NoInline);
529 F->addFnAttr(llvm::Attribute::AlwaysInline);
534 FO.UIntPtrCastRequired ? FO.Loc : FO.S->getBeginLoc(),
535 FO.UIntPtrCastRequired ? FO.Loc
538 I = FO.S->captures().begin();
542 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
550 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
551 const VarDecl *CurVD = I->getCapturedVar();
552 if (!FO.RegisterCastedArgsOnly)
553 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
561 if (FD->hasCapturedVLAType()) {
562 if (FO.UIntPtrCastRequired) {
565 Args[Cnt]->getName(), ArgLVal),
570 VLASizes.try_emplace(Args[Cnt], VAT->
getSizeExpr(), ExprArg);
571 }
else if (I->capturesVariable()) {
572 const VarDecl *Var = I->getCapturedVar();
582 if (!FO.RegisterCastedArgsOnly) {
586 }
else if (I->capturesVariableByCopy()) {
587 assert(!FD->getType()->isAnyPointerType() &&
588 "Not expecting a captured pointer.");
589 const VarDecl *Var = I->getCapturedVar();
590 LocalAddrs.insert({Args[Cnt],
591 {Var, FO.UIntPtrCastRequired
593 CGF, I->getLocation(), FD->getType(),
594 Args[Cnt]->getName(), ArgLVal)
598 assert(I->capturesThis());
600 LocalAddrs.insert({Args[Cnt], {
nullptr, ArgLVal.
getAddress(CGF)}});
614 "CapturedStmtInfo should be set when generating the captured function");
617 bool NeedWrapperFunction =
620 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
621 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
623 llvm::raw_svector_ostream Out(Buffer);
625 if (NeedWrapperFunction)
627 FunctionOptions FO(&S, !NeedWrapperFunction,
false,
630 VLASizes, CXXThisValue, FO);
631 CodeGenFunction::OMPPrivateScope LocalScope(*
this);
632 for (
const auto &LocalAddrPair : LocalAddrs) {
633 if (LocalAddrPair.second.first) {
634 LocalScope.addPrivate(LocalAddrPair.second.first,
635 LocalAddrPair.second.second);
638 (void)LocalScope.Privatize();
639 for (
const auto &VLASizePair : VLASizes)
640 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
643 (void)LocalScope.ForceCleanup();
645 if (!NeedWrapperFunction)
648 FunctionOptions WrapperFO(&S,
true,
656 llvm::Function *WrapperF =
658 WrapperCGF.CXXThisValue, WrapperFO);
660 auto *PI = F->arg_begin();
661 for (
const auto *Arg : Args) {
663 auto I = LocalAddrs.find(Arg);
664 if (I != LocalAddrs.end()) {
665 LValue LV = WrapperCGF.MakeAddrLValue(
667 I->second.first ? I->second.first->getType() : Arg->getType(),
670 LV.
setAddress(WrapperCGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
675 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
677 auto EI = VLASizes.find(Arg);
678 if (EI != VLASizes.end()) {
682 WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
684 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
687 CallArgs.emplace_back(WrapperCGF.EmitFromMemory(
CallArg, Arg->
getType()));
691 WrapperCGF.FinishFunction();
706 llvm::Value *NumElements =
emitArrayLength(ArrayTy, ElementTy, DestAddr);
710 llvm::Value *DestBegin = DestAddr.
getPointer();
713 DestBegin, NumElements);
718 llvm::Value *IsEmpty =
719 Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arraycpy.isempty");
720 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
723 llvm::BasicBlock *EntryBB =
Builder.GetInsertBlock();
728 llvm::PHINode *SrcElementPHI =
729 Builder.CreatePHI(SrcBegin->getType(), 2,
"omp.arraycpy.srcElementPast");
730 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
735 llvm::PHINode *DestElementPHI =
Builder.CreatePHI(
736 DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
737 DestElementPHI->addIncoming(DestBegin, EntryBB);
743 CopyGen(DestElementCurrent, SrcElementCurrent);
746 llvm::Value *DestElementNext =
748 1,
"omp.arraycpy.dest.element");
749 llvm::Value *SrcElementNext =
751 1,
"omp.arraycpy.src.element");
754 Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
755 Builder.CreateCondBr(Done, DoneBB, BodyBB);
756 DestElementPHI->addIncoming(DestElementNext,
Builder.GetInsertBlock());
757 SrcElementPHI->addIncoming(SrcElementNext,
Builder.GetInsertBlock());
767 const auto *BO = dyn_cast<BinaryOperator>(Copy);
768 if (BO && BO->getOpcode() == BO_Assign) {
777 DestAddr, SrcAddr, OriginalType,
778 [
this, Copy, SrcVD, DestVD](
Address DestElement,
Address SrcElement) {
782 CodeGenFunction::OMPPrivateScope Remap(*
this);
783 Remap.addPrivate(DestVD, DestElement);
784 Remap.addPrivate(SrcVD, SrcElement);
785 (void)Remap.Privatize();
791 CodeGenFunction::OMPPrivateScope Remap(*
this);
792 Remap.addPrivate(SrcVD, SrcAddr);
793 Remap.addPrivate(DestVD, DestAddr);
794 (void)Remap.Privatize();
801 OMPPrivateScope &PrivateScope) {
804 bool DeviceConstTarget =
807 bool FirstprivateIsLastprivate =
false;
808 llvm::DenseMap<const VarDecl *, OpenMPLastprivateModifier> Lastprivates;
810 for (
const auto *D :
C->varlists())
811 Lastprivates.try_emplace(
820 bool MustEmitFirstprivateCopy =
821 CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
823 const auto *IRef =
C->varlist_begin();
824 const auto *InitsRef =
C->inits().begin();
825 for (
const Expr *IInit :
C->private_copies()) {
826 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
827 bool ThisFirstprivateIsLastprivate =
828 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
830 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
831 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
833 (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())) {
834 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
841 if (DeviceConstTarget && OrigVD->getType().isConstant(
getContext()) &&
843 (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())) {
844 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
849 FirstprivateIsLastprivate =
850 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
851 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
853 cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
862 if (CE && !CE.isReference()) {
868 if (CE && CE.isReference()) {
869 OriginalLVal = CE.getReferenceLValue(*
this, &DRE);
871 assert(!CE &&
"Expected non-constant firstprivate.");
890 Emission.getAllocatedAddress(), OriginalLVal.
getAddress(*
this),
895 RunCleanupsScope InitScope(*this);
897 setAddrOfLocalVar(VDInit, SrcElement);
898 EmitAnyExprToMem(Init, DestElement,
899 Init->getType().getQualifiers(),
901 LocalDeclMap.erase(VDInit);
906 PrivateScope.addPrivate(OrigVD, Emission.getAllocatedAddress());
912 setAddrOfLocalVar(VDInit, OriginalAddr);
914 LocalDeclMap.erase(VDInit);
916 if (ThisFirstprivateIsLastprivate &&
917 Lastprivates[OrigVD->getCanonicalDecl()] ==
918 OMPC_LASTPRIVATE_conditional) {
923 (*IRef)->getExprLoc());
928 LocalDeclMap.erase(VD);
929 setAddrOfLocalVar(VD, VDAddr);
931 IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);
933 assert(IsRegistered &&
934 "firstprivate var already registered as private");
942 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
947 CodeGenFunction::OMPPrivateScope &PrivateScope) {
952 auto IRef =
C->varlist_begin();
953 for (
const Expr *IInit :
C->private_copies()) {
954 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
955 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
956 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
961 assert(IsRegistered &&
"private var already registered as private");
978 llvm::BasicBlock *CopyBegin =
nullptr, *CopyEnd =
nullptr;
980 auto IRef =
C->varlist_begin();
981 auto ISrcRef =
C->source_exprs().begin();
982 auto IDestRef =
C->destination_exprs().begin();
983 for (
const Expr *AssignOp :
C->assignment_ops()) {
984 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
992 getContext().getTargetInfo().isTLSSupported()) {
994 "Copyin threadprivates should have been captured!");
998 LocalDeclMap.erase(VD);
1008 if (CopiedVars.size() == 1) {
1014 auto *MasterAddrInt =
1016 auto *PrivateAddrInt =
1019 Builder.CreateICmpNE(MasterAddrInt, PrivateAddrInt), CopyBegin,
1024 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1025 const auto *DestVD =
1026 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1046 bool HasAtLeastOneLastprivate =
false;
1049 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1050 for (
const Expr *
C : LoopDirective->counters()) {
1057 HasAtLeastOneLastprivate =
true;
1061 const auto *IRef =
C->varlist_begin();
1062 const auto *IDestRef =
C->destination_exprs().begin();
1063 for (
const Expr *IInit :
C->private_copies()) {
1066 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1069 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
1070 const auto *DestVD =
1071 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1076 PrivateScope.addPrivate(DestVD,
EmitLValue(&DRE).getAddress(*
this));
1080 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
1081 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
1083 if (
C->getKind() == OMPC_LASTPRIVATE_conditional) {
1086 setAddrOfLocalVar(VD, VDAddr);
1092 bool IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);
1093 assert(IsRegistered &&
1094 "lastprivate var already registered as private");
1102 return HasAtLeastOneLastprivate;
1107 llvm::Value *IsLastIterCond) {
1116 llvm::BasicBlock *ThenBB =
nullptr;
1117 llvm::BasicBlock *DoneBB =
nullptr;
1118 if (IsLastIterCond) {
1124 return C->getKind() == OMPC_LASTPRIVATE_conditional;
1133 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1137 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1138 if (
const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1139 auto IC = LoopDirective->counters().begin();
1140 for (
const Expr *F : LoopDirective->finals()) {
1142 cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1144 AlreadyEmittedVars.insert(D);
1146 LoopCountersAndUpdates[D] = F;
1151 auto IRef =
C->varlist_begin();
1152 auto ISrcRef =
C->source_exprs().begin();
1153 auto IDestRef =
C->destination_exprs().begin();
1154 for (
const Expr *AssignOp :
C->assignment_ops()) {
1155 const auto *PrivateVD =
1156 cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1158 const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1159 if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1163 if (
const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1166 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1167 const auto *DestVD =
1168 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1171 if (
const auto *RefTy = PrivateVD->getType()->getAs<
ReferenceType>())
1177 if (
C->getKind() == OMPC_LASTPRIVATE_conditional)
1179 *
this,
MakeAddrLValue(PrivateAddr, (*IRef)->getType()), PrivateVD,
1180 (*IRef)->getExprLoc());
1183 EmitOMPCopy(
Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1189 if (
const Expr *PostUpdate =
C->getPostUpdateExpr())
1198 CodeGenFunction::OMPPrivateScope &PrivateScope,
bool ForInscan) {
1210 if (ForInscan != (
C->getModifier() == OMPC_REDUCTION_inscan))
1212 Shareds.append(
C->varlist_begin(),
C->varlist_end());
1213 Privates.append(
C->privates().begin(),
C->privates().end());
1214 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
1215 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
1216 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
1217 if (
C->getModifier() == OMPC_REDUCTION_task) {
1218 Data.ReductionVars.append(
C->privates().begin(),
C->privates().end());
1219 Data.ReductionOrigs.append(
C->varlist_begin(),
C->varlist_end());
1220 Data.ReductionCopies.append(
C->privates().begin(),
C->privates().end());
1221 Data.ReductionOps.append(
C->reduction_ops().begin(),
1222 C->reduction_ops().end());
1223 TaskLHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
1224 TaskRHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
1229 auto *ILHS = LHSs.begin();
1230 auto *IRHS = RHSs.begin();
1231 auto *IPriv = Privates.begin();
1232 for (
const Expr *IRef : Shareds) {
1233 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1235 RedCG.emitSharedOrigLValue(*
this, Count);
1236 RedCG.emitAggregateType(*
this, Count);
1238 RedCG.emitInitialization(*
this, Count, Emission.getAllocatedAddress(),
1239 RedCG.getSharedLValue(Count).getAddress(*
this),
1241 CGF.EmitAutoVarInit(Emission);
1245 Address BaseAddr = RedCG.adjustPrivateAddress(
1246 *
this, Count, Emission.getAllocatedAddress());
1248 PrivateScope.addPrivate(RedCG.getBaseDecl(Count), BaseAddr);
1249 assert(IsRegistered &&
"private var already registered as private");
1253 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1254 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1256 bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1260 PrivateScope.addPrivate(LHSVD,
1261 RedCG.getSharedLValue(Count).getAddress(*
this));
1264 isa<ArraySubscriptExpr>(IRef)) {
1267 PrivateScope.addPrivate(LHSVD,
1268 RedCG.getSharedLValue(Count).getAddress(*
this));
1276 Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress(*
this);
1283 PrivateScope.addPrivate(LHSVD, OriginalAddr);
1284 PrivateScope.addPrivate(
1295 if (!
Data.ReductionVars.empty()) {
1296 Data.IsReductionWithTaskMod =
true;
1297 Data.IsWorksharingReduction =
1301 const Expr *TaskRedRef =
nullptr;
1304 TaskRedRef = cast<OMPParallelDirective>(D).getTaskReductionRefExpr();
1307 TaskRedRef = cast<OMPForDirective>(D).getTaskReductionRefExpr();
1310 TaskRedRef = cast<OMPSectionsDirective>(D).getTaskReductionRefExpr();
1312 case OMPD_parallel_for:
1313 TaskRedRef = cast<OMPParallelForDirective>(D).getTaskReductionRefExpr();
1315 case OMPD_parallel_master:
1317 cast<OMPParallelMasterDirective>(D).getTaskReductionRefExpr();
1319 case OMPD_parallel_sections:
1321 cast<OMPParallelSectionsDirective>(D).getTaskReductionRefExpr();
1323 case OMPD_target_parallel:
1325 cast<OMPTargetParallelDirective>(D).getTaskReductionRefExpr();
1327 case OMPD_target_parallel_for:
1329 cast<OMPTargetParallelForDirective>(D).getTaskReductionRefExpr();
1331 case OMPD_distribute_parallel_for:
1333 cast<OMPDistributeParallelForDirective>(D).getTaskReductionRefExpr();
1335 case OMPD_teams_distribute_parallel_for:
1336 TaskRedRef = cast<OMPTeamsDistributeParallelForDirective>(D)
1337 .getTaskReductionRefExpr();
1339 case OMPD_target_teams_distribute_parallel_for:
1340 TaskRedRef = cast<OMPTargetTeamsDistributeParallelForDirective>(D)
1341 .getTaskReductionRefExpr();
1349 case OMPD_parallel_for_simd:
1351 case OMPD_taskyield:
1355 case OMPD_taskgroup:
1363 case OMPD_cancellation_point:
1365 case OMPD_target_data:
1366 case OMPD_target_enter_data:
1367 case OMPD_target_exit_data:
1369 case OMPD_taskloop_simd:
1370 case OMPD_master_taskloop:
1371 case OMPD_master_taskloop_simd:
1372 case OMPD_parallel_master_taskloop:
1373 case OMPD_parallel_master_taskloop_simd:
1374 case OMPD_distribute:
1375 case OMPD_target_update:
1376 case OMPD_distribute_parallel_for_simd:
1377 case OMPD_distribute_simd:
1378 case OMPD_target_parallel_for_simd:
1379 case OMPD_target_simd:
1380 case OMPD_teams_distribute:
1381 case OMPD_teams_distribute_simd:
1382 case OMPD_teams_distribute_parallel_for_simd:
1383 case OMPD_target_teams:
1384 case OMPD_target_teams_distribute:
1385 case OMPD_target_teams_distribute_parallel_for_simd:
1386 case OMPD_target_teams_distribute_simd:
1387 case OMPD_declare_target:
1388 case OMPD_end_declare_target:
1389 case OMPD_threadprivate:
1391 case OMPD_declare_reduction:
1392 case OMPD_declare_mapper:
1393 case OMPD_declare_simd:
1395 case OMPD_declare_variant:
1396 case OMPD_begin_declare_variant:
1397 case OMPD_end_declare_variant:
1400 llvm_unreachable(
"Enexpected directive with task reductions.");
1403 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(TaskRedRef)->getDecl());
1406 false, TaskRedRef->
getType());
1418 bool HasAtLeastOneReduction =
false;
1419 bool IsReductionWithTaskMod =
false;
1422 if (
C->getModifier() == OMPC_REDUCTION_inscan)
1424 HasAtLeastOneReduction =
true;
1425 Privates.append(
C->privates().begin(),
C->privates().end());
1426 LHSExprs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
1427 RHSExprs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
1428 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
1429 IsReductionWithTaskMod =
1430 IsReductionWithTaskMod ||
C->getModifier() == OMPC_REDUCTION_task;
1432 if (HasAtLeastOneReduction) {
1433 if (IsReductionWithTaskMod) {
1440 ReductionKind == OMPD_simd;
1441 bool SimpleReduction = ReductionKind == OMPD_simd;
1445 *
this, D.
getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1446 {WithNowait, SimpleReduction, ReductionKind});
1455 llvm::BasicBlock *DoneBB =
nullptr;
1457 if (
const Expr *PostUpdate =
C->getPostUpdateExpr()) {
1459 if (llvm::Value *Cond = CondGen(CGF)) {
1464 CGF.
Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1482 CodeGenBoundParametersTy;
1492 for (
const Expr *Ref :
C->varlists()) {
1493 if (!Ref->getType()->isScalarType())
1495 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1498 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1503 for (
const Expr *Ref :
C->varlists()) {
1504 if (!Ref->getType()->isScalarType())
1506 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1509 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1514 for (
const Expr *Ref :
C->varlists()) {
1515 if (!Ref->getType()->isScalarType())
1517 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1520 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1529 for (
const Expr *Ref :
C->varlists()) {
1530 if (!Ref->getType()->isScalarType())
1532 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1535 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1539 CGF, S, PrivateDecls);
1545 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1546 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1547 llvm::Value *NumThreads =
nullptr;
1548 llvm::Function *OutlinedFn =
1552 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1553 NumThreads = CGF.
EmitScalarExpr(NumThreadsClause->getNumThreads(),
1556 CGF, NumThreads, NumThreadsClause->getBeginLoc());
1559 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1561 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1563 const Expr *IfCond =
nullptr;
1564 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
1565 if (
C->getNameModifier() == OMPD_unknown ||
1566 C->getNameModifier() == OMPD_parallel) {
1567 IfCond =
C->getCondition();
1572 OMPParallelScope
Scope(CGF, S);
1578 CodeGenBoundParameters(CGF, S, CapturedVars);
1581 CapturedVars, IfCond, NumThreads);
1586 if (!CVD->
hasAttr<OMPAllocateDeclAttr>())
1588 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
1590 return !((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||
1591 AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&
1592 !AA->getAllocator());
1607 CGF, S.getBeginLoc(), OMPD_unknown,
false,
1627 Size = CGF.
Builder.CreateNUWAdd(
1636 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
1637 assert(AA->getAllocator() &&
1638 "Expected allocator expression for non-default allocator.");
1642 if (Allocator->getType()->isIntegerTy())
1644 else if (Allocator->getType()->isPointerTy())
1648 llvm::Value *Addr = OMPBuilder.createOMPAlloc(
1651 llvm::CallInst *FreeCI =
1652 OMPBuilder.createOMPFree(CGF.
Builder, Addr, Allocator);
1676 std::string Suffix = getNameWithSeparators({
"cache",
""});
1679 llvm::CallInst *ThreadPrivateCacheCall =
1680 OMPBuilder.createCachedThreadPrivate(CGF.
Builder,
Data, Size, CacheName);
1688 llvm::raw_svector_ostream OS(Buffer);
1689 StringRef Sep = FirstSeparator;
1690 for (StringRef Part : Parts) {
1694 return OS.str().str();
1702 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
Builder,
false,
1703 "." + RegionName +
".after");
1719 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
Builder,
false,
1720 "." + RegionName +
".after");
1735 llvm::Value *IfCond =
nullptr;
1740 llvm::Value *NumThreads =
nullptr;
1745 ProcBindKind ProcBind = OMP_PROC_BIND_default;
1747 ProcBind = ProcBindClause->getProcBindKind();
1749 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1753 auto FiniCB = [
this](InsertPointTy IP) {
1761 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
1762 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
1770 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1773 auto BodyGenCB = [&,
this](InsertPointTy AllocaIP,
1774 InsertPointTy CodeGenIP) {
1776 *
this, ParallelRegionBodyStmt, AllocaIP, CodeGenIP,
"parallel");
1779 CGCapturedStmtInfo CGSI(*CS,
CR_OpenMP);
1780 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*
this, &CGSI);
1781 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
1784 OMPBuilder.createParallel(
Builder, AllocaIP, BodyGenCB, PrivCB, FiniCB,
1785 IfCond, NumThreads, ProcBind, S.hasCancel()));
1792 OMPPrivateScope PrivateScope(CGF);
1797 (void)PrivateScope.Privatize();
1798 CGF.
EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1819class OMPTransformDirectiveScopeRAII {
1820 OMPLoopScope *
Scope =
nullptr;
1821 CodeGenFunction::CGCapturedStmtInfo *CGSI =
nullptr;
1822 CodeGenFunction::CGCapturedStmtRAII *CapInfoRAII =
nullptr;
1826 if (
const auto *Dir = dyn_cast<OMPLoopBasedDirective>(S)) {
1827 Scope =
new OMPLoopScope(CGF, *Dir);
1828 CGSI =
new CodeGenFunction::CGCapturedStmtInfo(
CR_OpenMP);
1829 CapInfoRAII =
new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI);
1832 ~OMPTransformDirectiveScopeRAII() {
1843 int MaxLevel,
int Level = 0) {
1844 assert(Level < MaxLevel &&
"Too deep lookup during loop body codegen.");
1845 const Stmt *SimplifiedS = S->IgnoreContainers();
1846 if (
const auto *CS = dyn_cast<CompoundStmt>(SimplifiedS)) {
1849 "LLVM IR generation of compound statement ('{}')");
1852 CodeGenFunction::LexicalScope
Scope(CGF, S->getSourceRange());
1853 for (
const Stmt *CurStmt : CS->body())
1854 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level);
1857 if (SimplifiedS == NextLoop) {
1858 if (
auto *Dir = dyn_cast<OMPLoopTransformationDirective>(SimplifiedS))
1859 SimplifiedS = Dir->getTransformedStmt();
1860 if (
const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(SimplifiedS))
1861 SimplifiedS = CanonLoop->getLoopStmt();
1862 if (
const auto *For = dyn_cast<ForStmt>(SimplifiedS)) {
1865 assert(isa<CXXForRangeStmt>(SimplifiedS) &&
1866 "Expected canonical for loop or range-based for loop.");
1867 const auto *CXXFor = cast<CXXForRangeStmt>(SimplifiedS);
1868 CGF.
EmitStmt(CXXFor->getLoopVarStmt());
1869 S = CXXFor->getBody();
1871 if (Level + 1 < MaxLevel) {
1874 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1);
1883 RunCleanupsScope BodyScope(*
this);
1892 for (
const Expr *UE :
C->updates())
1899 BreakContinueStack.push_back(BreakContinue(
LoopExit, Continue));
1911 OMPPrivateScope InscanScope(*
this);
1913 bool IsInscanRegion = InscanScope.Privatize();
1914 if (IsInscanRegion) {
1946 BreakContinueStack.pop_back();
1957 std::unique_ptr<CodeGenFunction::CGCapturedStmtInfo> CSI =
1958 std::make_unique<CodeGenFunction::CGCapturedStmtInfo>(*S);
1959 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, CSI.get());
1966static llvm::CallInst *
1971 EffectiveArgs.reserve(Args.size() + 1);
1972 llvm::append_range(EffectiveArgs, Args);
1973 EffectiveArgs.push_back(Cap.second);
1978llvm::CanonicalLoopInfo *
1980 assert(Depth == 1 &&
"Nested loops with OpenMPIRBuilder not yet implemented");
2006 const Stmt *SyntacticalLoop = S->getLoopStmt();
2013 LexicalScope ForScope(*
this, S->getSourceRange());
2017 const Stmt *BodyStmt;
2018 if (
const auto *For = dyn_cast<ForStmt>(SyntacticalLoop)) {
2019 if (
const Stmt *InitStmt = For->getInit())
2021 BodyStmt = For->getBody();
2022 }
else if (
const auto *RangeFor =
2023 dyn_cast<CXXForRangeStmt>(SyntacticalLoop)) {
2024 if (
const DeclStmt *RangeStmt = RangeFor->getRangeStmt())
2026 if (
const DeclStmt *BeginStmt = RangeFor->getBeginStmt())
2028 if (
const DeclStmt *EndStmt = RangeFor->getEndStmt())
2030 if (
const DeclStmt *LoopVarStmt = RangeFor->getLoopVarStmt())
2032 BodyStmt = RangeFor->getBody();
2034 llvm_unreachable(
"Expected for-stmt or range-based for-stmt");
2037 const CapturedStmt *DistanceFunc = S->getDistanceFunc();
2054 auto BodyGen = [&,
this](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP,
2055 llvm::Value *IndVar) {
2060 const DeclRefExpr *LoopVarRef = S->getLoopVarRef();
2066 RunCleanupsScope BodyScope(*
this);
2069 llvm::CanonicalLoopInfo *CL =
2070 OMPBuilder.createCanonicalLoop(
Builder, BodyGen, DistVal);
2073 Builder.restoreIP(CL->getAfterIP());
2074 ForScope.ForceCleanup();
2082 const Expr *IncExpr,
2093 const auto &OMPED = cast<OMPExecutableDirective>(S);
2094 const CapturedStmt *ICS = OMPED.getInnermostCapturedStmt();
2108 llvm::BasicBlock *ExitBlock =
LoopExit.getBlock();
2109 if (RequiresCleanup)
2116 if (ExitBlock !=
LoopExit.getBlock()) {
2126 BreakContinueStack.push_back(BreakContinue(
LoopExit, Continue));
2134 BreakContinueStack.pop_back();
2145 bool HasLinears =
false;
2147 for (
const Expr *Init :
C->inits()) {
2149 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
2150 if (
const auto *Ref =
2153 const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
2169 if (
const auto *CS = cast_or_null<BinaryOperator>(
C->getCalcStep()))
2170 if (
const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
2184 llvm::BasicBlock *DoneBB =
nullptr;
2187 auto IC =
C->varlist_begin();
2188 for (
const Expr *F :
C->finals()) {
2190 if (llvm::Value *Cond = CondGen(*
this)) {
2195 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2199 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
2204 CodeGenFunction::OMPPrivateScope VarScope(*
this);
2205 VarScope.addPrivate(OrigVD, OrigAddr);
2206 (void)VarScope.Privatize();
2210 if (
const Expr *PostUpdate =
C->getPostUpdateExpr())
2222 llvm::APInt ClauseAlignment(64, 0);
2223 if (
const Expr *AlignmentExpr = Clause->getAlignment()) {
2226 ClauseAlignment = AlignmentCI->getValue();
2228 for (
const Expr *E : Clause->varlists()) {
2229 llvm::APInt Alignment(ClauseAlignment);
2230 if (Alignment == 0) {
2240 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2241 "alignment is not power of 2");
2242 if (Alignment != 0) {
2256 auto I = S.private_counters().begin();
2257 for (
const Expr *E : S.counters()) {
2258 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2259 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
2263 LocalDeclMap.erase(PrivateVD);
2264 (void)LoopScope.addPrivate(VD, VarEmission.getAllocatedAddress());
2272 (void)LoopScope.addPrivate(PrivateVD, VarEmission.getAllocatedAddress());
2278 if (!
C->getNumForLoops())
2280 for (
unsigned I = S.getLoopsNumber(), E =
C->getLoopNumIterations().size();
2282 const auto *DRE = cast<DeclRefExpr>(
C->getLoopCounter(I));
2283 const auto *VD = cast<VarDecl>(DRE->getDecl());
2286 if (DRE->refersToEnclosingVariableOrCapture()) {
2287 (void)LoopScope.addPrivate(
2295 const Expr *Cond, llvm::BasicBlock *TrueBlock,
2296 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
2300 CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
2302 (void)PreCondScope.Privatize();
2304 for (
const Expr *I : S.inits()) {
2310 CodeGenFunction::OMPMapVars PreCondVars;
2311 for (
const Expr *E : S.dependent_counters()) {
2314 assert(!E->getType().getNonReferenceType()->isRecordType() &&
2315 "dependent counter must not be an iterator.");
2316 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2319 (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr);
2321 (void)PreCondVars.apply(CGF);
2322 for (
const Expr *E : S.dependent_inits()) {
2329 PreCondVars.restore(CGF);
2333 const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
2338 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
2339 for (
const Expr *
C : LoopDirective->counters()) {
2345 auto CurPrivate =
C->privates().begin();
2346 for (
const Expr *E :
C->varlists()) {
2347 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2348 const auto *PrivateVD =
2349 cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
2355 assert(IsRegistered &&
"linear var already registered as private");
2373 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
2382 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
2397 if (
C->getKind() == OMPC_ORDER_concurrent)
2404 return C->getModifier() == OMPC_REDUCTION_inscan;
2415 llvm::BasicBlock *DoneBB =
nullptr;
2419 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
2420 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
2421 const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
2423 OrigVD->hasGlobalStorage() || CED) {
2425 if (llvm::Value *Cond = CondGen(*
this)) {
2430 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2444 OMPPrivateScope VarScope(*
this);
2445 VarScope.addPrivate(OrigVD, OrigAddr);
2446 (void)VarScope.Privatize();
2458 CodeGenFunction::JumpDest
LoopExit) {
2466 auto VDecl = cast<VarDecl>(Helper->
getDecl());
2474 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](
CodeGenFunction &CGF,
2477 CodeGenFunction::OMPLocalDeclMapRAII
Scope(CGF);
2483 CodeGenFunction::OMPLocalDeclMapRAII
Scope(CGF);
2488 const Expr *IfCond =
nullptr;
2490 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
2492 (
C->getNameModifier() == OMPD_unknown ||
2493 C->getNameModifier() == OMPD_simd)) {
2494 IfCond =
C->getCondition();
2511 "Expected simd directive");
2512 OMPLoopScope PreInitScope(CGF, S);
2529 llvm::BasicBlock *ContBlock =
nullptr;
2536 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
2543 const Expr *IVExpr = S.getIterationVariable();
2544 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
2551 if (
const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2552 CGF.
EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2560 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2566 CGF, S, CGF.
EmitLValue(S.getIterationVariable()));
2568 (void)LoopScope.Privatize();
2579 S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),
2581 emitOMPLoopBodyWithStopPoint(CGF, S,
2582 CodeGenFunction::JumpDest());
2588 if (HasLastprivateClause)
2593 LoopScope.restoreMap();
2607 if (!(isa<OMPSimdlenClause>(
C) || isa<OMPSafelenClause>(
C) ||
2608 isa<OMPOrderClause>(
C) || isa<OMPAlignedClause>(
C)))
2615 if (
const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(S.getRawStmt())) {
2616 if (
const Stmt *SyntacticalLoop = CanonLoop->getLoopStmt()) {
2617 for (
const Stmt *SubStmt : SyntacticalLoop->
children()) {
2620 if (
const CompoundStmt *CS = dyn_cast<CompoundStmt>(SubStmt)) {
2624 if (isa<OMPOrderedDirective>(CSSubStmt)) {
2634static llvm::MapVector<llvm::Value *, llvm::Value *>
2636 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars;
2638 llvm::APInt ClauseAlignment(64, 0);
2639 if (
const Expr *AlignmentExpr = Clause->getAlignment()) {
2642 ClauseAlignment = AlignmentCI->getValue();
2644 for (
const Expr *E : Clause->varlists()) {
2645 llvm::APInt Alignment(ClauseAlignment);
2646 if (Alignment == 0) {
2653 E->getType()->getPointeeType()))
2656 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2657 "alignment is not power of 2");
2659 AlignedVars[PtrValue] = CGF.
Builder.getInt64(Alignment.getSExtValue());
2666 bool UseOMPIRBuilder =
2668 if (UseOMPIRBuilder) {
2669 auto &&CodeGenIRBuilder = [
this, &S, UseOMPIRBuilder](
CodeGenFunction &CGF,
2672 if (UseOMPIRBuilder) {
2673 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars =
2676 const Stmt *Inner = S.getRawStmt();
2677 llvm::CanonicalLoopInfo *CLI =
2680 llvm::OpenMPIRBuilder &OMPBuilder =
2683 llvm::ConstantInt *Simdlen =
nullptr;
2688 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
2691 llvm::ConstantInt *Safelen =
nullptr;
2696 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
2699 llvm::omp::OrderKind Order = llvm::omp::OrderKind::OMP_ORDER_unknown;
2701 if (
C->getKind() == OpenMPOrderClauseKind ::OMPC_ORDER_concurrent) {
2702 Order = llvm::omp::OrderKind::OMP_ORDER_concurrent;
2707 OMPBuilder.applySimd(CLI, AlignedVars,
2708 nullptr, Order, Simdlen, Safelen);
2715 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
2722 ParentLoopDirectiveForScanRegion ScanRegion(*
this, S);
2730 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
2739 OMPTransformDirectiveScopeRAII TileScope(*
this, &S);
2746 if (UseOMPIRBuilder) {
2748 const Stmt *Inner = S.getRawStmt();
2759 llvm::CanonicalLoopInfo *UnrolledCLI =
nullptr;
2763 OMPBuilder.unrollLoopFull(DL, CLI);
2766 if (
Expr *FactorExpr = PartialClause->getFactor()) {
2767 Factor = FactorExpr->EvaluateKnownConstInt(
getContext()).getZExtValue();
2768 assert(Factor >= 1 &&
"Only positive factors are valid");
2770 OMPBuilder.unrollLoopPartial(DL, CLI, Factor,
2771 NeedsUnrolledCLI ? &UnrolledCLI :
nullptr);
2773 OMPBuilder.unrollLoopHeuristic(DL, CLI);
2776 assert((!NeedsUnrolledCLI || UnrolledCLI) &&
2777 "NeedsUnrolledCLI implies UnrolledCLI to be set");
2794 if (
Expr *FactorExpr = PartialClause->getFactor()) {
2796 FactorExpr->EvaluateKnownConstInt(
getContext()).getZExtValue();
2797 assert(Factor >= 1 &&
"Only positive factors are valid");
2805void CodeGenFunction::EmitOMPOuterLoop(
2807 CodeGenFunction::OMPPrivateScope &LoopScope,
2808 const CodeGenFunction::OMPLoopArguments &LoopArgs,
2813 const Expr *IVExpr = S.getIterationVariable();
2827 llvm::Value *BoolCondVal =
nullptr;
2828 if (!DynamicOrOrdered) {
2839 RT.
emitForNext(*
this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
2840 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
2845 llvm::BasicBlock *ExitBlock =
LoopExit.getBlock();
2846 if (LoopScope.requiresCleanups())
2850 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
2851 if (ExitBlock !=
LoopExit.getBlock()) {
2859 if (DynamicOrOrdered)
2864 BreakContinueStack.push_back(BreakContinue(
LoopExit, Continue));
2872 CGF.LoopStack.setParallel(!IsMonotonic);
2873 if (const auto *C = S.getSingleClause<OMPOrderClause>())
2874 if (C->getKind() == OMPC_ORDER_concurrent)
2875 CGF.LoopStack.setParallel(true);
2877 CGF.EmitOMPSimdInit(S);
2880 [&S, &LoopArgs,
LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered,
2888 CGF.EmitOMPInnerLoop(
2889 S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
2891 CodeGenLoop(CGF, S, LoopExit);
2894 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
2899 BreakContinueStack.pop_back();
2900 if (!DynamicOrOrdered) {
2914 if (!DynamicOrOrdered)
2915 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2916 S.getDirectiveKind());
2918 OMPCancelStack.emitExit(*
this, S.getDirectiveKind(), CodeGen);
2921void CodeGenFunction::EmitOMPForOuterLoop(
2924 const OMPLoopArguments &LoopArgs,
2932 LoopArgs.Chunk !=
nullptr)) &&
2933 "static non-chunked schedule does not need outer loop");
2985 const Expr *IVExpr = S.getIterationVariable();
2989 if (DynamicOrOrdered) {
2990 const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
2991 CGDispatchBounds(*
this, S, LoopArgs.LB, LoopArgs.UB);
2992 llvm::Value *LBVal = DispatchBounds.first;
2993 llvm::Value *UBVal = DispatchBounds.second;
2997 IVSigned, Ordered, DipatchRTInputValues);
3000 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
3001 LoopArgs.ST, LoopArgs.Chunk);
3003 ScheduleKind, StaticInit);
3007 const unsigned IVSize,
3008 const bool IVSigned) {
3015 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
3016 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
3017 OuterLoopArgs.IncExpr = S.getInc();
3018 OuterLoopArgs.Init = S.getInit();
3019 OuterLoopArgs.Cond = S.getCond();
3020 OuterLoopArgs.NextLB = S.getNextLowerBound();
3021 OuterLoopArgs.NextUB = S.getNextUpperBound();
3022 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
3027 const unsigned IVSize,
const bool IVSigned) {}
3029void CodeGenFunction::EmitOMPDistributeOuterLoop(
3031 OMPPrivateScope &LoopScope,
const OMPLoopArguments &LoopArgs,
3041 const Expr *IVExpr = S.getIterationVariable();
3046 IVSize, IVSigned,
false, LoopArgs.IL, LoopArgs.LB,
3047 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
3054 IncExpr = S.getDistInc();
3056 IncExpr = S.getInc();
3061 OMPLoopArguments OuterLoopArgs;
3062 OuterLoopArgs.LB = LoopArgs.LB;
3063 OuterLoopArgs.UB = LoopArgs.UB;
3064 OuterLoopArgs.ST = LoopArgs.ST;
3065 OuterLoopArgs.IL = LoopArgs.IL;
3066 OuterLoopArgs.Chunk = LoopArgs.Chunk;
3068 ? S.getCombinedEnsureUpperBound()
3069 : S.getEnsureUpperBound();
3070 OuterLoopArgs.IncExpr = IncExpr;
3072 ? S.getCombinedInit()
3075 ? S.getCombinedCond()
3078 ? S.getCombinedNextLowerBound()
3079 : S.getNextLowerBound();
3081 ? S.getCombinedNextUpperBound()
3082 : S.getNextUpperBound();
3084 EmitOMPOuterLoop(
false,
false, S,
3085 LoopScope, OuterLoopArgs, CodeGenLoopContent,
3089static std::pair<LValue, LValue>
3132static std::pair<llvm::Value *, llvm::Value *>
3143 llvm::Value *LBVal =
3145 llvm::Value *UBVal =
3147 return {LBVal, UBVal};
3153 const auto &Dir = cast<OMPLoopDirective>(S);
3155 CGF.
EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
3156 llvm::Value *LBCast =
3159 CapturedVars.push_back(LBCast);
3161 CGF.
EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
3163 llvm::Value *UBCast =
3166 CapturedVars.push_back(UBCast);
3172 CodeGenFunction::JumpDest
LoopExit) {
3176 bool HasCancel =
false;
3178 if (
const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
3179 HasCancel = D->hasCancel();
3180 else if (
const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
3181 HasCancel = D->hasCancel();
3182 else if (
const auto *D =
3183 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
3184 HasCancel = D->hasCancel();
3186 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
3196 CGInlinedWorksharingLoop,
3206 OMPLexicalScope
Scope(*
this, S, OMPD_parallel);
3216 OMPLexicalScope
Scope(*
this, S, OMPD_parallel);
3225 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
3236 llvm::Constant *Addr;
3239 S, ParentName, Fn, Addr,
true, CodeGen);
3240 assert(Fn && Addr &&
"Target device function emission failed.");
3252struct ScheduleKindModifiersTy {
3268 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3269 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3275 if (
const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3283 bool HasLastprivateClause;
3286 OMPLoopScope PreInitScope(*
this, S);
3291 llvm::BasicBlock *ContBlock =
nullptr;
3298 emitPreCond(*
this, S, S.getPreCond(), ThenBlock, ContBlock,
3304 RunCleanupsScope DoacrossCleanupScope(*
this);
3305 bool Ordered =
false;
3307 if (OrderedClause->getNumForLoops())
3318 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*
this, S);
3319 LValue LB = Bounds.first;
3320 LValue UB = Bounds.second;
3328 OMPPrivateScope LoopScope(*
this);
3334 *
this, S.getBeginLoc(), OMPD_unknown,
false,
3339 *
this, S,
EmitLValue(S.getIterationVariable()));
3344 (void)LoopScope.Privatize();
3349 const Expr *ChunkExpr =
nullptr;
3352 ScheduleKind.
Schedule =
C->getScheduleKind();
3353 ScheduleKind.
M1 =
C->getFirstScheduleModifier();
3354 ScheduleKind.
M2 =
C->getSecondScheduleModifier();
3355 ChunkExpr =
C->getChunkSize();
3359 *
this, S, ScheduleKind.
Schedule, ChunkExpr);
3361 bool HasChunkSizeOne =
false;
3362 llvm::Value *Chunk =
nullptr;
3366 S.getIterationVariable()->getType(),
3370 llvm::APSInt EvaluatedChunk =
Result.Val.getInt();
3371 HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
3380 bool StaticChunkedOne =
3382 Chunk !=
nullptr) &&
3387 (ScheduleKind.
Schedule == OMPC_SCHEDULE_static &&
3388 !(ScheduleKind.
M1 == OMPC_SCHEDULE_MODIFIER_nonmonotonic ||
3389 ScheduleKind.
M2 == OMPC_SCHEDULE_MODIFIER_nonmonotonic)) ||
3390 ScheduleKind.
M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
3391 ScheduleKind.
M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
3393 Chunk !=
nullptr) ||
3394 StaticChunkedOne) &&
3402 CGF.EmitOMPSimdInit(S);
3404 if (C->getKind() == OMPC_ORDER_concurrent)
3405 CGF.LoopStack.setParallel(true);
3408 [IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk,
3417 IVSize, IVSigned, Ordered, IL.getAddress(CGF),
3418 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF),
3419 StaticChunkedOne ? Chunk :
nullptr);
3420 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
3421 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind,
3424 if (!StaticChunkedOne)
3425 CGF.EmitIgnoredExpr(S.getEnsureUpperBound());
3427 CGF.EmitIgnoredExpr(S.getInit());
3441 CGF.EmitOMPInnerLoop(
3442 S, LoopScope.requiresCleanups(),
3443 StaticChunkedOne ? S.getCombinedParForInDistCond()
3445 StaticChunkedOne ? S.getDistInc() : S.getInc(),
3447 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit);
3454 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
3455 S.getDirectiveKind());
3457 OMPCancelStack.emitExit(*
this, S.getDirectiveKind(), CodeGen);
3461 const OMPLoopArguments LoopArguments(
3462 LB.getAddress(*
this), UB.getAddress(*
this), ST.getAddress(*
this),
3463 IL.getAddress(*
this), Chunk, EUB);
3464 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
3465 LoopArguments, CGDispatchBounds);
3469 return CGF.
Builder.CreateIsNotNull(
3475 ? OMPD_parallel_for_simd
3480 return CGF.
Builder.CreateIsNotNull(
3484 if (HasLastprivateClause)
3488 LoopScope.restoreMap();
3490 return CGF.
Builder.CreateIsNotNull(
3494 DoacrossCleanupScope.ForceCleanup();
3501 return HasLastprivateClause;
3507static std::pair<LValue, LValue>
3509 const auto &LS = cast<OMPLoopDirective>(S);
3521static std::pair<llvm::Value *, llvm::Value *>
3524 const auto &LS = cast<OMPLoopDirective>(S);
3525 const Expr *IVExpr = LS.getIterationVariable();
3527 llvm::Value *LBVal = CGF.
Builder.getIntN(IVSize, 0);
3529 return {LBVal, UBVal};
3541 llvm::function_ref<llvm::Value *(
CodeGenFunction &)> NumIteratorsGen) {
3542 llvm::Value *OMPScanNumIterations = CGF.
Builder.CreateIntCast(
3543 NumIteratorsGen(CGF), CGF.
SizeTy,
false);
3549 assert(
C->getModifier() == OMPC_REDUCTION_inscan &&
3550 "Only inscan reductions are expected.");
3551 Shareds.append(
C->varlist_begin(),
C->varlist_end());
3552 Privates.append(
C->privates().begin(),
C->privates().end());
3553 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
3554 CopyArrayTemps.append(
C->copy_array_temps().begin(),
3555 C->copy_array_temps().end());
3563 auto *ITA = CopyArrayTemps.begin();
3564 for (
const Expr *IRef : Privates) {
3565 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
3568 if (PrivateVD->getType()->isVariablyModifiedType()) {
3572 CodeGenFunction::OpaqueValueMapping DimMapping(
3574 cast<OpaqueValueExpr>(
3575 cast<VariableArrayType>((*ITA)->getType()->getAsArrayTypeUnsafe())
3579 CGF.
EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(*ITA)->getDecl()));
3593 llvm::function_ref<llvm::Value *(
CodeGenFunction &)> NumIteratorsGen) {
3594 llvm::Value *OMPScanNumIterations = CGF.
Builder.CreateIntCast(
3595 NumIteratorsGen(CGF), CGF.
SizeTy,
false);
3603 assert(
C->getModifier() == OMPC_REDUCTION_inscan &&
3604 "Only inscan reductions are expected.");
3605 Shareds.append(
C->varlist_begin(),
C->varlist_end());
3606 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
3607 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
3608 Privates.append(
C->privates().begin(),
C->privates().end());
3609 CopyOps.append(
C->copy_ops().begin(),
C->copy_ops().end());
3610 CopyArrayElems.append(
C->copy_array_elems().begin(),
3611 C->copy_array_elems().end());
3615 llvm::Value *OMPLast = CGF.
Builder.CreateNSWSub(
3616 OMPScanNumIterations,
3617 llvm::ConstantInt::get(CGF.
SizeTy, 1,
false));
3618 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
3619 const Expr *PrivateExpr = Privates[I];
3620 const Expr *OrigExpr = Shareds[I];
3621 const Expr *CopyArrayElem = CopyArrayElems[I];
3622 CodeGenFunction::OpaqueValueMapping IdxMapping(
3624 cast<OpaqueValueExpr>(
3625 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3631 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
3632 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
3661 llvm::Value *OMPScanNumIterations = CGF.
Builder.CreateIntCast(
3662 NumIteratorsGen(CGF), CGF.
SizeTy,
false);
3669 assert(
C->getModifier() == OMPC_REDUCTION_inscan &&
3670 "Only inscan reductions are expected.");
3671 Privates.append(
C->privates().begin(),
C->privates().end());
3672 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
3673 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
3674 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
3675 CopyArrayElems.append(
C->copy_array_elems().begin(),
3676 C->copy_array_elems().end());
3678 CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S);
3687 CodeGenFunction::OMPLocalDeclMapRAII
Scope(CGF);
3691 auto &&CodeGen = [&S, OMPScanNumIterations, &LHSs, &RHSs, &CopyArrayElems,
3698 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock();
3699 llvm::BasicBlock *LoopBB = CGF.createBasicBlock(
"omp.outer.log.scan.body");
3700 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
"omp.outer.log.scan.exit");
3702 CGF.CGM.getIntrinsic(llvm::Intrinsic::log2, CGF.DoubleTy);
3704 CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy);
3705 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg);
3706 F = CGF.CGM.getIntrinsic(llvm::Intrinsic::ceil, CGF.DoubleTy);
3707 LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal);
3708 LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy);
3709 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub(
3710 OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1));
3712 CGF.EmitBlock(LoopBB);
3713 auto *Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2);
3715 auto *Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3716 Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB);
3717 Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB);
3720 llvm::BasicBlock *InnerLoopBB =
3721 CGF.createBasicBlock(
"omp.inner.log.scan.body");
3722 llvm::BasicBlock *InnerExitBB =
3723 CGF.createBasicBlock(
"omp.inner.log.scan.exit");
3724 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K);
3725 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3726 CGF.EmitBlock(InnerLoopBB);
3727 auto *IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3728 IVal->addIncoming(NMin1, LoopBB);
3730 CodeGenFunction::OMPPrivateScope PrivScope(CGF);
3731 auto *ILHS = LHSs.begin();
3732 auto *IRHS = RHSs.begin();
3733 for (
const Expr *CopyArrayElem : CopyArrayElems) {
3734 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
3735 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
3738 CodeGenFunction::OpaqueValueMapping IdxMapping(
3740 cast<OpaqueValueExpr>(
3741 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3743 LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF);
3745 PrivScope.addPrivate(LHSVD, LHSAddr);
3748 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K);
3749 CodeGenFunction::OpaqueValueMapping IdxMapping(
3751 cast<OpaqueValueExpr>(
3752 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3754 RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF);
3756 PrivScope.addPrivate(RHSVD, RHSAddr);
3760 PrivScope.Privatize();
3761 CGF.CGM.getOpenMPRuntime().emitReduction(
3762 CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
3763 {true, true, OMPD_unknown});
3765 llvm::Value *NextIVal =
3766 CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1));
3767 IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock());
3768 CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K);
3769 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3770 CGF.EmitBlock(InnerExitBB);
3772 CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1));
3773 Counter->addIncoming(Next, CGF.Builder.GetInsertBlock());
3775 llvm::Value *NextPow2K =
3776 CGF.Builder.CreateShl(Pow2K, 1,
"",
true);
3777 Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock());
3778 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(Next, LogVal);
3779 CGF.Builder.CreateCondBr(Cmp, LoopBB, ExitBB);
3781 CGF.EmitBlock(ExitBB);
3784 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());
3785 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
3786 CGF, S.getBeginLoc(), OMPD_unknown,
false,
3793 CGF.OMPFirstScanLoop =
false;
3800 bool HasLastprivates;
3803 return C->getModifier() == OMPC_REDUCTION_inscan;
3806 CodeGenFunction::OMPLocalDeclMapRAII
Scope(CGF);
3807 OMPLoopScope LoopScope(CGF, S);
3811 CodeGenFunction::OMPCancelStackRAII CancelRegion(
3812 CGF, S.getDirectiveKind(), HasCancel);
3820 const auto &&SecondGen = [&S, HasCancel,
3822 CodeGenFunction::OMPCancelStackRAII CancelRegion(
3823 CGF, S.getDirectiveKind(), HasCancel);
3834 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
3840 return HasLastprivates;
3847 if (isa<OMPNowaitClause>(
C))
3850 if (
auto *SC = dyn_cast<OMPScheduleClause>(
C)) {
3855 switch (SC->getScheduleKind()) {
3856 case OMPC_SCHEDULE_auto:
3857 case OMPC_SCHEDULE_dynamic:
3858 case OMPC_SCHEDULE_runtime:
3859 case OMPC_SCHEDULE_guided:
3860 case OMPC_SCHEDULE_static:
3873static llvm::omp::ScheduleKind
3875 switch (ScheduleClauseKind) {
3877 return llvm::omp::OMP_SCHEDULE_Default;
3878 case OMPC_SCHEDULE_auto:
3879 return llvm::omp::OMP_SCHEDULE_Auto;
3880 case OMPC_SCHEDULE_dynamic:
3881 return llvm::omp::OMP_SCHEDULE_Dynamic;
3882 case OMPC_SCHEDULE_guided:
3883 return llvm::omp::OMP_SCHEDULE_Guided;
3884 case OMPC_SCHEDULE_runtime:
3885 return llvm::omp::OMP_SCHEDULE_Runtime;
3886 case OMPC_SCHEDULE_static:
3887 return llvm::omp::OMP_SCHEDULE_Static;
3889 llvm_unreachable(
"Unhandled schedule kind");
3893 bool HasLastprivates =
false;
3894 bool UseOMPIRBuilder =
3896 auto &&CodeGen = [
this, &S, &HasLastprivates,
3899 if (UseOMPIRBuilder) {
3902 llvm::omp::ScheduleKind SchedKind = llvm::omp::OMP_SCHEDULE_Default;
3903 llvm::Value *ChunkSize =
nullptr;
3907 if (
const Expr *ChunkSizeExpr = SchedClause->getChunkSize())
3912 const Stmt *Inner = S.getRawStmt();
3913 llvm::CanonicalLoopInfo *CLI =
3916 llvm::OpenMPIRBuilder &OMPBuilder =
3918 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
3920 OMPBuilder.applyWorkshareLoop(
3921 Builder.getCurrentDebugLocation(), CLI, AllocaIP, NeedsBarrier,
3922 SchedKind, ChunkSize,
false,
3933 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
3938 if (!UseOMPIRBuilder) {
3948 bool HasLastprivates =
false;
3956 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
3969 llvm::Value *Init =
nullptr) {
3977 const Stmt *
CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
3979 bool HasLastprivates =
false;
3984 C.getIntTypeForBitwidth(32, 1);
3987 CGF.Builder.getInt32(0));
3988 llvm::ConstantInt *GlobalUBVal = CS !=
nullptr
3989 ? CGF.Builder.getInt32(CS->size() - 1)
3990 : CGF.Builder.getInt32(0);
3994 CGF.Builder.getInt32(1));
3996 CGF.Builder.getInt32(0));
4000 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
4002 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
4023 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
".omp.sections.exit");
4025 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
4026 ExitBB, CS ==
nullptr ? 1 : CS->size());
4028 unsigned CaseNumber = 0;
4030 auto CaseBB = CGF.createBasicBlock(
".omp.sections.case");
4031 CGF.EmitBlock(CaseBB);
4032 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
4033 CGF.EmitStmt(SubStmt);
4034 CGF.EmitBranch(ExitBB);
4038 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(
".omp.sections.case");
4039 CGF.EmitBlock(CaseBB);
4040 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
4042 CGF.EmitBranch(ExitBB);
4044 CGF.EmitBlock(ExitBB,
true);
4047 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
4048 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
4052 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
4053 CGF, S.getBeginLoc(), OMPD_unknown,
false,
4056 CGF.EmitOMPPrivateClause(S, LoopScope);
4058 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4059 CGF.EmitOMPReductionClauseInit(S, LoopScope);
4060 (void)LoopScope.Privatize();
4062 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4066 ScheduleKind.
Schedule = OMPC_SCHEDULE_static;
4070 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
4071 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit);
4073 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
4074 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
4075 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
4076 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
4078 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
4080 CGF.EmitOMPInnerLoop(S,
false, Cond, Inc, BodyGen,
4084 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
4085 S.getDirectiveKind());
4087 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
4088 CGF.EmitOMPReductionClauseFinal(S, OMPD_parallel);
4091 return CGF.
Builder.CreateIsNotNull(
4096 if (HasLastprivates)
4103 bool HasCancel =
false;
4104 if (
auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
4105 HasCancel = OSD->hasCancel();
4106 else if (
auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
4107 HasCancel = OPSD->hasCancel();
4108 OMPCancelStackRAII CancelRegion(*
this, S.getDirectiveKind(), HasCancel);
4125 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4126 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4128 auto FiniCB = [
this](InsertPointTy IP) {
4132 const CapturedStmt *ICS = S.getInnermostCapturedStmt();
4133 const Stmt *
CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4138 auto SectionCB = [
this, SubStmt](InsertPointTy AllocaIP,
4139 InsertPointTy CodeGenIP) {
4141 *
this, SubStmt, AllocaIP, CodeGenIP,
"section");
4143 SectionCBVector.push_back(SectionCB);
4146 auto SectionCB = [
this,
CapturedStmt](InsertPointTy AllocaIP,
4147 InsertPointTy CodeGenIP) {
4151 SectionCBVector.push_back(SectionCB);
4158 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
4159 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
4167 CGCapturedStmtInfo CGSI(*ICS,
CR_OpenMP);
4168 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*
this, &CGSI);
4169 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
4171 Builder.restoreIP(OMPBuilder.createSections(
4172 Builder, AllocaIP, SectionCBVector, PrivCB, FiniCB, S.hasCancel(),
4179 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
4194 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4196 const Stmt *SectionRegionBodyStmt = S.getAssociatedStmt();
4197 auto FiniCB = [
this](InsertPointTy IP) {
4201 auto BodyGenCB = [SectionRegionBodyStmt,
this](InsertPointTy AllocaIP,
4202 InsertPointTy CodeGenIP) {
4204 *
this, SectionRegionBodyStmt, AllocaIP, CodeGenIP,
"section");
4207 LexicalScope
Scope(*
this, S.getSourceRange());
4209 Builder.restoreIP(OMPBuilder.createSection(
Builder, BodyGenCB, FiniCB));
4213 LexicalScope
Scope(*
this, S.getSourceRange());
4228 CopyprivateVars.append(
C->varlists().begin(),
C->varlists().end());
4229 DestExprs.append(
C->destination_exprs().begin(),
4230 C->destination_exprs().end());
4231 SrcExprs.append(
C->source_exprs().begin(),
C->source_exprs().end());
4232 AssignmentOps.append(
C->assignment_ops().begin(),
4233 C->assignment_ops().end());
4238 OMPPrivateScope SingleScope(CGF);
4241 (void)SingleScope.Privatize();
4242 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4247 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
4249 CopyprivateVars, DestExprs,
4250 SrcExprs, AssignmentOps);
4254 if (!S.getSingleClause<
OMPNowaitClause>() && CopyprivateVars.empty()) {
4256 *
this, S.getBeginLoc(),
4274 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4276 const Stmt *MasterRegionBodyStmt = S.getAssociatedStmt();
4278 auto FiniCB = [
this](InsertPointTy IP) {
4282 auto BodyGenCB = [MasterRegionBodyStmt,
this](InsertPointTy AllocaIP,
4283 InsertPointTy CodeGenIP) {
4285 *
this, MasterRegionBodyStmt, AllocaIP, CodeGenIP,
"master");
4288 LexicalScope
Scope(*
this, S.getSourceRange());
4290 Builder.restoreIP(OMPBuilder.createMaster(
Builder, BodyGenCB, FiniCB));
4294 LexicalScope
Scope(*
this, S.getSourceRange());
4304 Expr *Filter =
nullptr;
4306 Filter = FilterClause->getThreadID();
4314 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4316 const Stmt *MaskedRegionBodyStmt = S.getAssociatedStmt();
4319 Filter = FilterClause->getThreadID();
4320 llvm::Value *FilterVal =
Filter
4324 auto FiniCB = [
this](InsertPointTy IP) {
4328 auto BodyGenCB = [MaskedRegionBodyStmt,
this](InsertPointTy AllocaIP,
4329 InsertPointTy CodeGenIP) {
4331 *
this, MaskedRegionBodyStmt, AllocaIP, CodeGenIP,
"masked");
4334 LexicalScope
Scope(*
this, S.getSourceRange());
4337 OMPBuilder.createMasked(
Builder, BodyGenCB, FiniCB, FilterVal));
4341 LexicalScope
Scope(*
this, S.getSourceRange());
4349 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4351 const Stmt *CriticalRegionBodyStmt = S.getAssociatedStmt();
4352 const Expr *Hint =
nullptr;
4353 if (
const auto *HintClause = S.getSingleClause<
OMPHintClause>())
4354 Hint = HintClause->getHint();
4359 llvm::Value *HintInst =
nullptr;
4364 auto FiniCB = [
this](InsertPointTy IP) {
4368 auto BodyGenCB = [CriticalRegionBodyStmt,
this](InsertPointTy AllocaIP,
4369 InsertPointTy CodeGenIP) {
4371 *
this, CriticalRegionBodyStmt, AllocaIP, CodeGenIP,
"critical");
4374 LexicalScope
Scope(*
this, S.getSourceRange());
4376 Builder.restoreIP(OMPBuilder.createCritical(
4377 Builder, BodyGenCB, FiniCB, S.getDirectiveName().getAsString(),
4385 CGF.
EmitStmt(S.getAssociatedStmt());
4387 const Expr *Hint =
nullptr;
4388 if (
const auto *HintClause = S.getSingleClause<
OMPHintClause>())
4389 Hint = HintClause->getHint();
4390 LexicalScope
Scope(*
this, S.getSourceRange());
4393 S.getDirectiveName().getAsString(),
4394 CodeGen, S.getBeginLoc(), Hint);
4408 CodeGenFunction::OMPLocalDeclMapRAII
Scope(CGF);
4410 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
4411 OMPLoopScope LoopScope(CGF, S);
4416 return C->getModifier() == OMPC_REDUCTION_inscan;
4442 CodeGenFunction::OMPLocalDeclMapRAII
Scope(CGF);
4444 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
4445 OMPLoopScope LoopScope(CGF, S);
4450 return C->getModifier() == OMPC_REDUCTION_inscan;
4471 OMPPrivateScope PrivateScope(CGF);
4476 (void)PrivateScope.Privatize();
4499 CGF.EmitSections(S);
4513class CheckVarsEscapingUntiedTaskDeclContext final
4518 explicit CheckVarsEscapingUntiedTaskDeclContext() =
default;
4519 virtual ~CheckVarsEscapingUntiedTaskDeclContext() =
default;
4520 void VisitDeclStmt(
const DeclStmt *S) {
4524 for (
const Decl *D : S->decls()) {
4525 if (
const auto *VD = dyn_cast_or_null<VarDecl>(D))
4527 PrivateDecls.push_back(VD);
4533 void VisitBlockExpr(
const BlockExpr *) {}
4534 void VisitStmt(
const Stmt *S) {
4537 for (
const Stmt *Child : S->children())
4551 bool OmpAllMemory =
false;
4554 return C->getDependencyKind() == OMPC_DEPEND_outallmemory ||
4555 C->getDependencyKind() == OMPC_DEPEND_inoutallmemory;
4557 OmpAllMemory =
true;
4562 Data.Dependences.emplace_back(OMPC_DEPEND_outallmemory,
4571 if (Kind == OMPC_DEPEND_outallmemory || Kind == OMPC_DEPEND_inoutallmemory)
4573 if (OmpAllMemory && (Kind == OMPC_DEPEND_out || Kind == OMPC_DEPEND_inout))
4576 Data.Dependences.emplace_back(
C->getDependencyKind(),
C->getModifier());
4577 DD.
DepExprs.append(
C->varlist_begin(),
C->varlist_end());
4586 const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
4588 auto PartId = std::next(I);
4589 auto TaskT = std::next(I, 4);
4594 const Expr *Cond = Clause->getCondition();
4597 Data.Final.setInt(CondConstant);
4602 Data.Final.setInt(
false);
4606 const Expr *Prio = Clause->getPriority();
4607 Data.Priority.setInt(
true);
4618 auto IRef =
C->varlist_begin();
4619 for (
const Expr *IInit :
C->private_copies()) {
4620 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4621 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4622 Data.PrivateVars.push_back(*IRef);
4623 Data.PrivateCopies.push_back(IInit);
4628 EmittedAsPrivate.clear();
4631 auto IRef =
C->varlist_begin();
4632 auto IElemInitRef =
C->inits().begin();
4633 for (
const Expr *IInit :
C->private_copies()) {
4634 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4635 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4636 Data.FirstprivateVars.push_back(*IRef);
4637 Data.FirstprivateCopies.push_back(IInit);
4638 Data.FirstprivateInits.push_back(*IElemInitRef);
4645 llvm::MapVector<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
4647 auto IRef =
C->varlist_begin();
4648 auto ID =
C->destination_exprs().begin();
4649 for (
const Expr *IInit :
C->private_copies()) {
4650 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4651 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4652 Data.LastprivateVars.push_back(*IRef);
4653 Data.LastprivateCopies.push_back(IInit);
4655 LastprivateDstsOrigs.insert(
4656 std::make_pair(cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
4657 cast<DeclRefExpr>(*IRef)));
4665 Data.ReductionVars.append(
C->varlist_begin(),
C->varlist_end());
4666 Data.ReductionOrigs.append(
C->varlist_begin(),
C->varlist_end());
4667 Data.ReductionCopies.append(
C->privates().begin(),
C->privates().end());
4668 Data.ReductionOps.append(
C->reduction_ops().begin(),
4669 C->reduction_ops().end());
4670 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
4671 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
4674 *
this, S.getBeginLoc(), LHSs, RHSs,
Data);
4679 CheckVarsEscapingUntiedTaskDeclContext Checker;
4680 Checker.Visit(S.getInnermostCapturedStmt()->getCapturedStmt());
4681 Data.PrivateLocals.append(Checker.getPrivateDecls().begin(),
4682 Checker.getPrivateDecls().end());
4684 auto &&CodeGen = [&
Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
4687 llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
4688 std::pair<Address, Address>>
4691 OMPPrivateScope
Scope(CGF);
4693 if (
auto *DI = CGF.getDebugInfo()) {
4694 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields =
4695 CGF.CapturedStmtInfo->getCaptureFields();
4696 llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue();
4697 if (CaptureFields.size() && ContextValue) {
4698 unsigned CharWidth = CGF.getContext().getCharWidth();
4712 for (
auto It = CaptureFields.begin(); It != CaptureFields.end(); ++It) {
4713 const VarDecl *SharedVar = It->first;
4716 CGF.getContext().getASTRecordLayout(CaptureRecord);
4719 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
4720 (void)DI->EmitDeclareOfAutoVariable(SharedVar, ContextValue,
4721 CGF.Builder,
false);
4722 llvm::Instruction &
Last = CGF.Builder.GetInsertBlock()->back();
4725 if (
auto DDI = dyn_cast<llvm::DbgVariableIntrinsic>(&
Last)) {
4729 Ops.push_back(llvm::dwarf::DW_OP_plus_uconst);
4732 Ops.push_back(llvm::dwarf::DW_OP_deref);
4733 auto &Ctx = DDI->getContext();
4734 llvm::DIExpression *DIExpr = llvm::DIExpression::get(Ctx, Ops);
4735 Last.setOperand(2, llvm::MetadataAsValue::get(Ctx, DIExpr));
4741 if (!
Data.PrivateVars.empty() || !
Data.FirstprivateVars.empty() ||
4742 !
Data.LastprivateVars.empty() || !
Data.PrivateLocals.empty()) {
4743 enum { PrivatesParam = 2, CopyFnParam = 3 };
4744 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
4745 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
4746 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
4747 CS->getCapturedDecl()->getParam(PrivatesParam)));
4752 CallArgs.push_back(PrivatesPtr);
4753 ParamTypes.push_back(PrivatesPtr->getType());
4754 for (
const Expr *E :
Data.PrivateVars) {
4755 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4756 Address PrivatePtr = CGF.CreateMemTemp(
4757 CGF.getContext().getPointerType(E->
getType()),
".priv.ptr.addr");
4758 PrivatePtrs.emplace_back(VD, PrivatePtr);
4760 ParamTypes.push_back(PrivatePtr.
getType());
4762 for (
const Expr *E :
Data.FirstprivateVars) {
4763 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4765 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->
getType()),
4766 ".firstpriv.ptr.addr");
4767 PrivatePtrs.emplace_back(VD, PrivatePtr);
4768 FirstprivatePtrs.emplace_back(VD, PrivatePtr);
4770 ParamTypes.push_back(PrivatePtr.
getType());
4772 for (
const Expr *E :
Data.LastprivateVars) {
4773 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4775 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->
getType()),
4776 ".lastpriv.ptr.addr");
4777 PrivatePtrs.emplace_back(VD, PrivatePtr);
4779 ParamTypes.push_back(PrivatePtr.
getType());
4784 Ty = CGF.getContext().getPointerType(Ty);
4786 Ty = CGF.getContext().getPointerType(Ty);
4787 Address PrivatePtr = CGF.CreateMemTemp(
4788 CGF.getContext().getPointerType(Ty),
".local.ptr.addr");
4789 auto Result = UntiedLocalVars.insert(
4792 if (
Result.second ==
false)
4793 *
Result.first = std::make_pair(
4796 ParamTypes.push_back(PrivatePtr.
getType());
4798 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
4800 CopyFn = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4801 CopyFn, CopyFnTy->getPointerTo());
4802 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
4803 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
4804 for (
const auto &Pair : LastprivateDstsOrigs) {
4805 const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
4808 CGF.CapturedStmtInfo->lookup(OrigVD) !=
nullptr,
4810 Pair.second->getExprLoc());
4811 Scope.addPrivate(Pair.first, CGF.EmitLValue(&DRE).getAddress(CGF));
4813 for (
const auto &Pair : PrivatePtrs) {
4815 CGF.Builder.CreateLoad(Pair.second),
4816 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
4817 CGF.getContext().getDeclAlign(Pair.first));
4818 Scope.addPrivate(Pair.first, Replacement);
4819 if (
auto *DI = CGF.getDebugInfo())
4820 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
4821 (void)DI->EmitDeclareOfAutoVariable(
4822 Pair.first, Pair.second.getPointer(), CGF.Builder,
4827 for (
auto &Pair : UntiedLocalVars) {
4828 QualType VDType = Pair.first->getType().getNonReferenceType();
4830 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
4833 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(VDType)),
4834 CGF.getPointerAlign());
4835 Pair.second.first = Replacement;
4836 Ptr = CGF.Builder.CreateLoad(Replacement);
4837 Replacement =
Address(Ptr, CGF.ConvertTypeForMem(VDType),
4838 CGF.getContext().getDeclAlign(Pair.first));
4839 Pair.second.second = Replacement;
4841 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
4842 Address Replacement(Ptr, CGF.ConvertTypeForMem(VDType),
4843 CGF.getContext().getDeclAlign(Pair.first));
4844 Pair.second.first = Replacement;
4848 if (
Data.Reductions) {
4849 OMPPrivateScope FirstprivateScope(CGF);
4850 for (
const auto &Pair : FirstprivatePtrs) {
4852 CGF.Builder.CreateLoad(Pair.second),
4853 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
4854 CGF.getContext().getDeclAlign(Pair.first));
4855 FirstprivateScope.addPrivate(Pair.first, Replacement);
4857 (void)FirstprivateScope.Privatize();
4858 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
4860 Data.ReductionCopies,
Data.ReductionOps);
4861 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
4862 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
4863 for (
unsigned Cnt = 0, E =
Data.ReductionVars.size(); Cnt < E; ++Cnt) {
4864 RedCG.emitSharedOrigLValue(CGF, Cnt);
4865 RedCG.emitAggregateType(CGF, Cnt);
4869 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),