27 #include "llvm/ADT/SmallSet.h"
28 #include "llvm/BinaryFormat/Dwarf.h"
29 #include "llvm/Frontend/OpenMP/OMPConstants.h"
30 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
31 #include "llvm/IR/Constants.h"
32 #include "llvm/IR/DebugInfoMetadata.h"
33 #include "llvm/IR/Instructions.h"
34 #include "llvm/IR/IntrinsicInst.h"
35 #include "llvm/IR/Metadata.h"
36 #include "llvm/Support/AtomicOrdering.h"
37 using namespace clang;
38 using namespace CodeGen;
39 using namespace llvm::omp;
48 for (
const auto *C : S.clauses()) {
50 if (
const auto *PreInit =
51 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
52 for (
const auto *I : PreInit->decls()) {
53 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
71 cast<BlockDecl>(CGF.
CurCodeDecl)->capturesVariable(VD));
78 const bool EmitPreInitStmt =
true)
82 emitPreInitStmt(CGF, S);
83 if (!CapturedRegion.hasValue())
85 assert(S.hasAssociatedStmt() &&
86 "Expected associated statement for inlined directive.");
87 const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
88 for (
const auto &C : CS->
captures()) {
89 if (
C.capturesVariable() ||
C.capturesVariableByCopy()) {
90 auto *VD =
C.getCapturedVar();
92 "Canonical decl must be captured.");
107 class OMPParallelScope final :
public OMPLexicalScope {
117 : OMPLexicalScope(CGF, S,
llvm::
None,
118 EmitPreInitStmt(S)) {}
123 class OMPTeamsScope final :
public OMPLexicalScope {
132 : OMPLexicalScope(CGF, S,
llvm::
None,
133 EmitPreInitStmt(S)) {}
142 if (
auto *LD = dyn_cast<OMPLoopDirective>(&S)) {
144 for (
const auto *E : LD->counters()) {
145 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
152 for (
const Expr *IRef :
C->varlists()) {
154 cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
155 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
156 QualType OrigVDTy = OrigVD->getType().getNonReferenceType();
166 (void)PreCondVars.
apply(CGF);
169 LD->getInnermostCapturedStmt()->getCapturedStmt(),
170 true, LD->getLoopsNumber(),
171 [&CGF](
unsigned Cnt,
const Stmt *CurStmt) {
172 if (
const auto *CXXFor = dyn_cast<CXXForRangeStmt>(CurStmt)) {
173 if (
const Stmt *Init = CXXFor->getInit())
175 CGF.
EmitStmt(CXXFor->getRangeStmt());
180 PreInits = cast_or_null<DeclStmt>(LD->getPreInits());
181 }
else if (
const auto *Tile = dyn_cast<OMPTileDirective>(&S)) {
182 PreInits = cast_or_null<DeclStmt>(Tile->getPreInits());
183 }
else if (
const auto *Unroll = dyn_cast<OMPUnrollDirective>(&S)) {
184 PreInits = cast_or_null<DeclStmt>(Unroll->getPreInits());
186 llvm_unreachable(
"Unknown loop-based directive kind.");
189 for (
const auto *I : PreInits->
decls())
198 emitPreInitStmt(CGF, S);
209 cast<BlockDecl>(CGF.
CurCodeDecl)->capturesVariable(VD));
215 InlinedShareds(CGF) {
216 for (
const auto *C : S.clauses()) {
218 if (
const auto *PreInit =
219 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
220 for (
const auto *I : PreInit->decls()) {
221 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
230 }
else if (
const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
231 for (
const Expr *E : UDP->varlists()) {
232 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
233 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
236 }
else if (
const auto *UDP = dyn_cast<OMPUseDeviceAddrClause>(C)) {
237 for (
const Expr *E : UDP->varlists()) {
239 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
246 if (
const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
247 if (
const Expr *E = TG->getReductionRef())
248 CGF.
EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
254 if (
C->getModifier() != OMPC_REDUCTION_inscan)
256 for (
const Expr *E :
C->copy_array_temps())
257 CopyArrayTemps.insert(cast<DeclRefExpr>(E)->getDecl());
259 const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
262 if (
C.capturesVariable() ||
C.capturesVariableByCopy()) {
263 auto *VD =
C.getCapturedVar();
264 if (CopyArrayTemps.contains(VD))
267 "Canonical decl must be captured.");
269 isCapturedVar(CGF, VD) ||
290 if (
const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
291 if (
const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
292 OrigVD = OrigVD->getCanonicalDecl();
294 LambdaCaptureFields.lookup(OrigVD) ||
295 (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
296 (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
299 return EmitLValue(&DRE);
302 return EmitLValue(E);
307 llvm::Value *Size =
nullptr;
308 auto SizeInChars = C.getTypeSizeInChars(Ty);
309 if (SizeInChars.isZero()) {
315 Size ? Builder.CreateNUWMul(Size, VlaSize.
NumElts) : VlaSize.
NumElts;
317 SizeInChars = C.getTypeSizeInChars(Ty);
318 if (SizeInChars.isZero())
319 return llvm::ConstantInt::get(SizeTy, 0);
320 return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
322 return CGM.getSize(SizeInChars);
327 const RecordDecl *RD = S.getCapturedRecordDecl();
329 auto CurCap = S.captures().begin();
331 E = S.capture_init_end();
332 I != E; ++I, ++CurField, ++CurCap) {
333 if (CurField->hasCapturedVLAType()) {
336 CapturedVars.push_back(Val);
337 }
else if (CurCap->capturesThis()) {
338 CapturedVars.push_back(CXXThisValue);
339 }
else if (CurCap->capturesVariableByCopy()) {
340 llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
344 if (!CurField->getType()->isAnyPointerType()) {
346 Address DstAddr = CreateMemTemp(
348 Twine(CurCap->getCapturedVar()->getName(),
".casted"));
351 llvm::Value *SrcAddrVal = EmitScalarConversion(
355 MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
361 CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
363 CapturedVars.push_back(CV);
365 assert(CurCap->capturesVariable() &&
"Expected capture by reference.");
366 CapturedVars.push_back(EmitLValue(*I).getAddress(*this).getPointer());
386 return C.getLValueReferenceType(
392 if (
const auto *VLA = dyn_cast<VariableArrayType>(A))
394 if (!A->isVariablyModifiedType())
395 return C.getCanonicalType(T);
397 return C.getCanonicalParamType(T);
402 struct FunctionOptions {
407 const bool UIntPtrCastRequired =
true;
410 const bool RegisterCastedArgsOnly =
false;
412 const StringRef FunctionName;
415 explicit FunctionOptions(
const CapturedStmt *S,
bool UIntPtrCastRequired,
416 bool RegisterCastedArgsOnly, StringRef FunctionName,
418 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
419 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
420 FunctionName(FunctionName), Loc(Loc) {}
426 llvm::MapVector<
const Decl *, std::pair<const VarDecl *, Address>>
428 llvm::DenseMap<
const Decl *, std::pair<const Expr *, llvm::Value *>>
430 llvm::Value *&CXXThisValue,
const FunctionOptions &FO) {
432 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
433 assert(CD->
hasBody() &&
"missing CapturedDecl body");
435 CXXThisValue =
nullptr;
445 auto I = FO.S->captures().begin();
447 if (!FO.UIntPtrCastRequired) {
467 if (FO.UIntPtrCastRequired &&
469 I->capturesVariableArrayType()))
472 if (I->capturesVariable() || I->capturesVariableByCopy()) {
473 CapVar = I->getCapturedVar();
475 }
else if (I->capturesThis()) {
478 assert(I->capturesVariableArrayType());
488 }
else if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
490 Ctx, DebugFunctionDecl,
491 CapVar ? CapVar->
getBeginLoc() : FD->getBeginLoc(),
492 CapVar ? CapVar->
getLocation() : FD->getLocation(), II, ArgType,
498 Args.emplace_back(Arg);
500 TargetArgs.emplace_back(
501 FO.UIntPtrCastRequired
522 F->setDoesNotThrow();
523 F->setDoesNotRecurse();
527 F->removeFnAttr(llvm::Attribute::NoInline);
528 F->addFnAttr(llvm::Attribute::AlwaysInline);
533 FO.UIntPtrCastRequired ? FO.Loc : FO.S->getBeginLoc(),
534 FO.UIntPtrCastRequired ? FO.Loc
537 I = FO.S->captures().begin();
541 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
549 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
550 const VarDecl *CurVD = I->getCapturedVar();
551 if (!FO.RegisterCastedArgsOnly)
552 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
560 if (FD->hasCapturedVLAType()) {
561 if (FO.UIntPtrCastRequired) {
564 Args[Cnt]->getName(), ArgLVal),
569 VLASizes.try_emplace(Args[Cnt], VAT->
getSizeExpr(), ExprArg);
570 }
else if (I->capturesVariable()) {
571 const VarDecl *Var = I->getCapturedVar();
581 if (!FO.RegisterCastedArgsOnly) {
585 }
else if (I->capturesVariableByCopy()) {
586 assert(!FD->getType()->isAnyPointerType() &&
587 "Not expecting a captured pointer.");
588 const VarDecl *Var = I->getCapturedVar();
589 LocalAddrs.insert({Args[Cnt],
590 {Var, FO.UIntPtrCastRequired
592 CGF, I->getLocation(), FD->getType(),
593 Args[Cnt]->getName(), ArgLVal)
597 assert(I->capturesThis());
599 LocalAddrs.insert({Args[Cnt], {
nullptr, ArgLVal.
getAddress(CGF)}});
613 "CapturedStmtInfo should be set when generating the captured function");
616 bool NeedWrapperFunction =
617 getDebugInfo() && CGM.getCodeGenOpts().hasReducedDebugInfo();
619 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
620 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
622 llvm::raw_svector_ostream Out(Buffer);
623 Out << CapturedStmtInfo->getHelperName();
624 if (NeedWrapperFunction)
626 FunctionOptions FO(&S, !NeedWrapperFunction,
false,
629 VLASizes, CXXThisValue, FO);
631 for (
const auto &LocalAddrPair : LocalAddrs) {
632 if (LocalAddrPair.second.first) {
633 LocalScope.
addPrivate(LocalAddrPair.second.first,
634 LocalAddrPair.second.second);
638 for (
const auto &VLASizePair : VLASizes)
639 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
641 CapturedStmtInfo->EmitBody(*
this, CD->
getBody());
644 if (!NeedWrapperFunction)
647 FunctionOptions WrapperFO(&S,
true,
649 CapturedStmtInfo->getHelperName(), Loc);
655 llvm::Function *WrapperF =
657 WrapperCGF.CXXThisValue, WrapperFO);
659 auto *PI = F->arg_begin();
660 for (
const auto *Arg : Args) {
662 auto I = LocalAddrs.find(Arg);
663 if (I != LocalAddrs.end()) {
666 I->second.first ? I->second.first->getType() : Arg->getType(),
676 auto EI = VLASizes.find(Arg);
677 if (EI != VLASizes.end()) {
689 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, Loc, F, CallArgs);
705 llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
706 SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.
getElementType());
709 llvm::Value *DestBegin = DestAddr.
getPointer();
711 llvm::Value *DestEnd =
712 Builder.CreateGEP(DestAddr.
getElementType(), DestBegin, NumElements);
714 llvm::BasicBlock *BodyBB = createBasicBlock(
"omp.arraycpy.body");
715 llvm::BasicBlock *DoneBB = createBasicBlock(
"omp.arraycpy.done");
716 llvm::Value *IsEmpty =
717 Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arraycpy.isempty");
718 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
721 llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
724 CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
726 llvm::PHINode *SrcElementPHI =
727 Builder.CreatePHI(SrcBegin->getType(), 2,
"omp.arraycpy.srcElementPast");
728 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
733 llvm::PHINode *DestElementPHI = Builder.CreatePHI(
734 DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
735 DestElementPHI->addIncoming(DestBegin, EntryBB);
741 CopyGen(DestElementCurrent, SrcElementCurrent);
744 llvm::Value *DestElementNext =
745 Builder.CreateConstGEP1_32(DestAddr.
getElementType(), DestElementPHI,
746 1,
"omp.arraycpy.dest.element");
747 llvm::Value *SrcElementNext =
748 Builder.CreateConstGEP1_32(SrcAddr.
getElementType(), SrcElementPHI,
749 1,
"omp.arraycpy.src.element");
752 Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
753 Builder.CreateCondBr(Done, DoneBB, BodyBB);
754 DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
755 SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
758 EmitBlock(DoneBB,
true);
765 const auto *BO = dyn_cast<BinaryOperator>(Copy);
766 if (BO && BO->getOpcode() == BO_Assign) {
768 LValue Dest = MakeAddrLValue(DestAddr, OriginalType);
769 LValue Src = MakeAddrLValue(SrcAddr, OriginalType);
770 EmitAggregateAssign(Dest, Src, OriginalType);
774 EmitOMPAggregateAssign(
775 DestAddr, SrcAddr, OriginalType,
776 [
this, Copy, SrcVD, DestVD](
Address DestElement,
Address SrcElement) {
784 EmitIgnoredExpr(Copy);
794 EmitIgnoredExpr(Copy);
800 if (!HaveInsertPoint())
802 bool DeviceConstTarget =
803 getLangOpts().OpenMPIsDevice &&
805 bool FirstprivateIsLastprivate =
false;
806 llvm::DenseMap<const VarDecl *, OpenMPLastprivateModifier> Lastprivates;
808 for (
const auto *D : C->varlists())
809 Lastprivates.try_emplace(
818 bool MustEmitFirstprivateCopy =
819 CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
821 const auto *IRef = C->varlist_begin();
822 const auto *InitsRef = C->inits().begin();
823 for (
const Expr *IInit : C->private_copies()) {
824 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
825 bool ThisFirstprivateIsLastprivate =
826 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
827 const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
828 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
829 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
831 (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())) {
832 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
839 if (DeviceConstTarget && OrigVD->getType().isConstant(getContext()) &&
841 (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())) {
842 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
847 FirstprivateIsLastprivate =
848 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
849 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
851 cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
869 assert(!CE &&
"Expected non-constant firstprivate.");
870 OriginalLVal = EmitLValue(&DRE);
873 OriginalLVal = EmitLValue(&DRE);
882 if (!isa<CXXConstructExpr>(Init) || isTrivialInitializer(Init)) {
885 EmitAggregateAssign(Dest, OriginalLVal,
Type);
887 EmitOMPAggregateAssign(
893 RunCleanupsScope InitScope(*this);
895 setAddrOfLocalVar(VDInit, SrcElement);
896 EmitAnyExprToMem(Init, DestElement,
897 Init->getType().getQualifiers(),
899 LocalDeclMap.erase(VDInit);
902 EmitAutoVarCleanups(Emission);
910 setAddrOfLocalVar(VDInit, OriginalAddr);
912 LocalDeclMap.erase(VDInit);
913 Address VDAddr = GetAddrOfLocalVar(VD);
914 if (ThisFirstprivateIsLastprivate &&
915 Lastprivates[OrigVD->getCanonicalDecl()] ==
916 OMPC_LASTPRIVATE_conditional) {
919 EmitLoadOfScalar(MakeAddrLValue(VDAddr, (*IRef)->
getType(),
921 (*IRef)->getExprLoc());
922 VDAddr = CGM.getOpenMPRuntime().emitLastprivateConditionalInit(
924 EmitStoreOfScalar(
V, MakeAddrLValue(VDAddr, (*IRef)->
getType(),
926 LocalDeclMap.erase(VD);
927 setAddrOfLocalVar(VD, VDAddr);
929 IsRegistered = PrivateScope.
addPrivate(OrigVD, VDAddr);
931 assert(IsRegistered &&
932 "firstprivate var already registered as private");
940 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
946 if (!HaveInsertPoint())
950 auto IRef = C->varlist_begin();
951 for (
const Expr *IInit : C->private_copies()) {
952 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
953 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
954 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
958 PrivateScope.
addPrivate(OrigVD, GetAddrOfLocalVar(VD));
959 assert(IsRegistered &&
"private var already registered as private");
969 if (!HaveInsertPoint())
976 llvm::BasicBlock *CopyBegin =
nullptr, *CopyEnd =
nullptr;
978 auto IRef = C->varlist_begin();
979 auto ISrcRef = C->source_exprs().begin();
980 auto IDestRef = C->destination_exprs().begin();
981 for (
const Expr *AssignOp : C->assignment_ops()) {
982 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
989 if (getLangOpts().OpenMPUseTLS &&
990 getContext().getTargetInfo().isTLSSupported()) {
991 assert(CapturedStmtInfo->lookup(VD) &&
992 "Copyin threadprivates should have been captured!");
995 MasterAddr = EmitLValue(&DRE).getAddress(*
this);
996 LocalDeclMap.erase(VD);
1000 : CGM.GetAddrOfGlobal(VD),
1001 CGM.getTypes().ConvertTypeForMem(VD->
getType()),
1002 getContext().getDeclAlign(VD));
1005 Address PrivateAddr = EmitLValue(*IRef).getAddress(*
this);
1006 if (CopiedVars.size() == 1) {
1009 CopyBegin = createBasicBlock(
"copyin.not.master");
1010 CopyEnd = createBasicBlock(
"copyin.not.master.end");
1012 auto *MasterAddrInt =
1013 Builder.CreatePtrToInt(MasterAddr.
getPointer(), CGM.IntPtrTy);
1014 auto *PrivateAddrInt =
1015 Builder.CreatePtrToInt(PrivateAddr.
getPointer(), CGM.IntPtrTy);
1016 Builder.CreateCondBr(
1017 Builder.CreateICmpNE(MasterAddrInt, PrivateAddrInt), CopyBegin,
1019 EmitBlock(CopyBegin);
1022 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1023 const auto *DestVD =
1024 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1025 EmitOMPCopy(
Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
1034 EmitBlock(CopyEnd,
true);
1042 if (!HaveInsertPoint())
1044 bool HasAtLeastOneLastprivate =
false;
1047 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1048 for (
const Expr *C : LoopDirective->counters()) {
1055 HasAtLeastOneLastprivate =
true;
1057 !getLangOpts().OpenMPSimd)
1059 const auto *IRef = C->varlist_begin();
1060 const auto *IDestRef = C->destination_exprs().begin();
1061 for (
const Expr *IInit : C->private_copies()) {
1064 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1067 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
1068 const auto *DestVD =
1069 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1072 CapturedStmtInfo->lookup(OrigVD) !=
nullptr,
1074 PrivateScope.
addPrivate(DestVD, EmitLValue(&DRE).getAddress(*
this));
1078 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
1079 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
1081 if (C->getKind() == OMPC_LASTPRIVATE_conditional) {
1082 VDAddr = CGM.getOpenMPRuntime().emitLastprivateConditionalInit(
1084 setAddrOfLocalVar(VD, VDAddr);
1088 VDAddr = GetAddrOfLocalVar(VD);
1090 bool IsRegistered = PrivateScope.
addPrivate(OrigVD, VDAddr);
1091 assert(IsRegistered &&
1092 "lastprivate var already registered as private");
1100 return HasAtLeastOneLastprivate;
1105 llvm::Value *IsLastIterCond) {
1106 if (!HaveInsertPoint())
1114 llvm::BasicBlock *ThenBB =
nullptr;
1115 llvm::BasicBlock *DoneBB =
nullptr;
1116 if (IsLastIterCond) {
1119 if (!getLangOpts().OpenMPSimd &&
1122 return C->getKind() == OMPC_LASTPRIVATE_conditional;
1124 CGM.getOpenMPRuntime().emitBarrierCall(*
this, D.
getBeginLoc(),
1129 ThenBB = createBasicBlock(
".omp.lastprivate.then");
1130 DoneBB = createBasicBlock(
".omp.lastprivate.done");
1131 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1135 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1136 if (
const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1137 auto IC = LoopDirective->counters().begin();
1138 for (
const Expr *F : LoopDirective->finals()) {
1140 cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1142 AlreadyEmittedVars.insert(D);
1144 LoopCountersAndUpdates[D] = F;
1149 auto IRef = C->varlist_begin();
1150 auto ISrcRef = C->source_exprs().begin();
1151 auto IDestRef = C->destination_exprs().begin();
1152 for (
const Expr *AssignOp : C->assignment_ops()) {
1153 const auto *PrivateVD =
1154 cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1156 const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1157 if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1161 if (
const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1162 EmitIgnoredExpr(FinalExpr);
1164 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1165 const auto *DestVD =
1166 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1168 Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
1169 if (
const auto *RefTy = PrivateVD->getType()->getAs<
ReferenceType>())
1171 Builder.CreateLoad(PrivateAddr),
1172 CGM.getTypes().ConvertTypeForMem(RefTy->getPointeeType()),
1173 CGM.getNaturalTypeAlignment(RefTy->getPointeeType()));
1175 if (C->getKind() == OMPC_LASTPRIVATE_conditional)
1176 CGM.getOpenMPRuntime().emitLastprivateConditionalFinalUpdate(
1177 *
this, MakeAddrLValue(PrivateAddr, (*IRef)->
getType()), PrivateVD,
1178 (*IRef)->getExprLoc());
1180 Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1181 EmitOMPCopy(
Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1187 if (
const Expr *PostUpdate = C->getPostUpdateExpr())
1188 EmitIgnoredExpr(PostUpdate);
1191 EmitBlock(DoneBB,
true);
1197 if (!HaveInsertPoint())
1208 if (ForInscan != (C->getModifier() == OMPC_REDUCTION_inscan))
1210 Shareds.append(C->varlist_begin(), C->varlist_end());
1211 Privates.append(C->privates().begin(), C->privates().end());
1212 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1213 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1214 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1215 if (C->getModifier() == OMPC_REDUCTION_task) {
1216 Data.
ReductionVars.append(C->privates().begin(), C->privates().end());
1217 Data.
ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
1218 Data.
ReductionCopies.append(C->privates().begin(), C->privates().end());
1220 C->reduction_ops().end());
1221 TaskLHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1222 TaskRHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1227 auto *ILHS = LHSs.begin();
1228 auto *IRHS = RHSs.begin();
1229 auto *IPriv = Privates.begin();
1230 for (
const Expr *IRef : Shareds) {
1231 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1239 CGF.EmitAutoVarInit(Emission);
1242 EmitAutoVarCleanups(Emission);
1247 assert(IsRegistered &&
"private var already registered as private");
1251 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1252 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1254 bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1260 PrivateScope.
addPrivate(RHSVD, GetAddrOfLocalVar(PrivateVD));
1262 isa<ArraySubscriptExpr>(IRef)) {
1267 PrivateScope.
addPrivate(RHSVD, Builder.CreateElementBitCast(
1268 GetAddrOfLocalVar(PrivateVD),
1269 ConvertTypeForMem(RHSVD->getType()),
1273 bool IsArray = getContext().getAsArrayType(
Type) !=
nullptr;
1278 OriginalAddr = Builder.CreateElementBitCast(
1279 OriginalAddr, ConvertTypeForMem(LHSVD->getType()),
"lhs.begin");
1281 PrivateScope.
addPrivate(LHSVD, OriginalAddr);
1283 RHSVD, IsArray ? Builder.CreateElementBitCast(
1284 GetAddrOfLocalVar(PrivateVD),
1285 ConvertTypeForMem(RHSVD->getType()),
"rhs.begin")
1286 : GetAddrOfLocalVar(PrivateVD));
1297 llvm::Value *ReductionDesc = CGM.getOpenMPRuntime().emitTaskReductionInit(
1298 *
this, D.
getBeginLoc(), TaskLHSs, TaskRHSs, Data);
1299 const Expr *TaskRedRef =
nullptr;
1302 TaskRedRef = cast<OMPParallelDirective>(D).getTaskReductionRefExpr();
1305 TaskRedRef = cast<OMPForDirective>(D).getTaskReductionRefExpr();
1308 TaskRedRef = cast<OMPSectionsDirective>(D).getTaskReductionRefExpr();
1310 case OMPD_parallel_for:
1311 TaskRedRef = cast<OMPParallelForDirective>(D).getTaskReductionRefExpr();
1313 case OMPD_parallel_master:
1315 cast<OMPParallelMasterDirective>(D).getTaskReductionRefExpr();
1317 case OMPD_parallel_sections:
1319 cast<OMPParallelSectionsDirective>(D).getTaskReductionRefExpr();
1321 case OMPD_target_parallel:
1323 cast<OMPTargetParallelDirective>(D).getTaskReductionRefExpr();
1325 case OMPD_target_parallel_for:
1327 cast<OMPTargetParallelForDirective>(D).getTaskReductionRefExpr();
1329 case OMPD_distribute_parallel_for:
1331 cast<OMPDistributeParallelForDirective>(D).getTaskReductionRefExpr();
1333 case OMPD_teams_distribute_parallel_for:
1334 TaskRedRef = cast<OMPTeamsDistributeParallelForDirective>(D)
1335 .getTaskReductionRefExpr();
1337 case OMPD_target_teams_distribute_parallel_for:
1338 TaskRedRef = cast<OMPTargetTeamsDistributeParallelForDirective>(D)
1339 .getTaskReductionRefExpr();
1347 case OMPD_parallel_for_simd:
1349 case OMPD_taskyield:
1352 case OMPD_taskgroup:
1360 case OMPD_cancellation_point:
1362 case OMPD_target_data:
1363 case OMPD_target_enter_data:
1364 case OMPD_target_exit_data:
1366 case OMPD_taskloop_simd:
1367 case OMPD_master_taskloop:
1368 case OMPD_master_taskloop_simd:
1369 case OMPD_parallel_master_taskloop:
1370 case OMPD_parallel_master_taskloop_simd:
1371 case OMPD_distribute:
1372 case OMPD_target_update:
1373 case OMPD_distribute_parallel_for_simd:
1374 case OMPD_distribute_simd:
1375 case OMPD_target_parallel_for_simd:
1376 case OMPD_target_simd:
1377 case OMPD_teams_distribute:
1378 case OMPD_teams_distribute_simd:
1379 case OMPD_teams_distribute_parallel_for_simd:
1380 case OMPD_target_teams:
1381 case OMPD_target_teams_distribute:
1382 case OMPD_target_teams_distribute_parallel_for_simd:
1383 case OMPD_target_teams_distribute_simd:
1384 case OMPD_declare_target:
1385 case OMPD_end_declare_target:
1386 case OMPD_threadprivate:
1388 case OMPD_declare_reduction:
1389 case OMPD_declare_mapper:
1390 case OMPD_declare_simd:
1392 case OMPD_declare_variant:
1393 case OMPD_begin_declare_variant:
1394 case OMPD_end_declare_variant:
1397 llvm_unreachable(
"Enexpected directive with task reductions.");
1400 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(TaskRedRef)->getDecl());
1402 EmitStoreOfScalar(ReductionDesc, GetAddrOfLocalVar(VD),
1403 false, TaskRedRef->
getType());
1409 if (!HaveInsertPoint())
1415 bool HasAtLeastOneReduction =
false;
1416 bool IsReductionWithTaskMod =
false;
1419 if (C->getModifier() == OMPC_REDUCTION_inscan)
1421 HasAtLeastOneReduction =
true;
1422 Privates.append(C->privates().begin(), C->privates().end());
1423 LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1424 RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1425 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1426 IsReductionWithTaskMod =
1427 IsReductionWithTaskMod || C->getModifier() == OMPC_REDUCTION_task;
1429 if (HasAtLeastOneReduction) {
1430 if (IsReductionWithTaskMod) {
1431 CGM.getOpenMPRuntime().emitTaskReductionFini(
1437 ReductionKind == OMPD_simd;
1438 bool SimpleReduction = ReductionKind == OMPD_simd;
1441 CGM.getOpenMPRuntime().emitReduction(
1442 *
this, D.
getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1443 {WithNowait, SimpleReduction, ReductionKind});
1452 llvm::BasicBlock *DoneBB =
nullptr;
1454 if (
const Expr *PostUpdate = C->getPostUpdateExpr()) {
1456 if (llvm::Value *Cond = CondGen(CGF)) {
1461 CGF.
Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1479 CodeGenBoundParametersTy;
1489 for (
const Expr *Ref : C->varlists()) {
1490 if (!Ref->getType()->isScalarType())
1492 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1495 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1500 for (
const Expr *Ref : C->varlists()) {
1501 if (!Ref->getType()->isScalarType())
1503 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1506 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1511 for (
const Expr *Ref : C->varlists()) {
1512 if (!Ref->getType()->isScalarType())
1514 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1517 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1526 for (
const Expr *Ref : C->varlists()) {
1527 if (!Ref->getType()->isScalarType())
1529 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1532 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1536 CGF, S, PrivateDecls);
1542 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1543 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1544 llvm::Value *NumThreads =
nullptr;
1545 llvm::Function *OutlinedFn =
1550 NumThreads = CGF.
EmitScalarExpr(NumThreadsClause->getNumThreads(),
1553 CGF, NumThreads, NumThreadsClause->getBeginLoc());
1558 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1560 const Expr *IfCond =
nullptr;
1561 for (
const auto *C : S.getClausesOfKind<
OMPIfClause>()) {
1562 if (C->getNameModifier() == OMPD_unknown ||
1563 C->getNameModifier() == OMPD_parallel) {
1564 IfCond = C->getCondition();
1569 OMPParallelScope
Scope(CGF, S);
1575 CodeGenBoundParameters(CGF, S, CapturedVars);
1578 CapturedVars, IfCond, NumThreads);
1583 if (!CVD->
hasAttr<OMPAllocateDeclAttr>())
1585 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
1587 return !((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||
1588 AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&
1589 !AA->getAllocator());
1611 Size = CGF.
Builder.CreateNUWAdd(
1620 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
1621 assert(AA->getAllocator() &&
1622 "Expected allocator expression for non-default allocator.");
1626 if (Allocator->getType()->isIntegerTy())
1628 else if (Allocator->getType()->isPointerTy())
1632 llvm::Value *Addr = OMPBuilder.createOMPAlloc(
1634 getNameWithSeparators({CVD->
getName(),
".void.addr"},
".",
"."));
1635 llvm::CallInst *FreeCI =
1636 OMPBuilder.createOMPFree(CGF.
Builder, Addr, Allocator);
1642 getNameWithSeparators({CVD->
getName(),
".addr"},
".",
"."));
1660 std::string Suffix = getNameWithSeparators({
"cache",
""});
1661 llvm::Twine CacheName = Twine(CGM.
getMangledName(VD)).concat(Suffix);
1663 llvm::CallInst *ThreadPrivateCacheCall =
1664 OMPBuilder.createCachedThreadPrivate(CGF.
Builder, Data, Size, CacheName);
1672 llvm::raw_svector_ostream OS(Buffer);
1673 StringRef Sep = FirstSeparator;
1674 for (StringRef Part : Parts) {
1678 return OS.str().str();
1685 Builder.restoreIP(CodeGenIP);
1686 llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder,
false,
1687 "." + RegionName +
".after");
1694 if (Builder.saveIP().isSet())
1695 Builder.CreateBr(FiniBB);
1702 Builder.restoreIP(CodeGenIP);
1703 llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder,
false,
1704 "." + RegionName +
".after");
1711 if (Builder.saveIP().isSet())
1712 Builder.CreateBr(FiniBB);
1716 if (CGM.getLangOpts().OpenMPIRBuilder) {
1717 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1719 llvm::Value *IfCond =
nullptr;
1720 if (
const auto *C = S.getSingleClause<
OMPIfClause>())
1721 IfCond = EmitScalarExpr(C->getCondition(),
1724 llvm::Value *NumThreads =
nullptr;
1726 NumThreads = EmitScalarExpr(NumThreadsClause->getNumThreads(),
1729 ProcBindKind ProcBind = OMP_PROC_BIND_default;
1731 ProcBind = ProcBindClause->getProcBindKind();
1733 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1737 auto FiniCB = [
this](InsertPointTy IP) {
1738 OMPBuilderCBHelpers::FinalizeOMPRegion(*
this, IP);
1745 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
1746 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
1754 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1757 auto BodyGenCB = [&,
this](InsertPointTy AllocaIP,
1758 InsertPointTy CodeGenIP) {
1759 OMPBuilderCBHelpers::EmitOMPOutlinedRegionBody(
1760 *
this, ParallelRegionBodyStmt, AllocaIP, CodeGenIP,
"parallel");
1765 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
1766 AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
1768 OMPBuilder.createParallel(Builder, AllocaIP, BodyGenCB, PrivCB, FiniCB,
1769 IfCond, NumThreads, ProcBind, S.hasCancel()));
1784 CGF, S.getBeginLoc(), OMPD_unknown,
false,
1790 CGF.
EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1806 EmitStmt(S.getIfStmt());
1811 class OMPTransformDirectiveScopeRAII {
1812 OMPLoopScope *
Scope =
nullptr;
1818 if (
const auto *Dir = dyn_cast<OMPLoopBasedDirective>(S)) {
1819 Scope =
new OMPLoopScope(CGF, *Dir);
1824 ~OMPTransformDirectiveScopeRAII() {
1835 int MaxLevel,
int Level = 0) {
1836 assert(
Level < MaxLevel &&
"Too deep lookup during loop body codegen.");
1837 const Stmt *SimplifiedS = S->IgnoreContainers();
1838 if (
const auto *CS = dyn_cast<CompoundStmt>(SimplifiedS)) {
1841 "LLVM IR generation of compound statement ('{}')");
1845 for (
const Stmt *CurStmt : CS->body())
1849 if (SimplifiedS == NextLoop) {
1850 if (
auto *Dir = dyn_cast<OMPLoopTransformationDirective>(SimplifiedS))
1851 SimplifiedS = Dir->getTransformedStmt();
1852 if (
const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(SimplifiedS))
1853 SimplifiedS = CanonLoop->getLoopStmt();
1854 if (
const auto *For = dyn_cast<ForStmt>(SimplifiedS)) {
1857 assert(isa<CXXForRangeStmt>(SimplifiedS) &&
1858 "Expected canonical for loop or range-based for loop.");
1859 const auto *CXXFor = cast<CXXForRangeStmt>(SimplifiedS);
1860 CGF.
EmitStmt(CXXFor->getLoopVarStmt());
1861 S = CXXFor->getBody();
1863 if (
Level + 1 < MaxLevel) {
1878 EmitIgnoredExpr(UE);
1884 for (
const Expr *UE : C->updates())
1885 EmitIgnoredExpr(UE);
1890 JumpDest Continue = getJumpDestInCurrentScope(
"omp.body.continue");
1891 BreakContinueStack.push_back(BreakContinue(
LoopExit, Continue));
1897 llvm::BasicBlock *NextBB = createBasicBlock(
"omp.body.next");
1898 EmitBranchOnBoolExpr(E, NextBB, Continue.
getBlock(),
1899 getProfileCount(D.
getBody()));
1904 EmitOMPReductionClauseInit(D, InscanScope,
true);
1905 bool IsInscanRegion = InscanScope.
Privatize();
1906 if (IsInscanRegion) {
1912 OMPBeforeScanBlock = createBasicBlock(
"omp.before.scan.bb");
1913 OMPAfterScanBlock = createBasicBlock(
"omp.after.scan.bb");
1917 OMPScanExitBlock = createBasicBlock(
"omp.exit.inscan.bb");
1918 OMPScanDispatch = createBasicBlock(
"omp.inscan.dispatch");
1919 EmitBranch(OMPScanDispatch);
1920 EmitBlock(OMPBeforeScanBlock);
1934 EmitBranch(OMPScanExitBlock);
1938 BreakContinueStack.pop_back();
1949 std::unique_ptr<CodeGenFunction::CGCapturedStmtInfo> CSI =
1950 std::make_unique<CodeGenFunction::CGCapturedStmtInfo>(*S);
1958 static llvm::CallInst *
1963 EffectiveArgs.reserve(Args.size() + 1);
1964 llvm::append_range(EffectiveArgs, Args);
1965 EffectiveArgs.push_back(Cap.second);
1967 return ParentCGF.
Builder.CreateCall(Cap.first, EffectiveArgs);
1970 llvm::CanonicalLoopInfo *
1972 assert(
Depth == 1 &&
"Nested loops with OpenMPIRBuilder not yet implemented");
1980 int ParentExpectedOMPLoopDepth = ExpectedOMPLoopDepth;
1981 ExpectedOMPLoopDepth =
Depth;
1984 assert(OMPLoopNestStack.size() >= (
size_t)
Depth &&
"Found too few loops");
1987 llvm::CanonicalLoopInfo *Result = OMPLoopNestStack.back();
1991 OMPLoopNestStack.pop_back_n(
Depth);
1992 ExpectedOMPLoopDepth = ParentExpectedOMPLoopDepth;
1998 const Stmt *SyntacticalLoop = S->getLoopStmt();
1999 if (!getLangOpts().OpenMPIRBuilder) {
2001 EmitStmt(SyntacticalLoop);
2009 const Stmt *BodyStmt;
2010 if (
const auto *For = dyn_cast<ForStmt>(SyntacticalLoop)) {
2011 if (
const Stmt *InitStmt = For->getInit())
2013 BodyStmt = For->getBody();
2014 }
else if (
const auto *RangeFor =
2015 dyn_cast<CXXForRangeStmt>(SyntacticalLoop)) {
2016 if (
const DeclStmt *RangeStmt = RangeFor->getRangeStmt())
2017 EmitStmt(RangeStmt);
2018 if (
const DeclStmt *BeginStmt = RangeFor->getBeginStmt())
2019 EmitStmt(BeginStmt);
2020 if (
const DeclStmt *EndStmt = RangeFor->getEndStmt())
2022 if (
const DeclStmt *LoopVarStmt = RangeFor->getLoopVarStmt())
2023 EmitStmt(LoopVarStmt);
2024 BodyStmt = RangeFor->getBody();
2026 llvm_unreachable(
"Expected for-stmt or range-based for-stmt");
2029 const CapturedStmt *DistanceFunc = S->getDistanceFunc();
2040 Address CountAddr = CreateMemTemp(LogicalTy,
".count.addr");
2042 llvm::Value *DistVal = Builder.CreateLoad(CountAddr,
".count");
2045 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2046 auto BodyGen = [&,
this](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP,
2047 llvm::Value *IndVar) {
2048 Builder.restoreIP(CodeGenIP);
2052 const DeclRefExpr *LoopVarRef = S->getLoopVarRef();
2053 LValue LCVal = EmitLValue(LoopVarRef);
2061 llvm::CanonicalLoopInfo *CL =
2062 OMPBuilder.createCanonicalLoop(Builder, BodyGen, DistVal);
2065 Builder.restoreIP(CL->getAfterIP());
2069 OMPLoopNestStack.push_back(CL);
2074 const Expr *IncExpr,
2077 auto LoopExit = getJumpDestInCurrentScope(
"omp.inner.for.end");
2080 auto CondBlock = createBasicBlock(
"omp.inner.for.cond");
2081 EmitBlock(CondBlock);
2085 const auto &OMPED = cast<OMPExecutableDirective>(S);
2086 const CapturedStmt *ICS = OMPED.getInnermostCapturedStmt();
2089 OMPLoopNestStack.clear();
2091 LoopStack.push(CondBlock, CGM.getContext(), CGM.getCodeGenOpts(),
2093 SourceLocToDebugLoc(R.
getEnd()));
2095 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.
getBegin()),
2096 SourceLocToDebugLoc(R.
getEnd()));
2100 llvm::BasicBlock *ExitBlock =
LoopExit.getBlock();
2101 if (RequiresCleanup)
2102 ExitBlock = createBasicBlock(
"omp.inner.for.cond.cleanup");
2104 llvm::BasicBlock *LoopBody = createBasicBlock(
"omp.inner.for.body");
2107 EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
2108 if (ExitBlock !=
LoopExit.getBlock()) {
2109 EmitBlock(ExitBlock);
2110 EmitBranchThroughCleanup(
LoopExit);
2113 EmitBlock(LoopBody);
2114 incrementProfileCounter(&S);
2117 JumpDest Continue = getJumpDestInCurrentScope(
"omp.inner.for.inc");
2118 BreakContinueStack.push_back(BreakContinue(
LoopExit, Continue));
2124 EmitIgnoredExpr(IncExpr);
2126 BreakContinueStack.pop_back();
2127 EmitBranch(CondBlock);
2134 if (!HaveInsertPoint())
2137 bool HasLinears =
false;
2139 for (
const Expr *Init : C->inits()) {
2141 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
2142 if (
const auto *Ref =
2145 const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
2147 CapturedStmtInfo->lookup(OrigVD) !=
nullptr,
2154 EmitAutoVarCleanups(Emission);
2161 if (
const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
2162 if (
const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
2163 EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
2165 EmitIgnoredExpr(CS);
2174 if (!HaveInsertPoint())
2176 llvm::BasicBlock *DoneBB =
nullptr;
2179 auto IC = C->varlist_begin();
2180 for (
const Expr *F : C->finals()) {
2182 if (llvm::Value *Cond = CondGen(*
this)) {
2185 llvm::BasicBlock *ThenBB = createBasicBlock(
".omp.linear.pu");
2186 DoneBB = createBasicBlock(
".omp.linear.pu.done");
2187 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2191 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
2193 CapturedStmtInfo->lookup(OrigVD) !=
nullptr,
2195 Address OrigAddr = EmitLValue(&DRE).getAddress(*
this);
2202 if (
const Expr *PostUpdate = C->getPostUpdateExpr())
2203 EmitIgnoredExpr(PostUpdate);
2206 EmitBlock(DoneBB,
true);
2215 if (
const Expr *AlignmentExpr = Clause->getAlignment()) {
2218 ClauseAlignment = AlignmentCI->getValue();
2220 for (
const Expr *E : Clause->varlists()) {
2222 if (Alignment == 0) {
2229 E->getType()->getPointeeType()))
2232 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2233 "alignment is not power of 2");
2234 if (Alignment != 0) {
2246 if (!HaveInsertPoint())
2248 auto I = S.private_counters().begin();
2249 for (
const Expr *E : S.counters()) {
2250 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2251 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
2254 EmitAutoVarCleanups(VarEmission);
2255 LocalDeclMap.erase(PrivateVD);
2257 if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
2260 LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
2262 (void)LoopScope.
addPrivate(PrivateVD, EmitLValue(&DRE).getAddress(*
this));
2270 if (!C->getNumForLoops())
2272 for (
unsigned I = S.getLoopsNumber(), E = C->getLoopNumIterations().size();
2274 const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I));
2275 const auto *VD = cast<VarDecl>(DRE->getDecl());
2278 if (DRE->refersToEnclosingVariableOrCapture()) {
2280 VD, CreateMemTemp(DRE->getType(), VD->
getName()));
2287 const Expr *Cond, llvm::BasicBlock *TrueBlock,
2288 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
2296 for (
const Expr *I : S.inits()) {
2303 for (
const Expr *E : S.dependent_counters()) {
2306 assert(!E->getType().getNonReferenceType()->isRecordType() &&
2307 "dependent counter must not be an iterator.");
2308 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2311 (void)PreCondVars.
setVarAddr(CGF, VD, CounterAddr);
2313 (void)PreCondVars.
apply(CGF);
2314 for (
const Expr *E : S.dependent_inits()) {
2326 if (!HaveInsertPoint())
2330 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
2331 for (
const Expr *C : LoopDirective->counters()) {
2337 auto CurPrivate = C->privates().begin();
2338 for (
const Expr *E : C->varlists()) {
2339 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2340 const auto *PrivateVD =
2341 cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
2344 EmitVarDecl(*PrivateVD);
2346 PrivateScope.
addPrivate(VD, GetAddrOfLocalVar(PrivateVD));
2347 assert(IsRegistered &&
"linear var already registered as private");
2351 EmitVarDecl(*PrivateVD);
2365 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
2374 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
2385 LoopStack.setParallel(
true);
2386 LoopStack.setVectorizeEnable();
2389 if (C->getKind() == OMPC_ORDER_concurrent)
2390 LoopStack.setParallel(
true);
2392 (getLangOpts().OpenMPSimd &&
2396 return C->getModifier() == OMPC_REDUCTION_inscan;
2399 LoopStack.setParallel(
false);
2405 if (!HaveInsertPoint())
2407 llvm::BasicBlock *DoneBB =
nullptr;
2411 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
2412 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
2413 const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
2414 if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
2415 OrigVD->hasGlobalStorage() || CED) {
2417 if (llvm::Value *Cond = CondGen(*
this)) {
2420 llvm::BasicBlock *ThenBB = createBasicBlock(
".omp.final.then");
2421 DoneBB = createBasicBlock(
".omp.final.done");
2422 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2429 EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress(*
this);
2434 OrigAddr = EmitLValue(&DRE).getAddress(*
this);
2445 EmitBlock(DoneBB,
true);
2458 auto VDecl = cast<VarDecl>(Helper->
getDecl());
2466 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](
CodeGenFunction &CGF,
2480 const Expr *IfCond =
nullptr;
2482 for (
const auto *C : S.getClausesOfKind<
OMPIfClause>()) {
2484 (C->getNameModifier() == OMPD_unknown ||
2485 C->getNameModifier() == OMPD_simd)) {
2486 IfCond = C->getCondition();
2503 "Expected simd directive");
2504 OMPLoopScope PreInitScope(CGF, S);
2521 llvm::BasicBlock *ContBlock =
nullptr;
2528 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
2535 const Expr *IVExpr = S.getIterationVariable();
2536 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
2543 if (
const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2544 CGF.
EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2558 CGF, S, CGF.
EmitLValue(S.getIterationVariable()));
2570 CGF.EmitOMPInnerLoop(
2573 emitOMPLoopBodyWithStopPoint(CGF, S,
2574 CodeGenFunction::JumpDest());
2580 if (HasLastprivateClause)
2581 CGF.EmitOMPLastprivateClauseFinal(S,
true);
2582 CGF.EmitOMPReductionClauseFinal(S, OMPD_simd);
2596 if (!S.clauses().empty()) {
2604 if (
const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(S.getRawStmt())) {
2605 if (
const Stmt *SyntacticalLoop = CanonLoop->getLoopStmt()) {
2606 for (
const Stmt *SubStmt : SyntacticalLoop->
children()) {
2609 if (
const CompoundStmt *CS = dyn_cast<CompoundStmt>(SubStmt)) {
2613 if (isa<OMPOrderedDirective>(CSSubStmt)) {
2625 bool UseOMPIRBuilder =
2627 if (UseOMPIRBuilder) {
2628 auto &&CodeGenIRBuilder = [
this, &S, UseOMPIRBuilder](
CodeGenFunction &CGF,
2631 if (UseOMPIRBuilder) {
2633 llvm::DebugLoc DL = SourceLocToDebugLoc(S.getBeginLoc());
2634 const Stmt *Inner = S.getRawStmt();
2635 llvm::CanonicalLoopInfo *CLI =
2636 EmitOMPCollapsedCanonicalLoopNest(Inner, 1);
2638 llvm::OpenMPIRBuilder &OMPBuilder =
2639 CGM.getOpenMPRuntime().getOMPBuilder();
2641 OMPBuilder.applySimd(DL, CLI);
2648 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
2649 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_simd,
2656 OMPFirstScanLoop =
true;
2663 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
2664 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_simd, CodeGen);
2672 OMPTransformDirectiveScopeRAII TileScope(*
this, &S);
2673 EmitStmt(S.getTransformedStmt());
2677 bool UseOMPIRBuilder = CGM.getLangOpts().OpenMPIRBuilder;
2679 if (UseOMPIRBuilder) {
2680 auto DL = SourceLocToDebugLoc(S.getBeginLoc());
2681 const Stmt *Inner = S.getRawStmt();
2686 llvm::CanonicalLoopInfo *CLI = EmitOMPCollapsedCanonicalLoopNest(Inner, 1);
2687 OMPLoopNestStack.clear();
2689 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2691 bool NeedsUnrolledCLI = ExpectedOMPLoopDepth >= 1;
2692 llvm::CanonicalLoopInfo *UnrolledCLI =
nullptr;
2695 assert(ExpectedOMPLoopDepth == 0);
2696 OMPBuilder.unrollLoopFull(DL, CLI);
2698 uint64_t Factor = 0;
2699 if (
Expr *FactorExpr = PartialClause->getFactor()) {
2700 Factor = FactorExpr->EvaluateKnownConstInt(getContext()).getZExtValue();
2701 assert(Factor >= 1 &&
"Only positive factors are valid");
2703 OMPBuilder.unrollLoopPartial(DL, CLI, Factor,
2704 NeedsUnrolledCLI ? &UnrolledCLI :
nullptr);
2706 OMPBuilder.unrollLoopHeuristic(DL, CLI);
2709 assert((!NeedsUnrolledCLI || UnrolledCLI) &&
2710 "NeedsUnrolledCLI implies UnrolledCLI to be set");
2712 OMPLoopNestStack.push_back(UnrolledCLI);
2727 if (
Expr *FactorExpr = PartialClause->getFactor()) {
2729 FactorExpr->EvaluateKnownConstInt(getContext()).getZExtValue();
2730 assert(Factor >= 1 &&
"Only positive factors are valid");
2731 LoopStack.setUnrollCount(Factor);
2735 EmitStmt(S.getAssociatedStmt());
2738 void CodeGenFunction::EmitOMPOuterLoop(
2741 const CodeGenFunction::OMPLoopArguments &LoopArgs,
2746 const Expr *IVExpr = S.getIterationVariable();
2747 const unsigned IVSize = getContext().getTypeSize(IVExpr->
getType());
2750 JumpDest
LoopExit = getJumpDestInCurrentScope(
"omp.dispatch.end");
2753 llvm::BasicBlock *CondBlock = createBasicBlock(
"omp.dispatch.cond");
2754 EmitBlock(CondBlock);
2756 OMPLoopNestStack.clear();
2757 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.
getBegin()),
2758 SourceLocToDebugLoc(R.
getEnd()));
2760 llvm::Value *BoolCondVal =
nullptr;
2761 if (!DynamicOrOrdered) {
2765 EmitIgnoredExpr(LoopArgs.EUB);
2767 EmitIgnoredExpr(LoopArgs.Init);
2769 BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
2772 RT.
emitForNext(*
this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
2773 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
2778 llvm::BasicBlock *ExitBlock =
LoopExit.getBlock();
2780 ExitBlock = createBasicBlock(
"omp.dispatch.cleanup");
2782 llvm::BasicBlock *LoopBody = createBasicBlock(
"omp.dispatch.body");
2783 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
2784 if (ExitBlock !=
LoopExit.getBlock()) {
2785 EmitBlock(ExitBlock);
2786 EmitBranchThroughCleanup(
LoopExit);
2788 EmitBlock(LoopBody);
2792 if (DynamicOrOrdered)
2793 EmitIgnoredExpr(LoopArgs.Init);
2796 JumpDest Continue = getJumpDestInCurrentScope(
"omp.dispatch.inc");
2797 BreakContinueStack.push_back(BreakContinue(
LoopExit, Continue));
2805 CGF.LoopStack.setParallel(!IsMonotonic);
2806 if (const auto *C = S.getSingleClause<OMPOrderClause>())
2807 if (C->getKind() == OMPC_ORDER_concurrent)
2808 CGF.LoopStack.setParallel(true);
2810 CGF.EmitOMPSimdInit(S);
2813 [&S, &LoopArgs,
LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered,
2821 CGF.EmitOMPInnerLoop(
2824 CodeGenLoop(CGF, S, LoopExit);
2827 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
2831 EmitBlock(Continue.getBlock());
2832 BreakContinueStack.pop_back();
2833 if (!DynamicOrOrdered) {
2835 EmitIgnoredExpr(LoopArgs.NextLB);
2836 EmitIgnoredExpr(LoopArgs.NextUB);
2839 EmitBranch(CondBlock);
2840 OMPLoopNestStack.clear();
2847 if (!DynamicOrOrdered)
2848 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2849 S.getDirectiveKind());
2851 OMPCancelStack.emitExit(*
this, S.getDirectiveKind(), CodeGen);
2854 void CodeGenFunction::EmitOMPForOuterLoop(
2857 const OMPLoopArguments &LoopArgs,
2858 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
2865 LoopArgs.Chunk !=
nullptr)) &&
2866 "static non-chunked schedule does not need outer loop");
2918 const Expr *IVExpr = S.getIterationVariable();
2919 const unsigned IVSize = getContext().getTypeSize(IVExpr->
getType());
2922 if (DynamicOrOrdered) {
2923 const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
2924 CGDispatchBounds(*
this, S, LoopArgs.LB, LoopArgs.UB);
2925 llvm::Value *LBVal = DispatchBounds.first;
2926 llvm::Value *UBVal = DispatchBounds.second;
2930 IVSigned, Ordered, DipatchRTInputValues);
2933 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
2934 LoopArgs.ST, LoopArgs.Chunk);
2936 ScheduleKind, StaticInit);
2940 const unsigned IVSize,
2941 const bool IVSigned) {
2948 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
2949 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
2950 OuterLoopArgs.IncExpr = S.getInc();
2951 OuterLoopArgs.Init = S.getInit();
2952 OuterLoopArgs.Cond = S.getCond();
2953 OuterLoopArgs.NextLB = S.getNextLowerBound();
2954 OuterLoopArgs.NextUB = S.getNextUpperBound();
2955 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
2960 const unsigned IVSize,
const bool IVSigned) {}
2962 void CodeGenFunction::EmitOMPDistributeOuterLoop(
2964 OMPPrivateScope &LoopScope,
const OMPLoopArguments &LoopArgs,
2965 const CodeGenLoopTy &CodeGenLoopContent) {
2974 const Expr *IVExpr = S.getIterationVariable();
2975 const unsigned IVSize = getContext().getTypeSize(IVExpr->
getType());
2979 IVSize, IVSigned,
false, LoopArgs.IL, LoopArgs.LB,
2980 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
2987 IncExpr = S.getDistInc();
2989 IncExpr = S.getInc();
2994 OMPLoopArguments OuterLoopArgs;
2995 OuterLoopArgs.LB = LoopArgs.LB;
2996 OuterLoopArgs.UB = LoopArgs.UB;
2997 OuterLoopArgs.ST = LoopArgs.ST;
2998 OuterLoopArgs.IL = LoopArgs.IL;
2999 OuterLoopArgs.Chunk = LoopArgs.Chunk;
3001 ? S.getCombinedEnsureUpperBound()
3002 : S.getEnsureUpperBound();
3003 OuterLoopArgs.IncExpr = IncExpr;
3005 ? S.getCombinedInit()
3008 ? S.getCombinedCond()
3011 ? S.getCombinedNextLowerBound()
3012 : S.getNextLowerBound();
3014 ? S.getCombinedNextUpperBound()
3015 : S.getNextUpperBound();
3017 EmitOMPOuterLoop(
false,
false, S,
3018 LoopScope, OuterLoopArgs, CodeGenLoopContent,
3022 static std::pair<LValue, LValue>
3065 static std::pair<llvm::Value *, llvm::Value *>
3076 llvm::Value *LBVal =
3078 llvm::Value *UBVal =
3080 return {LBVal, UBVal};
3086 const auto &Dir = cast<OMPLoopDirective>(S);
3088 CGF.
EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
3089 llvm::Value *LBCast =
3092 CapturedVars.push_back(LBCast);
3094 CGF.
EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
3096 llvm::Value *UBCast =
3099 CapturedVars.push_back(UBCast);
3109 bool HasCancel =
false;
3111 if (
const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
3112 HasCancel = D->hasCancel();
3113 else if (
const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
3114 HasCancel = D->hasCancel();
3115 else if (
const auto *D =
3116 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
3117 HasCancel = D->hasCancel();
3129 CGInlinedWorksharingLoop,
3139 OMPLexicalScope
Scope(*
this, S, OMPD_parallel);
3140 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_distribute, CodeGen);
3149 OMPLexicalScope
Scope(*
this, S, OMPD_parallel);
3150 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_distribute, CodeGen);
3158 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
3159 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_simd, CodeGen);
3169 llvm::Constant *Addr;
3172 S, ParentName, Fn, Addr,
true, CodeGen);
3173 assert(Fn && Addr &&
"Target device function emission failed.");
3185 struct ScheduleKindModifiersTy {
3201 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3202 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3203 EmitVarDecl(*IVDecl);
3208 if (
const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3209 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3211 EmitIgnoredExpr(S.getCalcLastIteration());
3216 bool HasLastprivateClause;
3219 OMPLoopScope PreInitScope(*
this, S);
3224 llvm::BasicBlock *ContBlock =
nullptr;
3225 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
3229 llvm::BasicBlock *ThenBlock = createBasicBlock(
"omp.precond.then");
3230 ContBlock = createBasicBlock(
"omp.precond.end");
3231 emitPreCond(*
this, S, S.getPreCond(), ThenBlock, ContBlock,
3232 getProfileCount(&S));
3233 EmitBlock(ThenBlock);
3234 incrementProfileCounter(&S);
3238 bool Ordered =
false;
3240 if (OrderedClause->getNumForLoops())
3248 bool HasLinears = EmitOMPLinearClauseInit(S);
3251 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*
this, S);
3252 LValue LB = Bounds.first;
3253 LValue UB = Bounds.second;
3262 if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
3266 CGM.getOpenMPRuntime().emitBarrierCall(
3267 *
this, S.getBeginLoc(), OMPD_unknown,
false,
3270 EmitOMPPrivateClause(S, LoopScope);
3272 *
this, S, EmitLValue(S.getIterationVariable()));
3273 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
3274 EmitOMPReductionClauseInit(S, LoopScope);
3275 EmitOMPPrivateLoopCounters(S, LoopScope);
3276 EmitOMPLinearClause(S, LoopScope);
3279 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*
this, S);
3282 const Expr *ChunkExpr =
nullptr;
3285 ScheduleKind.
Schedule = C->getScheduleKind();
3286 ScheduleKind.
M1 = C->getFirstScheduleModifier();
3287 ScheduleKind.
M2 = C->getSecondScheduleModifier();
3288 ChunkExpr = C->getChunkSize();
3291 CGM.getOpenMPRuntime().getDefaultScheduleAndChunk(
3292 *
this, S, ScheduleKind.
Schedule, ChunkExpr);
3294 bool HasChunkSizeOne =
false;
3295 llvm::Value *Chunk =
nullptr;
3297 Chunk = EmitScalarExpr(ChunkExpr);
3298 Chunk = EmitScalarConversion(Chunk, ChunkExpr->
getType(),
3299 S.getIterationVariable()->getType(),
3304 HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
3307 const unsigned IVSize = getContext().getTypeSize(IVExpr->
getType());
3313 bool StaticChunkedOne =
3315 Chunk !=
nullptr) &&
3320 (ScheduleKind.
Schedule == OMPC_SCHEDULE_static &&
3321 !(ScheduleKind.
M1 == OMPC_SCHEDULE_MODIFIER_nonmonotonic ||
3322 ScheduleKind.
M2 == OMPC_SCHEDULE_MODIFIER_nonmonotonic)) ||
3323 ScheduleKind.
M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
3324 ScheduleKind.
M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
3326 Chunk !=
nullptr) ||
3327 StaticChunkedOne) &&
3330 getJumpDestInCurrentScope(createBasicBlock(
"omp.loop.exit"));
3335 CGF.EmitOMPSimdInit(S);
3337 if (C->getKind() == OMPC_ORDER_concurrent)
3338 CGF.LoopStack.setParallel(true);
3341 [IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk,
3350 IVSize, IVSigned, Ordered, IL.getAddress(CGF),
3351 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF),
3352 StaticChunkedOne ? Chunk :
nullptr);
3353 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
3354 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind,
3357 if (!StaticChunkedOne)
3358 CGF.EmitIgnoredExpr(S.getEnsureUpperBound());
3360 CGF.EmitIgnoredExpr(S.getInit());
3374 CGF.EmitOMPInnerLoop(
3376 StaticChunkedOne ? S.getCombinedParForInDistCond()
3378 StaticChunkedOne ? S.getDistInc() : S.getInc(),
3380 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit);
3387 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
3388 S.getDirectiveKind());
3390 OMPCancelStack.emitExit(*
this, S.getDirectiveKind(), CodeGen);
3394 const OMPLoopArguments LoopArguments(
3397 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
3398 LoopArguments, CGDispatchBounds);
3402 return CGF.
Builder.CreateIsNotNull(
3406 EmitOMPReductionClauseFinal(
3408 ? OMPD_parallel_for_simd
3413 return CGF.
Builder.CreateIsNotNull(
3417 if (HasLastprivateClause)
3418 EmitOMPLastprivateClauseFinal(
3420 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
3423 return CGF.
Builder.CreateIsNotNull(
3429 EmitBranch(ContBlock);
3430 EmitBlock(ContBlock,
true);
3433 return HasLastprivateClause;
3439 static std::pair<LValue, LValue>
3441 const auto &LS = cast<OMPLoopDirective>(S);
3453 static std::pair<llvm::Value *, llvm::Value *>
3456 const auto &LS = cast<OMPLoopDirective>(S);
3457 const Expr *IVExpr = LS.getIterationVariable();
3459 llvm::Value *LBVal = CGF.
Builder.getIntN(IVSize, 0);
3461 return {LBVal, UBVal};
3473 llvm::function_ref<llvm::Value *(
CodeGenFunction &)> NumIteratorsGen) {
3474 llvm::Value *OMPScanNumIterations = CGF.
Builder.CreateIntCast(
3475 NumIteratorsGen(CGF), CGF.
SizeTy,
false);
3481 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
3482 "Only inscan reductions are expected.");
3483 Shareds.append(C->varlist_begin(), C->varlist_end());
3484 Privates.append(C->privates().begin(), C->privates().end());
3485 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
3486 CopyArrayTemps.append(C->copy_array_temps().begin(),
3487 C->copy_array_temps().end());
3495 auto *ITA = CopyArrayTemps.begin();
3496 for (
const Expr *IRef : Privates) {
3497 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
3500 if (PrivateVD->getType()->isVariablyModifiedType()) {
3506 cast<OpaqueValueExpr>(
3507 cast<VariableArrayType>((*ITA)->getType()->getAsArrayTypeUnsafe())
3511 CGF.
EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(*ITA)->getDecl()));
3525 llvm::function_ref<llvm::Value *(
CodeGenFunction &)> NumIteratorsGen) {
3526 llvm::Value *OMPScanNumIterations = CGF.
Builder.CreateIntCast(
3527 NumIteratorsGen(CGF), CGF.
SizeTy,
false);
3535 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
3536 "Only inscan reductions are expected.");
3537 Shareds.append(C->varlist_begin(), C->varlist_end());
3538 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
3539 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
3540 Privates.append(C->privates().begin(), C->privates().end());
3541 CopyOps.append(C->copy_ops().begin(), C->copy_ops().end());
3542 CopyArrayElems.append(C->copy_array_elems().begin(),
3543 C->copy_array_elems().end());
3547 llvm::Value *OMPLast = CGF.
Builder.CreateNSWSub(
3548 OMPScanNumIterations,
3549 llvm::ConstantInt::get(CGF.
SizeTy, 1,
false));
3550 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
3551 const Expr *PrivateExpr = Privates[I];
3552 const Expr *OrigExpr = Shareds[I];
3553 const Expr *CopyArrayElem = CopyArrayElems[I];
3556 cast<OpaqueValueExpr>(
3557 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3563 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
3564 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
3593 llvm::Value *OMPScanNumIterations = CGF.
Builder.CreateIntCast(
3594 NumIteratorsGen(CGF), CGF.
SizeTy,
false);
3601 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
3602 "Only inscan reductions are expected.");
3603 Privates.append(C->privates().begin(), C->privates().end());
3604 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
3605 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
3606 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
3607 CopyArrayElems.append(C->copy_array_elems().begin(),
3608 C->copy_array_elems().end());
3623 auto &&CodeGen = [&S, OMPScanNumIterations, &LHSs, &RHSs, &CopyArrayElems,
3630 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock();
3631 llvm::BasicBlock *LoopBB = CGF.createBasicBlock(
"omp.outer.log.scan.body");
3632 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
"omp.outer.log.scan.exit");
3636 CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy);
3637 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg);
3639 LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal);
3640 LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy);
3641 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub(
3642 OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1));
3644 CGF.EmitBlock(LoopBB);
3645 auto *Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2);
3647 auto *Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3648 Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB);
3649 Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB);
3652 llvm::BasicBlock *InnerLoopBB =
3653 CGF.createBasicBlock(
"omp.inner.log.scan.body");
3654 llvm::BasicBlock *InnerExitBB =
3655 CGF.createBasicBlock(
"omp.inner.log.scan.exit");
3656 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K);
3657 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3658 CGF.EmitBlock(InnerLoopBB);
3659 auto *IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3660 IVal->addIncoming(NMin1, LoopBB);
3663 auto *ILHS = LHSs.begin();
3664 auto *IRHS = RHSs.begin();
3665 for (
const Expr *CopyArrayElem : CopyArrayElems) {
3666 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
3667 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
3672 cast<OpaqueValueExpr>(
3673 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3675 LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF);
3680 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K);
3683 cast<OpaqueValueExpr>(
3684 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3686 RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF);
3693 CGF.CGM.getOpenMPRuntime().emitReduction(
3694 CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
3695 {true, true, OMPD_unknown});
3697 llvm::Value *NextIVal =
3698 CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1));
3699 IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock());
3700 CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K);
3701 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3702 CGF.EmitBlock(InnerExitBB);
3704 CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1));
3705 Counter->addIncoming(Next, CGF.Builder.GetInsertBlock());
3707 llvm::Value *NextPow2K =
3708 CGF.Builder.CreateShl(Pow2K, 1,
"",
true);
3709 Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock());
3710 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(Next, LogVal);
3711 CGF.Builder.CreateCondBr(Cmp, LoopBB, ExitBB);
3713 CGF.EmitBlock(ExitBB);
3716 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());
3717 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
3718 CGF, S.getBeginLoc(), OMPD_unknown,
false,
3725 CGF.OMPFirstScanLoop =
false;
3732 bool HasLastprivates;
3735 return C->getModifier() == OMPC_REDUCTION_inscan;
3739 OMPLoopScope LoopScope(CGF, S);
3744 CGF, S.getDirectiveKind(), HasCancel);
3752 const auto &&SecondGen = [&S, HasCancel,
3755 CGF, S.getDirectiveKind(), HasCancel);
3772 return HasLastprivates;
3779 if (isa<OMPNowaitClause>(C))
3782 if (
auto *SC = dyn_cast<OMPScheduleClause>(C)) {
3787 switch (SC->getScheduleKind()) {
3788 case OMPC_SCHEDULE_auto:
3789 case OMPC_SCHEDULE_dynamic:
3790 case OMPC_SCHEDULE_runtime:
3791 case OMPC_SCHEDULE_guided:
3792 case OMPC_SCHEDULE_static:
3805 static llvm::omp::ScheduleKind
3807 switch (ScheduleClauseKind) {
3809 return llvm::omp::OMP_SCHEDULE_Default;
3810 case OMPC_SCHEDULE_auto:
3811 return llvm::omp::OMP_SCHEDULE_Auto;
3812 case OMPC_SCHEDULE_dynamic:
3813 return llvm::omp::OMP_SCHEDULE_Dynamic;
3814 case OMPC_SCHEDULE_guided:
3815 return llvm::omp::OMP_SCHEDULE_Guided;
3816 case OMPC_SCHEDULE_runtime:
3817 return llvm::omp::OMP_SCHEDULE_Runtime;
3818 case OMPC_SCHEDULE_static:
3819 return llvm::omp::OMP_SCHEDULE_Static;
3821 llvm_unreachable(
"Unhandled schedule kind");
3825 bool HasLastprivates =
false;
3826 bool UseOMPIRBuilder =
3828 auto &&CodeGen = [
this, &S, &HasLastprivates,
3831 if (UseOMPIRBuilder) {
3834 llvm::omp::ScheduleKind SchedKind = llvm::omp::OMP_SCHEDULE_Default;
3835 llvm::Value *ChunkSize =
nullptr;
3839 if (
const Expr *ChunkSizeExpr = SchedClause->getChunkSize())
3840 ChunkSize = EmitScalarExpr(ChunkSizeExpr);
3844 const Stmt *Inner = S.getRawStmt();
3845 llvm::CanonicalLoopInfo *CLI =
3846 EmitOMPCollapsedCanonicalLoopNest(Inner, 1);
3848 llvm::OpenMPIRBuilder &OMPBuilder =
3849 CGM.getOpenMPRuntime().getOMPBuilder();
3850 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
3851 AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
3852 OMPBuilder.applyWorkshareLoop(
3853 Builder.getCurrentDebugLocation(), CLI, AllocaIP, NeedsBarrier,
3854 SchedKind, ChunkSize,
false,
3865 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
3866 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_for, CodeGen,
3870 if (!UseOMPIRBuilder) {
3873 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.getBeginLoc(), OMPD_for);
3880 bool HasLastprivates =
false;
3888 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
3889 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_simd, CodeGen);
3894 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.getBeginLoc(), OMPD_for);
3901 llvm::Value *Init =
nullptr) {
3909 const Stmt *
CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
3911 bool HasLastprivates =
false;
3916 C.getIntTypeForBitwidth(32, 1);
3919 CGF.Builder.getInt32(0));
3920 llvm::ConstantInt *GlobalUBVal = CS !=
nullptr
3921 ? CGF.Builder.getInt32(CS->size() - 1)
3922 : CGF.Builder.getInt32(0);
3926 CGF.Builder.getInt32(1));
3928 CGF.Builder.getInt32(0));
3955 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
".omp.sections.exit");
3957 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
3958 ExitBB, CS ==
nullptr ? 1 : CS->size());
3960 unsigned CaseNumber = 0;
3962 auto CaseBB = CGF.createBasicBlock(
".omp.sections.case");
3963 CGF.EmitBlock(CaseBB);
3964 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
3965 CGF.EmitStmt(SubStmt);
3966 CGF.EmitBranch(ExitBB);
3970 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(
".omp.sections.case");
3971 CGF.EmitBlock(CaseBB);
3972 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
3974 CGF.EmitBranch(ExitBB);
3976 CGF.EmitBlock(ExitBB,
true);
3980 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
3984 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
3985 CGF, S.getBeginLoc(), OMPD_unknown,
false,
3988 CGF.EmitOMPPrivateClause(S, LoopScope);
3990 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
3991 CGF.EmitOMPReductionClauseInit(S, LoopScope);
3992 (void)LoopScope.Privatize();
3994 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
3998 ScheduleKind.
Schedule = OMPC_SCHEDULE_static;
4002 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
4003 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit);
4005 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
4006 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
4007 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
4008 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
4010 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
4012 CGF.EmitOMPInnerLoop(S,
false, Cond, Inc, BodyGen,
4016 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
4017 S.getDirectiveKind());
4019 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
4020 CGF.EmitOMPReductionClauseFinal(S, OMPD_parallel);
4023 return CGF.
Builder.CreateIsNotNull(
4028 if (HasLastprivates)
4035 bool HasCancel =
false;
4036 if (
auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
4037 HasCancel = OSD->hasCancel();
4038 else if (
auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
4039 HasCancel = OPSD->hasCancel();
4040 OMPCancelStackRAII CancelRegion(*
this, S.getDirectiveKind(), HasCancel);
4041 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_sections, CodeGen,
4049 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.getBeginLoc(),
4055 if (CGM.getLangOpts().OpenMPIRBuilder) {
4056 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4057 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4058 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4060 auto FiniCB = [
this](InsertPointTy IP) {
4061 OMPBuilderCBHelpers::FinalizeOMPRegion(*
this, IP);
4064 const CapturedStmt *ICS = S.getInnermostCapturedStmt();
4065 const Stmt *
CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4070 auto SectionCB = [
this, SubStmt](InsertPointTy AllocaIP,
4071 InsertPointTy CodeGenIP) {
4072 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
4073 *
this, SubStmt, AllocaIP, CodeGenIP,
"section");
4075 SectionCBVector.push_back(SectionCB);
4078 auto SectionCB = [
this,
CapturedStmt](InsertPointTy AllocaIP,
4079 InsertPointTy CodeGenIP) {
4080 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
4083 SectionCBVector.push_back(SectionCB);
4090 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
4091 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
4101 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
4102 AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
4103 Builder.restoreIP(OMPBuilder.createSections(
4104 Builder, AllocaIP, SectionCBVector, PrivCB, FiniCB, S.hasCancel(),
4111 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
4116 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.getBeginLoc(),
4124 if (CGM.getLangOpts().OpenMPIRBuilder) {
4125 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4126 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4128 const Stmt *SectionRegionBodyStmt = S.getAssociatedStmt();
4129 auto FiniCB = [
this](InsertPointTy IP) {
4130 OMPBuilderCBHelpers::FinalizeOMPRegion(*
this, IP);
4133 auto BodyGenCB = [SectionRegionBodyStmt,
this](InsertPointTy AllocaIP,
4134 InsertPointTy CodeGenIP) {
4135 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
4136 *
this, SectionRegionBodyStmt, AllocaIP, CodeGenIP,
"section");
4141 Builder.restoreIP(OMPBuilder.createSection(Builder, BodyGenCB, FiniCB));
4147 EmitStmt(S.getAssociatedStmt());
4160 CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
4161 DestExprs.append(C->destination_exprs().begin(),
4162 C->destination_exprs().end());
4163 SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
4164 AssignmentOps.append(C->assignment_ops().begin(),
4165 C->assignment_ops().end());
4174 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4179 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
4180 CGM.getOpenMPRuntime().emitSingleRegion(*
this, CodeGen, S.getBeginLoc(),
4181 CopyprivateVars, DestExprs,
4182 SrcExprs, AssignmentOps);
4186 if (!S.getSingleClause<
OMPNowaitClause>() && CopyprivateVars.empty()) {
4187 CGM.getOpenMPRuntime().emitBarrierCall(
4188 *
this, S.getBeginLoc(),
4204 if (CGM.getLangOpts().OpenMPIRBuilder) {
4205 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4206 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4208 const Stmt *MasterRegionBodyStmt = S.getAssociatedStmt();
4210 auto FiniCB = [
this](InsertPointTy IP) {
4211 OMPBuilderCBHelpers::FinalizeOMPRegion(*
this, IP);
4214 auto BodyGenCB = [MasterRegionBodyStmt,
this](InsertPointTy AllocaIP,
4215 InsertPointTy CodeGenIP) {
4216 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
4217 *
this, MasterRegionBodyStmt, AllocaIP, CodeGenIP,
"master");
4222 Builder.restoreIP(OMPBuilder.createMaster(Builder, BodyGenCB, FiniCB));
4238 Filter = FilterClause->getThreadID();
4244 if (CGM.getLangOpts().OpenMPIRBuilder) {
4245 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4246 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4248 const Stmt *MaskedRegionBodyStmt = S.getAssociatedStmt();
4251 Filter = FilterClause->getThreadID();
4252 llvm::Value *FilterVal =
Filter
4253 ? EmitScalarExpr(
Filter, CGM.Int32Ty)
4254 : llvm::ConstantInt::get(CGM.Int32Ty, 0);
4256 auto FiniCB = [
this](InsertPointTy IP) {
4257 OMPBuilderCBHelpers::FinalizeOMPRegion(*
this, IP);
4260 auto BodyGenCB = [MaskedRegionBodyStmt,
this](InsertPointTy AllocaIP,
4261 InsertPointTy CodeGenIP) {
4262 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
4263 *
this, MaskedRegionBodyStmt, AllocaIP, CodeGenIP,
"masked");
4269 OMPBuilder.createMasked(Builder, BodyGenCB, FiniCB, FilterVal));
4279 if (CGM.getLangOpts().OpenMPIRBuilder) {
4280 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4281 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4283 const Stmt *CriticalRegionBodyStmt = S.getAssociatedStmt();
4284 const Expr *Hint =
nullptr;
4285 if (
const auto *HintClause = S.getSingleClause<
OMPHintClause>())
4286 Hint = HintClause->getHint();
4291 llvm::Value *HintInst =
nullptr;
4294 Builder.CreateIntCast(EmitScalarExpr(Hint), CGM.Int32Ty,
false);
4296 auto FiniCB = [
this](InsertPointTy IP) {
4297 OMPBuilderCBHelpers::FinalizeOMPRegion(*
this, IP);
4300 auto BodyGenCB = [CriticalRegionBodyStmt,
this](InsertPointTy AllocaIP,
4301 InsertPointTy CodeGenIP) {
4302 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
4303 *
this, CriticalRegionBodyStmt, AllocaIP, CodeGenIP,
"critical");
4308 Builder.restoreIP(OMPBuilder.createCritical(
4309 Builder, BodyGenCB, FiniCB, S.getDirectiveName().getAsString(),
4317 CGF.
EmitStmt(S.getAssociatedStmt());
4319 const Expr *Hint =
nullptr;
4320 if (
const auto *HintClause = S.getSingleClause<
OMPHintClause>())
4321 Hint = HintClause->getHint();
4324 CGM.getOpenMPRuntime().emitCriticalRegion(*
this,
4325 S.getDirectiveName().getAsString(),
4326 CodeGen, S.getBeginLoc(), Hint);
4342 OMPLoopScope LoopScope(CGF, S);
4347 return C->getModifier() == OMPC_REDUCTION_inscan;
4375 OMPLoopScope LoopScope(CGF, S);
4380 return C->getModifier() == OMPC_REDUCTION_inscan;
4409 CGF, S.getBeginLoc(), OMPD_unknown,
false,
4436 CGF.EmitSections(S);
4450 class CheckVarsEscapingUntiedTaskDeclContext final
4455 explicit CheckVarsEscapingUntiedTaskDeclContext() =
default;
4456 virtual ~CheckVarsEscapingUntiedTaskDeclContext() =
default;
4457 void VisitDeclStmt(
const DeclStmt *S) {
4461 for (
const Decl *D : S->decls()) {
4462 if (
const auto *VD = dyn_cast_or_null<VarDecl>(D))
4464 PrivateDecls.push_back(VD);
4470 void VisitBlockExpr(
const BlockExpr *) {}
4471 void VisitStmt(
const Stmt *S) {
4474 for (
const Stmt *Child : S->children())
4489 const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
4491 auto PartId = std::next(I);
4492 auto TaskT = std::next(I, 4);
4497 const Expr *Cond = Clause->getCondition();
4499 if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
4500 Data.
Final.setInt(CondConstant);
4502 Data.
Final.setPointer(EvaluateExprAsBool(Cond));
4505 Data.
Final.setInt(
false);
4509 const Expr *Prio = Clause->getPriority();
4511 Data.
Priority.setPointer(EmitScalarConversion(
4512 EmitScalarExpr(Prio), Prio->
getType(),
4513 getContext().getIntTypeForBitwidth(32, 1),
4521 auto IRef = C->varlist_begin();
4522 for (
const Expr *IInit : C->private_copies()) {
4523 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4524 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4531 EmittedAsPrivate.clear();
4534 auto IRef = C->varlist_begin();
4535 auto IElemInitRef = C->inits().begin();
4536 for (
const Expr *IInit : C->private_copies()) {
4537 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4538 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4548 llvm::MapVector<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
4550 auto IRef = C->varlist_begin();
4551 auto ID = C->destination_exprs().begin();
4552 for (
const Expr *IInit : C->private_copies()) {
4553 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4554 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4558 LastprivateDstsOrigs.insert(
4559 std::make_pair(cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
4560 cast<DeclRefExpr>(*IRef)));
4568 Data.
ReductionVars.append(C->varlist_begin(), C->varlist_end());
4569 Data.
ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
4570 Data.
ReductionCopies.append(C->privates().begin(), C->privates().end());
4572 C->reduction_ops().end());
4573 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
4574 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
4576 Data.
Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
4577 *
this, S.getBeginLoc(), LHSs, RHSs, Data);
4581 Data.
Dependences.emplace_back(C->getDependencyKind(), C->getModifier());
4582 DD.
DepExprs.append(C->varlist_begin(), C->varlist_end());
4586 CheckVarsEscapingUntiedTaskDeclContext Checker;
4587 Checker.Visit(S.getInnermostCapturedStmt()->getCapturedStmt());
4588 Data.
PrivateLocals.append(Checker.getPrivateDecls().begin(),
4589 Checker.getPrivateDecls().end());
4591 auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
4594 llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
4595 std::pair<Address, Address>>
4600 if (
auto *DI = CGF.getDebugInfo()) {
4601 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields =
4602 CGF.CapturedStmtInfo->getCaptureFields();
4603 llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue();
4604 if (CaptureFields.size() && ContextValue) {
4605 unsigned CharWidth = CGF.getContext().getCharWidth();
4619 for (
auto It = CaptureFields.begin(); It != CaptureFields.end(); ++It) {
4620 const VarDecl *SharedVar = It->first;
4623 CGF.getContext().getASTRecordLayout(CaptureRecord);
4626 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
4627 (void)DI->EmitDeclareOfAutoVariable(SharedVar, ContextValue,
4628 CGF.Builder,
false);
4629 llvm::Instruction &
Last = CGF.Builder.GetInsertBlock()->back();
4632 if (
auto DDI = dyn_cast<llvm::DbgVariableIntrinsic>(&
Last)) {
4636 Ops.push_back(llvm::dwarf::DW_OP_plus_uconst);
4639 Ops.push_back(llvm::dwarf::DW_OP_deref);
4640 auto &Ctx = DDI->getContext();
4641 llvm::DIExpression *DIExpr = llvm::DIExpression::get(Ctx, Ops);
4642 Last.setOperand(2, llvm::MetadataAsValue::get(Ctx, DIExpr));
4650 enum { PrivatesParam = 2, CopyFnParam = 3 };
4651 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
4652 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
4653 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
4654 CS->getCapturedDecl()->getParam(PrivatesParam)));
4659 CallArgs.push_back(PrivatesPtr);
4660 ParamTypes.push_back(PrivatesPtr->getType());
4662 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4663 Address PrivatePtr = CGF.CreateMemTemp(
4664 CGF.getContext().getPointerType(E->
getType()),
".priv.ptr.addr");
4665 PrivatePtrs.emplace_back(VD, PrivatePtr);
4667 ParamTypes.push_back(PrivatePtr.
getType());
4670 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4672 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->
getType()),
4673 ".firstpriv.ptr.addr");
4674 PrivatePtrs.emplace_back(VD, PrivatePtr);
4675 FirstprivatePtrs.emplace_back(VD, PrivatePtr);
4677 ParamTypes.push_back(PrivatePtr.
getType());
4680 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4682 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->
getType()),
4683 ".lastpriv.ptr.addr");
4684 PrivatePtrs.emplace_back(VD, PrivatePtr);
4686 ParamTypes.push_back(PrivatePtr.
getType());
4691 Ty = CGF.getContext().getPointerType(Ty);
4693 Ty = CGF.getContext().getPointerType(Ty);
4694 Address PrivatePtr = CGF.CreateMemTemp(
4695 CGF.getContext().getPointerType(Ty),
".local.ptr.addr");
4696 auto Result = UntiedLocalVars.insert(
4699 if (Result.second ==
false)
4700 *Result.first = std::make_pair(
4703 ParamTypes.push_back(PrivatePtr.
getType());
4705 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
4707 CopyFn = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4708 CopyFn, CopyFnTy->getPointerTo());
4709 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
4710 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
4711 for (
const auto &Pair : LastprivateDstsOrigs) {
4712 const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
4715 CGF.CapturedStmtInfo->lookup(OrigVD) !=
nullptr,
4717 Pair.second->getExprLoc());
4718 Scope.addPrivate(Pair.first, CGF.EmitLValue(&DRE).getAddress(CGF));
4720 for (
const auto &Pair : PrivatePtrs) {
4722 CGF.Builder.CreateLoad(Pair.second),
4723 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
4724 CGF.getContext().getDeclAlign(Pair.first));
4725 Scope.addPrivate(Pair.first, Replacement);
4726 if (
auto *DI = CGF.getDebugInfo())
4727 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
4728 (void)DI->EmitDeclareOfAutoVariable(
4729 Pair.first, Pair.second.getPointer(), CGF.Builder,
4734 for (
auto &Pair : UntiedLocalVars) {
4735 QualType VDType = Pair.first->getType().getNonReferenceType();
4737 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
4740 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(VDType)),
4741 CGF.getPointerAlign());
4742 Pair.second.first = Replacement;
4743 Ptr = CGF.Builder.CreateLoad(Replacement);
4744 Replacement =
Address(Ptr, CGF.ConvertTypeForMem(VDType),
4745 CGF.getContext().getDeclAlign(Pair.first));
4746 Pair.second.second = Replacement;
4748 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
4749 Address Replacement(Ptr, CGF.ConvertTypeForMem(VDType),
4750 CGF.getContext().getDeclAlign(Pair.first));
4751 Pair.second.first = Replacement;
4757 for (
const auto &Pair : FirstprivatePtrs) {
4759 CGF.Builder.CreateLoad(Pair.second),
4760 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
4761 CGF.getContext().getDeclAlign(Pair.first));
4762 FirstprivateScope.
addPrivate(Pair.first, Replacement);
4765 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
4768 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
4769 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
4770 for (
unsigned Cnt = 0, E = Data.
ReductionVars.size(); Cnt < E; ++Cnt) {
4776 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
4778 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
4781 Address(CGF.EmitScalarConversion(
4782 Replacement.
getPointer(), CGF.getContext().VoidPtrTy,
4783 CGF.getContext().getPointerType(
4793 (void)
Scope.Privatize();
4799 auto IPriv = C->privates().begin();
4800 auto IRed = C->reduction_ops().begin();
4801 auto ITD = C->taskgroup_descriptors().begin();
4802 for (
const Expr *Ref : C->varlists()) {
4803 InRedVars.emplace_back(Ref);
4804 InRedPrivs.emplace_back(*IPriv);
4805 InRedOps.emplace_back(*IRed);
4806 TaskgroupDescriptors.emplace_back(*ITD);
4815 if (!InRedVars.empty()) {
4817 for (
unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
4825 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
4827 llvm::Value *ReductionsPtr;
4828 if (
const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
4829 ReductionsPtr = CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr),
4830 TRExpr->getExprLoc());
4832 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4834 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
4837 CGF.EmitScalarConversion(
4838 Replacement.
getPointer(), CGF.getContext().VoidPtrTy,
4839 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
4840 InRedPrivs[Cnt]->getExprLoc()),
4841 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),
4854 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
4855 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.
Tied,
4857 OMPLexicalScope
Scope(*
this, S, llvm::None,
4860 TaskGen(*
this, OutlinedFn, Data);
4877 QualType ElemType = C.getBaseElementType(Ty);
4898 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
4901 auto PartId = std::next(I);
4902 auto TaskT = std::next(I, 4);
4905 Data.
Final.setInt(
false);
4908 auto IRef = C->varlist_begin();
4909 auto IElemInitRef = C->inits().begin();
4910 for (
auto *IInit : C->private_copies()) {
4925 getContext(), getContext().getTranslationUnitDecl(), 0);
4927 QualType BaseAndPointerAndMapperType = getContext().getConstantArrayType(
4931 getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
4933 getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
4934 QualType SizesType = getContext().getConstantArrayType(
4935 getContext().getIntTypeForBitwidth(64, 1),
4945 if (!isa_and_nonnull<llvm::ConstantPointerNull>(
4948 getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
4956 Data.
Dependences.emplace_back(C->getDependencyKind(), C->getModifier());
4957 DD.
DepExprs.append(C->varlist_begin(), C->varlist_end());
4959 auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD, MVD,
4964 enum { PrivatesParam = 2, CopyFnParam = 3 };
4965 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
4966 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
4967 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
4968 CS->getCapturedDecl()->getParam(PrivatesParam)));
4973 CallArgs.push_back(PrivatesPtr);
4974 ParamTypes.push_back(PrivatesPtr->getType());
4976 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4978 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->
getType()),
4979 ".firstpriv.ptr.addr");
4980 PrivatePtrs.emplace_back(VD, PrivatePtr);
4982 ParamTypes.push_back(PrivatePtr.
getType());
4984 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
4986 CopyFn = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4987 CopyFn, CopyFnTy->getPointerTo());
4988 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
4989 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
4990 for (
const auto &Pair : PrivatePtrs) {
4992 CGF.Builder.CreateLoad(Pair.second),
4993 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
4994 CGF.getContext().getDeclAlign(Pair.first));
4995 Scope.addPrivate(Pair.first, Replacement);
4999 (void)
Scope.Privatize();
5002 CGF.GetAddrOfLocalVar(BPVD), 0);
5004 CGF.GetAddrOfLocalVar(PVD), 0);
5005 InputInfo.
SizesArray = CGF.Builder.CreateConstArrayGEP(
5006 CGF.GetAddrOfLocalVar(SVD), 0);
5009 InputInfo.
MappersArray = CGF.Builder.CreateConstArrayGEP(
5010 CGF.GetAddrOfLocalVar(MVD), 0);
5014 OMPLexicalScope LexScope(CGF, S, OMPD_task,
false);
5017 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
5018 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen,
true,
5022 getContext().getIntTypeForBitwidth(32, 0),
5025 CGM.getOpenMPRuntime().emitTaskCall(*
this, S.getBeginLoc(), S, OutlinedFn,
5026 SharedsTy, CapturedStruct, &IfCond, Data);
5032 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
5034 const Expr *IfCond =
nullptr;
5035 for (
const auto *C : S.getClausesOfKind<
OMPIfClause>()) {
5036 if (C->getNameModifier() == OMPD_unknown ||
5037 C->getNameModifier() == OMPD_task) {
5038 IfCond = C->getCondition();
5049 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
5052 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn,
5053 SharedsTy, CapturedStruct, IfCond,
5058 EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data);
5063 CGM.getOpenMPRuntime().emitTaskyieldCall(*
this, S.getBeginLoc());
5067 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.getBeginLoc(), OMPD_barrier);
5075 Data.
Dependences.emplace_back(C->getDependencyKind(), C->getModifier());
5076 DD.
DepExprs.append(C->varlist_begin(), C->varlist_end());
5078 CGM.getOpenMPRuntime().emitTaskwaitCall(*
this, S.getBeginLoc(), Data);
5085 if (
const Expr *E = S.getReductionRef()) {
5090 Data.
ReductionVars.append(C->varlist_begin(), C->varlist_end());
5091 Data.
ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
5092 Data.
ReductionCopies.append(C->privates().begin(), C->privates().end());
5094 C->reduction_ops().end());
5095 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
5096 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
5098 llvm::Value *ReductionDesc =
5101 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5106 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
5108 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
5109 CGM.getOpenMPRuntime().emitTaskgroupRegion(*
this, CodeGen, S.getBeginLoc());
5114 ? llvm::AtomicOrdering::NotAtomic
5115 : llvm::AtomicOrdering::AcquireRelease;
5116 CGM.getOpenMPRuntime().emitFlush(
5119 if (
const auto *FlushClause = S.getSingleClause<
OMPFlushClause>())
5120 return llvm::makeArrayRef(FlushClause->varlist_begin(),
5121 FlushClause->varlist_end());
5124 S.getBeginLoc(), AO);
5129 LValue DOLVal = EmitLValue(DO->getDepobj());
5133 Dependencies.
DepExprs.append(DC->varlist_begin(), DC->varlist_end());
5134 Address DepAddr = CGM.getOpenMPRuntime().emitDepobjDependClause(
5135 *
this, Dependencies, DC->getBeginLoc());
5136 EmitStoreOfScalar(DepAddr.
getPointer(), DOLVal);
5140 CGM.getOpenMPRuntime().emitDestroyClause(*
this, DOLVal, DC->getBeginLoc());
5144 CGM.getOpenMPRuntime().emitUpdateClause(
5145 *
this, DOLVal, UC->getDependencyKind(), UC->getBeginLoc());
5151 if (!OMPParentLoopDirectiveForScan)
5164 if (C->getModifier() != OMPC_REDUCTION_inscan)
5166 Shareds.append(C->varlist_begin(), C->varlist_end());
5167 Privates.append(C->privates().begin(), C->privates().end());
5168 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
5169 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
5170 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
5171 CopyOps.append(C->copy_ops().begin(), C->copy_ops().end());
5172 CopyArrayTemps.append(C->copy_array_temps().begin(),
5173 C->copy_array_temps().end());
5174 CopyArrayElems.append(C->copy_array_elems().begin(),
5175 C->copy_array_elems().end());
5178 (getLangOpts().OpenMPSimd &&
5216 llvm::BasicBlock *OMPScanReduce = createBasicBlock(
"omp.inscan.reduce");
5217 EmitBranch(IsInclusive
5219 : BreakContinueStack.back().ContinueBlock.getBlock());
5220 EmitBlock(OMPScanDispatch);
5225 EmitBranch(IsInclusive ? OMPBeforeScanBlock : OMPAfterScanBlock);
5226 EmitBlock(OMPScanReduce);
5230 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5231 const Expr *PrivateExpr = Privates[I];
5232 const Expr *TempExpr = CopyArrayTemps[I];
5234 *cast<VarDecl>(cast<DeclRefExpr>(TempExpr)->getDecl()));
5235 LValue DestLVal = EmitLValue(TempExpr);
5236 LValue SrcLVal = EmitLValue(LHSs[I]);
5239 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5240 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
5244 CGM.getOpenMPRuntime().emitReduction(
5245 *
this, ParentDir.
getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
5246 {true, true, OMPD_simd});
5247 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5248 const Expr *PrivateExpr = Privates[I];
5252 DestLVal = EmitLValue(RHSs[I]);
5253 SrcLVal = EmitLValue(LHSs[I]);
5255 const Expr *TempExpr = CopyArrayTemps[I];
5256 DestLVal = EmitLValue(RHSs[I]);
5257 SrcLVal = EmitLValue(TempExpr);
5261 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5262 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
5266 EmitBranch(IsInclusive ? OMPAfterScanBlock : OMPBeforeScanBlock);
5267 OMPScanExitBlock = IsInclusive
5268 ? BreakContinueStack.back().ContinueBlock.getBlock()
5270 EmitBlock(OMPAfterScanBlock);
5274 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
5275 EmitBlock(OMPScanExitBlock);
5277 if (OMPFirstScanLoop) {
5279 const auto *IVExpr = cast<OMPLoopDirective>(ParentDir)
5280 .getIterationVariable()
5282 LValue IdxLVal = EmitLValue(IVExpr);
5283 llvm::Value *IdxVal = EmitLoadOfScalar(IdxLVal, IVExpr->
getExprLoc());
5284 IdxVal = Builder.CreateIntCast(IdxVal, SizeTy,
false);
5285 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5286 const Expr *PrivateExpr = Privates[I];
5287 const Expr *OrigExpr = Shareds[I];
5288 const Expr *CopyArrayElem = CopyArrayElems[I];
5291 cast<OpaqueValueExpr>(
5292 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
5294 LValue DestLVal = EmitLValue(CopyArrayElem);
5295 LValue SrcLVal = EmitLValue(OrigExpr);
5298 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5299 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
5303 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
5305 EmitBlock(OMPScanExitBlock);
5306 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
5308 EmitBlock(OMPScanDispatch);
5309 if (!OMPFirstScanLoop) {
5311 const auto *IVExpr = cast<OMPLoopDirective>(ParentDir)
5312 .getIterationVariable()
5314 LValue IdxLVal = EmitLValue(IVExpr);
5315 llvm::Value *IdxVal = EmitLoadOfScalar(IdxLVal, IVExpr->
getExprLoc());
5316 IdxVal = Builder.CreateIntCast(IdxVal, SizeTy,
false);
5317 llvm::BasicBlock *ExclusiveExitBB =
nullptr;
5319 llvm::BasicBlock *ContBB = createBasicBlock(
"omp.exclusive.dec");
5320 ExclusiveExitBB = createBasicBlock(
"omp.exclusive.copy.exit");
5321 llvm::Value *Cmp = Builder.CreateIsNull(IdxVal);
5322 Builder.CreateCondBr(Cmp, ExclusiveExitBB, ContBB);
5325 IdxVal = Builder.CreateNUWSub(IdxVal, llvm::ConstantInt::get(SizeTy, 1));
5327 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5328 const Expr *PrivateExpr = Privates[I];
5329 const Expr *OrigExpr = Shareds[I];
5330 const Expr *CopyArrayElem = CopyArrayElems[I];
5333 cast<OpaqueValueExpr>(
5334 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
5336 LValue SrcLVal = EmitLValue(CopyArrayElem);
5337 LValue DestLVal = EmitLValue(OrigExpr);
5340 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5341 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
5345 EmitBlock(ExclusiveExitBB);
5348 EmitBranch((OMPFirstScanLoop == IsInclusive) ? OMPBeforeScanBlock
5349 : OMPAfterScanBlock);
5350 EmitBlock(OMPAfterScanBlock);
5357 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
5358 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
5359 EmitVarDecl(*IVDecl);
5364 if (
const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
5365 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
5367 EmitIgnoredExpr(S.getCalcLastIteration());
5372 bool HasLastprivateClause =
false;
5375 OMPLoopScope PreInitScope(*
this, S);
5380 llvm::BasicBlock *ContBlock =
nullptr;
5381 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
5385 llvm::BasicBlock *ThenBlock = createBasicBlock(
"omp.precond.then");
5386 ContBlock = createBasicBlock(
"omp.precond.end");
5387 emitPreCond(*
this, S, S.getPreCond(), ThenBlock, ContBlock,
5388 getProfileCount(&S));
5389 EmitBlock(ThenBlock);
5390 incrementProfileCounter(&S);
5399 *
this, cast<DeclRefExpr>(
5401 ? S.getCombinedLowerBoundVariable()
5402 : S.getLowerBoundVariable())));
5404 *
this, cast<DeclRefExpr>(
5406 ? S.getCombinedUpperBoundVariable()
5407 : S.getUpperBoundVariable())));
5414 if (EmitOMPFirstprivateClause(S, LoopScope)) {
5418 CGM.getOpenMPRuntime().emitBarrierCall(
5419 *
this, S.getBeginLoc(), OMPD_unknown,
false,
5422 EmitOMPPrivateClause(S, LoopScope);
5426 EmitOMPReductionClauseInit(S, LoopScope);
5427 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
5428 EmitOMPPrivateLoopCounters(S, LoopScope);
5431 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*
this, S);
5434 llvm::Value *Chunk =
nullptr;
5437 ScheduleKind = C->getDistScheduleKind();
5438 if (
const Expr *Ch = C->getChunkSize()) {
5439 Chunk = EmitScalarExpr(Ch);
5440 Chunk = EmitScalarConversion(Chunk, Ch->getType(),
5441 S.getIterationVariable()->getType(),
5446 CGM.getOpenMPRuntime().getDefaultDistScheduleAndChunk(
5447 *
this, S, ScheduleKind, Chunk);
5449 const unsigned IVSize = getContext().getTypeSize(IVExpr->
getType());
5460 bool StaticChunked =
5464 Chunk !=
nullptr) ||
5467 IVSize, IVSigned,
false, IL.
getAddress(*
this),
5469 StaticChunked ? Chunk :
nullptr);
5473 getJumpDestInCurrentScope(createBasicBlock(
"omp.loop.exit"));
5476 ? S.getCombinedEnsureUpperBound()
5477 : S.getEnsureUpperBound());
5480 ? S.getCombinedInit()
5485 ? S.getCombinedCond()
5489 Cond = S.getCombinedDistCond();
5521 [&S, &LoopScope, Cond, IncExpr,
LoopExit, &CodeGenLoop,
5523 CGF.EmitOMPInnerLoop(
5524 S, LoopScope.requiresCleanups(), Cond, IncExpr,
5526 CodeGenLoop(CGF, S, LoopExit);
5529 if (StaticChunked) {
5530 CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());
5531 CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());
5532 CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());
5533 CGF.EmitIgnoredExpr(S.getCombinedInit());
5543 const OMPLoopArguments LoopArguments = {
5546 EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
5551 return CGF.
Builder.CreateIsNotNull(
5558 EmitOMPReductionClauseFinal(S, OMPD_simd);
5562 return CGF.
Builder.CreateIsNotNull(