28#include "llvm/ADT/SmallSet.h"
29#include "llvm/BinaryFormat/Dwarf.h"
30#include "llvm/Frontend/OpenMP/OMPConstants.h"
31#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
32#include "llvm/IR/Constants.h"
33#include "llvm/IR/DebugInfoMetadata.h"
34#include "llvm/IR/Instructions.h"
35#include "llvm/IR/IntrinsicInst.h"
36#include "llvm/IR/Metadata.h"
37#include "llvm/Support/AtomicOrdering.h"
38#include "llvm/Support/Debug.h"
41using namespace CodeGen;
42using namespace llvm::omp;
44#define TTL_CODEGEN_TYPE "target-teams-loop-codegen"
51class OMPLexicalScope :
public CodeGenFunction::LexicalScope {
53 for (
const auto *
C : S.clauses()) {
55 if (
const auto *PreInit =
56 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
57 for (
const auto *I : PreInit->decls()) {
58 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
61 CodeGenFunction::AutoVarEmission Emission =
70 CodeGenFunction::OMPPrivateScope InlinedShareds;
76 cast<BlockDecl>(CGF.
CurCodeDecl)->capturesVariable(VD));
82 const std::optional<OpenMPDirectiveKind> CapturedRegion = std::nullopt,
83 const bool EmitPreInitStmt =
true)
87 emitPreInitStmt(CGF, S);
90 assert(S.hasAssociatedStmt() &&
91 "Expected associated statement for inlined directive.");
92 const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
94 if (
C.capturesVariable() ||
C.capturesVariableByCopy()) {
95 auto *VD =
C.getCapturedVar();
97 "Canonical decl must be captured.");
101 InlinedShareds.isGlobalVarCaptured(VD)),
106 (void)InlinedShareds.Privatize();
112class OMPParallelScope final :
public OMPLexicalScope {
122 : OMPLexicalScope(CGF, S,
std::nullopt,
123 EmitPreInitStmt(S)) {}
128class OMPTeamsScope final :
public OMPLexicalScope {
137 : OMPLexicalScope(CGF, S,
std::nullopt,
138 EmitPreInitStmt(S)) {}
143class OMPLoopScope :
public CodeGenFunction::RunCleanupsScope {
146 CodeGenFunction::OMPMapVars PreCondVars;
147 if (
auto *LD = dyn_cast<OMPLoopDirective>(&S)) {
149 for (
const auto *E : LD->counters()) {
150 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
152 (void)PreCondVars.setVarAddr(
157 for (
const Expr *IRef :
C->varlists()) {
159 cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
160 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
161 QualType OrigVDTy = OrigVD->getType().getNonReferenceType();
162 (void)PreCondVars.setVarAddr(
171 (void)PreCondVars.apply(CGF);
174 LD->getInnermostCapturedStmt()->getCapturedStmt(),
175 true, LD->getLoopsNumber(),
176 [&CGF](
unsigned Cnt,
const Stmt *CurStmt) {
177 if (
const auto *CXXFor = dyn_cast<CXXForRangeStmt>(CurStmt)) {
178 if (
const Stmt *
Init = CXXFor->getInit())
180 CGF.
EmitStmt(CXXFor->getRangeStmt());
185 PreInits = cast_or_null<DeclStmt>(LD->getPreInits());
186 }
else if (
const auto *
Tile = dyn_cast<OMPTileDirective>(&S)) {
187 PreInits = cast_or_null<DeclStmt>(
Tile->getPreInits());
188 }
else if (
const auto *Unroll = dyn_cast<OMPUnrollDirective>(&S)) {
189 PreInits = cast_or_null<DeclStmt>(Unroll->getPreInits());
191 llvm_unreachable(
"Unknown loop-based directive kind.");
194 for (
const auto *I : PreInits->
decls())
197 PreCondVars.restore(CGF);
203 emitPreInitStmt(CGF, S);
207class OMPSimdLexicalScope :
public CodeGenFunction::LexicalScope {
208 CodeGenFunction::OMPPrivateScope InlinedShareds;
214 cast<BlockDecl>(CGF.
CurCodeDecl)->capturesVariable(VD));
220 InlinedShareds(CGF) {
221 for (
const auto *
C : S.clauses()) {
223 if (
const auto *PreInit =
224 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
225 for (
const auto *I : PreInit->decls()) {
226 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
229 CodeGenFunction::AutoVarEmission Emission =
235 }
else if (
const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(
C)) {
236 for (
const Expr *E : UDP->varlists()) {
237 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
238 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
241 }
else if (
const auto *UDP = dyn_cast<OMPUseDeviceAddrClause>(
C)) {
242 for (
const Expr *E : UDP->varlists()) {
244 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
251 if (
const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
252 if (
const Expr *E = TG->getReductionRef())
253 CGF.
EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
259 if (
C->getModifier() != OMPC_REDUCTION_inscan)
261 for (
const Expr *E :
C->copy_array_temps())
262 CopyArrayTemps.insert(cast<DeclRefExpr>(E)->getDecl());
264 const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
267 if (
C.capturesVariable() ||
C.capturesVariableByCopy()) {
268 auto *VD =
C.getCapturedVar();
269 if (CopyArrayTemps.contains(VD))
272 "Canonical decl must be captured.");
274 isCapturedVar(CGF, VD) ||
276 InlinedShareds.isGlobalVarCaptured(VD)),
284 (void)InlinedShareds.Privatize();
294LValue CodeGenFunction::EmitOMPSharedLValue(
const Expr *E) {
295 if (
const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
296 if (
const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
297 OrigVD = OrigVD->getCanonicalDecl();
312 llvm::Value *
Size =
nullptr;
313 auto SizeInChars =
C.getTypeSizeInChars(Ty);
314 if (SizeInChars.isZero()) {
320 Size ?
Builder.CreateNUWMul(Size, VlaSize.NumElts) : VlaSize.NumElts;
322 SizeInChars =
C.getTypeSizeInChars(Ty);
323 if (SizeInChars.isZero())
324 return llvm::ConstantInt::get(
SizeTy, 0);
332 const RecordDecl *RD = S.getCapturedRecordDecl();
334 auto CurCap = S.captures().begin();
336 E = S.capture_init_end();
337 I != E; ++I, ++CurField, ++CurCap) {
338 if (CurField->hasCapturedVLAType()) {
341 CapturedVars.push_back(Val);
342 }
else if (CurCap->capturesThis()) {
343 CapturedVars.push_back(CXXThisValue);
344 }
else if (CurCap->capturesVariableByCopy()) {
349 if (!CurField->getType()->isAnyPointerType()) {
353 Twine(CurCap->getCapturedVar()->getName(),
".casted"));
369 CapturedVars.push_back(CV);
371 assert(CurCap->capturesVariable() &&
"Expected capture by reference.");
372 CapturedVars.push_back(
394 return C.getLValueReferenceType(
400 if (
const auto *VLA = dyn_cast<VariableArrayType>(A))
402 if (!A->isVariablyModifiedType())
403 return C.getCanonicalType(
T);
405 return C.getCanonicalParamType(
T);
410struct FunctionOptions {
415 const bool UIntPtrCastRequired =
true;
418 const bool RegisterCastedArgsOnly =
false;
420 const StringRef FunctionName;
423 explicit FunctionOptions(
const CapturedStmt *S,
bool UIntPtrCastRequired,
424 bool RegisterCastedArgsOnly, StringRef FunctionName,
426 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
427 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
428 FunctionName(FunctionName), Loc(Loc) {}
434 llvm::MapVector<
const Decl *, std::pair<const VarDecl *, Address>>
436 llvm::DenseMap<
const Decl *, std::pair<const Expr *, llvm::Value *>>
438 llvm::Value *&CXXThisValue,
const FunctionOptions &FO) {
440 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
441 assert(CD->
hasBody() &&
"missing CapturedDecl body");
443 CXXThisValue =
nullptr;
453 auto I = FO.S->captures().begin();
455 if (!FO.UIntPtrCastRequired) {
475 if (FO.UIntPtrCastRequired &&
477 I->capturesVariableArrayType()))
480 if (I->capturesVariable() || I->capturesVariableByCopy()) {
481 CapVar = I->getCapturedVar();
483 }
else if (I->capturesThis()) {
486 assert(I->capturesVariableArrayType());
496 }
else if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
498 Ctx, DebugFunctionDecl,
499 CapVar ? CapVar->
getBeginLoc() : FD->getBeginLoc(),
500 CapVar ? CapVar->
getLocation() : FD->getLocation(), II, ArgType,
506 Args.emplace_back(Arg);
508 TargetArgs.emplace_back(
509 FO.UIntPtrCastRequired
526 llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
530 F->setDoesNotThrow();
531 F->setDoesNotRecurse();
535 F->removeFnAttr(llvm::Attribute::NoInline);
536 F->addFnAttr(llvm::Attribute::AlwaysInline);
541 FO.UIntPtrCastRequired ? FO.Loc : FO.S->getBeginLoc(),
542 FO.UIntPtrCastRequired ? FO.Loc
545 I = FO.S->captures().begin();
549 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
557 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
558 const VarDecl *CurVD = I->getCapturedVar();
559 if (!FO.RegisterCastedArgsOnly)
560 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
568 if (FD->hasCapturedVLAType()) {
569 if (FO.UIntPtrCastRequired) {
572 Args[Cnt]->getName(), ArgLVal),
577 VLASizes.try_emplace(Args[Cnt], VAT->
getSizeExpr(), ExprArg);
578 }
else if (I->capturesVariable()) {
579 const VarDecl *Var = I->getCapturedVar();
589 if (!FO.RegisterCastedArgsOnly) {
593 }
else if (I->capturesVariableByCopy()) {
594 assert(!FD->getType()->isAnyPointerType() &&
595 "Not expecting a captured pointer.");
596 const VarDecl *Var = I->getCapturedVar();
597 LocalAddrs.insert({Args[Cnt],
598 {Var, FO.UIntPtrCastRequired
600 CGF, I->getLocation(), FD->getType(),
601 Args[Cnt]->getName(), ArgLVal)
605 assert(I->capturesThis());
607 LocalAddrs.insert({Args[Cnt], {
nullptr, ArgLVal.
getAddress(CGF)}});
621 "CapturedStmtInfo should be set when generating the captured function");
624 bool NeedWrapperFunction =
627 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
628 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
630 llvm::raw_svector_ostream Out(Buffer);
632 if (NeedWrapperFunction)
634 FunctionOptions FO(&S, !NeedWrapperFunction,
false,
637 VLASizes, CXXThisValue, FO);
638 CodeGenFunction::OMPPrivateScope LocalScope(*
this);
639 for (
const auto &LocalAddrPair : LocalAddrs) {
640 if (LocalAddrPair.second.first) {
641 LocalScope.addPrivate(LocalAddrPair.second.first,
642 LocalAddrPair.second.second);
645 (void)LocalScope.Privatize();
646 for (
const auto &VLASizePair : VLASizes)
647 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
650 (void)LocalScope.ForceCleanup();
652 if (!NeedWrapperFunction)
655 FunctionOptions WrapperFO(&S,
true,
663 llvm::Function *WrapperF =
665 WrapperCGF.CXXThisValue, WrapperFO);
667 auto *PI = F->arg_begin();
668 for (
const auto *Arg : Args) {
670 auto I = LocalAddrs.find(Arg);
671 if (I != LocalAddrs.end()) {
672 LValue LV = WrapperCGF.MakeAddrLValue(
674 I->second.first ? I->second.first->getType() : Arg->getType(),
678 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
680 auto EI = VLASizes.find(Arg);
681 if (EI != VLASizes.end()) {
685 WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
687 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
690 CallArgs.emplace_back(WrapperCGF.EmitFromMemory(
CallArg, Arg->
getType()));
694 WrapperCGF.FinishFunction();
709 llvm::Value *NumElements =
emitArrayLength(ArrayTy, ElementTy, DestAddr);
716 DestBegin, NumElements);
721 llvm::Value *IsEmpty =
722 Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arraycpy.isempty");
723 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
726 llvm::BasicBlock *EntryBB =
Builder.GetInsertBlock();
731 llvm::PHINode *SrcElementPHI =
732 Builder.CreatePHI(SrcBegin->getType(), 2,
"omp.arraycpy.srcElementPast");
733 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
738 llvm::PHINode *DestElementPHI =
Builder.CreatePHI(
739 DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
740 DestElementPHI->addIncoming(DestBegin, EntryBB);
746 CopyGen(DestElementCurrent, SrcElementCurrent);
749 llvm::Value *DestElementNext =
751 1,
"omp.arraycpy.dest.element");
752 llvm::Value *SrcElementNext =
754 1,
"omp.arraycpy.src.element");
757 Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
758 Builder.CreateCondBr(Done, DoneBB, BodyBB);
759 DestElementPHI->addIncoming(DestElementNext,
Builder.GetInsertBlock());
760 SrcElementPHI->addIncoming(SrcElementNext,
Builder.GetInsertBlock());
770 const auto *BO = dyn_cast<BinaryOperator>(
Copy);
771 if (BO && BO->getOpcode() == BO_Assign) {
780 DestAddr, SrcAddr, OriginalType,
785 CodeGenFunction::OMPPrivateScope Remap(*
this);
786 Remap.addPrivate(DestVD, DestElement);
787 Remap.addPrivate(SrcVD, SrcElement);
788 (void)Remap.Privatize();
794 CodeGenFunction::OMPPrivateScope Remap(*
this);
795 Remap.addPrivate(SrcVD, SrcAddr);
796 Remap.addPrivate(DestVD, DestAddr);
797 (void)Remap.Privatize();
804 OMPPrivateScope &PrivateScope) {
807 bool DeviceConstTarget =
810 bool FirstprivateIsLastprivate =
false;
811 llvm::DenseMap<const VarDecl *, OpenMPLastprivateModifier> Lastprivates;
813 for (
const auto *D :
C->varlists())
814 Lastprivates.try_emplace(
823 bool MustEmitFirstprivateCopy =
824 CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
826 const auto *IRef =
C->varlist_begin();
827 const auto *InitsRef =
C->inits().begin();
828 for (
const Expr *IInit :
C->private_copies()) {
829 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
830 bool ThisFirstprivateIsLastprivate =
831 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
833 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
834 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
836 (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())) {
837 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
844 if (DeviceConstTarget && OrigVD->getType().isConstant(
getContext()) &&
846 (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())) {
847 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
852 FirstprivateIsLastprivate =
853 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
854 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
856 cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
865 if (CE && !CE.isReference()) {
871 if (CE && CE.isReference()) {
872 OriginalLVal = CE.getReferenceLValue(*
this, &DRE);
874 assert(!CE &&
"Expected non-constant firstprivate.");
893 Emission.getAllocatedAddress(), OriginalLVal.
getAddress(*
this),
898 RunCleanupsScope InitScope(*this);
900 setAddrOfLocalVar(VDInit, SrcElement);
901 EmitAnyExprToMem(Init, DestElement,
902 Init->getType().getQualifiers(),
904 LocalDeclMap.erase(VDInit);
909 PrivateScope.addPrivate(OrigVD, Emission.getAllocatedAddress());
915 setAddrOfLocalVar(VDInit, OriginalAddr);
917 LocalDeclMap.erase(VDInit);
919 if (ThisFirstprivateIsLastprivate &&
920 Lastprivates[OrigVD->getCanonicalDecl()] ==
921 OMPC_LASTPRIVATE_conditional) {
926 (*IRef)->getExprLoc());
931 LocalDeclMap.erase(VD);
932 setAddrOfLocalVar(VD, VDAddr);
934 IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);
936 assert(IsRegistered &&
937 "firstprivate var already registered as private");
945 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
950 CodeGenFunction::OMPPrivateScope &PrivateScope) {
955 auto IRef =
C->varlist_begin();
956 for (
const Expr *IInit :
C->private_copies()) {
957 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
958 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
959 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
964 assert(IsRegistered &&
"private var already registered as private");
981 llvm::BasicBlock *CopyBegin =
nullptr, *CopyEnd =
nullptr;
983 auto IRef =
C->varlist_begin();
984 auto ISrcRef =
C->source_exprs().begin();
985 auto IDestRef =
C->destination_exprs().begin();
986 for (
const Expr *AssignOp :
C->assignment_ops()) {
987 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
995 getContext().getTargetInfo().isTLSSupported()) {
997 "Copyin threadprivates should have been captured!");
1001 LocalDeclMap.erase(VD);
1011 if (CopiedVars.size() == 1) {
1017 auto *MasterAddrInt =
Builder.CreatePtrToInt(
1019 auto *PrivateAddrInt =
Builder.CreatePtrToInt(
1022 Builder.CreateICmpNE(MasterAddrInt, PrivateAddrInt), CopyBegin,
1027 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1028 const auto *DestVD =
1029 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1049 bool HasAtLeastOneLastprivate =
false;
1052 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1053 for (
const Expr *
C : LoopDirective->counters()) {
1060 HasAtLeastOneLastprivate =
true;
1064 const auto *IRef =
C->varlist_begin();
1065 const auto *IDestRef =
C->destination_exprs().begin();
1066 for (
const Expr *IInit :
C->private_copies()) {
1069 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1072 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
1073 const auto *DestVD =
1074 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1079 PrivateScope.addPrivate(DestVD,
EmitLValue(&DRE).getAddress(*
this));
1083 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
1084 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
1086 if (
C->getKind() == OMPC_LASTPRIVATE_conditional) {
1089 setAddrOfLocalVar(VD, VDAddr);
1095 bool IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);
1096 assert(IsRegistered &&
1097 "lastprivate var already registered as private");
1105 return HasAtLeastOneLastprivate;
1110 llvm::Value *IsLastIterCond) {
1119 llvm::BasicBlock *ThenBB =
nullptr;
1120 llvm::BasicBlock *DoneBB =
nullptr;
1121 if (IsLastIterCond) {
1127 return C->getKind() == OMPC_LASTPRIVATE_conditional;
1136 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1140 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1141 if (
const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1142 auto IC = LoopDirective->counters().begin();
1143 for (
const Expr *F : LoopDirective->finals()) {
1145 cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1147 AlreadyEmittedVars.insert(D);
1149 LoopCountersAndUpdates[D] = F;
1154 auto IRef =
C->varlist_begin();
1155 auto ISrcRef =
C->source_exprs().begin();
1156 auto IDestRef =
C->destination_exprs().begin();
1157 for (
const Expr *AssignOp :
C->assignment_ops()) {
1158 const auto *PrivateVD =
1159 cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1161 const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1162 if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1166 if (
const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1169 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1170 const auto *DestVD =
1171 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1174 if (
const auto *RefTy = PrivateVD->getType()->getAs<
ReferenceType>())
1180 if (
C->getKind() == OMPC_LASTPRIVATE_conditional)
1182 *
this,
MakeAddrLValue(PrivateAddr, (*IRef)->getType()), PrivateVD,
1183 (*IRef)->getExprLoc());
1186 EmitOMPCopy(
Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1192 if (
const Expr *PostUpdate =
C->getPostUpdateExpr())
1201 CodeGenFunction::OMPPrivateScope &PrivateScope,
bool ForInscan) {
1213 if (ForInscan != (
C->getModifier() == OMPC_REDUCTION_inscan))
1215 Shareds.append(
C->varlist_begin(),
C->varlist_end());
1216 Privates.append(
C->privates().begin(),
C->privates().end());
1217 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
1218 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
1219 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
1220 if (
C->getModifier() == OMPC_REDUCTION_task) {
1221 Data.ReductionVars.append(
C->privates().begin(),
C->privates().end());
1222 Data.ReductionOrigs.append(
C->varlist_begin(),
C->varlist_end());
1223 Data.ReductionCopies.append(
C->privates().begin(),
C->privates().end());
1224 Data.ReductionOps.append(
C->reduction_ops().begin(),
1225 C->reduction_ops().end());
1226 TaskLHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
1227 TaskRHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
1232 auto *ILHS = LHSs.begin();
1233 auto *IRHS = RHSs.begin();
1234 auto *IPriv = Privates.begin();
1235 for (
const Expr *IRef : Shareds) {
1236 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1238 RedCG.emitSharedOrigLValue(*
this, Count);
1239 RedCG.emitAggregateType(*
this, Count);
1241 RedCG.emitInitialization(*
this, Count, Emission.getAllocatedAddress(),
1242 RedCG.getSharedLValue(Count).getAddress(*
this),
1244 CGF.EmitAutoVarInit(Emission);
1248 Address BaseAddr = RedCG.adjustPrivateAddress(
1249 *
this, Count, Emission.getAllocatedAddress());
1251 PrivateScope.addPrivate(RedCG.getBaseDecl(Count), BaseAddr);
1252 assert(IsRegistered &&
"private var already registered as private");
1256 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1257 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1259 bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1263 PrivateScope.addPrivate(LHSVD,
1264 RedCG.getSharedLValue(Count).getAddress(*
this));
1267 isa<ArraySubscriptExpr>(IRef)) {
1270 PrivateScope.addPrivate(LHSVD,
1271 RedCG.getSharedLValue(Count).getAddress(*
this));
1272 PrivateScope.addPrivate(RHSVD,
1278 Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress(*
this);
1285 PrivateScope.addPrivate(LHSVD, OriginalAddr);
1286 PrivateScope.addPrivate(
1296 if (!
Data.ReductionVars.empty()) {
1297 Data.IsReductionWithTaskMod =
true;
1298 Data.IsWorksharingReduction =
1302 const Expr *TaskRedRef =
nullptr;
1305 TaskRedRef = cast<OMPParallelDirective>(D).getTaskReductionRefExpr();
1308 TaskRedRef = cast<OMPForDirective>(D).getTaskReductionRefExpr();
1311 TaskRedRef = cast<OMPSectionsDirective>(D).getTaskReductionRefExpr();
1313 case OMPD_parallel_for:
1314 TaskRedRef = cast<OMPParallelForDirective>(D).getTaskReductionRefExpr();
1316 case OMPD_parallel_master:
1318 cast<OMPParallelMasterDirective>(D).getTaskReductionRefExpr();
1320 case OMPD_parallel_sections:
1322 cast<OMPParallelSectionsDirective>(D).getTaskReductionRefExpr();
1324 case OMPD_target_parallel:
1326 cast<OMPTargetParallelDirective>(D).getTaskReductionRefExpr();
1328 case OMPD_target_parallel_for:
1330 cast<OMPTargetParallelForDirective>(D).getTaskReductionRefExpr();
1332 case OMPD_distribute_parallel_for:
1334 cast<OMPDistributeParallelForDirective>(D).getTaskReductionRefExpr();
1336 case OMPD_teams_distribute_parallel_for:
1337 TaskRedRef = cast<OMPTeamsDistributeParallelForDirective>(D)
1338 .getTaskReductionRefExpr();
1340 case OMPD_target_teams_distribute_parallel_for:
1341 TaskRedRef = cast<OMPTargetTeamsDistributeParallelForDirective>(D)
1342 .getTaskReductionRefExpr();
1350 case OMPD_parallel_for_simd:
1352 case OMPD_taskyield:
1356 case OMPD_taskgroup:
1364 case OMPD_cancellation_point:
1366 case OMPD_target_data:
1367 case OMPD_target_enter_data:
1368 case OMPD_target_exit_data:
1370 case OMPD_taskloop_simd:
1371 case OMPD_master_taskloop:
1372 case OMPD_master_taskloop_simd:
1373 case OMPD_parallel_master_taskloop:
1374 case OMPD_parallel_master_taskloop_simd:
1375 case OMPD_distribute:
1376 case OMPD_target_update:
1377 case OMPD_distribute_parallel_for_simd:
1378 case OMPD_distribute_simd:
1379 case OMPD_target_parallel_for_simd:
1380 case OMPD_target_simd:
1381 case OMPD_teams_distribute:
1382 case OMPD_teams_distribute_simd:
1383 case OMPD_teams_distribute_parallel_for_simd:
1384 case OMPD_target_teams:
1385 case OMPD_target_teams_distribute:
1386 case OMPD_target_teams_distribute_parallel_for_simd:
1387 case OMPD_target_teams_distribute_simd:
1388 case OMPD_declare_target:
1389 case OMPD_end_declare_target:
1390 case OMPD_threadprivate:
1392 case OMPD_declare_reduction:
1393 case OMPD_declare_mapper:
1394 case OMPD_declare_simd:
1396 case OMPD_declare_variant:
1397 case OMPD_begin_declare_variant:
1398 case OMPD_end_declare_variant:
1401 llvm_unreachable(
"Enexpected directive with task reductions.");
1404 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(TaskRedRef)->getDecl());
1407 false, TaskRedRef->
getType());
1419 bool HasAtLeastOneReduction =
false;
1420 bool IsReductionWithTaskMod =
false;
1423 if (
C->getModifier() == OMPC_REDUCTION_inscan)
1425 HasAtLeastOneReduction =
true;
1426 Privates.append(
C->privates().begin(),
C->privates().end());
1427 LHSExprs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
1428 RHSExprs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
1429 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
1430 IsReductionWithTaskMod =
1431 IsReductionWithTaskMod ||
C->getModifier() == OMPC_REDUCTION_task;
1433 if (HasAtLeastOneReduction) {
1434 if (IsReductionWithTaskMod) {
1439 bool TeamsLoopCanBeParallel =
false;
1440 if (
auto *TTLD = dyn_cast<OMPTargetTeamsGenericLoopDirective>(&D))
1441 TeamsLoopCanBeParallel = TTLD->canBeParallelFor();
1444 TeamsLoopCanBeParallel || ReductionKind == OMPD_simd;
1445 bool SimpleReduction = ReductionKind == OMPD_simd;
1449 *
this, D.
getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1450 {WithNowait, SimpleReduction, ReductionKind});
1459 llvm::BasicBlock *DoneBB =
nullptr;
1461 if (
const Expr *PostUpdate =
C->getPostUpdateExpr()) {
1463 if (llvm::Value *Cond = CondGen(CGF)) {
1468 CGF.
Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1486 CodeGenBoundParametersTy;
1496 for (
const Expr *Ref :
C->varlists()) {
1497 if (!Ref->getType()->isScalarType())
1499 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1502 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1507 for (
const Expr *Ref :
C->varlists()) {
1508 if (!Ref->getType()->isScalarType())
1510 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1513 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1518 for (
const Expr *Ref :
C->varlists()) {
1519 if (!Ref->getType()->isScalarType())
1521 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1524 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1533 for (
const Expr *Ref :
C->varlists()) {
1534 if (!Ref->getType()->isScalarType())
1536 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1539 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1543 CGF, S, PrivateDecls);
1549 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1550 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1551 llvm::Value *NumThreads =
nullptr;
1552 llvm::Function *OutlinedFn =
1557 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1558 NumThreads = CGF.
EmitScalarExpr(NumThreadsClause->getNumThreads(),
1561 CGF, NumThreads, NumThreadsClause->getBeginLoc());
1564 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1566 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1568 const Expr *IfCond =
nullptr;
1569 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
1570 if (
C->getNameModifier() == OMPD_unknown ||
1571 C->getNameModifier() == OMPD_parallel) {
1572 IfCond =
C->getCondition();
1577 OMPParallelScope
Scope(CGF, S);
1583 CodeGenBoundParameters(CGF, S, CapturedVars);
1586 CapturedVars, IfCond, NumThreads);
1591 if (!CVD->
hasAttr<OMPAllocateDeclAttr>())
1593 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
1595 return !((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||
1596 AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&
1597 !AA->getAllocator());
1612 CGF, S.getBeginLoc(), OMPD_unknown,
false,
1632 Size = CGF.
Builder.CreateNUWAdd(
1641 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
1642 assert(AA->getAllocator() &&
1643 "Expected allocator expression for non-default allocator.");
1647 if (Allocator->getType()->isIntegerTy())
1649 else if (Allocator->getType()->isPointerTy())
1653 llvm::Value *Addr = OMPBuilder.createOMPAlloc(
1656 llvm::CallInst *FreeCI =
1657 OMPBuilder.createOMPFree(CGF.
Builder, Addr, Allocator);
1681 std::string Suffix = getNameWithSeparators({
"cache",
""});
1684 llvm::CallInst *ThreadPrivateCacheCall =
1685 OMPBuilder.createCachedThreadPrivate(CGF.
Builder,
Data, Size, CacheName);
1693 llvm::raw_svector_ostream OS(Buffer);
1694 StringRef Sep = FirstSeparator;
1695 for (StringRef Part : Parts) {
1699 return OS.str().str();
1707 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
Builder,
false,
1708 "." + RegionName +
".after");
1724 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
Builder,
false,
1725 "." + RegionName +
".after");
1740 llvm::Value *IfCond =
nullptr;
1745 llvm::Value *NumThreads =
nullptr;
1750 ProcBindKind ProcBind = OMP_PROC_BIND_default;
1752 ProcBind = ProcBindClause->getProcBindKind();
1754 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1758 auto FiniCB = [
this](InsertPointTy IP) {
1766 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
1767 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
1775 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1778 auto BodyGenCB = [&,
this](InsertPointTy AllocaIP,
1779 InsertPointTy CodeGenIP) {
1781 *
this, ParallelRegionBodyStmt, AllocaIP, CodeGenIP,
"parallel");
1784 CGCapturedStmtInfo CGSI(*CS,
CR_OpenMP);
1785 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*
this, &CGSI);
1786 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
1789 OMPBuilder.createParallel(
Builder, AllocaIP, BodyGenCB, PrivCB, FiniCB,
1790 IfCond, NumThreads, ProcBind, S.hasCancel()));
1797 OMPPrivateScope PrivateScope(CGF);
1802 (void)PrivateScope.Privatize();
1803 CGF.
EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1824class OMPTransformDirectiveScopeRAII {
1825 OMPLoopScope *
Scope =
nullptr;
1826 CodeGenFunction::CGCapturedStmtInfo *CGSI =
nullptr;
1827 CodeGenFunction::CGCapturedStmtRAII *CapInfoRAII =
nullptr;
1829 OMPTransformDirectiveScopeRAII(
const OMPTransformDirectiveScopeRAII &) =
1831 OMPTransformDirectiveScopeRAII &
1832 operator=(
const OMPTransformDirectiveScopeRAII &) =
delete;
1836 if (
const auto *Dir = dyn_cast<OMPLoopBasedDirective>(S)) {
1837 Scope =
new OMPLoopScope(CGF, *Dir);
1838 CGSI =
new CodeGenFunction::CGCapturedStmtInfo(
CR_OpenMP);
1839 CapInfoRAII =
new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI);
1842 ~OMPTransformDirectiveScopeRAII() {
1853 int MaxLevel,
int Level = 0) {
1854 assert(Level < MaxLevel &&
"Too deep lookup during loop body codegen.");
1855 const Stmt *SimplifiedS = S->IgnoreContainers();
1856 if (
const auto *CS = dyn_cast<CompoundStmt>(SimplifiedS)) {
1859 "LLVM IR generation of compound statement ('{}')");
1862 CodeGenFunction::LexicalScope
Scope(CGF, S->getSourceRange());
1863 for (
const Stmt *CurStmt : CS->body())
1864 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level);
1867 if (SimplifiedS == NextLoop) {
1868 if (
auto *Dir = dyn_cast<OMPLoopTransformationDirective>(SimplifiedS))
1869 SimplifiedS = Dir->getTransformedStmt();
1870 if (
const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(SimplifiedS))
1871 SimplifiedS = CanonLoop->getLoopStmt();
1872 if (
const auto *For = dyn_cast<ForStmt>(SimplifiedS)) {
1875 assert(isa<CXXForRangeStmt>(SimplifiedS) &&
1876 "Expected canonical for loop or range-based for loop.");
1877 const auto *CXXFor = cast<CXXForRangeStmt>(SimplifiedS);
1878 CGF.
EmitStmt(CXXFor->getLoopVarStmt());
1879 S = CXXFor->getBody();
1881 if (Level + 1 < MaxLevel) {
1884 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1);
1893 RunCleanupsScope BodyScope(*
this);
1902 for (
const Expr *UE :
C->updates())
1909 BreakContinueStack.push_back(BreakContinue(
LoopExit, Continue));
1921 OMPPrivateScope InscanScope(*
this);
1923 bool IsInscanRegion = InscanScope.Privatize();
1924 if (IsInscanRegion) {
1956 BreakContinueStack.pop_back();
1967 std::unique_ptr<CodeGenFunction::CGCapturedStmtInfo> CSI =
1968 std::make_unique<CodeGenFunction::CGCapturedStmtInfo>(*S);
1969 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, CSI.get());
1976static llvm::CallInst *
1981 EffectiveArgs.reserve(Args.size() + 1);
1982 llvm::append_range(EffectiveArgs, Args);
1983 EffectiveArgs.push_back(Cap.second);
1988llvm::CanonicalLoopInfo *
1990 assert(Depth == 1 &&
"Nested loops with OpenMPIRBuilder not yet implemented");
2016 const Stmt *SyntacticalLoop = S->getLoopStmt();
2023 LexicalScope ForScope(*
this, S->getSourceRange());
2027 const Stmt *BodyStmt;
2028 if (
const auto *For = dyn_cast<ForStmt>(SyntacticalLoop)) {
2029 if (
const Stmt *InitStmt = For->getInit())
2031 BodyStmt = For->getBody();
2032 }
else if (
const auto *RangeFor =
2033 dyn_cast<CXXForRangeStmt>(SyntacticalLoop)) {
2034 if (
const DeclStmt *RangeStmt = RangeFor->getRangeStmt())
2036 if (
const DeclStmt *BeginStmt = RangeFor->getBeginStmt())
2038 if (
const DeclStmt *EndStmt = RangeFor->getEndStmt())
2040 if (
const DeclStmt *LoopVarStmt = RangeFor->getLoopVarStmt())
2042 BodyStmt = RangeFor->getBody();
2044 llvm_unreachable(
"Expected for-stmt or range-based for-stmt");
2047 const CapturedStmt *DistanceFunc = S->getDistanceFunc();
2064 auto BodyGen = [&,
this](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP,
2065 llvm::Value *IndVar) {
2070 const DeclRefExpr *LoopVarRef = S->getLoopVarRef();
2076 RunCleanupsScope BodyScope(*
this);
2079 llvm::CanonicalLoopInfo *CL =
2080 OMPBuilder.createCanonicalLoop(
Builder, BodyGen, DistVal);
2083 Builder.restoreIP(CL->getAfterIP());
2084 ForScope.ForceCleanup();
2092 const Expr *IncExpr,
2103 const auto &OMPED = cast<OMPExecutableDirective>(S);
2104 const CapturedStmt *ICS = OMPED.getInnermostCapturedStmt();
2118 llvm::BasicBlock *ExitBlock =
LoopExit.getBlock();
2119 if (RequiresCleanup)
2126 if (ExitBlock !=
LoopExit.getBlock()) {
2136 BreakContinueStack.push_back(BreakContinue(
LoopExit, Continue));
2144 BreakContinueStack.pop_back();
2155 bool HasLinears =
false;
2159 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
Init)->getDecl());
2160 if (
const auto *Ref =
2163 const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
2179 if (
const auto *CS = cast_or_null<BinaryOperator>(
C->getCalcStep()))
2180 if (
const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
2194 llvm::BasicBlock *DoneBB =
nullptr;
2197 auto IC =
C->varlist_begin();
2198 for (
const Expr *F :
C->finals()) {
2200 if (llvm::Value *Cond = CondGen(*
this)) {
2205 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2209 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
2214 CodeGenFunction::OMPPrivateScope VarScope(*
this);
2215 VarScope.addPrivate(OrigVD, OrigAddr);
2216 (void)VarScope.Privatize();
2220 if (
const Expr *PostUpdate =
C->getPostUpdateExpr())
2232 llvm::APInt ClauseAlignment(64, 0);
2233 if (
const Expr *AlignmentExpr = Clause->getAlignment()) {
2236 ClauseAlignment = AlignmentCI->getValue();
2238 for (
const Expr *E : Clause->varlists()) {
2239 llvm::APInt Alignment(ClauseAlignment);
2240 if (Alignment == 0) {
2250 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2251 "alignment is not power of 2");
2252 if (Alignment != 0) {
2266 auto I = S.private_counters().begin();
2267 for (
const Expr *E : S.counters()) {
2268 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2269 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
2273 LocalDeclMap.erase(PrivateVD);
2274 (void)LoopScope.addPrivate(VD, VarEmission.getAllocatedAddress());
2282 (void)LoopScope.addPrivate(PrivateVD, VarEmission.getAllocatedAddress());
2288 if (!
C->getNumForLoops())
2290 for (
unsigned I = S.getLoopsNumber(), E =
C->getLoopNumIterations().size();
2292 const auto *DRE = cast<DeclRefExpr>(
C->getLoopCounter(I));
2293 const auto *VD = cast<VarDecl>(DRE->getDecl());
2296 if (DRE->refersToEnclosingVariableOrCapture()) {
2297 (void)LoopScope.addPrivate(
2305 const Expr *Cond, llvm::BasicBlock *TrueBlock,
2306 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
2310 CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
2312 (void)PreCondScope.Privatize();
2314 for (
const Expr *I : S.inits()) {
2320 CodeGenFunction::OMPMapVars PreCondVars;
2321 for (
const Expr *E : S.dependent_counters()) {
2324 assert(!E->getType().getNonReferenceType()->isRecordType() &&
2325 "dependent counter must not be an iterator.");
2326 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2329 (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr);
2331 (void)PreCondVars.apply(CGF);
2332 for (
const Expr *E : S.dependent_inits()) {
2339 PreCondVars.restore(CGF);
2343 const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
2348 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
2349 for (
const Expr *
C : LoopDirective->counters()) {
2355 auto CurPrivate =
C->privates().begin();
2356 for (
const Expr *E :
C->varlists()) {
2357 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2358 const auto *PrivateVD =
2359 cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
2365 assert(IsRegistered &&
"linear var already registered as private");
2383 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
2392 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
2407 if (
C->getKind() == OMPC_ORDER_concurrent)
2414 return C->getModifier() == OMPC_REDUCTION_inscan;
2425 llvm::BasicBlock *DoneBB =
nullptr;
2429 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
2430 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
2431 const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
2433 OrigVD->hasGlobalStorage() || CED) {
2435 if (llvm::Value *Cond = CondGen(*
this)) {
2440 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2454 OMPPrivateScope VarScope(*
this);
2455 VarScope.addPrivate(OrigVD, OrigAddr);
2456 (void)VarScope.Privatize();
2468 CodeGenFunction::JumpDest
LoopExit) {
2476 auto VDecl = cast<VarDecl>(Helper->
getDecl());
2484 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](
CodeGenFunction &CGF,
2487 CodeGenFunction::OMPLocalDeclMapRAII
Scope(CGF);
2493 CodeGenFunction::OMPLocalDeclMapRAII
Scope(CGF);
2498 const Expr *IfCond =
nullptr;
2500 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
2502 (
C->getNameModifier() == OMPD_unknown ||
2503 C->getNameModifier() == OMPD_simd)) {
2504 IfCond =
C->getCondition();
2521 "Expected simd directive");
2522 OMPLoopScope PreInitScope(CGF, S);
2539 llvm::BasicBlock *ContBlock =
nullptr;
2546 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
2553 const Expr *IVExpr = S.getIterationVariable();
2554 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
2561 if (
const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2562 CGF.
EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2570 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2576 CGF, S, CGF.
EmitLValue(S.getIterationVariable()));
2578 (void)LoopScope.Privatize();
2589 S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),
2591 emitOMPLoopBodyWithStopPoint(CGF, S,
2592 CodeGenFunction::JumpDest());
2598 if (HasLastprivateClause)
2603 LoopScope.restoreMap();
2617 if (!(isa<OMPSimdlenClause>(
C) || isa<OMPSafelenClause>(
C) ||
2618 isa<OMPOrderClause>(
C) || isa<OMPAlignedClause>(
C)))
2625 if (
const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(S.getRawStmt())) {
2626 if (
const Stmt *SyntacticalLoop = CanonLoop->getLoopStmt()) {
2627 for (
const Stmt *SubStmt : SyntacticalLoop->
children()) {
2630 if (
const CompoundStmt *CS = dyn_cast<CompoundStmt>(SubStmt)) {
2634 if (isa<OMPOrderedDirective>(CSSubStmt)) {
2644static llvm::MapVector<llvm::Value *, llvm::Value *>
2646 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars;
2648 llvm::APInt ClauseAlignment(64, 0);
2649 if (
const Expr *AlignmentExpr = Clause->getAlignment()) {
2652 ClauseAlignment = AlignmentCI->getValue();
2654 for (
const Expr *E : Clause->varlists()) {
2655 llvm::APInt Alignment(ClauseAlignment);
2656 if (Alignment == 0) {
2663 E->getType()->getPointeeType()))
2666 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2667 "alignment is not power of 2");
2669 AlignedVars[PtrValue] = CGF.
Builder.getInt64(Alignment.getSExtValue());
2676 bool UseOMPIRBuilder =
2678 if (UseOMPIRBuilder) {
2679 auto &&CodeGenIRBuilder = [
this, &S, UseOMPIRBuilder](
CodeGenFunction &CGF,
2682 if (UseOMPIRBuilder) {
2683 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars =
2686 const Stmt *Inner = S.getRawStmt();
2687 llvm::CanonicalLoopInfo *CLI =
2690 llvm::OpenMPIRBuilder &OMPBuilder =
2693 llvm::ConstantInt *Simdlen =
nullptr;
2698 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
2701 llvm::ConstantInt *Safelen =
nullptr;
2706 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
2709 llvm::omp::OrderKind Order = llvm::omp::OrderKind::OMP_ORDER_unknown;
2711 if (
C->getKind() == OpenMPOrderClauseKind ::OMPC_ORDER_concurrent) {
2712 Order = llvm::omp::OrderKind::OMP_ORDER_concurrent;
2717 OMPBuilder.applySimd(CLI, AlignedVars,
2718 nullptr, Order, Simdlen, Safelen);
2725 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
2732 ParentLoopDirectiveForScanRegion ScanRegion(*
this, S);
2740 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
2749 OMPTransformDirectiveScopeRAII TileScope(*
this, &S);
2756 if (UseOMPIRBuilder) {
2758 const Stmt *Inner = S.getRawStmt();
2769 llvm::CanonicalLoopInfo *UnrolledCLI =
nullptr;
2773 OMPBuilder.unrollLoopFull(DL, CLI);
2776 if (
Expr *FactorExpr = PartialClause->getFactor()) {
2777 Factor = FactorExpr->EvaluateKnownConstInt(
getContext()).getZExtValue();
2778 assert(Factor >= 1 &&
"Only positive factors are valid");
2780 OMPBuilder.unrollLoopPartial(DL, CLI, Factor,
2781 NeedsUnrolledCLI ? &UnrolledCLI :
nullptr);
2783 OMPBuilder.unrollLoopHeuristic(DL, CLI);
2786 assert((!NeedsUnrolledCLI || UnrolledCLI) &&
2787 "NeedsUnrolledCLI implies UnrolledCLI to be set");
2804 if (
Expr *FactorExpr = PartialClause->getFactor()) {
2806 FactorExpr->EvaluateKnownConstInt(
getContext()).getZExtValue();
2807 assert(Factor >= 1 &&
"Only positive factors are valid");
2815void CodeGenFunction::EmitOMPOuterLoop(
2817 CodeGenFunction::OMPPrivateScope &LoopScope,
2818 const CodeGenFunction::OMPLoopArguments &LoopArgs,
2823 const Expr *IVExpr = S.getIterationVariable();
2837 llvm::Value *BoolCondVal =
nullptr;
2838 if (!DynamicOrOrdered) {
2849 RT.
emitForNext(*
this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
2850 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
2855 llvm::BasicBlock *ExitBlock =
LoopExit.getBlock();
2856 if (LoopScope.requiresCleanups())
2860 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
2861 if (ExitBlock !=
LoopExit.getBlock()) {
2869 if (DynamicOrOrdered)
2874 BreakContinueStack.push_back(BreakContinue(
LoopExit, Continue));
2882 CGF.LoopStack.setParallel(!IsMonotonic);
2883 if (const auto *C = S.getSingleClause<OMPOrderClause>())
2884 if (C->getKind() == OMPC_ORDER_concurrent)
2885 CGF.LoopStack.setParallel(true);
2887 CGF.EmitOMPSimdInit(S);
2890 [&S, &LoopArgs,
LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered,
2898 CGF.EmitOMPInnerLoop(
2899 S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
2901 CodeGenLoop(CGF, S, LoopExit);
2904 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
2909 BreakContinueStack.pop_back();
2910 if (!DynamicOrOrdered) {
2923 auto &&CodeGen = [DynamicOrOrdered, &S, &LoopArgs](
CodeGenFunction &CGF) {
2924 if (!DynamicOrOrdered)
2925 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2928 OMPCancelStack.emitExit(*
this, S.getDirectiveKind(), CodeGen);
2931void CodeGenFunction::EmitOMPForOuterLoop(
2934 const OMPLoopArguments &LoopArgs,
2942 LoopArgs.Chunk !=
nullptr)) &&
2943 "static non-chunked schedule does not need outer loop");
2995 const Expr *IVExpr = S.getIterationVariable();
2999 if (DynamicOrOrdered) {
3000 const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
3001 CGDispatchBounds(*
this, S, LoopArgs.LB, LoopArgs.UB);
3002 llvm::Value *LBVal = DispatchBounds.first;
3003 llvm::Value *UBVal = DispatchBounds.second;
3007 IVSigned, Ordered, DipatchRTInputValues);
3010 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
3011 LoopArgs.ST, LoopArgs.Chunk);
3013 ScheduleKind, StaticInit);
3017 const unsigned IVSize,
3018 const bool IVSigned) {
3025 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
3026 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
3027 OuterLoopArgs.IncExpr = S.getInc();
3028 OuterLoopArgs.Init = S.getInit();
3029 OuterLoopArgs.Cond = S.getCond();
3030 OuterLoopArgs.NextLB = S.getNextLowerBound();
3031 OuterLoopArgs.NextUB = S.getNextUpperBound();
3032 OuterLoopArgs.DKind = LoopArgs.DKind;
3033 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
3038 const unsigned IVSize,
const bool IVSigned) {}
3040void CodeGenFunction::EmitOMPDistributeOuterLoop(
3042 OMPPrivateScope &LoopScope,
const OMPLoopArguments &LoopArgs,
3052 const Expr *IVExpr = S.getIterationVariable();
3057 IVSize, IVSigned,
false, LoopArgs.IL, LoopArgs.LB,
3058 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
3065 IncExpr = S.getDistInc();
3067 IncExpr = S.getInc();
3072 OMPLoopArguments OuterLoopArgs;
3073 OuterLoopArgs.LB = LoopArgs.LB;
3074 OuterLoopArgs.UB = LoopArgs.UB;
3075 OuterLoopArgs.ST = LoopArgs.ST;
3076 OuterLoopArgs.IL = LoopArgs.IL;
3077 OuterLoopArgs.Chunk = LoopArgs.Chunk;
3079 ? S.getCombinedEnsureUpperBound()
3080 : S.getEnsureUpperBound();
3081 OuterLoopArgs.IncExpr = IncExpr;
3083 ? S.getCombinedInit()
3086 ? S.getCombinedCond()
3089 ? S.getCombinedNextLowerBound()
3090 : S.getNextLowerBound();
3092 ? S.getCombinedNextUpperBound()
3093 : S.getNextUpperBound();
3094 OuterLoopArgs.DKind = OMPD_distribute;
3096 EmitOMPOuterLoop(
false,
false, S,
3097 LoopScope, OuterLoopArgs, CodeGenLoopContent,
3101static std::pair<LValue, LValue>
3144static std::pair<llvm::Value *, llvm::Value *>
3155 llvm::Value *LBVal =
3157 llvm::Value *UBVal =
3159 return {LBVal, UBVal};
3165 const auto &Dir = cast<OMPLoopDirective>(S);
3167 CGF.
EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
3168 llvm::Value *LBCast =
3171 CapturedVars.push_back(LBCast);
3173 CGF.
EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
3175 llvm::Value *UBCast =
3178 CapturedVars.push_back(UBCast);
3184 CodeGenFunction::JumpDest
LoopExit) {
3188 bool HasCancel =
false;
3190 if (
const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
3191 HasCancel = D->hasCancel();
3192 else if (
const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
3193 HasCancel = D->hasCancel();
3194 else if (
const auto *D =
3195 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
3196 HasCancel = D->hasCancel();
3198 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
3208 CGInlinedWorksharingLoop,
3218 OMPLexicalScope
Scope(*
this, S, OMPD_parallel);
3228 OMPLexicalScope
Scope(*
this, S, OMPD_parallel);
3237 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
3248 llvm::Constant *Addr;
3251 S, ParentName, Fn, Addr,
true, CodeGen);
3252 assert(Fn && Addr &&
"Target device function emission failed.");
3264struct ScheduleKindModifiersTy {
3280 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3281 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3287 if (
const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3295 bool HasLastprivateClause;
3298 OMPLoopScope PreInitScope(*
this, S);
3303 llvm::BasicBlock *ContBlock =
nullptr;
3310 emitPreCond(*
this, S, S.getPreCond(), ThenBlock, ContBlock,
3316 RunCleanupsScope DoacrossCleanupScope(*
this);
3317 bool Ordered =
false;
3319 if (OrderedClause->getNumForLoops())
3330 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*
this, S);
3331 LValue LB = Bounds.first;
3332 LValue UB = Bounds.second;
3340 OMPPrivateScope LoopScope(*
this);
3346 *
this, S.getBeginLoc(), OMPD_unknown,
false,
3351 *
this, S,
EmitLValue(S.getIterationVariable()));
3356 (void)LoopScope.Privatize();
3361 const Expr *ChunkExpr =
nullptr;
3364 ScheduleKind.
Schedule =
C->getScheduleKind();
3365 ScheduleKind.
M1 =
C->getFirstScheduleModifier();
3366 ScheduleKind.
M2 =
C->getSecondScheduleModifier();
3367 ChunkExpr =
C->getChunkSize();
3371 *
this, S, ScheduleKind.
Schedule, ChunkExpr);
3373 bool HasChunkSizeOne =
false;
3374 llvm::Value *Chunk =
nullptr;
3378 S.getIterationVariable()->getType(),
3382 llvm::APSInt EvaluatedChunk =
Result.Val.getInt();
3383 HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
3392 bool StaticChunkedOne =
3394 Chunk !=
nullptr) &&
3399 (ScheduleKind.
Schedule == OMPC_SCHEDULE_static &&
3400 !(ScheduleKind.
M1 == OMPC_SCHEDULE_MODIFIER_nonmonotonic ||
3401 ScheduleKind.
M2 == OMPC_SCHEDULE_MODIFIER_nonmonotonic)) ||
3402 ScheduleKind.
M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
3403 ScheduleKind.
M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
3405 Chunk !=
nullptr) ||
3406 StaticChunkedOne) &&
3414 CGF.EmitOMPSimdInit(S);
3416 if (C->getKind() == OMPC_ORDER_concurrent)
3417 CGF.LoopStack.setParallel(true);
3420 [IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk,
3429 IVSize, IVSigned, Ordered, IL.getAddress(CGF),
3430 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF),
3431 StaticChunkedOne ? Chunk :
nullptr);
3432 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
3433 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind,
3436 if (!StaticChunkedOne)
3437 CGF.EmitIgnoredExpr(S.getEnsureUpperBound());
3439 CGF.EmitIgnoredExpr(S.getInit());
3453 CGF.EmitOMPInnerLoop(
3454 S, LoopScope.requiresCleanups(),
3455 StaticChunkedOne ? S.getCombinedParForInDistCond()
3457 StaticChunkedOne ? S.getDistInc() : S.getInc(),
3459 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit);
3466 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
3469 OMPCancelStack.emitExit(*
this, S.getDirectiveKind(), CodeGen);
3473 OMPLoopArguments LoopArguments(
3474 LB.getAddress(*
this), UB.getAddress(*
this), ST.getAddress(*
this),
3475 IL.getAddress(*
this), Chunk, EUB);
3476 LoopArguments.DKind = OMPD_for;
3477 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
3478 LoopArguments, CGDispatchBounds);
3482 return CGF.
Builder.CreateIsNotNull(
3488 ? OMPD_parallel_for_simd
3493 return CGF.
Builder.CreateIsNotNull(
3497 if (HasLastprivateClause)
3501 LoopScope.restoreMap();
3503 return CGF.
Builder.CreateIsNotNull(
3507 DoacrossCleanupScope.ForceCleanup();
3514 return HasLastprivateClause;
3520static std::pair<LValue, LValue>
3522 const auto &LS = cast<OMPLoopDirective>(S);
3534static std::pair<llvm::Value *, llvm::Value *>
3537 const auto &LS = cast<OMPLoopDirective>(S);
3538 const Expr *IVExpr = LS.getIterationVariable();
3540 llvm::Value *LBVal = CGF.
Builder.getIntN(IVSize, 0);
3542 return {LBVal, UBVal};
3554 llvm::function_ref<llvm::Value *(
CodeGenFunction &)> NumIteratorsGen) {
3555 llvm::Value *OMPScanNumIterations = CGF.
Builder.CreateIntCast(
3556 NumIteratorsGen(CGF), CGF.
SizeTy,
false);
3562 assert(
C->getModifier() == OMPC_REDUCTION_inscan &&
3563 "Only inscan reductions are expected.");
3564 Shareds.append(
C->varlist_begin(),
C->varlist_end());
3565 Privates.append(
C->privates().begin(),
C->privates().end());
3566 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
3567 CopyArrayTemps.append(
C->copy_array_temps().begin(),
3568 C->copy_array_temps().end());
3576 auto *ITA = CopyArrayTemps.begin();
3577 for (
const Expr *IRef : Privates) {
3578 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
3581 if (PrivateVD->getType()->isVariablyModifiedType()) {
3585 CodeGenFunction::OpaqueValueMapping DimMapping(
3587 cast<OpaqueValueExpr>(
3588 cast<VariableArrayType>((*ITA)->getType()->getAsArrayTypeUnsafe())
3592 CGF.
EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(*ITA)->getDecl()));
3606 llvm::function_ref<llvm::Value *(
CodeGenFunction &)> NumIteratorsGen) {
3607 llvm::Value *OMPScanNumIterations = CGF.
Builder.CreateIntCast(
3608 NumIteratorsGen(CGF), CGF.
SizeTy,
false);
3616 assert(
C->getModifier() == OMPC_REDUCTION_inscan &&
3617 "Only inscan reductions are expected.");
3618 Shareds.append(
C->varlist_begin(),
C->varlist_end());
3619 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
3620 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
3621 Privates.append(
C->privates().begin(),
C->privates().end());
3622 CopyOps.append(
C->copy_ops().begin(),
C->copy_ops().end());
3623 CopyArrayElems.append(
C->copy_array_elems().begin(),
3624 C->copy_array_elems().end());
3628 llvm::Value *OMPLast = CGF.
Builder.CreateNSWSub(
3629 OMPScanNumIterations,
3630 llvm::ConstantInt::get(CGF.
SizeTy, 1,
false));
3631 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
3632 const Expr *PrivateExpr = Privates[I];
3633 const Expr *OrigExpr = Shareds[I];
3634 const Expr *CopyArrayElem = CopyArrayElems[I];
3635 CodeGenFunction::OpaqueValueMapping IdxMapping(
3637 cast<OpaqueValueExpr>(
3638 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3644 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
3645 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
3674 llvm::Value *OMPScanNumIterations = CGF.
Builder.CreateIntCast(
3675 NumIteratorsGen(CGF), CGF.
SizeTy,
false);
3682 assert(
C->getModifier() == OMPC_REDUCTION_inscan &&
3683 "Only inscan reductions are expected.");
3684 Privates.append(
C->privates().begin(),
C->privates().end());
3685 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
3686 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
3687 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
3688 CopyArrayElems.append(
C->copy_array_elems().begin(),
3689 C->copy_array_elems().end());
3691 CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S);
3700 CodeGenFunction::OMPLocalDeclMapRAII
Scope(CGF);
3704 auto &&CodeGen = [&S, OMPScanNumIterations, &LHSs, &RHSs, &CopyArrayElems,
3711 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock();
3712 llvm::BasicBlock *LoopBB = CGF.createBasicBlock(
"omp.outer.log.scan.body");
3713 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
"omp.outer.log.scan.exit");
3715 CGF.CGM.getIntrinsic(llvm::Intrinsic::log2, CGF.DoubleTy);
3717 CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy);
3718 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg);
3719 F = CGF.CGM.getIntrinsic(llvm::Intrinsic::ceil, CGF.DoubleTy);
3720 LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal);
3721 LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy);
3722 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub(
3723 OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1));
3725 CGF.EmitBlock(LoopBB);
3726 auto *Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2);
3728 auto *Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3729 Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB);
3730 Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB);
3733 llvm::BasicBlock *InnerLoopBB =
3734 CGF.createBasicBlock(
"omp.inner.log.scan.body");
3735 llvm::BasicBlock *InnerExitBB =
3736 CGF.createBasicBlock(
"omp.inner.log.scan.exit");
3737 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K);
3738 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3739 CGF.EmitBlock(InnerLoopBB);
3740 auto *IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3741 IVal->addIncoming(NMin1, LoopBB);
3743 CodeGenFunction::OMPPrivateScope PrivScope(CGF);
3744 auto *ILHS = LHSs.begin();
3745 auto *IRHS = RHSs.begin();
3746 for (
const Expr *CopyArrayElem : CopyArrayElems) {
3747 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
3748 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
3751 CodeGenFunction::OpaqueValueMapping IdxMapping(
3753 cast<OpaqueValueExpr>(
3754 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3756 LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF);
3758 PrivScope.addPrivate(LHSVD, LHSAddr);
3761 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K);
3762 CodeGenFunction::OpaqueValueMapping IdxMapping(
3764 cast<OpaqueValueExpr>(
3765 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3767 RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF);
3769 PrivScope.addPrivate(RHSVD, RHSAddr);
3773 PrivScope.Privatize();
3774 CGF.CGM.getOpenMPRuntime().emitReduction(
3775 CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
3776 {true, true, OMPD_unknown});
3778 llvm::Value *NextIVal =
3779 CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1));
3780 IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock());
3781 CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K);
3782 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3783 CGF.EmitBlock(InnerExitBB);
3785 CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1));
3786 Counter->addIncoming(Next, CGF.Builder.GetInsertBlock());
3788 llvm::Value *NextPow2K =
3789 CGF.Builder.CreateShl(Pow2K, 1,
"",
true);
3790 Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock());
3791 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(Next, LogVal);
3792 CGF.Builder.CreateCondBr(Cmp, LoopBB, ExitBB);
3794 CGF.EmitBlock(ExitBB);
3797 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());
3798 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
3799 CGF, S.getBeginLoc(), OMPD_unknown,
false,
3806 CGF.OMPFirstScanLoop =
false;
3813 bool HasLastprivates;
3816 return C->getModifier() == OMPC_REDUCTION_inscan;
3819 CodeGenFunction::OMPLocalDeclMapRAII
Scope(CGF);
3820 OMPLoopScope LoopScope(CGF, S);
3824 CodeGenFunction::OMPCancelStackRAII CancelRegion(
3825 CGF, S.getDirectiveKind(), HasCancel);
3833 const auto &&SecondGen = [&S, HasCancel,
3835 CodeGenFunction::OMPCancelStackRAII CancelRegion(
3836 CGF, S.getDirectiveKind(), HasCancel);
3847 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
3853 return HasLastprivates;
3860 if (isa<OMPNowaitClause>(
C))
3863 if (
auto *SC = dyn_cast<OMPScheduleClause>(
C)) {
3868 switch (SC->getScheduleKind()) {
3869 case OMPC_SCHEDULE_auto:
3870 case OMPC_SCHEDULE_dynamic:
3871 case OMPC_SCHEDULE_runtime:
3872 case OMPC_SCHEDULE_guided:
3873 case OMPC_SCHEDULE_static:
3886static llvm::omp::ScheduleKind
3888 switch (ScheduleClauseKind) {
3890 return llvm::omp::OMP_SCHEDULE_Default;
3891 case OMPC_SCHEDULE_auto:
3892 return llvm::omp::OMP_SCHEDULE_Auto;
3893 case OMPC_SCHEDULE_dynamic:
3894 return llvm::omp::OMP_SCHEDULE_Dynamic;
3895 case OMPC_SCHEDULE_guided:
3896 return llvm::omp::OMP_SCHEDULE_Guided;
3897 case OMPC_SCHEDULE_runtime:
3898 return llvm::omp::OMP_SCHEDULE_Runtime;
3899 case OMPC_SCHEDULE_static:
3900 return llvm::omp::OMP_SCHEDULE_Static;
3902 llvm_unreachable(
"Unhandled schedule kind");
3906 bool HasLastprivates =
false;
3907 bool UseOMPIRBuilder =
3909 auto &&CodeGen = [
this, &S, &HasLastprivates,
3912 if (UseOMPIRBuilder) {
3915 llvm::omp::ScheduleKind SchedKind = llvm::omp::OMP_SCHEDULE_Default;
3916 llvm::Value *ChunkSize =
nullptr;
3920 if (
const Expr *ChunkSizeExpr = SchedClause->getChunkSize())
3925 const Stmt *Inner = S.getRawStmt();
3926 llvm::CanonicalLoopInfo *CLI =
3929 llvm::OpenMPIRBuilder &OMPBuilder =
3931 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
3933 OMPBuilder.applyWorkshareLoop(
3934 Builder.getCurrentDebugLocation(), CLI, AllocaIP, NeedsBarrier,
3935 SchedKind, ChunkSize,
false,
3946 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
3951 if (!UseOMPIRBuilder) {
3961 bool HasLastprivates =
false;
3969 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
3982 llvm::Value *
Init =
nullptr) {
3990 const Stmt *
CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
3992 bool HasLastprivates =
false;
3997 C.getIntTypeForBitwidth(32, 1);
4000 CGF.Builder.getInt32(0));
4001 llvm::ConstantInt *GlobalUBVal = CS !=
nullptr
4002 ? CGF.Builder.getInt32(CS->size() - 1)
4003 : CGF.Builder.getInt32(0);
4007 CGF.Builder.getInt32(1));
4009 CGF.Builder.getInt32(0));
4013 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
4015 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
4036 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
".omp.sections.exit");
4038 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
4039 ExitBB, CS ==
nullptr ? 1 : CS->size());
4041 unsigned CaseNumber = 0;
4043 auto CaseBB = CGF.createBasicBlock(
".omp.sections.case");
4044 CGF.EmitBlock(CaseBB);
4045 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
4046 CGF.EmitStmt(SubStmt);
4047 CGF.EmitBranch(ExitBB);
4051 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(
".omp.sections.case");
4052 CGF.EmitBlock(CaseBB);
4053 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
4055 CGF.EmitBranch(ExitBB);
4057 CGF.EmitBlock(ExitBB,
true);
4060 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
4061 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
4065 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
4066 CGF, S.getBeginLoc(), OMPD_unknown,
false,
4069 CGF.EmitOMPPrivateClause(S, LoopScope);
4071 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4072 CGF.EmitOMPReductionClauseInit(S, LoopScope);
4073 (void)LoopScope.Privatize();
4075 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4079 ScheduleKind.
Schedule = OMPC_SCHEDULE_static;
4083 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
4084 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit);
4086 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
4087 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
4088 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
4089 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
4091 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
4093 CGF.EmitOMPInnerLoop(S,
false, Cond, Inc, BodyGen,
4097 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
4100 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
4101 CGF.EmitOMPReductionClauseFinal(S, OMPD_parallel);
4104 return CGF.
Builder.CreateIsNotNull(
4109 if (HasLastprivates)
4116 bool HasCancel =
false;
4117 if (
auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
4118 HasCancel = OSD->hasCancel();
4119 else if (
auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
4120 HasCancel = OPSD->hasCancel();
4121 OMPCancelStackRAII CancelRegion(*
this, S.getDirectiveKind(), HasCancel);
4138 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4139 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4141 auto FiniCB = [
this](InsertPointTy IP) {
4145 const CapturedStmt *ICS = S.getInnermostCapturedStmt();
4146 const Stmt *
CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4151 auto SectionCB = [
this, SubStmt](InsertPointTy AllocaIP,
4152 InsertPointTy CodeGenIP) {
4154 *
this, SubStmt, AllocaIP, CodeGenIP,
"section");
4156 SectionCBVector.push_back(SectionCB);
4159 auto SectionCB = [
this,
CapturedStmt](InsertPointTy AllocaIP,
4160 InsertPointTy CodeGenIP) {
4164 SectionCBVector.push_back(SectionCB);
4171 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
4172 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
4180 CGCapturedStmtInfo CGSI(*ICS,
CR_OpenMP);
4181 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*
this, &CGSI);
4182 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
4184 Builder.restoreIP(OMPBuilder.createSections(
4185 Builder, AllocaIP, SectionCBVector, PrivCB, FiniCB, S.hasCancel(),
4192 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
4207 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4209 const Stmt *SectionRegionBodyStmt = S.getAssociatedStmt();
4210 auto FiniCB = [
this](InsertPointTy IP) {
4214 auto BodyGenCB = [SectionRegionBodyStmt,
this](InsertPointTy AllocaIP,
4215 InsertPointTy CodeGenIP) {
4217 *
this, SectionRegionBodyStmt, AllocaIP, CodeGenIP,
"section");
4220 LexicalScope
Scope(*
this, S.getSourceRange());
4222 Builder.restoreIP(OMPBuilder.createSection(
Builder, BodyGenCB, FiniCB));
4226 LexicalScope
Scope(*
this, S.getSourceRange());
4241 CopyprivateVars.append(
C->varlists().begin(),
C->varlists().end());
4242 DestExprs.append(
C->destination_exprs().begin(),
4243 C->destination_exprs().end());
4244 SrcExprs.append(
C->source_exprs().begin(),
C->source_exprs().end());
4245 AssignmentOps.append(
C->assignment_ops().begin(),
4246 C->assignment_ops().end());
4255 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4260 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
4262 CopyprivateVars, DestExprs,
4263 SrcExprs, AssignmentOps);
4267 if (!S.getSingleClause<
OMPNowaitClause>() && CopyprivateVars.empty()) {
4269 *
this, S.getBeginLoc(),
4287 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4289 const Stmt *MasterRegionBodyStmt = S.getAssociatedStmt();
4291 auto FiniCB = [
this](InsertPointTy IP) {
4295 auto BodyGenCB = [MasterRegionBodyStmt,
this](InsertPointTy AllocaIP,
4296 InsertPointTy CodeGenIP) {
4298 *
this, MasterRegionBodyStmt, AllocaIP, CodeGenIP,
"master");
4301 LexicalScope
Scope(*
this, S.getSourceRange());
4303 Builder.restoreIP(OMPBuilder.createMaster(
Builder, BodyGenCB, FiniCB));
4307 LexicalScope
Scope(*
this, S.getSourceRange());
4317 Expr *Filter =
nullptr;
4319 Filter = FilterClause->getThreadID();
4327 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4329 const Stmt *MaskedRegionBodyStmt = S.getAssociatedStmt();
4332 Filter = FilterClause->getThreadID();
4333 llvm::Value *FilterVal =
Filter
4337 auto FiniCB = [
this](InsertPointTy IP) {
4341 auto BodyGenCB = [MaskedRegionBodyStmt,
this](InsertPointTy AllocaIP,
4342 InsertPointTy CodeGenIP) {
4344 *
this, MaskedRegionBodyStmt, AllocaIP, CodeGenIP,
"masked");
4347 LexicalScope
Scope(*
this, S.getSourceRange());
4350 OMPBuilder.createMasked(
Builder, BodyGenCB, FiniCB, FilterVal));
4354 LexicalScope
Scope(*
this, S.getSourceRange());
4362 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4364 const Stmt *CriticalRegionBodyStmt = S.getAssociatedStmt();
4365 const Expr *Hint =
nullptr;
4366 if (
const auto *HintClause = S.getSingleClause<
OMPHintClause>())
4367 Hint = HintClause->getHint();
4372 llvm::Value *HintInst =
nullptr;
4377 auto FiniCB = [
this](InsertPointTy IP) {
4381 auto BodyGenCB = [CriticalRegionBodyStmt,
this](InsertPointTy AllocaIP,
4382 InsertPointTy CodeGenIP) {
4384 *
this, CriticalRegionBodyStmt, AllocaIP, CodeGenIP,
"critical");
4387 LexicalScope
Scope(*
this, S.getSourceRange());
4389 Builder.restoreIP(OMPBuilder.createCritical(
4390 Builder, BodyGenCB, FiniCB, S.getDirectiveName().getAsString(),
4398 CGF.
EmitStmt(S.getAssociatedStmt());
4400 const Expr *Hint =
nullptr;
4401 if (
const auto *HintClause = S.getSingleClause<
OMPHintClause>())
4402 Hint = HintClause->getHint();
4403 LexicalScope
Scope(*
this, S.getSourceRange());
4406 S.getDirectiveName().getAsString(),
4407 CodeGen, S.getBeginLoc(), Hint);
4421 CodeGenFunction::OMPLocalDeclMapRAII
Scope(CGF);
4423 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
4424 OMPLoopScope LoopScope(CGF, S);
4429 return C->getModifier() == OMPC_REDUCTION_inscan;
4455 CodeGenFunction::OMPLocalDeclMapRAII
Scope(CGF);
4457 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
4458 OMPLoopScope LoopScope(CGF, S);
4463 return C->getModifier() == OMPC_REDUCTION_inscan;
4484 OMPPrivateScope PrivateScope(CGF);
4489 (void)PrivateScope.Privatize();
4511 OMPPrivateScope PrivateScope(CGF);
4516 (void)PrivateScope.Privatize();
4539 CGF.EmitSections(S);
4553class CheckVarsEscapingUntiedTaskDeclContext final
4558 explicit CheckVarsEscapingUntiedTaskDeclContext() =
default;
4559 virtual ~CheckVarsEscapingUntiedTaskDeclContext() =
default;
4560 void VisitDeclStmt(
const DeclStmt *S) {
4564 for (
const Decl *D : S->decls()) {
4565 if (
const auto *VD = dyn_cast_or_null<VarDecl>(D))
4567 PrivateDecls.push_back(VD);
4573 void VisitBlockExpr(
const BlockExpr *) {}
4574 void VisitStmt(
const Stmt *S) {
4577 for (
const Stmt *Child : S->children())
4591 bool OmpAllMemory =
false;
4594 return C->getDependencyKind() == OMPC_DEPEND_outallmemory ||
4595 C->getDependencyKind() == OMPC_DEPEND_inoutallmemory;
4597 OmpAllMemory =
true;
4602 Data.Dependences.emplace_back(OMPC_DEPEND_outallmemory,
4611 if (Kind == OMPC_DEPEND_outallmemory || Kind == OMPC_DEPEND_inoutallmemory)
4613 if (OmpAllMemory && (Kind == OMPC_DEPEND_out || Kind == OMPC_DEPEND_inout))
4616 Data.Dependences.emplace_back(
C->getDependencyKind(),
C->getModifier());
4617 DD.
DepExprs.append(
C->varlist_begin(),
C->varlist_end());
4626 const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
4628 auto PartId = std::next(I);
4629 auto TaskT = std::next(I, 4);
4634 const Expr *Cond = Clause->getCondition();
4637 Data.Final.setInt(CondConstant);
4642 Data.Final.setInt(
false);
4646 const Expr *Prio = Clause->getPriority();
4647 Data.Priority.setInt(
true);
4658 auto IRef =
C->varlist_begin();
4659 for (
const Expr *IInit :
C->private_copies()) {
4660 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4661 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4662 Data.PrivateVars.push_back(*IRef);
4663 Data.PrivateCopies.push_back(IInit);
4668 EmittedAsPrivate.clear();
4671 auto IRef =
C->varlist_begin();
4672 auto IElemInitRef =
C->inits().begin();
4673 for (
const Expr *IInit :
C->private_copies()) {
4674 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4675 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4676 Data.FirstprivateVars.push_back(*IRef);
4677 Data.FirstprivateCopies.push_back(IInit);
4678 Data.FirstprivateInits.push_back(*IElemInitRef);
4685 llvm::MapVector<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
4687 auto IRef =
C->varlist_begin();
4688 auto ID =
C->destination_exprs().begin();
4689 for (
const Expr *IInit :
C->private_copies()) {
4690 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4691 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4692 Data.LastprivateVars.push_back(*IRef);
4693 Data.LastprivateCopies.push_back(IInit);
4695 LastprivateDstsOrigs.insert(
4696 std::make_pair(cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
4697 cast<DeclRefExpr>(*IRef)));
4705 Data.ReductionVars.append(
C->varlist_begin(),
C->varlist_end());
4706 Data.ReductionOrigs.append(
C->varlist_begin(),
C->varlist_end());
4707 Data.ReductionCopies.append(
C->privates().begin(),
C->privates().end());
4708 Data.ReductionOps.append(
C->reduction_ops().begin(),
4709 C->reduction_ops().end());
4710 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
4711 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
4714 *
this, S.getBeginLoc(), LHSs, RHSs,
Data);
4719 CheckVarsEscapingUntiedTaskDeclContext Checker;
4720 Checker.Visit(S.getInnermostCapturedStmt()->getCapturedStmt());
4721 Data.PrivateLocals.append(Checker.getPrivateDecls().begin(),
4722 Checker.getPrivateDecls().end());
4724 auto &&CodeGen = [&
Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
4727 llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
4728 std::pair<Address, Address>>
4731 OMPPrivateScope
Scope(CGF);
4733 if (
auto *DI = CGF.getDebugInfo()) {
4734 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields =
4735 CGF.CapturedStmtInfo->getCaptureFields();
4736 llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue();
4737 if (CaptureFields.size() && ContextValue) {
4738 unsigned CharWidth = CGF.getContext().getCharWidth();
4752 for (
auto It = CaptureFields.begin(); It != CaptureFields.end(); ++It) {
4753 const VarDecl *SharedVar = It->first;
4756 CGF.getContext().getASTRecordLayout(CaptureRecord);
4759 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
4760 (void)DI->EmitDeclareOfAutoVariable(SharedVar, ContextValue,
4761 CGF.Builder,
false);
4764 auto UpdateExpr = [](llvm::LLVMContext &Ctx,
auto *
Declare,
4769 Ops.push_back(llvm::dwarf::DW_OP_plus_uconst);
4770 Ops.push_back(Offset);
4772 Ops.push_back(llvm::dwarf::DW_OP_deref);
4773 Declare->setExpression(llvm::DIExpression::get(Ctx, Ops));
4775 llvm::Instruction &
Last = CGF.Builder.GetInsertBlock()->back();
4776 if (
auto DDI = dyn_cast<llvm::DbgVariableIntrinsic>(&
Last))
4777 UpdateExpr(DDI->getContext(), DDI, Offset);
4780 assert(!
Last.isTerminator() &&
"unexpected terminator");
4782 CGF.Builder.GetInsertBlock()->getTrailingDbgRecords()) {
4783 for (llvm::DbgVariableRecord &DVR : llvm::reverse(
4784 llvm::filterDbgVars(Marker->getDbgRecordRange()))) {
4785 UpdateExpr(
Last.getContext(), &DVR, Offset);
4793 if (!
Data.PrivateVars.empty() || !
Data.FirstprivateVars.empty() ||
4794 !
Data.LastprivateVars.empty() || !
Data.PrivateLocals.empty()) {
4795 enum { PrivatesParam = 2, CopyFnParam = 3 };
4796 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
4797 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
4798 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
4799 CS->getCapturedDecl()->getParam(PrivatesParam)));
4804 CallArgs.push_back(PrivatesPtr);
4805 ParamTypes.push_back(PrivatesPtr->getType());
4806 for (
const Expr *E :
Data.PrivateVars) {
4807 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4809 CGF.getContext().getPointerType(E->
getType()),
".priv.ptr.addr");
4810 PrivatePtrs.emplace_back(VD, PrivatePtr);
4812 ParamTypes.push_back(PrivatePtr.
getType());
4814 for (
const Expr *E :
Data.FirstprivateVars) {
4815 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4817 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->
getType()),
4818 ".firstpriv.ptr.addr");
4819 PrivatePtrs.emplace_back(VD, PrivatePtr);
4820 FirstprivatePtrs.emplace_back(VD, PrivatePtr);
4822 ParamTypes.push_back(PrivatePtr.
getType());
4824 for (
const Expr *E :
Data.LastprivateVars) {
4825 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4827 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->
getType()),
4828 ".lastpriv.ptr.addr");
4829 PrivatePtrs.emplace_back(VD, PrivatePtr);
4831 ParamTypes.push_back(PrivatePtr.
getType());
4836 Ty = CGF.getContext().getPointerType(Ty);
4838 Ty = CGF.getContext().getPointerType(Ty);
4840 CGF.getContext().getPointerType(Ty),
".local.ptr.addr");
4841 auto Result = UntiedLocalVars.insert(
4844 if (
Result.second ==
false)
4845 *
Result.first = std::make_pair(
4848 ParamTypes.push_back(PrivatePtr.
getType());
4850 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
4852 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
4853 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
4854 for (
const auto &Pair : LastprivateDstsOrigs) {
4855 const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
4858 CGF.CapturedStmtInfo->lookup(OrigVD) !=
nullptr,
4860 Pair.second->getExprLoc());
4861 Scope.addPrivate(Pair.first, CGF.EmitLValue(&DRE).getAddress(CGF));
4863 for (
const auto &Pair : PrivatePtrs) {
4865 CGF.Builder.CreateLoad(Pair.second),
4866 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
4867 CGF.getContext().getDeclAlign(Pair.first));
4868 Scope.addPrivate(Pair.first, Replacement);
4869 if (
auto *DI = CGF.getDebugInfo())
4870 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
4871 (void)DI->EmitDeclareOfAutoVariable(
4872 Pair.first, Pair.second.getBasePointer(), CGF.Builder,
4877 for (
auto &Pair : UntiedLocalVars) {
4878 QualType VDType = Pair.first->getType().getNonReferenceType();
4879 if (Pair.first->getType()->isLValueReferenceType())
4880 VDType = CGF.getContext().getPointerType(VDType);
4882 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
4885 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(VDType)),
4886 CGF.getPointerAlign());
4887 Pair.second.first = Replacement;
4888 Ptr = CGF.Builder.CreateLoad(Replacement);
4889 Replacement =
Address(Ptr, CGF.ConvertTypeForMem(VDType),
4890 CGF.getContext().getDeclAlign(Pair.first));
4891 Pair.second.second = Replacement;
4893 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
4894 Address Replacement(Ptr, CGF.ConvertTypeForMem(VDType),
4895 CGF.getContext().getDeclAlign(Pair.first));
4896 Pair.second.first = Replacement;
4900 if (
Data.Reductions) {
4901 OMPPrivateScope FirstprivateScope(CGF);
4902 for (
const auto &Pair : FirstprivatePtrs) {
4904 CGF.Builder.CreateLoad(Pair.second),
4905 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
4906 CGF.getContext().getDeclAlign(Pair.first));
4907 FirstprivateScope.addPrivate(Pair.first, Replacement);
4909 (void)FirstprivateScope.Privatize();
4910 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
4912 Data.ReductionCopies,
Data.ReductionOps);
4913 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
4914 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
4915 for (
unsigned Cnt = 0, E =
Data.ReductionVars.size(); Cnt < E; ++Cnt) {
4916 RedCG.emitSharedOrigLValue(CGF, Cnt);
4917 RedCG.emitAggregateType(CGF, Cnt);
4921 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
4923 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
4924 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
4926 CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF),
4927 CGF.getContext().VoidPtrTy,
4928 CGF.getContext().getPointerType(
4929 Data.ReductionCopies[Cnt]->getType()),
4930 Data.ReductionCopies[Cnt]->getExprLoc()),
4931 CGF.ConvertTypeForMem(
Data.ReductionCopies[Cnt]->getType()),
4932 Replacement.getAlignment());
4933 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
4934 Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
4938 (void)
Scope.Privatize();
4944 auto IPriv =
C->privates().begin();
4945 auto IRed =
C->reduction_ops().begin();
4946 auto ITD =
C->taskgroup_descriptors().begin();
4947 for (
const Expr *Ref :
C->varlists()) {
4948 InRedVars.emplace_back(Ref);
4949 InRedPrivs.emplace_back(*IPriv);
4950 InRedOps.emplace_back(*IRed);
4951 TaskgroupDescriptors.emplace_back(*ITD);
4952 std::advance(IPriv, 1);
4953 std::advance(IRed, 1);
4954 std::advance(ITD, 1);
4959 OMPPrivateScope InRedScope(CGF);
4960 if (!InRedVars.empty()) {
4962 for (
unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
4963 RedCG.emitSharedOrigLValue(CGF, Cnt);
4964 RedCG.emitAggregateType(CGF, Cnt);
4970 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
4972 llvm::Value *ReductionsPtr;
4973 if (
const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
4974 ReductionsPtr = CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr),
4975 TRExpr->getExprLoc());
4977 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4979 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
4980 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
4982 CGF.EmitScalarConversion(
4983 Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy,
4984 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
4985 InRedPrivs[Cnt]->getExprLoc()),
4986 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),
4987 Replacement.getAlignment());
4988 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
4989 InRedScope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
4992 (void)InRedScope.Privatize();
5000 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen,
Data.Tied,
5001 Data.NumberOfParts);
5002 OMPLexicalScope
Scope(*
this, S, std::nullopt,
5005 TaskGen(*
this, OutlinedFn,
Data);
5022 QualType ElemType =
C.getBaseElementType(Ty);
5032 Data.FirstprivateVars.emplace_back(OrigRef);
5033 Data.FirstprivateCopies.emplace_back(PrivateRef);
5034 Data.FirstprivateInits.emplace_back(InitRef);
5040 OMPTargetDataInfo &InputInfo) {
5046 auto PartId = std::next(I);
5047 auto TaskT = std::next(I, 4);
5050 Data.Final.setInt(
false);
5053 auto IRef =
C->varlist_begin();
5054 auto IElemInitRef =
C->inits().begin();
5055 for (
auto *IInit :
C->private_copies()) {
5056 Data.FirstprivateVars.push_back(*IRef);
5057 Data.FirstprivateCopies.push_back(IInit);
5058 Data.FirstprivateInits.push_back(*IElemInitRef);
5066 Data.ReductionVars.append(
C->varlist_begin(),
C->varlist_end());
5067 Data.ReductionOrigs.append(
C->varlist_begin(),
C->varlist_end());
5068 Data.ReductionCopies.append(
C->privates().begin(),
C->privates().end());
5069 Data.ReductionOps.append(
C->reduction_ops().begin(),
5070 C->reduction_ops().end());
5071 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
5072 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
5074 OMPPrivateScope TargetScope(*
this);
5079 if (InputInfo.NumberOfTargetItems > 0) {
5082 llvm::APInt ArrSize(32, InputInfo.NumberOfTargetItems);
5087 getContext(),
Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5089 getContext(),
Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5096 TargetScope.addPrivate(BPVD, InputInfo.BasePointersArray);
5097 TargetScope.addPrivate(PVD, InputInfo.PointersArray);
5098 TargetScope.addPrivate(SVD, InputInfo.SizesArray);
5101 if (!isa_and_nonnull<llvm::ConstantPointerNull>(
5102 InputInfo.MappersArray.emitRawPointer(*
this))) {
5104 getContext(),
Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5105 TargetScope.addPrivate(MVD, InputInfo.MappersArray);
5108 (void)TargetScope.Privatize();
5110 auto &&CodeGen = [&
Data, &S, CS, &BodyGen, BPVD, PVD, SVD, MVD,
5113 OMPPrivateScope
Scope(CGF);
5114 if (!
Data.FirstprivateVars.empty()) {
5115 enum { PrivatesParam = 2, CopyFnParam = 3 };
5116 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
5117 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
5118 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
5119 CS->getCapturedDecl()->getParam(PrivatesParam)));
5124 CallArgs.push_back(PrivatesPtr);
5125 ParamTypes.push_back(PrivatesPtr->getType());
5126 for (
const Expr *E :
Data.FirstprivateVars) {
5127 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5129 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->
getType()),
5130 ".firstpriv.ptr.addr");
5131 PrivatePtrs.emplace_back(VD, PrivatePtr);
5133 ParamTypes.push_back(PrivatePtr.
getType());
5135 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
5137 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
5138 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
5139 for (
const auto &Pair : PrivatePtrs) {
5141 CGF.Builder.CreateLoad(Pair.second),
5142 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
5143 CGF.getContext().getDeclAlign(Pair.first));
5144 Scope.addPrivate(Pair.first, Replacement);
5147 CGF.processInReduction(S,
Data, CGF, CS,
Scope);
5148 if (InputInfo.NumberOfTargetItems > 0) {
5149 InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
5150 CGF.GetAddrOfLocalVar(BPVD), 0);
5151 InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
5152 CGF.GetAddrOfLocalVar(PVD), 0);
5153 InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
5154 CGF.GetAddrOfLocalVar(SVD), 0);
5157 InputInfo.MappersArray = CGF.Builder.CreateConstArrayGEP(
5158 CGF.GetAddrOfLocalVar(MVD), 0);
5162 OMPLexicalScope LexScope(CGF, S, OMPD_task,
false);
5164 if (CGF.CGM.getLangOpts().OpenMP >= 51 &&
5169 CGF.CGM.getOpenMPRuntime().emitThreadLimitClause(
5170 CGF, TL->getThreadLimit(), S.getBeginLoc());
5175 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen,
true,
5176 Data.NumberOfParts);
5177 llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<
OMPNowaitClause>() ? 1 : 0);
5182 SharedsTy, CapturedStruct, &IfCond,
Data);
5189 OMPPrivateScope &
Scope) {
5190 if (
Data.Reductions) {
5192 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
5194 Data.ReductionCopies,
Data.ReductionOps);
5197 for (
unsigned Cnt = 0, E =
Data.ReductionVars.size(); Cnt < E; ++Cnt) {
5198 RedCG.emitSharedOrigLValue(CGF, Cnt);
5199 RedCG.emitAggregateType(CGF, Cnt);
5206 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
5211 Data.ReductionCopies[Cnt]->getType()),
5212 Data.ReductionCopies[Cnt]->getExprLoc()),
5214 Replacement.getAlignment());
5215 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
5216 Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
5219 (void)
Scope.Privatize();
5225 auto IPriv =
C->privates().begin();
5226 auto IRed =
C->reduction_ops().begin();
5227 auto ITD =
C->taskgroup_descriptors().begin();
5228 for (
const Expr *Ref :
C->varlists()) {
5229 InRedVars.emplace_back(Ref);
5230 InRedPrivs.emplace_back(*IPriv);
5231 InRedOps.emplace_back(*IRed);
5232 TaskgroupDescriptors.emplace_back(*ITD);
5233 std::advance(IPriv, 1);
5234 std::advance(IRed, 1);
5235 std::advance(ITD, 1);
5238 OMPPrivateScope InRedScope(CGF);
5239 if (!InRedVars.empty()) {
5241 for (
unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
5242 RedCG.emitSharedOrigLValue(CGF, Cnt);
5243 RedCG.emitAggregateType(CGF, Cnt);
5249 llvm::Value *ReductionsPtr;
5250 if (
const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
5254 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
5257 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
5262 InRedPrivs[Cnt]->getExprLoc()),
5264 Replacement.getAlignment());
5265 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
5266 InRedScope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
5269 (void)InRedScope.Privatize();
5277 const Expr *IfCond =
nullptr;
5278 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
5279 if (
C->getNameModifier() == OMPD_unknown ||
5280 C->getNameModifier() == OMPD_task) {
5281 IfCond =
C->getCondition();
5292 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
5295 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn,
5296 SharedsTy, CapturedStruct, IfCond,
5313 bool IsFatal =
false;
5332 return T.clauses().empty();
5337 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
5340 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5344 auto BodyGenCB = [&,
this](InsertPointTy AllocaIP,
5345 InsertPointTy CodeGenIP) {
5347 EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
5349 CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;
5352 Builder.restoreIP(OMPBuilder.createTaskgroup(
Builder, AllocaIP, BodyGenCB));
5357 if (
const Expr *E = S.getReductionRef()) {
5362 Data.ReductionVars.append(
C->varlist_begin(),
C->varlist_end());
5363 Data.ReductionOrigs.append(
C->varlist_begin(),
C->varlist_end());
5364 Data.ReductionCopies.append(
C->privates().begin(),
C->privates().end());
5365 Data.ReductionOps.append(
C->reduction_ops().begin(),
5366 C->reduction_ops().end());
5367 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
5368 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
5370 llvm::Value *ReductionDesc =
5373 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5378 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
5385 ? llvm::AtomicOrdering::NotAtomic
5386 : llvm::AtomicOrdering::AcquireRelease;
5390 if (
const auto *FlushClause = S.getSingleClause<
OMPFlushClause>())
5392 FlushClause->varlist_end());
5393 return std::nullopt;
5395 S.getBeginLoc(), AO);
5404 Dependencies.DepExprs.append(DC->varlist_begin(), DC->varlist_end());
5406 *
this, Dependencies, DC->getBeginLoc());
5416 *
this, DOLVal, UC->getDependencyKind(), UC->getBeginLoc());
5435 if (
C->getModifier() != OMPC_REDUCTION_inscan)
5437 Shareds.append(
C->varlist_begin(),
C->varlist_end());
5438 Privates.append(
C->privates().begin(),
C->privates().end());
5439 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
5440 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
5441 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
5442 CopyOps.append(
C->copy_ops().begin(),
C->copy_ops().end());
5443 CopyArrayTemps.append(
C->copy_array_temps().begin(),
5444 C->copy_array_temps().end());
5445 CopyArrayElems.append(
C->copy_array_elems().begin(),
5446 C->copy_array_elems().end());
5490 : BreakContinueStack.back().ContinueBlock.getBlock());
5495 LexicalScope
Scope(*
this, S.getSourceRange());
5501 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5502 const Expr *PrivateExpr = Privates[I];
5503 const Expr *TempExpr = CopyArrayTemps[I];
5505 *cast<VarDecl>(cast<DeclRefExpr>(TempExpr)->getDecl()));
5510 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5511 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
5516 *
this, ParentDir.
getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
5517 {true, true, OMPD_simd});
5518 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5519 const Expr *PrivateExpr = Privates[I];
5526 const Expr *TempExpr = CopyArrayTemps[I];
5532 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5533 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
5539 ? BreakContinueStack.back().ContinueBlock.getBlock()
5545 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
5550 const auto *IVExpr = cast<OMPLoopDirective>(ParentDir)
5551 .getIterationVariable()
5556 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5557 const Expr *PrivateExpr = Privates[I];
5558 const Expr *OrigExpr = Shareds[I];
5559 const Expr *CopyArrayElem = CopyArrayElems[I];
5560 OpaqueValueMapping IdxMapping(
5562 cast<OpaqueValueExpr>(
5563 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
5569 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5570 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
5574 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
5577 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
5582 const auto *IVExpr = cast<OMPLoopDirective>(ParentDir)
5583 .getIterationVariable()
5588 llvm::BasicBlock *ExclusiveExitBB =
nullptr;
5593 Builder.CreateCondBr(Cmp, ExclusiveExitBB, ContBB);
5596 IdxVal =
Builder.CreateNUWSub(IdxVal, llvm::ConstantInt::get(
SizeTy, 1));
5598 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5599 const Expr *PrivateExpr = Privates[I];
5600 const Expr *OrigExpr = Shareds[I];
5601 const Expr *CopyArrayElem = CopyArrayElems[I];
5602 OpaqueValueMapping IdxMapping(
5604 cast<OpaqueValueExpr>(
5605 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
5611 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5612 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
5628 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
5629 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
5635 if (
const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
5643 bool HasLastprivateClause =
false;
5646 OMPLoopScope PreInitScope(*
this, S);
5651 llvm::BasicBlock *ContBlock =
nullptr;
5658 emitPreCond(*
this, S, S.getPreCond(), ThenBlock, ContBlock,
5670 *
this, cast<DeclRefExpr>(
5672 ? S.getCombinedLowerBoundVariable()
5673 : S.getLowerBoundVariable())));
5675 *
this, cast<DeclRefExpr>(
5677 ? S.getCombinedUpperBoundVariable()
5678 : S.getUpperBoundVariable())));
5684 OMPPrivateScope LoopScope(*
this);
5690 *
this, S.getBeginLoc(), OMPD_unknown,
false,
5700 (void)LoopScope.Privatize();
5705 llvm::Value *Chunk =
nullptr;
5708 ScheduleKind =
C->getDistScheduleKind();
5709 if (
const Expr *Ch =
C->getChunkSize()) {
5712 S.getIterationVariable()->getType(),
5718 *
this, S, ScheduleKind, Chunk);
5731 bool StaticChunked =
5735 Chunk !=
nullptr) ||
5738 IVSize, IVSigned,
false, IL.
getAddress(*
this),
5740 StaticChunked ? Chunk :
nullptr);
5747 ? S.getCombinedEnsureUpperBound()
5748 : S.getEnsureUpperBound());
5751 ? S.getCombinedInit()
5756 ? S.getCombinedCond()
5760 Cond = S.getCombinedDistCond();
5792 [&S, &LoopScope, Cond, IncExpr,
LoopExit, &CodeGenLoop,
5794 CGF.EmitOMPInnerLoop(
5795 S, LoopScope.requiresCleanups(), Cond, IncExpr,
5797 CodeGenLoop(CGF, S, LoopExit);
5800 if (StaticChunked) {
5801 CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());
5802 CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());
5803 CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());
5804 CGF.EmitIgnoredExpr(S.getCombinedInit());
5814 const OMPLoopArguments LoopArguments = {
5817 EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
5822 return CGF.
Builder.CreateIsNotNull(
5833 return CGF.
Builder.CreateIsNotNull(
5838 if (HasLastprivateClause) {
5858 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
5866 CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;
5869 Fn->setDoesNotRecurse();
5873template <
typename T>
5875 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP,
5876 llvm::OpenMPIRBuilder &OMPBuilder) {
5878 unsigned NumLoops =
C->getNumLoops();
5882 for (
unsigned I = 0; I < NumLoops; I++) {
5883 const Expr *CounterVal =
C->getLoopData(I);
5888 StoreValues.emplace_back(StoreValue);
5890 OMPDoacrossKind<T> ODK;
5891 bool IsDependSource = ODK.isSource(
C);
5893 OMPBuilder.createOrderedDepend(CGF.
Builder, AllocaIP, NumLoops,
5894 StoreValues,
".cnt.addr", IsDependSource));
5900 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5905 assert(!S.hasAssociatedStmt() &&
"No associated statement must be in "
5906 "ordered depend|doacross construct.");
5918 auto FiniCB = [
this](InsertPointTy IP) {
5922 auto BodyGenCB = [&S,
C,
this](InsertPointTy AllocaIP,
5923 InsertPointTy CodeGenIP) {
5928 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
5929 Builder,
false,
".ordered.after");
5932 llvm::Function *OutlinedFn =
5934 assert(S.getBeginLoc().isValid() &&
5935 "Outlined function call location must be valid.");
5938 OutlinedFn, CapturedVars);
5945 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
5947 OMPBuilder.createOrderedThreadsSimd(
Builder, BodyGenCB, FiniCB, !
C));
5953 assert(!S.hasAssociatedStmt() &&
5954 "No associated statement must be in ordered depend construct.");
5960 assert(!S.hasAssociatedStmt() &&
5961 "No associated statement must be in ordered doacross construct.");
5972 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
5973 llvm::Function *OutlinedFn =
5976 OutlinedFn, CapturedVars);
5982 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
5990 "DestType must have scalar evaluation kind.");
5991 assert(!Val.
isAggregate() &&
"Must be a scalar or complex.");
6002 "DestType must have complex evaluation kind.");
6011 ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
6013 assert(Val.
isComplex() &&
"Must be a scalar or complex.");
6018 Val.
getComplexVal().first, SrcElementType, DestElementType, Loc);
6020 Val.
getComplexVal().second, SrcElementType, DestElementType, Loc);
6034 llvm::AtomicOrdering AO,
LValue LVal,
6039 LVal, Loc, llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO),
6048 *
this, RVal, RValTy, LVal.
getType(), Loc)),
6057 llvm_unreachable(
"Must be a scalar or complex.");
6065 assert(
V->isLValue() &&
"V of 'omp atomic read' is not lvalue");
6066 assert(
X->isLValue() &&
"X of 'omp atomic read' is not lvalue");
6075 case llvm::AtomicOrdering::Acquire:
6076 case llvm::AtomicOrdering::AcquireRelease:
6077 case llvm::AtomicOrdering::SequentiallyConsistent:
6079 llvm::AtomicOrdering::Acquire);
6081 case llvm::AtomicOrdering::Monotonic:
6082 case llvm::AtomicOrdering::Release:
6084 case llvm::AtomicOrdering::NotAtomic:
6085 case llvm::AtomicOrdering::Unordered:
6086 llvm_unreachable(
"Unexpected ordering.");
6093 llvm::AtomicOrdering AO,
const Expr *
X,
6096 assert(
X->isLValue() &&
"X of 'omp atomic write' is not lvalue");
6104 case llvm::AtomicOrdering::Release:
6105 case llvm::AtomicOrdering::AcquireRelease:
6106 case llvm::AtomicOrdering::SequentiallyConsistent:
6108 llvm::AtomicOrdering::Release);
6110 case llvm::AtomicOrdering::Acquire:
6111 case llvm::AtomicOrdering::Monotonic:
6113 case llvm::AtomicOrdering::NotAtomic:
6114 case llvm::AtomicOrdering::Unordered:
6115 llvm_unreachable(
"Unexpected ordering.");
6122 llvm::AtomicOrdering AO,
6123 bool IsXLHSInRHSPart) {
6128 if (BO == BO_Comma || !
Update.isScalar() || !
X.isSimple() ||
6129 (!isa<llvm::ConstantInt>(
Update.getScalarVal()) &&
6130 (
Update.getScalarVal()->getType() !=
6131 X.getAddress(CGF).getElementType())) ||
6134 return std::make_pair(
false,
RValue::get(
nullptr));
6137 if (
T->isIntegerTy())
6140 if (
T->isFloatingPointTy() && (BO == BO_Add || BO == BO_Sub))
6146 if (!CheckAtomicSupport(
Update.getScalarVal()->getType(), BO) ||
6147 !CheckAtomicSupport(
X.getAddress(CGF).getElementType(), BO))
6148 return std::make_pair(
false,
RValue::get(
nullptr));
6150 bool IsInteger =
X.getAddress(CGF).getElementType()->isIntegerTy();
6151 llvm::AtomicRMWInst::BinOp RMWOp;
6154 RMWOp = IsInteger ? llvm::AtomicRMWInst::Add : llvm::AtomicRMWInst::FAdd;
6157 if (!IsXLHSInRHSPart)
6158 return std::make_pair(
false,
RValue::get(
nullptr));
6159 RMWOp = IsInteger ? llvm::AtomicRMWInst::Sub : llvm::AtomicRMWInst::FSub;
6162 RMWOp = llvm::AtomicRMWInst::And;
6165 RMWOp = llvm::AtomicRMWInst::Or;
6168 RMWOp = llvm::AtomicRMWInst::Xor;
6172 RMWOp =
X.getType()->hasSignedIntegerRepresentation()
6173 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
6174 : llvm::AtomicRMWInst::Max)
6175 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
6176 : llvm::AtomicRMWInst::UMax);
6178 RMWOp = IsXLHSInRHSPart ? llvm::AtomicRMWInst::FMin
6179 : llvm::AtomicRMWInst::FMax;
6183 RMWOp =
X.getType()->hasSignedIntegerRepresentation()
6184 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
6185 : llvm::AtomicRMWInst::Min)
6186 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
6187 : llvm::AtomicRMWInst::UMin);
6189 RMWOp = IsXLHSInRHSPart ? llvm::AtomicRMWInst::FMax
6190 : llvm::AtomicRMWInst::FMin;
6193 RMWOp = llvm::AtomicRMWInst::Xchg;
6202 return std::make_pair(
false,
RValue::get(
nullptr));
6221 llvm_unreachable(
"Unsupported atomic update operation");
6223 llvm::Value *UpdateVal =
Update.getScalarVal();
6224 if (
auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
6226 UpdateVal = CGF.
Builder.CreateIntCast(
6227 IC,
X.getAddress(CGF).getElementType(),
6228 X.getType()->hasSignedIntegerRepresentation());
6230 UpdateVal = CGF.
Builder.CreateCast(llvm::Instruction::CastOps::UIToFP, IC,
6231 X.getAddress(CGF).getElementType());
6250 if (
X.isGlobalReg()) {
6263 llvm::AtomicOrdering AO,
const Expr *
X,
6267 "Update expr in 'atomic update' must be a binary operator.");
6275 assert(
X->isLValue() &&
"X of 'omp atomic update' is not lvalue");
6278 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
6279 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
6282 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](
RValue XRValue) {
6283 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
6284 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
6288 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
6295 case llvm::AtomicOrdering::Release:
6296 case llvm::AtomicOrdering::AcquireRelease:
6297 case llvm::AtomicOrdering::SequentiallyConsistent:
6299 llvm::AtomicOrdering::Release);
6301 case llvm::AtomicOrdering::Acquire:
6302 case llvm::AtomicOrdering::Monotonic:
6304 case llvm::AtomicOrdering::NotAtomic:
6305 case llvm::AtomicOrdering::Unordered:
6306 llvm_unreachable(
"Unexpected ordering.");
6324 llvm_unreachable(
"Must be a scalar or complex.");
6328 llvm::AtomicOrdering AO,
6329 bool IsPostfixUpdate,
const Expr *
V,
6331 const Expr *UE,
bool IsXLHSInRHSPart,
6333 assert(
X->isLValue() &&
"X of 'omp atomic capture' is not lvalue");
6334 assert(
V->isLValue() &&
"V of 'omp atomic capture' is not lvalue");
6343 "Update expr in 'atomic capture' must be a binary operator.");
6351 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
6352 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
6354 NewVValType = XRValExpr->
getType();
6356 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
6357 IsPostfixUpdate](
RValue XRValue) {
6358 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
6359 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
6361 NewVVal = IsPostfixUpdate ? XRValue : Res;
6365 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
6369 if (IsPostfixUpdate) {
6371 NewVVal = Res.second;
6375 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
6376 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
6382 NewVValType =
X->getType().getNonReferenceType();
6384 X->getType().getNonReferenceType(), Loc);
6385 auto &&Gen = [&NewVVal, ExprRValue](
RValue XRValue) {
6391 XLValue, ExprRValue, BO_Assign,
false, AO,
6396 NewVVal = IsPostfixUpdate ? Res.
second : ExprRValue;
6412 case llvm::AtomicOrdering::Release:
6414 llvm::AtomicOrdering::Release);
6416 case llvm::AtomicOrdering::Acquire:
6418 llvm::AtomicOrdering::Acquire);
6420 case llvm::AtomicOrdering::AcquireRelease:
6421 case llvm::AtomicOrdering::SequentiallyConsistent:
6423 CGF, std::nullopt, Loc, llvm::AtomicOrdering::AcquireRelease);
6425 case llvm::AtomicOrdering::Monotonic:
6427 case llvm::AtomicOrdering::NotAtomic:
6428 case llvm::AtomicOrdering::Unordered:
6429 llvm_unreachable(
"Unexpected ordering.");
6435 CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO,
6437 const Expr *CE,
bool IsXBinopExpr,
bool IsPostfixUpdate,
bool IsFailOnly,
6439 llvm::OpenMPIRBuilder &OMPBuilder =
6442 OMPAtomicCompareOp Op;
6443 assert(isa<BinaryOperator>(CE) &&
"CE is not a BinaryOperator");
6444 switch (cast<BinaryOperator>(CE)->getOpcode()) {
6446 Op = OMPAtomicCompareOp::EQ;
6449 Op = OMPAtomicCompareOp::MIN;
6452 Op = OMPAtomicCompareOp::MAX;
6455 llvm_unreachable(
"unsupported atomic compare binary operator");
6461 auto EmitRValueWithCastIfNeeded = [&CGF, Loc](
const Expr *
X,
const Expr *E) {
6466 if (NewE->
getType() ==
X->getType())
6471 llvm::Value *EVal = EmitRValueWithCastIfNeeded(
X, E);
6472 llvm::Value *DVal = D ? EmitRValueWithCastIfNeeded(
X, D) :
nullptr;
6473 if (
auto *CI = dyn_cast<llvm::ConstantInt>(EVal))
6474 EVal = CGF.
Builder.CreateIntCast(
6478 if (
auto *CI = dyn_cast<llvm::ConstantInt>(DVal))
6479 DVal = CGF.
Builder.CreateIntCast(
6483 llvm::OpenMPIRBuilder::AtomicOpValue XOpVal{
6485 X->getType()->hasSignedIntegerRepresentation(),
6486 X->getType().isVolatileQualified()};
6487 llvm::OpenMPIRBuilder::AtomicOpValue VOpVal, ROpVal;
6492 V->getType()->hasSignedIntegerRepresentation(),
6493 V->getType().isVolatileQualified()};
6503 if (FailAO == llvm::AtomicOrdering::NotAtomic) {
6506 CGF.
Builder.restoreIP(OMPBuilder.createAtomicCompare(
6507 CGF.
Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
6508 IsPostfixUpdate, IsFailOnly));
6510 CGF.
Builder.restoreIP(OMPBuilder.createAtomicCompare(
6511 CGF.
Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
6512 IsPostfixUpdate, IsFailOnly, FailAO));
6516 llvm::AtomicOrdering AO,
6517 llvm::AtomicOrdering FailAO,
bool IsPostfixUpdate,
6520 const Expr *CE,
bool IsXLHSInRHSPart,
6535 IsXLHSInRHSPart, Loc);
6537 case OMPC_compare: {
6539 IsXLHSInRHSPart, IsPostfixUpdate, IsFailOnly, Loc);
6543 llvm_unreachable(
"Clause is not allowed in 'omp atomic'.");
6548 llvm::AtomicOrdering AO = llvm::AtomicOrdering::Monotonic;
6550 llvm::AtomicOrdering FailAO = llvm::AtomicOrdering::NotAtomic;
6551 bool MemOrderingSpecified =
false;
6553 AO = llvm::AtomicOrdering::SequentiallyConsistent;
6554 MemOrderingSpecified =
true;
6556 AO = llvm::AtomicOrdering::AcquireRelease;
6557 MemOrderingSpecified =
true;
6559 AO = llvm::AtomicOrdering::Acquire;
6560 MemOrderingSpecified =
true;
6562 AO = llvm::AtomicOrdering::Release;
6563 MemOrderingSpecified =
true;
6565 AO = llvm::AtomicOrdering::Monotonic;
6566 MemOrderingSpecified =
true;
6568 llvm::SmallSet<OpenMPClauseKind, 2> KindsEncountered;
6577 if (K == OMPC_seq_cst || K == OMPC_acq_rel || K == OMPC_acquire ||
6578 K == OMPC_release || K == OMPC_relaxed || K == OMPC_hint)
6581 KindsEncountered.insert(K);
6586 if (KindsEncountered.contains(OMPC_compare) &&
6587 KindsEncountered.contains(OMPC_capture))
6588 Kind = OMPC_compare;
6589 if (!MemOrderingSpecified) {
6590 llvm::AtomicOrdering DefaultOrder =
6592 if (DefaultOrder == llvm::AtomicOrdering::Monotonic ||
6593 DefaultOrder == llvm::AtomicOrdering::SequentiallyConsistent ||
6594 (DefaultOrder == llvm::AtomicOrdering::AcquireRelease &&
6595 Kind == OMPC_capture)) {
6597 }
else if (DefaultOrder == llvm::AtomicOrdering::AcquireRelease) {
6598 if (Kind == OMPC_unknown || Kind == OMPC_update || Kind == OMPC_write) {
6599 AO = llvm::AtomicOrdering::Release;
6600 }
else if (Kind == OMPC_read) {
6601 assert(Kind == OMPC_read &&
"Unexpected atomic kind.");
6602 AO = llvm::AtomicOrdering::Acquire;
6607 if (KindsEncountered.contains(OMPC_compare) &&
6608 KindsEncountered.contains(OMPC_fail)) {
6609 Kind = OMPC_compare;
6610 const auto *FailClause = S.getSingleClause<
OMPFailClause>();
6613 if (FailParameter == llvm::omp::OMPC_relaxed)
6614 FailAO = llvm::AtomicOrdering::Monotonic;
6615 else if (FailParameter == llvm::omp::OMPC_acquire)
6616 FailAO = llvm::AtomicOrdering::Acquire;
6617 else if (FailParameter == llvm::omp::OMPC_seq_cst)
6618 FailAO = llvm::AtomicOrdering::SequentiallyConsistent;
6622 LexicalScope
Scope(*
this, S.getSourceRange());
6625 S.getV(), S.getR(), S.getExpr(), S.getUpdateExpr(),
6626 S.getD(), S.getCondExpr(), S.isXLHSInRHSPart(),
6627 S.isFailOnly(), S.getBeginLoc());
6638 OMPLexicalScope
Scope(CGF, S, OMPD_target);
6641 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
6647 llvm::Function *Fn =
nullptr;
6648 llvm::Constant *FnID =
nullptr;
6650 const Expr *IfCond =
nullptr;
6652 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
6653 if (
C->getNameModifier() == OMPD_unknown ||
6654 C->getNameModifier() == OMPD_target) {
6655 IfCond =
C->getCondition();
6661 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device(
6664 Device.setPointerAndInt(
C->getDevice(),
C->getModifier());
6669 bool IsOffloadEntry =
true;
6673 IsOffloadEntry =
false;
6676 IsOffloadEntry =
false;
6678 if (
CGM.
getLangOpts().OpenMPOffloadMandatory && !IsOffloadEntry) {
6681 "No offloading entry generated while offloading is mandatory.");
6685 assert(CGF.
CurFuncDecl &&
"No parent declaration for target region!");
6686 StringRef ParentName;
6689 if (
const auto *D = dyn_cast<CXXConstructorDecl>(CGF.
CurFuncDecl))
6691 else if (
const auto *D = dyn_cast<CXXDestructorDecl>(CGF.
CurFuncDecl))
6699 IsOffloadEntry, CodeGen);
6700 OMPLexicalScope
Scope(CGF, S, OMPD_task);
6701 auto &&SizeEmitter =
6704 if (IsOffloadEntry) {
6705 OMPLoopScope(CGF, D);
6708 NumIterations = CGF.
Builder.CreateIntCast(NumIterations, CGF.
Int64Ty,
6710 return NumIterations;
6721 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
6724 (void)PrivateScope.Privatize();
6728 CGF.
EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
6733 StringRef ParentName,
6739 llvm::Constant *Addr;
6742 S, ParentName, Fn, Addr,
true, CodeGen);
6743 assert(Fn && Addr &&
"Target device function emission failed.");
6757 const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
6758 llvm::Function *OutlinedFn =
6766 const Expr *NumTeams = NT ? NT->getNumTeams() :
nullptr;
6767 const Expr *ThreadLimit = TL ? TL->getThreadLimit() :
nullptr;
6773 OMPTeamsScope
Scope(CGF, S);
6784 OMPPrivateScope PrivateScope(CGF);
6788 (void)PrivateScope.Privatize();
6789 CGF.
EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
6799 auto *CS = S.getCapturedStmt(OMPD_teams);
6804 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
6808 (void)PrivateScope.Privatize();
6826 llvm::Constant *Addr;
6829 S, ParentName, Fn, Addr,
true, CodeGen);
6830 assert(Fn && Addr &&
"Target device function emission failed.");
6853 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
6855 (void)PrivateScope.Privatize();
6872 llvm::Constant *Addr;
6875 S, ParentName, Fn, Addr,
true, CodeGen);
6876 assert(Fn && Addr &&
"Target device function emission failed.");
6899 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
6901 (void)PrivateScope.Privatize();
6918 llvm::Constant *Addr;
6921 S, ParentName, Fn, Addr,
true, CodeGen);
6922 assert(Fn && Addr &&
"Target device function emission failed.");
6944 OMPPrivateScope PrivateScope(CGF);
6945 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
6946 (void)PrivateScope.Privatize();
6947 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
6949 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
6966 OMPPrivateScope PrivateScope(CGF);
6967 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
6968 (void)PrivateScope.Privatize();
6969 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
6971 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
6989 OMPPrivateScope PrivateScope(CGF);
6990 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
6991 (void)PrivateScope.Privatize();
6992 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
6994 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
7012 OMPPrivateScope PrivateScope(CGF);
7013 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7014 (void)PrivateScope.Privatize();
7015 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
7016 CGF, OMPD_distribute, CodeGenDistribute,
false);
7017 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
7027 llvm::Value *
Device =
nullptr;
7028 llvm::Value *NumDependences =
nullptr;
7029 llvm::Value *DependenceList =
nullptr;
7037 if (!
Data.Dependences.empty()) {
7039 std::tie(NumDependences, DependenciesArray) =
7049 "OMPNowaitClause clause is used separately in OMPInteropDirective.");
7052 if (!ItOMPInitClause.empty()) {
7055 llvm::Value *InteropvarPtr =
7057 llvm::omp::OMPInteropType InteropType =
7058 llvm::omp::OMPInteropType::Unknown;
7059 if (
C->getIsTarget()) {
7060 InteropType = llvm::omp::OMPInteropType::Target;
7062 assert(
C->getIsTargetSync() &&
7063 "Expected interop-type target/targetsync");
7064 InteropType = llvm::omp::OMPInteropType::TargetSync;
7066 OMPBuilder.createOMPInteropInit(
Builder, InteropvarPtr, InteropType,
7067 Device, NumDependences, DependenceList,
7068 Data.HasNowaitClause);
7072 if (!ItOMPDestroyClause.empty()) {
7075 llvm::Value *InteropvarPtr =
7077 OMPBuilder.createOMPInteropDestroy(
Builder, InteropvarPtr,
Device,
7078 NumDependences, DependenceList,
7079 Data.HasNowaitClause);
7082 auto ItOMPUseClause = S.getClausesOfKind<
OMPUseClause>();
7083 if (!ItOMPUseClause.empty()) {
7086 llvm::Value *InteropvarPtr =
7088 OMPBuilder.createOMPInteropUse(
Builder, InteropvarPtr,
Device,
7089 NumDependences, DependenceList,
7090 Data.HasNowaitClause);
7108 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7110 (void)PrivateScope.Privatize();
7112 CGF, OMPD_distribute, CodeGenDistribute,
false);
7131 llvm::Constant *Addr;
7134 S, ParentName, Fn, Addr,
true, CodeGen);
7135 assert(Fn && Addr &&
"Target device function emission failed.");
7160 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7162 (void)PrivateScope.Privatize();
7164 CGF, OMPD_distribute, CodeGenDistribute,
false);
7183 llvm::Constant *Addr;
7186 S, ParentName, Fn, Addr,
true, CodeGen);
7187 assert(Fn && Addr &&
"Target device function emission failed.");
7201 S.getCancelRegion());
7205 const Expr *IfCond =
nullptr;
7206 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
7207 if (
C->getNameModifier() == OMPD_unknown ||
7208 C->getNameModifier() == OMPD_cancel) {
7209 IfCond =
C->getCondition();
7217 if (S.getCancelRegion() == OMPD_parallel ||
7218 S.getCancelRegion() == OMPD_sections ||
7219 S.getCancelRegion() == OMPD_section) {
7220 llvm::Value *IfCondition =
nullptr;
7225 OMPBuilder.createCancel(
Builder, IfCondition, S.getCancelRegion()));
7230 S.getCancelRegion());
7233CodeGenFunction::JumpDest
7235 if (Kind == OMPD_parallel || Kind == OMPD_task ||
7236 Kind == OMPD_target_parallel || Kind == OMPD_taskloop ||
7237 Kind == OMPD_master_taskloop || Kind == OMPD_parallel_master_taskloop)
7239 assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
7240 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
7241 Kind == OMPD_distribute_parallel_for ||
7242 Kind == OMPD_target_parallel_for ||
7243 Kind == OMPD_teams_distribute_parallel_for ||
7244 Kind == OMPD_target_teams_distribute_parallel_for);
7245 return OMPCancelStack.getExitBlock();
7250 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
7251 CaptureDeviceAddrMap) {
7252 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
7253 for (
const Expr *OrigVarIt :
C.varlists()) {
7254 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(OrigVarIt)->getDecl());
7255 if (!Processed.insert(OrigVD).second)
7262 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
7265 const auto *ME = cast<MemberExpr>(OED->getInit());
7266 assert(isa<CXXThisExpr>(ME->getBase()->IgnoreImpCasts()) &&
7267 "Base should be the current struct!");
7268 MatchingVD = ME->getMemberDecl();
7273 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
7274 if (InitAddrIt == CaptureDeviceAddrMap.end())
7280 bool IsRegistered = PrivateScope.addPrivate(
7282 Address(InitAddrIt->second, Ty,
7284 assert(IsRegistered &&
"firstprivate var already registered as private");
7292 while (
const auto *OASE = dyn_cast<OMPArraySectionExpr>(
Base))
7293 Base = OASE->getBase()->IgnoreParenImpCasts();
7294 while (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(
Base))
7295 Base = ASE->getBase()->IgnoreParenImpCasts();
7296 return cast<VarDecl>(cast<DeclRefExpr>(
Base)->getDecl());
7301 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
7302 CaptureDeviceAddrMap) {
7303 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
7304 for (
const Expr *Ref :
C.varlists()) {
7306 if (!Processed.insert(OrigVD).second)
7312 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
7315 const auto *ME = cast<MemberExpr>(OED->getInit());
7316 assert(isa<CXXThisExpr>(ME->getBase()) &&
7317 "Base should be the current struct!");
7318 MatchingVD = ME->getMemberDecl();
7323 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
7324 if (InitAddrIt == CaptureDeviceAddrMap.end())
7330 Address(InitAddrIt->second, Ty,
7334 if (isa<DeclRefExpr>(Ref->IgnoreParenImpCasts()) ||
7343 (void)PrivateScope.addPrivate(OrigVD, PrivAddr);
7356 bool PrivatizeDevicePointers =
false;
7358 bool &PrivatizeDevicePointers;
7361 explicit DevicePointerPrivActionTy(
bool &PrivatizeDevicePointers)
7362 : PrivatizeDevicePointers(PrivatizeDevicePointers) {}
7364 PrivatizeDevicePointers =
true;
7367 DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
7371 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
7377 PrivatizeDevicePointers =
false;
7383 if (PrivatizeDevicePointers) {
7384 OMPPrivateScope PrivateScope(CGF);
7388 Info.CaptureDeviceAddrMap);
7391 Info.CaptureDeviceAddrMap);
7392 (void)PrivateScope.Privatize();
7397 std::optional<OpenMPDirectiveKind> CaptureRegion;
7401 for (
const Expr *E :
C->varlists()) {
7402 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
7403 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
7407 for (
const Expr *E :
C->varlists()) {
7409 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
7413 CaptureRegion = OMPD_unknown;
7416 OMPLexicalScope
Scope(CGF, S, CaptureRegion);
7423 PrivRCG.setAction(Action);
7428 OMPLexicalScope
Scope(CGF, S);
7443 const Expr *IfCond =
nullptr;
7445 IfCond =
C->getCondition();
7453 RCG.setAction(PrivAction);
7468 const Expr *IfCond =
nullptr;
7470 IfCond =
C->getCondition();
7477 OMPLexicalScope
Scope(*
this, S, OMPD_task);
7489 const Expr *IfCond =
nullptr;
7491 IfCond =
C->getCondition();
7498 OMPLexicalScope
Scope(*
this, S, OMPD_task);
7506 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
7510 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7514 (void)PrivateScope.Privatize();
7534 llvm::Constant *Addr;
7537 S, ParentName, Fn, Addr,
true, CodeGen);
7538 assert(Fn && Addr &&
"Target device function emission failed.");
7557 CodeGenFunction::OMPCancelStackRAII CancelRegion(
7558 CGF, OMPD_target_parallel_for, S.hasCancel());
7574 llvm::Constant *Addr;
7577 S, ParentName, Fn, Addr,
true, CodeGen);
7578 assert(Fn && Addr &&
"Target device function emission failed.");
7613 llvm::Constant *Addr;
7616 S, ParentName, Fn, Addr,
true, CodeGen);
7617 assert(Fn && Addr &&
"Target device function emission failed.");
7631 CodeGenFunction::OMPPrivateScope &Privates) {
7632 const auto *VDecl = cast<VarDecl>(Helper->
getDecl());
7639 const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);
7642 OMPLexicalScope
Scope(*
this, S, OMPD_taskloop,
false);
7646 const Expr *IfCond =
nullptr;
7647 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
7648 if (
C->getNameModifier() == OMPD_unknown ||
7649 C->getNameModifier() == OMPD_taskloop) {
7650 IfCond =
C->getCondition();
7663 Data.Schedule.setInt(
false);
7667 Data.Schedule.setInt(
true);
7682 llvm::BasicBlock *ContBlock =
nullptr;
7683 OMPLoopScope PreInitScope(CGF, S);
7684 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
7688 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock(
"taskloop.if.then");
7689 ContBlock = CGF.createBasicBlock(
"taskloop.if.end");
7690 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
7691 CGF.getProfileCount(&S));
7692 CGF.EmitBlock(ThenBlock);
7693 CGF.incrementProfileCounter(&S);
7696 (void)CGF.EmitOMPLinearClauseInit(S);
7698 OMPPrivateScope LoopScope(CGF);
7700 enum { LowerBound = 5, UpperBound, Stride, LastIter };
7702 auto *LBP = std::next(I, LowerBound);
7703 auto *UBP = std::next(I, UpperBound);
7704 auto *STP = std::next(I, Stride);
7705 auto *LIP = std::next(I, LastIter);
7706 mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
7708 mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
7710 mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
7711 mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
7713 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
7714 CGF.EmitOMPLinearClause(S, LoopScope);
7715 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
7716 (void)LoopScope.Privatize();
7718 const Expr *IVExpr = S.getIterationVariable();
7719 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
7720 CGF.EmitVarDecl(*IVDecl);
7721 CGF.EmitIgnoredExpr(S.getInit());
7726 if (
const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
7727 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
7729 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
7733 OMPLexicalScope
Scope(CGF, S, OMPD_taskloop,
false);
7741 CGF.EmitOMPInnerLoop(
7742 S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),
7744 emitOMPLoopBodyWithStopPoint(CGF, S,
7745 CodeGenFunction::JumpDest());
7752 CGF.EmitBranch(ContBlock);
7753 CGF.EmitBlock(ContBlock,
true);
7756 if (HasLastprivateClause) {
7757 CGF.EmitOMPLastprivateClauseFinal(
7759 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
7760 CGF.GetAddrOfLocalVar(*LIP),
false,
7761 (*LIP)->getType(), S.getBeginLoc())));
7763 LoopScope.restoreMap();
7765 return CGF.
Builder.CreateIsNotNull(
7767 (*LIP)->
getType(), S.getBeginLoc()));
7770 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
7773 auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
7775 OMPLoopScope PreInitScope(CGF, S);
7776 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
7777 OutlinedFn, SharedsTy,
7778 CapturedStruct, IfCond,
Data);
7780 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
7808 OMPLexicalScope
Scope(*
this, S);
7820 OMPLexicalScope
Scope(*
this, S, std::nullopt,
false);
7832 OMPLexicalScope
Scope(*
this, S);
7844 OMPLexicalScope
Scope(CGF, S, OMPD_parallel,
false);
7862 OMPLexicalScope
Scope(CGF, S, OMPD_parallel,
false);
7881 const Expr *IfCond =
nullptr;
7883 IfCond =
C->getCondition();
7890 OMPLexicalScope
Scope(*
this, S, OMPD_task);
7901 const auto *ForS = dyn_cast<ForStmt>(CS);
7902 if (ForS && !isa<DeclStmt>(ForS->getInit())) {
7903 OMPPrivateScope LoopScope(CGF);
7905 (void)LoopScope.Privatize();
7907 LoopScope.restoreMap();
7912 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
7947 OMPPrivateScope PrivateScope(CGF);
7948 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7949 (void)PrivateScope.Privatize();
7950 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
7952 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
7961 std::string StatusMsg,
7965 StatusMsg +=
": DEVICE";
7967 StatusMsg +=
": HOST";
7974 llvm::dbgs() << StatusMsg <<
": " <<
FileName <<
": " << LineNo <<
"\n";
7993 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7995 (void)PrivateScope.Privatize();
7997 CGF, OMPD_distribute, CodeGenDistribute,
false);
8022 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
8024 (void)PrivateScope.Privatize();
8026 CGF, OMPD_distribute, CodeGenDistribute,
false);
8040 if (S.canBeParallelFor())
8053 if (S.canBeParallelFor())
8059 llvm::Constant *Addr;
8062 S, ParentName, Fn, Addr,
true, CodeGen);
8063 assert(Fn && Addr &&
8064 "Target device function emission failed for 'target teams loop'.");
8074 CodeGenFunction::OMPCancelStackRAII CancelRegion(
8075 CGF, OMPD_target_parallel_loop,
false);
8091 llvm::Constant *Addr;
8094 S, ParentName, Fn, Addr,
true, CodeGen);
8095 assert(Fn && Addr &&
"Target device function emission failed.");
8110 if (
const auto *SD = dyn_cast<OMPScanDirective>(&D)) {
8117 OMPPrivateScope GlobalsScope(CGF);
8121 for (
const Expr *Ref :
C->varlists()) {
8122 const auto *DRE = cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
8125 const auto *VD = dyn_cast<VarDecl>(DRE->getDecl());
8128 if (!CGF.LocalDeclMap.count(VD)) {
8130 GlobalsScope.addPrivate(VD, GlobLVal.
getAddress(CGF));
8136 (void)GlobalsScope.Privatize();
8137 ParentLoopDirectiveForScanRegion ScanRegion(CGF, D);
8140 if (
const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
8141 for (
const Expr *E : LD->counters()) {
8142 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
8145 GlobalsScope.addPrivate(VD, GlobLVal.
getAddress(CGF));
8147 if (isa<OMPCapturedExprDecl>(VD)) {
8149 if (!CGF.LocalDeclMap.count(VD))
8154 if (!
C->getNumForLoops())
8156 for (
unsigned I = LD->getLoopsNumber(),
8157 E =
C->getLoopNumIterations().size();
8159 if (
const auto *VD = dyn_cast<OMPCapturedExprDecl>(
8160 cast<DeclRefExpr>(
C->getLoopCounter(I))->getDecl())) {
8162 if (!CGF.LocalDeclMap.count(VD))
8168 (void)GlobalsScope.Privatize();
8182 OMPSimdLexicalScope
Scope(*
this, D);
Defines the clang::ASTContext interface.
static bool isAllocatableDecl(const VarDecl *VD)
static const VarDecl * getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE)
static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, PrePostActionTy &Action)
static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, PrePostActionTy &Action)
static llvm::Function * emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S, SourceLocation Loc)
static const VarDecl * getBaseDecl(const Expr *Ref)
static void emitTargetTeamsGenericLoopRegionAsParallel(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsGenericLoopDirective &S)
static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *V, SourceLocation Loc)
static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, bool IsPostfixUpdate, const Expr *V, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
static void emitScanBasedDirective(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen, llvm::function_ref< void(CodeGenFunction &)> FirstGen, llvm::function_ref< void(CodeGenFunction &)> SecondGen)
Emits the code for the directive with inscan reductions.
static bool isSupportedByOpenMPIRBuilder(const OMPSimdDirective &S)
static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO, LValue LVal, RValue RVal)
static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, QualType DstType, StringRef Name, LValue AddrLV)
static void emitDistributeParallelForDistributeInnerBoundParams(CodeGenFunction &CGF, const OMPExecutableDirective &S, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars)
static void emitScanBasedDirectiveFinals(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen)
Copies final inscan reductions values to the original variables.
static void checkForLastprivateConditionalUpdate(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static std::pair< LValue, LValue > emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
The following two functions generate expressions for the loop lower and upper bounds in case of stati...
static void emitTargetParallelForRegion(CodeGenFunction &CGF, const OMPTargetParallelForDirective &S, PrePostActionTy &Action)
static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper)
Emit a helper variable and return corresponding lvalue.
static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
static llvm::Value * convertToScalarValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
static std::pair< bool, RValue > emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, RValue Update, BinaryOperatorKind BO, llvm::AtomicOrdering AO, bool IsXLHSInRHSPart)
static std::pair< LValue, LValue > emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitTargetTeamsGenericLoopRegionAsDistribute(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsGenericLoopDirective &S)
static void emitTargetParallelRegion(CodeGenFunction &CGF, const OMPTargetParallelDirective &S, PrePostActionTy &Action)
static std::pair< llvm::Value *, llvm::Value * > emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
When dealing with dispatch schedules (e.g.
static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitRestoreIP(CodeGenFunction &CGF, const T *C, llvm::OpenMPIRBuilder::InsertPointTy AllocaIP, llvm::OpenMPIRBuilder &OMPBuilder)
static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, const RegionCodeGenTy &CodeGen)
static void emitSimdlenSafelenClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
static void emitAlignedClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters)
static bool emitWorksharingDirective(CodeGenFunction &CGF, const OMPLoopDirective &S, bool HasCancel)
static void emitPostUpdateForReductionClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc, const unsigned IVSize, const bool IVSigned)
static void emitTargetTeamsLoopCodegenStatus(CodeGenFunction &CGF, std::string StatusMsg, const OMPExecutableDirective &D)
static RValue emitSimpleAtomicLoad(CodeGenFunction &CGF, llvm::AtomicOrdering AO, LValue LVal, SourceLocation Loc)
static std::pair< llvm::Value *, llvm::Value * > emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
if the 'for' loop has a dispatch schedule (e.g.
static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *UE, const Expr *D, const Expr *CE, bool IsXLHSInRHSPart, bool IsFailOnly, SourceLocation Loc)
static CodeGenFunction::ComplexPairTy convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static ImplicitParamDecl * createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data, QualType Ty, CapturedDecl *CD, SourceLocation Loc)
static EmittedClosureTy emitCapturedStmtFunc(CodeGenFunction &ParentCGF, const CapturedStmt *S)
Emit a captured statement and return the function as well as its captured closure context.
static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
static llvm::MapVector< llvm::Value *, llvm::Value * > GetAlignedMapping(const OMPSimdDirective &S, CodeGenFunction &CGF)
static void emitOMPCopyinClause(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitTargetTeamsDistributeParallelForRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, PrePostActionTy &Action)
static llvm::CallInst * emitCapturedStmtCall(CodeGenFunction &ParentCGF, EmittedClosureTy Cap, llvm::ArrayRef< llvm::Value * > Args)
Emit a call to a previously captured closure.
static void emitMasked(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop, int MaxLevel, int Level=0)
static void emitEmptyBoundParameters(CodeGenFunction &, const OMPExecutableDirective &, llvm::SmallVectorImpl< llvm::Value * > &)
static void emitTargetParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetParallelForSimdDirective &S, PrePostActionTy &Action)
static void emitOMPAtomicCompareExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO, const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *D, const Expr *CE, bool IsXBinopExpr, bool IsPostfixUpdate, bool IsFailOnly, SourceLocation Loc)
std::pair< llvm::Function *, llvm::Value * > EmittedClosureTy
static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDirective &S)
static void buildDependences(const OMPExecutableDirective &S, OMPTaskDataTy &Data)
static RValue convertToType(CodeGenFunction &CGF, RValue Value, QualType SourceType, QualType ResType, SourceLocation Loc)
static void emitScanBasedDirectiveDecls(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen)
Emits internal temp array declarations for the directive with inscan reductions.
static void emitTargetTeamsDistributeParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForSimdDirective &S, PrePostActionTy &Action)
static void emitTargetTeamsDistributeSimdRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeSimdDirective &S)
static llvm::omp::ScheduleKind convertClauseKindToSchedKind(OpenMPScheduleClauseKind ScheduleClauseKind)
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
static void emitTargetParallelGenericLoopRegion(CodeGenFunction &CGF, const OMPTargetParallelGenericLoopDirective &S, PrePostActionTy &Action)
static QualType getCanonicalParamType(ASTContext &C, QualType T)
static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S, const RegionCodeGenTy &SimdInitGen, const RegionCodeGenTy &BodyCodeGen)
static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, const Twine &Name, llvm::Value *Init=nullptr)
static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *E, SourceLocation Loc)
static llvm::Function * emitOutlinedFunctionPrologue(CodeGenFunction &CGF, FunctionArgList &Args, llvm::MapVector< const Decl *, std::pair< const VarDecl *, Address > > &LocalAddrs, llvm::DenseMap< const Decl *, std::pair< const Expr *, llvm::Value * > > &VLASizes, llvm::Value *&CXXThisValue, const FunctionOptions &FO)
static void emitInnerParallelForWhenCombined(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
static void emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeDirective &S)
This file defines OpenMP nodes for declarative directives.
static const Decl * getCanonicalDecl(const Decl *D)
This file defines OpenMP AST classes for clauses.
Defines some OpenMP-specific enums and functions.
Defines the PrettyStackTraceEntry class, which is used to make crashes give more contextual informati...
Defines the SourceManager interface.
This file defines OpenMP AST classes for executable directives and clauses.
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
SourceManager & getSourceManager()
TranslationUnitDecl * getTranslationUnitDecl() const
QualType getRecordType(const RecordDecl *Decl) const
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
QualType getUIntPtrType() const
Return a type compatible with "uintptr_t" (C99 7.18.1.4), as defined by the target.
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth,...
TypeSourceInfo * getTrivialTypeSourceInfo(QualType T, SourceLocation Loc=SourceLocation()) const
Allocate a TypeSourceInfo where all locations have been initialized to a given location,...
unsigned getOpenMPDefaultSimdAlign(QualType T) const
Get default simd alignment of the specified complete type in bits.
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
QualType getFunctionType(QualType ResultTy, ArrayRef< QualType > Args, const FunctionProtoType::ExtProtoInfo &EPI) const
Return a normal function type with a typed argument list.
const TargetInfo & getTargetInfo() const
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
ASTRecordLayout - This class contains layout information for one RecordDecl, which is a struct/union/...
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Represents an attribute applied to a statement.
ArrayRef< const Attr * > getAttrs() const
A builtin binary operation expression such as "x + y" or "x <= y".
static BinaryOperator * Create(const ASTContext &C, Expr *lhs, Expr *rhs, Opcode opc, QualType ResTy, ExprValueKind VK, ExprObjectKind OK, SourceLocation opLoc, FPOptionsOverride FPFeatures)
BlockExpr - Adaptor class for mixing a BlockDecl with expressions.
Represents the body of a CapturedStmt, and serves as its DeclContext.
unsigned getContextParamPosition() const
static CapturedDecl * Create(ASTContext &C, DeclContext *DC, unsigned NumParams)
param_iterator param_end() const
Retrieve an iterator one past the last parameter decl.
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
Stmt * getBody() const override
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
ImplicitParamDecl * getParam(unsigned i) const
This captures a statement into a function.
CapturedDecl * getCapturedDecl()
Retrieve the outlined function declaration.
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Stmt * getCapturedStmt()
Retrieve the statement being captured.
Expr *const * const_capture_init_iterator
Const iterator that walks over the capture initialization arguments.
CharUnits - This is an opaque type for sizes expressed in character units.
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
CharUnits alignTo(const CharUnits &Align) const
alignTo - Returns the next integer (mod 2**64) that is greater than or equal to this quantity and is ...
bool hasReducedDebugInfo() const
Check if type and variable info should be emitted.
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
CharUnits getAlignment() const
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Address withAlignment(CharUnits NewAlignment) const
Return address with different alignment, but same pointer and element type.
llvm::PointerType * getType() const
Return the type of the pointer value.
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored.
static ApplyDebugLocation CreateDefaultArtificial(CodeGenFunction &CGF, SourceLocation TemporaryLocation)
Apply TemporaryLocation if it is valid.
llvm::Value * CreateIsNull(Address Addr, const Twine &Name="")
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
llvm::AtomicRMWInst * CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Ordering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Address CreateInBoundsGEP(Address Addr, ArrayRef< llvm::Value * > IdxList, llvm::Type *ElementType, CharUnits Align, const Twine &Name="")
CGFunctionInfo - Class to encapsulate the information about a function definition.
Manages list of lastprivate conditional decls for the specified directive.
static LastprivateConditionalRAII disable(CodeGenFunction &CGF, const OMPExecutableDirective &S)
Manages list of nontemporal decls for the specified directive.
Struct that keeps all the relevant information that should be kept throughout a 'target data' region.
Manages list of nontemporal decls for the specified directive.
virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
Emit task region for the task directive.
virtual llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST)
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device)
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
virtual void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc)
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)...
virtual void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps)
Emits a single region.
virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal)
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF, const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const
Choose default schedule type and chunk value for the schedule clause.
virtual void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars)
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
virtual void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancellation point' construct.
virtual const VarDecl * translateParameter(const FieldDecl *FD, const VarDecl *NativeParam) const
Translates the native parameter of outlined function if this is required for target.
virtual llvm::Function * emitTeamsOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP teams directive D.
virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr * > NumIterations)
Emit initialization for doacross loop nesting support.
virtual void adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF, const OMPExecutableDirective &D) const
Adjust some parameters for the target-based directives, like addresses of the variables captured by r...
virtual void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, CGOpenMPRuntime::TargetDataInfo &Info)
Emit the target data mapping code associated with D.
virtual Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const
Gets the address of the native argument basing on the address of the target-specific parameter.
virtual void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc)
Emit a taskgroup region.
virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams,...
virtual llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data)
Emit a code for initialization of task reduction clause.
void emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, OpenMPDependClauseKind NewDepKind, SourceLocation Loc)
Updates the dependency kind in the specified depobj object.
virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD, SourceLocation Loc)
Gets the address of the global copy used for lastprivate conditional update, if any.
virtual void emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, Expr *ME, bool IsFatal)
Emit __kmpc_error call for error directive extern void __kmpc_error(ident_t *loc, int severity,...
virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc)
Emits code for a taskyield directive.
virtual void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc, llvm::AtomicOrdering AO)
Emit flush of the variables specified in 'omp flush' directive.
virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPTaskDataTy &Data)
Emit code for 'taskwait' directive.
virtual void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc)
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, int proc_bind) to generat...
virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false)
Emit an implicit/explicit barrier for OpenMP threads.
virtual void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values)
virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind)
Call the appropriate runtime routine to notify that we finished all the work with current loop.
void emitIfClause(CodeGenFunction &CGF, const Expr *Cond, const RegionCodeGenTy &ThenGen, const RegionCodeGenTy &ElseGen)
Emits code for OpenMP 'if' clause using specified CodeGen function.
Address emitDepobjDependClause(CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies, SourceLocation Loc)
Emits list of dependecies based on the provided data (array of dependence/expression pairs) for depob...
virtual llvm::Function * emitParallelOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP parallel directive D.
virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values)
Call the appropriate runtime routine to initialize it before start of loop.
llvm::AtomicOrdering getDefaultMemoryOrdering() const
Gets default memory ordering as specified in requires directive.
virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static non-chunked.
virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancel' construct.
virtual void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc)
Emits a master region.
virtual llvm::Function * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts)
Emits outlined function for the OpenMP task directive D.
void emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal, SourceLocation Loc)
Emits the code to destroy the dependency object provided in depobj directive.
virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Required to resolve existing problems in the runtime.
virtual void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C)
Emit code for doacross ordered directive with 'depend' clause.
virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF, const Expr *LHS)
Checks if the provided LVal is lastprivate conditional and emits the code to update the value of the ...
virtual void getDefaultDistScheduleAndChunk(CodeGenFunction &CGF, const OMPLoopDirective &S, OpenMPDistScheduleClauseKind &ScheduleKind, llvm::Value *&Chunk) const
Choose default schedule type and chunk value for the dist_schedule clause.
llvm::OpenMPIRBuilder & getOMPBuilder()
virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Emit outilined function for 'target' directive.
virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr)
Emits a critical region.
virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned)
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
virtual void checkAndEmitSharedLastprivateConditional(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::DenseSet< CanonicalDeclPtr< const VarDecl > > &IgnoredDecls)
Checks if the lastprivate conditional was updated in inner region and writes the value.
virtual void emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool HasCancel=false)
Emit code for the directive that does not require outlining.
virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond, llvm::Value *NumThreads)
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static chunked.
virtual void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Emit the target offloading code associated with D.
virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps, ReductionOptionsTy Options)
Emit a code for reduction clause.
std::pair< llvm::Value *, Address > emitDependClause(CodeGenFunction &CGF, ArrayRef< OMPTaskDataTy::DependData > Dependencies, SourceLocation Loc)
Emits list of dependecies based on the provided data (array of dependence/expression pairs).
virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const
Check if the specified ScheduleKind is dynamic.
Address emitLastprivateConditionalInit(CodeGenFunction &CGF, const VarDecl *VD)
Create specialized alloca to handle lastprivate conditionals.
virtual void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads)
Emit an ordered region.
virtual void emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, ArrayRef< llvm::Value * > Args=std::nullopt) const
Emits call of the outlined function with the provided arguments, translating these arguments to corre...
virtual void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, bool IsWorksharingReduction)
Emits the following code for reduction clause with task modifier:
virtual void emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter=nullptr)
Emits a masked region.
virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues)
Call the appropriate runtime routine to initialize it before start of loop.
virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S)
Emit the captured statement body.
virtual StringRef getHelperName() const
Get the name of the capture helper.
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
RAII for preserving necessary info during inlined region body codegen.
Cleanup action for allocate support.
RAII for preserving necessary info during Outlined region body codegen.
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion, const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen, OMPTaskDataTy &Data)
void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S)
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
void EmitOMPParallelGenericLoopDirective(const OMPLoopDirective &S)
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount, Stmt::Likelihood LH=Stmt::LH_None, const Expr *ConditionalOp=nullptr)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target)
The given basic block lies in the current EH scope, but may be a target of a potentially scope-crossi...
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
void EmitOMPOrderedDirective(const OMPOrderedDirective &S)
void EmitOMPTargetDirective(const OMPTargetDirective &S)
bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
void EmitOMPReductionClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope, bool ForInscan=false)
Emit initial code for reduction variables.
void EmitOMPAtomicDirective(const OMPAtomicDirective &S)
void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S)
void EmitOMPParallelMasterTaskLoopDirective(const OMPParallelMasterTaskLoopDirective &S)
void EmitAutoVarDecl(const VarDecl &D)
EmitAutoVarDecl - Emit an auto variable declaration.
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
static bool hasScalarEvaluationKind(QualType T)
llvm::function_ref< std::pair< llvm::Value *, llvm::Value * >(CodeGenFunction &, const OMPExecutableDirective &S, Address LB, Address UB)> CodeGenDispatchBoundsTy
CGCapturedStmtInfo * CapturedStmtInfo
void EmitDecl(const Decl &D)
EmitDecl - Emit a declaration.
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Load a pointer with type PtrTy stored at address Ptr.
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
llvm::Value * emitArrayLength(const ArrayType *arrayType, QualType &baseType, Address &addr)
emitArrayLength - Compute the length of an array, even if it's a VLA, and drill down to the base elem...
VlaSizePair getVLASize(const VariableArrayType *vla)
Returns an LLVM value that corresponds to the size, in non-variably-sized elements,...
const OMPExecutableDirective * OMPParentLoopDirectiveForScan
Parent loop-based directive for scan directive.
void EmitOMPTeamsDistributeParallelForDirective(const OMPTeamsDistributeParallelForDirective &S)
void EmitOMPTaskDirective(const OMPTaskDirective &S)
void EmitOMPScanDirective(const OMPScanDirective &S)
void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S)
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
void EmitOMPMasterTaskLoopSimdDirective(const OMPMasterTaskLoopSimdDirective &S)
void EmitOMPUseDevicePtrClause(const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap)
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
void EmitOMPDistributeLoop(const OMPLoopDirective &S, const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr)
Emit code for the distribute loop-based directive.
void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S)
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO, const llvm::function_ref< RValue(RValue)> &UpdateOp, bool IsVolatile)
const LangOptions & getLangOpts() const
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S)
Emit device code for the target simd directive.
void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S)
void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S, OMPPrivateScope &LoopScope)
Emit initial code for loop counters of loop-based directives.
void EmitOMPDistributeParallelForDirective(const OMPDistributeParallelForDirective &S)
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D, bool NoFinals, llvm::Value *IsLastIterCond=nullptr)
Emit final copying of lastprivate values to original variables at the end of the worksharing or simd ...
void EmitOMPTeamsGenericLoopDirective(const OMPTeamsGenericLoopDirective &S)
void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S)
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit)
EmitExprAsInit - Emits the code necessary to initialize a location in memory with the given initializ...
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
void EmitOMPParallelDirective(const OMPParallelDirective &S)
void EmitOMPSimdFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
llvm::Type * ConvertTypeForMem(QualType T)
void EmitOMPDistributeParallelForSimdDirective(const OMPDistributeParallelForSimdDirective &S)
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
void EmitOMPForSimdDirective(const OMPForSimdDirective &S)
llvm::AssertingVH< llvm::Instruction > AllocaInsertPt
AllocaInsertPoint - This is an instruction in the entry block before which we prefer to insert alloca...
void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy)
Emit an aggregate assignment.
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value * > &CapturedVars)
CodeGenFunction * ParentCGF
JumpDest ReturnBlock
ReturnBlock - Unified return block.
const llvm::function_ref< void(CodeGenFunction &, llvm::Function *, const OMPTaskDataTy &)> TaskGenTy
static void EmitOMPTargetTeamsGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsGenericLoopDirective &S)
Emit device code for the target teams loop directive.
RawAddress CreateMemTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
void EmitOMPFlushDirective(const OMPFlushDirective &S)
llvm::BasicBlock * OMPBeforeScanBlock
void EmitOMPCancelDirective(const OMPCancelDirective &S)
void EmitOMPGenericLoopDirective(const OMPGenericLoopDirective &S)
void EmitOMPTargetTeamsDistributeDirective(const OMPTargetTeamsDistributeDirective &S)
ConstantEmission tryEmitAsConstant(DeclRefExpr *refExpr)
void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S)
static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S)
void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy, SourceLocation Loc)
void EmitOMPInnerLoop(const OMPExecutableDirective &S, bool RequiresCleanup, const Expr *LoopCond, const Expr *IncExpr, const llvm::function_ref< void(CodeGenFunction &)> BodyGen, const llvm::function_ref< void(CodeGenFunction &)> PostIncGen)
Emit inner loop of the worksharing/simd construct.
llvm::DebugLoc SourceLocToDebugLoc(SourceLocation Location)
Converts Location to a DebugLoc, if debug information is enabled.
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
void EmitOMPDepobjDirective(const OMPDepobjDirective &S)
void EmitOMPSingleDirective(const OMPSingleDirective &S)
void EmitOMPTargetTeamsGenericLoopDirective(const OMPTargetTeamsGenericLoopDirective &S)
llvm::CanonicalLoopInfo * EmitOMPCollapsedCanonicalLoopNest(const Stmt *S, int Depth)
Emit the Stmt S and return its topmost canonical loop, if any.
void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D)
Emit simple code for OpenMP directives in Simd-only mode.
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
EmitAnyExpr - Emit code to compute the specified expression which can have any type.
void EmitOMPParallelForDirective(const OMPParallelForDirective &S)
void EmitOMPTeamsDirective(const OMPTeamsDirective &S)
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
void EmitOMPUnrollDirective(const OMPUnrollDirective &S)
void EmitOMPParallelMasterTaskLoopSimdDirective(const OMPParallelMasterTaskLoopSimdDirective &S)
void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S)
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
llvm::Value * EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified complex type to the specified destination type,...
bool isTrivialInitializer(const Expr *Init)
Determine whether the given initializer is trivial in the sense that it requires no code to be genera...
CGDebugInfo * getDebugInfo()
void EmitOMPTargetTeamsDistributeParallelForSimdDirective(const OMPTargetTeamsDistributeParallelForSimdDirective &S)
void EmitOMPTeamsDistributeParallelForSimdDirective(const OMPTeamsDistributeParallelForSimdDirective &S)
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block,...
llvm::Function * GenerateCapturedStmtFunction(const CapturedStmt &S)
void EmitOMPInteropDirective(const OMPInteropDirective &S)
void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind)
Emit final update of reduction values to original variables at the end of the directive.
llvm::BasicBlock * OMPScanDispatch
llvm::function_ref< std::pair< LValue, LValue >(CodeGenFunction &, const OMPExecutableDirective &S)> CodeGenLoopBoundsTy
void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty, SourceLocation Loc, SourceLocation AssumptionLoc, llvm::Value *Alignment, llvm::Value *OffsetValue=nullptr)
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S)
void EmitOMPParallelMaskedDirective(const OMPParallelMaskedDirective &S)
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
void EmitOMPSimdInit(const OMPLoopDirective &D)
Helpers for the OpenMP loop directives.
int ExpectedOMPLoopDepth
Number of nested loop to be consumed by the last surrounding loop-associated directive.
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S)
ASTContext & getContext() const
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
SmallVector< llvm::CanonicalLoopInfo *, 4 > OMPLoopNestStack
List of recently emitted OMPCanonicalLoops.
bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB, const CodeGenLoopBoundsTy &CodeGenLoopBounds, const CodeGenDispatchBoundsTy &CGDispatchBounds)
Emit code for the worksharing loop-based directive.
void EmitOMPLinearClause(const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope)
Emit initial code for linear clauses.
void EmitBranchThroughCleanup(JumpDest Dest)
EmitBranchThroughCleanup - Emit a branch from the current insert block through the normal cleanup han...
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant,...
void EmitAutoVarCleanups(const AutoVarEmission &emission)
void EmitOMPTileDirective(const OMPTileDirective &S)
bool EmitOMPLinearClauseInit(const OMPLoopDirective &D)
Emit initial code for linear variables.
llvm::DenseMap< const ValueDecl *, FieldDecl * > LambdaCaptureFields
void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit)
Helper for the OpenMP loop directives.
void EmitOMPLinearClauseFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
Emit final code for linear clauses.
void EmitOMPSectionsDirective(const OMPSectionsDirective &S)
llvm::BasicBlock * OMPScanExitBlock
static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Emit device code for the target teams distribute parallel for simd directive.
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
void EmitOMPUseDeviceAddrClause(const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap)
void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S)
void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S)
void EmitOMPSimdDirective(const OMPSimdDirective &S)
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
void EmitOMPCriticalDirective(const OMPCriticalDirective &S)
void EmitOMPForDirective(const OMPForDirective &S)
void EmitOMPMetaDirective(const OMPMetaDirective &S)
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S)
LValue InitCapturedStruct(const CapturedStmt &S)
void EmitOMPParallelMasterDirective(const OMPParallelMasterDirective &S)
void EmitOMPTargetTeamsDistributeSimdDirective(const OMPTargetTeamsDistributeSimdDirective &S)
void processInReduction(const OMPExecutableDirective &S, OMPTaskDataTy &Data, CodeGenFunction &CGF, const CapturedStmt *CS, OMPPrivateScope &Scope)
void EmitOMPMasterDirective(const OMPMasterDirective &S)
void EmitOMPMasterTaskLoopDirective(const OMPMasterTaskLoopDirective &S)
void EmitOMPTargetParallelGenericLoopDirective(const OMPTargetParallelGenericLoopDirective &S)
bool EmitOMPCopyinClause(const OMPExecutableDirective &D)
Emit code for copyin clause in D directive.
void EmitOMPMaskedDirective(const OMPMaskedDirective &S)
uint64_t getProfileCount(const Stmt *S)
Get the profiler's count for the given statement.
void EmitOMPPrivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S, SourceLocation Loc)
void EmitOMPTargetParallelForSimdDirective(const OMPTargetParallelForSimdDirective &S)
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
Address GenerateCapturedStmtArgument(const CapturedStmt &S)
static void EmitOMPTargetParallelGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelGenericLoopDirective &S)
Emit device code for the target parallel loop directive.
llvm::function_ref< void(CodeGenFunction &, SourceLocation, const unsigned, const bool)> CodeGenOrderedTy
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
llvm::BasicBlock * OMPAfterScanBlock
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S)
void EmitOMPErrorDirective(const OMPErrorDirective &S)
void EmitOMPSectionDirective(const OMPSectionDirective &S)
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
void EmitOMPBarrierDirective(const OMPBarrierDirective &S)
void EmitStopPoint(const Stmt *S)
EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
void EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S)
void EmitStmt(const Stmt *S, ArrayRef< const Attr * > Attrs=std::nullopt)
EmitStmt - Emit the code for the statement.
void EnsureInsertPoint()
EnsureInsertPoint - Ensure that an insertion point is defined so that emitted IR has a place to go.
llvm::LLVMContext & getLLVMContext()
void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
void EmitOMPTargetTeamsDistributeParallelForDirective(const OMPTargetTeamsDistributeParallelForDirective &S)
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler's counter for the given statement by StepV.
bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for lastprivate variables.
void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S)
llvm::function_ref< void(CodeGenFunction &, const OMPLoopDirective &, JumpDest)> CodeGenLoopTy
void EmitOMPCanonicalLoop(const OMPCanonicalLoop *S)
Emit an OMPCanonicalLoop using the OpenMPIRBuilder.
void EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S)
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
void EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S)
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
This class organizes the cross-function state that is used while generating LLVM code.
void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
llvm::Module & getModule() const
CharUnits GetTargetTypeStoreSize(llvm::Type *Ty) const
Return the store size, in character units, of the given LLVM type.
DiagnosticsEngine & getDiags() const
const LangOptions & getLangOpts() const
CharUnits getNaturalTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, bool forPointeeType=false)
CodeGenTypes & getTypes()
const llvm::DataLayout & getDataLayout() const
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
llvm::Constant * GetAddrOfGlobal(GlobalDecl GD, ForDefinition_t IsForDefinition=NotForDefinition)
ASTContext & getContext() const
const CodeGenOptions & getCodeGenOpts() const
StringRef getMangledName(GlobalDecl GD)
llvm::Constant * getStaticLocalDeclAddress(const VarDecl *D)
llvm::ConstantInt * getSize(CharUnits numChars)
Emit the given number of characters as a value of type size_t.
void assignRegionCounters(GlobalDecl GD, llvm::Function *Fn)
Assign counters to regions and configure them for PGO of a given function.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
llvm::Type * ConvertTypeForMem(QualType T, bool ForBitField=false)
ConvertTypeForMem - Convert type T into a llvm::Type.
FunctionArgList - Type for representing both the decl and type of parameters to a function.
LValue - This represents an lvalue references.
Address getAddress(CodeGenFunction &CGF) const
llvm::Value * getPointer(CodeGenFunction &CGF) const
void setAddress(Address address)
void setUnrollCount(unsigned C)
Set the unroll count for the next loop pushed.
void setVectorizeWidth(unsigned W)
Set the vectorize width for the next loop pushed.
void setParallel(bool Enable=true)
Set the next pushed loop as parallel.
void setUnrollState(const LoopAttributes::LVEnableState &State)
Set the next pushed loop unroll state.
void pop()
End the current loop.
void push(llvm::BasicBlock *Header, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
Begin a new structured loop.
void setVectorizeEnable(bool Enable=true)
Set the next pushed loop 'vectorize.enable'.
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
virtual void Enter(CodeGenFunction &CGF)
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
static RValue get(llvm::Value *V)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
An abstract representation of an aligned address.
llvm::PointerType * getType() const
Return the type of the pointer value.
llvm::Value * getPointer() const
Class intended to support codegen of all kind of the reduction clauses.
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N)
Emits lvalue for the shared and original reduction item.
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
Complex values, per C99 6.2.5p11.
CompoundStmt - This represents a group of statements like { stmt stmt }.
ConstStmtVisitor - This class implements a simple visitor for Stmt subclasses.
DeclContext * getParent()
getParent - Returns the containing DeclContext.
A reference to a declared variable, function, enum, etc.
static DeclRefExpr * Create(const ASTContext &Context, NestedNameSpecifierLoc QualifierLoc, SourceLocation TemplateKWLoc, ValueDecl *D, bool RefersToEnclosingVariableOrCapture, SourceLocation NameLoc, QualType T, ExprValueKind VK, NamedDecl *FoundD=nullptr, const TemplateArgumentListInfo *TemplateArgs=nullptr, NonOdrUseReason NOUR=NOUR_None)
DeclStmt - Adaptor class for mixing declarations with statements and expressions.
Decl - This represents one declaration (or definition), e.g.
SourceLocation getBodyRBrace() const
getBodyRBrace - Gets the right brace of the body, if a body exists.
virtual bool hasBody() const
Returns true if this Decl represents a declaration for a body of code, such as a function or method d...
SourceLocation getLocation() const
The name of a declaration.
SourceLocation getBeginLoc() const LLVM_READONLY
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
unsigned getCustomDiagID(Level L, const char(&FormatString)[N])
Return an ID for a diagnostic with the specified format string and level.
This represents one expression.
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer,...
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
Expr * IgnoreImplicitAsWritten() LLVM_READONLY
Skip past any implicit AST nodes which might surround this expression until reaching a fixed point.
Expr * IgnoreImpCasts() LLVM_READONLY
Skip past any implicit casts which might surround this expression until reaching a fixed point.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Represents difference between two FPOptions values.
Represents a member of a struct/union/class.
Represents a function declaration or definition.
static FunctionDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation NLoc, DeclarationName N, QualType T, TypeSourceInfo *TInfo, StorageClass SC, bool UsesFPIntrin=false, bool isInlineSpecified=false, bool hasWrittenPrototype=true, ConstexprSpecKind ConstexprKind=ConstexprSpecKind::Unspecified, Expr *TrailingRequiresClause=nullptr)
GlobalDecl - represents a global declaration.
One of these records is kept for each identifier that is lexed.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
static ImplicitCastExpr * Create(const ASTContext &Context, QualType T, CastKind Kind, Expr *Operand, const CXXCastPath *BasePath, ExprValueKind Cat, FPOptionsOverride FPO)
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
A C++ lambda expression, which produces a function object (of unspecified type) that can be invoked l...
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
Represents a point when we exit a loop.
IdentifierInfo * getIdentifier() const
Get the identifier that names this declaration, if there is one.
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
A C++ nested-name-specifier augmented with source location information.
This represents 'acq_rel' clause in the '#pragma omp atomic|flush' directives.
This represents 'acquire' clause in the '#pragma omp atomic|flush' directives.
This represents clause 'aligned' in the '#pragma omp ...' directives.
This represents '#pragma omp atomic' directive.
This represents '#pragma omp barrier' directive.
This represents '#pragma omp cancel' directive.
This represents '#pragma omp cancellation point' directive.
Representation of an OpenMP canonical loop.
static OMPClauseWithPreInit * get(OMPClause *C)
This is a basic class for representing single OpenMP clause.
This represents clause 'copyin' in the '#pragma omp ...' directives.
This represents clause 'copyprivate' in the '#pragma omp ...' directives.
This represents '#pragma omp critical' directive.
This represents implicit clause 'depend' for the '#pragma omp task' directive.
This represents implicit clause 'depobj' for the '#pragma omp depobj' directive.
This represents '#pragma omp depobj' directive.
This represents 'destroy' clause in the '#pragma omp depobj' directive or the '#pragma omp interop' d...
This represents 'device' clause in the '#pragma omp ...' directive.
This represents 'dist_schedule' clause in the '#pragma omp ...' directive.
This represents '#pragma omp distribute' directive.
This represents '#pragma omp distribute parallel for' composite directive.
This represents '#pragma omp distribute parallel for simd' composite directive.
This represents '#pragma omp distribute simd' composite directive.
This represents the 'doacross' clause for the '#pragma omp ordered' directive.
This represents '#pragma omp error' directive.
This is a basic class for representing single OpenMP executable directive.
CapturedStmt * getInnermostCapturedStmt()
Get innermost captured statement for the construct.
bool hasAssociatedStmt() const
Returns true if directive has associated statement.
OpenMPDirectiveKind getDirectiveKind() const
const Stmt * getAssociatedStmt() const
Returns statement associated with the directive.
SourceLocation getBeginLoc() const
Returns starting location of directive kind.
SourceLocation getEndLoc() const
Returns ending location of directive.
static const SpecificClause * getSingleClause(ArrayRef< OMPClause * > Clauses)
Gets a single clause of the specified kind associated with the current directive iff there is only on...
static llvm::iterator_range< specific_clause_iterator< SpecificClause > > getClausesOfKind(ArrayRef< OMPClause * > Clauses)
This represents 'fail' clause in the '#pragma omp atomic' directive.
OpenMPClauseKind getFailParameter() const
Gets the parameter (type memory-order-clause) in Fail clause.
This represents 'filter' clause in the '#pragma omp ...' directive.
This represents 'final' clause in the '#pragma omp ...' directive.
This represents clause 'firstprivate' in the '#pragma omp ...' directives.
This represents implicit clause 'flush' for the '#pragma omp flush' directive.
This represents '#pragma omp flush' directive.
This represents '#pragma omp for' directive.
This represents '#pragma omp for simd' directive.
Representation of the 'full' clause of the '#pragma omp unroll' directive.
This represents '#pragma omp loop' directive.
This represents 'grainsize' clause in the '#pragma omp ...' directive.
This represents 'hint' clause in the '#pragma omp ...' directive.
This represents 'if' clause in the '#pragma omp ...' directive.
This represents clause 'in_reduction' in the '#pragma omp task' directives.
This represents clause 'inclusive' in the '#pragma omp scan' directive.
This represents the 'init' clause in '#pragma omp ...' directives.
This represents '#pragma omp interop' directive.
This represents clause 'lastprivate' in the '#pragma omp ...' directives.
This represents clause 'linear' in the '#pragma omp ...' directives.
The base class for all loop-based directives, including loop transformation directives.
static Stmt * tryToFindNextInnerLoop(Stmt *CurStmt, bool TryImperfectlyNestedLoops)
Try to find the next loop sub-statement in the specified statement CurStmt.
static bool doForAllLoops(Stmt *CurStmt, bool TryImperfectlyNestedLoops, unsigned NumLoops, llvm::function_ref< bool(unsigned, Stmt *)> Callback, llvm::function_ref< void(OMPLoopTransformationDirective *)> OnTransformationCallback)
Calls the specified callback function for all the loops in CurStmt, from the outermost to the innermo...
unsigned getLoopsNumber() const
Get number of collapsed loops.
This is a common base class for loop directives ('omp simd', 'omp for', 'omp for simd' etc....
ArrayRef< Expr * > finals_conditions()
Expr * getPrevUpperBoundVariable() const
ArrayRef< Expr * > counters()
Expr * getNumIterations() const
ArrayRef< Expr * > finals()
Expr * getPrevLowerBoundVariable() const
Expr * getIterationVariable() const
Expr * getUpperBoundVariable() const
ArrayRef< Expr * > updates()
Expr * getLowerBoundVariable() const
ArrayRef< Expr * > private_counters()
This represents '#pragma omp masked' directive.
This represents '#pragma omp master' directive.
This represents '#pragma omp master taskloop' directive.
This represents '#pragma omp master taskloop simd' directive.
This represents 'message' clause in the '#pragma omp error' directive.
Expr * getMessageString() const
Returns message string of the clause.
This represents 'nogroup' clause in the '#pragma omp ...' directive.
This represents 'nowait' clause in the '#pragma omp ...' directive.
This represents 'num_tasks' clause in the '#pragma omp ...' directive.
This represents 'num_teams' clause in the '#pragma omp ...' directive.
This represents 'num_threads' clause in the '#pragma omp ...' directive.
This represents 'order' clause in the '#pragma omp ...' directive.
This represents 'ordered' clause in the '#pragma omp ...' directive.
This represents '#pragma omp ordered' directive.
This represents '#pragma omp parallel' directive.
This represents '#pragma omp parallel for' directive.
This represents '#pragma omp parallel for simd' directive.
This represents '#pragma omp parallel masked' directive.
This represents '#pragma omp parallel master' directive.
This represents '#pragma omp parallel master taskloop' directive.
This represents '#pragma omp parallel master taskloop simd' directive.
This represents '#pragma omp parallel sections' directive.
Representation of the 'partial' clause of the '#pragma omp unroll' directive.
This represents 'priority' clause in the '#pragma omp ...' directive.
This represents clause 'private' in the '#pragma omp ...' directives.
This represents 'proc_bind' clause in the '#pragma omp ...' directive.
This represents clause 'reduction' in the '#pragma omp ...' directives.
This represents 'relaxed' clause in the '#pragma omp atomic' directives.
This represents 'release' clause in the '#pragma omp atomic|flush' directives.
This represents 'simd' clause in the '#pragma omp ...' directive.
This represents 'safelen' clause in the '#pragma omp ...' directive.
This represents '#pragma omp scan' directive.
This represents 'schedule' clause in the '#pragma omp ...' directive.
This represents '#pragma omp section' directive.
This represents '#pragma omp sections' directive.
This represents 'seq_cst' clause in the '#pragma omp atomic' directive.
This represents 'severity' clause in the '#pragma omp error' directive.
OpenMPSeverityClauseKind getSeverityKind() const
Returns kind of the clause.
This represents '#pragma omp simd' directive.
This represents 'simdlen' clause in the '#pragma omp ...' directive.
This represents '#pragma omp single' directive.
This represents '#pragma omp target data' directive.
This represents '#pragma omp target' directive.
This represents '#pragma omp target enter data' directive.
This represents '#pragma omp target exit data' directive.
This represents '#pragma omp target parallel' directive.
This represents '#pragma omp target parallel for' directive.
This represents '#pragma omp target parallel for simd' directive.
This represents '#pragma omp target parallel loop' directive.
This represents '#pragma omp target simd' directive.
This represents '#pragma omp target teams' directive.
This represents '#pragma omp target teams distribute' combined directive.
This represents '#pragma omp target teams distribute parallel for' combined directive.
This represents '#pragma omp target teams distribute parallel for simd' combined directive.
This represents '#pragma omp target teams distribute simd' combined directive.
This represents '#pragma omp target teams loop' directive.
This represents '#pragma omp target update' directive.
This represents '#pragma omp task' directive.
This represents '#pragma omp taskloop' directive.
This represents '#pragma omp taskloop simd' directive.
This represents clause 'task_reduction' in the '#pragma omp taskgroup' directives.
This represents '#pragma omp taskgroup' directive.
This represents '#pragma omp taskwait' directive.
This represents '#pragma omp taskyield' directive.
This represents '#pragma omp teams' directive.
This represents '#pragma omp teams distribute' directive.
This represents '#pragma omp teams distribute parallel for' composite directive.
This represents '#pragma omp teams distribute parallel for simd' composite directive.
This represents '#pragma omp teams distribute simd' combined directive.
This represents '#pragma omp teams loop' directive.
This represents 'thread_limit' clause in the '#pragma omp ...' directive.
This represents the '#pragma omp tile' loop transformation directive.
This represents the '#pragma omp unroll' loop transformation directive.
This represents 'untied' clause in the '#pragma omp ...' directive.
This represents 'update' clause in the '#pragma omp atomic' directive.
This represents the 'use' clause in '#pragma omp ...' directives.
This represents clause 'use_device_addr' in the '#pragma omp ...' directives.
This represents clause 'use_device_ptr' in the '#pragma omp ...' directives.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class.
static ParmVarDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, const IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, StorageClass S, Expr *DefArg)
PointerType - C99 6.7.5.1 - Pointer Declarators.
Represents an unpacked "presumed" location which can be presented to the user.
const char * getFilename() const
Return the presumed filename of this location.
unsigned getLine() const
Return the presumed line number of this location.
If a crash happens while one of these objects are live, the message is printed out along with the spe...
A (possibly-)qualified type.
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Represents a struct/union/class.
field_range fields() const
field_iterator field_begin() const
Base for LValueReferenceType and RValueReferenceType.
Scope - A scope is a transient data structure that is used while parsing the program.
Encodes a location in the source.
A trivial tuple used to represent a source range.
SourceLocation getEnd() const
SourceLocation getBegin() const
Stmt - This represents one statement.
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top,...
SourceLocation getBeginLoc() const LLVM_READONLY
SwitchStmt - This represents a 'switch' stmt.
virtual bool hasBuiltinAtomic(uint64_t AtomicSizeInBits, uint64_t AlignmentInBits) const
Returns true if the given target supports lock-free atomic operations at the specified width and alig...
bool isTLSSupported() const
Whether the target supports thread-local storage.
The base class of the type hierarchy.
bool isPointerType() const
const T * castAs() const
Member-template castAs<specific type>.
bool isReferenceType() const
bool isScalarType() const
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
bool isLValueReferenceType() const
bool isAnyComplexType() const
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g....
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type.
bool isAnyPointerType() const
UnaryOperator - This represents the unary-expression's (except sizeof and alignof),...
static UnaryOperator * Create(const ASTContext &C, Expr *input, Opcode opc, QualType type, ExprValueKind VK, ExprObjectKind OK, SourceLocation l, bool CanOverflow, FPOptionsOverride FPFeatures)
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Represents a variable declaration or definition.
TLSKind getTLSKind() const
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
@ CInit
C-style initialization with assignment.
bool hasGlobalStorage() const
Returns true for all variables that do not have local storage.
bool isStaticLocal() const
Returns true if a variable with function scope is a static local variable.
const Expr * getInit() const
bool hasLocalStorage() const
Returns true if a variable with function scope is a non-static local variable.
@ TLS_None
Not a TLS variable.
Represents a C array with a specified size that is not an integer-constant-expression.
Expr * getSizeExpr() const
@ Decl
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
bool Inc(InterpState &S, CodePtr OpPC)
1) Pops a pointer from the stack 2) Load the value from the pointer 3) Writes the value increased by ...
The JSON file list parser is used to communicate input to InstallAPI.
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
@ Tile
'tile' clause, allowed on 'loop' and Combined constructs.
bool needsTaskBasedThreadLimit(OpenMPDirectiveKind DKind)
Checks if the specified target directive, combined or not, needs task based thread_limit.
@ Ctor_Complete
Complete object ctor.
if(T->getSizeExpr()) TRY_TO(TraverseStmt(const_cast< Expr * >(T -> getSizeExpr())))
llvm::omp::Directive OpenMPDirectiveKind
OpenMP directives.
@ OK_Ordinary
An ordinary object is located at an address in memory.
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
@ OMPC_SCHEDULE_MODIFIER_unknown
llvm::omp::Clause OpenMPClauseKind
OpenMP clauses.
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
OpenMPDistScheduleClauseKind
OpenMP attributes for 'dist_schedule' clause.
@ OMPC_DIST_SCHEDULE_unknown
bool isOpenMPTaskingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of tasking directives - task, taskloop,...
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
@ Result
The result type of a method or function.
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
OpenMPDependClauseKind
OpenMP attributes for 'depend' clause.
@ Dtor_Complete
Complete object dtor.
bool isOpenMPLoopBoundSharingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of the composite or combined directives that need loop ...
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
@ VK_PRValue
A pr-value expression (in the C++11 taxonomy) produces a temporary value.
@ VK_LValue
An l-value expression is a reference to an object with independent storage.
const FunctionProtoType * T
void getOpenMPCaptureRegions(llvm::SmallVectorImpl< OpenMPDirectiveKind > &CaptureRegions, OpenMPDirectiveKind DKind)
Return the captured regions of an OpenMP directive.
@ ThreadPrivateVar
Parameter for Thread private variable.
@ Other
Other implicit parameter.
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
Diagnostic wrappers for TextAPI types for error reporting.
static Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc)
Returns address of the threadprivate variable for the current thread.
llvm::OpenMPIRBuilder::InsertPointTy InsertPointTy
static void EmitOMPOutlinedRegionBody(CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Twine RegionName)
Emit the body of an OMP region that will be outlined in OpenMPIRBuilder::finalize().
static Address getAddressOfLocalVariable(CodeGenFunction &CGF, const VarDecl *VD)
Gets the OpenMP-specific address of the local variable /p VD.
static void EmitCaptureStmt(CodeGenFunction &CGF, InsertPointTy CodeGenIP, llvm::BasicBlock &FiniBB, llvm::Function *Fn, ArrayRef< llvm::Value * > Args)
static std::string getNameWithSeparators(ArrayRef< StringRef > Parts, StringRef FirstSeparator=".", StringRef Separator=".")
Get the platform-specific name separator.
static void FinalizeOMPRegion(CodeGenFunction &CGF, InsertPointTy IP)
Emit the Finalization for an OMP region.
static void EmitOMPInlinedRegionBody(CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Twine RegionName)
Emit the body of an OMP region.
llvm::PointerType * VoidPtrTy
llvm::IntegerType * Int64Ty
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::IntegerType * SizeTy
llvm::IntegerType * Int32Ty
llvm::IntegerType * IntPtrTy
llvm::PointerType * Int8PtrTy
SmallVector< const Expr *, 4 > DepExprs
EvalResult is a struct with detailed info about an evaluated expression.
Extra information about a function prototype.
Scheduling data for loop-based OpenMP directives.
OpenMPScheduleClauseModifier M2
OpenMPScheduleClauseModifier M1
OpenMPScheduleClauseKind Schedule