clang 24.0.0git
CGStmtOpenMP.cpp
Go to the documentation of this file.
1//===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit OpenMP nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCleanup.h"
14#include "CGDebugInfo.h"
15#include "CGOpenMPRuntime.h"
16#include "CodeGenFunction.h"
17#include "CodeGenModule.h"
18#include "CodeGenPGO.h"
19#include "TargetInfo.h"
21#include "clang/AST/Attr.h"
24#include "clang/AST/Stmt.h"
31#include "llvm/ADT/SmallSet.h"
32#include "llvm/BinaryFormat/Dwarf.h"
33#include "llvm/Frontend/OpenMP/OMPConstants.h"
34#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DebugInfoMetadata.h"
37#include "llvm/IR/Instructions.h"
38#include "llvm/IR/Metadata.h"
39#include "llvm/Support/AtomicOrdering.h"
40#include "llvm/Support/Debug.h"
41#include <optional>
42using namespace clang;
43using namespace CodeGen;
44using namespace llvm::omp;
45
46#define TTL_CODEGEN_TYPE "target-teams-loop-codegen"
47
48static const VarDecl *getBaseDecl(const Expr *Ref);
51
52/// Whether a combined `distribute parallel for` may use the fused
53/// distr_static_chunk + static_chunkone schedule (enum 93): one
54/// for_static_init, no surrounding distribute_static_init.
56 const OMPLoopDirective &S,
57 OpenMPDirectiveKind DKind) {
58 // 'teams loop' is always emitted as 'distribute', and 'target teams loop'
59 // only becomes 'distribute parallel for' if canBeParallelFor() holds.
60 // Without the inner worksharing loop, there is nothing that would schedule
61 // the iteration space if the outer distribute loop is omitted.
62 if (DKind == OMPD_teams_loop)
63 return false;
64 if (const auto *TTLD = dyn_cast<OMPTargetTeamsGenericLoopDirective>(&S);
65 TTLD && !TTLD->canBeParallelFor())
66 return false;
67 // Reduction-only for now. Non-reduction cases might follow in the future, but
68 // need more analysis for maximum profit.
69 return CGM.getLangOpts().OpenMPIsTargetDevice && CGM.getTriple().isGPU() &&
71 S.hasClausesOfKind<OMPReductionClause>() &&
72 !S.getSingleClause<OMPDistScheduleClause>() &&
73 !S.getSingleClause<OMPScheduleClause>() &&
74 !S.getSingleClause<OMPOrderedClause>();
75}
76
77namespace {
78/// Lexical scope for OpenMP executable constructs, that handles correct codegen
79/// for captured expressions.
80class OMPLexicalScope : public CodeGenFunction::LexicalScope {
81 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
82 for (const auto *C : S.clauses()) {
83 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
84 if (const auto *PreInit =
85 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
86 for (const auto *I : PreInit->decls()) {
87 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
89 } else {
90 CodeGenFunction::AutoVarEmission Emission =
92 CGF.EmitAutoVarCleanups(Emission);
93 }
94 }
95 }
96 }
97 }
98 }
99 CodeGenFunction::OMPPrivateScope InlinedShareds;
100
101 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
102 return CGF.LambdaCaptureFields.lookup(VD) ||
103 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
104 (isa_and_nonnull<BlockDecl>(CGF.CurCodeDecl) &&
105 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
106 }
107
108public:
109 OMPLexicalScope(
110 CodeGenFunction &CGF, const OMPExecutableDirective &S,
111 const std::optional<OpenMPDirectiveKind> CapturedRegion = std::nullopt,
112 const bool EmitPreInitStmt = true)
113 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
114 InlinedShareds(CGF) {
115 if (EmitPreInitStmt)
116 emitPreInitStmt(CGF, S);
117 if (!CapturedRegion)
118 return;
119 assert(S.hasAssociatedStmt() &&
120 "Expected associated statement for inlined directive.");
121 const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
122 for (const auto &C : CS->captures()) {
123 if (C.capturesVariable() || C.capturesVariableByCopy()) {
124 auto *VD = C.getCapturedVar();
125 assert(VD == VD->getCanonicalDecl() &&
126 "Canonical decl must be captured.");
127 DeclRefExpr DRE(
128 CGF.getContext(), const_cast<VarDecl *>(VD),
129 isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
130 InlinedShareds.isGlobalVarCaptured(VD)),
131 VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
132 InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress());
133 }
134 }
135 (void)InlinedShareds.Privatize();
136 }
137};
138
139/// Lexical scope for OpenMP parallel construct, that handles correct codegen
140/// for captured expressions.
141class OMPParallelScope final : public OMPLexicalScope {
142 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
144 return !(isOpenMPTargetExecutionDirective(EKind) ||
147 }
148
149public:
150 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
151 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt,
152 EmitPreInitStmt(S)) {}
153};
154
155/// Lexical scope for OpenMP teams construct, that handles correct codegen
156/// for captured expressions.
157class OMPTeamsScope final : public OMPLexicalScope {
158 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
160 return !isOpenMPTargetExecutionDirective(EKind) &&
162 }
163
164public:
165 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
166 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt,
167 EmitPreInitStmt(S)) {}
168};
169
170/// Private scope for OpenMP loop-based directives, that supports capturing
171/// of used expression from loop statement.
172class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
173 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopBasedDirective &S) {
174 const Stmt *PreInits;
175 CodeGenFunction::OMPMapVars PreCondVars;
176 SmallVector<const Decl *, 4> DoacrossOnlyDecls;
177 if (auto *LD = dyn_cast<OMPLoopDirective>(&S)) {
178 // Emit init, __range, __begin and __end variables for C++ range loops.
179 (void)OMPLoopBasedDirective::doForAllLoops(
180 LD->getInnermostCapturedStmt()->getCapturedStmt(),
181 /*TryImperfectlyNestedLoops=*/true, LD->getLoopsNumber(),
182 [&CGF](unsigned Cnt, const Stmt *CurStmt) {
183 if (const auto *CXXFor = dyn_cast<CXXForRangeStmt>(CurStmt)) {
184 if (const Stmt *Init = CXXFor->getInit())
185 CGF.EmitStmt(Init);
186 CGF.EmitStmt(CXXFor->getRangeStmt());
187 CGF.EmitStmt(CXXFor->getBeginStmt());
188 CGF.EmitStmt(CXXFor->getEndStmt());
189 }
190 return false;
191 });
192 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
193 for (const auto *E : LD->counters()) {
194 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
195 EmittedAsPrivate.insert(VD->getCanonicalDecl());
196 (void)PreCondVars.setVarAddr(
197 CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
198 }
199 // Mark private vars as undefs.
200 for (const auto *C : LD->getClausesOfKind<OMPPrivateClause>()) {
201 for (const Expr *IRef : C->varlist()) {
202 const auto *OrigDecl = cast<DeclRefExpr>(IRef)->getDecl();
203 const auto *OrigVD = dyn_cast<VarDecl>(OrigDecl);
204 if (!OrigVD)
205 continue;
206 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
207 QualType OrigVDTy = OrigVD->getType().getNonReferenceType();
208 (void)PreCondVars.setVarAddr(
209 CGF, OrigVD,
210 Address(llvm::UndefValue::get(CGF.ConvertTypeForMem(
211 CGF.getContext().getPointerType(OrigVDTy))),
212 CGF.ConvertTypeForMem(OrigVDTy),
213 CGF.getContext().getDeclAlign(OrigVD)));
214 }
215 }
216 }
217 (void)PreCondVars.apply(CGF);
218 // The pre-inits also use the bounds of the loops that an 'ordered(n)'
219 // clause counts beyond the collapsed ones. Such a loop is emitted as a
220 // regular loop in the body and declares its own helpers, so these copies
221 // only live until the pre-inits.
222 if (const auto *C = LD->getSingleClause<OMPOrderedClause>())
223 (void)OMPLoopBasedDirective::doForAllLoops(
224 LD->getInnermostCapturedStmt()->getCapturedStmt(),
225 /*TryImperfectlyNestedLoops=*/true,
226 C->getLoopNumIterations().size(),
227 [&CGF, LD, &DoacrossOnlyDecls](unsigned Cnt, const Stmt *CurStmt) {
228 const auto *CXXFor = dyn_cast<CXXForRangeStmt>(CurStmt);
229 if (!CXXFor || Cnt < LD->getLoopsNumber())
230 return false;
231 for (const Stmt *Helper : std::initializer_list<const Stmt *>{
232 CXXFor->getInit(), CXXFor->getRangeStmt(),
233 CXXFor->getBeginStmt(), CXXFor->getEndStmt()}) {
234 if (!Helper)
235 continue;
236 CGF.EmitStmt(Helper);
237 if (const auto *DS = dyn_cast<DeclStmt>(Helper))
238 llvm::append_range(DoacrossOnlyDecls, DS->decls());
239 }
240 return false;
241 });
242 PreInits = LD->getPreInits();
243 } else if (const auto *Tile = dyn_cast<OMPTileDirective>(&S)) {
244 PreInits = Tile->getPreInits();
245 } else if (const auto *Stripe = dyn_cast<OMPStripeDirective>(&S)) {
246 PreInits = Stripe->getPreInits();
247 } else if (const auto *Unroll = dyn_cast<OMPUnrollDirective>(&S)) {
248 PreInits = Unroll->getPreInits();
249 } else if (const auto *Reverse = dyn_cast<OMPReverseDirective>(&S)) {
250 PreInits = Reverse->getPreInits();
251 } else if (const auto *Split = dyn_cast<OMPSplitDirective>(&S)) {
252 PreInits = Split->getPreInits();
253 } else if (const auto *Interchange =
254 dyn_cast<OMPInterchangeDirective>(&S)) {
255 PreInits = Interchange->getPreInits();
256 } else if (const auto *Flatten = dyn_cast<OMPFlattenDirective>(&S)) {
257 PreInits = Flatten->getPreInits();
258 } else {
259 llvm_unreachable("Unknown loop-based directive kind.");
260 }
261 doEmitPreinits(PreInits);
262 for (const Decl *D : DoacrossOnlyDecls)
263 CGF.eraseLocalDecl(D);
264 PreCondVars.restore(CGF);
265 }
266
267 void
268 emitPreInitStmt(CodeGenFunction &CGF,
270 const Stmt *PreInits;
271 if (const auto *Fuse = dyn_cast<OMPFuseDirective>(&S)) {
272 PreInits = Fuse->getPreInits();
273 } else {
274 llvm_unreachable(
275 "Unknown canonical loop sequence transform directive kind.");
276 }
277 doEmitPreinits(PreInits);
278 }
279
280 void doEmitPreinits(const Stmt *PreInits) {
281 if (PreInits) {
282 // CompoundStmts and DeclStmts are used as lists of PreInit statements and
283 // declarations. Since declarations must be visible in the the following
284 // that they initialize, unpack the CompoundStmt they are nested in.
285 SmallVector<const Stmt *> PreInitStmts;
286 if (auto *PreInitCompound = dyn_cast<CompoundStmt>(PreInits))
287 llvm::append_range(PreInitStmts, PreInitCompound->body());
288 else
289 PreInitStmts.push_back(PreInits);
290
291 for (const Stmt *S : PreInitStmts) {
292 // EmitStmt skips any OMPCapturedExprDecls, but needs to be emitted
293 // here.
294 if (auto *PreInitDecl = dyn_cast<DeclStmt>(S)) {
295 for (Decl *I : PreInitDecl->decls())
296 CGF.EmitVarDecl(cast<VarDecl>(*I));
297 continue;
298 }
299 CGF.EmitStmt(S);
300 }
301 }
302 }
303
304public:
305 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopBasedDirective &S)
306 : CodeGenFunction::RunCleanupsScope(CGF) {
307 emitPreInitStmt(CGF, S);
308 }
309 OMPLoopScope(CodeGenFunction &CGF,
311 : CodeGenFunction::RunCleanupsScope(CGF) {
312 emitPreInitStmt(CGF, S);
313 }
314};
315
316class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
317 CodeGenFunction::OMPPrivateScope InlinedShareds;
318
319 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
320 return CGF.LambdaCaptureFields.lookup(VD) ||
321 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
322 (isa_and_nonnull<BlockDecl>(CGF.CurCodeDecl) &&
323 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
324 }
325
326public:
327 OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
328 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
329 InlinedShareds(CGF) {
330 for (const auto *C : S.clauses()) {
331 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
332 if (const auto *PreInit =
333 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
334 for (const auto *I : PreInit->decls()) {
335 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
336 CGF.EmitVarDecl(cast<VarDecl>(*I));
337 } else {
338 CodeGenFunction::AutoVarEmission Emission =
339 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
340 CGF.EmitAutoVarCleanups(Emission);
341 }
342 }
343 }
344 } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
345 for (const Expr *E : UDP->varlist()) {
346 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
347 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
348 CGF.EmitVarDecl(*OED);
349 }
350 } else if (const auto *UDP = dyn_cast<OMPUseDeviceAddrClause>(C)) {
351 for (const Expr *E : UDP->varlist()) {
352 const Decl *D = getBaseDecl(E);
353 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
354 CGF.EmitVarDecl(*OED);
355 }
356 }
357 }
358 if (!isOpenMPSimdDirective(S.getDirectiveKind()))
359 CGF.EmitOMPPrivateClause(S, InlinedShareds);
360 if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
361 if (const Expr *E = TG->getReductionRef())
362 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
363 }
364 // Temp copy arrays for inscan reductions should not be emitted as they are
365 // not used in simd only mode.
366 llvm::DenseSet<CanonicalDeclPtr<const Decl>> CopyArrayTemps;
367 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
368 if (C->getModifier() != OMPC_REDUCTION_inscan)
369 continue;
370 for (const Expr *E : C->copy_array_temps())
371 CopyArrayTemps.insert(cast<DeclRefExpr>(E)->getDecl());
372 }
373 const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
374 while (CS) {
375 for (auto &C : CS->captures()) {
376 if (C.capturesVariable() || C.capturesVariableByCopy()) {
377 auto *VD = C.getCapturedVar();
378 if (CopyArrayTemps.contains(VD))
379 continue;
380 assert(VD == VD->getCanonicalDecl() &&
381 "Canonical decl must be captured.");
382 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
383 isCapturedVar(CGF, VD) ||
384 (CGF.CapturedStmtInfo &&
385 InlinedShareds.isGlobalVarCaptured(VD)),
387 C.getLocation());
388 InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress());
389 }
390 }
391 CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
392 }
393 (void)InlinedShareds.Privatize();
394 }
395};
396
397} // namespace
398
399// Map `loop` with a bind clause, `teams loop`, `target loop`, and
400// `target teams loop` to the construct CodeGen emits.
403 OpenMPDirectiveKind Kind = S.getDirectiveKind();
404
405 if (Kind == OMPD_teams_loop)
406 return OMPD_teams_distribute;
407 if (Kind == OMPD_target_loop)
408 return OMPD_target_simd;
409 if (Kind == OMPD_target_teams_loop)
410 return cast<OMPTargetTeamsGenericLoopDirective>(S).canBeParallelFor()
411 ? OMPD_target_teams_distribute_parallel_for
412 : OMPD_target_teams_distribute;
413
414 if (Kind != OMPD_loop)
415 return Kind;
416
418 if (const auto *C = S.getSingleClause<OMPBindClause>())
419 BindKind = C->getBindKind();
420
421 switch (BindKind) {
422 case OMPC_BIND_parallel:
423 return OMPD_for;
424 case OMPC_BIND_teams:
425 return OMPD_distribute;
426 case OMPC_BIND_thread:
427 return OMPD_simd;
428 default:
429 return OMPD_loop;
430 }
431}
432
434 const OMPExecutableDirective &S,
435 const RegionCodeGenTy &CodeGen);
436
438 SourceLocation Loc) {
439 if (CapturedStmtInfo &&
441 if (const auto *DD = dyn_cast<VarDecl>(BD->getDecomposedDecl())) {
442 if (CapturedStmtInfo->lookup(DD))
444 }
445 }
446 DeclRefExpr DRE(getContext(), const_cast<BindingDecl *>(BD),
447 /*RefersToEnclosingVariableOrCapture=*/false, BD->getType(),
448 VK_LValue, Loc);
449 return EmitLValue(&DRE).getAddress();
450}
451
453 if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
454 if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
455 OrigVD = OrigVD->getCanonicalDecl();
456 bool IsCaptured =
457 LambdaCaptureFields.lookup(OrigVD) ||
458 (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
459 (isa_and_nonnull<BlockDecl>(CurCodeDecl));
460 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), IsCaptured,
461 OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
462 return EmitLValue(&DRE);
463 }
464 if (const auto *OrigBD = dyn_cast<BindingDecl>(OrigDRE->getDecl())) {
465 OrigBD = cast<BindingDecl>(OrigBD->getCanonicalDecl());
466 const auto *DD = cast<VarDecl>(OrigBD->getDecomposedDecl());
467 bool IsCaptured = LambdaCaptureFields.lookup(OrigBD) ||
468 (CapturedStmtInfo && CapturedStmtInfo->lookup(DD)) ||
469 isa_and_nonnull<BlockDecl>(CurCodeDecl);
470 DeclRefExpr DRE(getContext(), const_cast<BindingDecl *>(OrigBD),
471 IsCaptured, OrigDRE->getType(), VK_LValue,
472 OrigDRE->getExprLoc());
473 return EmitLValue(&DRE);
474 }
475 }
476 return EmitLValue(E);
477}
478
481 llvm::Value *Size = nullptr;
482 auto SizeInChars = C.getTypeSizeInChars(Ty);
483 if (SizeInChars.isZero()) {
484 // getTypeSizeInChars() returns 0 for a VLA.
485 while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) {
486 VlaSizePair VlaSize = getVLASize(VAT);
487 Ty = VlaSize.Type;
488 Size =
489 Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts) : VlaSize.NumElts;
490 }
491 SizeInChars = C.getTypeSizeInChars(Ty);
492 if (SizeInChars.isZero())
493 return llvm::ConstantInt::get(SizeTy, /*V=*/0);
494 return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
495 }
496 return CGM.getSize(SizeInChars);
497}
498
500 const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
501 const RecordDecl *RD = S.getCapturedRecordDecl();
502 auto CurField = RD->field_begin();
503 auto CurCap = S.captures().begin();
505 E = S.capture_init_end();
506 I != E; ++I, ++CurField, ++CurCap) {
507 if (CurField->hasCapturedVLAType()) {
508 const VariableArrayType *VAT = CurField->getCapturedVLAType();
509 llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
510 CapturedVars.push_back(Val);
511 } else if (CurCap->capturesThis()) {
512 CapturedVars.push_back(CXXThisValue);
513 } else if (CurCap->capturesVariableByCopy()) {
514 llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
515
516 // If the field is not a pointer, we need to save the actual value
517 // and load it as a void pointer.
518 if (!CurField->getType()->isAnyPointerType()) {
519 ASTContext &Ctx = getContext();
521 Ctx.getUIntPtrType(),
522 Twine(CurCap->getCapturedVar()->getName(), ".casted"));
523 LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
524
525 llvm::Value *SrcAddrVal = EmitScalarConversion(
526 DstAddr.emitRawPointer(*this),
528 Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
529 LValue SrcLV =
530 MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
531
532 // Store the value using the source type pointer.
534
535 // Load the value using the destination type pointer.
536 CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
537 }
538 CapturedVars.push_back(CV);
539 } else {
540 assert(CurCap->capturesVariable() && "Expected capture by reference.");
541 llvm::Value *Addr = EmitLValue(*I).getAddress().emitRawPointer(*this);
542 // Sema strips the address space from the type of the captured field.
543 llvm::Type *ArgTy = ConvertType(CurField->getType());
544 if (Addr->getType() != ArgTy)
546 CapturedVars.push_back(Addr);
547 }
548 }
549}
550
552 QualType DstType, StringRef Name,
553 LValue AddrLV) {
554 ASTContext &Ctx = CGF.getContext();
555
556 llvm::Value *CastedPtr = CGF.EmitScalarConversion(
557 AddrLV.getAddress().emitRawPointer(CGF), Ctx.getUIntPtrType(),
558 Ctx.getPointerType(DstType), Loc);
559 // FIXME: should the pointee type (DstType) be passed?
560 Address TmpAddr =
561 CGF.MakeNaturalAlignAddrLValue(CastedPtr, DstType).getAddress();
562 return TmpAddr;
563}
564
566 if (T->isLValueReferenceType())
567 return C.getLValueReferenceType(
568 getCanonicalParamType(C, T.getNonReferenceType()),
569 /*SpelledAsLValue=*/false);
570 if (T->isPointerType())
571 return C.getPointerType(getCanonicalParamType(C, T->getPointeeType()));
572 if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
573 if (const auto *VLA = dyn_cast<VariableArrayType>(A))
574 return getCanonicalParamType(C, VLA->getElementType());
575 if (!A->isVariablyModifiedType())
576 return C.getCanonicalType(T);
577 }
578 return C.getCanonicalParamType(T);
579}
580
581namespace {
582/// Contains required data for proper outlined function codegen.
583struct FunctionOptions {
584 /// Captured statement for which the function is generated.
585 const CapturedStmt *S = nullptr;
586 /// true if cast to/from UIntPtr is required for variables captured by
587 /// value.
588 const bool UIntPtrCastRequired = true;
589 /// true if only casted arguments must be registered as local args or VLA
590 /// sizes.
591 const bool RegisterCastedArgsOnly = false;
592 /// Name of the generated function.
593 const StringRef FunctionName;
594 /// Location of the non-debug version of the outlined function.
595 SourceLocation Loc;
596 const bool IsDeviceKernel = false;
597 explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
598 bool RegisterCastedArgsOnly, StringRef FunctionName,
599 SourceLocation Loc, bool IsDeviceKernel)
600 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
601 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
602 FunctionName(FunctionName), Loc(Loc), IsDeviceKernel(IsDeviceKernel) {}
603};
604} // namespace
605
606static llvm::Function *emitOutlinedFunctionPrologue(
608 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
609 &LocalAddrs,
610 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
611 &VLASizes,
612 llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
613 const CapturedDecl *CD = FO.S->getCapturedDecl();
614 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
615 assert(CD->hasBody() && "missing CapturedDecl body");
616
617 CXXThisValue = nullptr;
618 // Build the argument list.
619 CodeGenModule &CGM = CGF.CGM;
620 ASTContext &Ctx = CGM.getContext();
621 FunctionArgList TargetArgs;
622 Args.append(CD->param_begin(),
623 std::next(CD->param_begin(), CD->getContextParamPosition()));
624 TargetArgs.append(
625 CD->param_begin(),
626 std::next(CD->param_begin(), CD->getContextParamPosition()));
627 auto I = FO.S->captures().begin();
628 FunctionDecl *DebugFunctionDecl = nullptr;
629 if (!FO.UIntPtrCastRequired) {
631 QualType FunctionTy = Ctx.getFunctionType(Ctx.VoidTy, {}, EPI);
632 DebugFunctionDecl = FunctionDecl::Create(
633 Ctx, Ctx.getTranslationUnitDecl(), FO.S->getBeginLoc(),
634 SourceLocation(), DeclarationName(), FunctionTy,
635 Ctx.getTrivialTypeSourceInfo(FunctionTy), SC_Static,
636 /*UsesFPIntrin=*/false, /*isInlineSpecified=*/false,
637 /*hasWrittenPrototype=*/false);
638 }
639 for (const FieldDecl *FD : RD->fields()) {
640 QualType ArgType = FD->getType();
641 IdentifierInfo *II = nullptr;
642 VarDecl *CapVar = nullptr;
643
644 // If this is a capture by copy and the type is not a pointer, the outlined
645 // function argument type should be uintptr and the value properly casted to
646 // uintptr. This is necessary given that the runtime library is only able to
647 // deal with pointers. We can pass in the same way the VLA type sizes to the
648 // outlined function.
649 if (FO.UIntPtrCastRequired &&
650 ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
651 I->capturesVariableArrayType()))
652 ArgType = Ctx.getUIntPtrType();
653
654 if (I->capturesVariable() || I->capturesVariableByCopy()) {
655 CapVar = I->getCapturedVar();
656 II = CapVar->getIdentifier();
657 } else if (I->capturesThis()) {
658 II = &Ctx.Idents.get("this");
659 } else {
660 assert(I->capturesVariableArrayType());
661 II = &Ctx.Idents.get("vla");
662 }
663 if (ArgType->isVariablyModifiedType())
664 ArgType = getCanonicalParamType(Ctx, ArgType);
665 VarDecl *Arg;
666 if (CapVar && (CapVar->getTLSKind() != clang::VarDecl::TLS_None)) {
667 Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
668 II, ArgType,
670 } else if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
672 Ctx, DebugFunctionDecl,
673 CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),
674 CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
675 /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
676 } else {
677 Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
678 II, ArgType, ImplicitParamKind::Other);
679 }
680 Args.emplace_back(Arg);
681 // Do not cast arguments if we emit function with non-original types.
682 TargetArgs.emplace_back(
683 FO.UIntPtrCastRequired
684 ? Arg
685 : CGM.getOpenMPRuntime().translateParameter(FD, Arg));
686 ++I;
687 }
688 Args.append(std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
689 CD->param_end());
690 TargetArgs.append(
691 std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
692 CD->param_end());
693
694 // Create the function declaration.
695 const CGFunctionInfo &FuncInfo =
696 FO.IsDeviceKernel
698 TargetArgs)
700 TargetArgs);
701 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
702
703 auto *F =
704 llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
705 FO.FunctionName, &CGM.getModule());
706 CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
707
708 // Adjust the calling convention for SPIR-V targets to avoid mismatches
709 // between callee and caller.
710 if (CGM.getTriple().isSPIRV() && !FO.IsDeviceKernel)
711 F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
712
713 if (CD->isNothrow())
714 F->setDoesNotThrow();
715
716 // Always inline the outlined function if optimizations are enabled.
717 if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
718 F->removeFnAttr(llvm::Attribute::NoInline);
719 F->addFnAttr(llvm::Attribute::AlwaysInline);
720 }
721 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
722 F->addFnAttr("sample-profile-suffix-elision-policy", "selected");
723
724 // Generate the function.
725 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
726 FO.UIntPtrCastRequired ? FO.Loc : FO.S->getBeginLoc(),
727 FO.UIntPtrCastRequired ? FO.Loc
728 : CD->getBody()->getBeginLoc());
729 unsigned Cnt = CD->getContextParamPosition();
730 I = FO.S->captures().begin();
731 for (const FieldDecl *FD : RD->fields()) {
732 // Do not map arguments if we emit function with non-original types.
733 Address LocalAddr(Address::invalid());
734 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
735 LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
736 TargetArgs[Cnt]);
737 } else {
738 LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
739 }
740 // If we are capturing a pointer by copy we don't need to do anything, just
741 // use the value that we get from the arguments.
742 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
743 const VarDecl *CurVD = I->getCapturedVar();
744 if (!FO.RegisterCastedArgsOnly)
745 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
746 ++Cnt;
747 ++I;
748 continue;
749 }
750
751 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
753 if (FD->hasCapturedVLAType()) {
754 if (FO.UIntPtrCastRequired) {
755 ArgLVal = CGF.MakeAddrLValue(
756 castValueFromUintptr(CGF, I->getLocation(), FD->getType(),
757 Args[Cnt]->getName(), ArgLVal),
759 }
760 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
761 const VariableArrayType *VAT = FD->getCapturedVLAType();
762 VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg);
763 } else if (I->capturesVariable()) {
764 const VarDecl *Var = I->getCapturedVar();
765 QualType VarTy = Var->getType();
766 Address ArgAddr = ArgLVal.getAddress();
767 if (ArgLVal.getType()->isLValueReferenceType()) {
768 ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
769 } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) {
770 assert(ArgLVal.getType()->isPointerType());
771 ArgAddr = CGF.EmitLoadOfPointer(
772 ArgAddr, ArgLVal.getType()->castAs<PointerType>());
773 }
774 if (!FO.RegisterCastedArgsOnly) {
775 LocalAddrs.insert(
776 {Args[Cnt], {Var, ArgAddr.withAlignment(Ctx.getDeclAlign(Var))}});
777 }
778 } else if (I->capturesVariableByCopy()) {
779 assert(!FD->getType()->isAnyPointerType() &&
780 "Not expecting a captured pointer.");
781 const VarDecl *Var = I->getCapturedVar();
782 LocalAddrs.insert({Args[Cnt],
783 {Var, FO.UIntPtrCastRequired
785 CGF, I->getLocation(), FD->getType(),
786 Args[Cnt]->getName(), ArgLVal)
787 : ArgLVal.getAddress()}});
788 } else {
789 // If 'this' is captured, load it into CXXThisValue.
790 assert(I->capturesThis());
791 CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
792 LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress()}});
793 }
794 ++Cnt;
795 ++I;
796 }
797
798 return F;
799}
800
803 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
804 &LocalAddrs,
805 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
806 &VLASizes,
807 llvm::Value *&CXXThisValue, llvm::Value *&ContextV, const CapturedStmt &CS,
808 SourceLocation Loc, StringRef FunctionName) {
809 const CapturedDecl *CD = CS.getCapturedDecl();
810 const RecordDecl *RD = CS.getCapturedRecordDecl();
811
812 CXXThisValue = nullptr;
813 CodeGenModule &CGM = CGF.CGM;
814 ASTContext &Ctx = CGM.getContext();
815 Args.push_back(CD->getContextParam());
816
817 const CGFunctionInfo &FuncInfo =
819 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
820
821 auto *F =
822 llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
823 FunctionName, &CGM.getModule());
824 CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
825 if (CD->isNothrow())
826 F->setDoesNotThrow();
827
828 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, Args, Loc, Loc);
829 Address ContextAddr = CGF.GetAddrOfLocalVar(CD->getContextParam());
830 ContextV = CGF.Builder.CreateLoad(ContextAddr);
831
832 // The runtime passes arguments as an array of pointers.
833 llvm::Type *PtrTy = CGF.Builder.getPtrTy();
834 llvm::Align PtrAlign = CGM.getDataLayout().getPointerABIAlignment(0);
835 CharUnits SlotAlign = CharUnits::fromQuantity(PtrAlign.value());
836
837 for (auto [FD, C, FieldIdx] :
838 llvm::zip(RD->fields(), CS.captures(),
839 llvm::seq<unsigned>(RD->getNumFields()))) {
840 llvm::Value *SlotPtr =
841 CGF.Builder.CreateConstInBoundsGEP1_32(PtrTy, ContextV, FieldIdx);
842 llvm::Value *Slot = CGF.Builder.CreateAlignedLoad(PtrTy, SlotPtr, PtrAlign);
843
844 // Generate the appropriate load from the per-argument storage. This
845 // includes all of the user arguments as well as the implicit kernel
846 // argument pointer.
847 if (C.capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
848 const VarDecl *CurVD = C.getCapturedVar();
849 Slot->setName(CurVD->getName());
850 Address SlotAddr(Slot, PtrTy, SlotAlign);
851 LocalAddrs.insert({FD, {CurVD, SlotAddr}});
852 } else if (FD->hasCapturedVLAType()) {
853 // VLA size is stored as intptr_t directly in the slot.
854 Address SlotAddr(Slot, CGF.ConvertTypeForMem(FD->getType()), SlotAlign);
855 LValue ArgLVal =
856 CGF.MakeAddrLValue(SlotAddr, FD->getType(), AlignmentSource::Decl);
857 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, C.getLocation());
858 const VariableArrayType *VAT = FD->getCapturedVLAType();
859 VLASizes.try_emplace(FD, VAT->getSizeExpr(), ExprArg);
860 } else if (C.capturesVariable()) {
861 const VarDecl *Var = C.getCapturedVar();
862 QualType VarTy = Var->getType();
863
864 if (VarTy->isVariablyModifiedType() && VarTy->isPointerType()) {
865 Slot->setName(Var->getName() + ".addr");
866 Address SlotAddr(Slot, PtrTy, SlotAlign);
867 LocalAddrs.insert({FD, {Var, SlotAddr}});
868 } else {
869 llvm::Value *VarAddr = CGF.Builder.CreateAlignedLoad(
870 PtrTy, Slot, PtrAlign, Var->getName());
871 LocalAddrs.insert({FD,
872 {Var, Address(VarAddr, CGF.ConvertTypeForMem(VarTy),
873 Ctx.getDeclAlign(Var))}});
874 }
875 } else if (C.capturesVariableByCopy()) {
876 assert(!FD->getType()->isAnyPointerType() &&
877 "Not expecting a captured pointer.");
878 const VarDecl *Var = C.getCapturedVar();
879 QualType FieldTy = FD->getType();
880
881 // Scalar values are promoted and stored directly in the slot.
882 Address SlotAddr(Slot, CGF.ConvertTypeForMem(FieldTy), SlotAlign);
883 Address CopyAddr =
884 CGF.CreateMemTemp(FieldTy, Ctx.getDeclAlign(FD), Var->getName());
885 LValue SrcLVal =
886 CGF.MakeAddrLValue(SlotAddr, FieldTy, AlignmentSource::Decl);
887 LValue CopyLVal =
888 CGF.MakeAddrLValue(CopyAddr, FieldTy, AlignmentSource::Decl);
889
890 RValue ArgRVal = CGF.EmitLoadOfLValue(SrcLVal, C.getLocation());
891 CGF.EmitStoreThroughLValue(ArgRVal, CopyLVal);
892
893 LocalAddrs.insert({FD, {Var, CopyAddr}});
894 } else {
895 assert(C.capturesThis() && "Default case expected to be CXX 'this'");
896 CXXThisValue =
897 CGF.Builder.CreateAlignedLoad(PtrTy, Slot, PtrAlign, "this");
898 Address SlotAddr(Slot, PtrTy, SlotAlign);
899 LocalAddrs.insert({FD, {nullptr, SlotAddr}});
900 }
901 }
902
903 return F;
904}
905
907 const CapturedStmt &S, const OMPExecutableDirective &D) {
908 SourceLocation Loc = D.getBeginLoc();
909 assert(
911 "CapturedStmtInfo should be set when generating the captured function");
912 const CapturedDecl *CD = S.getCapturedDecl();
913 // Build the argument list.
914 bool NeedWrapperFunction =
915 getDebugInfo() && CGM.getCodeGenOpts().hasReducedDebugInfo();
916 FunctionArgList Args, WrapperArgs;
917 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs,
918 WrapperLocalAddrs;
919 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes,
920 WrapperVLASizes;
921 SmallString<256> Buffer;
922 llvm::raw_svector_ostream Out(Buffer);
923 Out << CapturedStmtInfo->getHelperName();
925 bool IsDeviceKernel = CGM.getOpenMPRuntime().isGPU() &&
927 D.getCapturedStmt(OMPD_target) == &S;
928 CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
929 llvm::Function *WrapperF = nullptr;
930 if (NeedWrapperFunction) {
931 // Emit the final kernel early to allow attributes to be added by the
932 // OpenMPI-IR-Builder.
933 FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
934 /*RegisterCastedArgsOnly=*/true,
935 CapturedStmtInfo->getHelperName(), Loc,
936 IsDeviceKernel);
938 WrapperF =
939 emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
940 WrapperCGF.CXXThisValue, WrapperFO);
941 Out << "_debug__";
942 }
943 FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
944 Out.str(), Loc, !NeedWrapperFunction && IsDeviceKernel);
945 llvm::Function *F = emitOutlinedFunctionPrologue(
946 *this, WrapperArgs, WrapperLocalAddrs, WrapperVLASizes, CXXThisValue, FO);
947 CodeGenFunction::OMPPrivateScope LocalScope(*this);
948 for (const auto &LocalAddrPair : WrapperLocalAddrs) {
949 if (LocalAddrPair.second.first) {
950 LocalScope.addPrivate(LocalAddrPair.second.first,
951 LocalAddrPair.second.second);
952 }
953 }
954 (void)LocalScope.Privatize();
955 for (const auto &VLASizePair : WrapperVLASizes)
956 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
957 PGO->assignRegionCounters(GlobalDecl(CD), F);
958 CapturedStmtInfo->EmitBody(*this, CD->getBody());
959 LocalScope.ForceCleanup();
961 if (!NeedWrapperFunction)
962 return F;
963
964 // Reverse the order.
965 WrapperF->removeFromParent();
966 F->getParent()->getFunctionList().insertAfter(F->getIterator(), WrapperF);
967
969 auto *PI = F->arg_begin();
970 for (const auto *Arg : Args) {
971 llvm::Value *CallArg;
972 auto I = LocalAddrs.find(Arg);
973 if (I != LocalAddrs.end()) {
974 LValue LV = WrapperCGF.MakeAddrLValue(
975 I->second.second,
976 I->second.first ? I->second.first->getType() : Arg->getType(),
978 if (LV.getType()->isAnyComplexType())
979 LV.setAddress(LV.getAddress().withElementType(PI->getType()));
980 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
981 } else {
982 auto EI = VLASizes.find(Arg);
983 if (EI != VLASizes.end()) {
984 CallArg = EI->second.second;
985 } else {
986 LValue LV =
987 WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
989 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
990 }
991 }
992 CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
993 ++PI;
994 }
995 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, Loc, F, CallArgs);
996 WrapperCGF.FinishFunction();
997 return WrapperF;
998}
999
1001 const CapturedStmt &S, const OMPExecutableDirective &D) {
1002 SourceLocation Loc = D.getBeginLoc();
1003 assert(
1005 "CapturedStmtInfo should be set when generating the captured function");
1006 const CapturedDecl *CD = S.getCapturedDecl();
1007 const RecordDecl *RD = S.getCapturedRecordDecl();
1008 StringRef FunctionName = CapturedStmtInfo->getHelperName();
1009 bool NeedWrapperFunction =
1010 getDebugInfo() && CGM.getCodeGenOpts().hasReducedDebugInfo();
1011
1012 CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
1013 llvm::Function *WrapperF = nullptr;
1014 llvm::Value *WrapperContextV = nullptr;
1015 if (NeedWrapperFunction) {
1017 FunctionArgList WrapperArgs;
1018 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
1019 WrapperLocalAddrs;
1020 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
1021 WrapperVLASizes;
1023 WrapperCGF, WrapperArgs, WrapperLocalAddrs, WrapperVLASizes,
1024 WrapperCGF.CXXThisValue, WrapperContextV, S, Loc, FunctionName);
1025 }
1026
1027 FunctionArgList Args;
1028 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
1029 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
1030 llvm::Function *F;
1031
1032 if (NeedWrapperFunction) {
1033 SmallString<256> Buffer;
1034 llvm::raw_svector_ostream Out(Buffer);
1035 Out << FunctionName << "_debug__";
1036
1037 FunctionOptions FO(&S, /*UIntPtrCastRequired=*/false,
1038 /*RegisterCastedArgsOnly=*/false, Out.str(), Loc,
1039 /*IsDeviceKernel=*/false);
1040 F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs, VLASizes,
1041 CXXThisValue, FO);
1042 } else {
1043 llvm::Value *ContextV = nullptr;
1044 F = emitOutlinedFunctionPrologueAggregate(*this, Args, LocalAddrs, VLASizes,
1045 CXXThisValue, ContextV, S, Loc,
1046 FunctionName);
1047
1048 const RecordDecl *RD = S.getCapturedRecordDecl();
1049 unsigned FieldIdx = RD->getNumFields();
1050 for (unsigned I = 0; I < CD->getNumParams(); ++I) {
1051 const ImplicitParamDecl *Param = CD->getParam(I);
1052 if (Param == CD->getContextParam())
1053 continue;
1054 llvm::Align PtrAlign = CGM.getDataLayout().getPointerABIAlignment(0);
1055 llvm::Value *SlotPtr = Builder.CreateConstInBoundsGEP1_32(
1056 Builder.getPtrTy(), ContextV, FieldIdx,
1057 Twine(Param->getName()) + ".addr");
1058 llvm::Value *ParamAddr =
1059 Builder.CreateAlignedLoad(Builder.getPtrTy(), SlotPtr, PtrAlign);
1060 llvm::Value *ParamVal = Builder.CreateAlignedLoad(
1061 Builder.getPtrTy(), ParamAddr, PtrAlign, Param->getName());
1062 Address ParamLocalAddr =
1063 CreateMemTemp(Param->getType(), Param->getName());
1064 Builder.CreateStore(ParamVal, ParamLocalAddr);
1065 LocalAddrs.insert({Param, {Param, ParamLocalAddr}});
1066 ++FieldIdx;
1067 }
1068 }
1069
1070 CodeGenFunction::OMPPrivateScope LocalScope(*this);
1071 for (const auto &LocalAddrPair : LocalAddrs) {
1072 if (LocalAddrPair.second.first)
1073 LocalScope.addPrivate(LocalAddrPair.second.first,
1074 LocalAddrPair.second.second);
1075 }
1076 (void)LocalScope.Privatize();
1077 for (const auto &VLASizePair : VLASizes)
1078 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
1079 PGO->assignRegionCounters(GlobalDecl(CD), F);
1080 CapturedStmtInfo->EmitBody(*this, CD->getBody());
1081 (void)LocalScope.ForceCleanup();
1083
1084 if (!NeedWrapperFunction)
1085 return F;
1086
1087 // Reverse the order.
1088 WrapperF->removeFromParent();
1089 F->getParent()->getFunctionList().insertAfter(F->getIterator(), WrapperF);
1090
1091 llvm::Align PtrAlign = CGM.getDataLayout().getPointerABIAlignment(0);
1093 assert(CD->getContextParamPosition() == 0 &&
1094 "Expected context param at position 0 for target regions");
1095 assert(RD->getNumFields() + 1 == F->arg_size() && "Argument count mismatch");
1096
1097 for (auto [FD, InnerParam, SlotIdx] : llvm::zip(
1098 RD->fields(), F->args(), llvm::seq<unsigned>(RD->getNumFields()))) {
1099 llvm::Value *SlotPtr = WrapperCGF.Builder.CreateConstInBoundsGEP1_32(
1100 WrapperCGF.Builder.getPtrTy(), WrapperContextV, SlotIdx);
1101 llvm::Value *Slot = WrapperCGF.Builder.CreateAlignedLoad(
1102 WrapperCGF.Builder.getPtrTy(), SlotPtr, PtrAlign);
1103 llvm::Value *Val = WrapperCGF.Builder.CreateAlignedLoad(
1104 InnerParam.getType(), Slot, PtrAlign, InnerParam.getName());
1105 CallArgs.push_back(Val);
1106 }
1107
1108 // Handle the load from the implicit dyn_ptr at the end of the __context.
1109 unsigned SlotIdx = RD->getNumFields();
1110 auto InnerParam = F->arg_begin() + SlotIdx;
1111 llvm::Value *SlotPtr = WrapperCGF.Builder.CreateConstInBoundsGEP1_32(
1112 WrapperCGF.Builder.getPtrTy(), WrapperContextV, SlotIdx);
1113 llvm::Value *Slot = WrapperCGF.Builder.CreateAlignedLoad(
1114 WrapperCGF.Builder.getPtrTy(), SlotPtr, PtrAlign);
1115 llvm::Value *Val = WrapperCGF.Builder.CreateAlignedLoad(
1116 InnerParam->getType(), Slot, PtrAlign, InnerParam->getName());
1117 CallArgs.push_back(Val);
1118
1119 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, Loc, F, CallArgs);
1120 WrapperCGF.FinishFunction();
1121 return WrapperF;
1122}
1123
1124//===----------------------------------------------------------------------===//
1125// OpenMP Directive Emission
1126//===----------------------------------------------------------------------===//
1128 Address DestAddr, Address SrcAddr, QualType OriginalType,
1129 const llvm::function_ref<void(Address, Address)> CopyGen) {
1130 // Perform element-by-element initialization.
1131 QualType ElementTy;
1132
1133 // Drill down to the base element type on both arrays.
1134 const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
1135 llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
1136 SrcAddr = SrcAddr.withElementType(DestAddr.getElementType());
1137
1138 llvm::Value *SrcBegin = SrcAddr.emitRawPointer(*this);
1139 llvm::Value *DestBegin = DestAddr.emitRawPointer(*this);
1140 // Cast from pointer to array type to pointer to single element.
1141 llvm::Value *DestEnd = Builder.CreateInBoundsGEP(DestAddr.getElementType(),
1142 DestBegin, NumElements);
1143
1144 // The basic structure here is a while-do loop.
1145 llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body");
1146 llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done");
1147 llvm::Value *IsEmpty =
1148 Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
1149 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
1150
1151 // Enter the loop body, making that address the current address.
1152 llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
1153 EmitBlock(BodyBB);
1154
1155 CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
1156
1157 llvm::PHINode *SrcElementPHI =
1158 Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
1159 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
1160 Address SrcElementCurrent =
1161 Address(SrcElementPHI, SrcAddr.getElementType(),
1162 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
1163
1164 llvm::PHINode *DestElementPHI = Builder.CreatePHI(
1165 DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
1166 DestElementPHI->addIncoming(DestBegin, EntryBB);
1167 Address DestElementCurrent =
1168 Address(DestElementPHI, DestAddr.getElementType(),
1169 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
1170
1171 // Emit copy.
1172 CopyGen(DestElementCurrent, SrcElementCurrent);
1173
1174 // Shift the address forward by one element.
1175 llvm::Value *DestElementNext =
1176 Builder.CreateConstGEP1_32(DestAddr.getElementType(), DestElementPHI,
1177 /*Idx0=*/1, "omp.arraycpy.dest.element");
1178 llvm::Value *SrcElementNext =
1179 Builder.CreateConstGEP1_32(SrcAddr.getElementType(), SrcElementPHI,
1180 /*Idx0=*/1, "omp.arraycpy.src.element");
1181 // Check whether we've reached the end.
1182 llvm::Value *Done =
1183 Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
1184 Builder.CreateCondBr(Done, DoneBB, BodyBB);
1185 DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
1186 SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
1187
1188 // Done.
1189 EmitBlock(DoneBB, /*IsFinished=*/true);
1190}
1191
1193 Address SrcAddr, const VarDecl *DestVD,
1194 const VarDecl *SrcVD, const Expr *Copy) {
1195 if (OriginalType->isArrayType()) {
1196 const auto *BO = dyn_cast<BinaryOperator>(Copy);
1197 if (BO && BO->getOpcode() == BO_Assign) {
1198 // Perform simple memcpy for simple copying.
1199 LValue Dest = MakeAddrLValue(DestAddr, OriginalType);
1200 LValue Src = MakeAddrLValue(SrcAddr, OriginalType);
1201 EmitAggregateAssign(Dest, Src, OriginalType);
1202 } else {
1203 // For arrays with complex element types perform element by element
1204 // copying.
1206 DestAddr, SrcAddr, OriginalType,
1207 [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
1208 // Working with the single array element, so have to remap
1209 // destination and source variables to corresponding array
1210 // elements.
1212 Remap.addPrivate(DestVD, DestElement);
1213 Remap.addPrivate(SrcVD, SrcElement);
1214 (void)Remap.Privatize();
1216 });
1217 }
1218 } else {
1219 // Remap pseudo source variable to private copy.
1221 Remap.addPrivate(SrcVD, SrcAddr);
1222 Remap.addPrivate(DestVD, DestAddr);
1223 (void)Remap.Privatize();
1224 // Emit copying of the whole variable.
1226 }
1227}
1228
1230 OMPPrivateScope &PrivateScope) {
1231 if (!HaveInsertPoint())
1232 return false;
1234 bool DeviceConstTarget = getLangOpts().OpenMPIsTargetDevice &&
1236 bool FirstprivateIsLastprivate = false;
1237 llvm::SmallDenseMap<const Decl *, OpenMPLastprivateModifier> Lastprivates;
1238 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1239 for (const auto *D : C->varlist()) {
1240 const auto *VD = cast<DeclRefExpr>(D)->getDecl();
1241 Lastprivates.try_emplace(VD->getCanonicalDecl(), C->getKind());
1242 }
1243 }
1244 llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
1246 getOpenMPCaptureRegions(CaptureRegions, EKind);
1247 // Force emission of the firstprivate copy if the directive does not emit
1248 // outlined function, like omp for, omp simd, omp distribute etc.
1249 bool MustEmitFirstprivateCopy =
1250 CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
1251 for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
1252 const auto *IRef = C->varlist_begin();
1253 const auto *InitsRef = C->inits().begin();
1254 for (const Expr *IInit : C->private_copies()) {
1255 const auto *OrigDecl = cast<DeclRefExpr>(*IRef)->getDecl();
1256 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
1257
1258 if (const auto *BD = dyn_cast<BindingDecl>(OrigDecl)) {
1259 // Check if this binding is also lastprivate.
1260 bool ThisFirstprivateIsLastprivate =
1261 Lastprivates.count(BD->getCanonicalDecl()) > 0;
1262 const auto *DD = cast<VarDecl>(BD->getDecomposedDecl());
1263 // If the decomposition is captured by copy, the captured field is
1264 // already a private copy; map the binding to its member directly.
1265 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate) {
1266 if (const FieldDecl *FD = CapturedStmtInfo->lookup(DD)) {
1267 if (!FD->getType()->isReferenceType()) {
1268 bool IsRegistered = PrivateScope.addPrivate(
1269 BD, EmitOMPCapturedBindingLValue(BD).getAddress());
1270 assert(IsRegistered &&
1271 "firstprivate var already registered as firstprivate");
1272 (void)IsRegistered;
1273 ++IRef;
1274 ++InitsRef;
1275 continue;
1276 }
1277 }
1278 }
1279 const auto *VDInit =
1280 cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
1281 Address OriginalAddr =
1282 EmitOMPBindingOriginalAddr(BD, (*IRef)->getExprLoc());
1283
1284 QualType Type = VD->getType();
1285 bool IsRegistered;
1286 if (Type->isArrayType()) {
1287 // For array bindings, use array copy logic
1288 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
1289 const Expr *Init = VD->getInit();
1290 LValue OriginalLVal = MakeAddrLValue(OriginalAddr, Type);
1291 if (!Init || !isa<CXXConstructExpr>(Init) ||
1293 // Perform simple memcpy.
1294 LValue Dest = MakeAddrLValue(Emission.getAllocatedAddress(), Type);
1295 EmitAggregateAssign(Dest, OriginalLVal, Type);
1296 } else {
1298 Emission.getAllocatedAddress(), OriginalAddr, Type,
1299 [this, VDInit, Init](Address DestElement, Address SrcElement) {
1300 // Clean up any temporaries needed by the initialization.
1301 RunCleanupsScope InitScope(*this);
1302 // Emit initialization for single element.
1303 setAddrOfLocalVar(VDInit, SrcElement);
1304 EmitAnyExprToMem(Init, DestElement,
1305 Init->getType().getQualifiers(),
1306 /*IsInitializer*/ false);
1307 LocalDeclMap.erase(VDInit);
1308 });
1309 }
1310 EmitAutoVarCleanups(Emission);
1311 IsRegistered =
1312 PrivateScope.addPrivate(BD, Emission.getAllocatedAddress());
1313 } else {
1314 // VD now has the binding's type (e.g., int), not the struct type.
1315 // Emit VD initialized from the binding's field address.
1316 setAddrOfLocalVar(VDInit, OriginalAddr);
1317 EmitDecl(*VD);
1318 LocalDeclMap.erase(VDInit);
1319 Address VDAddr = GetAddrOfLocalVar(VD);
1320 // VD is the private copy of the binding, map BD to VDAddr directly
1321 IsRegistered = PrivateScope.addPrivate(BD, VDAddr);
1322 }
1323
1324 assert(IsRegistered &&
1325 "firstprivate var already registered as firstprivate");
1326 (void)IsRegistered;
1327 FirstprivateIsLastprivate =
1328 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
1329 ++IRef;
1330 ++InitsRef;
1331 continue;
1332 }
1333
1334 // Original VarDecl logic.
1335 const VarDecl *OrigVD = dyn_cast<VarDecl>(OrigDecl);
1336 assert(OrigVD && "Expected VarDecl for non-BindingDecl firstprivate");
1337 bool ThisFirstprivateIsLastprivate =
1338 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
1339 const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
1340 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
1341 !FD->getType()->isReferenceType() &&
1342 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
1343 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
1344 ++IRef;
1345 ++InitsRef;
1346 continue;
1347 }
1348 // Do not emit copy for firstprivate constant variables in target regions,
1349 // captured by reference.
1350 if (DeviceConstTarget && OrigVD->getType().isConstant(getContext()) &&
1351 FD && FD->getType()->isReferenceType() &&
1352 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
1353 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
1354 ++IRef;
1355 ++InitsRef;
1356 continue;
1357 }
1358 FirstprivateIsLastprivate =
1359 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
1360 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
1361 const auto *VDInit =
1362 cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
1363 bool IsRegistered;
1364 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1365 /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
1366 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
1367 LValue OriginalLVal;
1368 if (!FD) {
1369 // Check if the firstprivate variable is just a constant value.
1371 if (CE && !CE.isReference()) {
1372 // Constant value, no need to create a copy.
1373 ++IRef;
1374 ++InitsRef;
1375 continue;
1376 }
1377 if (CE && CE.isReference()) {
1378 OriginalLVal = CE.getReferenceLValue(*this, &DRE);
1379 } else {
1380 assert(!CE && "Expected non-constant firstprivate.");
1381 OriginalLVal = EmitLValue(&DRE);
1382 }
1383 } else {
1384 OriginalLVal = EmitLValue(&DRE);
1385 }
1386 QualType Type = VD->getType();
1387 if (Type->isArrayType()) {
1388 // Emit VarDecl with copy init for arrays.
1389 // Get the address of the original variable captured in current
1390 // captured region.
1391 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
1392 const Expr *Init = VD->getInit();
1394 // Perform simple memcpy.
1395 LValue Dest = MakeAddrLValue(Emission.getAllocatedAddress(), Type);
1396 EmitAggregateAssign(Dest, OriginalLVal, Type);
1397 } else {
1399 Emission.getAllocatedAddress(), OriginalLVal.getAddress(), Type,
1400 [this, VDInit, Init](Address DestElement, Address SrcElement) {
1401 // Clean up any temporaries needed by the
1402 // initialization.
1403 RunCleanupsScope InitScope(*this);
1404 // Emit initialization for single element.
1405 setAddrOfLocalVar(VDInit, SrcElement);
1406 EmitAnyExprToMem(Init, DestElement,
1407 Init->getType().getQualifiers(),
1408 /*IsInitializer*/ false);
1409 LocalDeclMap.erase(VDInit);
1410 });
1411 }
1412 EmitAutoVarCleanups(Emission);
1413 IsRegistered =
1414 PrivateScope.addPrivate(OrigVD, Emission.getAllocatedAddress());
1415 } else {
1416 Address OriginalAddr = OriginalLVal.getAddress();
1417 // Emit private VarDecl with copy init.
1418 // Remap temp VDInit variable to the address of the original
1419 // variable (for proper handling of captured global variables).
1420 setAddrOfLocalVar(VDInit, OriginalAddr);
1421 EmitDecl(*VD);
1422 LocalDeclMap.erase(VDInit);
1423 Address VDAddr = GetAddrOfLocalVar(VD);
1424 if (ThisFirstprivateIsLastprivate &&
1425 Lastprivates[OrigVD->getCanonicalDecl()] ==
1426 OMPC_LASTPRIVATE_conditional) {
1427 // Create/init special variable for lastprivate conditionals.
1428 llvm::Value *V =
1429 EmitLoadOfScalar(MakeAddrLValue(VDAddr, (*IRef)->getType(),
1431 (*IRef)->getExprLoc());
1432 VDAddr = CGM.getOpenMPRuntime().emitLastprivateConditionalInit(
1433 *this, OrigVD);
1434 EmitStoreOfScalar(V, MakeAddrLValue(VDAddr, (*IRef)->getType(),
1436 LocalDeclMap.erase(VD);
1437 setAddrOfLocalVar(VD, VDAddr);
1438 }
1439 IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);
1440 }
1441 assert(IsRegistered &&
1442 "firstprivate var already registered as private");
1443 // Silence the warning about unused variable.
1444 (void)IsRegistered;
1445 }
1446 ++IRef;
1447 ++InitsRef;
1448 }
1449 }
1450 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
1451}
1452
1454 const OMPExecutableDirective &D,
1455 CodeGenFunction::OMPPrivateScope &PrivateScope) {
1456 if (!HaveInsertPoint())
1457 return;
1458 llvm::SmallDenseSet<const ValueDecl *> EmittedAsPrivate;
1459 for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
1460 auto IRef = C->varlist_begin();
1461 for (const Expr *IInit : C->private_copies()) {
1462 const auto *OrigDecl = cast<DeclRefExpr>(*IRef)->getDecl();
1463 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
1464 if (EmittedAsPrivate.insert(cast<ValueDecl>(OrigDecl->getCanonicalDecl()))
1465 .second) {
1466 EmitDecl(*VD);
1467 bool IsRegistered =
1468 PrivateScope.addPrivate(OrigDecl, GetAddrOfLocalVar(VD));
1469 assert(IsRegistered && "private var already registered as private");
1470 (void)IsRegistered;
1471 }
1472 ++IRef;
1473 }
1474 }
1475}
1476
1478 if (!HaveInsertPoint())
1479 return false;
1480 // threadprivate_var1 = master_threadprivate_var1;
1481 // operator=(threadprivate_var2, master_threadprivate_var2);
1482 // ...
1483 // __kmpc_barrier(&loc, global_tid);
1484 llvm::DenseSet<const VarDecl *> CopiedVars;
1485 llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
1486 for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
1487 auto IRef = C->varlist_begin();
1488 auto ISrcRef = C->source_exprs().begin();
1489 auto IDestRef = C->destination_exprs().begin();
1490 for (const Expr *AssignOp : C->assignment_ops()) {
1491 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1492 QualType Type = VD->getType();
1493 if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
1494 // Get the address of the master variable. If we are emitting code with
1495 // TLS support, the address is passed from the master as field in the
1496 // captured declaration.
1497 Address MasterAddr = Address::invalid();
1498 if (getLangOpts().OpenMPUseTLS &&
1499 getContext().getTargetInfo().isTLSSupported()) {
1500 assert(CapturedStmtInfo->lookup(VD) &&
1501 "Copyin threadprivates should have been captured!");
1502 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), true,
1503 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
1504 MasterAddr = EmitLValue(&DRE).getAddress();
1505 LocalDeclMap.erase(VD);
1506 } else {
1507 MasterAddr =
1508 Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
1509 : CGM.GetAddrOfGlobal(VD),
1510 CGM.getTypes().ConvertTypeForMem(VD->getType()),
1511 getContext().getDeclAlign(VD));
1512 }
1513 // Get the address of the threadprivate variable.
1514 Address PrivateAddr = EmitLValue(*IRef).getAddress();
1515 if (CopiedVars.size() == 1) {
1516 // At first check if current thread is a master thread. If it is, no
1517 // need to copy data.
1518 CopyBegin = createBasicBlock("copyin.not.master");
1519 CopyEnd = createBasicBlock("copyin.not.master.end");
1520 // TODO: Avoid ptrtoint conversion.
1521 auto *MasterAddrInt = Builder.CreatePtrToInt(
1522 MasterAddr.emitRawPointer(*this), CGM.IntPtrTy);
1523 auto *PrivateAddrInt = Builder.CreatePtrToInt(
1524 PrivateAddr.emitRawPointer(*this), CGM.IntPtrTy);
1525 Builder.CreateCondBr(
1526 Builder.CreateICmpNE(MasterAddrInt, PrivateAddrInt), CopyBegin,
1527 CopyEnd);
1528 EmitBlock(CopyBegin);
1529 }
1530 const auto *SrcVD =
1531 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1532 const auto *DestVD =
1533 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1534 EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
1535 }
1536 ++IRef;
1537 ++ISrcRef;
1538 ++IDestRef;
1539 }
1540 }
1541 if (CopyEnd) {
1542 // Exit out of copying procedure for non-master thread.
1543 EmitBlock(CopyEnd, /*IsFinished=*/true);
1544 return true;
1545 }
1546 return false;
1547}
1548
1550 const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
1551 if (!HaveInsertPoint())
1552 return false;
1553 bool HasAtLeastOneLastprivate = false;
1555 llvm::DenseSet<const VarDecl *> SIMDLCVs;
1556 if (isOpenMPSimdDirective(EKind)) {
1557 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1558 for (const Expr *C : LoopDirective->counters()) {
1559 SIMDLCVs.insert(
1561 }
1562 }
1563 llvm::SmallDenseSet<const ValueDecl *> AlreadyEmittedVars;
1564 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1565 HasAtLeastOneLastprivate = true;
1566 if (isOpenMPTaskLoopDirective(EKind) && !getLangOpts().OpenMPSimd)
1567 break;
1568 const auto *IRef = C->varlist_begin();
1569 const auto *IDestRef = C->destination_exprs().begin();
1570 for (const Expr *IInit : C->private_copies()) {
1571 // Keep the address of the original variable for future update at the end
1572 // of the loop.
1573 const auto *OrigDecl = cast<DeclRefExpr>(*IRef)->getDecl();
1574 // Handle BindingDecls with the same level of support as VarDecls.
1575 if (const auto *BD = dyn_cast<BindingDecl>(OrigDecl)) {
1576 if (AlreadyEmittedVars.insert(cast<ValueDecl>(BD->getCanonicalDecl()))
1577 .second) {
1578 const auto *DestVD =
1579 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1580
1581 // Get the original binding address.
1582 Address OrigAddr =
1583 EmitOMPBindingOriginalAddr(BD, (*IRef)->getExprLoc());
1584 PrivateScope.addPrivate(DestVD, OrigAddr);
1585 if (IInit) {
1586 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
1587 // Emit private VarDecl with copy init.
1588 EmitDecl(*VD);
1589 Address VDAddr = GetAddrOfLocalVar(VD);
1590 bool IsRegistered = PrivateScope.addPrivate(BD, VDAddr);
1591 assert(IsRegistered &&
1592 "lastprivate binding already registered as private");
1593 (void)IsRegistered;
1594 }
1595 }
1596 ++IRef;
1597 ++IDestRef;
1598 continue;
1599 }
1600 const auto *OrigVD = cast<VarDecl>(OrigDecl);
1601 // Taskloops do not require additional initialization, it is done in
1602 // runtime support library.
1603 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
1604 const auto *DestVD =
1605 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1606 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1607 /*RefersToEnclosingVariableOrCapture=*/
1608 CapturedStmtInfo->lookup(OrigVD) != nullptr,
1609 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
1610 PrivateScope.addPrivate(DestVD, EmitLValue(&DRE).getAddress());
1611 // Check if the variable is also a firstprivate: in this case IInit is
1612 // not generated. Initialization of this variable will happen in codegen
1613 // for 'firstprivate' clause.
1614 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
1615 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
1616 Address VDAddr = Address::invalid();
1617 if (C->getKind() == OMPC_LASTPRIVATE_conditional) {
1618 VDAddr = CGM.getOpenMPRuntime().emitLastprivateConditionalInit(
1619 *this, OrigVD);
1620 setAddrOfLocalVar(VD, VDAddr);
1621 } else {
1622 // Emit private VarDecl with copy init.
1623 EmitDecl(*VD);
1624 VDAddr = GetAddrOfLocalVar(VD);
1625 }
1626 bool IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);
1627 assert(IsRegistered &&
1628 "lastprivate var already registered as private");
1629 (void)IsRegistered;
1630 }
1631 }
1632 ++IRef;
1633 ++IDestRef;
1634 }
1635 }
1636 return HasAtLeastOneLastprivate;
1637}
1638
1640 const OMPExecutableDirective &D, bool NoFinals,
1641 llvm::Value *IsLastIterCond) {
1642 if (!HaveInsertPoint())
1643 return;
1644 // Emit following code:
1645 // if (<IsLastIterCond>) {
1646 // orig_var1 = private_orig_var1;
1647 // ...
1648 // orig_varn = private_orig_varn;
1649 // }
1650 llvm::BasicBlock *ThenBB = nullptr;
1651 llvm::BasicBlock *DoneBB = nullptr;
1652 if (IsLastIterCond) {
1653 // Emit implicit barrier if at least one lastprivate conditional is found
1654 // and this is not a simd mode.
1655 if (!getLangOpts().OpenMPSimd &&
1656 llvm::any_of(D.getClausesOfKind<OMPLastprivateClause>(),
1657 [](const OMPLastprivateClause *C) {
1658 return C->getKind() == OMPC_LASTPRIVATE_conditional;
1659 })) {
1660 CGM.getOpenMPRuntime().emitBarrierCall(*this, D.getBeginLoc(),
1661 OMPD_unknown,
1662 /*EmitChecks=*/false,
1663 /*ForceSimpleCall=*/true);
1664 }
1665 ThenBB = createBasicBlock(".omp.lastprivate.then");
1666 DoneBB = createBasicBlock(".omp.lastprivate.done");
1667 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1668 EmitBlock(ThenBB);
1669 }
1670 llvm::DenseSet<const ValueDecl *> AlreadyEmittedVars;
1671 llvm::SmallDenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1672 if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1673 auto IC = LoopDirective->counters().begin();
1674 for (const Expr *F : LoopDirective->finals()) {
1675 const auto *D =
1676 cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1677 if (NoFinals)
1678 AlreadyEmittedVars.insert(D);
1679 else
1680 LoopCountersAndUpdates[D] = F;
1681 ++IC;
1682 }
1683 }
1684 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1685 auto IRef = C->varlist_begin();
1686 auto ISrcRef = C->source_exprs().begin();
1687 auto IDestRef = C->destination_exprs().begin();
1688 for (const Expr *AssignOp : C->assignment_ops()) {
1689 const auto *PrivateDecl = cast<DeclRefExpr>(*IRef)->getDecl();
1690
1691 // For BindingDecls, check if we should use .lastprivate.src or the BD
1692 // itself.
1693 const VarDecl *PrivateVD = nullptr;
1694 const BindingDecl *BD = nullptr;
1695 if ((BD = dyn_cast<BindingDecl>(PrivateDecl))) {
1696 // Check if .lastprivate.src is available (taskloop case).
1697 const auto *SrcVD =
1698 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1699
1700 if (LocalDeclMap.count(SrcVD)) {
1701 // Taskloop case: use .lastprivate.src.
1702 PrivateVD = SrcVD;
1703 } else if (OMPPrivatizedBindings.count(BD)) {
1704 // Parallel for case: BindingDecl is directly privatized.
1705 // Leave PrivateVD as nullptr to handle specially below.
1706 }
1707 } else {
1708 PrivateVD = cast<VarDecl>(PrivateDecl);
1709 }
1710
1711 QualType Type = PrivateVD ? PrivateVD->getType() : BD->getType();
1712 const auto *CanonicalVD =
1713 PrivateVD ? PrivateVD->getCanonicalDecl() : nullptr;
1714
1715 // Check if already emitted.
1716 bool ShouldEmit =
1717 AlreadyEmittedVars
1718 .insert(PrivateVD ? static_cast<const ValueDecl *>(CanonicalVD)
1719 : static_cast<const ValueDecl *>(BD))
1720 .second;
1721 if (ShouldEmit) {
1722 // If lastprivate variable is a loop control variable for loop-based
1723 // directive, update its value before copyin back to original
1724 // variable.
1725 if (CanonicalVD) {
1726 if (const Expr *FinalExpr =
1727 LoopCountersAndUpdates.lookup(CanonicalVD))
1728 EmitIgnoredExpr(FinalExpr);
1729 }
1730 const auto *SrcVD =
1731 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1732 const auto *DestVD =
1733 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1734
1735 // Get the address of the private variable.
1736 Address PrivateAddr = Address::invalid();
1737 if (PrivateVD) {
1738 PrivateAddr = GetAddrOfLocalVar(PrivateVD);
1739 } else {
1740 auto It = OMPPrivatizedBindings.find(BD);
1741 assert(It != OMPPrivatizedBindings.end() &&
1742 "BindingDecl should be privatized");
1743 PrivateAddr = It->second;
1744 }
1745 if (PrivateVD) {
1746 if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1747 PrivateAddr = Address(
1748 Builder.CreateLoad(PrivateAddr),
1749 CGM.getTypes().ConvertTypeForMem(RefTy->getPointeeType()),
1750 CGM.getNaturalTypeAlignment(RefTy->getPointeeType()));
1751 }
1752 // Store the last value to the private copy in the last iteration.
1753 if (C->getKind() == OMPC_LASTPRIVATE_conditional)
1754 CGM.getOpenMPRuntime().emitLastprivateConditionalFinalUpdate(
1755 *this, MakeAddrLValue(PrivateAddr, (*IRef)->getType()), PrivateVD,
1756 (*IRef)->getExprLoc());
1757 // Get the address of the original variable.
1758 Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1759 EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1760 }
1761 ++IRef;
1762 ++ISrcRef;
1763 ++IDestRef;
1764 }
1765 if (const Expr *PostUpdate = C->getPostUpdateExpr())
1766 EmitIgnoredExpr(PostUpdate);
1767 }
1768 if (IsLastIterCond)
1769 EmitBlock(DoneBB, /*IsFinished=*/true);
1770}
1771
1773 const OMPExecutableDirective &D,
1774 CodeGenFunction::OMPPrivateScope &PrivateScope, bool ForInscan) {
1775 if (!HaveInsertPoint())
1776 return;
1779 SmallVector<const Expr *, 4> ReductionOps;
1785 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1786 if (ForInscan != (C->getModifier() == OMPC_REDUCTION_inscan))
1787 continue;
1788 Shareds.append(C->varlist_begin(), C->varlist_end());
1789 Privates.append(C->privates().begin(), C->privates().end());
1790 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1791 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1792 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1793 if (C->getModifier() == OMPC_REDUCTION_task) {
1794 Data.ReductionVars.append(C->privates().begin(), C->privates().end());
1795 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
1796 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
1797 Data.ReductionOps.append(C->reduction_ops().begin(),
1798 C->reduction_ops().end());
1799 TaskLHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1800 TaskRHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1801 }
1802 }
1803 ReductionCodeGen RedCG(Shareds, Shareds, Privates, ReductionOps);
1804 unsigned Count = 0;
1805 auto *ILHS = LHSs.begin();
1806 auto *IRHS = RHSs.begin();
1807 auto *IPriv = Privates.begin();
1808 for (const Expr *IRef : Shareds) {
1809 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1810 RedCG.emitSharedOrigLValue(*this, Count);
1811 RedCG.emitAggregateType(*this, Count);
1812 AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);
1813 RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
1814 RedCG.getSharedLValue(Count).getAddress(),
1815 [&Emission](CodeGenFunction &CGF) {
1816 CGF.EmitAutoVarInit(Emission);
1817 return true;
1818 });
1819 EmitAutoVarCleanups(Emission);
1820 Address BaseAddr = RedCG.adjustPrivateAddress(
1821 *this, Count, Emission.getAllocatedAddress());
1822 bool IsRegistered =
1823 PrivateScope.addPrivate(RedCG.getBaseDecl(Count), BaseAddr);
1824 assert(IsRegistered && "private var already registered as private");
1825 // Silence the warning about unused variable.
1826 (void)IsRegistered;
1827
1828 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1829 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1830 QualType Type = PrivateVD->getType();
1831 bool isaOMPArraySectionExpr = isa<ArraySectionExpr>(IRef);
1832 if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1833 // Store the address of the original variable associated with the LHS
1834 // implicit variable.
1835 PrivateScope.addPrivate(LHSVD, RedCG.getSharedLValue(Count).getAddress());
1836 PrivateScope.addPrivate(RHSVD, GetAddrOfLocalVar(PrivateVD));
1837 } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1839 // Store the address of the original variable associated with the LHS
1840 // implicit variable.
1841 PrivateScope.addPrivate(LHSVD, RedCG.getSharedLValue(Count).getAddress());
1842 PrivateScope.addPrivate(RHSVD,
1843 GetAddrOfLocalVar(PrivateVD).withElementType(
1844 ConvertTypeForMem(RHSVD->getType())));
1845 } else {
1846 QualType Type = PrivateVD->getType();
1847 bool IsArray = getContext().getAsArrayType(Type) != nullptr;
1848 Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress();
1849 // Store the address of the original variable associated with the LHS
1850 // implicit variable.
1851 if (IsArray) {
1852 OriginalAddr =
1853 OriginalAddr.withElementType(ConvertTypeForMem(LHSVD->getType()));
1854 }
1855 PrivateScope.addPrivate(LHSVD, OriginalAddr);
1856 PrivateScope.addPrivate(
1857 RHSVD, IsArray ? GetAddrOfLocalVar(PrivateVD).withElementType(
1858 ConvertTypeForMem(RHSVD->getType()))
1859 : GetAddrOfLocalVar(PrivateVD));
1860 }
1861 ++ILHS;
1862 ++IRHS;
1863 ++IPriv;
1864 ++Count;
1865 }
1866 if (!Data.ReductionVars.empty()) {
1868 Data.IsReductionWithTaskMod = true;
1869 Data.IsWorksharingReduction = isOpenMPWorksharingDirective(EKind);
1870 llvm::Value *ReductionDesc = CGM.getOpenMPRuntime().emitTaskReductionInit(
1871 *this, D.getBeginLoc(), TaskLHSs, TaskRHSs, Data);
1872 const Expr *TaskRedRef = nullptr;
1873 switch (EKind) {
1874 case OMPD_parallel:
1875 TaskRedRef = cast<OMPParallelDirective>(D).getTaskReductionRefExpr();
1876 break;
1877 case OMPD_for:
1878 TaskRedRef = cast<OMPForDirective>(D).getTaskReductionRefExpr();
1879 break;
1880 case OMPD_sections:
1881 TaskRedRef = cast<OMPSectionsDirective>(D).getTaskReductionRefExpr();
1882 break;
1883 case OMPD_parallel_for:
1884 TaskRedRef = cast<OMPParallelForDirective>(D).getTaskReductionRefExpr();
1885 break;
1886 case OMPD_parallel_master:
1887 TaskRedRef =
1888 cast<OMPParallelMasterDirective>(D).getTaskReductionRefExpr();
1889 break;
1890 case OMPD_parallel_sections:
1891 TaskRedRef =
1892 cast<OMPParallelSectionsDirective>(D).getTaskReductionRefExpr();
1893 break;
1894 case OMPD_target_parallel:
1895 TaskRedRef =
1896 cast<OMPTargetParallelDirective>(D).getTaskReductionRefExpr();
1897 break;
1898 case OMPD_target_parallel_for:
1899 TaskRedRef =
1900 cast<OMPTargetParallelForDirective>(D).getTaskReductionRefExpr();
1901 break;
1902 case OMPD_distribute_parallel_for:
1903 TaskRedRef =
1904 cast<OMPDistributeParallelForDirective>(D).getTaskReductionRefExpr();
1905 break;
1906 case OMPD_teams_distribute_parallel_for:
1908 .getTaskReductionRefExpr();
1909 break;
1910 case OMPD_target_teams_distribute_parallel_for:
1912 .getTaskReductionRefExpr();
1913 break;
1914 case OMPD_simd:
1915 case OMPD_for_simd:
1916 case OMPD_section:
1917 case OMPD_single:
1918 case OMPD_master:
1919 case OMPD_critical:
1920 case OMPD_parallel_for_simd:
1921 case OMPD_task:
1922 case OMPD_taskyield:
1923 case OMPD_error:
1924 case OMPD_barrier:
1925 case OMPD_taskwait:
1926 case OMPD_taskgroup:
1927 case OMPD_flush:
1928 case OMPD_depobj:
1929 case OMPD_scan:
1930 case OMPD_ordered_standalone:
1931 case OMPD_ordered_blockassoc:
1932 case OMPD_atomic:
1933 case OMPD_teams:
1934 case OMPD_target:
1935 case OMPD_cancellation_point:
1936 case OMPD_cancel:
1937 case OMPD_target_data:
1938 case OMPD_target_enter_data:
1939 case OMPD_target_exit_data:
1940 case OMPD_taskloop:
1941 case OMPD_taskloop_simd:
1942 case OMPD_master_taskloop:
1943 case OMPD_master_taskloop_simd:
1944 case OMPD_parallel_master_taskloop:
1945 case OMPD_parallel_master_taskloop_simd:
1946 case OMPD_distribute:
1947 case OMPD_target_update:
1948 case OMPD_distribute_parallel_for_simd:
1949 case OMPD_distribute_simd:
1950 case OMPD_target_parallel_for_simd:
1951 case OMPD_target_simd:
1952 case OMPD_teams_distribute:
1953 case OMPD_teams_distribute_simd:
1954 case OMPD_teams_distribute_parallel_for_simd:
1955 case OMPD_target_teams:
1956 case OMPD_target_teams_distribute:
1957 case OMPD_target_teams_distribute_parallel_for_simd:
1958 case OMPD_target_teams_distribute_simd:
1959 case OMPD_declare_target:
1960 case OMPD_end_declare_target:
1961 case OMPD_threadprivate:
1962 case OMPD_allocate:
1963 case OMPD_declare_reduction:
1964 case OMPD_declare_mapper:
1965 case OMPD_declare_simd:
1966 case OMPD_requires:
1967 case OMPD_declare_variant:
1968 case OMPD_begin_declare_variant:
1969 case OMPD_end_declare_variant:
1970 case OMPD_unknown:
1971 default:
1972 llvm_unreachable("Unexpected directive with task reductions.");
1973 }
1974
1975 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(TaskRedRef)->getDecl());
1976 EmitVarDecl(*VD);
1977 EmitStoreOfScalar(ReductionDesc, GetAddrOfLocalVar(VD),
1978 /*Volatile=*/false, TaskRedRef->getType());
1979 }
1980}
1981
1983 const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1984 if (!HaveInsertPoint())
1985 return;
1990 llvm::SmallVector<bool, 8> IsPrivateVarReduction;
1991 bool HasAtLeastOneReduction = false;
1992 bool IsReductionWithTaskMod = false;
1993 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1994 // Do not emit for inscan reductions.
1995 if (C->getModifier() == OMPC_REDUCTION_inscan)
1996 continue;
1997 HasAtLeastOneReduction = true;
1998 Privates.append(C->privates().begin(), C->privates().end());
1999 LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
2000 RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
2001 IsPrivateVarReduction.append(C->private_var_reduction_flags().begin(),
2002 C->private_var_reduction_flags().end());
2003 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
2004 IsReductionWithTaskMod =
2005 IsReductionWithTaskMod || C->getModifier() == OMPC_REDUCTION_task;
2006 }
2007 if (HasAtLeastOneReduction) {
2009 if (IsReductionWithTaskMod) {
2010 CGM.getOpenMPRuntime().emitTaskReductionFini(
2011 *this, D.getBeginLoc(), isOpenMPWorksharingDirective(EKind));
2012 }
2013 bool TeamsLoopCanBeParallel = false;
2014 if (auto *TTLD = dyn_cast<OMPTargetTeamsGenericLoopDirective>(&D))
2015 TeamsLoopCanBeParallel = TTLD->canBeParallelFor();
2016 bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
2018 TeamsLoopCanBeParallel || ReductionKind == OMPD_simd;
2019 bool SimpleReduction = ReductionKind == OMPD_simd;
2020 // Emit nowait reduction if nowait clause is present or directive is a
2021 // parallel directive (it always has implicit barrier).
2022 CGM.getOpenMPRuntime().emitReduction(
2023 *this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
2024 {WithNowait, SimpleReduction, IsPrivateVarReduction, ReductionKind});
2025 }
2026}
2027
2030 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
2031 if (!CGF.HaveInsertPoint())
2032 return;
2033 llvm::BasicBlock *DoneBB = nullptr;
2034 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
2035 if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
2036 if (!DoneBB) {
2037 if (llvm::Value *Cond = CondGen(CGF)) {
2038 // If the first post-update expression is found, emit conditional
2039 // block if it was requested.
2040 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
2041 DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
2042 CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2043 CGF.EmitBlock(ThenBB);
2044 }
2045 }
2046 CGF.EmitIgnoredExpr(PostUpdate);
2047 }
2048 }
2049 if (DoneBB)
2050 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
2051}
2052
2053namespace {
2054/// Codegen lambda for appending distribute lower and upper bounds to outlined
2055/// parallel function. This is necessary for combined constructs such as
2056/// 'distribute parallel for'
2057typedef llvm::function_ref<void(CodeGenFunction &,
2058 const OMPExecutableDirective &,
2059 llvm::SmallVectorImpl<llvm::Value *> &)>
2060 CodeGenBoundParametersTy;
2061} // anonymous namespace
2062
2063static void
2065 const OMPExecutableDirective &S) {
2066 if (CGF.getLangOpts().OpenMP < 50)
2067 return;
2068 llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> PrivateDecls;
2069 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
2070 for (const Expr *Ref : C->varlist()) {
2071 if (!Ref->getType()->isScalarType())
2072 continue;
2073 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
2074 if (!DRE)
2075 continue;
2076 // Skip BindingDecls - lastprivate conditional only applies to VarDecls.
2077 const auto *VD = dyn_cast<VarDecl>(DRE->getDecl());
2078 if (!VD)
2079 continue;
2080 PrivateDecls.insert(VD);
2082 }
2083 }
2084 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
2085 for (const Expr *Ref : C->varlist()) {
2086 if (!Ref->getType()->isScalarType())
2087 continue;
2088 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
2089 if (!DRE)
2090 continue;
2091 // Skip BindingDecls - they don't use the same conditional lastprivate
2092 // mechanism.
2093 if (const auto *VD = dyn_cast<VarDecl>(DRE->getDecl())) {
2094 PrivateDecls.insert(VD);
2096 }
2097 }
2098 }
2099 for (const auto *C : S.getClausesOfKind<OMPLinearClause>()) {
2100 for (const Expr *Ref : C->varlist()) {
2101 if (!Ref->getType()->isScalarType())
2102 continue;
2103 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
2104 if (!DRE)
2105 continue;
2106 if (const auto *VD = dyn_cast<VarDecl>(DRE->getDecl())) {
2107 PrivateDecls.insert(VD);
2109 }
2110 }
2111 }
2112 // Privates should ne analyzed since they are not captured at all.
2113 // Task reductions may be skipped - tasks are ignored.
2114 // Firstprivates do not return value but may be passed by reference - no need
2115 // to check for updated lastprivate conditional.
2116 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
2117 for (const Expr *Ref : C->varlist()) {
2118 if (!Ref->getType()->isScalarType())
2119 continue;
2120 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
2121 if (!DRE)
2122 continue;
2123 // Only track VarDecl, not BindingDecl.
2124 if (const auto *VD = dyn_cast<VarDecl>(DRE->getDecl()))
2125 PrivateDecls.insert(VD);
2126 }
2127 }
2129 CGF, S, PrivateDecls);
2130}
2131
2134 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
2135 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
2136 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
2137 llvm::Value *NumThreads = nullptr;
2139 // OpenMP 6.0, 10.4: "If no severity clause is specified then the effect is as
2140 // if sev-level is fatal."
2141 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal;
2142 clang::Expr *Message = nullptr;
2143 SourceLocation SeverityLoc = SourceLocation();
2144 SourceLocation MessageLoc = SourceLocation();
2145
2146 llvm::Function *OutlinedFn =
2148 CGF, S, *CS->getCapturedDecl()->param_begin(), InnermostKind,
2149 CodeGen);
2150
2151 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
2152 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
2153 NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads().front(),
2154 /*IgnoreResultAssign=*/true);
2155 Modifier = NumThreadsClause->getPrescriptivenessModifier();
2156 if (const auto *MessageClause = S.getSingleClause<OMPMessageClause>()) {
2157 Message = MessageClause->getMessageString();
2158 MessageLoc = MessageClause->getBeginLoc();
2159 }
2160 if (const auto *SeverityClause = S.getSingleClause<OMPSeverityClause>()) {
2161 Severity = SeverityClause->getSeverityKind();
2162 SeverityLoc = SeverityClause->getBeginLoc();
2163 }
2165 CGF, NumThreads, NumThreadsClause->getBeginLoc(), Modifier, Severity,
2166 SeverityLoc, Message, MessageLoc);
2167 }
2168 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
2169 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
2171 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
2172 }
2173 const Expr *IfCond = nullptr;
2174 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
2175 if (C->getNameModifier() == OMPD_unknown ||
2176 C->getNameModifier() == OMPD_parallel) {
2177 IfCond = C->getCondition();
2178 break;
2179 }
2180 }
2181
2182 OMPParallelScope Scope(CGF, S);
2184 // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
2185 // lower and upper bounds with the pragma 'for' chunking mechanism.
2186 // The following lambda takes care of appending the lower and upper bound
2187 // parameters when necessary
2188 CodeGenBoundParameters(CGF, S, CapturedVars);
2189 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
2190 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn,
2191 CapturedVars, IfCond, NumThreads,
2192 Modifier, Severity, Message);
2193}
2194
2195static bool isAllocatableDecl(const VarDecl *VD) {
2196 const VarDecl *CVD = VD->getCanonicalDecl();
2197 if (!CVD->hasAttr<OMPAllocateDeclAttr>())
2198 return false;
2199 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
2200 // Use the default allocation.
2201 return !((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||
2202 AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&
2203 !AA->getAllocator());
2204}
2205
2209
2211 const OMPExecutableDirective &S) {
2212 bool Copyins = CGF.EmitOMPCopyinClause(S);
2213 if (Copyins) {
2214 // Emit implicit barrier to synchronize threads and avoid data races on
2215 // propagation master's thread values of threadprivate variables to local
2216 // instances of that variables of all other implicit threads.
2218 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
2219 /*ForceSimpleCall=*/true);
2220 }
2221}
2222
2224 CodeGenFunction &CGF, const VarDecl *VD) {
2225 CodeGenModule &CGM = CGF.CGM;
2226 auto &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2227
2228 // Unreachable declaration: nothing can run the allocation.
2229 if (!VD || !CGF.HaveInsertPoint())
2230 return Address::invalid();
2231 const VarDecl *CVD = VD->getCanonicalDecl();
2232 if (!isAllocatableDecl(CVD))
2233 return Address::invalid();
2234 llvm::Value *Size;
2235 CharUnits Align = CGM.getContext().getDeclAlign(CVD);
2236 if (CVD->getType()->isVariablyModifiedType()) {
2237 Size = CGF.getTypeSize(CVD->getType());
2238 // Align the size: ((size + align - 1) / align) * align
2239 Size = CGF.Builder.CreateNUWAdd(
2240 Size, CGM.getSize(Align - CharUnits::fromQuantity(1)));
2241 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align));
2242 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align));
2243 } else {
2244 CharUnits Sz = CGM.getContext().getTypeSizeInChars(CVD->getType());
2245 Size = CGM.getSize(Sz.alignTo(Align));
2246 }
2247
2248 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
2249 assert(AA->getAllocator() &&
2250 "Expected allocator expression for non-default allocator.");
2251 llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator());
2252 // According to the standard, the original allocator type is a enum (integer).
2253 // Convert to pointer type, if required.
2254 if (Allocator->getType()->isIntegerTy())
2255 Allocator = CGF.Builder.CreateIntToPtr(Allocator, CGM.VoidPtrTy);
2256 else if (Allocator->getType()->isPointerTy())
2257 Allocator = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Allocator,
2258 CGM.VoidPtrTy);
2259
2260 llvm::Value *Addr = OMPBuilder.createOMPAlloc(
2261 CGF.Builder, Size, Allocator,
2262 getNameWithSeparators({CVD->getName(), ".void.addr"}, ".", "."));
2263 llvm::CallInst *FreeCI =
2264 OMPBuilder.createOMPFree(CGF.Builder, Addr, Allocator);
2265
2266 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(NormalAndEHCleanup, FreeCI);
2268 Addr,
2269 CGF.ConvertTypeForMem(CGM.getContext().getPointerType(CVD->getType())),
2270 getNameWithSeparators({CVD->getName(), ".addr"}, ".", "."));
2271 return Address(Addr, CGF.ConvertTypeForMem(CVD->getType()), Align);
2272}
2273
2275 CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr,
2276 SourceLocation Loc) {
2277 CodeGenModule &CGM = CGF.CGM;
2278 if (CGM.getLangOpts().OpenMPUseTLS &&
2279 CGM.getContext().getTargetInfo().isTLSSupported())
2280 return VDAddr;
2281
2282 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2283
2284 llvm::Type *VarTy = VDAddr.getElementType();
2285 llvm::Value *Data =
2286 CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.Int8PtrTy);
2287 llvm::ConstantInt *Size = CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy));
2288 std::string Suffix = getNameWithSeparators({"cache", ""});
2289 llvm::Twine CacheName = Twine(CGM.getMangledName(VD)).concat(Suffix);
2290
2291 llvm::CallInst *ThreadPrivateCacheCall =
2292 OMPBuilder.createCachedThreadPrivate(CGF.Builder, Data, Size, CacheName);
2293
2294 return Address(ThreadPrivateCacheCall, CGM.Int8Ty, VDAddr.getAlignment());
2295}
2296
2298 ArrayRef<StringRef> Parts, StringRef FirstSeparator, StringRef Separator) {
2299 SmallString<128> Buffer;
2300 llvm::raw_svector_ostream OS(Buffer);
2301 StringRef Sep = FirstSeparator;
2302 for (StringRef Part : Parts) {
2303 OS << Sep << Part;
2304 Sep = Separator;
2305 }
2306 return OS.str().str();
2307}
2308
2310 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
2311 InsertPointTy CodeGenIP, Twine RegionName) {
2313 Builder.restoreIP(CodeGenIP);
2314 llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder, /*CreateBranch=*/false,
2315 "." + RegionName + ".after");
2316
2317 {
2318 OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);
2319 CGF.EmitStmt(RegionBodyStmt);
2320 }
2321
2322 if (Builder.saveIP().isValid())
2323 Builder.CreateBr(FiniBB);
2324}
2325
2327 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
2328 InsertPointTy CodeGenIP, Twine RegionName) {
2330 Builder.restoreIP(CodeGenIP);
2331 llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder, /*CreateBranch=*/false,
2332 "." + RegionName + ".after");
2333
2334 {
2335 OMPBuilderCBHelpers::OutlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);
2336 CGF.EmitStmt(RegionBodyStmt);
2337 }
2338
2339 if (Builder.saveIP().isValid())
2340 Builder.CreateBr(FiniBB);
2341}
2342
2343void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
2344 if (CGM.getLangOpts().OpenMPIRBuilder) {
2345 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2346 // Check if we have any if clause associated with the directive.
2347 llvm::Value *IfCond = nullptr;
2348 if (const auto *C = S.getSingleClause<OMPIfClause>())
2349 IfCond = EmitScalarExpr(C->getCondition(),
2350 /*IgnoreResultAssign=*/true);
2351
2352 llvm::Value *NumThreads = nullptr;
2353 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>())
2354 NumThreads = EmitScalarExpr(NumThreadsClause->getNumThreads().front(),
2355 /*IgnoreResultAssign=*/true);
2356
2357 ProcBindKind ProcBind = OMP_PROC_BIND_default;
2358 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>())
2359 ProcBind = ProcBindClause->getProcBindKind();
2360
2361 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
2362
2363 // The cleanup callback that finalizes all variables at the given location,
2364 // thus calls destructors etc.
2365 auto FiniCB = [this](InsertPointTy IP) {
2367 return llvm::Error::success();
2368 };
2369
2370 // Privatization callback that performs appropriate action for
2371 // shared/private/firstprivate/lastprivate/copyin/... variables.
2372 //
2373 // TODO: This defaults to shared right now.
2374 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
2375 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
2376 // The next line is appropriate only for variables (Val) with the
2377 // data-sharing attribute "shared".
2378 ReplVal = &Val;
2379
2380 return CodeGenIP;
2381 };
2382
2383 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
2384 const Stmt *ParallelRegionBodyStmt = CS->getCapturedStmt();
2385
2386 auto BodyGenCB = [&, this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
2387 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
2389 *this, ParallelRegionBodyStmt, AllocIP, CodeGenIP, "parallel");
2390 return llvm::Error::success();
2391 };
2392
2393 CGCapturedStmtInfo CGSI(*CS, CR_OpenMP);
2394 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
2395 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
2396 AllocaInsertPt->getIterator());
2397 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
2398 cantFail(OMPBuilder.createParallel(
2399 Builder, AllocaIP, /*DeallocBlocks=*/{}, BodyGenCB, PrivCB, FiniCB,
2400 IfCond, NumThreads, ProcBind, S.hasCancel()));
2401 Builder.restoreIP(AfterIP);
2402 return;
2403 }
2404
2405 // Emit parallel region as a standalone region.
2406 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2407 Action.Enter(CGF);
2408 OMPPrivateScope PrivateScope(CGF);
2409 emitOMPCopyinClause(CGF, S);
2410 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
2411 CGF.EmitOMPPrivateClause(S, PrivateScope);
2412 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
2413 (void)PrivateScope.Privatize();
2414 CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
2415 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
2416 };
2417 {
2418 auto LPCRegion =
2420 emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
2423 [](CodeGenFunction &) { return nullptr; });
2424 }
2425 // Check for outer lastprivate conditional update.
2427}
2428
2430 // TODO: Select the variant at runtime.
2431 EmitStmt(S.variants().front());
2432}
2433
2434namespace {
2435/// RAII to handle scopes for loop transformation directives.
2436class OMPTransformDirectiveScopeRAII {
2437 OMPLoopScope *Scope = nullptr;
2439 CodeGenFunction::CGCapturedStmtRAII *CapInfoRAII = nullptr;
2440
2441 OMPTransformDirectiveScopeRAII(const OMPTransformDirectiveScopeRAII &) =
2442 delete;
2443 OMPTransformDirectiveScopeRAII &
2444 operator=(const OMPTransformDirectiveScopeRAII &) = delete;
2445
2446public:
2447 OMPTransformDirectiveScopeRAII(CodeGenFunction &CGF, const Stmt *S) {
2448 if (const auto *Dir = dyn_cast<OMPLoopBasedDirective>(S)) {
2449 Scope = new OMPLoopScope(CGF, *Dir);
2451 CapInfoRAII = new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI);
2452 } else if (const auto *Dir =
2453 dyn_cast<OMPCanonicalLoopSequenceTransformationDirective>(
2454 S)) {
2455 // For simplicity we reuse the loop scope similarly to what we do with
2456 // OMPCanonicalLoopNestTransformationDirective do by being a subclass
2457 // of OMPLoopBasedDirective.
2458 Scope = new OMPLoopScope(CGF, *Dir);
2460 CapInfoRAII = new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI);
2461 }
2462 }
2463 ~OMPTransformDirectiveScopeRAII() {
2464 if (!Scope)
2465 return;
2466 delete CapInfoRAII;
2467 delete CGSI;
2468 delete Scope;
2469 }
2470};
2471} // namespace
2472
2473static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
2474 int MaxLevel, int Level = 0) {
2475 assert(Level < MaxLevel && "Too deep lookup during loop body codegen.");
2476 const Stmt *SimplifiedS = S->IgnoreContainers();
2477 if (const auto *CS = dyn_cast<CompoundStmt>(SimplifiedS)) {
2478 PrettyStackTraceLoc CrashInfo(
2479 CGF.getContext().getSourceManager(), CS->getLBracLoc(),
2480 "LLVM IR generation of compound statement ('{}')");
2481
2482 // Keep track of the current cleanup stack depth, including debug scopes.
2484 for (const Stmt *CurStmt : CS->body())
2485 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level);
2486 return;
2487 }
2488
2489 // `tryToFindNextInnerLoop` keeps the intra-tile hint wrapper around, so match
2490 // against the loop it annotates. The tile overshoot guard is emitted
2491 // separately via EmitOMPLoopBody's finals-conditions handling.
2492 if (SimplifiedS == OMPLoopBasedDirective::ignoreIntraTileHint(NextLoop)) {
2493 if (auto *Dir = dyn_cast<OMPLoopTransformationDirective>(SimplifiedS))
2494 SimplifiedS = Dir->getTransformedStmt();
2495 if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(SimplifiedS))
2496 SimplifiedS = CanonLoop->getLoopStmt();
2497 if (const auto *For = dyn_cast<ForStmt>(SimplifiedS)) {
2498 S = For->getBody();
2499 } else {
2500 assert(isa<CXXForRangeStmt>(SimplifiedS) &&
2501 "Expected canonical for loop or range-based for loop.");
2502 const auto *CXXFor = cast<CXXForRangeStmt>(SimplifiedS);
2503 CGF.EmitStmt(CXXFor->getLoopVarStmt());
2504 S = CXXFor->getBody();
2505 }
2506 if (Level + 1 < MaxLevel) {
2507 NextLoop = OMPLoopDirective::tryToFindNextInnerLoop(
2508 S, /*TryImperfectlyNestedLoops=*/true);
2509 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1);
2510 return;
2511 }
2512 }
2513 CGF.EmitStmt(S);
2514}
2515
2518 RunCleanupsScope BodyScope(*this);
2519 // Update counters values on current iteration.
2520 for (const Expr *UE : D.updates())
2521 EmitIgnoredExpr(UE);
2522 // Update the linear variables.
2523 // In distribute directives only loop counters may be marked as linear, no
2524 // need to generate the code for them.
2526 if (!isOpenMPDistributeDirective(EKind)) {
2527 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2528 for (const Expr *UE : C->updates())
2529 EmitIgnoredExpr(UE);
2530 }
2531 }
2532
2533 // On a continue in the body, jump to the end.
2534 JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue");
2535 BreakContinueStack.push_back(BreakContinue(D, LoopExit, Continue));
2536 for (const Expr *E : D.finals_conditions()) {
2537 if (!E)
2538 continue;
2539 // Check that loop counter in non-rectangular nest fits into the iteration
2540 // space.
2541 llvm::BasicBlock *NextBB = createBasicBlock("omp.body.next");
2542 EmitBranchOnBoolExpr(E, NextBB, Continue.getBlock(),
2544 EmitBlock(NextBB);
2545 }
2546
2547 OMPPrivateScope InscanScope(*this);
2548 EmitOMPReductionClauseInit(D, InscanScope, /*ForInscan=*/true);
2549 bool IsInscanRegion = InscanScope.Privatize();
2550 if (IsInscanRegion) {
2551 // Need to remember the block before and after scan directive
2552 // to dispatch them correctly depending on the clause used in
2553 // this directive, inclusive or exclusive. For inclusive scan the natural
2554 // order of the blocks is used, for exclusive clause the blocks must be
2555 // executed in reverse order.
2556 OMPBeforeScanBlock = createBasicBlock("omp.before.scan.bb");
2557 OMPAfterScanBlock = createBasicBlock("omp.after.scan.bb");
2558 // No need to allocate inscan exit block, in simd mode it is selected in the
2559 // codegen for the scan directive.
2560 if (EKind != OMPD_simd && !getLangOpts().OpenMPSimd)
2561 OMPScanExitBlock = createBasicBlock("omp.exit.inscan.bb");
2562 OMPScanDispatch = createBasicBlock("omp.inscan.dispatch");
2565 }
2566
2567 // Emit loop variables for C++ range loops.
2568 const Stmt *Body =
2569 D.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers();
2570 // Emit loop body.
2571 emitBody(*this, Body,
2572 OMPLoopBasedDirective::tryToFindNextInnerLoop(
2573 Body, /*TryImperfectlyNestedLoops=*/true),
2574 D.getLoopsNumber());
2575
2576 // Jump to the dispatcher at the end of the loop body.
2577 if (IsInscanRegion)
2579
2580 // The end (updates/cleanups).
2581 EmitBlock(Continue.getBlock());
2582 BreakContinueStack.pop_back();
2583}
2584
2585using EmittedClosureTy = std::pair<llvm::Function *, llvm::Value *>;
2586
2587/// Emit a captured statement and return the function as well as its captured
2588/// closure context.
2590 const CapturedStmt *S) {
2591 LValue CapStruct = ParentCGF.InitCapturedStruct(*S);
2592 CodeGenFunction CGF(ParentCGF.CGM, /*suppressNewContext=*/true);
2593 std::unique_ptr<CodeGenFunction::CGCapturedStmtInfo> CSI =
2594 std::make_unique<CodeGenFunction::CGCapturedStmtInfo>(*S);
2595 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, CSI.get());
2596 llvm::Function *F = CGF.GenerateCapturedStmtFunction(*S);
2597
2598 return {F, CapStruct.getPointer(ParentCGF)};
2599}
2600
2601/// Emit a call to a previously captured closure.
2602static llvm::CallInst *
2605 // Append the closure context to the argument.
2606 SmallVector<llvm::Value *> EffectiveArgs;
2607 EffectiveArgs.reserve(Args.size() + 1);
2608 llvm::append_range(EffectiveArgs, Args);
2609 EffectiveArgs.push_back(Cap.second);
2610
2611 return ParentCGF.Builder.CreateCall(Cap.first, EffectiveArgs);
2612}
2613
2614llvm::CanonicalLoopInfo *
2616 assert(Depth == 1 && "Nested loops with OpenMPIRBuilder not yet implemented");
2617
2618 // The caller is processing the loop-associated directive processing the \p
2619 // Depth loops nested in \p S. Put the previous pending loop-associated
2620 // directive to the stack. If the current loop-associated directive is a loop
2621 // transformation directive, it will push its generated loops onto the stack
2622 // such that together with the loops left here they form the combined loop
2623 // nest for the parent loop-associated directive.
2624 int ParentExpectedOMPLoopDepth = ExpectedOMPLoopDepth;
2625 ExpectedOMPLoopDepth = Depth;
2626
2627 EmitStmt(S);
2628 assert(OMPLoopNestStack.size() >= (size_t)Depth && "Found too few loops");
2629
2630 // The last added loop is the outermost one.
2631 llvm::CanonicalLoopInfo *Result = OMPLoopNestStack.back();
2632
2633 // Pop the \p Depth loops requested by the call from that stack and restore
2634 // the previous context.
2635 OMPLoopNestStack.pop_back_n(Depth);
2636 ExpectedOMPLoopDepth = ParentExpectedOMPLoopDepth;
2637
2638 return Result;
2639}
2640
2641void CodeGenFunction::EmitOMPCanonicalLoop(const OMPCanonicalLoop *S) {
2642 const Stmt *SyntacticalLoop = S->getLoopStmt();
2643 if (!getLangOpts().OpenMPIRBuilder) {
2644 // Ignore if OpenMPIRBuilder is not enabled.
2645 EmitStmt(SyntacticalLoop);
2646 return;
2647 }
2648
2649 LexicalScope ForScope(*this, S->getSourceRange());
2650
2651 // Emit init statements. The Distance/LoopVar funcs may reference variable
2652 // declarations they contain.
2653 const Stmt *BodyStmt;
2654 if (const auto *For = dyn_cast<ForStmt>(SyntacticalLoop)) {
2655 if (const Stmt *InitStmt = For->getInit())
2656 EmitStmt(InitStmt);
2657 BodyStmt = For->getBody();
2658 } else if (const auto *RangeFor =
2659 dyn_cast<CXXForRangeStmt>(SyntacticalLoop)) {
2660 if (const DeclStmt *RangeStmt = RangeFor->getRangeStmt())
2661 EmitStmt(RangeStmt);
2662 if (const DeclStmt *BeginStmt = RangeFor->getBeginStmt())
2663 EmitStmt(BeginStmt);
2664 if (const DeclStmt *EndStmt = RangeFor->getEndStmt())
2665 EmitStmt(EndStmt);
2666 if (const DeclStmt *LoopVarStmt = RangeFor->getLoopVarStmt())
2667 EmitStmt(LoopVarStmt);
2668 BodyStmt = RangeFor->getBody();
2669 } else
2670 llvm_unreachable("Expected for-stmt or range-based for-stmt");
2671
2672 // Emit closure for later use. By-value captures will be captured here.
2673 const CapturedStmt *DistanceFunc = S->getDistanceFunc();
2674 EmittedClosureTy DistanceClosure = emitCapturedStmtFunc(*this, DistanceFunc);
2675 const CapturedStmt *LoopVarFunc = S->getLoopVarFunc();
2676 EmittedClosureTy LoopVarClosure = emitCapturedStmtFunc(*this, LoopVarFunc);
2677
2678 // Call the distance function to get the number of iterations of the loop to
2679 // come.
2680 QualType LogicalTy = DistanceFunc->getCapturedDecl()
2681 ->getParam(0)
2682 ->getType()
2684 RawAddress CountAddr = CreateMemTemp(LogicalTy, ".count.addr");
2685 emitCapturedStmtCall(*this, DistanceClosure, {CountAddr.getPointer()});
2686 llvm::Value *DistVal = Builder.CreateLoad(CountAddr, ".count");
2687
2688 // Emit the loop structure.
2689 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2690 auto BodyGen = [&, this](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP,
2691 llvm::Value *IndVar) {
2692 Builder.restoreIP(CodeGenIP);
2693
2694 // Emit the loop body: Convert the logical iteration number to the loop
2695 // variable and emit the body.
2696 const DeclRefExpr *LoopVarRef = S->getLoopVarRef();
2697 LValue LCVal = EmitLValue(LoopVarRef);
2698 Address LoopVarAddress = LCVal.getAddress();
2699 emitCapturedStmtCall(*this, LoopVarClosure,
2700 {LoopVarAddress.emitRawPointer(*this), IndVar});
2701
2702 RunCleanupsScope BodyScope(*this);
2703 EmitStmt(BodyStmt);
2704 return llvm::Error::success();
2705 };
2706
2707 llvm::CanonicalLoopInfo *CL =
2708 cantFail(OMPBuilder.createCanonicalLoop(Builder, BodyGen, DistVal));
2709
2710 // Finish up the loop.
2711 Builder.restoreIP(CL->getAfterIP());
2712 ForScope.ForceCleanup();
2713
2714 // Remember the CanonicalLoopInfo for parent AST nodes consuming it.
2715 OMPLoopNestStack.push_back(CL);
2716}
2717
2718/// A directive's own source range covers only the pragma, so use the range of
2719/// the associated loop, looking through loop transformation directives.
2721 if (!isa<OMPLoopDirective>(S))
2722 return S.getSourceRange();
2723 const Stmt *Loop = S.getRawStmt()->IgnoreContainers();
2725 Loop = cast<OMPExecutableDirective>(Loop)->getRawStmt()->IgnoreContainers();
2726 return Loop->getSourceRange();
2727}
2728
2730 const OMPExecutableDirective &S, bool RequiresCleanup, const Expr *LoopCond,
2731 const Expr *IncExpr,
2732 const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
2733 const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
2734 auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
2735
2736 // Start the loop with a block that tests the condition.
2737 auto CondBlock = createBasicBlock("omp.inner.for.cond");
2738 EmitBlock(CondBlock);
2740
2741 // If attributes are attached, push to the basic block with them.
2742 const auto &OMPED = cast<OMPExecutableDirective>(S);
2743 const CapturedStmt *ICS = OMPED.getInnermostCapturedStmt();
2744 const Stmt *SS = ICS->getCapturedStmt();
2745 const AttributedStmt *AS = dyn_cast_or_null<AttributedStmt>(SS);
2746 OMPLoopNestStack.clear();
2747 if (AS)
2748 LoopStack.push(CondBlock, CGM.getContext(), CGM.getCodeGenOpts(),
2749 AS->getAttrs(), SourceLocToDebugLoc(R.getBegin()),
2750 SourceLocToDebugLoc(R.getEnd()));
2751 else
2752 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
2753 SourceLocToDebugLoc(R.getEnd()));
2754
2755 // If there are any cleanups between here and the loop-exit scope,
2756 // create a block to stage a loop exit along.
2757 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
2758 if (RequiresCleanup)
2759 ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
2760
2761 llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body");
2762
2763 // Emit condition.
2764 EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
2765 if (ExitBlock != LoopExit.getBlock()) {
2766 EmitBlock(ExitBlock);
2768 }
2769
2770 EmitBlock(LoopBody);
2772
2773 // Create a block for the increment.
2774 JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
2775 BreakContinueStack.push_back(BreakContinue(S, LoopExit, Continue));
2776
2777 BodyGen(*this);
2778
2779 // Emit "IV = IV + 1" and a back-edge to the condition block.
2780 EmitBlock(Continue.getBlock());
2781 EmitIgnoredExpr(IncExpr);
2782 PostIncGen(*this);
2783 BreakContinueStack.pop_back();
2784 EmitBranch(CondBlock);
2785 LoopStack.pop();
2786 // Emit the fall-through block.
2787 EmitBlock(LoopExit.getBlock());
2788}
2789
2791 if (!HaveInsertPoint())
2792 return false;
2793 // Emit inits for the linear variables.
2794 bool HasLinears = false;
2795 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2796 for (const Expr *Init : C->inits()) {
2797 HasLinears = true;
2798 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
2799 if (const auto *Ref =
2800 dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
2801 if (isa<BindingDecl>(Ref->getDecl())) {
2802 EmitVarDecl(*VD);
2803 } else {
2804 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
2805 const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
2806 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
2807 CapturedStmtInfo->lookup(OrigVD) != nullptr,
2808 VD->getInit()->getType(), VK_LValue,
2809 VD->getInit()->getExprLoc());
2811 &DRE, VD,
2812 MakeAddrLValue(Emission.getAllocatedAddress(), VD->getType()),
2813 /*capturedByInit=*/false);
2814 EmitAutoVarCleanups(Emission);
2815 }
2816 } else {
2817 EmitVarDecl(*VD);
2818 }
2819 }
2820 // Emit the linear steps for the linear clauses.
2821 // If a step is not constant, it is pre-calculated before the loop.
2822 if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
2823 if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
2824 EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
2825 // Emit calculation of the linear step.
2826 EmitIgnoredExpr(CS);
2827 }
2828 }
2829 return HasLinears;
2830}
2831
2833 const OMPLoopDirective &D,
2834 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
2835 if (!HaveInsertPoint())
2836 return;
2837 llvm::BasicBlock *DoneBB = nullptr;
2838 // Emit the final values of the linear variables.
2839 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2840 auto IC = C->varlist_begin();
2841 for (const Expr *F : C->finals()) {
2842 if (!DoneBB) {
2843 if (llvm::Value *Cond = CondGen(*this)) {
2844 // If the first post-update expression is found, emit conditional
2845 // block if it was requested.
2846 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu");
2847 DoneBB = createBasicBlock(".omp.linear.pu.done");
2848 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2849 EmitBlock(ThenBB);
2850 }
2851 }
2852 const auto *OrigDecl = cast<DeclRefExpr>(*IC)->getDecl();
2853 Address OrigAddr = Address::invalid();
2854 if (const auto *BD = dyn_cast<BindingDecl>(OrigDecl)) {
2855 OrigAddr = EmitOMPBindingOriginalAddr(BD, (*IC)->getExprLoc());
2856 } else {
2857 const auto *OrigVD = cast<VarDecl>(OrigDecl);
2858 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
2859 CapturedStmtInfo->lookup(OrigVD) != nullptr,
2860 (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
2861 OrigAddr = EmitLValue(&DRE).getAddress();
2862 }
2863 CodeGenFunction::OMPPrivateScope VarScope(*this);
2864 VarScope.addPrivate(OrigDecl, OrigAddr);
2865 (void)VarScope.Privatize();
2866 EmitIgnoredExpr(F);
2867 ++IC;
2868 }
2869 if (const Expr *PostUpdate = C->getPostUpdateExpr())
2870 EmitIgnoredExpr(PostUpdate);
2871 }
2872 if (DoneBB)
2873 EmitBlock(DoneBB, /*IsFinished=*/true);
2874}
2875
2877 const OMPExecutableDirective &D) {
2878 if (!CGF.HaveInsertPoint())
2879 return;
2880 for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
2881 llvm::APInt ClauseAlignment(64, 0);
2882 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
2883 auto *AlignmentCI =
2884 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
2885 ClauseAlignment = AlignmentCI->getValue();
2886 }
2887 for (const Expr *E : Clause->varlist()) {
2888 llvm::APInt Alignment(ClauseAlignment);
2889 if (Alignment == 0) {
2890 // OpenMP [2.8.1, Description]
2891 // If no optional parameter is specified, implementation-defined default
2892 // alignments for SIMD instructions on the target platforms are assumed.
2893 Alignment =
2894 CGF.getContext()
2896 E->getType()->getPointeeType()))
2897 .getQuantity();
2898 }
2899 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2900 "alignment is not power of 2");
2901 if (Alignment != 0) {
2902 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
2904 PtrValue, E, /*No second loc needed*/ SourceLocation(),
2905 llvm::ConstantInt::get(CGF.getLLVMContext(), Alignment));
2906 }
2907 }
2908 }
2909}
2910
2913 if (!HaveInsertPoint())
2914 return;
2915 auto I = S.private_counters().begin();
2916 for (const Expr *E : S.counters()) {
2917 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2918 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
2919 // Emit var without initialization.
2920 AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD);
2921 EmitAutoVarCleanups(VarEmission);
2922 LocalDeclMap.erase(PrivateVD);
2923 (void)LoopScope.addPrivate(VD, VarEmission.getAllocatedAddress());
2924 if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
2925 VD->hasGlobalStorage()) {
2926 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD),
2927 LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
2928 E->getType(), VK_LValue, E->getExprLoc());
2929 (void)LoopScope.addPrivate(PrivateVD, EmitLValue(&DRE).getAddress());
2930 } else {
2931 (void)LoopScope.addPrivate(PrivateVD, VarEmission.getAllocatedAddress());
2932 }
2933 ++I;
2934 }
2935 // Privatize extra loop counters used in loops for ordered(n) clauses.
2936 for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) {
2937 if (!C->getNumForLoops())
2938 continue;
2939 for (unsigned I = S.getLoopsNumber(), E = C->getLoopNumIterations().size();
2940 I < E; ++I) {
2941 const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I));
2942 const auto *VD = cast<VarDecl>(DRE->getDecl());
2943 // Override only those variables that can be captured to avoid re-emission
2944 // of the variables declared within the loops.
2945 if (DRE->refersToEnclosingVariableOrCapture()) {
2946 (void)LoopScope.addPrivate(
2947 VD, CreateMemTemp(DRE->getType(), VD->getName()));
2948 }
2949 }
2950 }
2951}
2952
2954 const Expr *Cond, llvm::BasicBlock *TrueBlock,
2955 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
2956 if (!CGF.HaveInsertPoint())
2957 return;
2958 {
2959 CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
2960 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
2961 (void)PreCondScope.Privatize();
2962 // Get initial values of real counters.
2963 for (const Expr *I : S.inits()) {
2964 CGF.EmitIgnoredExpr(I);
2965 }
2966 }
2967 // Create temp loop control variables with their init values to support
2968 // non-rectangular loops.
2969 CodeGenFunction::OMPMapVars PreCondVars;
2970 for (const Expr *E : S.dependent_counters()) {
2971 if (!E)
2972 continue;
2973 assert(!E->getType().getNonReferenceType()->isRecordType() &&
2974 "dependent counter must not be an iterator.");
2975 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2976 Address CounterAddr =
2978 (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr);
2979 }
2980 (void)PreCondVars.apply(CGF);
2981 for (const Expr *E : S.dependent_inits()) {
2982 if (!E)
2983 continue;
2984 CGF.EmitIgnoredExpr(E);
2985 }
2986 // Check that loop is executed at least one time.
2987 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
2988 PreCondVars.restore(CGF);
2989}
2990
2992 const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
2993 if (!HaveInsertPoint())
2994 return;
2995 llvm::DenseSet<const VarDecl *> SIMDLCVs;
2997 if (isOpenMPSimdDirective(EKind)) {
2998 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
2999 for (const Expr *C : LoopDirective->counters()) {
3000 SIMDLCVs.insert(
3002 }
3003 }
3004 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
3005 auto CurPrivate = C->privates().begin();
3006 for (const Expr *E : C->varlist()) {
3007 const auto *VD = cast<DeclRefExpr>(E)->getDecl();
3008 const auto *PrivateVD =
3009 cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
3010 bool IsSIMDLCV = false;
3011 if (const auto *VarD = dyn_cast<VarDecl>(VD))
3012 IsSIMDLCV = SIMDLCVs.count(VarD->getCanonicalDecl());
3013 if (!IsSIMDLCV) {
3014 // Emit private VarDecl with copy init.
3015 EmitVarDecl(*PrivateVD);
3016 Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
3017 bool IsRegistered = PrivateScope.addPrivate(VD, PrivateAddr);
3018 assert(IsRegistered && "linear var already registered as private");
3019 // Silence the warning about unused variable.
3020 (void)IsRegistered;
3021 } else {
3022 EmitVarDecl(*PrivateVD);
3023 }
3024 ++CurPrivate;
3025 }
3026 }
3027}
3028
3030 const OMPExecutableDirective &D) {
3031 if (!CGF.HaveInsertPoint())
3032 return;
3033 if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
3034 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
3035 /*ignoreResult=*/true);
3036 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
3037 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
3038 // In presence of finite 'safelen', it may be unsafe to mark all
3039 // the memory instructions parallel, because loop-carried
3040 // dependences of 'safelen' iterations are possible.
3041 CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>());
3042 } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
3043 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
3044 /*ignoreResult=*/true);
3045 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
3046 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
3047 // In presence of finite 'safelen', it may be unsafe to mark all
3048 // the memory instructions parallel, because loop-carried
3049 // dependences of 'safelen' iterations are possible.
3050 CGF.LoopStack.setParallel(/*Enable=*/false);
3051 }
3052}
3053
3054// Check for the presence of an `OMPOrderedBlockAssocDirective`,
3055// i.e., `ordered` in `#pragma omp ordered simd`.
3056//
3057// Consider the following source code:
3058// ```
3059// __attribute__((noinline)) void omp_simd_loop(float X[ARRAY_SIZE][ARRAY_SIZE])
3060// {
3061// for (int r = 1; r < ARRAY_SIZE; ++r) {
3062// for (int c = 1; c < ARRAY_SIZE; ++c) {
3063// #pragma omp simd
3064// for (int k = 2; k < ARRAY_SIZE; ++k) {
3065// #pragma omp ordered simd
3066// X[r][k] = X[r][k - 2] + sinf((float)(r / c));
3067// }
3068// }
3069// }
3070// }
3071// ```
3072//
3073// Suppose we are in `CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective
3074// &D)`. By examining `D.dump()` we have the following AST containing
3075// `OMPOrderedBlockAssocDirective`:
3076//
3077// ```
3078// OMPSimdDirective 0x1c32950
3079// `-CapturedStmt 0x1c32028
3080// |-CapturedDecl 0x1c310e8
3081// | |-ForStmt 0x1c31e30
3082// | | |-DeclStmt 0x1c31298
3083// | | | `-VarDecl 0x1c31208 used k 'int' cinit
3084// | | | `-IntegerLiteral 0x1c31278 'int' 2
3085// | | |-<<<NULL>>>
3086// | | |-BinaryOperator 0x1c31308 'int' '<'
3087// | | | |-ImplicitCastExpr 0x1c312f0 'int' <LValueToRValue>
3088// | | | | `-DeclRefExpr 0x1c312b0 'int' lvalue Var 0x1c31208 'k' 'int'
3089// | | | `-IntegerLiteral 0x1c312d0 'int' 256
3090// | | |-UnaryOperator 0x1c31348 'int' prefix '++'
3091// | | | `-DeclRefExpr 0x1c31328 'int' lvalue Var 0x1c31208 'k' 'int'
3092// | | `-CompoundStmt 0x1c31e18
3093// | | `-OMPOrderedBlockAssocDirective 0x1c31dd8
3094// | | |-OMPSimdClause 0x1c31380
3095// | | `-CapturedStmt 0x1c31cd0
3096// ```
3097//
3098// Note the presence of `OMPOrderedBlockAssocDirective` above:
3099// It's (transitively) nested in a `CapturedStmt` representing the pragma
3100// annotated compound statement. Thus, we need to consider this nesting and
3101// include checking the `getCapturedStmt` in this case.
3104 return true;
3105
3106 if (const auto *CS = dyn_cast<CapturedStmt>(S))
3108
3109 for (const Stmt *Child : S->children()) {
3110 if (Child && hasOrderedBlockAssocDirective(Child))
3111 return true;
3112 }
3113
3114 return false;
3115}
3116
3117static void applyConservativeSimdOrderedDirective(const Stmt &AssociatedStmt,
3119 // Check for the presence of an `OMPOrderedBlockAssocDirective`
3120 // i.e., `ordered` in `#pragma omp ordered simd`
3121 bool HasOrderedDirective = hasOrderedBlockAssocDirective(&AssociatedStmt);
3122 // If present then conservatively disable loop vectorization
3123 // analogously to how `emitSimdlenSafelenClause` does.
3124 if (HasOrderedDirective)
3125 LoopStack.setParallel(/*Enable=*/false);
3126}
3127
3129 // Walk clauses and process safelen/lastprivate.
3130 LoopStack.setParallel(/*Enable=*/true);
3131 LoopStack.setVectorizeEnable();
3132 const Stmt *AssociatedStmt = D.getAssociatedStmt();
3134 emitSimdlenSafelenClause(*this, D);
3135 if (const auto *C = D.getSingleClause<OMPOrderClause>())
3136 if (C->getKind() == OMPC_ORDER_concurrent)
3137 LoopStack.setParallel(/*Enable=*/true);
3139 if ((EKind == OMPD_simd ||
3140 (getLangOpts().OpenMPSimd && isOpenMPSimdDirective(EKind))) &&
3141 llvm::any_of(D.getClausesOfKind<OMPReductionClause>(),
3142 [](const OMPReductionClause *C) {
3143 return C->getModifier() == OMPC_REDUCTION_inscan;
3144 }))
3145 // Disable parallel access in case of prefix sum.
3146 LoopStack.setParallel(/*Enable=*/false);
3147}
3148
3150 const OMPLoopDirective &D,
3151 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
3152 if (!HaveInsertPoint())
3153 return;
3154 llvm::BasicBlock *DoneBB = nullptr;
3155 auto IC = D.counters().begin();
3156 auto IPC = D.private_counters().begin();
3157 for (const Expr *F : D.finals()) {
3158 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
3159 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
3160 const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
3161 if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
3162 OrigVD->hasGlobalStorage() || CED) {
3163 if (!DoneBB) {
3164 if (llvm::Value *Cond = CondGen(*this)) {
3165 // If the first post-update expression is found, emit conditional
3166 // block if it was requested.
3167 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then");
3168 DoneBB = createBasicBlock(".omp.final.done");
3169 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
3170 EmitBlock(ThenBB);
3171 }
3172 }
3173 Address OrigAddr = Address::invalid();
3174 if (CED) {
3175 OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();
3176 } else {
3177 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(PrivateVD),
3178 /*RefersToEnclosingVariableOrCapture=*/false,
3179 (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
3180 OrigAddr = EmitLValue(&DRE).getAddress();
3181 }
3182 OMPPrivateScope VarScope(*this);
3183 VarScope.addPrivate(OrigVD, OrigAddr);
3184 (void)VarScope.Privatize();
3185 EmitIgnoredExpr(F);
3186 }
3187 ++IC;
3188 ++IPC;
3189 }
3190 if (DoneBB)
3191 EmitBlock(DoneBB, /*IsFinished=*/true);
3192}
3193
3200
3201/// Emit a helper variable and return corresponding lvalue.
3203 const DeclRefExpr *Helper) {
3204 auto VDecl = cast<VarDecl>(Helper->getDecl());
3205 CGF.EmitVarDecl(*VDecl);
3206 return CGF.EmitLValue(Helper);
3207}
3208
3210 const RegionCodeGenTy &SimdInitGen,
3211 const RegionCodeGenTy &BodyCodeGen) {
3212 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](CodeGenFunction &CGF,
3213 PrePostActionTy &) {
3214 CGOpenMPRuntime::NontemporalDeclsRAII NontemporalsRegion(CGF.CGM, S);
3216 SimdInitGen(CGF);
3217
3218 BodyCodeGen(CGF);
3219 };
3220 auto &&ElseGen = [&BodyCodeGen](CodeGenFunction &CGF, PrePostActionTy &) {
3222 CGF.LoopStack.setVectorizeEnable(/*Enable=*/false);
3223
3224 BodyCodeGen(CGF);
3225 };
3226 const Expr *IfCond = nullptr;
3227 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
3228 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3229 if (CGF.getLangOpts().OpenMP >= 50 &&
3230 (C->getNameModifier() == OMPD_unknown ||
3231 C->getNameModifier() == OMPD_simd)) {
3232 IfCond = C->getCondition();
3233 break;
3234 }
3235 }
3236 }
3237 if (IfCond) {
3238 CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, IfCond, ThenGen, ElseGen);
3239 } else {
3240 RegionCodeGenTy ThenRCG(ThenGen);
3241 ThenRCG(CGF);
3242 }
3243}
3244
3246 PrePostActionTy &Action) {
3247 Action.Enter(CGF);
3248 OMPLoopScope PreInitScope(CGF, S);
3249 // if (PreCond) {
3250 // for (IV in 0..LastIteration) BODY;
3251 // <Final counter/linear vars updates>;
3252 // }
3253
3254 // The presence of lower/upper bound variable depends on the actual directive
3255 // kind in the AST node. The variables must be emitted because some of the
3256 // expressions associated with the loop will use them.
3257 OpenMPDirectiveKind DKind = S.getDirectiveKind();
3258 if (isOpenMPDistributeDirective(DKind) ||
3263 }
3264
3266 // Emit: if (PreCond) - begin.
3267 // If the condition constant folds and can be elided, avoid emitting the
3268 // whole loop.
3269 bool CondConstant;
3270 llvm::BasicBlock *ContBlock = nullptr;
3271 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
3272 if (!CondConstant)
3273 return;
3274 } else {
3275 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");
3276 ContBlock = CGF.createBasicBlock("simd.if.end");
3277 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
3278 CGF.getProfileCount(&S));
3279 CGF.EmitBlock(ThenBlock);
3281 }
3282
3283 // Emit the loop iteration variable.
3284 const Expr *IVExpr = S.getIterationVariable();
3285 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
3286 CGF.EmitVarDecl(*IVDecl);
3287 CGF.EmitIgnoredExpr(S.getInit());
3288
3289 // Emit the iterations count variable.
3290 // If it is not a variable, Sema decided to calculate iterations count on
3291 // each iteration (e.g., it is foldable into a constant).
3292 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3293 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3294 // Emit calculation of the iterations count.
3296 }
3297
3298 emitAlignedClause(CGF, S);
3299 (void)CGF.EmitOMPLinearClauseInit(S);
3300 {
3301 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
3302 CGF.EmitOMPPrivateClause(S, LoopScope);
3303 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
3304 CGF.EmitOMPLinearClause(S, LoopScope);
3305 CGF.EmitOMPReductionClauseInit(S, LoopScope);
3307 CGF, S, CGF.EmitLValue(S.getIterationVariable()));
3308 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
3309 (void)LoopScope.Privatize();
3312
3314 CGF, S,
3315 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3316 CGF.EmitOMPSimdInit(S);
3317 },
3318 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
3319 CGF.EmitOMPInnerLoop(
3320 S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),
3321 [&S](CodeGenFunction &CGF) {
3322 emitOMPLoopBodyWithStopPoint(CGF, S,
3323 CodeGenFunction::JumpDest());
3324 },
3325 [](CodeGenFunction &) {});
3326 });
3327 CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; });
3328 // Emit final copy of the lastprivate variables at the end of loops.
3329 if (HasLastprivateClause)
3330 CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
3331 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
3333 [](CodeGenFunction &) { return nullptr; });
3334 LoopScope.restoreMap();
3335 CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; });
3336 }
3337 // Emit: if (PreCond) - end.
3338 if (ContBlock) {
3339 CGF.EmitBranch(ContBlock);
3340 CGF.EmitBlock(ContBlock, true);
3341 }
3342}
3343
3344// Pass OMPLoopDirective (instead of OMPSimdDirective) to make this function
3345// available for "loop bind(thread)", which maps to "simd".
3347 // Check for unsupported clauses
3348 for (OMPClause *C : S.clauses()) {
3349 // Currently only order, simdlen and safelen clauses are supported
3352 return false;
3353 }
3354
3355 // Check if we have a statement with the ordered-blockassoc directive.
3356 // Visit the statement hierarchy to find a compound statement
3357 // with a ordered-blockassoc directive in it.
3358 if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(S.getRawStmt())) {
3359 if (const Stmt *SyntacticalLoop = CanonLoop->getLoopStmt()) {
3360 for (const Stmt *SubStmt : SyntacticalLoop->children()) {
3361 if (!SubStmt)
3362 continue;
3363 if (const CompoundStmt *CS = dyn_cast<CompoundStmt>(SubStmt)) {
3364 for (const Stmt *CSSubStmt : CS->children()) {
3365 if (!CSSubStmt)
3366 continue;
3367 if (isa<OMPOrderedBlockAssocDirective>(CSSubStmt)) {
3368 return false;
3369 }
3370 }
3371 }
3372 }
3373 }
3374 }
3375 return true;
3376}
3377
3378static llvm::MapVector<llvm::Value *, llvm::Value *>
3380 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars;
3381 for (const auto *Clause : S.getClausesOfKind<OMPAlignedClause>()) {
3382 llvm::APInt ClauseAlignment(64, 0);
3383 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
3384 auto *AlignmentCI =
3385 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
3386 ClauseAlignment = AlignmentCI->getValue();
3387 }
3388 for (const Expr *E : Clause->varlist()) {
3389 llvm::APInt Alignment(ClauseAlignment);
3390 if (Alignment == 0) {
3391 // OpenMP [2.8.1, Description]
3392 // If no optional parameter is specified, implementation-defined default
3393 // alignments for SIMD instructions on the target platforms are assumed.
3394 Alignment =
3395 CGF.getContext()
3397 E->getType()->getPointeeType()))
3398 .getQuantity();
3399 }
3400 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
3401 "alignment is not power of 2");
3402 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
3403 AlignedVars[PtrValue] = CGF.Builder.getInt64(Alignment.getSExtValue());
3404 }
3405 }
3406 return AlignedVars;
3407}
3408
3409// Pass OMPLoopDirective (instead of OMPSimdDirective) to make this function
3410// available for "loop bind(thread)", which maps to "simd".
3413 bool UseOMPIRBuilder =
3414 CGM.getLangOpts().OpenMPIRBuilder && isSimdSupportedByOpenMPIRBuilder(S);
3415 if (UseOMPIRBuilder) {
3416 auto &&CodeGenIRBuilder = [&S, &CGM, UseOMPIRBuilder](CodeGenFunction &CGF,
3417 PrePostActionTy &) {
3418 // Use the OpenMPIRBuilder if enabled.
3419 if (UseOMPIRBuilder) {
3420 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars =
3421 GetAlignedMapping(S, CGF);
3422 // Emit the associated statement and get its loop representation.
3423 const Stmt *Inner = S.getRawStmt();
3424 llvm::CanonicalLoopInfo *CLI =
3425 CGF.EmitOMPCollapsedCanonicalLoopNest(Inner, 1);
3426
3427 llvm::OpenMPIRBuilder &OMPBuilder =
3429 // Add SIMD specific metadata
3430 llvm::ConstantInt *Simdlen = nullptr;
3431 if (const auto *C = S.getSingleClause<OMPSimdlenClause>()) {
3432 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
3433 /*ignoreResult=*/true);
3434 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
3435 Simdlen = Val;
3436 }
3437 llvm::ConstantInt *Safelen = nullptr;
3438 if (const auto *C = S.getSingleClause<OMPSafelenClause>()) {
3439 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
3440 /*ignoreResult=*/true);
3441 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
3442 Safelen = Val;
3443 }
3444 llvm::omp::OrderKind Order = llvm::omp::OrderKind::OMP_ORDER_unknown;
3445 if (const auto *C = S.getSingleClause<OMPOrderClause>()) {
3446 if (C->getKind() == OpenMPOrderClauseKind::OMPC_ORDER_concurrent) {
3447 Order = llvm::omp::OrderKind::OMP_ORDER_concurrent;
3448 }
3449 }
3450 // Add simd metadata to the collapsed loop. Do not generate
3451 // another loop for if clause. Support for if clause is done earlier.
3452 OMPBuilder.applySimd(CLI, AlignedVars,
3453 /*IfCond*/ nullptr, Order, Simdlen, Safelen);
3454 return;
3455 }
3456 };
3457 {
3458 auto LPCRegion =
3460 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
3461 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
3462 CodeGenIRBuilder);
3463 }
3464 return;
3465 }
3466
3468 CGF.OMPFirstScanLoop = true;
3469 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3470 emitOMPSimdRegion(CGF, S, Action);
3471 };
3472 {
3473 auto LPCRegion =
3475 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
3477 }
3478 // Check for outer lastprivate conditional update.
3480}
3481
3485
3486/// Emit a loop-nest transformation followed by its loop variable finalization,
3487/// which restores the loop-exit values as required by OpenMP 6.0.
3489 CodeGenFunction &CGF,
3491 OMPTransformDirectiveScopeRAII Scope(CGF, &S);
3493 if (Stmt *Finals = S.getFinals())
3494 CGF.EmitStmt(Finals);
3495}
3496
3500
3504
3508
3512
3517
3519 // Emit the de-sugared statement.
3520 OMPTransformDirectiveScopeRAII FlattenScope(*this, &S);
3522 EmitStmt(S.getFinals());
3523}
3524
3526 // Emit the de-sugared statement
3527 OMPTransformDirectiveScopeRAII FuseScope(*this, &S);
3528
3530 if (Stmt *Finals = S.getFinals())
3531 EmitStmt(Finals);
3532}
3533
3535 bool UseOMPIRBuilder = CGM.getLangOpts().OpenMPIRBuilder;
3536
3537 if (UseOMPIRBuilder) {
3538 auto DL = SourceLocToDebugLoc(S.getBeginLoc());
3539 const Stmt *Inner = S.getRawStmt();
3540
3541 // Consume nested loop. Clear the entire remaining loop stack because a
3542 // fully unrolled loop is non-transformable. For partial unrolling the
3543 // generated outer loop is pushed back to the stack.
3544 llvm::CanonicalLoopInfo *CLI = EmitOMPCollapsedCanonicalLoopNest(Inner, 1);
3545 OMPLoopNestStack.clear();
3546
3547 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
3548
3549 bool NeedsUnrolledCLI = ExpectedOMPLoopDepth >= 1;
3550 llvm::CanonicalLoopInfo *UnrolledCLI = nullptr;
3551
3552 if (S.hasClausesOfKind<OMPFullClause>()) {
3553 assert(ExpectedOMPLoopDepth == 0);
3554 OMPBuilder.unrollLoopFull(DL, CLI);
3555 } else if (auto *PartialClause = S.getSingleClause<OMPPartialClause>()) {
3556 uint64_t Factor = 0;
3557 if (Expr *FactorExpr = PartialClause->getFactor()) {
3558 Factor =
3559 FactorExpr->EvaluateKnownConstInt(getContext()).getLimitedValue();
3560 assert(Factor >= 1 && "Only positive factors are valid");
3561 }
3562 OMPBuilder.unrollLoopPartial(DL, CLI, Factor,
3563 NeedsUnrolledCLI ? &UnrolledCLI : nullptr);
3564 } else {
3565 OMPBuilder.unrollLoopHeuristic(DL, CLI);
3566 }
3567
3568 assert((!NeedsUnrolledCLI || UnrolledCLI) &&
3569 "NeedsUnrolledCLI implies UnrolledCLI to be set");
3570 if (UnrolledCLI)
3571 OMPLoopNestStack.push_back(UnrolledCLI);
3572
3573 return;
3574 }
3575
3576 // This function is only called if the unrolled loop is not consumed by any
3577 // other loop-associated construct. Such a loop-associated construct will have
3578 // used the transformed AST.
3579
3580 // Set the unroll metadata for the next emitted loop.
3581 LoopStack.setUnrollState(LoopAttributes::Enable);
3582
3583 if (S.hasClausesOfKind<OMPFullClause>()) {
3584 LoopStack.setUnrollState(LoopAttributes::Full);
3585 } else if (auto *PartialClause = S.getSingleClause<OMPPartialClause>()) {
3586 if (Expr *FactorExpr = PartialClause->getFactor()) {
3587 uint64_t Factor =
3588 FactorExpr->EvaluateKnownConstInt(getContext()).getLimitedValue();
3589 assert(Factor >= 1 && "Only positive factors are valid");
3590 LoopStack.setUnrollCount(Factor);
3591 }
3592 }
3593
3594 EmitStmt(S.getAssociatedStmt());
3595}
3596
3597void CodeGenFunction::EmitOMPOuterLoop(
3598 bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
3600 const CodeGenFunction::OMPLoopArguments &LoopArgs,
3601 const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
3602 const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
3604
3605 const Expr *IVExpr = S.getIterationVariable();
3606 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3607 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3608
3609 JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
3610
3611 // Start the loop with a block that tests the condition.
3612 llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond");
3613 EmitBlock(CondBlock);
3615 OMPLoopNestStack.clear();
3616 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
3617 SourceLocToDebugLoc(R.getEnd()));
3618
3619 llvm::Value *BoolCondVal = nullptr;
3620 if (!DynamicOrOrdered) {
3621 // UB = min(UB, GlobalUB) or
3622 // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
3623 // 'distribute parallel for')
3624 EmitIgnoredExpr(LoopArgs.EUB);
3625 // IV = LB
3626 EmitIgnoredExpr(LoopArgs.Init);
3627 // IV < UB
3628 BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
3629 } else {
3630 BoolCondVal =
3631 RT.emitForNext(*this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
3632 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
3633 }
3634
3635 // If there are any cleanups between here and the loop-exit scope,
3636 // create a block to stage a loop exit along.
3637 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
3638 if (LoopScope.requiresCleanups())
3639 ExitBlock = createBasicBlock("omp.dispatch.cleanup");
3640
3641 llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body");
3642 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
3643 if (ExitBlock != LoopExit.getBlock()) {
3644 EmitBlock(ExitBlock);
3646 }
3647 EmitBlock(LoopBody);
3648
3649 // Emit "IV = LB" (in case of static schedule, we have already calculated new
3650 // LB for loop condition and emitted it above).
3651 if (DynamicOrOrdered)
3652 EmitIgnoredExpr(LoopArgs.Init);
3653
3654 // Create a block for the increment.
3655 JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
3656 BreakContinueStack.push_back(BreakContinue(S, LoopExit, Continue));
3657
3660 *this, S,
3661 [&S, IsMonotonic, EKind](CodeGenFunction &CGF, PrePostActionTy &) {
3662 // Generate !llvm.loop.parallel metadata for loads and stores for loops
3663 // with dynamic/guided scheduling and without ordered clause.
3664 if (!isOpenMPSimdDirective(EKind)) {
3665 CGF.LoopStack.setParallel(!IsMonotonic);
3666 if (const auto *C = S.getSingleClause<OMPOrderClause>())
3667 if (C->getKind() == OMPC_ORDER_concurrent)
3668 CGF.LoopStack.setParallel(/*Enable=*/true);
3669 } else {
3670 CGF.EmitOMPSimdInit(S);
3671 }
3672 },
3673 [&S, &LoopArgs, LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered,
3674 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
3675 SourceLocation Loc = S.getBeginLoc();
3676 // when 'distribute' is not combined with a 'for':
3677 // while (idx <= UB) { BODY; ++idx; }
3678 // when 'distribute' is combined with a 'for'
3679 // (e.g. 'distribute parallel for')
3680 // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
3681 CGF.EmitOMPInnerLoop(
3682 S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
3683 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3684 CodeGenLoop(CGF, S, LoopExit);
3685 },
3686 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
3687 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
3688 });
3689 });
3690
3691 EmitBlock(Continue.getBlock());
3692 BreakContinueStack.pop_back();
3693 if (!DynamicOrOrdered) {
3694 // Emit "LB = LB + Stride", "UB = UB + Stride".
3695 EmitIgnoredExpr(LoopArgs.NextLB);
3696 EmitIgnoredExpr(LoopArgs.NextUB);
3697 }
3698
3699 EmitBranch(CondBlock);
3700 OMPLoopNestStack.clear();
3701 LoopStack.pop();
3702 // Emit the fall-through block.
3703 EmitBlock(LoopExit.getBlock());
3704
3705 // Tell the runtime we are done.
3706 auto &&CodeGen = [DynamicOrOrdered, &S, &LoopArgs](CodeGenFunction &CGF) {
3707 if (!DynamicOrOrdered)
3708 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
3709 LoopArgs.DKind);
3710 };
3711 OMPCancelStack.emitExit(*this, EKind, CodeGen);
3712}
3713
3714void CodeGenFunction::EmitOMPForOuterLoop(
3715 const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
3716 const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
3717 const OMPLoopArguments &LoopArgs,
3718 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
3719 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3720
3721 // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
3722 const bool DynamicOrOrdered = Ordered || RT.isDynamic(ScheduleKind.Schedule);
3723
3724 assert((Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule,
3725 LoopArgs.Chunk != nullptr)) &&
3726 "static non-chunked schedule does not need outer loop");
3727
3728 // Emit outer loop.
3729 //
3730 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
3731 // When schedule(dynamic,chunk_size) is specified, the iterations are
3732 // distributed to threads in the team in chunks as the threads request them.
3733 // Each thread executes a chunk of iterations, then requests another chunk,
3734 // until no chunks remain to be distributed. Each chunk contains chunk_size
3735 // iterations, except for the last chunk to be distributed, which may have
3736 // fewer iterations. When no chunk_size is specified, it defaults to 1.
3737 //
3738 // When schedule(guided,chunk_size) is specified, the iterations are assigned
3739 // to threads in the team in chunks as the executing threads request them.
3740 // Each thread executes a chunk of iterations, then requests another chunk,
3741 // until no chunks remain to be assigned. For a chunk_size of 1, the size of
3742 // each chunk is proportional to the number of unassigned iterations divided
3743 // by the number of threads in the team, decreasing to 1. For a chunk_size
3744 // with value k (greater than 1), the size of each chunk is determined in the
3745 // same way, with the restriction that the chunks do not contain fewer than k
3746 // iterations (except for the last chunk to be assigned, which may have fewer
3747 // than k iterations).
3748 //
3749 // When schedule(auto) is specified, the decision regarding scheduling is
3750 // delegated to the compiler and/or runtime system. The programmer gives the
3751 // implementation the freedom to choose any possible mapping of iterations to
3752 // threads in the team.
3753 //
3754 // When schedule(runtime) is specified, the decision regarding scheduling is
3755 // deferred until run time, and the schedule and chunk size are taken from the
3756 // run-sched-var ICV. If the ICV is set to auto, the schedule is
3757 // implementation defined
3758 //
3759 // __kmpc_dispatch_init();
3760 // while(__kmpc_dispatch_next(&LB, &UB)) {
3761 // idx = LB;
3762 // while (idx <= UB) { BODY; ++idx;
3763 // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
3764 // } // inner loop
3765 // }
3766 // __kmpc_dispatch_deinit();
3767 //
3768 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
3769 // When schedule(static, chunk_size) is specified, iterations are divided into
3770 // chunks of size chunk_size, and the chunks are assigned to the threads in
3771 // the team in a round-robin fashion in the order of the thread number.
3772 //
3773 // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
3774 // while (idx <= UB) { BODY; ++idx; } // inner loop
3775 // LB = LB + ST;
3776 // UB = UB + ST;
3777 // }
3778 //
3779
3780 const Expr *IVExpr = S.getIterationVariable();
3781 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3782 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3783
3784 if (DynamicOrOrdered) {
3785 const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
3786 CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
3787 llvm::Value *LBVal = DispatchBounds.first;
3788 llvm::Value *UBVal = DispatchBounds.second;
3789 CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
3790 LoopArgs.Chunk};
3791 RT.emitForDispatchInit(*this, S.getBeginLoc(), ScheduleKind, IVSize,
3792 IVSigned, Ordered, DipatchRTInputValues);
3793 } else {
3794 CGOpenMPRuntime::StaticRTInput StaticInit(
3795 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
3796 LoopArgs.ST, LoopArgs.Chunk);
3798 RT.emitForStaticInit(*this, S.getBeginLoc(), EKind, ScheduleKind,
3799 StaticInit);
3800 }
3801
3802 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
3803 const unsigned IVSize,
3804 const bool IVSigned) {
3805 if (Ordered) {
3806 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
3807 IVSigned);
3808 }
3809 };
3810
3811 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
3812 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
3813 OuterLoopArgs.IncExpr = S.getInc();
3814 OuterLoopArgs.Init = S.getInit();
3815 OuterLoopArgs.Cond = S.getCond();
3816 OuterLoopArgs.NextLB = S.getNextLowerBound();
3817 OuterLoopArgs.NextUB = S.getNextUpperBound();
3818 OuterLoopArgs.DKind = LoopArgs.DKind;
3819 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
3820 emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
3821 if (DynamicOrOrdered) {
3822 RT.emitForDispatchDeinit(*this, S.getBeginLoc());
3823 }
3824}
3825
3827 const unsigned IVSize, const bool IVSigned) {}
3828
3829void CodeGenFunction::EmitOMPDistributeOuterLoop(
3830 OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
3831 OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
3832 const CodeGenLoopTy &CodeGenLoopContent) {
3833
3834 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3835
3836 // Emit outer loop.
3837 // Same behavior as a OMPForOuterLoop, except that schedule cannot be
3838 // dynamic
3839 //
3840
3841 const Expr *IVExpr = S.getIterationVariable();
3842 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3843 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3845
3846 CGOpenMPRuntime::StaticRTInput StaticInit(
3847 IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
3848 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
3849 RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, StaticInit);
3850
3851 // for combined 'distribute' and 'for' the increment expression of distribute
3852 // is stored in DistInc. For 'distribute' alone, it is in Inc.
3853 Expr *IncExpr;
3855 IncExpr = S.getDistInc();
3856 else
3857 IncExpr = S.getInc();
3858
3859 // this routine is shared by 'omp distribute parallel for' and
3860 // 'omp distribute': select the right EUB expression depending on the
3861 // directive
3862 OMPLoopArguments OuterLoopArgs;
3863 OuterLoopArgs.LB = LoopArgs.LB;
3864 OuterLoopArgs.UB = LoopArgs.UB;
3865 OuterLoopArgs.ST = LoopArgs.ST;
3866 OuterLoopArgs.IL = LoopArgs.IL;
3867 OuterLoopArgs.Chunk = LoopArgs.Chunk;
3868 OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(EKind)
3870 : S.getEnsureUpperBound();
3871 OuterLoopArgs.IncExpr = IncExpr;
3872 OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(EKind)
3873 ? S.getCombinedInit()
3874 : S.getInit();
3875 OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(EKind)
3876 ? S.getCombinedCond()
3877 : S.getCond();
3878 OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(EKind)
3880 : S.getNextLowerBound();
3881 OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(EKind)
3883 : S.getNextUpperBound();
3884 OuterLoopArgs.DKind = OMPD_distribute;
3885
3886 EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
3887 LoopScope, OuterLoopArgs, CodeGenLoopContent,
3889}
3890
3891static std::pair<LValue, LValue>
3893 const OMPExecutableDirective &S) {
3895 LValue LB =
3897 LValue UB =
3899
3900 // When composing 'distribute' with 'for' (e.g. as in 'distribute
3901 // parallel for') we need to use the 'distribute'
3902 // chunk lower and upper bounds rather than the whole loop iteration
3903 // space. These are parameters to the outlined function for 'parallel'
3904 // and we copy the bounds of the previous schedule into the
3905 // the current ones.
3906 LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
3907 LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
3908 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
3909 PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());
3910 PrevLBVal = CGF.EmitScalarConversion(
3911 PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
3914 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
3915 PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());
3916 PrevUBVal = CGF.EmitScalarConversion(
3917 PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
3920
3921 CGF.EmitStoreOfScalar(PrevLBVal, LB);
3922 CGF.EmitStoreOfScalar(PrevUBVal, UB);
3923
3924 return {LB, UB};
3925}
3926
3927/// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
3928/// we need to use the LB and UB expressions generated by the worksharing
3929/// code generation support, whereas in non combined situations we would
3930/// just emit 0 and the LastIteration expression
3931/// This function is necessary due to the difference of the LB and UB
3932/// types for the RT emission routines for 'for_static_init' and
3933/// 'for_dispatch_init'
3934static std::pair<llvm::Value *, llvm::Value *>
3936 const OMPExecutableDirective &S,
3937 Address LB, Address UB) {
3939 const Expr *IVExpr = LS.getIterationVariable();
3940 // when implementing a dynamic schedule for a 'for' combined with a
3941 // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
3942 // is not normalized as each team only executes its own assigned
3943 // distribute chunk
3944 QualType IteratorTy = IVExpr->getType();
3945 llvm::Value *LBVal =
3946 CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
3947 llvm::Value *UBVal =
3948 CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
3949 return {LBVal, UBVal};
3950}
3951
3955 const auto &Dir = cast<OMPLoopDirective>(S);
3956 LValue LB =
3957 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
3958 llvm::Value *LBCast = CGF.Builder.CreateIntCast(
3959 CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
3960 CapturedVars.push_back(LBCast);
3961 LValue UB =
3962 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
3963
3964 llvm::Value *UBCast = CGF.Builder.CreateIntCast(
3965 CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
3966 CapturedVars.push_back(UBCast);
3967}
3968
3969static void
3971 const OMPLoopDirective &S,
3974 auto &&CGInlinedWorksharingLoop = [&S, EKind](CodeGenFunction &CGF,
3975 PrePostActionTy &Action) {
3976 Action.Enter(CGF);
3977 bool HasCancel = false;
3978 if (!isOpenMPSimdDirective(EKind)) {
3979 if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
3980 HasCancel = D->hasCancel();
3981 else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
3982 HasCancel = D->hasCancel();
3983 else if (const auto *D =
3984 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
3985 HasCancel = D->hasCancel();
3986 }
3987 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);
3991 };
3992
3994 CGF, S, isOpenMPSimdDirective(EKind) ? OMPD_for_simd : OMPD_for,
3995 CGInlinedWorksharingLoop,
3997}
3998
4001 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4003 S.getDistInc());
4004 };
4005 OMPLexicalScope Scope(*this, S, OMPD_parallel);
4006 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
4007}
4008
4011 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4013 S.getDistInc());
4014 };
4015 OMPLexicalScope Scope(*this, S, OMPD_parallel);
4016 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
4017}
4018
4020 const OMPDistributeSimdDirective &S) {
4021 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4023 };
4024 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4025 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
4026}
4027
4029 CodeGenModule &CGM, StringRef ParentName, const OMPLoopDirective &S) {
4030 // Emit SPMD target parallel for region as a standalone region.
4031 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4032 emitOMPSimdRegion(CGF, S, Action);
4033 };
4034 llvm::Function *Fn;
4035 llvm::Constant *Addr;
4036 // Emit target region as a standalone region.
4037 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4038 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4039 assert(Fn && Addr && "Target device function emission failed.");
4040}
4041
4043 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4044 emitOMPSimdRegion(CGF, S, Action);
4045 };
4047}
4048
4049namespace {
4050struct ScheduleKindModifiersTy {
4054 ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
4057 : Kind(Kind), M1(M1), M2(M2) {}
4058};
4059} // namespace
4060
4062 const OMPLoopDirective &S, Expr *EUB,
4063 const CodeGenLoopBoundsTy &CodeGenLoopBounds,
4064 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
4065 // Emit the loop iteration variable.
4066 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
4067 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
4068 EmitVarDecl(*IVDecl);
4069
4070 // Emit the iterations count variable.
4071 // If it is not a variable, Sema decided to calculate iterations count on each
4072 // iteration (e.g., it is foldable into a constant).
4073 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
4074 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
4075 // Emit calculation of the iterations count.
4077 }
4078
4079 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
4080
4081 bool HasLastprivateClause;
4082 // Check pre-condition.
4083 {
4084 OMPLoopScope PreInitScope(*this, S);
4085 // Skip the entire loop if we don't meet the precondition.
4086 // If the condition constant folds and can be elided, avoid emitting the
4087 // whole loop.
4088 bool CondConstant;
4089 llvm::BasicBlock *ContBlock = nullptr;
4090 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
4091 if (!CondConstant)
4092 return false;
4093 } else {
4094 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
4095 ContBlock = createBasicBlock("omp.precond.end");
4096 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
4097 getProfileCount(&S));
4098 EmitBlock(ThenBlock);
4100 }
4101
4102 RunCleanupsScope DoacrossCleanupScope(*this);
4103 bool Ordered = false;
4104 if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
4105 if (OrderedClause->getNumForLoops())
4106 RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations());
4107 else
4108 Ordered = true;
4109 }
4110
4111 emitAlignedClause(*this, S);
4112 bool HasLinears = EmitOMPLinearClauseInit(S);
4113 // Emit helper vars inits.
4114
4115 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
4116 LValue LB = Bounds.first;
4117 LValue UB = Bounds.second;
4118 LValue ST =
4120 LValue IL =
4122
4123 // Emit 'then' code.
4124 {
4126 OMPPrivateScope LoopScope(*this);
4127 if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
4128 // Emit implicit barrier to synchronize threads and avoid data races on
4129 // initialization of firstprivate variables and post-update of
4130 // lastprivate variables.
4131 CGM.getOpenMPRuntime().emitBarrierCall(
4132 *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
4133 /*ForceSimpleCall=*/true);
4134 }
4135 EmitOMPPrivateClause(S, LoopScope);
4137 *this, S, EmitLValue(S.getIterationVariable()));
4138 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
4139 EmitOMPReductionClauseInit(S, LoopScope);
4140 EmitOMPPrivateLoopCounters(S, LoopScope);
4141 EmitOMPLinearClause(S, LoopScope);
4142 (void)LoopScope.Privatize();
4144 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
4145
4146 // Detect the loop schedule kind and chunk.
4147 const Expr *ChunkExpr = nullptr;
4148 OpenMPScheduleTy ScheduleKind;
4149 if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
4150 ScheduleKind.Schedule = C->getScheduleKind();
4151 ScheduleKind.M1 = C->getFirstScheduleModifier();
4152 ScheduleKind.M2 = C->getSecondScheduleModifier();
4153 ChunkExpr = C->getChunkSize();
4154 } else {
4155 // Default behaviour for schedule clause.
4156 CGM.getOpenMPRuntime().getDefaultScheduleAndChunk(
4157 *this, S, ScheduleKind.Schedule, ChunkExpr);
4158 }
4159 bool HasChunkSizeOne = false;
4160 llvm::Value *Chunk = nullptr;
4161 if (ChunkExpr) {
4162 Chunk = EmitScalarExpr(ChunkExpr);
4163 Chunk = EmitScalarConversion(Chunk, ChunkExpr->getType(),
4165 S.getBeginLoc());
4167 if (ChunkExpr->EvaluateAsInt(Result, getContext())) {
4168 llvm::APSInt EvaluatedChunk = Result.Val.getInt();
4169 HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
4170 }
4171 }
4172 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
4173 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
4174 // OpenMP 4.5, 2.7.1 Loop Construct, Description.
4175 // If the static schedule kind is specified or if the ordered clause is
4176 // specified, and if no monotonic modifier is specified, the effect will
4177 // be as if the monotonic modifier was specified.
4178 bool StaticChunkedOne =
4179 RT.isStaticChunked(ScheduleKind.Schedule,
4180 /* Chunked */ Chunk != nullptr) &&
4181 HasChunkSizeOne && isOpenMPLoopBoundSharingDirective(EKind);
4182 // GPU combined `distribute parallel for`: emit a single
4183 // for_static_init with the fused distr_static_chunk + static_chunkone
4184 // schedule (enum 93). The surrounding EmitOMPDistributeLoop must skip
4185 // its distribute_static_init under the same conditions. Both sites are
4186 // guarded by canEmitGPUFusedDistSchedule() alone so they cannot
4187 // disagree; the assert guards the invariant that makes this safe today,
4188 // aka that the implicit GPU default schedule is always static chunk-one.
4189 ScheduleKind.UseFusedDistChunkSchedule =
4191 assert((!ScheduleKind.UseFusedDistChunkSchedule || StaticChunkedOne) &&
4192 "fused distribute schedule requires a static chunk-one schedule");
4193 bool IsMonotonic =
4194 Ordered ||
4195 (ScheduleKind.Schedule == OMPC_SCHEDULE_static &&
4196 !(ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_nonmonotonic ||
4197 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_nonmonotonic)) ||
4198 ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
4199 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
4200 if ((RT.isStaticNonchunked(ScheduleKind.Schedule,
4201 /* Chunked */ Chunk != nullptr) ||
4202 StaticChunkedOne) &&
4203 !Ordered) {
4207 *this, S,
4208 [&S, EKind](CodeGenFunction &CGF, PrePostActionTy &) {
4209 if (isOpenMPSimdDirective(EKind)) {
4210 CGF.EmitOMPSimdInit(S);
4211 } else if (const auto *C = S.getSingleClause<OMPOrderClause>()) {
4212 if (C->getKind() == OMPC_ORDER_concurrent)
4213 CGF.LoopStack.setParallel(/*Enable=*/true);
4214 }
4215 },
4216 [IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk,
4217 &S, ScheduleKind, LoopExit, EKind,
4218 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
4219 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
4220 // When no chunk_size is specified, the iteration space is divided
4221 // into chunks that are approximately equal in size, and at most
4222 // one chunk is distributed to each thread. Note that the size of
4223 // the chunks is unspecified in this case.
4225 IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
4226 UB.getAddress(), ST.getAddress(),
4227 StaticChunkedOne ? Chunk : nullptr);
4229 CGF, S.getBeginLoc(), EKind, ScheduleKind, StaticInit);
4230 // UB = min(UB, GlobalUB);
4231 if (!StaticChunkedOne)
4232 CGF.EmitIgnoredExpr(S.getEnsureUpperBound());
4233 // IV = LB;
4234 CGF.EmitIgnoredExpr(S.getInit());
4235 // For unchunked static schedule generate:
4236 //
4237 // while (idx <= UB) {
4238 // BODY;
4239 // ++idx;
4240 // }
4241 //
4242 // For static schedule with chunk one:
4243 //
4244 // while (IV <= PrevUB) {
4245 // BODY;
4246 // IV += ST;
4247 // }
4248 CGF.EmitOMPInnerLoop(
4249 S, LoopScope.requiresCleanups(),
4250 StaticChunkedOne ? S.getCombinedParForInDistCond()
4251 : S.getCond(),
4252 StaticChunkedOne ? S.getDistInc() : S.getInc(),
4253 [&S, LoopExit](CodeGenFunction &CGF) {
4254 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit);
4255 },
4256 [](CodeGenFunction &) {});
4257 });
4258 EmitBlock(LoopExit.getBlock());
4259 // Tell the runtime we are done.
4260 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
4261 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
4262 OMPD_for);
4263 };
4264 OMPCancelStack.emitExit(*this, EKind, CodeGen);
4265 } else {
4266 // Emit the outer loop, which requests its work chunk [LB..UB] from
4267 // runtime and runs the inner loop to process it.
4268 OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),
4269 ST.getAddress(), IL.getAddress(), Chunk,
4270 EUB);
4271 LoopArguments.DKind = OMPD_for;
4272 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
4273 LoopArguments, CGDispatchBounds);
4274 }
4275 if (isOpenMPSimdDirective(EKind)) {
4276 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
4277 return CGF.Builder.CreateIsNotNull(
4278 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
4279 });
4280 }
4282 S, /*ReductionKind=*/isOpenMPSimdDirective(EKind)
4283 ? /*Parallel and Simd*/ OMPD_parallel_for_simd
4284 : /*Parallel only*/ OMPD_parallel);
4285 // Emit post-update of the reduction variables if IsLastIter != 0.
4287 *this, S, [IL, &S](CodeGenFunction &CGF) {
4288 return CGF.Builder.CreateIsNotNull(
4289 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
4290 });
4291 // Emit final copy of the lastprivate variables if IsLastIter != 0.
4292 if (HasLastprivateClause)
4294 S, isOpenMPSimdDirective(EKind),
4295 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
4296 LoopScope.restoreMap();
4297 EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) {
4298 return CGF.Builder.CreateIsNotNull(
4299 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
4300 });
4301 }
4302 DoacrossCleanupScope.ForceCleanup();
4303 // We're now done with the loop, so jump to the continuation block.
4304 if (ContBlock) {
4305 EmitBranch(ContBlock);
4306 EmitBlock(ContBlock, /*IsFinished=*/true);
4307 }
4308 }
4309 return HasLastprivateClause;
4310}
4311
4312/// The following two functions generate expressions for the loop lower
4313/// and upper bounds in case of static and dynamic (dispatch) schedule
4314/// of the associated 'for' or 'distribute' loop.
4315static std::pair<LValue, LValue>
4317 const auto &LS = cast<OMPLoopDirective>(S);
4318 LValue LB =
4319 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
4320 LValue UB =
4321 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
4322 return {LB, UB};
4323}
4324
4325/// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
4326/// consider the lower and upper bound expressions generated by the
4327/// worksharing loop support, but we use 0 and the iteration space size as
4328/// constants
4329static std::pair<llvm::Value *, llvm::Value *>
4331 Address LB, Address UB) {
4332 const auto &LS = cast<OMPLoopDirective>(S);
4333 const Expr *IVExpr = LS.getIterationVariable();
4334 const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
4335 llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
4336 llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
4337 return {LBVal, UBVal};
4338}
4339
4340/// Emits internal temp array declarations for the directive with inscan
4341/// reductions.
4342/// The code is the following:
4343/// \code
4344/// size num_iters = <num_iters>;
4345/// <type> buffer[num_iters];
4346/// \endcode
4348 CodeGenFunction &CGF, const OMPLoopDirective &S,
4349 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen) {
4350 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
4351 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
4354 SmallVector<const Expr *, 4> ReductionOps;
4355 SmallVector<const Expr *, 4> CopyArrayTemps;
4356 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
4357 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
4358 "Only inscan reductions are expected.");
4359 Shareds.append(C->varlist_begin(), C->varlist_end());
4360 Privates.append(C->privates().begin(), C->privates().end());
4361 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
4362 CopyArrayTemps.append(C->copy_array_temps().begin(),
4363 C->copy_array_temps().end());
4364 }
4365 {
4366 // Emit buffers for each reduction variables.
4367 // ReductionCodeGen is required to emit correctly the code for array
4368 // reductions.
4369 ReductionCodeGen RedCG(Shareds, Shareds, Privates, ReductionOps);
4370 unsigned Count = 0;
4371 auto *ITA = CopyArrayTemps.begin();
4372 for (const Expr *IRef : Privates) {
4373 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
4374 // Emit variably modified arrays, used for arrays/array sections
4375 // reductions.
4376 if (PrivateVD->getType()->isVariablyModifiedType()) {
4377 RedCG.emitSharedOrigLValue(CGF, Count);
4378 RedCG.emitAggregateType(CGF, Count);
4379 }
4381 CGF,
4383 cast<VariableArrayType>((*ITA)->getType()->getAsArrayTypeUnsafe())
4384 ->getSizeExpr()),
4385 RValue::get(OMPScanNumIterations));
4386 // Emit temp buffer.
4387 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(*ITA)->getDecl()));
4388 ++ITA;
4389 ++Count;
4390 }
4391 }
4392}
4393
4394/// Copies final inscan reductions values to the original variables.
4395/// The code is the following:
4396/// \code
4397/// <orig_var> = buffer[num_iters-1];
4398/// \endcode
4400 CodeGenFunction &CGF, const OMPLoopDirective &S,
4401 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen) {
4402 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
4403 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
4409 SmallVector<const Expr *, 4> CopyArrayElems;
4410 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
4411 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
4412 "Only inscan reductions are expected.");
4413 Shareds.append(C->varlist_begin(), C->varlist_end());
4414 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
4415 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
4416 Privates.append(C->privates().begin(), C->privates().end());
4417 CopyOps.append(C->copy_ops().begin(), C->copy_ops().end());
4418 CopyArrayElems.append(C->copy_array_elems().begin(),
4419 C->copy_array_elems().end());
4420 }
4421 // Create temp var and copy LHS value to this temp value.
4422 // LHS = TMP[LastIter];
4423 llvm::Value *OMPLast = CGF.Builder.CreateNSWSub(
4424 OMPScanNumIterations,
4425 llvm::ConstantInt::get(CGF.SizeTy, 1, /*isSigned=*/false));
4426 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
4427 const Expr *PrivateExpr = Privates[I];
4428 const Expr *OrigExpr = Shareds[I];
4429 const Expr *CopyArrayElem = CopyArrayElems[I];
4431 CGF,
4433 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
4434 RValue::get(OMPLast));
4435 LValue DestLVal = CGF.EmitLValue(OrigExpr);
4436 LValue SrcLVal = CGF.EmitLValue(CopyArrayElem);
4437 CGF.EmitOMPCopy(
4438 PrivateExpr->getType(), DestLVal.getAddress(), SrcLVal.getAddress(),
4439 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
4440 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]);
4441 }
4442}
4443
4444/// Emits the code for the directive with inscan reductions.
4445/// The code is the following:
4446/// \code
4447/// #pragma omp ...
4448/// for (i: 0..<num_iters>) {
4449/// <input phase>;
4450/// buffer[i] = red;
4451/// }
4452/// #pragma omp master // in parallel region
4453/// for (int k = 0; k != ceil(log2(num_iters)); ++k)
4454/// for (size cnt = last_iter; cnt >= pow(2, k); --k)
4455/// buffer[i] op= buffer[i-pow(2,k)];
4456/// #pragma omp barrier // in parallel region
4457/// #pragma omp ...
4458/// for (0..<num_iters>) {
4459/// red = InclusiveScan ? buffer[i] : buffer[i-1];
4460/// <scan phase>;
4461/// }
4462/// \endcode
4464 CodeGenFunction &CGF, const OMPLoopDirective &S,
4465 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen,
4466 llvm::function_ref<void(CodeGenFunction &)> FirstGen,
4467 llvm::function_ref<void(CodeGenFunction &)> SecondGen) {
4468 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
4469 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
4471 SmallVector<const Expr *, 4> ReductionOps;
4474 SmallVector<const Expr *, 4> CopyArrayElems;
4475 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
4476 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
4477 "Only inscan reductions are expected.");
4478 Privates.append(C->privates().begin(), C->privates().end());
4479 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
4480 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
4481 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
4482 CopyArrayElems.append(C->copy_array_elems().begin(),
4483 C->copy_array_elems().end());
4484 }
4486 {
4487 // Emit loop with input phase:
4488 // #pragma omp ...
4489 // for (i: 0..<num_iters>) {
4490 // <input phase>;
4491 // buffer[i] = red;
4492 // }
4493 CGF.OMPFirstScanLoop = true;
4495 FirstGen(CGF);
4496 }
4497 // #pragma omp barrier // in parallel region
4498 auto &&CodeGen = [&S, OMPScanNumIterations, &LHSs, &RHSs, &CopyArrayElems,
4499 &ReductionOps,
4500 &Privates](CodeGenFunction &CGF, PrePostActionTy &Action) {
4501 Action.Enter(CGF);
4502 // Emit prefix reduction:
4503 // #pragma omp master // in parallel region
4504 // for (int k = 0; k <= ceil(log2(n)); ++k)
4505 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock();
4506 llvm::BasicBlock *LoopBB = CGF.createBasicBlock("omp.outer.log.scan.body");
4507 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.outer.log.scan.exit");
4508 llvm::Function *F =
4509 CGF.CGM.getIntrinsic(llvm::Intrinsic::log2, CGF.DoubleTy);
4510 llvm::Value *Arg =
4511 CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy);
4512 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg);
4513 F = CGF.CGM.getIntrinsic(llvm::Intrinsic::ceil, CGF.DoubleTy);
4514 LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal);
4515 LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy);
4516 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub(
4517 OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1));
4518 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getBeginLoc());
4519 CGF.EmitBlock(LoopBB);
4520 auto *Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2);
4521 // size pow2k = 1;
4522 auto *Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
4523 Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB);
4524 Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB);
4525 // for (size i = n - 1; i >= 2 ^ k; --i)
4526 // tmp[i] op= tmp[i-pow2k];
4527 llvm::BasicBlock *InnerLoopBB =
4528 CGF.createBasicBlock("omp.inner.log.scan.body");
4529 llvm::BasicBlock *InnerExitBB =
4530 CGF.createBasicBlock("omp.inner.log.scan.exit");
4531 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K);
4532 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
4533 CGF.EmitBlock(InnerLoopBB);
4534 auto *IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
4535 IVal->addIncoming(NMin1, LoopBB);
4536 {
4537 CodeGenFunction::OMPPrivateScope PrivScope(CGF);
4538 auto *ILHS = LHSs.begin();
4539 auto *IRHS = RHSs.begin();
4540 for (const Expr *CopyArrayElem : CopyArrayElems) {
4541 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
4542 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
4543 Address LHSAddr = Address::invalid();
4544 {
4546 CGF,
4548 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
4549 RValue::get(IVal));
4550 LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress();
4551 }
4552 PrivScope.addPrivate(LHSVD, LHSAddr);
4553 Address RHSAddr = Address::invalid();
4554 {
4555 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K);
4557 CGF,
4559 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
4560 RValue::get(OffsetIVal));
4561 RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress();
4562 }
4563 PrivScope.addPrivate(RHSVD, RHSAddr);
4564 ++ILHS;
4565 ++IRHS;
4566 }
4567 PrivScope.Privatize();
4568 CGF.CGM.getOpenMPRuntime().emitReduction(
4569 CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
4570 {/*WithNowait=*/true, /*SimpleReduction=*/true,
4571 /*IsPrivateVarReduction*/ {}, OMPD_unknown});
4572 }
4573 llvm::Value *NextIVal =
4574 CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1));
4575 IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock());
4576 CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K);
4577 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
4578 CGF.EmitBlock(InnerExitBB);
4579 llvm::Value *Next =
4580 CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1));
4581 Counter->addIncoming(Next, CGF.Builder.GetInsertBlock());
4582 // pow2k <<= 1;
4583 llvm::Value *NextPow2K =
4584 CGF.Builder.CreateShl(Pow2K, 1, "", /*HasNUW=*/true);
4585 Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock());
4586 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(Next, LogVal);
4587 CGF.Builder.CreateCondBr(Cmp, LoopBB, ExitBB);
4588 auto DL1 = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getEndLoc());
4589 CGF.EmitBlock(ExitBB);
4590 };
4592 if (isOpenMPParallelDirective(EKind)) {
4593 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());
4595 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
4596 /*ForceSimpleCall=*/true);
4597 } else {
4598 RegionCodeGenTy RCG(CodeGen);
4599 RCG(CGF);
4600 }
4601
4602 CGF.OMPFirstScanLoop = false;
4603 SecondGen(CGF);
4604}
4605
4607 const OMPLoopDirective &S,
4608 bool HasCancel) {
4609 bool HasLastprivates;
4611 if (llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
4612 [](const OMPReductionClause *C) {
4613 return C->getModifier() == OMPC_REDUCTION_inscan;
4614 })) {
4615 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
4617 OMPLoopScope LoopScope(CGF, S);
4618 return CGF.EmitScalarExpr(S.getNumIterations());
4619 };
4620 const auto &&FirstGen = [&S, HasCancel, EKind](CodeGenFunction &CGF) {
4621 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);
4625 // Emit an implicit barrier at the end.
4626 CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getBeginLoc(),
4627 OMPD_for);
4628 };
4629 const auto &&SecondGen = [&S, HasCancel, EKind,
4630 &HasLastprivates](CodeGenFunction &CGF) {
4631 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);
4632 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
4635 };
4636 if (!isOpenMPParallelDirective(EKind))
4637 emitScanBasedDirectiveDecls(CGF, S, NumIteratorsGen);
4638 emitScanBasedDirective(CGF, S, NumIteratorsGen, FirstGen, SecondGen);
4639 if (!isOpenMPParallelDirective(EKind))
4640 emitScanBasedDirectiveFinals(CGF, S, NumIteratorsGen);
4641 } else {
4642 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);
4643 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
4646 }
4647 return HasLastprivates;
4648}
4649
4650// Pass OMPLoopDirective (instead of OMPForDirective) to make this check
4651// available for "loop bind(parallel)", which maps to "for".
4653 bool HasCancel) {
4654 if (HasCancel)
4655 return false;
4656 for (OMPClause *C : S.clauses()) {
4658 continue;
4659
4660 if (auto *SC = dyn_cast<OMPScheduleClause>(C)) {
4661 if (SC->getFirstScheduleModifier() != OMPC_SCHEDULE_MODIFIER_unknown)
4662 return false;
4663 if (SC->getSecondScheduleModifier() != OMPC_SCHEDULE_MODIFIER_unknown)
4664 return false;
4665 switch (SC->getScheduleKind()) {
4666 case OMPC_SCHEDULE_auto:
4667 case OMPC_SCHEDULE_dynamic:
4668 case OMPC_SCHEDULE_runtime:
4669 case OMPC_SCHEDULE_guided:
4670 case OMPC_SCHEDULE_static:
4671 continue;
4673 return false;
4674 }
4675 }
4676
4677 return false;
4678 }
4679
4680 return true;
4681}
4682
4683static llvm::omp::ScheduleKind
4685 switch (ScheduleClauseKind) {
4687 return llvm::omp::OMP_SCHEDULE_Default;
4688 case OMPC_SCHEDULE_auto:
4689 return llvm::omp::OMP_SCHEDULE_Auto;
4690 case OMPC_SCHEDULE_dynamic:
4691 return llvm::omp::OMP_SCHEDULE_Dynamic;
4692 case OMPC_SCHEDULE_guided:
4693 return llvm::omp::OMP_SCHEDULE_Guided;
4694 case OMPC_SCHEDULE_runtime:
4695 return llvm::omp::OMP_SCHEDULE_Runtime;
4696 case OMPC_SCHEDULE_static:
4697 return llvm::omp::OMP_SCHEDULE_Static;
4698 }
4699 llvm_unreachable("Unhandled schedule kind");
4700}
4701
4702// Pass OMPLoopDirective (instead of OMPForDirective) to make this function
4703// available for "loop bind(parallel)", which maps to "for".
4705 CodeGenModule &CGM, bool HasCancel) {
4706 bool HasLastprivates = false;
4707 bool UseOMPIRBuilder = CGM.getLangOpts().OpenMPIRBuilder &&
4708 isForSupportedByOpenMPIRBuilder(S, HasCancel);
4709 auto &&CodeGen = [&S, &CGM, HasCancel, &HasLastprivates,
4710 UseOMPIRBuilder](CodeGenFunction &CGF, PrePostActionTy &) {
4711 // Use the OpenMPIRBuilder if enabled.
4712 if (UseOMPIRBuilder) {
4713 bool NeedsBarrier = !S.getSingleClause<OMPNowaitClause>();
4714
4715 llvm::omp::ScheduleKind SchedKind = llvm::omp::OMP_SCHEDULE_Default;
4716 llvm::Value *ChunkSize = nullptr;
4717 if (auto *SchedClause = S.getSingleClause<OMPScheduleClause>()) {
4718 SchedKind =
4719 convertClauseKindToSchedKind(SchedClause->getScheduleKind());
4720 if (const Expr *ChunkSizeExpr = SchedClause->getChunkSize())
4721 ChunkSize = CGF.EmitScalarExpr(ChunkSizeExpr);
4722 }
4723
4724 // Emit the associated statement and get its loop representation.
4725 const Stmt *Inner = S.getRawStmt();
4726 llvm::CanonicalLoopInfo *CLI =
4728
4729 llvm::OpenMPIRBuilder &OMPBuilder =
4731 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
4732 CGF.AllocaInsertPt->getIterator());
4733 cantFail(OMPBuilder.applyWorkshareLoop(
4734 CGF.Builder.getCurrentDebugLocation(), CLI, AllocaIP, NeedsBarrier,
4735 SchedKind, ChunkSize, /*HasSimdModifier=*/false,
4736 /*HasMonotonicModifier=*/false, /*HasNonmonotonicModifier=*/false,
4737 /*HasOrderedClause=*/false));
4738 return;
4739 }
4740
4741 HasLastprivates = emitWorksharingDirective(CGF, S, HasCancel);
4742 };
4743 {
4744 auto LPCRegion =
4746 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
4748 HasCancel);
4749 }
4750
4751 if (!UseOMPIRBuilder) {
4752 // Emit an implicit barrier at the end.
4753 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
4754 CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getBeginLoc(), OMPD_for);
4755 }
4756 // Check for outer lastprivate conditional update.
4758}
4759
4763
4765 bool HasLastprivates = false;
4766 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
4767 PrePostActionTy &) {
4768 HasLastprivates = emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
4769 };
4770 {
4771 auto LPCRegion =
4773 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4774 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
4775 }
4776
4777 // Emit an implicit barrier at the end.
4778 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
4779 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
4780 // Check for outer lastprivate conditional update.
4782}
4783
4785 const Twine &Name,
4786 llvm::Value *Init = nullptr) {
4787 LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
4788 if (Init)
4789 CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
4790 return LVal;
4791}
4792
4793void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
4794 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4795 const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
4796 bool HasLastprivates = false;
4798 auto &&CodeGen = [&S, CapturedStmt, CS, EKind,
4799 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
4800 const ASTContext &C = CGF.getContext();
4801 QualType KmpInt32Ty =
4802 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
4803 // Emit helper vars inits.
4804 LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
4805 CGF.Builder.getInt32(0));
4806 llvm::ConstantInt *GlobalUBVal = CS != nullptr
4807 ? CGF.Builder.getInt32(CS->size() - 1)
4808 : CGF.Builder.getInt32(0);
4809 LValue UB =
4810 createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
4811 LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
4812 CGF.Builder.getInt32(1));
4813 LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
4814 CGF.Builder.getInt32(0));
4815 // Loop counter.
4816 LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
4817 OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
4818 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
4819 OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
4820 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
4821 // Generate condition for loop.
4822 BinaryOperator *Cond = BinaryOperator::Create(
4823 C, &IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_PRValue, OK_Ordinary,
4824 S.getBeginLoc(), FPOptionsOverride());
4825 // Increment for loop counter.
4826 UnaryOperator *Inc = UnaryOperator::Create(
4827 C, &IVRefExpr, UO_PreInc, KmpInt32Ty, VK_PRValue, OK_Ordinary,
4828 S.getBeginLoc(), true, FPOptionsOverride());
4829 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
4830 // Iterate through all sections and emit a switch construct:
4831 // switch (IV) {
4832 // case 0:
4833 // <SectionStmt[0]>;
4834 // break;
4835 // ...
4836 // case <NumSection> - 1:
4837 // <SectionStmt[<NumSection> - 1]>;
4838 // break;
4839 // }
4840 // .omp.sections.exit:
4841 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
4842 llvm::SwitchInst *SwitchStmt =
4843 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
4844 ExitBB, CS == nullptr ? 1 : CS->size());
4845 if (CS) {
4846 unsigned CaseNumber = 0;
4847 for (const Stmt *SubStmt : CS->children()) {
4848 auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
4849 CGF.EmitBlock(CaseBB);
4850 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
4851 CGF.EmitStmt(SubStmt);
4852 CGF.EmitBranch(ExitBB);
4853 ++CaseNumber;
4854 }
4855 } else {
4856 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");
4857 CGF.EmitBlock(CaseBB);
4858 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
4859 CGF.EmitStmt(CapturedStmt);
4860 CGF.EmitBranch(ExitBB);
4861 }
4862 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
4863 };
4864
4865 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
4866 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
4867 // Emit implicit barrier to synchronize threads and avoid data races on
4868 // initialization of firstprivate variables and post-update of lastprivate
4869 // variables.
4870 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
4871 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
4872 /*ForceSimpleCall=*/true);
4873 }
4874 CGF.EmitOMPPrivateClause(S, LoopScope);
4875 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(CGF, S, IV);
4876 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4877 CGF.EmitOMPReductionClauseInit(S, LoopScope);
4878 (void)LoopScope.Privatize();
4880 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4881
4882 // Emit static non-chunked loop.
4883 OpenMPScheduleTy ScheduleKind;
4884 ScheduleKind.Schedule = OMPC_SCHEDULE_static;
4885 CGOpenMPRuntime::StaticRTInput StaticInit(
4886 /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(),
4887 LB.getAddress(), UB.getAddress(), ST.getAddress());
4888 CGF.CGM.getOpenMPRuntime().emitForStaticInit(CGF, S.getBeginLoc(), EKind,
4889 ScheduleKind, StaticInit);
4890 // UB = min(UB, GlobalUB);
4891 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
4892 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
4893 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
4894 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
4895 // IV = LB;
4896 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
4897 // while (idx <= UB) { BODY; ++idx; }
4898 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, Cond, Inc, BodyGen,
4899 [](CodeGenFunction &) {});
4900 // Tell the runtime we are done.
4901 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
4902 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
4903 OMPD_sections);
4904 };
4905 CGF.OMPCancelStack.emitExit(CGF, EKind, CodeGen);
4906 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4907 // Emit post-update of the reduction variables if IsLastIter != 0.
4908 emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) {
4909 return CGF.Builder.CreateIsNotNull(
4910 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
4911 });
4912
4913 // Emit final copy of the lastprivate variables if IsLastIter != 0.
4914 if (HasLastprivates)
4916 S, /*NoFinals=*/false,
4917 CGF.Builder.CreateIsNotNull(
4918 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())));
4919 };
4920
4921 bool HasCancel = false;
4922 if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
4923 HasCancel = OSD->hasCancel();
4924 else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
4925 HasCancel = OPSD->hasCancel();
4926 OMPCancelStackRAII CancelRegion(*this, EKind, HasCancel);
4927 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
4928 HasCancel);
4929 // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
4930 // clause. Otherwise the barrier will be generated by the codegen for the
4931 // directive.
4932 if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
4933 // Emit implicit barrier to synchronize threads and avoid data races on
4934 // initialization of firstprivate variables.
4935 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
4936 OMPD_unknown);
4937 }
4938}
4939
4941 {
4942 // Emit code for 'scope' region
4943 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4944 Action.Enter(CGF);
4945 OMPPrivateScope PrivateScope(CGF);
4946 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4947 CGF.EmitOMPPrivateClause(S, PrivateScope);
4948 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4949 (void)PrivateScope.Privatize();
4950 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4951 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4952 };
4953 auto LPCRegion =
4955 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4956 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_scope, CodeGen);
4957 }
4958 // Emit an implicit barrier at the end.
4959 if (!S.getSingleClause<OMPNowaitClause>()) {
4960 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_scope);
4961 }
4962 // Check for outer lastprivate conditional update.
4964}
4965
4967 if (CGM.getLangOpts().OpenMPIRBuilder) {
4968 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4969 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4970 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4971
4972 auto FiniCB = [](InsertPointTy IP) {
4973 // Don't FinalizeOMPRegion because this is done inside of OMPIRBuilder for
4974 // sections.
4975 return llvm::Error::success();
4976 };
4977
4978 const CapturedStmt *ICS = S.getInnermostCapturedStmt();
4979 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4980 const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
4982 if (CS) {
4983 for (const Stmt *SubStmt : CS->children()) {
4984 auto SectionCB = [this, SubStmt](
4985 InsertPointTy AllocIP, InsertPointTy CodeGenIP,
4986 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
4987 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(*this, SubStmt, AllocIP,
4988 CodeGenIP, "section");
4989 return llvm::Error::success();
4990 };
4991 SectionCBVector.push_back(SectionCB);
4992 }
4993 } else {
4994 auto SectionCB =
4995 [this, CapturedStmt](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
4996 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
4998 *this, CapturedStmt, AllocIP, CodeGenIP, "section");
4999 return llvm::Error::success();
5000 };
5001 SectionCBVector.push_back(SectionCB);
5002 }
5003
5004 // Privatization callback that performs appropriate action for
5005 // shared/private/firstprivate/lastprivate/copyin/... variables.
5006 //
5007 // TODO: This defaults to shared right now.
5008 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
5009 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
5010 // The next line is appropriate only for variables (Val) with the
5011 // data-sharing attribute "shared".
5012 ReplVal = &Val;
5013
5014 return CodeGenIP;
5015 };
5016
5017 CGCapturedStmtInfo CGSI(*ICS, CR_OpenMP);
5018 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
5019 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
5020 AllocaInsertPt->getIterator());
5021 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
5022 cantFail(OMPBuilder.createSections(
5023 Builder, AllocaIP, SectionCBVector, PrivCB, FiniCB, S.hasCancel(),
5024 S.getSingleClause<OMPNowaitClause>()));
5025 Builder.restoreIP(AfterIP);
5026 return;
5027 }
5028 {
5029 auto LPCRegion =
5031 OMPLexicalScope Scope(*this, S, OMPD_unknown);
5032 EmitSections(S);
5033 }
5034 // Emit an implicit barrier at the end.
5035 if (!S.getSingleClause<OMPNowaitClause>()) {
5036 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
5037 OMPD_sections);
5038 }
5039 // Check for outer lastprivate conditional update.
5041}
5042
5044 if (CGM.getLangOpts().OpenMPIRBuilder) {
5045 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
5046 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5047
5048 const Stmt *SectionRegionBodyStmt = S.getAssociatedStmt();
5049 auto FiniCB = [this](InsertPointTy IP) {
5051 return llvm::Error::success();
5052 };
5053
5054 auto BodyGenCB = [SectionRegionBodyStmt,
5055 this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
5056 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
5058 *this, SectionRegionBodyStmt, AllocIP, CodeGenIP, "section");
5059 return llvm::Error::success();
5060 };
5061
5062 LexicalScope Scope(*this, S.getSourceRange());
5063 EmitStopPoint(&S);
5064 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
5065 cantFail(OMPBuilder.createSection(Builder, BodyGenCB, FiniCB));
5066 Builder.restoreIP(AfterIP);
5067
5068 return;
5069 }
5070 LexicalScope Scope(*this, S.getSourceRange());
5071 EmitStopPoint(&S);
5072 EmitStmt(S.getAssociatedStmt());
5073}
5074
5076 llvm::SmallVector<const Expr *, 8> CopyprivateVars;
5080 // Check if there are any 'copyprivate' clauses associated with this
5081 // 'single' construct.
5082 // Build a list of copyprivate variables along with helper expressions
5083 // (<source>, <destination>, <destination>=<source> expressions)
5084 for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
5085 CopyprivateVars.append(C->varlist_begin(), C->varlist_end());
5086 DestExprs.append(C->destination_exprs().begin(),
5087 C->destination_exprs().end());
5088 SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
5089 AssignmentOps.append(C->assignment_ops().begin(),
5090 C->assignment_ops().end());
5091 }
5092 // Emit code for 'single' region along with 'copyprivate' clauses
5093 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5094 Action.Enter(CGF);
5098 (void)SingleScope.Privatize();
5099 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
5100 };
5101 {
5102 auto LPCRegion =
5104 OMPLexicalScope Scope(*this, S, OMPD_unknown);
5105 CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getBeginLoc(),
5106 CopyprivateVars, DestExprs,
5107 SrcExprs, AssignmentOps);
5108 }
5109 // Emit an implicit barrier at the end (to avoid data race on firstprivate
5110 // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
5111 if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
5112 CGM.getOpenMPRuntime().emitBarrierCall(
5113 *this, S.getBeginLoc(),
5114 S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
5115 }
5116 // Check for outer lastprivate conditional update.
5118}
5119
5121 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5122 Action.Enter(CGF);
5123 CGF.EmitStmt(S.getRawStmt());
5124 };
5125 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());
5126}
5127
5129 if (CGM.getLangOpts().OpenMPIRBuilder) {
5130 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
5131 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5132
5133 const Stmt *MasterRegionBodyStmt = S.getAssociatedStmt();
5134
5135 auto FiniCB = [this](InsertPointTy IP) {
5137 return llvm::Error::success();
5138 };
5139
5140 auto BodyGenCB = [MasterRegionBodyStmt,
5141 this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
5142 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
5144 *this, MasterRegionBodyStmt, AllocIP, CodeGenIP, "master");
5145 return llvm::Error::success();
5146 };
5147
5148 LexicalScope Scope(*this, S.getSourceRange());
5149 EmitStopPoint(&S);
5150 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
5151 cantFail(OMPBuilder.createMaster(Builder, BodyGenCB, FiniCB));
5152 Builder.restoreIP(AfterIP);
5153
5154 return;
5155 }
5156 LexicalScope Scope(*this, S.getSourceRange());
5157 EmitStopPoint(&S);
5158 emitMaster(*this, S);
5159}
5160
5162 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5163 Action.Enter(CGF);
5164 CGF.EmitStmt(S.getRawStmt());
5165 };
5166 Expr *Filter = nullptr;
5167 if (const auto *FilterClause = S.getSingleClause<OMPFilterClause>())
5168 Filter = FilterClause->getThreadID();
5169 CGF.CGM.getOpenMPRuntime().emitMaskedRegion(CGF, CodeGen, S.getBeginLoc(),
5170 Filter);
5171}
5172
5174 if (CGM.getLangOpts().OpenMPIRBuilder) {
5175 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
5176 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5177
5178 const Stmt *MaskedRegionBodyStmt = S.getAssociatedStmt();
5179 const Expr *Filter = nullptr;
5180 if (const auto *FilterClause = S.getSingleClause<OMPFilterClause>())
5181 Filter = FilterClause->getThreadID();
5182 llvm::Value *FilterVal = Filter
5183 ? EmitScalarExpr(Filter, CGM.Int32Ty)
5184 : llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/0);
5185
5186 auto FiniCB = [this](InsertPointTy IP) {
5188 return llvm::Error::success();
5189 };
5190
5191 auto BodyGenCB = [MaskedRegionBodyStmt,
5192 this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
5193 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
5195 *this, MaskedRegionBodyStmt, AllocIP, CodeGenIP, "masked");
5196 return llvm::Error::success();
5197 };
5198
5199 LexicalScope Scope(*this, S.getSourceRange());
5200 EmitStopPoint(&S);
5201 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
5202 OMPBuilder.createMasked(Builder, BodyGenCB, FiniCB, FilterVal));
5203 Builder.restoreIP(AfterIP);
5204
5205 return;
5206 }
5207 LexicalScope Scope(*this, S.getSourceRange());
5208 EmitStopPoint(&S);
5209 emitMasked(*this, S);
5210}
5211
5213 if (CGM.getLangOpts().OpenMPIRBuilder) {
5214 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
5215 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5216
5217 const Stmt *CriticalRegionBodyStmt = S.getAssociatedStmt();
5218 const Expr *Hint = nullptr;
5219 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
5220 Hint = HintClause->getHint();
5221
5222 // TODO: This is slightly different from what's currently being done in
5223 // clang. Fix the Int32Ty to IntPtrTy (pointer width size) when everything
5224 // about typing is final.
5225 llvm::Value *HintInst = nullptr;
5226 if (Hint)
5227 HintInst =
5228 Builder.CreateIntCast(EmitScalarExpr(Hint), CGM.Int32Ty, false);
5229
5230 auto FiniCB = [this](InsertPointTy IP) {
5232 return llvm::Error::success();
5233 };
5234
5235 auto BodyGenCB = [CriticalRegionBodyStmt,
5236 this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
5237 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
5239 *this, CriticalRegionBodyStmt, AllocIP, CodeGenIP, "critical");
5240 return llvm::Error::success();
5241 };
5242
5243 LexicalScope Scope(*this, S.getSourceRange());
5244 EmitStopPoint(&S);
5245 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
5246 cantFail(OMPBuilder.createCritical(Builder, BodyGenCB, FiniCB,
5248 HintInst));
5249 Builder.restoreIP(AfterIP);
5250
5251 return;
5252 }
5253
5254 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5255 Action.Enter(CGF);
5256 CGF.EmitStmt(S.getAssociatedStmt());
5257 };
5258 const Expr *Hint = nullptr;
5259 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
5260 Hint = HintClause->getHint();
5261 LexicalScope Scope(*this, S.getSourceRange());
5262 EmitStopPoint(&S);
5263 CGM.getOpenMPRuntime().emitCriticalRegion(*this,
5265 CodeGen, S.getBeginLoc(), Hint);
5266}
5267
5269 const OMPParallelForDirective &S) {
5270 // Emit directive as a combined directive that consists of two implicit
5271 // directives: 'parallel' with 'for' directive.
5272 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5273 Action.Enter(CGF);
5274 emitOMPCopyinClause(CGF, S);
5275 (void)emitWorksharingDirective(CGF, S, S.hasCancel());
5276 };
5277 {
5278 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
5281 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
5282 OMPLoopScope LoopScope(CGF, S);
5283 return CGF.EmitScalarExpr(S.getNumIterations());
5284 };
5285 bool IsInscan = llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
5286 [](const OMPReductionClause *C) {
5287 return C->getModifier() == OMPC_REDUCTION_inscan;
5288 });
5289 if (IsInscan)
5290 emitScanBasedDirectiveDecls(*this, S, NumIteratorsGen);
5291 auto LPCRegion =
5293 emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
5295 if (IsInscan)
5296 emitScanBasedDirectiveFinals(*this, S, NumIteratorsGen);
5297 }
5298 // Check for outer lastprivate conditional update.
5300}
5301
5303 const OMPParallelForSimdDirective &S) {
5304 // Emit directive as a combined directive that consists of two implicit
5305 // directives: 'parallel' with 'for' directive.
5306 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5307 Action.Enter(CGF);
5308 emitOMPCopyinClause(CGF, S);
5309 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
5310 };
5311 {
5312 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
5315 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
5316 OMPLoopScope LoopScope(CGF, S);
5317 return CGF.EmitScalarExpr(S.getNumIterations());
5318 };
5319 bool IsInscan = llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
5320 [](const OMPReductionClause *C) {
5321 return C->getModifier() == OMPC_REDUCTION_inscan;
5322 });
5323 if (IsInscan)
5324 emitScanBasedDirectiveDecls(*this, S, NumIteratorsGen);
5325 auto LPCRegion =
5327 emitCommonOMPParallelDirective(*this, S, OMPD_for_simd, CodeGen,
5329 if (IsInscan)
5330 emitScanBasedDirectiveFinals(*this, S, NumIteratorsGen);
5331 }
5332 // Check for outer lastprivate conditional update.
5334}
5335
5337 const OMPParallelMasterDirective &S) {
5338 // Emit directive as a combined directive that consists of two implicit
5339 // directives: 'parallel' with 'master' directive.
5340 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5341 Action.Enter(CGF);
5342 OMPPrivateScope PrivateScope(CGF);
5343 emitOMPCopyinClause(CGF, S);
5344 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
5345 CGF.EmitOMPPrivateClause(S, PrivateScope);
5346 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
5347 (void)PrivateScope.Privatize();
5348 emitMaster(CGF, S);
5349 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
5350 };
5351 {
5352 auto LPCRegion =
5354 emitCommonOMPParallelDirective(*this, S, OMPD_master, CodeGen,
5357 [](CodeGenFunction &) { return nullptr; });
5358 }
5359 // Check for outer lastprivate conditional update.
5361}
5362
5364 const OMPParallelMaskedDirective &S) {
5365 // Emit directive as a combined directive that consists of two implicit
5366 // directives: 'parallel' with 'masked' directive.
5367 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5368 Action.Enter(CGF);
5369 OMPPrivateScope PrivateScope(CGF);
5370 emitOMPCopyinClause(CGF, S);
5371 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
5372 CGF.EmitOMPPrivateClause(S, PrivateScope);
5373 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
5374 (void)PrivateScope.Privatize();
5375 emitMasked(CGF, S);
5376 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
5377 };
5378 {
5379 auto LPCRegion =
5381 emitCommonOMPParallelDirective(*this, S, OMPD_masked, CodeGen,
5384 [](CodeGenFunction &) { return nullptr; });
5385 }
5386 // Check for outer lastprivate conditional update.
5388}
5389
5392 // Emit directive as a combined directive that consists of two implicit
5393 // directives: 'parallel' with 'sections' directive.
5394 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5395 Action.Enter(CGF);
5396 emitOMPCopyinClause(CGF, S);
5397 CGF.EmitSections(S);
5398 };
5399 {
5400 auto LPCRegion =
5402 emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
5404 }
5405 // Check for outer lastprivate conditional update.
5407}
5408
5409namespace {
5410/// Get the list of variables declared in the context of the untied tasks.
5411class CheckVarsEscapingUntiedTaskDeclContext final
5412 : public ConstStmtVisitor<CheckVarsEscapingUntiedTaskDeclContext> {
5414
5415public:
5416 explicit CheckVarsEscapingUntiedTaskDeclContext() = default;
5417 ~CheckVarsEscapingUntiedTaskDeclContext() = default;
5418 void VisitDeclStmt(const DeclStmt *S) {
5419 if (!S)
5420 return;
5421 // Need to privatize only local vars, static locals can be processed as is.
5422 for (const Decl *D : S->decls()) {
5423 if (const auto *VD = dyn_cast_or_null<VarDecl>(D))
5424 if (VD->hasLocalStorage())
5425 PrivateDecls.push_back(VD);
5426 }
5427 }
5428 void VisitOMPExecutableDirective(const OMPExecutableDirective *) {}
5429 void VisitCapturedStmt(const CapturedStmt *) {}
5430 void VisitLambdaExpr(const LambdaExpr *) {}
5431 void VisitBlockExpr(const BlockExpr *) {}
5432 void VisitStmt(const Stmt *S) {
5433 if (!S)
5434 return;
5435 for (const Stmt *Child : S->children())
5436 if (Child)
5437 Visit(Child);
5438 }
5439
5440 /// Swaps list of vars with the provided one.
5441 ArrayRef<const VarDecl *> getPrivateDecls() const { return PrivateDecls; }
5442};
5443} // anonymous namespace
5444
5447
5448 // First look for 'omp_all_memory' and add this first.
5449 bool OmpAllMemory = false;
5450 if (llvm::any_of(
5451 S.getClausesOfKind<OMPDependClause>(), [](const OMPDependClause *C) {
5452 return C->getDependencyKind() == OMPC_DEPEND_outallmemory ||
5453 C->getDependencyKind() == OMPC_DEPEND_inoutallmemory;
5454 })) {
5455 OmpAllMemory = true;
5456 // Since both OMPC_DEPEND_outallmemory and OMPC_DEPEND_inoutallmemory are
5457 // equivalent to the runtime, always use OMPC_DEPEND_outallmemory to
5458 // simplify.
5460 Data.Dependences.emplace_back(OMPC_DEPEND_outallmemory,
5461 /*IteratorExpr=*/nullptr);
5462 // Add a nullptr Expr to simplify the codegen in emitDependData.
5463 DD.DepExprs.push_back(nullptr);
5464 }
5465 // Add remaining dependences skipping any 'out' or 'inout' if they are
5466 // overridden by 'omp_all_memory'.
5467 for (const auto *C : S.getClausesOfKind<OMPDependClause>()) {
5468 OpenMPDependClauseKind Kind = C->getDependencyKind();
5469 if (Kind == OMPC_DEPEND_outallmemory || Kind == OMPC_DEPEND_inoutallmemory)
5470 continue;
5471 if (OmpAllMemory && (Kind == OMPC_DEPEND_out || Kind == OMPC_DEPEND_inout))
5472 continue;
5474 Data.Dependences.emplace_back(C->getDependencyKind(), C->getModifier());
5475 DD.DepExprs.append(C->varlist_begin(), C->varlist_end());
5476 }
5477}
5478
5480 const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
5481 const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
5483 // Emit outlined function for task construct.
5484 const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
5485 auto I = CS->getCapturedDecl()->param_begin();
5486 auto PartId = std::next(I);
5487 auto TaskT = std::next(I, 4);
5488 // Check if the task is final
5489 if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
5490 // If the condition constant folds and can be elided, try to avoid emitting
5491 // the condition and the dead arm of the if/else.
5492 const Expr *Cond = Clause->getCondition();
5493 bool CondConstant;
5494 if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
5495 Data.Final.setInt(CondConstant);
5496 else
5497 Data.Final.setPointer(EvaluateExprAsBool(Cond));
5498 } else {
5499 // By default the task is not final.
5500 Data.Final.setInt(/*IntVal=*/false);
5501 }
5502 // Check if the task has 'priority' clause.
5503 if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
5504 const Expr *Prio = Clause->getPriority();
5505 Data.Priority.setInt(/*IntVal=*/true);
5506 Data.Priority.setPointer(EmitScalarConversion(
5507 EmitScalarExpr(Prio), Prio->getType(),
5508 getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
5509 Prio->getExprLoc()));
5510 }
5511 // The first function argument for tasks is a thread id, the second one is a
5512 // part id (0 for tied tasks, >=0 for untied task).
5513 llvm::DenseSet<const ValueDecl *> EmittedAsPrivate;
5514 // Get list of private variables.
5515 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
5516 auto IRef = C->varlist_begin();
5517 for (const Expr *IInit : C->private_copies()) {
5518 const auto *OrigDecl = cast<DeclRefExpr>(*IRef)->getDecl();
5519 if (EmittedAsPrivate.insert(cast<ValueDecl>(OrigDecl->getCanonicalDecl()))
5520 .second) {
5521 Data.PrivateVars.push_back(*IRef);
5522 Data.PrivateCopies.push_back(IInit);
5523 }
5524 ++IRef;
5525 }
5526 }
5527 EmittedAsPrivate.clear();
5528 // Get list of firstprivate variables.
5529 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
5530 auto IRef = C->varlist_begin();
5531 auto IElemInitRef = C->inits().begin();
5532 for (const Expr *IInit : C->private_copies()) {
5533 const auto *OrigDecl = cast<DeclRefExpr>(*IRef)->getDecl();
5534 if (EmittedAsPrivate.insert(cast<ValueDecl>(OrigDecl->getCanonicalDecl()))
5535 .second) {
5536 Data.FirstprivateVars.push_back(*IRef);
5537 Data.FirstprivateCopies.push_back(IInit);
5538 Data.FirstprivateInits.push_back(*IElemInitRef);
5539 }
5540 ++IRef;
5541 ++IElemInitRef;
5542 }
5543 }
5544 // Get list of lastprivate variables (for taskloops).
5545 llvm::MapVector<const ValueDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
5546 llvm::MapVector<const ValueDecl *, const DeclRefExpr *> LastprivateSrcsOrigs;
5547 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
5548 auto IRef = C->varlist_begin();
5549 auto ID = C->destination_exprs().begin();
5550 auto IS = C->source_exprs().begin();
5551 for (const Expr *IInit : C->private_copies()) {
5552 const auto *OrigDecl = cast<DeclRefExpr>(*IRef)->getDecl();
5553 if (EmittedAsPrivate.insert(cast<ValueDecl>(OrigDecl->getCanonicalDecl()))
5554 .second) {
5555 Data.LastprivateVars.push_back(*IRef);
5556 Data.LastprivateCopies.push_back(IInit);
5557 }
5558 LastprivateDstsOrigs.insert(std::make_pair(
5559 cast<DeclRefExpr>(*ID)->getDecl(), cast<DeclRefExpr>(*IRef)));
5560 LastprivateSrcsOrigs.insert(std::make_pair(
5561 cast<DeclRefExpr>(*IS)->getDecl(), cast<DeclRefExpr>(*IRef)));
5562 ++IRef;
5563 ++ID;
5564 ++IS;
5565 }
5566 }
5569 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
5570 Data.ReductionVars.append(C->varlist_begin(), C->varlist_end());
5571 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
5572 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
5573 Data.ReductionOps.append(C->reduction_ops().begin(),
5574 C->reduction_ops().end());
5575 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
5576 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
5577 }
5578 Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
5579 *this, S.getBeginLoc(), LHSs, RHSs, Data);
5580 // Build list of dependences.
5582 // Get list of local vars for untied tasks.
5583 if (!Data.Tied) {
5584 CheckVarsEscapingUntiedTaskDeclContext Checker;
5585 Checker.Visit(S.getInnermostCapturedStmt()->getCapturedStmt());
5586 Data.PrivateLocals.append(Checker.getPrivateDecls().begin(),
5587 Checker.getPrivateDecls().end());
5588 }
5589 auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
5590 &LastprivateSrcsOrigs, CapturedRegion](
5591 CodeGenFunction &CGF, PrePostActionTy &Action) {
5592 llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
5593 std::pair<Address, Address>>
5594 UntiedLocalVars;
5595 // Set proper addresses for generated private copies.
5597 // Generate debug info for variables present in shared clause.
5598 if (auto *DI = CGF.getDebugInfo()) {
5599 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields =
5600 CGF.CapturedStmtInfo->getCaptureFields();
5601 llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue();
5602 if (CaptureFields.size() && ContextValue) {
5603 unsigned CharWidth = CGF.getContext().getCharWidth();
5604 // The shared variables are packed together as members of structure.
5605 // So the address of each shared variable can be computed by adding
5606 // offset of it (within record) to the base address of record. For each
5607 // shared variable, debug intrinsic llvm.dbg.declare is generated with
5608 // appropriate expressions (DIExpression).
5609 // Ex:
5610 // %12 = load %struct.anon*, %struct.anon** %__context.addr.i
5611 // call void @llvm.dbg.declare(metadata %struct.anon* %12,
5612 // metadata !svar1,
5613 // metadata !DIExpression(DW_OP_deref))
5614 // call void @llvm.dbg.declare(metadata %struct.anon* %12,
5615 // metadata !svar2,
5616 // metadata !DIExpression(DW_OP_plus_uconst, 8, DW_OP_deref))
5617 for (auto It = CaptureFields.begin(); It != CaptureFields.end(); ++It) {
5618 const VarDecl *SharedVar = It->first;
5619 RecordDecl *CaptureRecord = It->second->getParent();
5620 const ASTRecordLayout &Layout =
5621 CGF.getContext().getASTRecordLayout(CaptureRecord);
5622 unsigned Offset =
5623 Layout.getFieldOffset(It->second->getFieldIndex()) / CharWidth;
5624 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
5625 (void)DI->EmitDeclareOfAutoVariable(SharedVar, ContextValue,
5626 CGF.Builder, false);
5627 // Get the call dbg.declare instruction we just created and update
5628 // its DIExpression to add offset to base address.
5629 auto UpdateExpr = [](llvm::LLVMContext &Ctx, auto *Declare,
5630 unsigned Offset) {
5632 // Add offset to the base address if non zero.
5633 if (Offset) {
5634 Ops.push_back(llvm::dwarf::DW_OP_plus_uconst);
5635 Ops.push_back(Offset);
5636 }
5637 Ops.push_back(llvm::dwarf::DW_OP_deref);
5638 Declare->setExpression(llvm::DIExpression::get(Ctx, Ops));
5639 };
5640 llvm::Instruction &Last = CGF.Builder.GetInsertBlock()->back();
5641 // If we're emitting using the new debug info format into a block
5642 // without a terminator, the record will be "trailing".
5643 assert(!Last.isTerminator() && "unexpected terminator");
5644 if (auto *Marker =
5645 CGF.Builder.GetInsertBlock()->getTrailingDbgRecords()) {
5646 for (llvm::DbgVariableRecord &DVR : llvm::reverse(
5647 llvm::filterDbgVars(Marker->getDbgRecordRange()))) {
5648 UpdateExpr(Last.getContext(), &DVR, Offset);
5649 break;
5650 }
5651 }
5652 }
5653 }
5654 }
5656 FirstprivatePtrs;
5657 if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
5658 !Data.LastprivateVars.empty() || !Data.PrivateLocals.empty()) {
5659 enum { PrivatesParam = 2, CopyFnParam = 3 };
5660 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
5661 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
5662 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
5663 CS->getCapturedDecl()->getParam(PrivatesParam)));
5664 // Map privates.
5668 CallArgs.push_back(PrivatesPtr);
5669 ParamTypes.push_back(PrivatesPtr->getType());
5670 for (const Expr *E : Data.PrivateVars) {
5671 const auto *VD = cast<DeclRefExpr>(E)->getDecl();
5672 RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
5673 CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
5674 PrivatePtrs.emplace_back(VD, PrivatePtr);
5675 CallArgs.push_back(PrivatePtr.getPointer());
5676 ParamTypes.push_back(PrivatePtr.getType());
5677 }
5678 for (const Expr *E : Data.FirstprivateVars) {
5679 const auto *VD = cast<DeclRefExpr>(E)->getDecl();
5680 RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
5681 CGF.getContext().getPointerType(E->getType()),
5682 ".firstpriv.ptr.addr");
5683 PrivatePtrs.emplace_back(VD, PrivatePtr);
5684 FirstprivatePtrs.emplace_back(VD, PrivatePtr);
5685 CallArgs.push_back(PrivatePtr.getPointer());
5686 ParamTypes.push_back(PrivatePtr.getType());
5687 }
5688 for (const Expr *E : Data.LastprivateVars) {
5689 const auto *VD = cast<DeclRefExpr>(E)->getDecl();
5690 RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
5691 CGF.getContext().getPointerType(E->getType()),
5692 ".lastpriv.ptr.addr");
5693 PrivatePtrs.emplace_back(VD, PrivatePtr);
5694 CallArgs.push_back(PrivatePtr.getPointer());
5695 ParamTypes.push_back(PrivatePtr.getType());
5696 }
5697 for (const VarDecl *VD : Data.PrivateLocals) {
5699 if (VD->getType()->isLValueReferenceType())
5700 Ty = CGF.getContext().getPointerType(Ty);
5701 if (isAllocatableDecl(VD))
5702 Ty = CGF.getContext().getPointerType(Ty);
5703 RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
5704 CGF.getContext().getPointerType(Ty), ".local.ptr.addr");
5705 auto Result = UntiedLocalVars.insert(
5706 std::make_pair(VD, std::make_pair(PrivatePtr, Address::invalid())));
5707 // If key exists update in place.
5708 if (Result.second == false)
5709 *Result.first = std::make_pair(
5710 VD, std::make_pair(PrivatePtr, Address::invalid()));
5711 CallArgs.push_back(PrivatePtr.getPointer());
5712 ParamTypes.push_back(PrivatePtr.getType());
5713 }
5714 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
5715 ParamTypes, /*isVarArg=*/false);
5716 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
5717 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
5718 for (const auto &Pair : LastprivateDstsOrigs) {
5719 const auto *OrigDecl = Pair.second->getDecl();
5720 if (const auto *BD = dyn_cast<BindingDecl>(OrigDecl)) {
5721 // For BindingDecls, emit the binding's LValue directly.
5722 Address OrigAddr =
5723 CGF.EmitOMPBindingOriginalAddr(BD, Pair.second->getExprLoc());
5724 Scope.addPrivate(Pair.first, OrigAddr);
5725 } else {
5726 const auto *OrigVD = cast<VarDecl>(OrigDecl);
5727 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD),
5728 /*RefersToEnclosingVariableOrCapture=*/
5729 CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr,
5730 Pair.second->getType(), VK_LValue,
5731 Pair.second->getExprLoc());
5732 Scope.addPrivate(Pair.first, CGF.EmitLValue(&DRE).getAddress());
5733 }
5734 }
5735 for (const auto &Pair : PrivatePtrs) {
5736 Address Replacement = Address(
5737 CGF.Builder.CreateLoad(Pair.second),
5738 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
5739 CGF.getContext().getDeclAlign(Pair.first));
5740 Scope.addPrivate(Pair.first, Replacement);
5741
5742 // For BindingDecls with lastprivate, also map the .lastprivate.src
5743 // pseudo-variable to the same private address.
5744 if (isa<BindingDecl>(Pair.first)) {
5745 for (const auto &SrcPair : LastprivateSrcsOrigs) {
5746 if (SrcPair.second->getDecl() == Pair.first) {
5747 Scope.addPrivate(SrcPair.first, Replacement);
5748 break;
5749 }
5750 }
5751 }
5752
5753 if (auto *DI = CGF.getDebugInfo())
5754 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
5755 // Only emit debug info for VarDecls, not BindingDecls.
5756 if (const auto *VD = dyn_cast<VarDecl>(Pair.first))
5757 (void)DI->EmitDeclareOfAutoVariable(
5758 VD, Pair.second.getBasePointer(), CGF.Builder,
5759 /*UsePointerValue*/ true);
5760 }
5761 // Adjust mapping for internal locals by mapping actual memory instead of
5762 // a pointer to this memory.
5763 for (auto &Pair : UntiedLocalVars) {
5764 QualType VDType = Pair.first->getType().getNonReferenceType();
5765 if (Pair.first->getType()->isLValueReferenceType())
5766 VDType = CGF.getContext().getPointerType(VDType);
5767 if (isAllocatableDecl(Pair.first)) {
5768 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
5769 Address Replacement(
5770 Ptr,
5771 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(VDType)),
5772 CGF.getPointerAlign());
5773 Pair.second.first = Replacement;
5774 Ptr = CGF.Builder.CreateLoad(Replacement);
5775 Replacement = Address(Ptr, CGF.ConvertTypeForMem(VDType),
5776 CGF.getContext().getDeclAlign(Pair.first));
5777 Pair.second.second = Replacement;
5778 } else {
5779 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
5780 Address Replacement(Ptr, CGF.ConvertTypeForMem(VDType),
5781 CGF.getContext().getDeclAlign(Pair.first));
5782 Pair.second.first = Replacement;
5783 }
5784 }
5785 }
5786 if (Data.Reductions) {
5787 OMPPrivateScope FirstprivateScope(CGF);
5788 for (const auto &Pair : FirstprivatePtrs) {
5789 Address Replacement(
5790 CGF.Builder.CreateLoad(Pair.second),
5791 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
5792 CGF.getContext().getDeclAlign(Pair.first));
5793 FirstprivateScope.addPrivate(Pair.first, Replacement);
5794 }
5795 (void)FirstprivateScope.Privatize();
5796 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
5797 ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionVars,
5798 Data.ReductionCopies, Data.ReductionOps);
5799 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
5800 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
5801 for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
5802 RedCG.emitSharedOrigLValue(CGF, Cnt);
5803 RedCG.emitAggregateType(CGF, Cnt);
5804 // FIXME: This must removed once the runtime library is fixed.
5805 // Emit required threadprivate variables for
5806 // initializer/combiner/finalizer.
5807 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
5808 RedCG, Cnt);
5809 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5810 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
5811 Replacement = Address(
5812 CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF),
5813 CGF.getContext().VoidPtrTy,
5814 CGF.getContext().getPointerType(
5815 Data.ReductionCopies[Cnt]->getType()),
5816 Data.ReductionCopies[Cnt]->getExprLoc()),
5817 CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()),
5818 Replacement.getAlignment());
5819 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
5820 Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
5821 }
5822 }
5823 // Privatize all private variables except for in_reduction items.
5824 (void)Scope.Privatize();
5828 SmallVector<const Expr *, 4> TaskgroupDescriptors;
5829 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
5830 auto IPriv = C->privates().begin();
5831 auto IRed = C->reduction_ops().begin();
5832 auto ITD = C->taskgroup_descriptors().begin();
5833 for (const Expr *Ref : C->varlist()) {
5834 InRedVars.emplace_back(Ref);
5835 InRedPrivs.emplace_back(*IPriv);
5836 InRedOps.emplace_back(*IRed);
5837 TaskgroupDescriptors.emplace_back(*ITD);
5838 std::advance(IPriv, 1);
5839 std::advance(IRed, 1);
5840 std::advance(ITD, 1);
5841 }
5842 }
5843 // Privatize in_reduction items here, because taskgroup descriptors must be
5844 // privatized earlier.
5845 OMPPrivateScope InRedScope(CGF);
5846 if (!InRedVars.empty()) {
5847 ReductionCodeGen RedCG(InRedVars, InRedVars, InRedPrivs, InRedOps);
5848 for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
5849 RedCG.emitSharedOrigLValue(CGF, Cnt);
5850 RedCG.emitAggregateType(CGF, Cnt);
5851 // The taskgroup descriptor variable is always implicit firstprivate and
5852 // privatized already during processing of the firstprivates.
5853 // FIXME: This must removed once the runtime library is fixed.
5854 // Emit required threadprivate variables for
5855 // initializer/combiner/finalizer.
5856 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
5857 RedCG, Cnt);
5858 llvm::Value *ReductionsPtr;
5859 if (const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
5860 ReductionsPtr = CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr),
5861 TRExpr->getExprLoc());
5862 } else {
5863 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5864 }
5865 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5866 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
5867 Replacement = Address(
5868 CGF.EmitScalarConversion(
5869 Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy,
5870 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
5871 InRedPrivs[Cnt]->getExprLoc()),
5872 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),
5873 Replacement.getAlignment());
5874 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
5875 InRedScope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
5876 }
5877 }
5878 (void)InRedScope.Privatize();
5879
5881 UntiedLocalVars);
5882 Action.Enter(CGF);
5883 BodyGen(CGF);
5884 };
5886 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
5887 S, *I, *PartId, *TaskT, EKind, CodeGen, Data.Tied, Data.NumberOfParts);
5888 OMPLexicalScope Scope(*this, S, std::nullopt,
5889 !isOpenMPParallelDirective(EKind) &&
5890 !isOpenMPSimdDirective(EKind));
5891 TaskGen(*this, OutlinedFn, Data);
5892}
5893
5894static ImplicitParamDecl *
5896 QualType Ty, CapturedDecl *CD,
5897 SourceLocation Loc) {
5898 auto *OrigVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
5900 auto *OrigRef = DeclRefExpr::Create(
5902 /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
5903 auto *PrivateVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
5905 auto *PrivateRef = DeclRefExpr::Create(
5906 C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD,
5907 /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
5908 QualType ElemType = C.getBaseElementType(Ty);
5909 auto *InitVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, ElemType,
5911 auto *InitRef = DeclRefExpr::Create(
5913 /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue);
5914 PrivateVD->setInitStyle(VarDecl::CInit);
5915 PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue,
5916 InitRef, /*BasePath=*/nullptr,
5918 Data.FirstprivateVars.emplace_back(OrigRef);
5919 Data.FirstprivateCopies.emplace_back(PrivateRef);
5920 Data.FirstprivateInits.emplace_back(InitRef);
5921 return OrigVD;
5922}
5923
5925 const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,
5926 OMPTargetDataInfo &InputInfo) {
5927 // Emit outlined function for task construct.
5928 const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
5929 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
5930 CanQualType SharedsTy =
5932 auto I = CS->getCapturedDecl()->param_begin();
5933 auto PartId = std::next(I);
5934 auto TaskT = std::next(I, 4);
5936 // The task is not final.
5937 Data.Final.setInt(/*IntVal=*/false);
5938 // Get list of firstprivate variables.
5939 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
5940 auto IRef = C->varlist_begin();
5941 auto IElemInitRef = C->inits().begin();
5942 for (auto *IInit : C->private_copies()) {
5943 Data.FirstprivateVars.push_back(*IRef);
5944 Data.FirstprivateCopies.push_back(IInit);
5945 Data.FirstprivateInits.push_back(*IElemInitRef);
5946 ++IRef;
5947 ++IElemInitRef;
5948 }
5949 }
5952 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
5953 Data.ReductionVars.append(C->varlist_begin(), C->varlist_end());
5954 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
5955 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
5956 Data.ReductionOps.append(C->reduction_ops().begin(),
5957 C->reduction_ops().end());
5958 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
5959 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
5960 }
5961 OMPPrivateScope TargetScope(*this);
5962 VarDecl *BPVD = nullptr;
5963 VarDecl *PVD = nullptr;
5964 VarDecl *SVD = nullptr;
5965 VarDecl *MVD = nullptr;
5966 if (InputInfo.NumberOfTargetItems > 0) {
5967 auto *CD = CapturedDecl::Create(
5968 getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0);
5969 llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);
5970 QualType BaseAndPointerAndMapperType = getContext().getConstantArrayType(
5971 getContext().VoidPtrTy, ArrSize, nullptr, ArraySizeModifier::Normal,
5972 /*IndexTypeQuals=*/0);
5974 getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5976 getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5978 getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1),
5979 ArrSize, nullptr, ArraySizeModifier::Normal,
5980 /*IndexTypeQuals=*/0);
5981 SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD,
5982 S.getBeginLoc());
5983 TargetScope.addPrivate(BPVD, InputInfo.BasePointersArray);
5984 TargetScope.addPrivate(PVD, InputInfo.PointersArray);
5985 TargetScope.addPrivate(SVD, InputInfo.SizesArray);
5986 // If there is no user-defined mapper, the mapper array will be nullptr. In
5987 // this case, we don't need to privatize it.
5988 if (!isa_and_nonnull<llvm::ConstantPointerNull>(
5989 InputInfo.MappersArray.emitRawPointer(*this))) {
5991 getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5992 TargetScope.addPrivate(MVD, InputInfo.MappersArray);
5993 }
5994 }
5995 (void)TargetScope.Privatize();
5998 auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD, MVD, EKind,
5999 &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
6000 // Set proper addresses for generated private copies.
6002 if (!Data.FirstprivateVars.empty()) {
6003 enum { PrivatesParam = 2, CopyFnParam = 3 };
6004 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
6005 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
6006 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
6007 CS->getCapturedDecl()->getParam(PrivatesParam)));
6008 // Map privates.
6012 CallArgs.push_back(PrivatesPtr);
6013 ParamTypes.push_back(PrivatesPtr->getType());
6014 for (const Expr *E : Data.FirstprivateVars) {
6015 const auto *VD = cast<DeclRefExpr>(E)->getDecl();
6016 RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
6017 CGF.getContext().getPointerType(E->getType()),
6018 ".firstpriv.ptr.addr");
6019 PrivatePtrs.emplace_back(VD, PrivatePtr);
6020 CallArgs.push_back(PrivatePtr.getPointer());
6021 ParamTypes.push_back(PrivatePtr.getType());
6022 }
6023 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
6024 ParamTypes, /*isVarArg=*/false);
6025 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
6026 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
6027 for (const auto &Pair : PrivatePtrs) {
6028 Address Replacement(
6029 CGF.Builder.CreateLoad(Pair.second),
6030 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
6031 CGF.getContext().getDeclAlign(Pair.first));
6032 Scope.addPrivate(Pair.first, Replacement);
6033 }
6034 }
6035 CGF.processInReduction(S, Data, CGF, CS, Scope);
6036 if (InputInfo.NumberOfTargetItems > 0) {
6037 InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
6038 CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0);
6039 InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
6040 CGF.GetAddrOfLocalVar(PVD), /*Index=*/0);
6041 InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
6042 CGF.GetAddrOfLocalVar(SVD), /*Index=*/0);
6043 // If MVD is nullptr, the mapper array is not privatized
6044 if (MVD)
6045 InputInfo.MappersArray = CGF.Builder.CreateConstArrayGEP(
6046 CGF.GetAddrOfLocalVar(MVD), /*Index=*/0);
6047 }
6048
6049 Action.Enter(CGF);
6050 OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);
6051 auto *TL = S.getSingleClause<OMPThreadLimitClause>();
6052 if (CGF.CGM.getLangOpts().OpenMP >= 51 &&
6053 needsTaskBasedThreadLimit(EKind) && TL) {
6054 // Emit __kmpc_set_thread_limit() to set the thread_limit for the task
6055 // enclosing this target region. This will indirectly set the thread_limit
6056 // for every applicable construct within target region.
6057 CGF.CGM.getOpenMPRuntime().emitThreadLimitClause(
6058 CGF, TL->getThreadLimit().front(), S.getBeginLoc());
6059 }
6060 BodyGen(CGF);
6061 };
6062 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
6063 S, *I, *PartId, *TaskT, EKind, CodeGen, /*Tied=*/true,
6064 Data.NumberOfParts);
6065 llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
6066 IntegerLiteral IfCond(getContext(), TrueOrFalse,
6067 getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
6068 SourceLocation());
6069 CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn,
6070 SharedsTy, CapturedStruct, &IfCond, Data);
6071}
6072
6075 CodeGenFunction &CGF,
6076 const CapturedStmt *CS,
6079 if (Data.Reductions) {
6080 OpenMPDirectiveKind CapturedRegion = EKind;
6081 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
6082 ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionVars,
6083 Data.ReductionCopies, Data.ReductionOps);
6084 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
6086 for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
6087 RedCG.emitSharedOrigLValue(CGF, Cnt);
6088 RedCG.emitAggregateType(CGF, Cnt);
6089 // FIXME: This must removed once the runtime library is fixed.
6090 // Emit required threadprivate variables for
6091 // initializer/combiner/finalizer.
6092 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
6093 RedCG, Cnt);
6095 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
6096 Replacement = Address(
6097 CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF),
6098 CGF.getContext().VoidPtrTy,
6100 Data.ReductionCopies[Cnt]->getType()),
6101 Data.ReductionCopies[Cnt]->getExprLoc()),
6102 CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()),
6103 Replacement.getAlignment());
6104 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
6105 Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
6106 }
6107 }
6108 (void)Scope.Privatize();
6112 SmallVector<const Expr *, 4> TaskgroupDescriptors;
6113 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
6114 auto IPriv = C->privates().begin();
6115 auto IRed = C->reduction_ops().begin();
6116 auto ITD = C->taskgroup_descriptors().begin();
6117 for (const Expr *Ref : C->varlist()) {
6118 InRedVars.emplace_back(Ref);
6119 InRedPrivs.emplace_back(*IPriv);
6120 InRedOps.emplace_back(*IRed);
6121 TaskgroupDescriptors.emplace_back(*ITD);
6122 std::advance(IPriv, 1);
6123 std::advance(IRed, 1);
6124 std::advance(ITD, 1);
6125 }
6126 }
6127 OMPPrivateScope InRedScope(CGF);
6128 if (!InRedVars.empty()) {
6129 ReductionCodeGen RedCG(InRedVars, InRedVars, InRedPrivs, InRedOps);
6130 for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
6131 RedCG.emitSharedOrigLValue(CGF, Cnt);
6132 RedCG.emitAggregateType(CGF, Cnt);
6133 // FIXME: This must removed once the runtime library is fixed.
6134 // Emit required threadprivate variables for
6135 // initializer/combiner/finalizer.
6136 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
6137 RedCG, Cnt);
6138 llvm::Value *ReductionsPtr;
6139 if (const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
6140 ReductionsPtr =
6141 CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr), TRExpr->getExprLoc());
6142 } else {
6143 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
6144 }
6146 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
6147 Replacement = Address(
6149 Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy,
6150 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
6151 InRedPrivs[Cnt]->getExprLoc()),
6152 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),
6153 Replacement.getAlignment());
6154 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
6155 InRedScope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
6156 }
6157 }
6158 (void)InRedScope.Privatize();
6159}
6160
6162 // Emit outlined function for task construct.
6163 const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
6164 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
6165 CanQualType SharedsTy =
6167 const Expr *IfCond = nullptr;
6168 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
6169 if (C->getNameModifier() == OMPD_unknown ||
6170 C->getNameModifier() == OMPD_task) {
6171 IfCond = C->getCondition();
6172 break;
6173 }
6174 }
6175
6177 // Check if we should emit tied or untied task.
6178 Data.Tied = !S.getSingleClause<OMPUntiedClause>();
6179 auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
6180 CGF.EmitStmt(CS->getCapturedStmt());
6181 };
6182 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
6183 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
6184 const OMPTaskDataTy &Data) {
6185 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn,
6186 SharedsTy, CapturedStruct, IfCond,
6187 Data);
6188 };
6189 auto LPCRegion =
6191 EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data);
6192}
6193
6195 const OMPTaskyieldDirective &S) {
6196 CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getBeginLoc());
6197}
6198
6200 const OMPMessageClause *MC = S.getSingleClause<OMPMessageClause>();
6201 Expr *ME = MC ? MC->getMessageString() : nullptr;
6202 const OMPSeverityClause *SC = S.getSingleClause<OMPSeverityClause>();
6203 bool IsFatal = false;
6204 if (!SC || SC->getSeverityKind() == OMPC_SEVERITY_fatal)
6205 IsFatal = true;
6206 CGM.getOpenMPRuntime().emitErrorCall(*this, S.getBeginLoc(), ME, IsFatal);
6207}
6208
6210 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_barrier);
6211}
6212
6215 // Build list of dependences
6217 Data.HasNowaitClause = S.hasClausesOfKind<OMPNowaitClause>();
6218 CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc(), Data);
6219}
6220
6222 return T.clauses().empty();
6223}
6224
6226 const OMPTaskgroupDirective &S) {
6227 OMPLexicalScope Scope(*this, S, OMPD_unknown);
6228 if (CGM.getLangOpts().OpenMPIRBuilder && isSupportedByOpenMPIRBuilder(S)) {
6229 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
6230 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
6231 InsertPointTy AllocaIP(AllocaInsertPt->getIterator());
6232
6233 auto BodyGenCB = [&, this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
6234 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
6235 Builder.restoreIP(CodeGenIP);
6236 EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
6237 return llvm::Error::success();
6238 };
6241 : &CapStmtInfo);
6242 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
6243 cantFail(OMPBuilder.createTaskgroup(Builder, AllocaIP,
6244 /*DeallocBlocks=*/{}, BodyGenCB));
6245 Builder.restoreIP(AfterIP);
6246 return;
6247 }
6248 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6249 Action.Enter(CGF);
6250 if (const Expr *E = S.getReductionRef()) {
6254 for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
6255 Data.ReductionVars.append(C->varlist_begin(), C->varlist_end());
6256 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
6257 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
6258 Data.ReductionOps.append(C->reduction_ops().begin(),
6259 C->reduction_ops().end());
6260 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
6261 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
6262 }
6263 llvm::Value *ReductionDesc =
6264 CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(),
6265 LHSs, RHSs, Data);
6266 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
6267 CGF.EmitVarDecl(*VD);
6268 CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),
6269 /*Volatile=*/false, E->getType());
6270 }
6271 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
6272 };
6273 CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getBeginLoc());
6274}
6275
6277 llvm::AtomicOrdering AO = S.getSingleClause<OMPFlushClause>()
6278 ? llvm::AtomicOrdering::NotAtomic
6279 : llvm::AtomicOrdering::AcquireRelease;
6280 CGM.getOpenMPRuntime().emitFlush(
6281 *this,
6282 [&S]() -> ArrayRef<const Expr *> {
6283 if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>())
6284 return llvm::ArrayRef(FlushClause->varlist_begin(),
6285 FlushClause->varlist_end());
6286 return {};
6287 }(),
6288 S.getBeginLoc(), AO);
6289}
6290
6292 const auto *DO = S.getSingleClause<OMPDepobjClause>();
6293 LValue DOLVal = EmitLValue(DO->getDepobj());
6294 if (const auto *DC = S.getSingleClause<OMPDependClause>()) {
6295 // Build list and emit dependences
6298 for (auto &Dep : Data.Dependences) {
6299 Address DepAddr = CGM.getOpenMPRuntime().emitDepobjDependClause(
6300 *this, Dep, DC->getBeginLoc());
6301 EmitStoreOfScalar(DepAddr.emitRawPointer(*this), DOLVal);
6302 }
6303 return;
6304 }
6305 if (const auto *DC = S.getSingleClause<OMPDestroyClause>()) {
6306 CGM.getOpenMPRuntime().emitDestroyClause(*this, DOLVal, DC->getBeginLoc());
6307 return;
6308 }
6309 if (const auto *UC = S.getSingleClause<OMPUpdateDependObjectsClause>()) {
6310 CGM.getOpenMPRuntime().emitUpdateDependObjectsClause(
6311 *this, DOLVal, UC->getDependencyKind(), UC->getBeginLoc());
6312 return;
6313 }
6314}
6315
6318 return;
6320 bool IsInclusive = S.hasClausesOfKind<OMPInclusiveClause>();
6325 SmallVector<const Expr *, 4> ReductionOps;
6327 SmallVector<const Expr *, 4> CopyArrayTemps;
6328 SmallVector<const Expr *, 4> CopyArrayElems;
6329 for (const auto *C : ParentDir.getClausesOfKind<OMPReductionClause>()) {
6330 if (C->getModifier() != OMPC_REDUCTION_inscan)
6331 continue;
6332 Shareds.append(C->varlist_begin(), C->varlist_end());
6333 Privates.append(C->privates().begin(), C->privates().end());
6334 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
6335 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
6336 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
6337 CopyOps.append(C->copy_ops().begin(), C->copy_ops().end());
6338 CopyArrayTemps.append(C->copy_array_temps().begin(),
6339 C->copy_array_temps().end());
6340 CopyArrayElems.append(C->copy_array_elems().begin(),
6341 C->copy_array_elems().end());
6342 }
6343 if (ParentDir.getDirectiveKind() == OMPD_simd ||
6344 (getLangOpts().OpenMPSimd &&
6345 isOpenMPSimdDirective(ParentDir.getDirectiveKind()))) {
6346 // For simd directive and simd-based directives in simd only mode, use the
6347 // following codegen:
6348 // int x = 0;
6349 // #pragma omp simd reduction(inscan, +: x)
6350 // for (..) {
6351 // <first part>
6352 // #pragma omp scan inclusive(x)
6353 // <second part>
6354 // }
6355 // is transformed to:
6356 // int x = 0;
6357 // for (..) {
6358 // int x_priv = 0;
6359 // <first part>
6360 // x = x_priv + x;
6361 // x_priv = x;
6362 // <second part>
6363 // }
6364 // and
6365 // int x = 0;
6366 // #pragma omp simd reduction(inscan, +: x)
6367 // for (..) {
6368 // <first part>
6369 // #pragma omp scan exclusive(x)
6370 // <second part>
6371 // }
6372 // to
6373 // int x = 0;
6374 // for (..) {
6375 // int x_priv = 0;
6376 // <second part>
6377 // int temp = x;
6378 // x = x_priv + x;
6379 // x_priv = temp;
6380 // <first part>
6381 // }
6382 llvm::BasicBlock *OMPScanReduce = createBasicBlock("omp.inscan.reduce");
6383 EmitBranch(IsInclusive
6384 ? OMPScanReduce
6385 : BreakContinueStack.back().ContinueBlock.getBlock());
6387 {
6388 // New scope for correct construction/destruction of temp variables for
6389 // exclusive scan.
6390 LexicalScope Scope(*this, S.getSourceRange());
6392 EmitBlock(OMPScanReduce);
6393 if (!IsInclusive) {
6394 // Create temp var and copy LHS value to this temp value.
6395 // TMP = LHS;
6396 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
6397 const Expr *PrivateExpr = Privates[I];
6398 const Expr *TempExpr = CopyArrayTemps[I];
6400 *cast<VarDecl>(cast<DeclRefExpr>(TempExpr)->getDecl()));
6401 LValue DestLVal = EmitLValue(TempExpr);
6402 LValue SrcLVal = EmitLValue(LHSs[I]);
6403 EmitOMPCopy(PrivateExpr->getType(), DestLVal.getAddress(),
6404 SrcLVal.getAddress(),
6405 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
6406 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
6407 CopyOps[I]);
6408 }
6409 }
6410 CGM.getOpenMPRuntime().emitReduction(
6411 *this, ParentDir.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
6412 {/*WithNowait=*/true, /*SimpleReduction=*/true,
6413 /*IsPrivateVarReduction*/ {}, OMPD_simd});
6414 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
6415 const Expr *PrivateExpr = Privates[I];
6416 LValue DestLVal;
6417 LValue SrcLVal;
6418 if (IsInclusive) {
6419 DestLVal = EmitLValue(RHSs[I]);
6420 SrcLVal = EmitLValue(LHSs[I]);
6421 } else {
6422 const Expr *TempExpr = CopyArrayTemps[I];
6423 DestLVal = EmitLValue(RHSs[I]);
6424 SrcLVal = EmitLValue(TempExpr);
6425 }
6427 PrivateExpr->getType(), DestLVal.getAddress(), SrcLVal.getAddress(),
6428 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
6429 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]);
6430 }
6431 }
6433 OMPScanExitBlock = IsInclusive
6434 ? BreakContinueStack.back().ContinueBlock.getBlock()
6435 : OMPScanReduce;
6437 return;
6438 }
6439 if (!IsInclusive) {
6440 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
6442 }
6443 if (OMPFirstScanLoop) {
6444 // Emit buffer[i] = red; at the end of the input phase.
6445 const auto *IVExpr = cast<OMPLoopDirective>(ParentDir)
6446 .getIterationVariable()
6447 ->IgnoreParenImpCasts();
6448 LValue IdxLVal = EmitLValue(IVExpr);
6449 llvm::Value *IdxVal = EmitLoadOfScalar(IdxLVal, IVExpr->getExprLoc());
6450 IdxVal = Builder.CreateIntCast(IdxVal, SizeTy, /*isSigned=*/false);
6451 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
6452 const Expr *PrivateExpr = Privates[I];
6453 const Expr *OrigExpr = Shareds[I];
6454 const Expr *CopyArrayElem = CopyArrayElems[I];
6455 OpaqueValueMapping IdxMapping(
6456 *this,
6458 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
6459 RValue::get(IdxVal));
6460 LValue DestLVal = EmitLValue(CopyArrayElem);
6461 LValue SrcLVal = EmitLValue(OrigExpr);
6463 PrivateExpr->getType(), DestLVal.getAddress(), SrcLVal.getAddress(),
6464 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
6465 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]);
6466 }
6467 }
6468 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
6469 if (IsInclusive) {
6471 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
6472 }
6474 if (!OMPFirstScanLoop) {
6475 // Emit red = buffer[i]; at the entrance to the scan phase.
6476 const auto *IVExpr = cast<OMPLoopDirective>(ParentDir)
6477 .getIterationVariable()
6478 ->IgnoreParenImpCasts();
6479 LValue IdxLVal = EmitLValue(IVExpr);
6480 llvm::Value *IdxVal = EmitLoadOfScalar(IdxLVal, IVExpr->getExprLoc());
6481 IdxVal = Builder.CreateIntCast(IdxVal, SizeTy, /*isSigned=*/false);
6482 llvm::BasicBlock *ExclusiveExitBB = nullptr;
6483 if (!IsInclusive) {
6484 llvm::BasicBlock *ContBB = createBasicBlock("omp.exclusive.dec");
6485 ExclusiveExitBB = createBasicBlock("omp.exclusive.copy.exit");
6486 llvm::Value *Cmp = Builder.CreateIsNull(IdxVal);
6487 Builder.CreateCondBr(Cmp, ExclusiveExitBB, ContBB);
6488 EmitBlock(ContBB);
6489 // Use idx - 1 iteration for exclusive scan.
6490 IdxVal = Builder.CreateNUWSub(IdxVal, llvm::ConstantInt::get(SizeTy, 1));
6491 }
6492 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
6493 const Expr *PrivateExpr = Privates[I];
6494 const Expr *OrigExpr = Shareds[I];
6495 const Expr *CopyArrayElem = CopyArrayElems[I];
6496 OpaqueValueMapping IdxMapping(
6497 *this,
6499 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
6500 RValue::get(IdxVal));
6501 LValue SrcLVal = EmitLValue(CopyArrayElem);
6502 LValue DestLVal = EmitLValue(OrigExpr);
6504 PrivateExpr->getType(), DestLVal.getAddress(), SrcLVal.getAddress(),
6505 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
6506 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]);
6507 }
6508 if (!IsInclusive) {
6509 EmitBlock(ExclusiveExitBB);
6510 }
6511 }
6515}
6516
6518 const CodeGenLoopTy &CodeGenLoop,
6519 Expr *IncExpr) {
6520 // Emit the loop iteration variable.
6521 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
6522 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
6523 EmitVarDecl(*IVDecl);
6524
6525 // Emit the iterations count variable.
6526 // If it is not a variable, Sema decided to calculate iterations count on each
6527 // iteration (e.g., it is foldable into a constant).
6528 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
6529 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
6530 // Emit calculation of the iterations count.
6532 }
6533
6534 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
6535
6537 bool HasLastprivateClause = false;
6538 // Check pre-condition.
6539 {
6540 OMPLoopScope PreInitScope(*this, S);
6541 // Skip the entire loop if we don't meet the precondition.
6542 // If the condition constant folds and can be elided, avoid emitting the
6543 // whole loop.
6544 bool CondConstant;
6545 llvm::BasicBlock *ContBlock = nullptr;
6546 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
6547 if (!CondConstant)
6548 return;
6549 } else {
6550 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
6551 ContBlock = createBasicBlock("omp.precond.end");
6552 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
6553 getProfileCount(&S));
6554 EmitBlock(ThenBlock);
6556 }
6557
6558 emitAlignedClause(*this, S);
6559 // Emit 'then' code.
6560 {
6561 // Emit helper vars inits.
6562
6566 : S.getLowerBoundVariable()));
6570 : S.getUpperBoundVariable()));
6571 LValue ST =
6573 LValue IL =
6575
6576 OMPPrivateScope LoopScope(*this);
6577 if (EmitOMPFirstprivateClause(S, LoopScope)) {
6578 // Emit implicit barrier to synchronize threads and avoid data races
6579 // on initialization of firstprivate variables and post-update of
6580 // lastprivate variables.
6581 CGM.getOpenMPRuntime().emitBarrierCall(
6582 *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
6583 /*ForceSimpleCall=*/true);
6584 }
6585 EmitOMPPrivateClause(S, LoopScope);
6586 if (isOpenMPSimdDirective(S.getDirectiveKind()) &&
6587 !isOpenMPParallelDirective(S.getDirectiveKind()) &&
6588 !isOpenMPTeamsDirective(S.getDirectiveKind()))
6589 EmitOMPReductionClauseInit(S, LoopScope);
6590 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
6591 EmitOMPPrivateLoopCounters(S, LoopScope);
6592 (void)LoopScope.Privatize();
6593 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
6594 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
6595
6596 // Detect the distribute schedule kind and chunk.
6597 llvm::Value *Chunk = nullptr;
6599 if (const auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
6600 ScheduleKind = C->getDistScheduleKind();
6601 if (const Expr *Ch = C->getChunkSize()) {
6602 Chunk = EmitScalarExpr(Ch);
6603 Chunk = EmitScalarConversion(Chunk, Ch->getType(),
6605 S.getBeginLoc());
6606 }
6607 } else {
6608 // Default behaviour for dist_schedule clause.
6609 CGM.getOpenMPRuntime().getDefaultDistScheduleAndChunk(
6610 *this, S, ScheduleKind, Chunk);
6611 }
6612 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
6613 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
6614
6615 // GPU fused schedule: omit the outer distribute loop and let the inner
6616 // worksharing loop schedule the flattened team/thread iteration space.
6617 if (canEmitGPUFusedDistSchedule(CGM, S, EKind)) {
6620 CodeGenLoop(*this, S, LoopExit);
6621 EmitBlock(LoopExit.getBlock());
6622 } else {
6623 // OpenMP [2.10.8, distribute Construct, Description]
6624 // If dist_schedule is specified, kind must be static. If specified,
6625 // iterations are divided into chunks of size chunk_size, chunks are
6626 // assigned to the teams of the league in a round-robin fashion in the
6627 // order of the team number. When no chunk_size is specified, the
6628 // iteration space is divided into chunks that are approximately equal
6629 // in size, and at most one chunk is distributed to each team of the
6630 // league. The size of the chunks is unspecified in this case.
6631 bool StaticChunked =
6632 RT.isStaticChunked(ScheduleKind, /* Chunked */ Chunk != nullptr) &&
6634 if (RT.isStaticNonchunked(ScheduleKind,
6635 /* Chunked */ Chunk != nullptr) ||
6636 StaticChunked) {
6638 IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(),
6639 LB.getAddress(), UB.getAddress(), ST.getAddress(),
6640 StaticChunked ? Chunk : nullptr);
6641 RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind,
6642 StaticInit);
6645 // UB = min(UB, GlobalUB);
6648 : S.getEnsureUpperBound());
6649 // IV = LB;
6651 ? S.getCombinedInit()
6652 : S.getInit());
6653
6654 const Expr *Cond = isOpenMPLoopBoundSharingDirective(EKind)
6655 ? S.getCombinedCond()
6656 : S.getCond();
6657
6658 if (StaticChunked)
6659 Cond = S.getCombinedDistCond();
6660
6661 // For static unchunked schedules generate:
6662 //
6663 // 1. For distribute alone, codegen
6664 // while (idx <= UB) {
6665 // BODY;
6666 // ++idx;
6667 // }
6668 //
6669 // 2. When combined with 'for' (e.g. as in 'distribute parallel for')
6670 // while (idx <= UB) {
6671 // <CodeGen rest of pragma>(LB, UB);
6672 // idx += ST;
6673 // }
6674 //
6675 // For static chunk one schedule generate:
6676 //
6677 // while (IV <= GlobalUB) {
6678 // <CodeGen rest of pragma>(LB, UB);
6679 // LB += ST;
6680 // UB += ST;
6681 // UB = min(UB, GlobalUB);
6682 // IV = LB;
6683 // }
6684 //
6686 *this, S,
6687 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
6688 if (isOpenMPSimdDirective(S.getDirectiveKind()))
6689 CGF.EmitOMPSimdInit(S);
6690 },
6691 [&S, &LoopScope, Cond, IncExpr, LoopExit, &CodeGenLoop,
6692 StaticChunked](CodeGenFunction &CGF, PrePostActionTy &) {
6693 CGF.EmitOMPInnerLoop(
6694 S, LoopScope.requiresCleanups(), Cond, IncExpr,
6695 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
6696 CodeGenLoop(CGF, S, LoopExit);
6697 },
6698 [&S, StaticChunked](CodeGenFunction &CGF) {
6699 if (StaticChunked) {
6700 CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());
6701 CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());
6702 CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());
6703 CGF.EmitIgnoredExpr(S.getCombinedInit());
6704 }
6705 });
6706 });
6707 EmitBlock(LoopExit.getBlock());
6708 // Tell the runtime we are done.
6709 RT.emitForStaticFinish(*this, S.getEndLoc(), OMPD_distribute);
6710 } else {
6711 // Emit the outer loop, which requests its work chunk [LB..UB] from
6712 // runtime and runs the inner loop to process it.
6713 const OMPLoopArguments LoopArguments = {
6714 LB.getAddress(), UB.getAddress(), ST.getAddress(),
6715 IL.getAddress(), Chunk};
6716 EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
6717 CodeGenLoop);
6718 }
6719 }
6720 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
6721 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
6722 return CGF.Builder.CreateIsNotNull(
6723 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
6724 });
6725 }
6726 if (isOpenMPSimdDirective(S.getDirectiveKind()) &&
6727 !isOpenMPParallelDirective(S.getDirectiveKind()) &&
6728 !isOpenMPTeamsDirective(S.getDirectiveKind())) {
6729 EmitOMPReductionClauseFinal(S, OMPD_simd);
6730 // Emit post-update of the reduction variables if IsLastIter != 0.
6732 *this, S, [IL, &S](CodeGenFunction &CGF) {
6733 return CGF.Builder.CreateIsNotNull(
6734 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
6735 });
6736 }
6737 // Emit final copy of the lastprivate variables if IsLastIter != 0.
6738 if (HasLastprivateClause) {
6740 S, /*NoFinals=*/false,
6741 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
6742 }
6743 }
6744
6745 // We're now done with the loop, so jump to the continuation block.
6746 if (ContBlock) {
6747 EmitBranch(ContBlock);
6748 EmitBlock(ContBlock, true);
6749 }
6750 }
6751}
6752
6753// Pass OMPLoopDirective (instead of OMPDistributeDirective) to make this
6754// function available for "loop bind(teams)", which maps to "distribute".
6756 CodeGenFunction &CGF,
6757 CodeGenModule &CGM) {
6758 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
6760 };
6761 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
6762 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, CodeGen);
6763}
6764
6769
6770static llvm::Function *
6772 const OMPExecutableDirective &D) {
6773 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
6775 CGF.CapturedStmtInfo = &CapStmtInfo;
6776 llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S, D);
6777 Fn->setDoesNotRecurse();
6778 return Fn;
6779}
6780
6781template <typename T>
6782static void emitRestoreIP(CodeGenFunction &CGF, const T *C,
6783 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP,
6784 llvm::OpenMPIRBuilder &OMPBuilder) {
6785
6786 unsigned NumLoops = C->getNumLoops();
6788 /*DestWidth=*/64, /*Signed=*/1);
6790 for (unsigned I = 0; I < NumLoops; I++) {
6791 const Expr *CounterVal = C->getLoopData(I);
6792 assert(CounterVal);
6793 llvm::Value *StoreValue = CGF.EmitScalarConversion(
6794 CGF.EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty,
6795 CounterVal->getExprLoc());
6796 StoreValues.emplace_back(StoreValue);
6797 }
6798 OMPDoacrossKind<T> ODK;
6799 bool IsDependSource = ODK.isSource(C);
6800 CGF.Builder.restoreIP(
6801 OMPBuilder.createOrderedDepend(CGF.Builder, AllocaIP, NumLoops,
6802 StoreValues, ".cnt.addr", IsDependSource));
6803}
6804
6807 assert((S.hasClausesOfKind<OMPDependClause>() ||
6808 S.hasClausesOfKind<OMPDoacrossClause>()) &&
6809 "Standalone ordered directive should have either depend or doacross "
6810 "clause");
6811 // The ordered-standalone directive.
6812 assert(!S.hasAssociatedStmt() && "No associated statement must be in "
6813 "ordered depend|doacross construct.");
6814
6815 if (CGM.getLangOpts().OpenMPIRBuilder) {
6816 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
6817 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
6818
6819 InsertPointTy AllocaIP(AllocaInsertPt->getIterator());
6820 for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
6821 emitRestoreIP(*this, DC, AllocaIP, OMPBuilder);
6822 for (const auto *DC : S.getClausesOfKind<OMPDoacrossClause>())
6823 emitRestoreIP(*this, DC, AllocaIP, OMPBuilder);
6824 return;
6825 }
6826
6827 if (S.hasClausesOfKind<OMPDependClause>()) {
6828 for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
6829 CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
6830 } else if (S.hasClausesOfKind<OMPDoacrossClause>()) {
6831 for (const auto *DC : S.getClausesOfKind<OMPDoacrossClause>())
6832 CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
6833 }
6834}
6835
6838 if (CGM.getLangOpts().OpenMPIRBuilder) {
6839 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
6840 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
6841
6842 // The ordered directive with threads or simd clause, or without clause.
6843 // Without clause, it behaves as if the threads clause is specified.
6844 const auto *C = S.getSingleClause<OMPSIMDClause>();
6845
6846 auto FiniCB = [this](InsertPointTy IP) {
6848 return llvm::Error::success();
6849 };
6850
6851 auto BodyGenCB = [&S, C, this](InsertPointTy AllocIP,
6852 InsertPointTy CodeGenIP,
6853 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
6854 Builder.restoreIP(CodeGenIP);
6855
6856 const CapturedStmt *CS = S.getInnermostCapturedStmt();
6857 if (C) {
6858 llvm::BasicBlock *CodeGenBB = CodeGenIP.getNodeParent();
6859 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
6860 Builder, /*CreateBranch=*/false, ".ordered.after");
6862 GenerateOpenMPCapturedVars(*CS, CapturedVars);
6863 llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS, S);
6864 assert(S.getBeginLoc().isValid() &&
6865 "Outlined function call location must be valid.");
6866 ApplyDebugLocation::CreateDefaultArtificial(*this, S.getBeginLoc());
6867 OMPBuilderCBHelpers::EmitCaptureStmt(*this, CodeGenBB, *FiniBB,
6868 OutlinedFn, CapturedVars);
6869 } else {
6871 *this, CS->getCapturedStmt(), AllocIP, CodeGenIP, "ordered");
6872 }
6873 return llvm::Error::success();
6874 };
6875
6876 OMPLexicalScope Scope(*this, S, OMPD_unknown);
6877 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
6878 OMPBuilder.createOrderedThreadsSimd(Builder, BodyGenCB, FiniCB, !C));
6879 Builder.restoreIP(AfterIP);
6880 return;
6881 }
6882
6883 const auto *C = S.getSingleClause<OMPSIMDClause>();
6884 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
6885 PrePostActionTy &Action) {
6886 const CapturedStmt *CS = S.getInnermostCapturedStmt();
6887 if (C) {
6889 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
6890 llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS, S);
6891 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
6892 OutlinedFn, CapturedVars);
6893 } else {
6894 Action.Enter(CGF);
6895 CGF.EmitStmt(CS->getCapturedStmt());
6896 }
6897 };
6898 OMPLexicalScope Scope(*this, S, OMPD_unknown);
6899 CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getBeginLoc(), !C);
6900}
6901
6902static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val,
6903 QualType SrcType, QualType DestType,
6904 SourceLocation Loc) {
6905 assert(CGF.hasScalarEvaluationKind(DestType) &&
6906 "DestType must have scalar evaluation kind.");
6907 assert(!Val.isAggregate() && "Must be a scalar or complex.");
6908 return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
6909 DestType, Loc)
6911 Val.getComplexVal(), SrcType, DestType, Loc);
6912}
6913
6916 QualType DestType, SourceLocation Loc) {
6917 assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
6918 "DestType must have complex evaluation kind.");
6920 if (Val.isScalar()) {
6921 // Convert the input element to the element type of the complex.
6922 QualType DestElementType =
6923 DestType->castAs<ComplexType>()->getElementType();
6924 llvm::Value *ScalarVal = CGF.EmitScalarConversion(
6925 Val.getScalarVal(), SrcType, DestElementType, Loc);
6926 ComplexVal = CodeGenFunction::ComplexPairTy(
6927 ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
6928 } else {
6929 assert(Val.isComplex() && "Must be a scalar or complex.");
6930 QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
6931 QualType DestElementType =
6932 DestType->castAs<ComplexType>()->getElementType();
6933 ComplexVal.first = CGF.EmitScalarConversion(
6934 Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
6935 ComplexVal.second = CGF.EmitScalarConversion(
6936 Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
6937 }
6938 return ComplexVal;
6939}
6940
6941static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO,
6942 LValue LVal, RValue RVal) {
6943 if (LVal.isGlobalReg())
6944 CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
6945 else
6946 CGF.EmitAtomicStore(RVal, LVal, AO, LVal.isVolatile(), /*isInit=*/false);
6947}
6948
6950 llvm::AtomicOrdering AO, LValue LVal,
6951 SourceLocation Loc) {
6952 if (LVal.isGlobalReg())
6953 return CGF.EmitLoadOfLValue(LVal, Loc);
6954 return CGF.EmitAtomicLoad(
6955 LVal, Loc, llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO),
6956 LVal.isVolatile());
6957}
6958
6960 QualType RValTy, SourceLocation Loc) {
6961 switch (getEvaluationKind(LVal.getType())) {
6962 case TEK_Scalar:
6964 *this, RVal, RValTy, LVal.getType(), Loc)),
6965 LVal);
6966 break;
6967 case TEK_Complex:
6969 convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,
6970 /*isInit=*/false);
6971 break;
6972 case TEK_Aggregate:
6973 llvm_unreachable("Must be a scalar or complex.");
6974 }
6975}
6976
6977static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO,
6978 const Expr *X, const Expr *V,
6979 SourceLocation Loc) {
6980 // v = x;
6981 assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
6982 assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
6983 LValue XLValue = CGF.EmitLValue(X);
6984 LValue VLValue = CGF.EmitLValue(V);
6985 RValue Res = emitSimpleAtomicLoad(CGF, AO, XLValue, Loc);
6986 // OpenMP, 2.17.7, atomic Construct
6987 // If the read or capture clause is specified and the acquire, acq_rel, or
6988 // seq_cst clause is specified then the strong flush on exit from the atomic
6989 // operation is also an acquire flush.
6990 switch (AO) {
6991 case llvm::AtomicOrdering::Acquire:
6992 case llvm::AtomicOrdering::AcquireRelease:
6993 case llvm::AtomicOrdering::SequentiallyConsistent:
6994 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, {}, Loc,
6995 llvm::AtomicOrdering::Acquire);
6996 break;
6997 case llvm::AtomicOrdering::Monotonic:
6998 case llvm::AtomicOrdering::Release:
6999 break;
7000 case llvm::AtomicOrdering::NotAtomic:
7001 case llvm::AtomicOrdering::Unordered:
7002 llvm_unreachable("Unexpected ordering.");
7003 }
7004 CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
7006}
7007
7009 llvm::AtomicOrdering AO, const Expr *X,
7010 const Expr *E, SourceLocation Loc) {
7011 // x = expr;
7012 assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
7013 emitSimpleAtomicStore(CGF, AO, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
7015 // OpenMP, 2.17.7, atomic Construct
7016 // If the write, update, or capture clause is specified and the release,
7017 // acq_rel, or seq_cst clause is specified then the strong flush on entry to
7018 // the atomic operation is also a release flush.
7019 switch (AO) {
7020 case llvm::AtomicOrdering::Release:
7021 case llvm::AtomicOrdering::AcquireRelease:
7022 case llvm::AtomicOrdering::SequentiallyConsistent:
7023 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, {}, Loc,
7024 llvm::AtomicOrdering::Release);
7025 break;
7026 case llvm::AtomicOrdering::Acquire:
7027 case llvm::AtomicOrdering::Monotonic:
7028 break;
7029 case llvm::AtomicOrdering::NotAtomic:
7030 case llvm::AtomicOrdering::Unordered:
7031 llvm_unreachable("Unexpected ordering.");
7032 }
7033}
7034
7035static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
7036 RValue Update,
7038 llvm::AtomicOrdering AO,
7039 bool IsXLHSInRHSPart) {
7040 ASTContext &Context = CGF.getContext();
7041 // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
7042 // expression is simple and atomic is allowed for the given type for the
7043 // target platform.
7044 if (BO == BO_Comma || !Update.isScalar() || !X.isSimple() ||
7045 (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
7046 (Update.getScalarVal()->getType() != X.getAddress().getElementType())) ||
7047 !Context.getTargetInfo().hasBuiltinAtomic(
7048 Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
7049 return std::make_pair(false, RValue::get(nullptr));
7050
7051 auto &&CheckAtomicSupport = [&CGF](llvm::Type *T, BinaryOperatorKind BO) {
7052 if (T->isIntegerTy())
7053 return true;
7054
7055 if (T->isFloatingPointTy() && (BO == BO_Add || BO == BO_Sub))
7056 return llvm::isPowerOf2_64(CGF.CGM.getDataLayout().getTypeStoreSize(T));
7057
7058 return false;
7059 };
7060
7061 if (!CheckAtomicSupport(Update.getScalarVal()->getType(), BO) ||
7062 !CheckAtomicSupport(X.getAddress().getElementType(), BO))
7063 return std::make_pair(false, RValue::get(nullptr));
7064
7065 bool IsInteger = X.getAddress().getElementType()->isIntegerTy();
7066 llvm::AtomicRMWInst::BinOp RMWOp;
7067 switch (BO) {
7068 case BO_Add:
7069 RMWOp = IsInteger ? llvm::AtomicRMWInst::Add : llvm::AtomicRMWInst::FAdd;
7070 break;
7071 case BO_Sub:
7072 if (!IsXLHSInRHSPart)
7073 return std::make_pair(false, RValue::get(nullptr));
7074 RMWOp = IsInteger ? llvm::AtomicRMWInst::Sub : llvm::AtomicRMWInst::FSub;
7075 break;
7076 case BO_And:
7077 RMWOp = llvm::AtomicRMWInst::And;
7078 break;
7079 case BO_Or:
7080 RMWOp = llvm::AtomicRMWInst::Or;
7081 break;
7082 case BO_Xor:
7083 RMWOp = llvm::AtomicRMWInst::Xor;
7084 break;
7085 case BO_LT:
7086 if (IsInteger)
7087 RMWOp = X.getType()->hasSignedIntegerRepresentation()
7088 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
7089 : llvm::AtomicRMWInst::Max)
7090 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
7091 : llvm::AtomicRMWInst::UMax);
7092 else
7093 RMWOp = IsXLHSInRHSPart ? llvm::AtomicRMWInst::FMin
7094 : llvm::AtomicRMWInst::FMax;
7095 break;
7096 case BO_GT:
7097 if (IsInteger)
7098 RMWOp = X.getType()->hasSignedIntegerRepresentation()
7099 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
7100 : llvm::AtomicRMWInst::Min)
7101 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
7102 : llvm::AtomicRMWInst::UMin);
7103 else
7104 RMWOp = IsXLHSInRHSPart ? llvm::AtomicRMWInst::FMax
7105 : llvm::AtomicRMWInst::FMin;
7106 break;
7107 case BO_Assign:
7108 RMWOp = llvm::AtomicRMWInst::Xchg;
7109 break;
7110 case BO_Mul:
7111 case BO_Div:
7112 case BO_Rem:
7113 case BO_Shl:
7114 case BO_Shr:
7115 case BO_LAnd:
7116 case BO_LOr:
7117 return std::make_pair(false, RValue::get(nullptr));
7118 case BO_PtrMemD:
7119 case BO_PtrMemI:
7120 case BO_LE:
7121 case BO_GE:
7122 case BO_EQ:
7123 case BO_NE:
7124 case BO_Cmp:
7125 case BO_AddAssign:
7126 case BO_SubAssign:
7127 case BO_AndAssign:
7128 case BO_OrAssign:
7129 case BO_XorAssign:
7130 case BO_MulAssign:
7131 case BO_DivAssign:
7132 case BO_RemAssign:
7133 case BO_ShlAssign:
7134 case BO_ShrAssign:
7135 case BO_Comma:
7136 llvm_unreachable("Unsupported atomic update operation");
7137 }
7138 llvm::Value *UpdateVal = Update.getScalarVal();
7139 if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
7140 if (IsInteger)
7141 UpdateVal = CGF.Builder.CreateIntCast(
7142 IC, X.getAddress().getElementType(),
7143 X.getType()->hasSignedIntegerRepresentation());
7144 else
7145 UpdateVal = CGF.Builder.CreateCast(llvm::Instruction::CastOps::UIToFP, IC,
7146 X.getAddress().getElementType());
7147 }
7148 llvm::AtomicRMWInst *Res =
7149 CGF.emitAtomicRMWInst(RMWOp, X.getAddress(), UpdateVal, AO);
7150 return std::make_pair(true, RValue::get(Res));
7151}
7152
7154 LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
7155 llvm::AtomicOrdering AO, SourceLocation Loc,
7156 const llvm::function_ref<RValue(RValue)> CommonGen) {
7157 // Update expressions are allowed to have the following forms:
7158 // x binop= expr; -> xrval + expr;
7159 // x++, ++x -> xrval + 1;
7160 // x--, --x -> xrval - 1;
7161 // x = x binop expr; -> xrval binop expr
7162 // x = expr Op x; - > expr binop xrval;
7163 auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);
7164 if (!Res.first) {
7165 if (X.isGlobalReg()) {
7166 // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
7167 // 'xrval'.
7168 EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
7169 } else {
7170 // Perform compare-and-swap procedure.
7171 EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
7172 }
7173 }
7174 return Res;
7175}
7176
7178 llvm::AtomicOrdering AO, const Expr *X,
7179 const Expr *E, const Expr *UE,
7180 bool IsXLHSInRHSPart, SourceLocation Loc) {
7181 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
7182 "Update expr in 'atomic update' must be a binary operator.");
7183 const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
7184 // Update expressions are allowed to have the following forms:
7185 // x binop= expr; -> xrval + expr;
7186 // x++, ++x -> xrval + 1;
7187 // x--, --x -> xrval - 1;
7188 // x = x binop expr; -> xrval binop expr
7189 // x = expr Op x; - > expr binop xrval;
7190 assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
7191 LValue XLValue = CGF.EmitLValue(X);
7192 RValue ExprRValue = CGF.EmitAnyExpr(E);
7193 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
7194 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
7195 const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
7196 const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
7197 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {
7198 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
7199 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
7200 return CGF.EmitAnyExpr(UE);
7201 };
7203 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
7205 // OpenMP, 2.17.7, atomic Construct
7206 // If the write, update, or capture clause is specified and the release,
7207 // acq_rel, or seq_cst clause is specified then the strong flush on entry to
7208 // the atomic operation is also a release flush.
7209 switch (AO) {
7210 case llvm::AtomicOrdering::Release:
7211 case llvm::AtomicOrdering::AcquireRelease:
7212 case llvm::AtomicOrdering::SequentiallyConsistent:
7213 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, {}, Loc,
7214 llvm::AtomicOrdering::Release);
7215 break;
7216 case llvm::AtomicOrdering::Acquire:
7217 case llvm::AtomicOrdering::Monotonic:
7218 break;
7219 case llvm::AtomicOrdering::NotAtomic:
7220 case llvm::AtomicOrdering::Unordered:
7221 llvm_unreachable("Unexpected ordering.");
7222 }
7223}
7224
7226 QualType SourceType, QualType ResType,
7227 SourceLocation Loc) {
7228 switch (CGF.getEvaluationKind(ResType)) {
7229 case TEK_Scalar:
7230 return RValue::get(
7231 convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
7232 case TEK_Complex: {
7233 auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
7234 return RValue::getComplex(Res.first, Res.second);
7235 }
7236 case TEK_Aggregate:
7237 break;
7238 }
7239 llvm_unreachable("Must be a scalar or complex.");
7240}
7241
7243 llvm::AtomicOrdering AO,
7244 bool IsPostfixUpdate, const Expr *V,
7245 const Expr *X, const Expr *E,
7246 const Expr *UE, bool IsXLHSInRHSPart,
7247 SourceLocation Loc) {
7248 assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
7249 assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
7250 RValue NewVVal;
7251 LValue VLValue = CGF.EmitLValue(V);
7252 LValue XLValue = CGF.EmitLValue(X);
7253 RValue ExprRValue = CGF.EmitAnyExpr(E);
7254 QualType NewVValType;
7255 if (UE) {
7256 // 'x' is updated with some additional value.
7257 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
7258 "Update expr in 'atomic capture' must be a binary operator.");
7259 const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
7260 // Update expressions are allowed to have the following forms:
7261 // x binop= expr; -> xrval + expr;
7262 // x++, ++x -> xrval + 1;
7263 // x--, --x -> xrval - 1;
7264 // x = x binop expr; -> xrval binop expr
7265 // x = expr Op x; - > expr binop xrval;
7266 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
7267 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
7268 const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
7269 NewVValType = XRValExpr->getType();
7270 const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
7271 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
7272 IsPostfixUpdate](RValue XRValue) {
7273 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
7274 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
7275 RValue Res = CGF.EmitAnyExpr(UE);
7276 NewVVal = IsPostfixUpdate ? XRValue : Res;
7277 return Res;
7278 };
7279 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
7280 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
7282 if (Res.first) {
7283 // 'atomicrmw' instruction was generated.
7284 if (IsPostfixUpdate) {
7285 // Use old value from 'atomicrmw'.
7286 NewVVal = Res.second;
7287 } else {
7288 // 'atomicrmw' does not provide new value, so evaluate it using old
7289 // value of 'x'.
7290 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
7291 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
7292 NewVVal = CGF.EmitAnyExpr(UE);
7293 }
7294 }
7295 } else {
7296 // 'x' is simply rewritten with some 'expr'.
7297 NewVValType = X->getType().getNonReferenceType();
7298 ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
7299 X->getType().getNonReferenceType(), Loc);
7300 auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) {
7301 NewVVal = XRValue;
7302 return ExprRValue;
7303 };
7304 // Try to perform atomicrmw xchg, otherwise simple exchange.
7305 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
7306 XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
7307 Loc, Gen);
7309 if (Res.first) {
7310 // 'atomicrmw' instruction was generated.
7311 NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
7312 }
7313 }
7314 // Emit post-update store to 'v' of old/new 'x' value.
7315 CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
7317 // OpenMP 5.1 removes the required flush for capture clause.
7318 if (CGF.CGM.getLangOpts().OpenMP < 51) {
7319 // OpenMP, 2.17.7, atomic Construct
7320 // If the write, update, or capture clause is specified and the release,
7321 // acq_rel, or seq_cst clause is specified then the strong flush on entry to
7322 // the atomic operation is also a release flush.
7323 // If the read or capture clause is specified and the acquire, acq_rel, or
7324 // seq_cst clause is specified then the strong flush on exit from the atomic
7325 // operation is also an acquire flush.
7326 switch (AO) {
7327 case llvm::AtomicOrdering::Release:
7328 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, {}, Loc,
7329 llvm::AtomicOrdering::Release);
7330 break;
7331 case llvm::AtomicOrdering::Acquire:
7332 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, {}, Loc,
7333 llvm::AtomicOrdering::Acquire);
7334 break;
7335 case llvm::AtomicOrdering::AcquireRelease:
7336 case llvm::AtomicOrdering::SequentiallyConsistent:
7338 CGF, {}, Loc, llvm::AtomicOrdering::AcquireRelease);
7339 break;
7340 case llvm::AtomicOrdering::Monotonic:
7341 break;
7342 case llvm::AtomicOrdering::NotAtomic:
7343 case llvm::AtomicOrdering::Unordered:
7344 llvm_unreachable("Unexpected ordering.");
7345 }
7346 }
7347}
7348
7350 CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO,
7351 const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *D,
7352 const Expr *CE, bool IsXBinopExpr, bool IsPostfixUpdate, bool IsFailOnly,
7353 SourceLocation Loc) {
7354 llvm::OpenMPIRBuilder &OMPBuilder =
7356
7357 OMPAtomicCompareOp Op;
7358 assert(isa<BinaryOperator>(CE) && "CE is not a BinaryOperator");
7359 switch (cast<BinaryOperator>(CE)->getOpcode()) {
7360 case BO_EQ:
7361 Op = OMPAtomicCompareOp::EQ;
7362 break;
7363 case BO_LT:
7364 Op = OMPAtomicCompareOp::MIN;
7365 break;
7366 case BO_GT:
7367 Op = OMPAtomicCompareOp::MAX;
7368 break;
7369 default:
7370 llvm_unreachable("unsupported atomic compare binary operator");
7371 }
7372
7373 LValue XLVal = CGF.EmitLValue(X);
7374 Address XAddr = XLVal.getAddress();
7375
7376 auto EmitRValueWithCastIfNeeded = [&CGF, Loc](const Expr *X, const Expr *E) {
7377 if (X->getType() == E->getType())
7378 return CGF.EmitScalarExpr(E);
7379 const Expr *NewE = E->IgnoreImplicitAsWritten();
7380 llvm::Value *V = CGF.EmitScalarExpr(NewE);
7381 if (NewE->getType() == X->getType())
7382 return V;
7383 return CGF.EmitScalarConversion(V, NewE->getType(), X->getType(), Loc);
7384 };
7385
7386 llvm::Value *EVal = EmitRValueWithCastIfNeeded(X, E);
7387 llvm::Value *DVal = D ? EmitRValueWithCastIfNeeded(X, D) : nullptr;
7388 if (auto *CI = dyn_cast<llvm::ConstantInt>(EVal))
7389 EVal = CGF.Builder.CreateIntCast(
7390 CI, XLVal.getAddress().getElementType(),
7392 if (DVal)
7393 if (auto *CI = dyn_cast<llvm::ConstantInt>(DVal))
7394 DVal = CGF.Builder.CreateIntCast(
7395 CI, XLVal.getAddress().getElementType(),
7397
7398 llvm::OpenMPIRBuilder::AtomicOpValue XOpVal{
7399 XAddr.emitRawPointer(CGF), XAddr.getElementType(),
7400 X->getType()->hasSignedIntegerRepresentation(),
7401 X->getType().isVolatileQualified()};
7402 llvm::OpenMPIRBuilder::AtomicOpValue VOpVal, ROpVal;
7403 if (V) {
7404 LValue LV = CGF.EmitLValue(V);
7405 Address Addr = LV.getAddress();
7406 VOpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(),
7407 V->getType()->hasSignedIntegerRepresentation(),
7408 V->getType().isVolatileQualified()};
7409 }
7410 if (R) {
7411 LValue LV = CGF.EmitLValue(R);
7412 Address Addr = LV.getAddress();
7413 ROpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(),
7414 R->getType()->hasSignedIntegerRepresentation(),
7415 R->getType().isVolatileQualified()};
7416 }
7417
7418 if (FailAO == llvm::AtomicOrdering::NotAtomic) {
7419 // fail clause was not mentioned on the
7420 // "#pragma omp atomic compare" construct.
7421 CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare(
7422 CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
7423 IsPostfixUpdate, IsFailOnly));
7424 } else
7425 CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare(
7426 CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
7427 IsPostfixUpdate, IsFailOnly, FailAO));
7428}
7429
7431 llvm::AtomicOrdering AO,
7432 llvm::AtomicOrdering FailAO, bool IsPostfixUpdate,
7433 const Expr *X, const Expr *V, const Expr *R,
7434 const Expr *E, const Expr *UE, const Expr *D,
7435 const Expr *CE, bool IsXLHSInRHSPart,
7436 bool IsFailOnly, SourceLocation Loc) {
7437 switch (Kind) {
7438 case OMPC_read:
7439 emitOMPAtomicReadExpr(CGF, AO, X, V, Loc);
7440 break;
7441 case OMPC_write:
7442 emitOMPAtomicWriteExpr(CGF, AO, X, E, Loc);
7443 break;
7444 case OMPC_unknown:
7445 case OMPC_update:
7446 emitOMPAtomicUpdateExpr(CGF, AO, X, E, UE, IsXLHSInRHSPart, Loc);
7447 break;
7448 case OMPC_capture:
7449 emitOMPAtomicCaptureExpr(CGF, AO, IsPostfixUpdate, V, X, E, UE,
7450 IsXLHSInRHSPart, Loc);
7451 break;
7452 case OMPC_compare: {
7453 emitOMPAtomicCompareExpr(CGF, AO, FailAO, X, V, R, E, D, CE,
7454 IsXLHSInRHSPart, IsPostfixUpdate, IsFailOnly, Loc);
7455 break;
7456 }
7457 default:
7458 llvm_unreachable("Clause is not allowed in 'omp atomic'.");
7459 }
7460}
7461
7463 llvm::AtomicOrdering AO = CGM.getOpenMPRuntime().getDefaultMemoryOrdering();
7464 // Fail Memory Clause Ordering.
7465 llvm::AtomicOrdering FailAO = llvm::AtomicOrdering::NotAtomic;
7466 bool MemOrderingSpecified = false;
7467 if (S.getSingleClause<OMPSeqCstClause>()) {
7468 AO = llvm::AtomicOrdering::SequentiallyConsistent;
7469 MemOrderingSpecified = true;
7470 } else if (S.getSingleClause<OMPAcqRelClause>()) {
7471 AO = llvm::AtomicOrdering::AcquireRelease;
7472 MemOrderingSpecified = true;
7473 } else if (S.getSingleClause<OMPAcquireClause>()) {
7474 AO = llvm::AtomicOrdering::Acquire;
7475 MemOrderingSpecified = true;
7476 } else if (S.getSingleClause<OMPReleaseClause>()) {
7477 AO = llvm::AtomicOrdering::Release;
7478 MemOrderingSpecified = true;
7479 } else if (S.getSingleClause<OMPRelaxedClause>()) {
7480 AO = llvm::AtomicOrdering::Monotonic;
7481 MemOrderingSpecified = true;
7482 }
7483 llvm::SmallSet<OpenMPClauseKind, 2> KindsEncountered;
7484 OpenMPClauseKind Kind = OMPC_unknown;
7485 for (const OMPClause *C : S.clauses()) {
7486 // Find first clause (skip seq_cst|acq_rel|aqcuire|release|relaxed clause,
7487 // if it is first).
7488 OpenMPClauseKind K = C->getClauseKind();
7489 // TBD
7490 if (K == OMPC_weak)
7491 return;
7492 if (K == OMPC_seq_cst || K == OMPC_acq_rel || K == OMPC_acquire ||
7493 K == OMPC_release || K == OMPC_relaxed || K == OMPC_hint)
7494 continue;
7495 Kind = K;
7496 KindsEncountered.insert(K);
7497 }
7498 // We just need to correct Kind here. No need to set a bool saying it is
7499 // actually compare capture because we can tell from whether V and R are
7500 // nullptr.
7501 if (KindsEncountered.contains(OMPC_compare) &&
7502 KindsEncountered.contains(OMPC_capture))
7503 Kind = OMPC_compare;
7504 if (!MemOrderingSpecified) {
7505 llvm::AtomicOrdering DefaultOrder =
7506 CGM.getOpenMPRuntime().getDefaultMemoryOrdering();
7507 if (DefaultOrder == llvm::AtomicOrdering::Monotonic ||
7508 DefaultOrder == llvm::AtomicOrdering::SequentiallyConsistent ||
7509 (DefaultOrder == llvm::AtomicOrdering::AcquireRelease &&
7510 Kind == OMPC_capture)) {
7511 AO = DefaultOrder;
7512 } else if (DefaultOrder == llvm::AtomicOrdering::AcquireRelease) {
7513 if (Kind == OMPC_unknown || Kind == OMPC_update || Kind == OMPC_write) {
7514 AO = llvm::AtomicOrdering::Release;
7515 } else if (Kind == OMPC_read) {
7516 assert(Kind == OMPC_read && "Unexpected atomic kind.");
7517 AO = llvm::AtomicOrdering::Acquire;
7518 }
7519 }
7520 }
7521
7522 if (KindsEncountered.contains(OMPC_compare) &&
7523 KindsEncountered.contains(OMPC_fail)) {
7524 Kind = OMPC_compare;
7525 const auto *FailClause = S.getSingleClause<OMPFailClause>();
7526 if (FailClause) {
7527 OpenMPClauseKind FailParameter = FailClause->getFailParameter();
7528 if (FailParameter == llvm::omp::OMPC_relaxed)
7529 FailAO = llvm::AtomicOrdering::Monotonic;
7530 else if (FailParameter == llvm::omp::OMPC_acquire)
7531 FailAO = llvm::AtomicOrdering::Acquire;
7532 else if (FailParameter == llvm::omp::OMPC_seq_cst)
7533 FailAO = llvm::AtomicOrdering::SequentiallyConsistent;
7534 }
7535 }
7536
7537 // The fp atomics hints override -f[no-]atomic-ignore-denormal-mode for this
7538 // construct. The safe hint takes precedence if both are given.
7539 AtomicOptions AtomicOpts = CGM.getAtomicOpts();
7540 if (const auto *HintClause = S.getSingleClause<OMPHintClause>()) {
7541 uint64_t Hint = HintClause->getHint()
7542 ->EvaluateKnownConstInt(getContext())
7543 .getZExtValue();
7544 if (Hint & llvm::to_underlying(OMPSyncHint::OMPXSafeFPAtomics))
7545 AtomicOpts.ignore_denormal_mode = false;
7546 else if (Hint & llvm::to_underlying(OMPSyncHint::OMPXFastFPAtomics))
7547 AtomicOpts.ignore_denormal_mode = true;
7548 }
7549 CGAtomicOptionsRAII AtomicOptsRAII(CGM, AtomicOpts);
7550
7551 LexicalScope Scope(*this, S.getSourceRange());
7552 EmitStopPoint(S.getAssociatedStmt());
7553 emitOMPAtomicExpr(*this, Kind, AO, FailAO, S.isPostfixUpdate(), S.getX(),
7554 S.getV(), S.getR(), S.getExpr(), S.getUpdateExpr(),
7555 S.getD(), S.getCondExpr(), S.isXLHSInRHSPart(),
7556 S.isFailOnly(), S.getBeginLoc());
7557}
7558
7560 const OMPExecutableDirective &S,
7561 const RegionCodeGenTy &CodeGen) {
7562 assert(isOpenMPTargetExecutionDirective(S.getDirectiveKind()));
7563 CodeGenModule &CGM = CGF.CGM;
7564
7565 // On device emit this construct as inlined code.
7566 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
7567 OMPLexicalScope Scope(CGF, S, OMPD_target);
7569 CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7570 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
7571 });
7572 return;
7573 }
7574
7576 llvm::Function *Fn = nullptr;
7577 llvm::Constant *FnID = nullptr;
7578
7579 const Expr *IfCond = nullptr;
7580 // Check for the at most one if clause associated with the target region.
7581 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
7582 if (C->getNameModifier() == OMPD_unknown ||
7583 C->getNameModifier() == OMPD_target) {
7584 IfCond = C->getCondition();
7585 break;
7586 }
7587 }
7588
7589 // Check if we have any device clause associated with the directive.
7590 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device(
7591 nullptr, OMPC_DEVICE_unknown);
7592 if (auto *C = S.getSingleClause<OMPDeviceClause>())
7593 Device.setPointerAndInt(C->getDevice(), C->getModifier());
7594
7595 // Check if we have an if clause whose conditional always evaluates to false
7596 // or if we do not have any targets specified. If so the target region is not
7597 // an offload entry point.
7598 bool IsOffloadEntry = true;
7599 if (IfCond) {
7600 bool Val;
7601 if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
7602 IsOffloadEntry = false;
7603 }
7604 if (CGM.getLangOpts().OMPTargetTriples.empty())
7605 IsOffloadEntry = false;
7606
7607 if (CGM.getLangOpts().OpenMPOffloadMandatory && !IsOffloadEntry) {
7608 CGM.getDiags().Report(diag::err_missing_mandatory_offloading);
7609 }
7610
7611 // Lambdas and blocks at namespace scope have no parent function. Regions in
7612 // constructs outlined from such a lambda are named after its call operator,
7613 // as the device names them.
7614 const Decl *CodeDecl = CGF.CurFuncDecl ? CGF.CurFuncDecl : CGF.CurCodeDecl;
7615 while (const auto *CD = dyn_cast_or_null<CapturedDecl>(CodeDecl))
7616 CodeDecl = Decl::castFromDeclContext(CD->getParent());
7617 const auto *ParentFD = dyn_cast_or_null<FunctionDecl>(CodeDecl);
7618 StringRef ParentName =
7619 ParentFD ? CGM.getOpenMPRuntime().getTargetRegionParentName(ParentFD)
7620 : CGF.CurFn->getName();
7621
7622 // Emit target region as a standalone region.
7623 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
7624 IsOffloadEntry, CodeGen);
7625 OMPLexicalScope Scope(CGF, S, OMPD_task);
7626 auto &&SizeEmitter =
7627 [IsOffloadEntry](CodeGenFunction &CGF,
7628 const OMPLoopDirective &D) -> llvm::Value * {
7629 if (IsOffloadEntry) {
7630 OMPLoopScope PreInitScope(CGF, D);
7631 // Emit calculation of the iterations count.
7632 llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations());
7633 NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty,
7634 /*isSigned=*/false);
7635 return NumIterations;
7636 }
7637 return nullptr;
7638 };
7639 CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device,
7640 SizeEmitter);
7641}
7642
7644 PrePostActionTy &Action) {
7645 Action.Enter(CGF);
7646 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7647 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
7648 CGF.EmitOMPPrivateClause(S, PrivateScope);
7649 (void)PrivateScope.Privatize();
7650 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
7652
7653 CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
7654 CGF.EnsureInsertPoint();
7655}
7656
7658 StringRef ParentName,
7659 const OMPTargetDirective &S) {
7660 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7661 emitTargetRegion(CGF, S, Action);
7662 };
7663 llvm::Function *Fn;
7664 llvm::Constant *Addr;
7665 // Emit target region as a standalone region.
7666 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7667 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
7668 assert(Fn && Addr && "Target device function emission failed.");
7669}
7670
7672 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7673 emitTargetRegion(CGF, S, Action);
7674 };
7676}
7677
7679 const OMPExecutableDirective &S,
7680 OpenMPDirectiveKind InnermostKind,
7681 const RegionCodeGenTy &CodeGen) {
7682 const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
7683 llvm::Function *OutlinedFn =
7685 CGF, S, *CS->getCapturedDecl()->param_begin(), InnermostKind,
7686 CodeGen);
7687
7688 OMPTeamsScope Scope(CGF, S);
7689 auto ParallelLeague = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7690 const auto *NT = S.getSingleClause<OMPNumTeamsClause>();
7691 const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
7692 if (NT || TL) {
7693 const Expr *NumTeams = NT ? NT->getNumTeams().front() : nullptr;
7694 const Expr *ThreadLimit = TL ? TL->getThreadLimit().front() : nullptr;
7695
7696 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
7697 S.getBeginLoc());
7698 }
7699 };
7700
7701 const Expr *IfCond = nullptr;
7702 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
7703 if (C->getNameModifier() == OMPD_unknown ||
7704 C->getNameModifier() == OMPD_teams) {
7705 IfCond = C->getCondition();
7706 break;
7707 }
7708 }
7709 if (IfCond && CGF.CGM.getLangOpts().OpenMP >= 52) {
7710 auto SerialLeague = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7711 // OpenMP 5.2, 10.2, teams Construct
7712 // When an if clause is present on a teams construct and the if clause
7713 // expression evaluates to false, the number of created teams is one.
7714 const llvm::APInt One(32, 1);
7715 IntegerLiteral NumTeams(
7716 CGF.getContext(), One,
7717 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
7718 SourceLocation());
7719 // The thread_limit clause is unaffected by the if clause.
7720 const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
7721 const Expr *ThreadLimit = TL ? TL->getThreadLimit().front() : nullptr;
7722 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, &NumTeams, ThreadLimit,
7723 S.getBeginLoc());
7724 };
7725 CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, IfCond, ParallelLeague,
7726 SerialLeague);
7727 } else {
7728 const RegionCodeGenTy ThenRCG(ParallelLeague);
7729 ThenRCG(CGF);
7730 }
7731
7733 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
7734 CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn,
7735 CapturedVars);
7736}
7737
7739 // Emit teams region as a standalone region.
7740 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7741 Action.Enter(CGF);
7742 OMPPrivateScope PrivateScope(CGF);
7743 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
7744 CGF.EmitOMPPrivateClause(S, PrivateScope);
7745 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7746 (void)PrivateScope.Privatize();
7747 CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
7748 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7749 };
7750 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
7752 [](CodeGenFunction &) { return nullptr; });
7753}
7754
7756 const OMPTargetTeamsDirective &S) {
7757 auto *CS = S.getCapturedStmt(OMPD_teams);
7758 Action.Enter(CGF);
7759 // Emit teams region as a standalone region.
7760 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
7761 Action.Enter(CGF);
7762 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7763 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
7764 CGF.EmitOMPPrivateClause(S, PrivateScope);
7765 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7766 (void)PrivateScope.Privatize();
7767 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
7769 CGF.EmitStmt(CS->getCapturedStmt());
7770 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7771 };
7772 emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
7774 [](CodeGenFunction &) { return nullptr; });
7775}
7776
7778 CodeGenModule &CGM, StringRef ParentName,
7779 const OMPTargetTeamsDirective &S) {
7780 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7781 emitTargetTeamsRegion(CGF, Action, S);
7782 };
7783 llvm::Function *Fn;
7784 llvm::Constant *Addr;
7785 // Emit target region as a standalone region.
7786 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7787 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
7788 assert(Fn && Addr && "Target device function emission failed.");
7789}
7790
7792 const OMPTargetTeamsDirective &S) {
7793 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7794 emitTargetTeamsRegion(CGF, Action, S);
7795 };
7797}
7798
7799static void
7802 Action.Enter(CGF);
7803 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7805 };
7806
7807 // Emit teams region as a standalone region.
7808 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7809 PrePostActionTy &Action) {
7810 Action.Enter(CGF);
7811 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7812 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7813 (void)PrivateScope.Privatize();
7814 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
7815 CodeGenDistribute);
7816 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7817 };
7818 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
7820 [](CodeGenFunction &) { return nullptr; });
7821}
7822
7824 CodeGenModule &CGM, StringRef ParentName,
7826 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7827 emitTargetTeamsDistributeRegion(CGF, Action, S);
7828 };
7829 llvm::Function *Fn;
7830 llvm::Constant *Addr;
7831 // Emit target region as a standalone region.
7832 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7833 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
7834 assert(Fn && Addr && "Target device function emission failed.");
7835}
7836
7839 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7840 emitTargetTeamsDistributeRegion(CGF, Action, S);
7841 };
7843}
7844
7846 CodeGenFunction &CGF, PrePostActionTy &Action,
7848 Action.Enter(CGF);
7849 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7851 };
7852
7853 // Emit teams region as a standalone region.
7854 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7855 PrePostActionTy &Action) {
7856 Action.Enter(CGF);
7857 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7858 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7859 (void)PrivateScope.Privatize();
7860 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
7861 CodeGenDistribute);
7862 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7863 };
7864 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);
7866 [](CodeGenFunction &) { return nullptr; });
7867}
7868
7870 CodeGenModule &CGM, StringRef ParentName,
7872 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7874 };
7875 llvm::Function *Fn;
7876 llvm::Constant *Addr;
7877 // Emit target region as a standalone region.
7878 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7879 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
7880 assert(Fn && Addr && "Target device function emission failed.");
7881}
7882
7885 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7887 };
7889}
7890
7892 const OMPTeamsDistributeDirective &S) {
7893
7894 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7896 };
7897
7898 // Emit teams region as a standalone region.
7899 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7900 PrePostActionTy &Action) {
7901 Action.Enter(CGF);
7902 OMPPrivateScope PrivateScope(CGF);
7903 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7904 (void)PrivateScope.Privatize();
7905 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
7906 CodeGenDistribute);
7907 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7908 };
7909 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
7911 [](CodeGenFunction &) { return nullptr; });
7912}
7913
7916 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7918 };
7919
7920 // Emit teams region as a standalone region.
7921 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7922 PrePostActionTy &Action) {
7923 Action.Enter(CGF);
7924 OMPPrivateScope PrivateScope(CGF);
7925 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7926 (void)PrivateScope.Privatize();
7927 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
7928 CodeGenDistribute);
7929 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7930 };
7931 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen);
7933 [](CodeGenFunction &) { return nullptr; });
7934}
7935
7938 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7940 S.getDistInc());
7941 };
7942
7943 // Emit teams region as a standalone region.
7944 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7945 PrePostActionTy &Action) {
7946 Action.Enter(CGF);
7947 OMPPrivateScope PrivateScope(CGF);
7948 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7949 (void)PrivateScope.Privatize();
7950 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
7951 CodeGenDistribute);
7952 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7953 };
7954 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
7956 [](CodeGenFunction &) { return nullptr; });
7957}
7958
7961 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7963 S.getDistInc());
7964 };
7965
7966 // Emit teams region as a standalone region.
7967 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7968 PrePostActionTy &Action) {
7969 Action.Enter(CGF);
7970 OMPPrivateScope PrivateScope(CGF);
7971 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7972 (void)PrivateScope.Privatize();
7974 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
7975 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7976 };
7977 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for_simd,
7978 CodeGen);
7980 [](CodeGenFunction &) { return nullptr; });
7981}
7982
7984 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
7985 llvm::Value *Device = nullptr;
7986 llvm::Value *NumDependences = nullptr;
7987 llvm::Value *DependenceList = nullptr;
7988
7989 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
7990 Device = EmitScalarExpr(C->getDevice());
7991
7992 // Build list and emit dependences
7995 if (!Data.Dependences.empty()) {
7996 Address DependenciesArray = Address::invalid();
7997 std::tie(NumDependences, DependenciesArray) =
7998 CGM.getOpenMPRuntime().emitDependClause(*this, Data.Dependences,
7999 S.getBeginLoc());
8000 DependenceList = DependenciesArray.emitRawPointer(*this);
8001 }
8002 Data.HasNowaitClause = S.hasClausesOfKind<OMPNowaitClause>();
8003
8004 assert(!(Data.HasNowaitClause && !(S.getSingleClause<OMPInitClause>() ||
8005 S.getSingleClause<OMPDestroyClause>() ||
8006 S.getSingleClause<OMPUseClause>())) &&
8007 "OMPNowaitClause clause is used separately in OMPInteropDirective.");
8008
8009 auto ItOMPInitClause = S.getClausesOfKind<OMPInitClause>();
8010 if (!ItOMPInitClause.empty()) {
8011 // Look at the multiple init clauses
8012 for (const OMPInitClause *C : ItOMPInitClause) {
8013 llvm::Value *InteropvarPtr =
8014 EmitLValue(C->getInteropVar()).getPointer(*this);
8015 llvm::omp::OMPInteropType InteropType =
8016 llvm::omp::OMPInteropType::Unknown;
8017 if (C->getIsTarget()) {
8018 InteropType = llvm::omp::OMPInteropType::Target;
8019 } else {
8020 assert(C->getIsTargetSync() &&
8021 "Expected interop-type target/targetsync");
8022 InteropType = llvm::omp::OMPInteropType::TargetSync;
8023 }
8024 OMPBuilder.createOMPInteropInit(Builder, InteropvarPtr, InteropType,
8025 Device, NumDependences, DependenceList,
8026 Data.HasNowaitClause);
8027 }
8028 }
8029 auto ItOMPDestroyClause = S.getClausesOfKind<OMPDestroyClause>();
8030 if (!ItOMPDestroyClause.empty()) {
8031 // Look at the multiple destroy clauses
8032 for (const OMPDestroyClause *C : ItOMPDestroyClause) {
8033 llvm::Value *InteropvarPtr =
8034 EmitLValue(C->getInteropVar()).getPointer(*this);
8035 OMPBuilder.createOMPInteropDestroy(Builder, InteropvarPtr, Device,
8036 NumDependences, DependenceList,
8037 Data.HasNowaitClause);
8038 }
8039 }
8040 auto ItOMPUseClause = S.getClausesOfKind<OMPUseClause>();
8041 if (!ItOMPUseClause.empty()) {
8042 // Look at the multiple use clauses
8043 for (const OMPUseClause *C : ItOMPUseClause) {
8044 llvm::Value *InteropvarPtr =
8045 EmitLValue(C->getInteropVar()).getPointer(*this);
8046 OMPBuilder.createOMPInteropUse(Builder, InteropvarPtr, Device,
8047 NumDependences, DependenceList,
8048 Data.HasNowaitClause);
8049 }
8050 }
8051}
8052
8055 PrePostActionTy &Action) {
8056 Action.Enter(CGF);
8057 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8059 S.getDistInc());
8060 };
8061
8062 // Emit teams region as a standalone region.
8063 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
8064 PrePostActionTy &Action) {
8065 Action.Enter(CGF);
8066 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
8067 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
8068 (void)PrivateScope.Privatize();
8070 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
8071 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
8072 };
8073
8074 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
8075 CodeGenTeams);
8077 [](CodeGenFunction &) { return nullptr; });
8078}
8079
8081 CodeGenModule &CGM, StringRef ParentName,
8083 // Emit SPMD target teams distribute parallel for region as a standalone
8084 // region.
8085 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8087 };
8088 llvm::Function *Fn;
8089 llvm::Constant *Addr;
8090 // Emit target region as a standalone region.
8091 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8092 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
8093 assert(Fn && Addr && "Target device function emission failed.");
8094}
8095
8103
8105 CodeGenFunction &CGF,
8107 PrePostActionTy &Action) {
8108 Action.Enter(CGF);
8109 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8111 S.getDistInc());
8112 };
8113
8114 // Emit teams region as a standalone region.
8115 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
8116 PrePostActionTy &Action) {
8117 Action.Enter(CGF);
8118 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
8119 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
8120 (void)PrivateScope.Privatize();
8122 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
8123 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
8124 };
8125
8126 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd,
8127 CodeGenTeams);
8129 [](CodeGenFunction &) { return nullptr; });
8130}
8131
8133 CodeGenModule &CGM, StringRef ParentName,
8135 // Emit SPMD target teams distribute parallel for simd region as a standalone
8136 // region.
8137 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8139 };
8140 llvm::Function *Fn;
8141 llvm::Constant *Addr;
8142 // Emit target region as a standalone region.
8143 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8144 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
8145 assert(Fn && Addr && "Target device function emission failed.");
8146}
8147
8155
8158 CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getBeginLoc(),
8159 S.getCancelRegion());
8160}
8161
8163 const Expr *IfCond = nullptr;
8164 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
8165 if (C->getNameModifier() == OMPD_unknown ||
8166 C->getNameModifier() == OMPD_cancel) {
8167 IfCond = C->getCondition();
8168 break;
8169 }
8170 }
8171 if (CGM.getLangOpts().OpenMPIRBuilder) {
8172 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
8173 // TODO: This check is necessary as we only generate `omp parallel` through
8174 // the OpenMPIRBuilder for now.
8175 if (S.getCancelRegion() == OMPD_parallel ||
8176 S.getCancelRegion() == OMPD_sections ||
8177 S.getCancelRegion() == OMPD_section) {
8178 llvm::Value *IfCondition = nullptr;
8179 if (IfCond)
8180 IfCondition = EmitScalarExpr(IfCond,
8181 /*IgnoreResultAssign=*/true);
8182 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
8183 OMPBuilder.createCancel(Builder, IfCondition, S.getCancelRegion()));
8184 return Builder.restoreIP(AfterIP);
8185 }
8186 }
8187
8188 CGM.getOpenMPRuntime().emitCancelCall(*this, S.getBeginLoc(), IfCond,
8189 S.getCancelRegion());
8190}
8191
8194 if (Kind == OMPD_parallel || Kind == OMPD_target_parallel ||
8196 return ReturnBlock;
8197 assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
8198 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
8199 Kind == OMPD_distribute_parallel_for ||
8200 Kind == OMPD_target_parallel_for ||
8201 Kind == OMPD_teams_distribute_parallel_for ||
8202 Kind == OMPD_target_teams_distribute_parallel_for);
8203 return OMPCancelStack.getExitBlock();
8204}
8205
8207 const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope,
8208 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
8209 CaptureDeviceAddrMap) {
8210 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
8211 for (const Expr *OrigVarIt : C.varlist()) {
8212 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(OrigVarIt)->getDecl());
8213 if (!Processed.insert(OrigVD).second)
8214 continue;
8215
8216 // In order to identify the right initializer we need to match the
8217 // declaration used by the mapping logic. In some cases we may get
8218 // OMPCapturedExprDecl that refers to the original declaration.
8219 const ValueDecl *MatchingVD = OrigVD;
8220 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
8221 // OMPCapturedExprDecl are used to privative fields of the current
8222 // structure.
8223 const auto *ME = cast<MemberExpr>(OED->getInit());
8224 assert(isa<CXXThisExpr>(ME->getBase()->IgnoreImpCasts()) &&
8225 "Base should be the current struct!");
8226 MatchingVD = ME->getMemberDecl();
8227 }
8228
8229 // If we don't have information about the current list item, move on to
8230 // the next one.
8231 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
8232 if (InitAddrIt == CaptureDeviceAddrMap.end())
8233 continue;
8234
8235 llvm::Type *Ty = ConvertTypeForMem(OrigVD->getType().getNonReferenceType());
8236
8237 // Return the address of the private variable.
8238 bool IsRegistered = PrivateScope.addPrivate(
8239 OrigVD,
8240 Address(InitAddrIt->second, Ty,
8241 getContext().getTypeAlignInChars(getContext().VoidPtrTy)));
8242 assert(IsRegistered && "firstprivate var already registered as private");
8243 // Silence the warning about unused variable.
8244 (void)IsRegistered;
8245 }
8246}
8247
8248static const VarDecl *getBaseDecl(const Expr *Ref) {
8249 const Expr *Base = Ref->IgnoreParenImpCasts();
8250 while (const auto *OASE = dyn_cast<ArraySectionExpr>(Base))
8251 Base = OASE->getBase()->IgnoreParenImpCasts();
8252 while (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Base))
8253 Base = ASE->getBase()->IgnoreParenImpCasts();
8254 return cast<VarDecl>(cast<DeclRefExpr>(Base)->getDecl());
8255}
8256
8258 const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope,
8259 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
8260 CaptureDeviceAddrMap) {
8261 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
8262 for (const Expr *Ref : C.varlist()) {
8263 const VarDecl *OrigVD = getBaseDecl(Ref);
8264 if (!Processed.insert(OrigVD).second)
8265 continue;
8266 // In order to identify the right initializer we need to match the
8267 // declaration used by the mapping logic. In some cases we may get
8268 // OMPCapturedExprDecl that refers to the original declaration.
8269 const ValueDecl *MatchingVD = OrigVD;
8270 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
8271 // OMPCapturedExprDecl are used to privative fields of the current
8272 // structure.
8273 const auto *ME = cast<MemberExpr>(OED->getInit());
8274 assert(isa<CXXThisExpr>(ME->getBase()) &&
8275 "Base should be the current struct!");
8276 MatchingVD = ME->getMemberDecl();
8277 }
8278
8279 // If we don't have information about the current list item, move on to
8280 // the next one.
8281 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
8282 if (InitAddrIt == CaptureDeviceAddrMap.end())
8283 continue;
8284
8285 llvm::Type *Ty = ConvertTypeForMem(OrigVD->getType().getNonReferenceType());
8286
8287 Address PrivAddr =
8288 Address(InitAddrIt->second, Ty,
8289 getContext().getTypeAlignInChars(getContext().VoidPtrTy));
8290 // For declrefs and variable length array need to load the pointer for
8291 // correct mapping, since the pointer to the data was passed to the runtime.
8292 if (isa<DeclRefExpr>(Ref->IgnoreParenImpCasts()) ||
8293 MatchingVD->getType()->isArrayType()) {
8295 OrigVD->getType().getNonReferenceType());
8296 PrivAddr =
8298 PtrTy->castAs<PointerType>());
8299 }
8300
8301 (void)PrivateScope.addPrivate(OrigVD, PrivAddr);
8302 }
8303}
8304
8305// Generate the instructions for '#pragma omp target data' directive.
8307 const OMPTargetDataDirective &S) {
8308 // Emit vtable only from host for target data directive.
8309 if (!CGM.getLangOpts().OpenMPIsTargetDevice)
8310 CGM.getOpenMPRuntime().registerVTable(S);
8311
8312 CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true,
8313 /*SeparateBeginEndCalls=*/true);
8314
8315 // Create a pre/post action to signal the privatization of the device pointer.
8316 // This action can be replaced by the OpenMP runtime code generation to
8317 // deactivate privatization.
8318 bool PrivatizeDevicePointers = false;
8319 class DevicePointerPrivActionTy : public PrePostActionTy {
8320 bool &PrivatizeDevicePointers;
8321
8322 public:
8323 explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
8324 : PrivatizeDevicePointers(PrivatizeDevicePointers) {}
8325 void Enter(CodeGenFunction &CGF) override {
8326 PrivatizeDevicePointers = true;
8327 }
8328 };
8329 DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
8330
8331 auto &&CodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
8332 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8333 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
8334 };
8335
8336 // Codegen that selects whether to generate the privatization code or not.
8337 auto &&PrivCodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
8338 RegionCodeGenTy RCG(InnermostCodeGen);
8339 PrivatizeDevicePointers = false;
8340
8341 // Call the pre-action to change the status of PrivatizeDevicePointers if
8342 // needed.
8343 Action.Enter(CGF);
8344
8345 if (PrivatizeDevicePointers) {
8346 OMPPrivateScope PrivateScope(CGF);
8347 // Emit all instances of the use_device_ptr clause.
8348 for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
8349 CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
8351 for (const auto *C : S.getClausesOfKind<OMPUseDeviceAddrClause>())
8352 CGF.EmitOMPUseDeviceAddrClause(*C, PrivateScope,
8354 (void)PrivateScope.Privatize();
8355 RCG(CGF);
8356 } else {
8357 // If we don't have target devices, don't bother emitting the data
8358 // mapping code.
8359 std::optional<OpenMPDirectiveKind> CaptureRegion;
8360 if (CGM.getLangOpts().OMPTargetTriples.empty()) {
8361 // Emit helper decls of the use_device_ptr/use_device_addr clauses.
8362 for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
8363 for (const Expr *E : C->varlist()) {
8364 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
8365 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
8366 CGF.EmitVarDecl(*OED);
8367 }
8368 for (const auto *C : S.getClausesOfKind<OMPUseDeviceAddrClause>())
8369 for (const Expr *E : C->varlist()) {
8370 const Decl *D = getBaseDecl(E);
8371 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
8372 CGF.EmitVarDecl(*OED);
8373 }
8374 } else {
8375 CaptureRegion = OMPD_unknown;
8376 }
8377
8378 OMPLexicalScope Scope(CGF, S, CaptureRegion);
8379 RCG(CGF);
8380 }
8381 };
8382
8383 // Forward the provided action to the privatization codegen.
8384 RegionCodeGenTy PrivRCG(PrivCodeGen);
8385 PrivRCG.setAction(Action);
8386
8387 // Notwithstanding the body of the region is emitted as inlined directive,
8388 // we don't use an inline scope as changes in the references inside the
8389 // region are expected to be visible outside, so we do not privative them.
8390 OMPLexicalScope Scope(CGF, S);
8391 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
8392 PrivRCG);
8393 };
8394
8396
8397 // If we don't have target devices, don't bother emitting the data mapping
8398 // code.
8399 if (CGM.getLangOpts().OMPTargetTriples.empty()) {
8400 RCG(*this);
8401 return;
8402 }
8403
8404 // Check if we have any if clause associated with the directive.
8405 const Expr *IfCond = nullptr;
8406 if (const auto *C = S.getSingleClause<OMPIfClause>())
8407 IfCond = C->getCondition();
8408
8409 // Check if we have any device clause associated with the directive.
8410 const Expr *Device = nullptr;
8411 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
8412 Device = C->getDevice();
8413
8414 // Set the action to signal privatization of device pointers.
8415 RCG.setAction(PrivAction);
8416
8417 // Emit region code.
8418 CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
8419 Info);
8420}
8421
8423 const OMPTargetEnterDataDirective &S) {
8424 // If we don't have target devices, don't bother emitting the data mapping
8425 // code.
8426 if (CGM.getLangOpts().OMPTargetTriples.empty())
8427 return;
8428
8429 // Check if we have any if clause associated with the directive.
8430 const Expr *IfCond = nullptr;
8431 if (const auto *C = S.getSingleClause<OMPIfClause>())
8432 IfCond = C->getCondition();
8433
8434 // Check if we have any device clause associated with the directive.
8435 const Expr *Device = nullptr;
8436 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
8437 Device = C->getDevice();
8438
8439 OMPLexicalScope Scope(*this, S, OMPD_task);
8440 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
8441}
8442
8444 const OMPTargetExitDataDirective &S) {
8445 // If we don't have target devices, don't bother emitting the data mapping
8446 // code.
8447 if (CGM.getLangOpts().OMPTargetTriples.empty())
8448 return;
8449
8450 // Check if we have any if clause associated with the directive.
8451 const Expr *IfCond = nullptr;
8452 if (const auto *C = S.getSingleClause<OMPIfClause>())
8453 IfCond = C->getCondition();
8454
8455 // Check if we have any device clause associated with the directive.
8456 const Expr *Device = nullptr;
8457 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
8458 Device = C->getDevice();
8459
8460 OMPLexicalScope Scope(*this, S, OMPD_task);
8461 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
8462}
8463
8466 PrePostActionTy &Action) {
8467 // Get the captured statement associated with the 'parallel' region.
8468 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
8469 Action.Enter(CGF);
8470 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
8471 Action.Enter(CGF);
8472 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
8473 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
8474 CGF.EmitOMPPrivateClause(S, PrivateScope);
8475 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
8476 (void)PrivateScope.Privatize();
8477 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
8479 // TODO: Add support for clauses.
8480 CGF.EmitStmt(CS->getCapturedStmt());
8481 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
8482 };
8483 emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
8486 [](CodeGenFunction &) { return nullptr; });
8487}
8488
8490 CodeGenModule &CGM, StringRef ParentName,
8491 const OMPTargetParallelDirective &S) {
8492 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8493 emitTargetParallelRegion(CGF, S, Action);
8494 };
8495 llvm::Function *Fn;
8496 llvm::Constant *Addr;
8497 // Emit target region as a standalone region.
8498 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8499 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
8500 assert(Fn && Addr && "Target device function emission failed.");
8501}
8502
8504 const OMPTargetParallelDirective &S) {
8505 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8506 emitTargetParallelRegion(CGF, S, Action);
8507 };
8509}
8510
8513 PrePostActionTy &Action) {
8514 Action.Enter(CGF);
8515 // Emit directive as a combined directive that consists of two implicit
8516 // directives: 'parallel' with 'for' directive.
8517 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8518 Action.Enter(CGF);
8520 CGF, OMPD_target_parallel_for, S.hasCancel());
8523 };
8524 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
8526}
8527
8529 CodeGenModule &CGM, StringRef ParentName,
8531 // Emit SPMD target parallel for region as a standalone region.
8532 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8533 emitTargetParallelForRegion(CGF, S, Action);
8534 };
8535 llvm::Function *Fn;
8536 llvm::Constant *Addr;
8537 // Emit target region as a standalone region.
8538 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8539 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
8540 assert(Fn && Addr && "Target device function emission failed.");
8541}
8542
8545 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8546 emitTargetParallelForRegion(CGF, S, Action);
8547 };
8549}
8550
8551static void
8554 PrePostActionTy &Action) {
8555 Action.Enter(CGF);
8556 // Emit directive as a combined directive that consists of two implicit
8557 // directives: 'parallel' with 'for' directive.
8558 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8559 Action.Enter(CGF);
8562 };
8563 emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
8565}
8566
8568 CodeGenModule &CGM, StringRef ParentName,
8570 // Emit SPMD target parallel for region as a standalone region.
8571 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8572 emitTargetParallelForSimdRegion(CGF, S, Action);
8573 };
8574 llvm::Function *Fn;
8575 llvm::Constant *Addr;
8576 // Emit target region as a standalone region.
8577 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8578 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
8579 assert(Fn && Addr && "Target device function emission failed.");
8580}
8581
8584 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8585 emitTargetParallelForSimdRegion(CGF, S, Action);
8586 };
8588}
8589
8590/// Emit a helper variable and return corresponding lvalue.
8591static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
8592 const ImplicitParamDecl *PVD,
8594 const auto *VDecl = cast<VarDecl>(Helper->getDecl());
8595 Privates.addPrivate(VDecl, CGF.GetAddrOfLocalVar(PVD));
8596}
8597
8599 assert(isOpenMPTaskLoopDirective(S.getDirectiveKind()));
8600 // Emit outlined function for task construct.
8601 const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);
8602 Address CapturedStruct = Address::invalid();
8603 {
8604 OMPLexicalScope Scope(*this, S, OMPD_taskloop, /*EmitPreInitStmt=*/false);
8605 CapturedStruct = GenerateCapturedStmtArgument(*CS);
8606 }
8607 CanQualType SharedsTy =
8609 const Expr *IfCond = nullptr;
8610 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
8611 if (C->getNameModifier() == OMPD_unknown ||
8612 C->getNameModifier() == OMPD_taskloop) {
8613 IfCond = C->getCondition();
8614 break;
8615 }
8616 }
8617
8619 // Check if taskloop must be emitted without taskgroup.
8620 Data.Nogroup = S.getSingleClause<OMPNogroupClause>();
8621 // TODO: Check if we should emit tied or untied task.
8622 Data.Tied = true;
8623 // Set scheduling for taskloop
8624 if (const auto *Clause = S.getSingleClause<OMPGrainsizeClause>()) {
8625 // grainsize clause
8626 Data.Schedule.setInt(/*IntVal=*/false);
8627 Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
8628 Data.HasModifier =
8629 (Clause->getModifier() == OMPC_GRAINSIZE_strict) ? true : false;
8630 } else if (const auto *Clause = S.getSingleClause<OMPNumTasksClause>()) {
8631 // num_tasks clause
8632 Data.Schedule.setInt(/*IntVal=*/true);
8633 Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
8634 Data.HasModifier =
8635 (Clause->getModifier() == OMPC_NUMTASKS_strict) ? true : false;
8636 }
8637
8638 auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
8639 // if (PreCond) {
8640 // for (IV in 0..LastIteration) BODY;
8641 // <Final counter/linear vars updates>;
8642 // }
8643 //
8644
8645 // Emit: if (PreCond) - begin.
8646 // If the condition constant folds and can be elided, avoid emitting the
8647 // whole loop.
8648 bool CondConstant;
8649 llvm::BasicBlock *ContBlock = nullptr;
8650 OMPLoopScope PreInitScope(CGF, S);
8651 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
8652 if (!CondConstant)
8653 return;
8654 } else {
8655 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
8656 ContBlock = CGF.createBasicBlock("taskloop.if.end");
8657 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
8658 CGF.getProfileCount(&S));
8659 CGF.EmitBlock(ThenBlock);
8660 CGF.incrementProfileCounter(&S);
8661 }
8662
8663 (void)CGF.EmitOMPLinearClauseInit(S);
8664
8665 OMPPrivateScope LoopScope(CGF);
8666 // Emit helper vars inits.
8667 enum { LowerBound = 5, UpperBound, Stride, LastIter };
8668 auto *I = CS->getCapturedDecl()->param_begin();
8669 auto *LBP = std::next(I, LowerBound);
8670 auto *UBP = std::next(I, UpperBound);
8671 auto *STP = std::next(I, Stride);
8672 auto *LIP = std::next(I, LastIter);
8674 LoopScope);
8676 LoopScope);
8677 mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
8679 LoopScope);
8680 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
8681 CGF.EmitOMPLinearClause(S, LoopScope);
8682 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
8683 (void)LoopScope.Privatize();
8684 // Emit the loop iteration variable.
8685 const Expr *IVExpr = S.getIterationVariable();
8686 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
8687 CGF.EmitVarDecl(*IVDecl);
8688 CGF.EmitIgnoredExpr(S.getInit());
8689
8690 // Emit the iterations count variable.
8691 // If it is not a variable, Sema decided to calculate iterations count on
8692 // each iteration (e.g., it is foldable into a constant).
8693 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
8694 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
8695 // Emit calculation of the iterations count.
8696 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
8697 }
8698
8699 {
8700 OMPLexicalScope Scope(CGF, S, OMPD_taskloop, /*EmitPreInitStmt=*/false);
8702 CGF, S,
8703 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8704 if (isOpenMPSimdDirective(S.getDirectiveKind()))
8705 CGF.EmitOMPSimdInit(S);
8706 },
8707 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
8708 CGF.EmitOMPInnerLoop(
8709 S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),
8710 [&S](CodeGenFunction &CGF) {
8711 emitOMPLoopBodyWithStopPoint(CGF, S,
8712 CodeGenFunction::JumpDest());
8713 },
8714 [](CodeGenFunction &) {});
8715 });
8716 }
8717 // Emit: if (PreCond) - end.
8718 if (ContBlock) {
8719 CGF.EmitBranch(ContBlock);
8720 CGF.EmitBlock(ContBlock, true);
8721 }
8722 // Emit final copy of the lastprivate variables if IsLastIter != 0.
8723 if (HasLastprivateClause) {
8724 CGF.EmitOMPLastprivateClauseFinal(
8725 S, isOpenMPSimdDirective(S.getDirectiveKind()),
8726 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
8727 CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
8728 (*LIP)->getType(), S.getBeginLoc())));
8729 }
8730 LoopScope.restoreMap();
8731 CGF.EmitOMPLinearClauseFinal(S, [LIP, &S](CodeGenFunction &CGF) {
8732 return CGF.Builder.CreateIsNotNull(
8733 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
8734 (*LIP)->getType(), S.getBeginLoc()));
8735 });
8736 };
8737 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
8738 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
8739 const OMPTaskDataTy &Data) {
8740 auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
8741 &Data](CodeGenFunction &CGF, PrePostActionTy &) {
8742 OMPLoopScope PreInitScope(CGF, S);
8743 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
8744 OutlinedFn, SharedsTy,
8745 CapturedStruct, IfCond, Data);
8746 };
8747 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
8748 CodeGen);
8749 };
8750 if (Data.Nogroup) {
8751 EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data);
8752 } else {
8753 CGM.getOpenMPRuntime().emitTaskgroupRegion(
8754 *this,
8755 [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
8756 PrePostActionTy &Action) {
8757 Action.Enter(CGF);
8758 CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen,
8759 Data);
8760 },
8761 S.getBeginLoc());
8762 }
8763}
8764
8770
8772 const OMPTaskLoopSimdDirective &S) {
8773 auto LPCRegion =
8775 OMPLexicalScope Scope(*this, S);
8777}
8778
8780 const OMPMasterTaskLoopDirective &S) {
8781 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8782 Action.Enter(CGF);
8784 };
8785 auto LPCRegion =
8787 OMPLexicalScope Scope(*this, S, std::nullopt, /*EmitPreInitStmt=*/false);
8788 CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
8789}
8790
8792 const OMPMaskedTaskLoopDirective &S) {
8793 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8794 Action.Enter(CGF);
8796 };
8797 auto LPCRegion =
8799 OMPLexicalScope Scope(*this, S, std::nullopt, /*EmitPreInitStmt=*/false);
8800 CGM.getOpenMPRuntime().emitMaskedRegion(*this, CodeGen, S.getBeginLoc());
8801}
8802
8805 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8806 Action.Enter(CGF);
8808 };
8809 auto LPCRegion =
8811 OMPLexicalScope Scope(*this, S);
8812 CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
8813}
8814
8817 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8818 Action.Enter(CGF);
8820 };
8821 auto LPCRegion =
8823 OMPLexicalScope Scope(*this, S);
8824 CGM.getOpenMPRuntime().emitMaskedRegion(*this, CodeGen, S.getBeginLoc());
8825}
8826
8829 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8830 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8831 PrePostActionTy &Action) {
8832 Action.Enter(CGF);
8834 };
8835 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
8836 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
8837 S.getBeginLoc());
8838 };
8839 auto LPCRegion =
8841 emitCommonOMPParallelDirective(*this, S, OMPD_master_taskloop, CodeGen,
8843}
8844
8847 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8848 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8849 PrePostActionTy &Action) {
8850 Action.Enter(CGF);
8852 };
8853 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
8854 CGM.getOpenMPRuntime().emitMaskedRegion(CGF, TaskLoopCodeGen,
8855 S.getBeginLoc());
8856 };
8857 auto LPCRegion =
8859 emitCommonOMPParallelDirective(*this, S, OMPD_masked_taskloop, CodeGen,
8861}
8862
8865 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8866 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8867 PrePostActionTy &Action) {
8868 Action.Enter(CGF);
8870 };
8871 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
8872 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
8873 S.getBeginLoc());
8874 };
8875 auto LPCRegion =
8877 emitCommonOMPParallelDirective(*this, S, OMPD_master_taskloop_simd, CodeGen,
8879}
8880
8883 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8884 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8885 PrePostActionTy &Action) {
8886 Action.Enter(CGF);
8888 };
8889 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
8890 CGM.getOpenMPRuntime().emitMaskedRegion(CGF, TaskLoopCodeGen,
8891 S.getBeginLoc());
8892 };
8893 auto LPCRegion =
8895 emitCommonOMPParallelDirective(*this, S, OMPD_masked_taskloop_simd, CodeGen,
8897}
8898
8899// Generate the instructions for '#pragma omp target update' directive.
8901 const OMPTargetUpdateDirective &S) {
8902 // If we don't have target devices, don't bother emitting the data mapping
8903 // code.
8904 if (CGM.getLangOpts().OMPTargetTriples.empty())
8905 return;
8906
8907 // Check if we have any if clause associated with the directive.
8908 const Expr *IfCond = nullptr;
8909 if (const auto *C = S.getSingleClause<OMPIfClause>())
8910 IfCond = C->getCondition();
8911
8912 // Check if we have any device clause associated with the directive.
8913 const Expr *Device = nullptr;
8914 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
8915 Device = C->getDevice();
8916
8917 OMPLexicalScope Scope(*this, S, OMPD_task);
8918 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
8919}
8920
8922 const OMPGenericLoopDirective &S) {
8923 // Always expect a bind clause on the loop directive. It it wasn't
8924 // in the source, it should have been added in sema.
8925
8927 if (const auto *C = S.getSingleClause<OMPBindClause>())
8928 BindKind = C->getBindKind();
8929
8930 switch (BindKind) {
8931 case OMPC_BIND_parallel: // for
8932 return emitOMPForDirective(S, *this, CGM, /*HasCancel=*/false);
8933 case OMPC_BIND_teams: // distribute
8934 return emitOMPDistributeDirective(S, *this, CGM);
8935 case OMPC_BIND_thread: // simd
8936 return emitOMPSimdDirective(S, *this, CGM);
8937 case OMPC_BIND_unknown:
8938 break;
8939 }
8940
8941 // Unimplemented, just inline the underlying statement for now.
8942 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8943 // Emit the loop iteration variable.
8944 const Stmt *CS =
8945 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt();
8946 const auto *ForS = dyn_cast<ForStmt>(CS);
8947 if (ForS && !isa<DeclStmt>(ForS->getInit())) {
8948 OMPPrivateScope LoopScope(CGF);
8949 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
8950 (void)LoopScope.Privatize();
8951 CGF.EmitStmt(CS);
8952 LoopScope.restoreMap();
8953 } else {
8954 CGF.EmitStmt(CS);
8955 }
8956 };
8957 OMPLexicalScope Scope(*this, S, OMPD_unknown);
8958 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_loop, CodeGen);
8959}
8960
8962 const OMPLoopDirective &S) {
8963 // Emit combined directive as if its constituent constructs are 'parallel'
8964 // and 'for'.
8965 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8966 Action.Enter(CGF);
8967 emitOMPCopyinClause(CGF, S);
8968 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
8969 };
8970 {
8971 auto LPCRegion =
8973 emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
8975 }
8976 // Check for outer lastprivate conditional update.
8978}
8979
8982 // To be consistent with current behavior of 'target teams loop', emit
8983 // 'teams loop' as if its constituent constructs are 'teams' and 'distribute'.
8984 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8986 };
8987
8988 // Emit teams region as a standalone region.
8989 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
8990 PrePostActionTy &Action) {
8991 Action.Enter(CGF);
8992 OMPPrivateScope PrivateScope(CGF);
8993 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
8994 (void)PrivateScope.Privatize();
8995 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
8996 CodeGenDistribute);
8997 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
8998 };
8999 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
9001 [](CodeGenFunction &) { return nullptr; });
9002}
9003
9004#ifndef NDEBUG
9006 std::string StatusMsg,
9007 const OMPExecutableDirective &D) {
9008 bool IsDevice = CGF.CGM.getLangOpts().OpenMPIsTargetDevice;
9009 if (IsDevice)
9010 StatusMsg += ": DEVICE";
9011 else
9012 StatusMsg += ": HOST";
9013 SourceLocation L = D.getBeginLoc();
9014 auto &SM = CGF.getContext().getSourceManager();
9015 PresumedLoc PLoc = SM.getPresumedLoc(L);
9016 const char *FileName = PLoc.isValid() ? PLoc.getFilename() : nullptr;
9017 unsigned LineNo =
9018 PLoc.isValid() ? PLoc.getLine() : SM.getExpansionLineNumber(L);
9019 llvm::dbgs() << StatusMsg << ": " << FileName << ": " << LineNo << "\n";
9020}
9021#endif
9022
9024 CodeGenFunction &CGF, PrePostActionTy &Action,
9026 Action.Enter(CGF);
9027 // Emit 'teams loop' as if its constituent constructs are 'distribute,
9028 // 'parallel, and 'for'.
9029 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
9031 S.getDistInc());
9032 };
9033
9034 // Emit teams region as a standalone region.
9035 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
9036 PrePostActionTy &Action) {
9037 Action.Enter(CGF);
9038 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
9039 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
9040 (void)PrivateScope.Privatize();
9042 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
9043 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
9044 };
9045 DEBUG_WITH_TYPE(TTL_CODEGEN_TYPE,
9047 CGF, TTL_CODEGEN_TYPE " as parallel for", S));
9048 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
9049 CodeGenTeams);
9051 [](CodeGenFunction &) { return nullptr; });
9052}
9053
9055 CodeGenFunction &CGF, PrePostActionTy &Action,
9057 Action.Enter(CGF);
9058 // Emit 'teams loop' as if its constituent construct is 'distribute'.
9059 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
9061 };
9062
9063 // Emit teams region as a standalone region.
9064 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
9065 PrePostActionTy &Action) {
9066 Action.Enter(CGF);
9067 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
9068 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
9069 (void)PrivateScope.Privatize();
9071 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
9072 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
9073 };
9074 DEBUG_WITH_TYPE(TTL_CODEGEN_TYPE,
9076 CGF, TTL_CODEGEN_TYPE " as distribute", S));
9077 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
9079 [](CodeGenFunction &) { return nullptr; });
9080}
9081
9084 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
9085 if (S.canBeParallelFor())
9087 else
9089 };
9091}
9092
9094 CodeGenModule &CGM, StringRef ParentName,
9096 // Emit SPMD target parallel loop region as a standalone region.
9097 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
9098 if (S.canBeParallelFor())
9100 else
9102 };
9103 llvm::Function *Fn;
9104 llvm::Constant *Addr;
9105 // Emit target region as a standalone region.
9106 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
9107 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
9108 assert(Fn && Addr &&
9109 "Target device function emission failed for 'target teams loop'.");
9110}
9111
9114 PrePostActionTy &Action) {
9115 Action.Enter(CGF);
9116 // Emit as 'parallel for'.
9117 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
9118 Action.Enter(CGF);
9120 CGF, OMPD_target_parallel_loop, /*hasCancel=*/false);
9123 };
9124 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
9126}
9127
9129 CodeGenModule &CGM, StringRef ParentName,
9131 // Emit target parallel loop region as a standalone region.
9132 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
9134 };
9135 llvm::Function *Fn;
9136 llvm::Constant *Addr;
9137 // Emit target region as a standalone region.
9138 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
9139 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
9140 assert(Fn && Addr && "Target device function emission failed.");
9141}
9142
9143/// Emit combined directive 'target parallel loop' as if its constituent
9144/// constructs are 'target', 'parallel', and 'for'.
9147 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
9149 };
9151}
9152
9154 const OMPExecutableDirective &D) {
9155 if (const auto *SD = dyn_cast<OMPScanDirective>(&D)) {
9157 return;
9158 }
9159 // A metadirective has no associated statement; emit its variants.
9160 if (const auto *MD = dyn_cast<OMPMetaDirective>(&D)) {
9162 return;
9163 }
9164 if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
9165 return;
9166 auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {
9167 OMPPrivateScope GlobalsScope(CGF);
9168 if (isOpenMPTaskingDirective(D.getDirectiveKind())) {
9169 // Capture global firstprivates to avoid crash.
9170 for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
9171 for (const Expr *Ref : C->varlist()) {
9172 const auto *DRE = cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
9173 if (!DRE)
9174 continue;
9175 const auto *VD = dyn_cast<VarDecl>(DRE->getDecl());
9176 if (!VD || VD->hasLocalStorage())
9177 continue;
9178 if (!CGF.LocalDeclMap.count(VD)) {
9179 LValue GlobLVal = CGF.EmitLValue(Ref);
9180 GlobalsScope.addPrivate(VD, GlobLVal.getAddress());
9181 }
9182 }
9183 }
9184 }
9185 if (isOpenMPSimdDirective(D.getDirectiveKind())) {
9186 (void)GlobalsScope.Privatize();
9187 ParentLoopDirectiveForScanRegion ScanRegion(CGF, D);
9189 } else {
9190 if (const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
9191 for (const Expr *E : LD->counters()) {
9192 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
9193 if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) {
9194 LValue GlobLVal = CGF.EmitLValue(E);
9195 GlobalsScope.addPrivate(VD, GlobLVal.getAddress());
9196 }
9197 if (isa<OMPCapturedExprDecl>(VD)) {
9198 // Emit only those that were not explicitly referenced in clauses.
9199 if (!CGF.LocalDeclMap.count(VD))
9200 CGF.EmitVarDecl(*VD);
9201 }
9202 }
9203 for (const auto *C : D.getClausesOfKind<OMPOrderedClause>()) {
9204 if (!C->getNumForLoops())
9205 continue;
9206 for (unsigned I = LD->getLoopsNumber(),
9207 E = C->getLoopNumIterations().size();
9208 I < E; ++I) {
9209 if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
9210 cast<DeclRefExpr>(C->getLoopCounter(I))->getDecl())) {
9211 // Emit only those that were not explicitly referenced in clauses.
9212 if (!CGF.LocalDeclMap.count(VD))
9213 CGF.EmitVarDecl(*VD);
9214 }
9215 }
9216 }
9217 }
9218 (void)GlobalsScope.Privatize();
9219 CGF.EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt());
9220 }
9221 };
9222 if (D.getDirectiveKind() == OMPD_atomic ||
9223 D.getDirectiveKind() == OMPD_critical ||
9224 D.getDirectiveKind() == OMPD_section ||
9225 D.getDirectiveKind() == OMPD_master ||
9226 D.getDirectiveKind() == OMPD_masked ||
9227 D.getDirectiveKind() == OMPD_assume ||
9228 isOpenMPLoopTransformationDirective(D.getDirectiveKind())) {
9229 EmitStmt(D.getAssociatedStmt());
9230 } else {
9231 auto LPCRegion =
9233 OMPSimdLexicalScope Scope(*this, D);
9234 CGM.getOpenMPRuntime().emitInlinedDirective(
9235 *this,
9236 isOpenMPSimdDirective(D.getDirectiveKind()) ? OMPD_simd
9237 : D.getDirectiveKind(),
9238 CodeGen);
9239 }
9240 // Check for outer lastprivate conditional update.
9242}
9243
9245 for (const auto *C : S.getClausesOfKind<OMPHoldsClause>()) {
9246 const Expr *E = C->getExpr();
9247 assert(E && "holds clause requires an expression");
9248 if (!E->HasSideEffects(getContext()))
9249 Builder.CreateAssumption(EvaluateExprAsBool(E));
9250 }
9251 EmitStmt(S.getAssociatedStmt());
9252}
Defines the clang::ASTContext interface.
#define V(N, I)
static bool isAllocatableDecl(const VarDecl *VD)
static const VarDecl * getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE)
static SourceRange getAssociatedLoopRange(const OMPExecutableDirective &S)
A directive's own source range covers only the pragma, so use the range of the associated loop,...
static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, PrePostActionTy &Action)
static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, PrePostActionTy &Action)
static const VarDecl * getBaseDecl(const Expr *Ref)
static void emitTargetTeamsGenericLoopRegionAsParallel(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsGenericLoopDirective &S)
static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *V, SourceLocation Loc)
static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, bool IsPostfixUpdate, const Expr *V, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
static void emitScanBasedDirective(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen, llvm::function_ref< void(CodeGenFunction &)> FirstGen, llvm::function_ref< void(CodeGenFunction &)> SecondGen)
Emits the code for the directive with inscan reductions.
static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO, LValue LVal, RValue RVal)
static bool isSupportedByOpenMPIRBuilder(const OMPTaskgroupDirective &T)
static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, QualType DstType, StringRef Name, LValue AddrLV)
static void emitLoopNestTransformation(CodeGenFunction &CGF, const OMPCanonicalLoopNestTransformationDirective &S)
Emit a loop-nest transformation followed by its loop variable finalization, which restores the loop-e...
static bool canEmitGPUFusedDistSchedule(const CodeGenModule &CGM, const OMPLoopDirective &S, OpenMPDirectiveKind DKind)
Whether a combined distribute parallel for may use the fused distr_static_chunk + static_chunkone sch...
static void emitDistributeParallelForDistributeInnerBoundParams(CodeGenFunction &CGF, const OMPExecutableDirective &S, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars)
static void emitScanBasedDirectiveFinals(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen)
Copies final inscan reductions values to the original variables.
static void checkForLastprivateConditionalUpdate(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static std::pair< LValue, LValue > emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
The following two functions generate expressions for the loop lower and upper bounds in case of stati...
static void emitTargetParallelForRegion(CodeGenFunction &CGF, const OMPTargetParallelForDirective &S, PrePostActionTy &Action)
static llvm::Function * emitOutlinedFunctionPrologueAggregate(CodeGenFunction &CGF, FunctionArgList &Args, llvm::MapVector< const Decl *, std::pair< const VarDecl *, Address > > &LocalAddrs, llvm::DenseMap< const Decl *, std::pair< const Expr *, llvm::Value * > > &VLASizes, llvm::Value *&CXXThisValue, llvm::Value *&ContextV, const CapturedStmt &CS, SourceLocation Loc, StringRef FunctionName)
static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper)
Emit a helper variable and return corresponding lvalue.
static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
static llvm::Value * convertToScalarValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static llvm::Function * emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S, const OMPExecutableDirective &D)
static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
static std::pair< bool, RValue > emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, RValue Update, BinaryOperatorKind BO, llvm::AtomicOrdering AO, bool IsXLHSInRHSPart)
static std::pair< LValue, LValue > emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitTargetTeamsGenericLoopRegionAsDistribute(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsGenericLoopDirective &S)
static void emitTargetParallelRegion(CodeGenFunction &CGF, const OMPTargetParallelDirective &S, PrePostActionTy &Action)
static std::pair< llvm::Value *, llvm::Value * > emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
When dealing with dispatch schedules (e.g.
static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitRestoreIP(CodeGenFunction &CGF, const T *C, llvm::OpenMPIRBuilder::InsertPointTy AllocaIP, llvm::OpenMPIRBuilder &OMPBuilder)
static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, const RegionCodeGenTy &CodeGen)
static void emitSimdlenSafelenClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
static void emitAlignedClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
static bool isSimdSupportedByOpenMPIRBuilder(const OMPLoopDirective &S)
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters)
static void applyConservativeSimdOrderedDirective(const Stmt &AssociatedStmt, LoopInfoStack &LoopStack)
static bool emitWorksharingDirective(CodeGenFunction &CGF, const OMPLoopDirective &S, bool HasCancel)
static void emitPostUpdateForReductionClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc, const unsigned IVSize, const bool IVSigned)
static void emitTargetTeamsLoopCodegenStatus(CodeGenFunction &CGF, std::string StatusMsg, const OMPExecutableDirective &D)
static bool isForSupportedByOpenMPIRBuilder(const OMPLoopDirective &S, bool HasCancel)
static RValue emitSimpleAtomicLoad(CodeGenFunction &CGF, llvm::AtomicOrdering AO, LValue LVal, SourceLocation Loc)
static std::pair< llvm::Value *, llvm::Value * > emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
if the 'for' loop has a dispatch schedule (e.g.
static bool hasOrderedBlockAssocDirective(const Stmt *S)
static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *UE, const Expr *D, const Expr *CE, bool IsXLHSInRHSPart, bool IsFailOnly, SourceLocation Loc)
#define TTL_CODEGEN_TYPE
static CodeGenFunction::ComplexPairTy convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static ImplicitParamDecl * createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data, QualType Ty, CapturedDecl *CD, SourceLocation Loc)
static EmittedClosureTy emitCapturedStmtFunc(CodeGenFunction &ParentCGF, const CapturedStmt *S)
Emit a captured statement and return the function as well as its captured closure context.
static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
static void emitOMPDistributeDirective(const OMPLoopDirective &S, CodeGenFunction &CGF, CodeGenModule &CGM)
static void emitOMPCopyinClause(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitTargetTeamsDistributeParallelForRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, PrePostActionTy &Action)
static llvm::CallInst * emitCapturedStmtCall(CodeGenFunction &ParentCGF, EmittedClosureTy Cap, llvm::ArrayRef< llvm::Value * > Args)
Emit a call to a previously captured closure.
static void emitMasked(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop, int MaxLevel, int Level=0)
static void emitOMPForDirective(const OMPLoopDirective &S, CodeGenFunction &CGF, CodeGenModule &CGM, bool HasCancel)
static void emitEmptyBoundParameters(CodeGenFunction &, const OMPExecutableDirective &, llvm::SmallVectorImpl< llvm::Value * > &)
static void emitTargetParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetParallelForSimdDirective &S, PrePostActionTy &Action)
static void emitOMPSimdDirective(const OMPLoopDirective &S, CodeGenFunction &CGF, CodeGenModule &CGM)
static void emitOMPAtomicCompareExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO, const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *D, const Expr *CE, bool IsXBinopExpr, bool IsPostfixUpdate, bool IsFailOnly, SourceLocation Loc)
std::pair< llvm::Function *, llvm::Value * > EmittedClosureTy
static OpenMPDirectiveKind getEffectiveDirectiveKind(const OMPExecutableDirective &S)
static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDirective &S)
static void buildDependences(const OMPExecutableDirective &S, OMPTaskDataTy &Data)
static RValue convertToType(CodeGenFunction &CGF, RValue Value, QualType SourceType, QualType ResType, SourceLocation Loc)
static void emitScanBasedDirectiveDecls(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen)
Emits internal temp array declarations for the directive with inscan reductions.
static void emitTargetTeamsDistributeParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForSimdDirective &S, PrePostActionTy &Action)
static void emitTargetTeamsDistributeSimdRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeSimdDirective &S)
static llvm::MapVector< llvm::Value *, llvm::Value * > GetAlignedMapping(const OMPLoopDirective &S, CodeGenFunction &CGF)
static llvm::omp::ScheduleKind convertClauseKindToSchedKind(OpenMPScheduleClauseKind ScheduleClauseKind)
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
static void emitTargetParallelGenericLoopRegion(CodeGenFunction &CGF, const OMPTargetParallelGenericLoopDirective &S, PrePostActionTy &Action)
static QualType getCanonicalParamType(ASTContext &C, QualType T)
static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S, const RegionCodeGenTy &SimdInitGen, const RegionCodeGenTy &BodyCodeGen)
static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, const Twine &Name, llvm::Value *Init=nullptr)
static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *E, SourceLocation Loc)
static llvm::Function * emitOutlinedFunctionPrologue(CodeGenFunction &CGF, FunctionArgList &Args, llvm::MapVector< const Decl *, std::pair< const VarDecl *, Address > > &LocalAddrs, llvm::DenseMap< const Decl *, std::pair< const Expr *, llvm::Value * > > &VLASizes, llvm::Value *&CXXThisValue, const FunctionOptions &FO)
static void emitInnerParallelForWhenCombined(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
static void emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeDirective &S)
This file defines OpenMP nodes for declarative directives.
TokenType getType() const
Returns the token's type, e.g.
FormatToken * Next
The next token in the unwrapped line.
static const Decl * getCanonicalDecl(const Decl *D)
#define X(type, name)
Definition Value.h:97
This file defines OpenMP AST classes for clauses.
Defines some OpenMP-specific enums and functions.
Defines the PrettyStackTraceEntry class, which is used to make crashes give more contextual informati...
Defines the SourceManager interface.
This file defines OpenMP AST classes for executable directives and clauses.
This represents clause 'aligned' in the 'pragma omp ...' directives.
This represents 'pragma omp atomic' directive.
Expr * getR()
Get 'r' part of the associated expression/statement.
Expr * getX()
Get 'x' part of the associated expression/statement.
bool isFailOnly() const
Return true if 'v' is updated only when the condition is evaluated false (compare capture only).
bool isPostfixUpdate() const
Return true if 'v' expression must be updated to original value of 'x', false if 'v' must be updated ...
Expr * getExpr()
Get 'expr' part of the associated expression/statement.
Expr * getV()
Get 'v' part of the associated expression/statement.
bool isXLHSInRHSPart() const
Return true if helper update expression has form 'OpaqueValueExpr(x) binop OpaqueValueExpr(expr)' and...
Expr * getD()
Get 'd' part of the associated expression/statement.
Expr * getUpdateExpr()
Get helper expression of the form 'OpaqueValueExpr(x) binop OpaqueValueExpr(expr)' or 'OpaqueValueExp...
Expr * getCondExpr()
Get the 'cond' part of the source atomic expression.
This represents 'pragma omp barrier' directive.
This represents 'bind' clause in the 'pragma omp ...' directives.
This represents 'pragma omp cancel' directive.
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
This represents 'pragma omp cancellation point' directive.
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
The base class for all transformation directives of canonical loop nests.
Stmt * getTransformedStmt() const
Get the de-sugared statements after the loop transformation.
Stmt * getFinals() const
Return loop variable finalization statement, or nullptr if the concrete transformation does not produ...
The base class for all transformation directives of canonical loop sequences (currently only 'fuse')
This represents clause 'copyin' in the 'pragma omp ...' directives.
This represents clause 'copyprivate' in the 'pragma omp ...' directives.
This represents 'pragma omp critical' directive.
DeclarationNameInfo getDirectiveName() const
Return name of the directive.
This represents implicit clause 'depend' for the 'pragma omp task' directive.
This represents implicit clause 'depobj' for the 'pragma omp depobj' directive. This clause does not ...
This represents 'pragma omp depobj' directive.
This represents 'destroy' clause in the 'pragma omp depobj' directive or the 'pragma omp interop' dir...
This represents 'device' clause in the 'pragma omp ...' directive.
This represents 'dist_schedule' clause in the 'pragma omp ...' directive.
This represents 'pragma omp distribute' directive.
This represents 'pragma omp distribute parallel for' composite directive.
This represents 'pragma omp distribute parallel for simd' composite directive.
This represents 'pragma omp distribute simd' composite directive.
This represents the 'doacross' clause for the 'pragma omp ordered' directive.
This represents 'pragma omp error' directive.
This represents 'filter' clause in the 'pragma omp ...' directive.
Represents the 'pragma omp flatten' loop transformation directive.
Stmt * getTransformedStmt() const
Gets the flattened loop after the transformation.
Stmt * getFinals() const
Return updates to the original loop variables after the flattened loop.
This represents implicit clause 'flush' for the 'pragma omp flush' directive. This clause does not ex...
This represents 'pragma omp flush' directive.
This represents 'pragma omp for' directive.
bool hasCancel() const
Return true if current directive has inner cancel directive.
This represents 'pragma omp for simd' directive.
Represents the 'pragma omp fuse' loop transformation directive.
Stmt * getFinals() const
Return finals statement.
Stmt * getTransformedStmt() const
Gets the associated loops after the transformation.
This represents 'pragma omp loop' directive.
This represents 'grainsize' clause in the 'pragma omp ...' directive.
This represents 'hint' clause in the 'pragma omp ...' directive.
This represents clause 'inclusive' in the 'pragma omp scan' directive.
This represents the 'init' clause in 'pragma omp ...' directives.
Represents the 'pragma omp interchange' loop transformation directive.
This represents 'pragma omp interop' directive.
This is a common base class for loop directives ('omp simd', 'omp for', 'omp for simd' etc....
Expr * getCombinedUpperBoundVariable() const
Expr * getPreCond() const
Expr * getPrevUpperBoundVariable() const
Expr * getIsLastIterVariable() const
Expr * getCombinedLowerBoundVariable() const
Expr * getCombinedCond() const
Expr * getCombinedInit() const
Expr * getLowerBoundVariable() const
Expr * getCombinedNextLowerBound() const
ArrayRef< Expr * > finals_conditions()
ArrayRef< Expr * > counters()
Expr * getInc() const
ArrayRef< Expr * > private_counters()
Expr * getUpperBoundVariable() const
Expr * getCombinedDistCond() const
Expr * getPrevLowerBoundVariable() const
ArrayRef< Expr * > dependent_inits()
Expr * getNextLowerBound() const
Expr * getDistInc() const
ArrayRef< Expr * > updates()
Expr * getCond() const
Expr * getNextUpperBound() const
Expr * getLastIteration() const
Expr * getEnsureUpperBound() const
Expr * getCalcLastIteration() const
Expr * getCombinedNextUpperBound() const
Expr * getIterationVariable() const
Expr * getStrideVariable() const
Expr * getNumIterations() const
Expr * getInit() const
ArrayRef< Expr * > finals()
Expr * getPrevEnsureUpperBound() const
Expr * getCombinedEnsureUpperBound() const
ArrayRef< Expr * > dependent_counters()
ArrayRef< Expr * > inits()
This represents 'pragma omp masked' directive.
This represents 'pragma omp masked taskloop' directive.
This represents 'pragma omp masked taskloop simd' directive.
This represents 'pragma omp master' directive.
This represents 'pragma omp master taskloop' directive.
This represents 'pragma omp master taskloop simd' directive.
This represents 'pragma omp metadirective' directive.
ArrayRef< Stmt * > variants() const
This represents 'nogroup' clause in the 'pragma omp ...' directive.
This represents 'num_tasks' clause in the 'pragma omp ...' directive.
This represents 'num_teams' clause in the 'pragma omp ...' directive.
This represents 'order' clause in the 'pragma omp ...' directive.
This represents block-associated 'pragma omp ordered' directive.
This represents standalone 'pragma omp ordered' directive.
This represents 'pragma omp parallel for' directive.
bool hasCancel() const
Return true if current directive has inner cancel directive.
This represents 'pragma omp parallel for simd' directive.
This represents 'pragma omp parallel masked' directive.
This represents 'pragma omp parallel masked taskloop' directive.
This represents 'pragma omp parallel masked taskloop simd' directive.
This represents 'pragma omp parallel master' directive.
This represents 'pragma omp parallel master taskloop' directive.
This represents 'pragma omp parallel master taskloop simd' directive.
This represents 'pragma omp parallel sections' directive.
This represents 'priority' clause in the 'pragma omp ...' directive.
Represents the 'pragma omp reverse' loop transformation directive.
This represents 'simd' clause in the 'pragma omp ...' directive.
This represents 'pragma omp scan' directive.
This represents 'pragma omp scope' directive.
This represents 'pragma omp section' directive.
This represents 'pragma omp sections' directive.
bool hasCancel() const
Return true if current directive has inner cancel directive.
This represents 'pragma omp simd' directive.
This represents 'pragma omp single' directive.
Represents the 'pragma omp split' loop transformation directive.
This represents the 'pragma omp stripe' loop transformation directive.
This represents 'pragma omp target data' directive.
This represents 'pragma omp target' directive.
This represents 'pragma omp target enter data' directive.
This represents 'pragma omp target exit data' directive.
This represents 'pragma omp target parallel' directive.
This represents 'pragma omp target parallel for' directive.
bool hasCancel() const
Return true if current directive has inner cancel directive.
This represents 'pragma omp target parallel for simd' directive.
This represents 'pragma omp target parallel loop' directive.
This represents 'pragma omp target teams' directive.
This represents 'pragma omp target teams distribute' combined directive.
This represents 'pragma omp target teams distribute parallel for' combined directive.
This represents 'pragma omp target teams distribute parallel for simd' combined directive.
This represents 'pragma omp target teams distribute simd' combined directive.
This represents 'pragma omp target teams loop' directive.
bool canBeParallelFor() const
Return true if current loop directive's associated loop can be a parallel for.
This represents 'pragma omp target update' directive.
This represents 'pragma omp task' directive.
This represents 'pragma omp taskloop' directive.
This represents 'pragma omp taskloop simd' directive.
This represents 'pragma omp taskgroup' directive.
const Expr * getReductionRef() const
Returns reference to the task_reduction return variable.
This represents 'pragma omp taskwait' directive.
This represents 'pragma omp taskyield' directive.
This represents 'pragma omp teams' directive.
This represents 'pragma omp teams distribute' directive.
This represents 'pragma omp teams distribute parallel for' composite directive.
This represents 'pragma omp teams distribute parallel for simd' composite directive.
This represents 'pragma omp teams distribute simd' combined directive.
This represents 'pragma omp teams loop' directive.
This represents 'thread_limit' clause in the 'pragma omp ...' directive.
This represents the 'pragma omp tile' loop transformation directive.
This represents the 'pragma omp unroll' loop transformation directive.
This represents the 'use' clause in 'pragma omp ...' directives.
This represents clause 'use_device_addr' in the 'pragma omp ...' directives.
This represents clause 'use_device_ptr' in the 'pragma omp ...' directives.
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition ASTContext.h:245
SourceManager & getSourceManager()
Definition ASTContext.h:916
TranslationUnitDecl * getTranslationUnitDecl() const
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
CanQualType VoidPtrTy
IdentifierTable & Idents
Definition ASTContext.h:855
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
QualType getUIntPtrType() const
Return a type compatible with "uintptr_t" (C99 7.18.1.4), as defined by the target.
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth,...
TypeSourceInfo * getTrivialTypeSourceInfo(QualType T, SourceLocation Loc=SourceLocation()) const
Allocate a TypeSourceInfo where all locations have been initialized to a given location,...
unsigned getOpenMPDefaultSimdAlign(QualType T) const
Get default simd alignment of the specified complete type in bits.
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
CanQualType VoidTy
QualType getFunctionType(QualType ResultTy, ArrayRef< QualType > Args, const FunctionProtoType::ExtProtoInfo &EPI) const
Return a normal function type with a typed argument list.
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
CanQualType getCanonicalTagType(const TagDecl *TD) const
ASTRecordLayout - This class contains layout information for one RecordDecl, which is a struct/union/...
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Definition TypeBase.h:3820
Represents an attribute applied to a statement.
Definition Stmt.h:2214
ArrayRef< const Attr * > getAttrs() const
Definition Stmt.h:2246
static BinaryOperator * Create(const ASTContext &C, Expr *lhs, Expr *rhs, Opcode opc, QualType ResTy, ExprValueKind VK, ExprObjectKind OK, SourceLocation opLoc, FPOptionsOverride FPFeatures)
Definition Expr.cpp:5144
A binding in a decomposition declaration.
Definition DeclCXX.h:4232
DecompositionDecl * getDecomposedDecl() const
Get the decomposition declaration that this binding represents a decomposition of.
Definition DeclCXX.h:4265
Represents the body of a CapturedStmt, and serves as its DeclContext.
Definition Decl.h:5131
unsigned getNumParams() const
Definition Decl.h:5169
ImplicitParamDecl * getContextParam() const
Retrieve the parameter containing captured variables.
Definition Decl.h:5189
unsigned getContextParamPosition() const
Definition Decl.h:5198
bool isNothrow() const
Definition Decl.cpp:5814
static CapturedDecl * Create(ASTContext &C, DeclContext *DC, unsigned NumParams)
Definition Decl.cpp:5799
param_iterator param_end() const
Retrieve an iterator one past the last parameter decl.
Definition Decl.h:5206
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
Definition Decl.h:5204
Stmt * getBody() const override
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
Definition Decl.cpp:5811
ImplicitParamDecl * getParam(unsigned i) const
Definition Decl.h:5171
This captures a statement into a function.
Definition Stmt.h:3950
SourceLocation getEndLoc() const LLVM_READONLY
Definition Stmt.h:4149
CapturedDecl * getCapturedDecl()
Retrieve the outlined function declaration.
Definition Stmt.cpp:1493
child_range children()
Definition Stmt.cpp:1484
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Definition Stmt.h:4071
Stmt * getCapturedStmt()
Retrieve the statement being captured.
Definition Stmt.h:4054
capture_init_iterator capture_init_begin()
Retrieve the first initialization argument.
Definition Stmt.h:4127
SourceLocation getBeginLoc() const LLVM_READONLY
Definition Stmt.h:4145
capture_init_iterator capture_init_end()
Retrieve the iterator pointing one past the last initialization argument.
Definition Stmt.h:4137
capture_range captures()
Definition Stmt.h:4088
Expr *const * const_capture_init_iterator
Const iterator that walks over the capture initialization arguments.
Definition Stmt.h:4114
This is an opaque type for sizes expressed in character units.
Definition CharUnits.h:38
CharUnits alignTo(CharUnits Align) const
Returns the next integer (mod 2**64) that is greater than or equal to this quantity and is a multiple...
Definition CharUnits.h:171
QuantityType getQuantity() const
Get the raw integer representation of this quantity.
Definition CharUnits.h:155
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
Definition CharUnits.h:184
static CharUnits fromQuantity(QuantityType Quantity)
Construct a CharUnits quantity from a raw integer type.
Definition CharUnits.h:58
std::string SampleProfileFile
Name of the profile file to use with -fprofile-sample-use.
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
Definition Address.h:128
static Address invalid()
Definition Address.h:176
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
Definition Address.h:253
CharUnits getAlignment() const
Definition Address.h:194
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Definition Address.h:209
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Definition Address.h:276
Address withAlignment(CharUnits NewAlignment) const
Return address with different alignment, but same pointer and element type.
Definition Address.h:269
llvm::PointerType * getType() const
Return the type of the pointer value.
Definition Address.h:204
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored.
Definition CGValue.h:619
static ApplyDebugLocation CreateDefaultArtificial(CodeGenFunction &CGF, SourceLocation TemporaryLocation)
Apply TemporaryLocation if it is valid.
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Definition CGBuilder.h:223
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Definition CGBuilder.h:128
llvm::LoadInst * CreateAlignedLoad(llvm::Type *Ty, llvm::Value *Addr, CharUnits Align, const llvm::Twine &Name="")
Definition CGBuilder.h:148
CGFunctionInfo - Class to encapsulate the information about a function definition.
Manages list of lastprivate conditional decls for the specified directive.
static LastprivateConditionalRAII disable(CodeGenFunction &CGF, const OMPExecutableDirective &S)
Manages list of nontemporal decls for the specified directive.
Struct that keeps all the relevant information that should be kept throughout a 'target data' region.
llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap
Map between the a declaration of a capture and the corresponding new llvm address where the runtime r...
Manages list of nontemporal decls for the specified directive.
virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
Emit task region for the task directive.
virtual llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST)
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond, llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, const Expr *Message=nullptr)
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
virtual void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc, OpenMPNumThreadsClauseModifier Modifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, SourceLocation SeverityLoc=SourceLocation(), const Expr *Message=nullptr, SourceLocation MessageLoc=SourceLocation())
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32global_tid, kmp_int32 num_threads) ...
StringRef getTargetRegionParentName(const FunctionDecl *FD) const
Returns the parent name of the target regions in FD: its enclosing non-closure function if FD is a cl...
virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal)
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
virtual void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc)
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
virtual void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars)
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
virtual const VarDecl * translateParameter(const FieldDecl *FD, const VarDecl *NativeParam) const
Translates the native parameter of outlined function if this is required for target.
virtual llvm::Function * emitTeamsOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP teams directive D.
virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr * > NumIterations)
Emit initialization for doacross loop nesting support.
virtual void adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF, const OMPExecutableDirective &D) const
Adjust some parameters for the target-based directives, like addresses of the variables captured by r...
virtual Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const
Gets the address of the native argument basing on the address of the target-specific parameter.
virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32global_tid, kmp_int32 num_teams,...
virtual llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data)
Emit a code for initialization of task reduction clause.
virtual void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc, llvm::AtomicOrdering AO)
Emit flush of the variables specified in 'omp flush' directive.
virtual void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc)
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32global_tid, int proc_bind) to generate...
virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false)
Emit an implicit/explicit barrier for OpenMP threads.
virtual void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values)
virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind)
Call the appropriate runtime routine to notify that we finished all the work with current loop.
void emitIfClause(CodeGenFunction &CGF, const Expr *Cond, const RegionCodeGenTy &ThenGen, const RegionCodeGenTy &ElseGen)
Emits code for OpenMP 'if' clause using specified CodeGen function.
virtual llvm::Function * emitParallelOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP parallel directive D.
virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values)
Call the appropriate runtime routine to initialize it before start of loop.
virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static non-chunked.
virtual void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc)
Emits a master region.
virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Required to resolve existing problems in the runtime.
virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF, const Expr *LHS)
Checks if the provided LVal is lastprivate conditional and emits the code to update the value of the ...
llvm::OpenMPIRBuilder & getOMPBuilder()
virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Emit outilined function for 'target' directive.
virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned)
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
virtual void checkAndEmitSharedLastprivateConditional(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::DenseSet< CanonicalDeclPtr< const VarDecl > > &IgnoredDecls)
Checks if the lastprivate conditional was updated in inner region and writes the value.
virtual void emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool HasCancel=false)
Emit code for the directive that does not require outlining.
virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static chunked.
virtual void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Emit the target offloading code associated with D.
virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const
Check if the specified ScheduleKind is dynamic.
virtual void emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter=nullptr)
Emits a masked region.
virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues)
Call the appropriate runtime routine to initialize it before start of loop.
Address getAllocatedAddress() const
Returns the raw, allocated address, which is not necessarily the address of the object itself.
API for captured statement code generation.
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
RAII for correct setting/restoring of CapturedStmtInfo.
LValue getReferenceLValue(CodeGenFunction &CGF, const Expr *RefExpr) const
void ForceCleanup()
Force the emission of cleanups now, instead of waiting until this object is destroyed.
RAII for preserving necessary info during inlined region body codegen.
RAII for preserving necessary info during Outlined region body codegen.
Controls insertion of cancellation exit blocks in worksharing constructs.
Save/restore original map of previously emitted local vars in case when we need to duplicate emission...
The class used to assign some variables some temporarily addresses.
bool setVarAddr(CodeGenFunction &CGF, const ValueDecl *LocalVD, Address TempAddr)
Sets the address of the variable LocalVD to be TempAddr in function CGF.
bool apply(CodeGenFunction &CGF)
Applies new addresses to the list of the variables.
void restore(CodeGenFunction &CGF)
Restores original addresses of the variables.
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
void restoreMap()
Restore all mapped variables w/o clean up.
bool Privatize()
Privatizes local variables previously registered as private.
bool addPrivate(const ValueDecl *LocalVD, Address Addr)
Registers LocalVD variable as a private with Addr as the address of the corresponding private variabl...
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited.
void ForceCleanup(std::initializer_list< llvm::Value ** > ValuesToReload={})
Force the emission of cleanups now, instead of waiting until this object is destroyed.
bool requiresCleanups() const
Determine whether this scope requires any cleanups.
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
void EmitOMPParallelMaskedTaskLoopDirective(const OMPParallelMaskedTaskLoopDirective &S)
void EmitOMPParallelMaskedDirective(const OMPParallelMaskedDirective &S)
void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S)
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount, Stmt::Likelihood LH=Stmt::LH_None, const Expr *ConditionalOp=nullptr, const VarDecl *ConditionalDecl=nullptr)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D, bool NoFinals, llvm::Value *IsLastIterCond=nullptr)
Emit final copying of lastprivate values to original variables at the end of the worksharing or simd ...
void processInReduction(const OMPExecutableDirective &S, OMPTaskDataTy &Data, CodeGenFunction &CGF, const CapturedStmt *CS, OMPPrivateScope &Scope)
llvm::Value * performAddrSpaceCast(llvm::Value *Src, llvm::Type *DestTy)
JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target)
The given basic block lies in the current EH scope, but may be a target of a potentially scope-crossi...
void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S)
void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy, SourceLocation Loc)
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
void EmitOMPCanonicalLoop(const OMPCanonicalLoop *S)
Emit an OMPCanonicalLoop using the OpenMPIRBuilder.
void EmitOMPGenericLoopDirective(const OMPGenericLoopDirective &S)
void EmitOMPScanDirective(const OMPScanDirective &S)
static bool hasScalarEvaluationKind(QualType T)
llvm::Type * ConvertType(QualType T)
llvm::function_ref< std::pair< llvm::Value *, llvm::Value * >(CodeGenFunction &, const OMPExecutableDirective &S, Address LB, Address UB)> CodeGenDispatchBoundsTy
LValue InitCapturedStruct(const CapturedStmt &S)
Definition CGStmt.cpp:3435
void EmitOMPFlattenDirective(const OMPFlattenDirective &S)
CGCapturedStmtInfo * CapturedStmtInfo
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
void EmitOMPParallelForDirective(const OMPParallelForDirective &S)
void EmitOMPMasterDirective(const OMPMasterDirective &S)
void EmitOMPParallelMasterTaskLoopSimdDirective(const OMPParallelMasterTaskLoopSimdDirective &S)
void EmitOMPSimdInit(const OMPLoopDirective &D)
Helpers for the OpenMP loop directives.
const OMPExecutableDirective * OMPParentLoopDirectiveForScan
Parent loop-based directive for scan directive.
void EmitOMPFlushDirective(const OMPFlushDirective &S)
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S)
void EmitOMPTargetTeamsDistributeParallelForSimdDirective(const OMPTargetTeamsDistributeParallelForSimdDirective &S)
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
void EmitOMPReductionClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope, bool ForInscan=false)
Emit initial code for reduction variables.
void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S)
void EmitAutoVarDecl(const VarDecl &D)
EmitAutoVarDecl - Emit an auto variable declaration.
Definition CGDecl.cpp:1366
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
llvm::Function * GenerateOpenMPCapturedStmtFunctionAggregate(const CapturedStmt &S, const OMPExecutableDirective &D)
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
void EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S)
const LangOptions & getLangOpts() const
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
EmitAutoVarAlloca - Emit the alloca and debug information for a local variable.
Definition CGDecl.cpp:1497
void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO, const llvm::function_ref< RValue(RValue)> &UpdateOp, bool IsVolatile)
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPLoopDirective &S)
Emit device code for the target simd and target loop directives.
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Load a pointer with type PtrTy stored at address Ptr.
Definition CGExpr.cpp:3465
void EmitOMPSplitDirective(const OMPSplitDirective &S)
void EmitBranchThroughCleanup(JumpDest Dest)
EmitBranchThroughCleanup - Emit a branch from the current insert block through the normal cleanup han...
void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind)
Emit final update of reduction values to original variables at the end of the directive.
void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit)
Helper for the OpenMP loop directives.
void EmitOMPScopeDirective(const OMPScopeDirective &S)
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
llvm::AssertingVH< llvm::Instruction > AllocaInsertPt
AllocaInsertPoint - This is an instruction in the entry block before which we prefer to insert alloca...
void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy)
Emit an aggregate assignment.
JumpDest ReturnBlock
ReturnBlock - Unified return block.
void EmitOMPTargetTeamsDistributeSimdDirective(const OMPTargetTeamsDistributeSimdDirective &S)
const llvm::function_ref< void(CodeGenFunction &, llvm::Function *, const OMPTaskDataTy &)> TaskGenTy
llvm::DebugLoc SourceLocToDebugLoc(SourceLocation Location)
Converts Location to a DebugLoc, if debug information is enabled.
bool EmitOMPCopyinClause(const OMPExecutableDirective &D)
Emit code for copyin clause in D directive.
void EmitOMPLinearClause(const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope)
Emit initial code for linear clauses.
llvm::BasicBlock * OMPBeforeScanBlock
void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S)
void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S, OMPPrivateScope &LoopScope)
Emit initial code for loop counters of loop-based directives.
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value * > &CapturedVars)
void EmitOMPDepobjDirective(const OMPDepobjDirective &S)
void EmitOMPMetaDirective(const OMPMetaDirective &S)
void EmitOMPCriticalDirective(const OMPCriticalDirective &S)
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
Definition CGExpr.cpp:263
void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S)
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
Definition CGExpr.cpp:2561
void EmitOMPCancelDirective(const OMPCancelDirective &S)
void EmitOMPBarrierDirective(const OMPBarrierDirective &S)
llvm::Value * EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified complex type to the specified destination type,...
bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB, const CodeGenLoopBoundsTy &CodeGenLoopBounds, const CodeGenDispatchBoundsTy &CGDispatchBounds)
Emit code for the worksharing loop-based directive.
LValue EmitOMPSharedLValue(const Expr *E)
Emits the lvalue for the expression with possibly captured variable.
llvm::CanonicalLoopInfo * EmitOMPCollapsedCanonicalLoopNest(const Stmt *S, int Depth)
Emit the Stmt S and return its topmost canonical loop, if any.
void EmitOMPSectionsDirective(const OMPSectionsDirective &S)
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
void EmitOMPInteropDirective(const OMPInteropDirective &S)
void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S)
void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S)
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
Definition CGExpr.cpp:244
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
void EmitOMPTargetParallelForSimdDirective(const OMPTargetParallelForSimdDirective &S)
void EmitOMPTargetParallelGenericLoopDirective(const OMPTargetParallelGenericLoopDirective &S)
Emit combined directive 'target parallel loop' as if its constituent constructs are 'target',...
void EmitOMPUseDeviceAddrClause(const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap)
void EmitOMPTeamsDistributeParallelForSimdDirective(const OMPTeamsDistributeParallelForSimdDirective &S)
void EmitOMPMaskedDirective(const OMPMaskedDirective &S)
llvm::Value * emitArrayLength(const ArrayType *arrayType, QualType &baseType, Address &addr)
emitArrayLength - Compute the length of an array, even if it's a VLA, and drill down to the base elem...
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
void EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S)
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
void EmitOMPOrderedBlockAssocDirective(const OMPOrderedBlockAssocDirective &S)
void EmitOMPDistributeLoop(const OMPLoopDirective &S, const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr)
Emit code for the distribute loop-based directive.
void EmitOMPMasterTaskLoopDirective(const OMPMasterTaskLoopDirective &S)
void EmitOMPReverseDirective(const OMPReverseDirective &S)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
void EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S)
void EmitOMPTargetTeamsDistributeParallelForDirective(const OMPTargetTeamsDistributeParallelForDirective &S)
void EmitOMPMaskedTaskLoopDirective(const OMPMaskedTaskLoopDirective &S)
llvm::function_ref< std::pair< LValue, LValue >(CodeGenFunction &, const OMPExecutableDirective &S)> CodeGenLoopBoundsTy
void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S)
RawAddress CreateMemTempWithoutCast(QualType T, const Twine &Name="tmp")
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen without...
Definition CGExpr.cpp:236
void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S)
void EmitOMPMaskedTaskLoopSimdDirective(const OMPMaskedTaskLoopSimdDirective &S)
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
VlaSizePair getVLASize(const VariableArrayType *vla)
Returns an LLVM value that corresponds to the size, in non-variably-sized elements,...
void EmitOMPParallelDirective(const OMPParallelDirective &S)
void EmitOMPTaskDirective(const OMPTaskDirective &S)
void EmitOMPMasterTaskLoopSimdDirective(const OMPMasterTaskLoopSimdDirective &S)
void EmitOMPDistributeParallelForDirective(const OMPDistributeParallelForDirective &S)
void EmitOMPAssumeDirective(const OMPAssumeDirective &S)
int ExpectedOMPLoopDepth
Number of nested loop to be consumed by the last surrounding loop-associated directive.
void EmitOMPPrivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S)
void EmitStopPoint(const Stmt *S)
EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
Definition CGStmt.cpp:49
void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S)
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
void EmitOMPTargetTeamsGenericLoopDirective(const OMPTargetTeamsGenericLoopDirective &S)
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
void EmitAutoVarCleanups(const AutoVarEmission &emission)
Definition CGDecl.cpp:2287
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
Definition CGExpr.cpp:2813
SmallVector< llvm::CanonicalLoopInfo *, 4 > OMPLoopNestStack
List of recently emitted OMPCanonicalLoops.
Address EmitOMPBindingOriginalAddr(const BindingDecl *BD, SourceLocation Loc)
Emits the original address for a structured binding.
void EmitOMPTeamsDistributeParallelForDirective(const OMPTeamsDistributeParallelForDirective &S)
llvm::AtomicRMWInst * emitAtomicRMWInst(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Order=llvm::AtomicOrdering::SequentiallyConsistent, llvm::SyncScope::ID SSID=llvm::SyncScope::System, const AtomicExpr *AE=nullptr)
Emit an atomicrmw instruction, and applying relevant metadata when applicable.
void EmitOMPFuseDirective(const OMPFuseDirective &S)
void EmitOMPTargetTeamsDistributeDirective(const OMPTargetTeamsDistributeDirective &S)
void EmitOMPUseDevicePtrClause(const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap)
LValue EmitOMPCapturedBindingLValue(const BindingDecl *BD)
Emit an LValue for a structured binding captured in an OpenMP region.
Definition CGExpr.cpp:3643
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
EmitAnyExpr - Emit code to compute the specified expression which can have any type.
Definition CGExpr.cpp:285
void EmitStmt(const Stmt *S, ArrayRef< const Attr * > Attrs={})
EmitStmt - Emit the code for the statement.
Definition CGStmt.cpp:59
llvm::DenseMap< const ValueDecl *, FieldDecl * > LambdaCaptureFields
void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S)
llvm::Type * ConvertTypeForMem(QualType T)
void EmitOMPInnerLoop(const OMPExecutableDirective &S, bool RequiresCleanup, const Expr *LoopCond, const Expr *IncExpr, const llvm::function_ref< void(CodeGenFunction &)> BodyGen, const llvm::function_ref< void(CodeGenFunction &)> PostIncGen)
Emit inner loop of the worksharing/simd construct.
void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S)
static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S)
void EmitOMPTargetDirective(const OMPTargetDirective &S)
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
void EmitOMPTeamsDirective(const OMPTeamsDirective &S)
void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D)
Emit simple code for OpenMP directives in Simd-only mode.
void EmitOMPErrorDirective(const OMPErrorDirective &S)
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
void EmitOMPParallelMaskedTaskLoopSimdDirective(const OMPParallelMaskedTaskLoopSimdDirective &S)
void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S)
void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S)
Address GenerateCapturedStmtArgument(const CapturedStmt &S)
Definition CGStmt.cpp:3476
bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for lastprivate variables.
static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Emit device code for the target teams distribute parallel for simd directive.
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block,...
Definition CGStmt.cpp:678
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S, const OMPExecutableDirective &D)
void EmitOMPSimdDirective(const OMPSimdDirective &S)
RawAddress CreateMemTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Definition CGExpr.cpp:200
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
Definition CGExpr.cpp:3423
void EmitOMPParallelGenericLoopDirective(const OMPLoopDirective &S)
void EmitOMPTeamsGenericLoopDirective(const OMPTeamsGenericLoopDirective &S)
void EmitOMPTargetSimdDirective(const OMPLoopDirective &S)
Also emits 'target loop'.
void EmitOMPOrderedStandaloneDirective(const OMPOrderedStandaloneDirective &S)
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
Definition CGDecl.cpp:217
bool EmitOMPLinearClauseInit(const OMPLoopDirective &D)
Emit initial code for linear variables.
static void EmitOMPTargetParallelGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelGenericLoopDirective &S)
Emit device code for the target parallel loop directive.
void EmitOMPUnrollDirective(const OMPUnrollDirective &S)
void EmitOMPStripeDirective(const OMPStripeDirective &S)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
void EmitOMPSingleDirective(const OMPSingleDirective &S)
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
llvm::function_ref< void(CodeGenFunction &, SourceLocation, const unsigned, const bool)> CodeGenOrderedTy
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation.
Definition CGExpr.cpp:2307
llvm::Function * GenerateCapturedStmtFunction(const CapturedStmt &S)
Creates the outlined function for a CapturedStmt.
Definition CGStmt.cpp:3483
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
uint64_t getProfileCount(const Stmt *S)
Get the profiler's count for the given statement.
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant,...
static void EmitOMPTargetTeamsGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsGenericLoopDirective &S)
Emit device code for the target teams loop directive.
void EmitOMPTileDirective(const OMPTileDirective &S)
void EmitDecl(const Decl &D, bool EvaluateConditionDecl=false)
EmitDecl - Emit a declaration.
Definition CGDecl.cpp:52
void EmitOMPAtomicDirective(const OMPAtomicDirective &S)
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
ConstantEmission tryEmitAsConstant(const DeclRefExpr *RefExpr)
Try to emit a reference to the given value without producing it as an l-value.
Definition CGExpr.cpp:1969
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
Definition CGExpr.cpp:1789
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
Store of global named registers are always calls to intrinsics.
Definition CGExpr.cpp:3263
void EmitOMPParallelMasterTaskLoopDirective(const OMPParallelMasterTaskLoopDirective &S)
void EmitOMPDistributeParallelForSimdDirective(const OMPDistributeParallelForSimdDirective &S)
void EmitOMPSectionDirective(const OMPSectionDirective &S)
void EnsureInsertPoint()
EnsureInsertPoint - Ensure that an insertion point is defined so that emitted IR has a place to go.
void EmitOMPForSimdDirective(const OMPForSimdDirective &S)
llvm::LLVMContext & getLLVMContext()
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler's counter for the given statement by StepV.
void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty, SourceLocation Loc, SourceLocation AssumptionLoc, llvm::Value *Alignment, llvm::Value *OffsetValue=nullptr)
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
llvm::function_ref< void(CodeGenFunction &, const OMPLoopDirective &, JumpDest)> CodeGenLoopTy
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
bool isTrivialInitializer(const Expr *Init)
Determine whether the given initializer is trivial in the sense that it requires no code to be genera...
Definition CGDecl.cpp:1867
void EmitOMPParallelMasterDirective(const OMPParallelMasterDirective &S)
void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion, const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen, OMPTaskDataTy &Data)
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
void EmitOMPForDirective(const OMPForDirective &S)
void EmitOMPLinearClauseFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
Emit final code for linear clauses.
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
Definition CGStmt.cpp:658
void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit)
EmitExprAsInit - Emits the code necessary to initialize a location in memory with the given initializ...
Definition CGDecl.cpp:2177
void EmitOMPSimdFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
This class organizes the cross-function state that is used while generating LLVM code.
void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
llvm::Module & getModule() const
DiagnosticsEngine & getDiags() const
const LangOptions & getLangOpts() const
const llvm::DataLayout & getDataLayout() const
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
const llvm::Triple & getTriple() const
ASTContext & getContext() const
const CodeGenOptions & getCodeGenOpts() const
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
Definition CGCall.cpp:2098
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
Definition CGCall.cpp:774
const CGFunctionInfo & arrangeDeviceKernelCallerDeclaration(QualType resultType, const FunctionArgList &args)
A device kernel caller function is an offload device entry point function with a target device depend...
Definition CGCall.cpp:790
FunctionArgList - Type for representing both the decl and type of parameters to a function.
Definition CGCall.h:378
LValue - This represents an lvalue references.
Definition CGValue.h:183
llvm::Value * getPointer(CodeGenFunction &CGF) const
Address getAddress() const
Definition CGValue.h:373
QualType getType() const
Definition CGValue.h:303
void setAddress(Address address)
Definition CGValue.h:375
A stack of loop information corresponding to loop nesting levels.
Definition CGLoopInfo.h:158
void setVectorizeWidth(unsigned W)
Set the vectorize width for the next loop pushed.
Definition CGLoopInfo.h:228
void setParallel(bool Enable=true)
Set the next pushed loop as parallel.
Definition CGLoopInfo.h:193
void push(llvm::BasicBlock *Header, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
Begin a new structured loop.
void setVectorizeEnable(bool Enable=true)
Set the next pushed loop 'vectorize.enable'.
Definition CGLoopInfo.h:196
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
virtual void Enter(CodeGenFunction &CGF)
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
Definition CGValue.h:42
bool isScalar() const
Definition CGValue.h:64
static RValue get(llvm::Value *V)
Definition CGValue.h:99
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
Definition CGValue.h:109
bool isAggregate() const
Definition CGValue.h:66
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
Definition CGValue.h:72
bool isComplex() const
Definition CGValue.h:65
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
Definition CGValue.h:79
An abstract representation of an aligned address.
Definition Address.h:42
llvm::PointerType * getType() const
Return the type of the pointer value.
Definition Address.h:72
llvm::Value * getPointer() const
Definition Address.h:66
Class intended to support codegen of all kind of the reduction clauses.
LValue getSharedLValue(unsigned N) const
Returns LValue for the reduction item.
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
const VarDecl * getBaseDecl(unsigned N) const
Returns the base declaration of the reduction item.
void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N)
Emits lvalue for the shared and original reduction item.
void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, llvm::function_ref< bool(CodeGenFunction &)> DefaultInit)
Performs initialization of the private copy for the reduction item.
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations.
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
void setAction(PrePostActionTy &Action) const
Complex values, per C99 6.2.5p11.
Definition TypeBase.h:3362
CompoundStmt - This represents a group of statements like { stmt stmt }.
Definition Stmt.h:1751
ConstStmtVisitor - This class implements a simple visitor for Stmt subclasses.
DeclContext * getParent()
getParent - Returns the containing DeclContext.
Definition DeclBase.h:2106
A reference to a declared variable, function, enum, etc.
Definition Expr.h:1315
static DeclRefExpr * Create(const ASTContext &Context, NestedNameSpecifierLoc QualifierLoc, SourceLocation TemplateKWLoc, ValueDecl *D, bool RefersToEnclosingVariableOrCapture, SourceLocation NameLoc, QualType T, ExprValueKind VK, NamedDecl *FoundD=nullptr, const TemplateArgumentListInfo *TemplateArgs=nullptr, NonOdrUseReason NOUR=NOUR_None)
Definition Expr.cpp:498
ValueDecl * getDecl()
Definition Expr.h:1383
DeclStmt - Adaptor class for mixing declarations with statements and expressions.
Definition Stmt.h:1642
decl_range decls()
Definition Stmt.h:1690
Decl - This represents one declaration (or definition), e.g.
Definition DeclBase.h:86
T * getAttr() const
Definition DeclBase.h:582
static Decl * castFromDeclContext(const DeclContext *)
SourceLocation getBodyRBrace() const
getBodyRBrace - Gets the right brace of the body, if a body exists.
virtual bool hasBody() const
Returns true if this Decl represents a declaration for a body of code, such as a function or method d...
Definition DeclBase.h:1111
SourceLocation getLocation() const
Definition DeclBase.h:448
bool hasAttr() const
Definition DeclBase.h:586
The name of a declaration.
SourceLocation getBeginLoc() const LLVM_READONLY
Definition Decl.h:833
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
This represents one expression.
Definition Expr.h:114
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer,...
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
Definition Expr.cpp:3129
Expr * IgnoreImplicitAsWritten() LLVM_READONLY
Skip past any implicit AST nodes which might surround this expression until reaching a fixed point.
Definition Expr.cpp:3121
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
Definition Expr.cpp:3728
Expr * IgnoreImpCasts() LLVM_READONLY
Skip past any implicit casts which might surround this expression until reaching a fixed point.
Definition Expr.cpp:3109
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Definition Expr.cpp:283
QualType getType() const
Definition Expr.h:146
Represents difference between two FPOptions values.
Represents a member of a struct/union/class.
Definition Decl.h:3341
Represents a function declaration or definition.
Definition Decl.h:2082
static FunctionDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation NLoc, DeclarationName N, QualType T, TypeSourceInfo *TInfo, StorageClass SC, bool UsesFPIntrin=false, bool isInlineSpecified=false, bool hasWrittenPrototype=true, ConstexprSpecKind ConstexprKind=ConstexprSpecKind::Unspecified, const AssociatedConstraint &TrailingRequiresClause={})
Definition Decl.h:2345
GlobalDecl - represents a global declaration.
Definition GlobalDecl.h:60
One of these records is kept for each identifier that is lexed.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
static ImplicitCastExpr * Create(const ASTContext &Context, QualType T, CastKind Kind, Expr *Operand, const CXXCastPath *BasePath, ExprValueKind Cat, FPOptionsOverride FPO)
Definition Expr.cpp:2109
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, const IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
Definition Decl.cpp:5710
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
Represents a point when we exit a loop.
IdentifierInfo * getIdentifier() const
Get the identifier that names this declaration, if there is one.
Definition Decl.h:297
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
Definition Decl.h:303
A C++ nested-name-specifier augmented with source location information.
This is a basic class for representing single OpenMP clause.
This represents 'final' clause in the 'pragma omp ...' directive.
Representation of the 'full' clause of the 'pragma omp unroll' directive.
This represents 'if' clause in the 'pragma omp ...' directive.
This represents 'num_threads' clause in the 'pragma omp ...' directive.
Representation of the 'partial' clause of the 'pragma omp unroll' directive.
This represents 'safelen' clause in the 'pragma omp ...' directive.
This represents 'simdlen' clause in the 'pragma omp ...' directive.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class.
Definition Expr.h:1223
static ParmVarDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, const IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, StorageClass S, Expr *DefArg)
Definition Decl.cpp:2963
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition TypeBase.h:3403
Represents an unpacked "presumed" location which can be presented to the user.
const char * getFilename() const
Return the presumed filename of this location.
unsigned getLine() const
Return the presumed line number of this location.
If a crash happens while one of these objects are live, the message is printed out along with the spe...
A (possibly-)qualified type.
Definition TypeBase.h:938
bool isConstant(const ASTContext &Ctx) const
Definition TypeBase.h:1098
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Definition TypeBase.h:8763
Represents a struct/union/class.
Definition Decl.h:4506
unsigned getNumFields() const
Returns the number of fields (non-static data members) in this record.
Definition Decl.h:4722
field_range fields() const
Definition Decl.h:4709
field_iterator field_begin() const
Definition Decl.cpp:5384
Base for LValueReferenceType and RValueReferenceType.
Definition TypeBase.h:3678
Scope - A scope is a transient data structure that is used while parsing the program.
Definition Scope.h:41
Encodes a location in the source.
A trivial tuple used to represent a source range.
Stmt - This represents one statement.
Definition Stmt.h:85
child_range children()
Definition Stmt.cpp:304
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
Definition Stmt.cpp:343
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top,...
Definition Stmt.cpp:210
SourceLocation getBeginLoc() const LLVM_READONLY
Definition Stmt.cpp:355
bool isArrayType() const
Definition TypeBase.h:8914
bool isPointerType() const
Definition TypeBase.h:8815
const T * castAs() const
Member-template castAs<specific type>.
Definition TypeBase.h:9498
bool isReferenceType() const
Definition TypeBase.h:8839
bool isLValueReferenceType() const
Definition TypeBase.h:8843
bool isAnyComplexType() const
Definition TypeBase.h:8950
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g....
Definition Type.cpp:2438
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
Definition TypeBase.h:2881
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type.
Definition TypeBase.h:9484
const T * getAs() const
Member-template getAs<specific type>'.
Definition TypeBase.h:9431
static UnaryOperator * Create(const ASTContext &C, Expr *input, Opcode opc, QualType type, ExprValueKind VK, ExprObjectKind OK, SourceLocation l, bool CanOverflow, FPOptionsOverride FPFeatures)
Definition Expr.cpp:5201
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Definition Decl.h:714
QualType getType() const
Definition Decl.h:725
Represents a variable declaration or definition.
Definition Decl.h:934
TLSKind getTLSKind() const
Definition Decl.cpp:2149
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
Definition Decl.cpp:2238
@ CInit
C-style initialization with assignment.
Definition Decl.h:939
bool hasGlobalStorage() const
Returns true for all variables that do not have local storage.
Definition Decl.h:1249
bool isStaticLocal() const
Returns true if a variable with function scope is a static local variable.
Definition Decl.h:1216
const Expr * getInit() const
Definition Decl.h:1393
bool hasLocalStorage() const
Returns true if a variable with function scope is a non-static local variable.
Definition Decl.h:1192
@ TLS_None
Not a TLS variable.
Definition Decl.h:954
Represents a C array with a specified size that is not an integer-constant-expression.
Definition TypeBase.h:4064
Expr * getSizeExpr() const
Definition TypeBase.h:4078
Definition SPIR.cpp:35
@ Type
The l-value was considered opaque, so the alignment was determined from a type.
Definition CGValue.h:155
@ Decl
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
Definition CGValue.h:146
@ Address
A pointer to a ValueDecl.
Definition Primitives.h:28
bool Inc(InterpState &S, CodePtr OpPC, bool CanOverflow)
1) Pops a pointer from the stack 2) Load the value from the pointer 3) Writes the value increased by ...
Definition Interp.h:960
CharSourceRange getSourceRange(const SourceRange &Range)
Returns the token CharSourceRange corresponding to Range.
Definition FixIt.h:32
Top level wrappers for InstallAPI frontend operations.
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
CanQual< Type > CanQualType
Represents a canonical, potentially-qualified type.
bool needsTaskBasedThreadLimit(OpenMPDirectiveKind DKind)
Checks if the specified target directive, combined or not, needs task based thread_limit.
Privates[]
This class represents the 'transparent' clause in the 'pragma omp task' directive.
bool isa(CodeGen::Address addr)
Definition Address.h:330
if(T->getSizeExpr()) TRY_TO(TraverseStmt(const_cast< Expr * >(T -> getSizeExpr())))
bool isOpenMPLoopTransformationDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a loop transformation directive.
@ OK_Ordinary
An ordinary object is located at an address in memory.
Definition Specifiers.h:155
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
@ Tile
'tile' clause, allowed on 'loop' and Combined constructs.
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
Definition OpenMPKinds.h:39
@ OMPC_SCHEDULE_MODIFIER_unknown
Definition OpenMPKinds.h:40
@ CR_OpenMP
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
@ SC_Static
Definition Specifiers.h:256
@ SC_None
Definition Specifiers.h:254
OpenMPDistScheduleClauseKind
OpenMP attributes for 'dist_schedule' clause.
@ OMPC_DIST_SCHEDULE_unknown
bool isOpenMPTaskingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of tasking directives - task or any taskloop-based dire...
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
@ Result
The result type of a method or function.
Definition TypeBase.h:906
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
bool isOpenMPGenericLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive constitutes a 'loop' directive in the outermost nest.
OpenMPBindClauseKind
OpenMP bindings for the 'bind' clause.
@ OMPC_BIND_unknown
const FunctionProtoType * T
OpenMPDependClauseKind
OpenMP attributes for 'depend' clause.
Definition OpenMPKinds.h:55
OpenMPSeverityClauseKind
OpenMP attributes for 'severity' clause.
bool isOpenMPLoopBoundSharingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of the composite or combined directives that need loop ...
llvm::omp::Directive OpenMPDirectiveKind
OpenMP directives.
Definition OpenMPKinds.h:25
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
@ VK_PRValue
A pr-value expression (in the C++11 taxonomy) produces a temporary value.
Definition Specifiers.h:139
@ VK_LValue
An l-value expression is a reference to an object with independent storage.
Definition Specifiers.h:143
void getOpenMPCaptureRegions(llvm::SmallVectorImpl< OpenMPDirectiveKind > &CaptureRegions, OpenMPDirectiveKind DKind)
Return the captured regions of an OpenMP directive.
OpenMPNumThreadsClauseModifier
@ OMPC_NUMTHREADS_unknown
U cast(CodeGen::Address addr)
Definition Address.h:327
@ OMPC_DEVICE_unknown
Definition OpenMPKinds.h:51
llvm::omp::Clause OpenMPClauseKind
OpenMP clauses.
Definition OpenMPKinds.h:28
@ ThreadPrivateVar
Parameter for Thread private variable.
Definition Decl.h:1784
@ Other
Other implicit parameter.
Definition Decl.h:1787
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
Definition OpenMPKinds.h:31
@ OMPC_SCHEDULE_unknown
Definition OpenMPKinds.h:35
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
__builtin_elementwise_add_sat __builtin_elementwise_sub_sat uint32_t uint64_t __builtin_riscv_ssha_i32 __builtin_riscv_sshl_u32 uint64_t
#define true
Definition stdbool.h:25
unsigned ignore_denormal_mode
Struct with the values to be passed to the static runtime function.
QualType getType() const
Definition CGCall.h:251
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
static Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc)
Returns address of the threadprivate variable for the current thread.
llvm::OpenMPIRBuilder::InsertPointTy InsertPointTy
static void EmitOMPOutlinedRegionBody(CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Twine RegionName)
Emit the body of an OMP region that will be outlined in OpenMPIRBuilder::finalize().
static Address getAddressOfLocalVariable(CodeGenFunction &CGF, const VarDecl *VD)
Gets the OpenMP-specific address of the local variable /p VD.
static std::string getNameWithSeparators(ArrayRef< StringRef > Parts, StringRef FirstSeparator=".", StringRef Separator=".")
Get the platform-specific name separator.
static void FinalizeOMPRegion(CodeGenFunction &CGF, InsertPointTy IP)
Emit the Finalization for an OMP region.
static void EmitOMPInlinedRegionBody(CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Twine RegionName)
Emit the body of an OMP region.
static void EmitCaptureStmt(CodeGenFunction &CGF, llvm::BasicBlock *CodeGenIPBB, llvm::BasicBlock &FiniBB, llvm::Function *Fn, ArrayRef< llvm::Value * > Args)
SmallVector< const Expr *, 4 > DepExprs
std::string getAsString() const
getAsString - Retrieve the human-readable string for this name.
EvalResult is a struct with detailed info about an evaluated expression.
Definition Expr.h:667
Extra information about a function prototype.
Definition TypeBase.h:5633
Scheduling data for loop-based OpenMP directives.
bool UseFusedDistChunkSchedule
Request the fused distr_static_chunk + static_chunkone runtime schedule in for_static_init.
OpenMPScheduleClauseModifier M2
OpenMPScheduleClauseModifier M1
OpenMPScheduleClauseKind Schedule