clang 19.0.0git
CGStmtOpenMP.cpp
Go to the documentation of this file.
1//===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit OpenMP nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCleanup.h"
14#include "CGOpenMPRuntime.h"
15#include "CodeGenFunction.h"
16#include "CodeGenModule.h"
17#include "TargetInfo.h"
19#include "clang/AST/Attr.h"
22#include "clang/AST/Stmt.h"
27#include "llvm/ADT/SmallSet.h"
28#include "llvm/BinaryFormat/Dwarf.h"
29#include "llvm/Frontend/OpenMP/OMPConstants.h"
30#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
31#include "llvm/IR/Constants.h"
32#include "llvm/IR/DebugInfoMetadata.h"
33#include "llvm/IR/Instructions.h"
34#include "llvm/IR/IntrinsicInst.h"
35#include "llvm/IR/Metadata.h"
36#include "llvm/Support/AtomicOrdering.h"
37#include <optional>
38using namespace clang;
39using namespace CodeGen;
40using namespace llvm::omp;
41
42static const VarDecl *getBaseDecl(const Expr *Ref);
43
44namespace {
45/// Lexical scope for OpenMP executable constructs, that handles correct codegen
46/// for captured expressions.
47class OMPLexicalScope : public CodeGenFunction::LexicalScope {
48 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
49 for (const auto *C : S.clauses()) {
50 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
51 if (const auto *PreInit =
52 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
53 for (const auto *I : PreInit->decls()) {
54 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
55 CGF.EmitVarDecl(cast<VarDecl>(*I));
56 } else {
57 CodeGenFunction::AutoVarEmission Emission =
58 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
59 CGF.EmitAutoVarCleanups(Emission);
60 }
61 }
62 }
63 }
64 }
65 }
66 CodeGenFunction::OMPPrivateScope InlinedShareds;
67
68 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
69 return CGF.LambdaCaptureFields.lookup(VD) ||
70 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
71 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) &&
72 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
73 }
74
75public:
76 OMPLexicalScope(
78 const std::optional<OpenMPDirectiveKind> CapturedRegion = std::nullopt,
79 const bool EmitPreInitStmt = true)
80 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
81 InlinedShareds(CGF) {
82 if (EmitPreInitStmt)
83 emitPreInitStmt(CGF, S);
84 if (!CapturedRegion)
85 return;
86 assert(S.hasAssociatedStmt() &&
87 "Expected associated statement for inlined directive.");
88 const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
89 for (const auto &C : CS->captures()) {
90 if (C.capturesVariable() || C.capturesVariableByCopy()) {
91 auto *VD = C.getCapturedVar();
92 assert(VD == VD->getCanonicalDecl() &&
93 "Canonical decl must be captured.");
94 DeclRefExpr DRE(
95 CGF.getContext(), const_cast<VarDecl *>(VD),
96 isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
97 InlinedShareds.isGlobalVarCaptured(VD)),
98 VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
99 InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress(CGF));
100 }
101 }
102 (void)InlinedShareds.Privatize();
103 }
104};
105
106/// Lexical scope for OpenMP parallel construct, that handles correct codegen
107/// for captured expressions.
108class OMPParallelScope final : public OMPLexicalScope {
109 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
110 OpenMPDirectiveKind Kind = S.getDirectiveKind();
111 return !(isOpenMPTargetExecutionDirective(Kind) ||
114 }
115
116public:
117 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
118 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt,
119 EmitPreInitStmt(S)) {}
120};
121
122/// Lexical scope for OpenMP teams construct, that handles correct codegen
123/// for captured expressions.
124class OMPTeamsScope final : public OMPLexicalScope {
125 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
126 OpenMPDirectiveKind Kind = S.getDirectiveKind();
127 return !isOpenMPTargetExecutionDirective(Kind) &&
129 }
130
131public:
132 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
133 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt,
134 EmitPreInitStmt(S)) {}
135};
136
137/// Private scope for OpenMP loop-based directives, that supports capturing
138/// of used expression from loop statement.
139class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
140 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopBasedDirective &S) {
141 const DeclStmt *PreInits;
142 CodeGenFunction::OMPMapVars PreCondVars;
143 if (auto *LD = dyn_cast<OMPLoopDirective>(&S)) {
144 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
145 for (const auto *E : LD->counters()) {
146 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
147 EmittedAsPrivate.insert(VD->getCanonicalDecl());
148 (void)PreCondVars.setVarAddr(
149 CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
150 }
151 // Mark private vars as undefs.
152 for (const auto *C : LD->getClausesOfKind<OMPPrivateClause>()) {
153 for (const Expr *IRef : C->varlists()) {
154 const auto *OrigVD =
155 cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
156 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
157 QualType OrigVDTy = OrigVD->getType().getNonReferenceType();
158 (void)PreCondVars.setVarAddr(
159 CGF, OrigVD,
160 Address(llvm::UndefValue::get(CGF.ConvertTypeForMem(
161 CGF.getContext().getPointerType(OrigVDTy))),
162 CGF.ConvertTypeForMem(OrigVDTy),
163 CGF.getContext().getDeclAlign(OrigVD)));
164 }
165 }
166 }
167 (void)PreCondVars.apply(CGF);
168 // Emit init, __range and __end variables for C++ range loops.
170 LD->getInnermostCapturedStmt()->getCapturedStmt(),
171 /*TryImperfectlyNestedLoops=*/true, LD->getLoopsNumber(),
172 [&CGF](unsigned Cnt, const Stmt *CurStmt) {
173 if (const auto *CXXFor = dyn_cast<CXXForRangeStmt>(CurStmt)) {
174 if (const Stmt *Init = CXXFor->getInit())
175 CGF.EmitStmt(Init);
176 CGF.EmitStmt(CXXFor->getRangeStmt());
177 CGF.EmitStmt(CXXFor->getEndStmt());
178 }
179 return false;
180 });
181 PreInits = cast_or_null<DeclStmt>(LD->getPreInits());
182 } else if (const auto *Tile = dyn_cast<OMPTileDirective>(&S)) {
183 PreInits = cast_or_null<DeclStmt>(Tile->getPreInits());
184 } else if (const auto *Unroll = dyn_cast<OMPUnrollDirective>(&S)) {
185 PreInits = cast_or_null<DeclStmt>(Unroll->getPreInits());
186 } else {
187 llvm_unreachable("Unknown loop-based directive kind.");
188 }
189 if (PreInits) {
190 for (const auto *I : PreInits->decls())
191 CGF.EmitVarDecl(cast<VarDecl>(*I));
192 }
193 PreCondVars.restore(CGF);
194 }
195
196public:
197 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopBasedDirective &S)
198 : CodeGenFunction::RunCleanupsScope(CGF) {
199 emitPreInitStmt(CGF, S);
200 }
201};
202
203class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
204 CodeGenFunction::OMPPrivateScope InlinedShareds;
205
206 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
207 return CGF.LambdaCaptureFields.lookup(VD) ||
208 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
209 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) &&
210 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
211 }
212
213public:
214 OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
215 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
216 InlinedShareds(CGF) {
217 for (const auto *C : S.clauses()) {
218 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
219 if (const auto *PreInit =
220 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
221 for (const auto *I : PreInit->decls()) {
222 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
223 CGF.EmitVarDecl(cast<VarDecl>(*I));
224 } else {
225 CodeGenFunction::AutoVarEmission Emission =
226 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
227 CGF.EmitAutoVarCleanups(Emission);
228 }
229 }
230 }
231 } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
232 for (const Expr *E : UDP->varlists()) {
233 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
234 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
235 CGF.EmitVarDecl(*OED);
236 }
237 } else if (const auto *UDP = dyn_cast<OMPUseDeviceAddrClause>(C)) {
238 for (const Expr *E : UDP->varlists()) {
239 const Decl *D = getBaseDecl(E);
240 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
241 CGF.EmitVarDecl(*OED);
242 }
243 }
244 }
245 if (!isOpenMPSimdDirective(S.getDirectiveKind()))
246 CGF.EmitOMPPrivateClause(S, InlinedShareds);
247 if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
248 if (const Expr *E = TG->getReductionRef())
249 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
250 }
251 // Temp copy arrays for inscan reductions should not be emitted as they are
252 // not used in simd only mode.
254 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
255 if (C->getModifier() != OMPC_REDUCTION_inscan)
256 continue;
257 for (const Expr *E : C->copy_array_temps())
258 CopyArrayTemps.insert(cast<DeclRefExpr>(E)->getDecl());
259 }
260 const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
261 while (CS) {
262 for (auto &C : CS->captures()) {
263 if (C.capturesVariable() || C.capturesVariableByCopy()) {
264 auto *VD = C.getCapturedVar();
265 if (CopyArrayTemps.contains(VD))
266 continue;
267 assert(VD == VD->getCanonicalDecl() &&
268 "Canonical decl must be captured.");
269 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
270 isCapturedVar(CGF, VD) ||
271 (CGF.CapturedStmtInfo &&
272 InlinedShareds.isGlobalVarCaptured(VD)),
274 C.getLocation());
275 InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress(CGF));
276 }
277 }
278 CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
279 }
280 (void)InlinedShareds.Privatize();
281 }
282};
283
284} // namespace
285
287 const OMPExecutableDirective &S,
288 const RegionCodeGenTy &CodeGen);
289
290LValue CodeGenFunction::EmitOMPSharedLValue(const Expr *E) {
291 if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
292 if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
293 OrigVD = OrigVD->getCanonicalDecl();
294 bool IsCaptured =
295 LambdaCaptureFields.lookup(OrigVD) ||
297 (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
298 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), IsCaptured,
299 OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
300 return EmitLValue(&DRE);
301 }
302 }
303 return EmitLValue(E);
304}
305
306llvm::Value *CodeGenFunction::getTypeSize(QualType Ty) {
308 llvm::Value *Size = nullptr;
309 auto SizeInChars = C.getTypeSizeInChars(Ty);
310 if (SizeInChars.isZero()) {
311 // getTypeSizeInChars() returns 0 for a VLA.
312 while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) {
313 VlaSizePair VlaSize = getVLASize(VAT);
314 Ty = VlaSize.Type;
315 Size =
316 Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts) : VlaSize.NumElts;
317 }
318 SizeInChars = C.getTypeSizeInChars(Ty);
319 if (SizeInChars.isZero())
320 return llvm::ConstantInt::get(SizeTy, /*V=*/0);
321 return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
322 }
323 return CGM.getSize(SizeInChars);
324}
325
327 const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
328 const RecordDecl *RD = S.getCapturedRecordDecl();
329 auto CurField = RD->field_begin();
330 auto CurCap = S.captures().begin();
331 for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
332 E = S.capture_init_end();
333 I != E; ++I, ++CurField, ++CurCap) {
334 if (CurField->hasCapturedVLAType()) {
335 const VariableArrayType *VAT = CurField->getCapturedVLAType();
336 llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
337 CapturedVars.push_back(Val);
338 } else if (CurCap->capturesThis()) {
339 CapturedVars.push_back(CXXThisValue);
340 } else if (CurCap->capturesVariableByCopy()) {
341 llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
342
343 // If the field is not a pointer, we need to save the actual value
344 // and load it as a void pointer.
345 if (!CurField->getType()->isAnyPointerType()) {
346 ASTContext &Ctx = getContext();
347 Address DstAddr = CreateMemTemp(
348 Ctx.getUIntPtrType(),
349 Twine(CurCap->getCapturedVar()->getName(), ".casted"));
350 LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
351
352 llvm::Value *SrcAddrVal = EmitScalarConversion(
353 DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
354 Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
355 LValue SrcLV =
356 MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
357
358 // Store the value using the source type pointer.
360
361 // Load the value using the destination type pointer.
362 CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
363 }
364 CapturedVars.push_back(CV);
365 } else {
366 assert(CurCap->capturesVariable() && "Expected capture by reference.");
367 CapturedVars.push_back(EmitLValue(*I).getAddress(*this).getPointer());
368 }
369 }
370}
371
373 QualType DstType, StringRef Name,
374 LValue AddrLV) {
375 ASTContext &Ctx = CGF.getContext();
376
377 llvm::Value *CastedPtr = CGF.EmitScalarConversion(
378 AddrLV.getAddress(CGF).getPointer(), Ctx.getUIntPtrType(),
379 Ctx.getPointerType(DstType), Loc);
380 Address TmpAddr =
381 CGF.MakeNaturalAlignAddrLValue(CastedPtr, DstType).getAddress(CGF);
382 return TmpAddr;
383}
384
386 if (T->isLValueReferenceType())
387 return C.getLValueReferenceType(
389 /*SpelledAsLValue=*/false);
390 if (T->isPointerType())
391 return C.getPointerType(getCanonicalParamType(C, T->getPointeeType()));
392 if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
393 if (const auto *VLA = dyn_cast<VariableArrayType>(A))
394 return getCanonicalParamType(C, VLA->getElementType());
395 if (!A->isVariablyModifiedType())
396 return C.getCanonicalType(T);
397 }
398 return C.getCanonicalParamType(T);
399}
400
401namespace {
402/// Contains required data for proper outlined function codegen.
403struct FunctionOptions {
404 /// Captured statement for which the function is generated.
405 const CapturedStmt *S = nullptr;
406 /// true if cast to/from UIntPtr is required for variables captured by
407 /// value.
408 const bool UIntPtrCastRequired = true;
409 /// true if only casted arguments must be registered as local args or VLA
410 /// sizes.
411 const bool RegisterCastedArgsOnly = false;
412 /// Name of the generated function.
413 const StringRef FunctionName;
414 /// Location of the non-debug version of the outlined function.
415 SourceLocation Loc;
416 explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
417 bool RegisterCastedArgsOnly, StringRef FunctionName,
418 SourceLocation Loc)
419 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
420 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
421 FunctionName(FunctionName), Loc(Loc) {}
422};
423} // namespace
424
425static llvm::Function *emitOutlinedFunctionPrologue(
427 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
428 &LocalAddrs,
429 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
430 &VLASizes,
431 llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
432 const CapturedDecl *CD = FO.S->getCapturedDecl();
433 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
434 assert(CD->hasBody() && "missing CapturedDecl body");
435
436 CXXThisValue = nullptr;
437 // Build the argument list.
438 CodeGenModule &CGM = CGF.CGM;
439 ASTContext &Ctx = CGM.getContext();
440 FunctionArgList TargetArgs;
441 Args.append(CD->param_begin(),
442 std::next(CD->param_begin(), CD->getContextParamPosition()));
443 TargetArgs.append(
444 CD->param_begin(),
445 std::next(CD->param_begin(), CD->getContextParamPosition()));
446 auto I = FO.S->captures().begin();
447 FunctionDecl *DebugFunctionDecl = nullptr;
448 if (!FO.UIntPtrCastRequired) {
450 QualType FunctionTy = Ctx.getFunctionType(Ctx.VoidTy, std::nullopt, EPI);
451 DebugFunctionDecl = FunctionDecl::Create(
452 Ctx, Ctx.getTranslationUnitDecl(), FO.S->getBeginLoc(),
453 SourceLocation(), DeclarationName(), FunctionTy,
454 Ctx.getTrivialTypeSourceInfo(FunctionTy), SC_Static,
455 /*UsesFPIntrin=*/false, /*isInlineSpecified=*/false,
456 /*hasWrittenPrototype=*/false);
457 }
458 for (const FieldDecl *FD : RD->fields()) {
459 QualType ArgType = FD->getType();
460 IdentifierInfo *II = nullptr;
461 VarDecl *CapVar = nullptr;
462
463 // If this is a capture by copy and the type is not a pointer, the outlined
464 // function argument type should be uintptr and the value properly casted to
465 // uintptr. This is necessary given that the runtime library is only able to
466 // deal with pointers. We can pass in the same way the VLA type sizes to the
467 // outlined function.
468 if (FO.UIntPtrCastRequired &&
469 ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
470 I->capturesVariableArrayType()))
471 ArgType = Ctx.getUIntPtrType();
472
473 if (I->capturesVariable() || I->capturesVariableByCopy()) {
474 CapVar = I->getCapturedVar();
475 II = CapVar->getIdentifier();
476 } else if (I->capturesThis()) {
477 II = &Ctx.Idents.get("this");
478 } else {
479 assert(I->capturesVariableArrayType());
480 II = &Ctx.Idents.get("vla");
481 }
482 if (ArgType->isVariablyModifiedType())
483 ArgType = getCanonicalParamType(Ctx, ArgType);
484 VarDecl *Arg;
485 if (CapVar && (CapVar->getTLSKind() != clang::VarDecl::TLS_None)) {
486 Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
487 II, ArgType,
489 } else if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
491 Ctx, DebugFunctionDecl,
492 CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),
493 CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
494 /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
495 } else {
496 Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
497 II, ArgType, ImplicitParamKind::Other);
498 }
499 Args.emplace_back(Arg);
500 // Do not cast arguments if we emit function with non-original types.
501 TargetArgs.emplace_back(
502 FO.UIntPtrCastRequired
503 ? Arg
504 : CGM.getOpenMPRuntime().translateParameter(FD, Arg));
505 ++I;
506 }
507 Args.append(std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
508 CD->param_end());
509 TargetArgs.append(
510 std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
511 CD->param_end());
512
513 // Create the function declaration.
514 const CGFunctionInfo &FuncInfo =
516 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
517
518 auto *F =
519 llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
520 FO.FunctionName, &CGM.getModule());
521 CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
522 if (CD->isNothrow())
523 F->setDoesNotThrow();
524 F->setDoesNotRecurse();
525
526 // Always inline the outlined function if optimizations are enabled.
527 if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
528 F->removeFnAttr(llvm::Attribute::NoInline);
529 F->addFnAttr(llvm::Attribute::AlwaysInline);
530 }
531
532 // Generate the function.
533 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
534 FO.UIntPtrCastRequired ? FO.Loc : FO.S->getBeginLoc(),
535 FO.UIntPtrCastRequired ? FO.Loc
536 : CD->getBody()->getBeginLoc());
537 unsigned Cnt = CD->getContextParamPosition();
538 I = FO.S->captures().begin();
539 for (const FieldDecl *FD : RD->fields()) {
540 // Do not map arguments if we emit function with non-original types.
541 Address LocalAddr(Address::invalid());
542 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
543 LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
544 TargetArgs[Cnt]);
545 } else {
546 LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
547 }
548 // If we are capturing a pointer by copy we don't need to do anything, just
549 // use the value that we get from the arguments.
550 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
551 const VarDecl *CurVD = I->getCapturedVar();
552 if (!FO.RegisterCastedArgsOnly)
553 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
554 ++Cnt;
555 ++I;
556 continue;
557 }
558
559 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
561 if (FD->hasCapturedVLAType()) {
562 if (FO.UIntPtrCastRequired) {
563 ArgLVal = CGF.MakeAddrLValue(
564 castValueFromUintptr(CGF, I->getLocation(), FD->getType(),
565 Args[Cnt]->getName(), ArgLVal),
567 }
568 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
569 const VariableArrayType *VAT = FD->getCapturedVLAType();
570 VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg);
571 } else if (I->capturesVariable()) {
572 const VarDecl *Var = I->getCapturedVar();
573 QualType VarTy = Var->getType();
574 Address ArgAddr = ArgLVal.getAddress(CGF);
575 if (ArgLVal.getType()->isLValueReferenceType()) {
576 ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
577 } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) {
578 assert(ArgLVal.getType()->isPointerType());
579 ArgAddr = CGF.EmitLoadOfPointer(
580 ArgAddr, ArgLVal.getType()->castAs<PointerType>());
581 }
582 if (!FO.RegisterCastedArgsOnly) {
583 LocalAddrs.insert(
584 {Args[Cnt], {Var, ArgAddr.withAlignment(Ctx.getDeclAlign(Var))}});
585 }
586 } else if (I->capturesVariableByCopy()) {
587 assert(!FD->getType()->isAnyPointerType() &&
588 "Not expecting a captured pointer.");
589 const VarDecl *Var = I->getCapturedVar();
590 LocalAddrs.insert({Args[Cnt],
591 {Var, FO.UIntPtrCastRequired
593 CGF, I->getLocation(), FD->getType(),
594 Args[Cnt]->getName(), ArgLVal)
595 : ArgLVal.getAddress(CGF)}});
596 } else {
597 // If 'this' is captured, load it into CXXThisValue.
598 assert(I->capturesThis());
599 CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
600 LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress(CGF)}});
601 }
602 ++Cnt;
603 ++I;
604 }
605
606 return F;
607}
608
609llvm::Function *
611 SourceLocation Loc) {
612 assert(
614 "CapturedStmtInfo should be set when generating the captured function");
615 const CapturedDecl *CD = S.getCapturedDecl();
616 // Build the argument list.
617 bool NeedWrapperFunction =
619 FunctionArgList Args;
620 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
621 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
622 SmallString<256> Buffer;
623 llvm::raw_svector_ostream Out(Buffer);
625 if (NeedWrapperFunction)
626 Out << "_debug__";
627 FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
628 Out.str(), Loc);
629 llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs,
630 VLASizes, CXXThisValue, FO);
631 CodeGenFunction::OMPPrivateScope LocalScope(*this);
632 for (const auto &LocalAddrPair : LocalAddrs) {
633 if (LocalAddrPair.second.first) {
634 LocalScope.addPrivate(LocalAddrPair.second.first,
635 LocalAddrPair.second.second);
636 }
637 }
638 (void)LocalScope.Privatize();
639 for (const auto &VLASizePair : VLASizes)
640 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
642 CapturedStmtInfo->EmitBody(*this, CD->getBody());
643 (void)LocalScope.ForceCleanup();
645 if (!NeedWrapperFunction)
646 return F;
647
648 FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
649 /*RegisterCastedArgsOnly=*/true,
651 CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
652 WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
653 Args.clear();
654 LocalAddrs.clear();
655 VLASizes.clear();
656 llvm::Function *WrapperF =
657 emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
658 WrapperCGF.CXXThisValue, WrapperFO);
660 auto *PI = F->arg_begin();
661 for (const auto *Arg : Args) {
662 llvm::Value *CallArg;
663 auto I = LocalAddrs.find(Arg);
664 if (I != LocalAddrs.end()) {
665 LValue LV = WrapperCGF.MakeAddrLValue(
666 I->second.second,
667 I->second.first ? I->second.first->getType() : Arg->getType(),
669 if (LV.getType()->isAnyComplexType())
670 LV.setAddress(LV.getAddress(WrapperCGF).withElementType(PI->getType()));
671 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
672 } else {
673 auto EI = VLASizes.find(Arg);
674 if (EI != VLASizes.end()) {
675 CallArg = EI->second.second;
676 } else {
677 LValue LV =
678 WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
679 Arg->getType(), AlignmentSource::Decl);
680 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
681 }
682 }
683 CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
684 ++PI;
685 }
686 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, Loc, F, CallArgs);
687 WrapperCGF.FinishFunction();
688 return WrapperF;
689}
690
691//===----------------------------------------------------------------------===//
692// OpenMP Directive Emission
693//===----------------------------------------------------------------------===//
695 Address DestAddr, Address SrcAddr, QualType OriginalType,
696 const llvm::function_ref<void(Address, Address)> CopyGen) {
697 // Perform element-by-element initialization.
698 QualType ElementTy;
699
700 // Drill down to the base element type on both arrays.
701 const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
702 llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
703 SrcAddr = SrcAddr.withElementType(DestAddr.getElementType());
704
705 llvm::Value *SrcBegin = SrcAddr.getPointer();
706 llvm::Value *DestBegin = DestAddr.getPointer();
707 // Cast from pointer to array type to pointer to single element.
708 llvm::Value *DestEnd = Builder.CreateInBoundsGEP(DestAddr.getElementType(),
709 DestBegin, NumElements);
710
711 // The basic structure here is a while-do loop.
712 llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body");
713 llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done");
714 llvm::Value *IsEmpty =
715 Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
716 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
717
718 // Enter the loop body, making that address the current address.
719 llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
720 EmitBlock(BodyBB);
721
722 CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
723
724 llvm::PHINode *SrcElementPHI =
725 Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
726 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
727 Address SrcElementCurrent =
728 Address(SrcElementPHI, SrcAddr.getElementType(),
729 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
730
731 llvm::PHINode *DestElementPHI = Builder.CreatePHI(
732 DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
733 DestElementPHI->addIncoming(DestBegin, EntryBB);
734 Address DestElementCurrent =
735 Address(DestElementPHI, DestAddr.getElementType(),
736 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
737
738 // Emit copy.
739 CopyGen(DestElementCurrent, SrcElementCurrent);
740
741 // Shift the address forward by one element.
742 llvm::Value *DestElementNext =
743 Builder.CreateConstGEP1_32(DestAddr.getElementType(), DestElementPHI,
744 /*Idx0=*/1, "omp.arraycpy.dest.element");
745 llvm::Value *SrcElementNext =
746 Builder.CreateConstGEP1_32(SrcAddr.getElementType(), SrcElementPHI,
747 /*Idx0=*/1, "omp.arraycpy.src.element");
748 // Check whether we've reached the end.
749 llvm::Value *Done =
750 Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
751 Builder.CreateCondBr(Done, DoneBB, BodyBB);
752 DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
753 SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
754
755 // Done.
756 EmitBlock(DoneBB, /*IsFinished=*/true);
757}
758
759void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
760 Address SrcAddr, const VarDecl *DestVD,
761 const VarDecl *SrcVD, const Expr *Copy) {
762 if (OriginalType->isArrayType()) {
763 const auto *BO = dyn_cast<BinaryOperator>(Copy);
764 if (BO && BO->getOpcode() == BO_Assign) {
765 // Perform simple memcpy for simple copying.
766 LValue Dest = MakeAddrLValue(DestAddr, OriginalType);
767 LValue Src = MakeAddrLValue(SrcAddr, OriginalType);
768 EmitAggregateAssign(Dest, Src, OriginalType);
769 } else {
770 // For arrays with complex element types perform element by element
771 // copying.
773 DestAddr, SrcAddr, OriginalType,
774 [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
775 // Working with the single array element, so have to remap
776 // destination and source variables to corresponding array
777 // elements.
778 CodeGenFunction::OMPPrivateScope Remap(*this);
779 Remap.addPrivate(DestVD, DestElement);
780 Remap.addPrivate(SrcVD, SrcElement);
781 (void)Remap.Privatize();
783 });
784 }
785 } else {
786 // Remap pseudo source variable to private copy.
787 CodeGenFunction::OMPPrivateScope Remap(*this);
788 Remap.addPrivate(SrcVD, SrcAddr);
789 Remap.addPrivate(DestVD, DestAddr);
790 (void)Remap.Privatize();
791 // Emit copying of the whole variable.
793 }
794}
795
797 OMPPrivateScope &PrivateScope) {
798 if (!HaveInsertPoint())
799 return false;
800 bool DeviceConstTarget =
801 getLangOpts().OpenMPIsTargetDevice &&
803 bool FirstprivateIsLastprivate = false;
804 llvm::DenseMap<const VarDecl *, OpenMPLastprivateModifier> Lastprivates;
805 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
806 for (const auto *D : C->varlists())
807 Lastprivates.try_emplace(
808 cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl(),
809 C->getKind());
810 }
811 llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
813 getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind());
814 // Force emission of the firstprivate copy if the directive does not emit
815 // outlined function, like omp for, omp simd, omp distribute etc.
816 bool MustEmitFirstprivateCopy =
817 CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
818 for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
819 const auto *IRef = C->varlist_begin();
820 const auto *InitsRef = C->inits().begin();
821 for (const Expr *IInit : C->private_copies()) {
822 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
823 bool ThisFirstprivateIsLastprivate =
824 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
825 const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
826 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
827 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
828 !FD->getType()->isReferenceType() &&
829 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
830 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
831 ++IRef;
832 ++InitsRef;
833 continue;
834 }
835 // Do not emit copy for firstprivate constant variables in target regions,
836 // captured by reference.
837 if (DeviceConstTarget && OrigVD->getType().isConstant(getContext()) &&
838 FD && FD->getType()->isReferenceType() &&
839 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
840 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
841 ++IRef;
842 ++InitsRef;
843 continue;
844 }
845 FirstprivateIsLastprivate =
846 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
847 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
848 const auto *VDInit =
849 cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
850 bool IsRegistered;
851 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
852 /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
853 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
854 LValue OriginalLVal;
855 if (!FD) {
856 // Check if the firstprivate variable is just a constant value.
857 ConstantEmission CE = tryEmitAsConstant(&DRE);
858 if (CE && !CE.isReference()) {
859 // Constant value, no need to create a copy.
860 ++IRef;
861 ++InitsRef;
862 continue;
863 }
864 if (CE && CE.isReference()) {
865 OriginalLVal = CE.getReferenceLValue(*this, &DRE);
866 } else {
867 assert(!CE && "Expected non-constant firstprivate.");
868 OriginalLVal = EmitLValue(&DRE);
869 }
870 } else {
871 OriginalLVal = EmitLValue(&DRE);
872 }
873 QualType Type = VD->getType();
874 if (Type->isArrayType()) {
875 // Emit VarDecl with copy init for arrays.
876 // Get the address of the original variable captured in current
877 // captured region.
878 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
879 const Expr *Init = VD->getInit();
880 if (!isa<CXXConstructExpr>(Init) || isTrivialInitializer(Init)) {
881 // Perform simple memcpy.
882 LValue Dest = MakeAddrLValue(Emission.getAllocatedAddress(), Type);
883 EmitAggregateAssign(Dest, OriginalLVal, Type);
884 } else {
886 Emission.getAllocatedAddress(), OriginalLVal.getAddress(*this),
887 Type,
888 [this, VDInit, Init](Address DestElement, Address SrcElement) {
889 // Clean up any temporaries needed by the
890 // initialization.
891 RunCleanupsScope InitScope(*this);
892 // Emit initialization for single element.
893 setAddrOfLocalVar(VDInit, SrcElement);
894 EmitAnyExprToMem(Init, DestElement,
895 Init->getType().getQualifiers(),
896 /*IsInitializer*/ false);
897 LocalDeclMap.erase(VDInit);
898 });
899 }
900 EmitAutoVarCleanups(Emission);
901 IsRegistered =
902 PrivateScope.addPrivate(OrigVD, Emission.getAllocatedAddress());
903 } else {
904 Address OriginalAddr = OriginalLVal.getAddress(*this);
905 // Emit private VarDecl with copy init.
906 // Remap temp VDInit variable to the address of the original
907 // variable (for proper handling of captured global variables).
908 setAddrOfLocalVar(VDInit, OriginalAddr);
909 EmitDecl(*VD);
910 LocalDeclMap.erase(VDInit);
911 Address VDAddr = GetAddrOfLocalVar(VD);
912 if (ThisFirstprivateIsLastprivate &&
913 Lastprivates[OrigVD->getCanonicalDecl()] ==
914 OMPC_LASTPRIVATE_conditional) {
915 // Create/init special variable for lastprivate conditionals.
916 llvm::Value *V =
917 EmitLoadOfScalar(MakeAddrLValue(VDAddr, (*IRef)->getType(),
919 (*IRef)->getExprLoc());
921 *this, OrigVD);
922 EmitStoreOfScalar(V, MakeAddrLValue(VDAddr, (*IRef)->getType(),
924 LocalDeclMap.erase(VD);
925 setAddrOfLocalVar(VD, VDAddr);
926 }
927 IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);
928 }
929 assert(IsRegistered &&
930 "firstprivate var already registered as private");
931 // Silence the warning about unused variable.
932 (void)IsRegistered;
933 }
934 ++IRef;
935 ++InitsRef;
936 }
937 }
938 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
939}
940
942 const OMPExecutableDirective &D,
943 CodeGenFunction::OMPPrivateScope &PrivateScope) {
944 if (!HaveInsertPoint())
945 return;
946 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
947 for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
948 auto IRef = C->varlist_begin();
949 for (const Expr *IInit : C->private_copies()) {
950 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
951 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
952 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
953 EmitDecl(*VD);
954 // Emit private VarDecl with copy init.
955 bool IsRegistered =
956 PrivateScope.addPrivate(OrigVD, GetAddrOfLocalVar(VD));
957 assert(IsRegistered && "private var already registered as private");
958 // Silence the warning about unused variable.
959 (void)IsRegistered;
960 }
961 ++IRef;
962 }
963 }
964}
965
967 if (!HaveInsertPoint())
968 return false;
969 // threadprivate_var1 = master_threadprivate_var1;
970 // operator=(threadprivate_var2, master_threadprivate_var2);
971 // ...
972 // __kmpc_barrier(&loc, global_tid);
974 llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
975 for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
976 auto IRef = C->varlist_begin();
977 auto ISrcRef = C->source_exprs().begin();
978 auto IDestRef = C->destination_exprs().begin();
979 for (const Expr *AssignOp : C->assignment_ops()) {
980 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
981 QualType Type = VD->getType();
982 if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
983 // Get the address of the master variable. If we are emitting code with
984 // TLS support, the address is passed from the master as field in the
985 // captured declaration.
986 Address MasterAddr = Address::invalid();
987 if (getLangOpts().OpenMPUseTLS &&
988 getContext().getTargetInfo().isTLSSupported()) {
989 assert(CapturedStmtInfo->lookup(VD) &&
990 "Copyin threadprivates should have been captured!");
991 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), true,
992 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
993 MasterAddr = EmitLValue(&DRE).getAddress(*this);
994 LocalDeclMap.erase(VD);
995 } else {
996 MasterAddr =
998 : CGM.GetAddrOfGlobal(VD),
1000 getContext().getDeclAlign(VD));
1001 }
1002 // Get the address of the threadprivate variable.
1003 Address PrivateAddr = EmitLValue(*IRef).getAddress(*this);
1004 if (CopiedVars.size() == 1) {
1005 // At first check if current thread is a master thread. If it is, no
1006 // need to copy data.
1007 CopyBegin = createBasicBlock("copyin.not.master");
1008 CopyEnd = createBasicBlock("copyin.not.master.end");
1009 // TODO: Avoid ptrtoint conversion.
1010 auto *MasterAddrInt =
1011 Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy);
1012 auto *PrivateAddrInt =
1013 Builder.CreatePtrToInt(PrivateAddr.getPointer(), CGM.IntPtrTy);
1014 Builder.CreateCondBr(
1015 Builder.CreateICmpNE(MasterAddrInt, PrivateAddrInt), CopyBegin,
1016 CopyEnd);
1017 EmitBlock(CopyBegin);
1018 }
1019 const auto *SrcVD =
1020 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1021 const auto *DestVD =
1022 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1023 EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
1024 }
1025 ++IRef;
1026 ++ISrcRef;
1027 ++IDestRef;
1028 }
1029 }
1030 if (CopyEnd) {
1031 // Exit out of copying procedure for non-master thread.
1032 EmitBlock(CopyEnd, /*IsFinished=*/true);
1033 return true;
1034 }
1035 return false;
1036}
1037
1039 const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
1040 if (!HaveInsertPoint())
1041 return false;
1042 bool HasAtLeastOneLastprivate = false;
1045 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1046 for (const Expr *C : LoopDirective->counters()) {
1047 SIMDLCVs.insert(
1048 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
1049 }
1050 }
1051 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1052 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1053 HasAtLeastOneLastprivate = true;
1055 !getLangOpts().OpenMPSimd)
1056 break;
1057 const auto *IRef = C->varlist_begin();
1058 const auto *IDestRef = C->destination_exprs().begin();
1059 for (const Expr *IInit : C->private_copies()) {
1060 // Keep the address of the original variable for future update at the end
1061 // of the loop.
1062 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1063 // Taskloops do not require additional initialization, it is done in
1064 // runtime support library.
1065 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
1066 const auto *DestVD =
1067 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1068 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1069 /*RefersToEnclosingVariableOrCapture=*/
1070 CapturedStmtInfo->lookup(OrigVD) != nullptr,
1071 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
1072 PrivateScope.addPrivate(DestVD, EmitLValue(&DRE).getAddress(*this));
1073 // Check if the variable is also a firstprivate: in this case IInit is
1074 // not generated. Initialization of this variable will happen in codegen
1075 // for 'firstprivate' clause.
1076 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
1077 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
1078 Address VDAddr = Address::invalid();
1079 if (C->getKind() == OMPC_LASTPRIVATE_conditional) {
1081 *this, OrigVD);
1082 setAddrOfLocalVar(VD, VDAddr);
1083 } else {
1084 // Emit private VarDecl with copy init.
1085 EmitDecl(*VD);
1086 VDAddr = GetAddrOfLocalVar(VD);
1087 }
1088 bool IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);
1089 assert(IsRegistered &&
1090 "lastprivate var already registered as private");
1091 (void)IsRegistered;
1092 }
1093 }
1094 ++IRef;
1095 ++IDestRef;
1096 }
1097 }
1098 return HasAtLeastOneLastprivate;
1099}
1100
1102 const OMPExecutableDirective &D, bool NoFinals,
1103 llvm::Value *IsLastIterCond) {
1104 if (!HaveInsertPoint())
1105 return;
1106 // Emit following code:
1107 // if (<IsLastIterCond>) {
1108 // orig_var1 = private_orig_var1;
1109 // ...
1110 // orig_varn = private_orig_varn;
1111 // }
1112 llvm::BasicBlock *ThenBB = nullptr;
1113 llvm::BasicBlock *DoneBB = nullptr;
1114 if (IsLastIterCond) {
1115 // Emit implicit barrier if at least one lastprivate conditional is found
1116 // and this is not a simd mode.
1117 if (!getLangOpts().OpenMPSimd &&
1118 llvm::any_of(D.getClausesOfKind<OMPLastprivateClause>(),
1119 [](const OMPLastprivateClause *C) {
1120 return C->getKind() == OMPC_LASTPRIVATE_conditional;
1121 })) {
1123 OMPD_unknown,
1124 /*EmitChecks=*/false,
1125 /*ForceSimpleCall=*/true);
1126 }
1127 ThenBB = createBasicBlock(".omp.lastprivate.then");
1128 DoneBB = createBasicBlock(".omp.lastprivate.done");
1129 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1130 EmitBlock(ThenBB);
1131 }
1132 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1133 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1134 if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1135 auto IC = LoopDirective->counters().begin();
1136 for (const Expr *F : LoopDirective->finals()) {
1137 const auto *D =
1138 cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1139 if (NoFinals)
1140 AlreadyEmittedVars.insert(D);
1141 else
1142 LoopCountersAndUpdates[D] = F;
1143 ++IC;
1144 }
1145 }
1146 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1147 auto IRef = C->varlist_begin();
1148 auto ISrcRef = C->source_exprs().begin();
1149 auto IDestRef = C->destination_exprs().begin();
1150 for (const Expr *AssignOp : C->assignment_ops()) {
1151 const auto *PrivateVD =
1152 cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1153 QualType Type = PrivateVD->getType();
1154 const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1155 if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1156 // If lastprivate variable is a loop control variable for loop-based
1157 // directive, update its value before copyin back to original
1158 // variable.
1159 if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1160 EmitIgnoredExpr(FinalExpr);
1161 const auto *SrcVD =
1162 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1163 const auto *DestVD =
1164 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1165 // Get the address of the private variable.
1166 Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
1167 if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1168 PrivateAddr = Address(
1169 Builder.CreateLoad(PrivateAddr),
1170 CGM.getTypes().ConvertTypeForMem(RefTy->getPointeeType()),
1171 CGM.getNaturalTypeAlignment(RefTy->getPointeeType()));
1172 // Store the last value to the private copy in the last iteration.
1173 if (C->getKind() == OMPC_LASTPRIVATE_conditional)
1175 *this, MakeAddrLValue(PrivateAddr, (*IRef)->getType()), PrivateVD,
1176 (*IRef)->getExprLoc());
1177 // Get the address of the original variable.
1178 Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1179 EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1180 }
1181 ++IRef;
1182 ++ISrcRef;
1183 ++IDestRef;
1184 }
1185 if (const Expr *PostUpdate = C->getPostUpdateExpr())
1186 EmitIgnoredExpr(PostUpdate);
1187 }
1188 if (IsLastIterCond)
1189 EmitBlock(DoneBB, /*IsFinished=*/true);
1190}
1191
1193 const OMPExecutableDirective &D,
1194 CodeGenFunction::OMPPrivateScope &PrivateScope, bool ForInscan) {
1195 if (!HaveInsertPoint())
1196 return;
1199 SmallVector<const Expr *, 4> ReductionOps;
1205 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1206 if (ForInscan != (C->getModifier() == OMPC_REDUCTION_inscan))
1207 continue;
1208 Shareds.append(C->varlist_begin(), C->varlist_end());
1209 Privates.append(C->privates().begin(), C->privates().end());
1210 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1211 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1212 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1213 if (C->getModifier() == OMPC_REDUCTION_task) {
1214 Data.ReductionVars.append(C->privates().begin(), C->privates().end());
1215 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
1216 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
1217 Data.ReductionOps.append(C->reduction_ops().begin(),
1218 C->reduction_ops().end());
1219 TaskLHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1220 TaskRHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1221 }
1222 }
1223 ReductionCodeGen RedCG(Shareds, Shareds, Privates, ReductionOps);
1224 unsigned Count = 0;
1225 auto *ILHS = LHSs.begin();
1226 auto *IRHS = RHSs.begin();
1227 auto *IPriv = Privates.begin();
1228 for (const Expr *IRef : Shareds) {
1229 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1230 // Emit private VarDecl with reduction init.
1231 RedCG.emitSharedOrigLValue(*this, Count);
1232 RedCG.emitAggregateType(*this, Count);
1233 AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);
1234 RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
1235 RedCG.getSharedLValue(Count).getAddress(*this),
1236 [&Emission](CodeGenFunction &CGF) {
1237 CGF.EmitAutoVarInit(Emission);
1238 return true;
1239 });
1240 EmitAutoVarCleanups(Emission);
1241 Address BaseAddr = RedCG.adjustPrivateAddress(
1242 *this, Count, Emission.getAllocatedAddress());
1243 bool IsRegistered =
1244 PrivateScope.addPrivate(RedCG.getBaseDecl(Count), BaseAddr);
1245 assert(IsRegistered && "private var already registered as private");
1246 // Silence the warning about unused variable.
1247 (void)IsRegistered;
1248
1249 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1250 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1251 QualType Type = PrivateVD->getType();
1252 bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1253 if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1254 // Store the address of the original variable associated with the LHS
1255 // implicit variable.
1256 PrivateScope.addPrivate(LHSVD,
1257 RedCG.getSharedLValue(Count).getAddress(*this));
1258 PrivateScope.addPrivate(RHSVD, GetAddrOfLocalVar(PrivateVD));
1259 } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1260 isa<ArraySubscriptExpr>(IRef)) {
1261 // Store the address of the original variable associated with the LHS
1262 // implicit variable.
1263 PrivateScope.addPrivate(LHSVD,
1264 RedCG.getSharedLValue(Count).getAddress(*this));
1265 PrivateScope.addPrivate(RHSVD,
1266 GetAddrOfLocalVar(PrivateVD).withElementType(
1267 ConvertTypeForMem(RHSVD->getType())));
1268 } else {
1269 QualType Type = PrivateVD->getType();
1270 bool IsArray = getContext().getAsArrayType(Type) != nullptr;
1271 Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress(*this);
1272 // Store the address of the original variable associated with the LHS
1273 // implicit variable.
1274 if (IsArray) {
1275 OriginalAddr =
1276 OriginalAddr.withElementType(ConvertTypeForMem(LHSVD->getType()));
1277 }
1278 PrivateScope.addPrivate(LHSVD, OriginalAddr);
1279 PrivateScope.addPrivate(
1280 RHSVD, IsArray ? GetAddrOfLocalVar(PrivateVD).withElementType(
1281 ConvertTypeForMem(RHSVD->getType()))
1282 : GetAddrOfLocalVar(PrivateVD));
1283 }
1284 ++ILHS;
1285 ++IRHS;
1286 ++IPriv;
1287 ++Count;
1288 }
1289 if (!Data.ReductionVars.empty()) {
1290 Data.IsReductionWithTaskMod = true;
1291 Data.IsWorksharingReduction =
1293 llvm::Value *ReductionDesc = CGM.getOpenMPRuntime().emitTaskReductionInit(
1294 *this, D.getBeginLoc(), TaskLHSs, TaskRHSs, Data);
1295 const Expr *TaskRedRef = nullptr;
1296 switch (D.getDirectiveKind()) {
1297 case OMPD_parallel:
1298 TaskRedRef = cast<OMPParallelDirective>(D).getTaskReductionRefExpr();
1299 break;
1300 case OMPD_for:
1301 TaskRedRef = cast<OMPForDirective>(D).getTaskReductionRefExpr();
1302 break;
1303 case OMPD_sections:
1304 TaskRedRef = cast<OMPSectionsDirective>(D).getTaskReductionRefExpr();
1305 break;
1306 case OMPD_parallel_for:
1307 TaskRedRef = cast<OMPParallelForDirective>(D).getTaskReductionRefExpr();
1308 break;
1309 case OMPD_parallel_master:
1310 TaskRedRef =
1311 cast<OMPParallelMasterDirective>(D).getTaskReductionRefExpr();
1312 break;
1313 case OMPD_parallel_sections:
1314 TaskRedRef =
1315 cast<OMPParallelSectionsDirective>(D).getTaskReductionRefExpr();
1316 break;
1317 case OMPD_target_parallel:
1318 TaskRedRef =
1319 cast<OMPTargetParallelDirective>(D).getTaskReductionRefExpr();
1320 break;
1321 case OMPD_target_parallel_for:
1322 TaskRedRef =
1323 cast<OMPTargetParallelForDirective>(D).getTaskReductionRefExpr();
1324 break;
1325 case OMPD_distribute_parallel_for:
1326 TaskRedRef =
1327 cast<OMPDistributeParallelForDirective>(D).getTaskReductionRefExpr();
1328 break;
1329 case OMPD_teams_distribute_parallel_for:
1330 TaskRedRef = cast<OMPTeamsDistributeParallelForDirective>(D)
1331 .getTaskReductionRefExpr();
1332 break;
1333 case OMPD_target_teams_distribute_parallel_for:
1334 TaskRedRef = cast<OMPTargetTeamsDistributeParallelForDirective>(D)
1335 .getTaskReductionRefExpr();
1336 break;
1337 case OMPD_simd:
1338 case OMPD_for_simd:
1339 case OMPD_section:
1340 case OMPD_single:
1341 case OMPD_master:
1342 case OMPD_critical:
1343 case OMPD_parallel_for_simd:
1344 case OMPD_task:
1345 case OMPD_taskyield:
1346 case OMPD_error:
1347 case OMPD_barrier:
1348 case OMPD_taskwait:
1349 case OMPD_taskgroup:
1350 case OMPD_flush:
1351 case OMPD_depobj:
1352 case OMPD_scan:
1353 case OMPD_ordered:
1354 case OMPD_atomic:
1355 case OMPD_teams:
1356 case OMPD_target:
1357 case OMPD_cancellation_point:
1358 case OMPD_cancel:
1359 case OMPD_target_data:
1360 case OMPD_target_enter_data:
1361 case OMPD_target_exit_data:
1362 case OMPD_taskloop:
1363 case OMPD_taskloop_simd:
1364 case OMPD_master_taskloop:
1365 case OMPD_master_taskloop_simd:
1366 case OMPD_parallel_master_taskloop:
1367 case OMPD_parallel_master_taskloop_simd:
1368 case OMPD_distribute:
1369 case OMPD_target_update:
1370 case OMPD_distribute_parallel_for_simd:
1371 case OMPD_distribute_simd:
1372 case OMPD_target_parallel_for_simd:
1373 case OMPD_target_simd:
1374 case OMPD_teams_distribute:
1375 case OMPD_teams_distribute_simd:
1376 case OMPD_teams_distribute_parallel_for_simd:
1377 case OMPD_target_teams:
1378 case OMPD_target_teams_distribute:
1379 case OMPD_target_teams_distribute_parallel_for_simd:
1380 case OMPD_target_teams_distribute_simd:
1381 case OMPD_declare_target:
1382 case OMPD_end_declare_target:
1383 case OMPD_threadprivate:
1384 case OMPD_allocate:
1385 case OMPD_declare_reduction:
1386 case OMPD_declare_mapper:
1387 case OMPD_declare_simd:
1388 case OMPD_requires:
1389 case OMPD_declare_variant:
1390 case OMPD_begin_declare_variant:
1391 case OMPD_end_declare_variant:
1392 case OMPD_unknown:
1393 default:
1394 llvm_unreachable("Enexpected directive with task reductions.");
1395 }
1396
1397 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(TaskRedRef)->getDecl());
1398 EmitVarDecl(*VD);
1399 EmitStoreOfScalar(ReductionDesc, GetAddrOfLocalVar(VD),
1400 /*Volatile=*/false, TaskRedRef->getType());
1401 }
1402}
1403
1405 const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1406 if (!HaveInsertPoint())
1407 return;
1412 bool HasAtLeastOneReduction = false;
1413 bool IsReductionWithTaskMod = false;
1414 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1415 // Do not emit for inscan reductions.
1416 if (C->getModifier() == OMPC_REDUCTION_inscan)
1417 continue;
1418 HasAtLeastOneReduction = true;
1419 Privates.append(C->privates().begin(), C->privates().end());
1420 LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1421 RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1422 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1423 IsReductionWithTaskMod =
1424 IsReductionWithTaskMod || C->getModifier() == OMPC_REDUCTION_task;
1425 }
1426 if (HasAtLeastOneReduction) {
1427 if (IsReductionWithTaskMod) {
1429 *this, D.getBeginLoc(),
1431 }
1432 bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1434 ReductionKind == OMPD_simd;
1435 bool SimpleReduction = ReductionKind == OMPD_simd;
1436 // Emit nowait reduction if nowait clause is present or directive is a
1437 // parallel directive (it always has implicit barrier).
1439 *this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1440 {WithNowait, SimpleReduction, ReductionKind});
1441 }
1442}
1443
1446 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1447 if (!CGF.HaveInsertPoint())
1448 return;
1449 llvm::BasicBlock *DoneBB = nullptr;
1450 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1451 if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
1452 if (!DoneBB) {
1453 if (llvm::Value *Cond = CondGen(CGF)) {
1454 // If the first post-update expression is found, emit conditional
1455 // block if it was requested.
1456 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1457 DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1458 CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1459 CGF.EmitBlock(ThenBB);
1460 }
1461 }
1462 CGF.EmitIgnoredExpr(PostUpdate);
1463 }
1464 }
1465 if (DoneBB)
1466 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1467}
1468
1469namespace {
1470/// Codegen lambda for appending distribute lower and upper bounds to outlined
1471/// parallel function. This is necessary for combined constructs such as
1472/// 'distribute parallel for'
1473typedef llvm::function_ref<void(CodeGenFunction &,
1474 const OMPExecutableDirective &,
1476 CodeGenBoundParametersTy;
1477} // anonymous namespace
1478
1479static void
1481 const OMPExecutableDirective &S) {
1482 if (CGF.getLangOpts().OpenMP < 50)
1483 return;
1485 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
1486 for (const Expr *Ref : C->varlists()) {
1487 if (!Ref->getType()->isScalarType())
1488 continue;
1489 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1490 if (!DRE)
1491 continue;
1492 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1494 }
1495 }
1496 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
1497 for (const Expr *Ref : C->varlists()) {
1498 if (!Ref->getType()->isScalarType())
1499 continue;
1500 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1501 if (!DRE)
1502 continue;
1503 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1505 }
1506 }
1507 for (const auto *C : S.getClausesOfKind<OMPLinearClause>()) {
1508 for (const Expr *Ref : C->varlists()) {
1509 if (!Ref->getType()->isScalarType())
1510 continue;
1511 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1512 if (!DRE)
1513 continue;
1514 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1516 }
1517 }
1518 // Privates should ne analyzed since they are not captured at all.
1519 // Task reductions may be skipped - tasks are ignored.
1520 // Firstprivates do not return value but may be passed by reference - no need
1521 // to check for updated lastprivate conditional.
1522 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
1523 for (const Expr *Ref : C->varlists()) {
1524 if (!Ref->getType()->isScalarType())
1525 continue;
1526 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1527 if (!DRE)
1528 continue;
1529 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1530 }
1531 }
1533 CGF, S, PrivateDecls);
1534}
1535
1538 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1539 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1540 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1541 llvm::Value *NumThreads = nullptr;
1542 llvm::Function *OutlinedFn =
1544 CGF, S, *CS->getCapturedDecl()->param_begin(), InnermostKind,
1545 CodeGen);
1546 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
1547 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1548 NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1549 /*IgnoreResultAssign=*/true);
1551 CGF, NumThreads, NumThreadsClause->getBeginLoc());
1552 }
1553 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
1554 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1556 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1557 }
1558 const Expr *IfCond = nullptr;
1559 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1560 if (C->getNameModifier() == OMPD_unknown ||
1561 C->getNameModifier() == OMPD_parallel) {
1562 IfCond = C->getCondition();
1563 break;
1564 }
1565 }
1566
1567 OMPParallelScope Scope(CGF, S);
1569 // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
1570 // lower and upper bounds with the pragma 'for' chunking mechanism.
1571 // The following lambda takes care of appending the lower and upper bound
1572 // parameters when necessary
1573 CodeGenBoundParameters(CGF, S, CapturedVars);
1574 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
1575 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn,
1576 CapturedVars, IfCond, NumThreads);
1577}
1578
1579static bool isAllocatableDecl(const VarDecl *VD) {
1580 const VarDecl *CVD = VD->getCanonicalDecl();
1581 if (!CVD->hasAttr<OMPAllocateDeclAttr>())
1582 return false;
1583 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
1584 // Use the default allocation.
1585 return !((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||
1586 AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&
1587 !AA->getAllocator());
1588}
1589
1591 const OMPExecutableDirective &,
1593
1595 const OMPExecutableDirective &S) {
1596 bool Copyins = CGF.EmitOMPCopyinClause(S);
1597 if (Copyins) {
1598 // Emit implicit barrier to synchronize threads and avoid data races on
1599 // propagation master's thread values of threadprivate variables to local
1600 // instances of that variables of all other implicit threads.
1602 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
1603 /*ForceSimpleCall=*/true);
1604 }
1605}
1606
1608 CodeGenFunction &CGF, const VarDecl *VD) {
1609 CodeGenModule &CGM = CGF.CGM;
1610 auto &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1611
1612 if (!VD)
1613 return Address::invalid();
1614 const VarDecl *CVD = VD->getCanonicalDecl();
1615 if (!isAllocatableDecl(CVD))
1616 return Address::invalid();
1617 llvm::Value *Size;
1618 CharUnits Align = CGM.getContext().getDeclAlign(CVD);
1619 if (CVD->getType()->isVariablyModifiedType()) {
1620 Size = CGF.getTypeSize(CVD->getType());
1621 // Align the size: ((size + align - 1) / align) * align
1622 Size = CGF.Builder.CreateNUWAdd(
1623 Size, CGM.getSize(Align - CharUnits::fromQuantity(1)));
1624 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align));
1625 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align));
1626 } else {
1628 Size = CGM.getSize(Sz.alignTo(Align));
1629 }
1630
1631 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
1632 assert(AA->getAllocator() &&
1633 "Expected allocator expression for non-default allocator.");
1634 llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator());
1635 // According to the standard, the original allocator type is a enum (integer).
1636 // Convert to pointer type, if required.
1637 if (Allocator->getType()->isIntegerTy())
1638 Allocator = CGF.Builder.CreateIntToPtr(Allocator, CGM.VoidPtrTy);
1639 else if (Allocator->getType()->isPointerTy())
1640 Allocator = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Allocator,
1641 CGM.VoidPtrTy);
1642
1643 llvm::Value *Addr = OMPBuilder.createOMPAlloc(
1644 CGF.Builder, Size, Allocator,
1645 getNameWithSeparators({CVD->getName(), ".void.addr"}, ".", "."));
1646 llvm::CallInst *FreeCI =
1647 OMPBuilder.createOMPFree(CGF.Builder, Addr, Allocator);
1648
1649 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(NormalAndEHCleanup, FreeCI);
1651 Addr,
1653 getNameWithSeparators({CVD->getName(), ".addr"}, ".", "."));
1654 return Address(Addr, CGF.ConvertTypeForMem(CVD->getType()), Align);
1655}
1656
1658 CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr,
1659 SourceLocation Loc) {
1660 CodeGenModule &CGM = CGF.CGM;
1661 if (CGM.getLangOpts().OpenMPUseTLS &&
1663 return VDAddr;
1664
1665 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1666
1667 llvm::Type *VarTy = VDAddr.getElementType();
1668 llvm::Value *Data =
1669 CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy);
1670 llvm::ConstantInt *Size = CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy));
1671 std::string Suffix = getNameWithSeparators({"cache", ""});
1672 llvm::Twine CacheName = Twine(CGM.getMangledName(VD)).concat(Suffix);
1673
1674 llvm::CallInst *ThreadPrivateCacheCall =
1675 OMPBuilder.createCachedThreadPrivate(CGF.Builder, Data, Size, CacheName);
1676
1677 return Address(ThreadPrivateCacheCall, CGM.Int8Ty, VDAddr.getAlignment());
1678}
1679
1681 ArrayRef<StringRef> Parts, StringRef FirstSeparator, StringRef Separator) {
1682 SmallString<128> Buffer;
1683 llvm::raw_svector_ostream OS(Buffer);
1684 StringRef Sep = FirstSeparator;
1685 for (StringRef Part : Parts) {
1686 OS << Sep << Part;
1687 Sep = Separator;
1688 }
1689 return OS.str().str();
1690}
1691
1693 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
1694 InsertPointTy CodeGenIP, Twine RegionName) {
1696 Builder.restoreIP(CodeGenIP);
1697 llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder, /*CreateBranch=*/false,
1698 "." + RegionName + ".after");
1699
1700 {
1701 OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);
1702 CGF.EmitStmt(RegionBodyStmt);
1703 }
1704
1705 if (Builder.saveIP().isSet())
1706 Builder.CreateBr(FiniBB);
1707}
1708
1710 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
1711 InsertPointTy CodeGenIP, Twine RegionName) {
1713 Builder.restoreIP(CodeGenIP);
1714 llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder, /*CreateBranch=*/false,
1715 "." + RegionName + ".after");
1716
1717 {
1718 OMPBuilderCBHelpers::OutlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);
1719 CGF.EmitStmt(RegionBodyStmt);
1720 }
1721
1722 if (Builder.saveIP().isSet())
1723 Builder.CreateBr(FiniBB);
1724}
1725
1727 if (CGM.getLangOpts().OpenMPIRBuilder) {
1728 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1729 // Check if we have any if clause associated with the directive.
1730 llvm::Value *IfCond = nullptr;
1731 if (const auto *C = S.getSingleClause<OMPIfClause>())
1732 IfCond = EmitScalarExpr(C->getCondition(),
1733 /*IgnoreResultAssign=*/true);
1734
1735 llvm::Value *NumThreads = nullptr;
1736 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>())
1737 NumThreads = EmitScalarExpr(NumThreadsClause->getNumThreads(),
1738 /*IgnoreResultAssign=*/true);
1739
1740 ProcBindKind ProcBind = OMP_PROC_BIND_default;
1741 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>())
1742 ProcBind = ProcBindClause->getProcBindKind();
1743
1744 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1745
1746 // The cleanup callback that finalizes all variabels at the given location,
1747 // thus calls destructors etc.
1748 auto FiniCB = [this](InsertPointTy IP) {
1750 };
1751
1752 // Privatization callback that performs appropriate action for
1753 // shared/private/firstprivate/lastprivate/copyin/... variables.
1754 //
1755 // TODO: This defaults to shared right now.
1756 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
1757 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
1758 // The next line is appropriate only for variables (Val) with the
1759 // data-sharing attribute "shared".
1760 ReplVal = &Val;
1761
1762 return CodeGenIP;
1763 };
1764
1765 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1766 const Stmt *ParallelRegionBodyStmt = CS->getCapturedStmt();
1767
1768 auto BodyGenCB = [&, this](InsertPointTy AllocaIP,
1769 InsertPointTy CodeGenIP) {
1771 *this, ParallelRegionBodyStmt, AllocaIP, CodeGenIP, "parallel");
1772 };
1773
1774 CGCapturedStmtInfo CGSI(*CS, CR_OpenMP);
1775 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
1776 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
1777 AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
1778 Builder.restoreIP(
1779 OMPBuilder.createParallel(Builder, AllocaIP, BodyGenCB, PrivCB, FiniCB,
1780 IfCond, NumThreads, ProcBind, S.hasCancel()));
1781 return;
1782 }
1783
1784 // Emit parallel region as a standalone region.
1785 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1786 Action.Enter(CGF);
1787 OMPPrivateScope PrivateScope(CGF);
1788 emitOMPCopyinClause(CGF, S);
1789 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1790 CGF.EmitOMPPrivateClause(S, PrivateScope);
1791 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1792 (void)PrivateScope.Privatize();
1793 CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1794 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
1795 };
1796 {
1797 auto LPCRegion =
1799 emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
1802 [](CodeGenFunction &) { return nullptr; });
1803 }
1804 // Check for outer lastprivate conditional update.
1806}
1807
1809 EmitStmt(S.getIfStmt());
1810}
1811
1812namespace {
1813/// RAII to handle scopes for loop transformation directives.
1814class OMPTransformDirectiveScopeRAII {
1815 OMPLoopScope *Scope = nullptr;
1816 CodeGenFunction::CGCapturedStmtInfo *CGSI = nullptr;
1817 CodeGenFunction::CGCapturedStmtRAII *CapInfoRAII = nullptr;
1818
1819 OMPTransformDirectiveScopeRAII(const OMPTransformDirectiveScopeRAII &) =
1820 delete;
1821 OMPTransformDirectiveScopeRAII &
1822 operator=(const OMPTransformDirectiveScopeRAII &) = delete;
1823
1824public:
1825 OMPTransformDirectiveScopeRAII(CodeGenFunction &CGF, const Stmt *S) {
1826 if (const auto *Dir = dyn_cast<OMPLoopBasedDirective>(S)) {
1827 Scope = new OMPLoopScope(CGF, *Dir);
1828 CGSI = new CodeGenFunction::CGCapturedStmtInfo(CR_OpenMP);
1829 CapInfoRAII = new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI);
1830 }
1831 }
1832 ~OMPTransformDirectiveScopeRAII() {
1833 if (!Scope)
1834 return;
1835 delete CapInfoRAII;
1836 delete CGSI;
1837 delete Scope;
1838 }
1839};
1840} // namespace
1841
1842static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
1843 int MaxLevel, int Level = 0) {
1844 assert(Level < MaxLevel && "Too deep lookup during loop body codegen.");
1845 const Stmt *SimplifiedS = S->IgnoreContainers();
1846 if (const auto *CS = dyn_cast<CompoundStmt>(SimplifiedS)) {
1847 PrettyStackTraceLoc CrashInfo(
1848 CGF.getContext().getSourceManager(), CS->getLBracLoc(),
1849 "LLVM IR generation of compound statement ('{}')");
1850
1851 // Keep track of the current cleanup stack depth, including debug scopes.
1852 CodeGenFunction::LexicalScope Scope(CGF, S->getSourceRange());
1853 for (const Stmt *CurStmt : CS->body())
1854 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level);
1855 return;
1856 }
1857 if (SimplifiedS == NextLoop) {
1858 if (auto *Dir = dyn_cast<OMPLoopTransformationDirective>(SimplifiedS))
1859 SimplifiedS = Dir->getTransformedStmt();
1860 if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(SimplifiedS))
1861 SimplifiedS = CanonLoop->getLoopStmt();
1862 if (const auto *For = dyn_cast<ForStmt>(SimplifiedS)) {
1863 S = For->getBody();
1864 } else {
1865 assert(isa<CXXForRangeStmt>(SimplifiedS) &&
1866 "Expected canonical for loop or range-based for loop.");
1867 const auto *CXXFor = cast<CXXForRangeStmt>(SimplifiedS);
1868 CGF.EmitStmt(CXXFor->getLoopVarStmt());
1869 S = CXXFor->getBody();
1870 }
1871 if (Level + 1 < MaxLevel) {
1873 S, /*TryImperfectlyNestedLoops=*/true);
1874 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1);
1875 return;
1876 }
1877 }
1878 CGF.EmitStmt(S);
1879}
1880
1882 JumpDest LoopExit) {
1883 RunCleanupsScope BodyScope(*this);
1884 // Update counters values on current iteration.
1885 for (const Expr *UE : D.updates())
1886 EmitIgnoredExpr(UE);
1887 // Update the linear variables.
1888 // In distribute directives only loop counters may be marked as linear, no
1889 // need to generate the code for them.
1891 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1892 for (const Expr *UE : C->updates())
1893 EmitIgnoredExpr(UE);
1894 }
1895 }
1896
1897 // On a continue in the body, jump to the end.
1898 JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue");
1899 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1900 for (const Expr *E : D.finals_conditions()) {
1901 if (!E)
1902 continue;
1903 // Check that loop counter in non-rectangular nest fits into the iteration
1904 // space.
1905 llvm::BasicBlock *NextBB = createBasicBlock("omp.body.next");
1906 EmitBranchOnBoolExpr(E, NextBB, Continue.getBlock(),
1908 EmitBlock(NextBB);
1909 }
1910
1911 OMPPrivateScope InscanScope(*this);
1912 EmitOMPReductionClauseInit(D, InscanScope, /*ForInscan=*/true);
1913 bool IsInscanRegion = InscanScope.Privatize();
1914 if (IsInscanRegion) {
1915 // Need to remember the block before and after scan directive
1916 // to dispatch them correctly depending on the clause used in
1917 // this directive, inclusive or exclusive. For inclusive scan the natural
1918 // order of the blocks is used, for exclusive clause the blocks must be
1919 // executed in reverse order.
1920 OMPBeforeScanBlock = createBasicBlock("omp.before.scan.bb");
1921 OMPAfterScanBlock = createBasicBlock("omp.after.scan.bb");
1922 // No need to allocate inscan exit block, in simd mode it is selected in the
1923 // codegen for the scan directive.
1924 if (D.getDirectiveKind() != OMPD_simd && !getLangOpts().OpenMPSimd)
1925 OMPScanExitBlock = createBasicBlock("omp.exit.inscan.bb");
1926 OMPScanDispatch = createBasicBlock("omp.inscan.dispatch");
1929 }
1930
1931 // Emit loop variables for C++ range loops.
1932 const Stmt *Body =
1934 // Emit loop body.
1935 emitBody(*this, Body,
1937 Body, /*TryImperfectlyNestedLoops=*/true),
1938 D.getLoopsNumber());
1939
1940 // Jump to the dispatcher at the end of the loop body.
1941 if (IsInscanRegion)
1943
1944 // The end (updates/cleanups).
1945 EmitBlock(Continue.getBlock());
1946 BreakContinueStack.pop_back();
1947}
1948
1949using EmittedClosureTy = std::pair<llvm::Function *, llvm::Value *>;
1950
1951/// Emit a captured statement and return the function as well as its captured
1952/// closure context.
1954 const CapturedStmt *S) {
1955 LValue CapStruct = ParentCGF.InitCapturedStruct(*S);
1956 CodeGenFunction CGF(ParentCGF.CGM, /*suppressNewContext=*/true);
1957 std::unique_ptr<CodeGenFunction::CGCapturedStmtInfo> CSI =
1958 std::make_unique<CodeGenFunction::CGCapturedStmtInfo>(*S);
1959 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, CSI.get());
1960 llvm::Function *F = CGF.GenerateCapturedStmtFunction(*S);
1961
1962 return {F, CapStruct.getPointer(ParentCGF)};
1963}
1964
1965/// Emit a call to a previously captured closure.
1966static llvm::CallInst *
1969 // Append the closure context to the argument.
1970 SmallVector<llvm::Value *> EffectiveArgs;
1971 EffectiveArgs.reserve(Args.size() + 1);
1972 llvm::append_range(EffectiveArgs, Args);
1973 EffectiveArgs.push_back(Cap.second);
1974
1975 return ParentCGF.Builder.CreateCall(Cap.first, EffectiveArgs);
1976}
1977
1978llvm::CanonicalLoopInfo *
1980 assert(Depth == 1 && "Nested loops with OpenMPIRBuilder not yet implemented");
1981
1982 // The caller is processing the loop-associated directive processing the \p
1983 // Depth loops nested in \p S. Put the previous pending loop-associated
1984 // directive to the stack. If the current loop-associated directive is a loop
1985 // transformation directive, it will push its generated loops onto the stack
1986 // such that together with the loops left here they form the combined loop
1987 // nest for the parent loop-associated directive.
1988 int ParentExpectedOMPLoopDepth = ExpectedOMPLoopDepth;
1989 ExpectedOMPLoopDepth = Depth;
1990
1991 EmitStmt(S);
1992 assert(OMPLoopNestStack.size() >= (size_t)Depth && "Found too few loops");
1993
1994 // The last added loop is the outermost one.
1995 llvm::CanonicalLoopInfo *Result = OMPLoopNestStack.back();
1996
1997 // Pop the \p Depth loops requested by the call from that stack and restore
1998 // the previous context.
1999 OMPLoopNestStack.pop_back_n(Depth);
2000 ExpectedOMPLoopDepth = ParentExpectedOMPLoopDepth;
2001
2002 return Result;
2003}
2004
2006 const Stmt *SyntacticalLoop = S->getLoopStmt();
2007 if (!getLangOpts().OpenMPIRBuilder) {
2008 // Ignore if OpenMPIRBuilder is not enabled.
2009 EmitStmt(SyntacticalLoop);
2010 return;
2011 }
2012
2013 LexicalScope ForScope(*this, S->getSourceRange());
2014
2015 // Emit init statements. The Distance/LoopVar funcs may reference variable
2016 // declarations they contain.
2017 const Stmt *BodyStmt;
2018 if (const auto *For = dyn_cast<ForStmt>(SyntacticalLoop)) {
2019 if (const Stmt *InitStmt = For->getInit())
2020 EmitStmt(InitStmt);
2021 BodyStmt = For->getBody();
2022 } else if (const auto *RangeFor =
2023 dyn_cast<CXXForRangeStmt>(SyntacticalLoop)) {
2024 if (const DeclStmt *RangeStmt = RangeFor->getRangeStmt())
2025 EmitStmt(RangeStmt);
2026 if (const DeclStmt *BeginStmt = RangeFor->getBeginStmt())
2027 EmitStmt(BeginStmt);
2028 if (const DeclStmt *EndStmt = RangeFor->getEndStmt())
2029 EmitStmt(EndStmt);
2030 if (const DeclStmt *LoopVarStmt = RangeFor->getLoopVarStmt())
2031 EmitStmt(LoopVarStmt);
2032 BodyStmt = RangeFor->getBody();
2033 } else
2034 llvm_unreachable("Expected for-stmt or range-based for-stmt");
2035
2036 // Emit closure for later use. By-value captures will be captured here.
2037 const CapturedStmt *DistanceFunc = S->getDistanceFunc();
2038 EmittedClosureTy DistanceClosure = emitCapturedStmtFunc(*this, DistanceFunc);
2039 const CapturedStmt *LoopVarFunc = S->getLoopVarFunc();
2040 EmittedClosureTy LoopVarClosure = emitCapturedStmtFunc(*this, LoopVarFunc);
2041
2042 // Call the distance function to get the number of iterations of the loop to
2043 // come.
2044 QualType LogicalTy = DistanceFunc->getCapturedDecl()
2045 ->getParam(0)
2046 ->getType()
2048 Address CountAddr = CreateMemTemp(LogicalTy, ".count.addr");
2049 emitCapturedStmtCall(*this, DistanceClosure, {CountAddr.getPointer()});
2050 llvm::Value *DistVal = Builder.CreateLoad(CountAddr, ".count");
2051
2052 // Emit the loop structure.
2053 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2054 auto BodyGen = [&, this](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP,
2055 llvm::Value *IndVar) {
2056 Builder.restoreIP(CodeGenIP);
2057
2058 // Emit the loop body: Convert the logical iteration number to the loop
2059 // variable and emit the body.
2060 const DeclRefExpr *LoopVarRef = S->getLoopVarRef();
2061 LValue LCVal = EmitLValue(LoopVarRef);
2062 Address LoopVarAddress = LCVal.getAddress(*this);
2063 emitCapturedStmtCall(*this, LoopVarClosure,
2064 {LoopVarAddress.getPointer(), IndVar});
2065
2066 RunCleanupsScope BodyScope(*this);
2067 EmitStmt(BodyStmt);
2068 };
2069 llvm::CanonicalLoopInfo *CL =
2070 OMPBuilder.createCanonicalLoop(Builder, BodyGen, DistVal);
2071
2072 // Finish up the loop.
2073 Builder.restoreIP(CL->getAfterIP());
2074 ForScope.ForceCleanup();
2075
2076 // Remember the CanonicalLoopInfo for parent AST nodes consuming it.
2077 OMPLoopNestStack.push_back(CL);
2078}
2079
2081 const OMPExecutableDirective &S, bool RequiresCleanup, const Expr *LoopCond,
2082 const Expr *IncExpr,
2083 const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
2084 const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
2085 auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
2086
2087 // Start the loop with a block that tests the condition.
2088 auto CondBlock = createBasicBlock("omp.inner.for.cond");
2089 EmitBlock(CondBlock);
2090 const SourceRange R = S.getSourceRange();
2091
2092 // If attributes are attached, push to the basic block with them.
2093 const auto &OMPED = cast<OMPExecutableDirective>(S);
2094 const CapturedStmt *ICS = OMPED.getInnermostCapturedStmt();
2095 const Stmt *SS = ICS->getCapturedStmt();
2096 const AttributedStmt *AS = dyn_cast_or_null<AttributedStmt>(SS);
2097 OMPLoopNestStack.clear();
2098 if (AS)
2102 else
2105
2106 // If there are any cleanups between here and the loop-exit scope,
2107 // create a block to stage a loop exit along.
2108 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
2109 if (RequiresCleanup)
2110 ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
2111
2112 llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body");
2113
2114 // Emit condition.
2115 EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
2116 if (ExitBlock != LoopExit.getBlock()) {
2117 EmitBlock(ExitBlock);
2119 }
2120
2121 EmitBlock(LoopBody);
2123
2124 // Create a block for the increment.
2125 JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
2126 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
2127
2128 BodyGen(*this);
2129
2130 // Emit "IV = IV + 1" and a back-edge to the condition block.
2131 EmitBlock(Continue.getBlock());
2132 EmitIgnoredExpr(IncExpr);
2133 PostIncGen(*this);
2134 BreakContinueStack.pop_back();
2135 EmitBranch(CondBlock);
2136 LoopStack.pop();
2137 // Emit the fall-through block.
2138 EmitBlock(LoopExit.getBlock());
2139}
2140
2142 if (!HaveInsertPoint())
2143 return false;
2144 // Emit inits for the linear variables.
2145 bool HasLinears = false;
2146 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2147 for (const Expr *Init : C->inits()) {
2148 HasLinears = true;
2149 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
2150 if (const auto *Ref =
2151 dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
2152 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
2153 const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
2154 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
2155 CapturedStmtInfo->lookup(OrigVD) != nullptr,
2156 VD->getInit()->getType(), VK_LValue,
2157 VD->getInit()->getExprLoc());
2159 &DRE, VD,
2160 MakeAddrLValue(Emission.getAllocatedAddress(), VD->getType()),
2161 /*capturedByInit=*/false);
2162 EmitAutoVarCleanups(Emission);
2163 } else {
2164 EmitVarDecl(*VD);
2165 }
2166 }
2167 // Emit the linear steps for the linear clauses.
2168 // If a step is not constant, it is pre-calculated before the loop.
2169 if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
2170 if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
2171 EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
2172 // Emit calculation of the linear step.
2173 EmitIgnoredExpr(CS);
2174 }
2175 }
2176 return HasLinears;
2177}
2178
2180 const OMPLoopDirective &D,
2181 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
2182 if (!HaveInsertPoint())
2183 return;
2184 llvm::BasicBlock *DoneBB = nullptr;
2185 // Emit the final values of the linear variables.
2186 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2187 auto IC = C->varlist_begin();
2188 for (const Expr *F : C->finals()) {
2189 if (!DoneBB) {
2190 if (llvm::Value *Cond = CondGen(*this)) {
2191 // If the first post-update expression is found, emit conditional
2192 // block if it was requested.
2193 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu");
2194 DoneBB = createBasicBlock(".omp.linear.pu.done");
2195 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2196 EmitBlock(ThenBB);
2197 }
2198 }
2199 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
2200 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
2201 CapturedStmtInfo->lookup(OrigVD) != nullptr,
2202 (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
2203 Address OrigAddr = EmitLValue(&DRE).getAddress(*this);
2204 CodeGenFunction::OMPPrivateScope VarScope(*this);
2205 VarScope.addPrivate(OrigVD, OrigAddr);
2206 (void)VarScope.Privatize();
2207 EmitIgnoredExpr(F);
2208 ++IC;
2209 }
2210 if (const Expr *PostUpdate = C->getPostUpdateExpr())
2211 EmitIgnoredExpr(PostUpdate);
2212 }
2213 if (DoneBB)
2214 EmitBlock(DoneBB, /*IsFinished=*/true);
2215}
2216
2218 const OMPExecutableDirective &D) {
2219 if (!CGF.HaveInsertPoint())
2220 return;
2221 for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
2222 llvm::APInt ClauseAlignment(64, 0);
2223 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
2224 auto *AlignmentCI =
2225 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
2226 ClauseAlignment = AlignmentCI->getValue();
2227 }
2228 for (const Expr *E : Clause->varlists()) {
2229 llvm::APInt Alignment(ClauseAlignment);
2230 if (Alignment == 0) {
2231 // OpenMP [2.8.1, Description]
2232 // If no optional parameter is specified, implementation-defined default
2233 // alignments for SIMD instructions on the target platforms are assumed.
2234 Alignment =
2235 CGF.getContext()
2237 E->getType()->getPointeeType()))
2238 .getQuantity();
2239 }
2240 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2241 "alignment is not power of 2");
2242 if (Alignment != 0) {
2243 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
2245 PtrValue, E, /*No second loc needed*/ SourceLocation(),
2246 llvm::ConstantInt::get(CGF.getLLVMContext(), Alignment));
2247 }
2248 }
2249 }
2250}
2251
2253 const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
2254 if (!HaveInsertPoint())
2255 return;
2256 auto I = S.private_counters().begin();
2257 for (const Expr *E : S.counters()) {
2258 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2259 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
2260 // Emit var without initialization.
2261 AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD);
2262 EmitAutoVarCleanups(VarEmission);
2263 LocalDeclMap.erase(PrivateVD);
2264 (void)LoopScope.addPrivate(VD, VarEmission.getAllocatedAddress());
2265 if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
2266 VD->hasGlobalStorage()) {
2267 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD),
2268 LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
2269 E->getType(), VK_LValue, E->getExprLoc());
2270 (void)LoopScope.addPrivate(PrivateVD, EmitLValue(&DRE).getAddress(*this));
2271 } else {
2272 (void)LoopScope.addPrivate(PrivateVD, VarEmission.getAllocatedAddress());
2273 }
2274 ++I;
2275 }
2276 // Privatize extra loop counters used in loops for ordered(n) clauses.
2277 for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) {
2278 if (!C->getNumForLoops())
2279 continue;
2280 for (unsigned I = S.getLoopsNumber(), E = C->getLoopNumIterations().size();
2281 I < E; ++I) {
2282 const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I));
2283 const auto *VD = cast<VarDecl>(DRE->getDecl());
2284 // Override only those variables that can be captured to avoid re-emission
2285 // of the variables declared within the loops.
2286 if (DRE->refersToEnclosingVariableOrCapture()) {
2287 (void)LoopScope.addPrivate(
2288 VD, CreateMemTemp(DRE->getType(), VD->getName()));
2289 }
2290 }
2291 }
2292}
2293
2295 const Expr *Cond, llvm::BasicBlock *TrueBlock,
2296 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
2297 if (!CGF.HaveInsertPoint())
2298 return;
2299 {
2300 CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
2301 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
2302 (void)PreCondScope.Privatize();
2303 // Get initial values of real counters.
2304 for (const Expr *I : S.inits()) {
2305 CGF.EmitIgnoredExpr(I);
2306 }
2307 }
2308 // Create temp loop control variables with their init values to support
2309 // non-rectangular loops.
2310 CodeGenFunction::OMPMapVars PreCondVars;
2311 for (const Expr *E : S.dependent_counters()) {
2312 if (!E)
2313 continue;
2314 assert(!E->getType().getNonReferenceType()->isRecordType() &&
2315 "dependent counter must not be an iterator.");
2316 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2317 Address CounterAddr =
2319 (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr);
2320 }
2321 (void)PreCondVars.apply(CGF);
2322 for (const Expr *E : S.dependent_inits()) {
2323 if (!E)
2324 continue;
2325 CGF.EmitIgnoredExpr(E);
2326 }
2327 // Check that loop is executed at least one time.
2328 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
2329 PreCondVars.restore(CGF);
2330}
2331
2333 const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
2334 if (!HaveInsertPoint())
2335 return;
2338 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
2339 for (const Expr *C : LoopDirective->counters()) {
2340 SIMDLCVs.insert(
2341 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
2342 }
2343 }
2344 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2345 auto CurPrivate = C->privates().begin();
2346 for (const Expr *E : C->varlists()) {
2347 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2348 const auto *PrivateVD =
2349 cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
2350 if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
2351 // Emit private VarDecl with copy init.
2352 EmitVarDecl(*PrivateVD);
2353 bool IsRegistered =
2354 PrivateScope.addPrivate(VD, GetAddrOfLocalVar(PrivateVD));
2355 assert(IsRegistered && "linear var already registered as private");
2356 // Silence the warning about unused variable.
2357 (void)IsRegistered;
2358 } else {
2359 EmitVarDecl(*PrivateVD);
2360 }
2361 ++CurPrivate;
2362 }
2363 }
2364}
2365
2367 const OMPExecutableDirective &D) {
2368 if (!CGF.HaveInsertPoint())
2369 return;
2370 if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
2371 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
2372 /*ignoreResult=*/true);
2373 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
2374 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
2375 // In presence of finite 'safelen', it may be unsafe to mark all
2376 // the memory instructions parallel, because loop-carried
2377 // dependences of 'safelen' iterations are possible.
2379 } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
2380 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
2381 /*ignoreResult=*/true);
2382 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
2383 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
2384 // In presence of finite 'safelen', it may be unsafe to mark all
2385 // the memory instructions parallel, because loop-carried
2386 // dependences of 'safelen' iterations are possible.
2387 CGF.LoopStack.setParallel(/*Enable=*/false);
2388 }
2389}
2390
2392 // Walk clauses and process safelen/lastprivate.
2393 LoopStack.setParallel(/*Enable=*/true);
2395 emitSimdlenSafelenClause(*this, D);
2396 if (const auto *C = D.getSingleClause<OMPOrderClause>())
2397 if (C->getKind() == OMPC_ORDER_concurrent)
2398 LoopStack.setParallel(/*Enable=*/true);
2399 if ((D.getDirectiveKind() == OMPD_simd ||
2400 (getLangOpts().OpenMPSimd &&
2402 llvm::any_of(D.getClausesOfKind<OMPReductionClause>(),
2403 [](const OMPReductionClause *C) {
2404 return C->getModifier() == OMPC_REDUCTION_inscan;
2405 }))
2406 // Disable parallel access in case of prefix sum.
2407 LoopStack.setParallel(/*Enable=*/false);
2408}
2409
2411 const OMPLoopDirective &D,
2412 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
2413 if (!HaveInsertPoint())
2414 return;
2415 llvm::BasicBlock *DoneBB = nullptr;
2416 auto IC = D.counters().begin();
2417 auto IPC = D.private_counters().begin();
2418 for (const Expr *F : D.finals()) {
2419 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
2420 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
2421 const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
2422 if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
2423 OrigVD->hasGlobalStorage() || CED) {
2424 if (!DoneBB) {
2425 if (llvm::Value *Cond = CondGen(*this)) {
2426 // If the first post-update expression is found, emit conditional
2427 // block if it was requested.
2428 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then");
2429 DoneBB = createBasicBlock(".omp.final.done");
2430 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2431 EmitBlock(ThenBB);
2432 }
2433 }
2434 Address OrigAddr = Address::invalid();
2435 if (CED) {
2436 OrigAddr =
2437 EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress(*this);
2438 } else {
2439 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(PrivateVD),
2440 /*RefersToEnclosingVariableOrCapture=*/false,
2441 (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
2442 OrigAddr = EmitLValue(&DRE).getAddress(*this);
2443 }
2444 OMPPrivateScope VarScope(*this);
2445 VarScope.addPrivate(OrigVD, OrigAddr);
2446 (void)VarScope.Privatize();
2447 EmitIgnoredExpr(F);
2448 }
2449 ++IC;
2450 ++IPC;
2451 }
2452 if (DoneBB)
2453 EmitBlock(DoneBB, /*IsFinished=*/true);
2454}
2455
2457 const OMPLoopDirective &S,
2458 CodeGenFunction::JumpDest LoopExit) {
2459 CGF.EmitOMPLoopBody(S, LoopExit);
2460 CGF.EmitStopPoint(&S);
2461}
2462
2463/// Emit a helper variable and return corresponding lvalue.
2465 const DeclRefExpr *Helper) {
2466 auto VDecl = cast<VarDecl>(Helper->getDecl());
2467 CGF.EmitVarDecl(*VDecl);
2468 return CGF.EmitLValue(Helper);
2469}
2470
2472 const RegionCodeGenTy &SimdInitGen,
2473 const RegionCodeGenTy &BodyCodeGen) {
2474 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](CodeGenFunction &CGF,
2475 PrePostActionTy &) {
2476 CGOpenMPRuntime::NontemporalDeclsRAII NontemporalsRegion(CGF.CGM, S);
2477 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
2478 SimdInitGen(CGF);
2479
2480 BodyCodeGen(CGF);
2481 };
2482 auto &&ElseGen = [&BodyCodeGen](CodeGenFunction &CGF, PrePostActionTy &) {
2483 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
2484 CGF.LoopStack.setVectorizeEnable(/*Enable=*/false);
2485
2486 BodyCodeGen(CGF);
2487 };
2488 const Expr *IfCond = nullptr;
2489 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
2490 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
2491 if (CGF.getLangOpts().OpenMP >= 50 &&
2492 (C->getNameModifier() == OMPD_unknown ||
2493 C->getNameModifier() == OMPD_simd)) {
2494 IfCond = C->getCondition();
2495 break;
2496 }
2497 }
2498 }
2499 if (IfCond) {
2500 CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, IfCond, ThenGen, ElseGen);
2501 } else {
2502 RegionCodeGenTy ThenRCG(ThenGen);
2503 ThenRCG(CGF);
2504 }
2505}
2506
2508 PrePostActionTy &Action) {
2509 Action.Enter(CGF);
2510 assert(isOpenMPSimdDirective(S.getDirectiveKind()) &&
2511 "Expected simd directive");
2512 OMPLoopScope PreInitScope(CGF, S);
2513 // if (PreCond) {
2514 // for (IV in 0..LastIteration) BODY;
2515 // <Final counter/linear vars updates>;
2516 // }
2517 //
2518 if (isOpenMPDistributeDirective(S.getDirectiveKind()) ||
2519 isOpenMPWorksharingDirective(S.getDirectiveKind()) ||
2520 isOpenMPTaskLoopDirective(S.getDirectiveKind())) {
2521 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));
2522 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));
2523 }
2524
2525 // Emit: if (PreCond) - begin.
2526 // If the condition constant folds and can be elided, avoid emitting the
2527 // whole loop.
2528 bool CondConstant;
2529 llvm::BasicBlock *ContBlock = nullptr;
2530 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2531 if (!CondConstant)
2532 return;
2533 } else {
2534 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");
2535 ContBlock = CGF.createBasicBlock("simd.if.end");
2536 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
2537 CGF.getProfileCount(&S));
2538 CGF.EmitBlock(ThenBlock);
2540 }
2541
2542 // Emit the loop iteration variable.
2543 const Expr *IVExpr = S.getIterationVariable();
2544 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
2545 CGF.EmitVarDecl(*IVDecl);
2546 CGF.EmitIgnoredExpr(S.getInit());
2547
2548 // Emit the iterations count variable.
2549 // If it is not a variable, Sema decided to calculate iterations count on
2550 // each iteration (e.g., it is foldable into a constant).
2551 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2552 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2553 // Emit calculation of the iterations count.
2554 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
2555 }
2556
2557 emitAlignedClause(CGF, S);
2558 (void)CGF.EmitOMPLinearClauseInit(S);
2559 {
2560 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2561 CGF.EmitOMPPrivateClause(S, LoopScope);
2562 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
2563 CGF.EmitOMPLinearClause(S, LoopScope);
2564 CGF.EmitOMPReductionClauseInit(S, LoopScope);
2566 CGF, S, CGF.EmitLValue(S.getIterationVariable()));
2567 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2568 (void)LoopScope.Privatize();
2569 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
2571
2573 CGF, S,
2574 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2575 CGF.EmitOMPSimdInit(S);
2576 },
2577 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
2578 CGF.EmitOMPInnerLoop(
2579 S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),
2580 [&S](CodeGenFunction &CGF) {
2581 emitOMPLoopBodyWithStopPoint(CGF, S,
2582 CodeGenFunction::JumpDest());
2583 },
2584 [](CodeGenFunction &) {});
2585 });
2586 CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; });
2587 // Emit final copy of the lastprivate variables at the end of loops.
2588 if (HasLastprivateClause)
2589 CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
2590 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
2592 [](CodeGenFunction &) { return nullptr; });
2593 LoopScope.restoreMap();
2594 CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; });
2595 }
2596 // Emit: if (PreCond) - end.
2597 if (ContBlock) {
2598 CGF.EmitBranch(ContBlock);
2599 CGF.EmitBlock(ContBlock, true);
2600 }
2601}
2602
2604 // Check for unsupported clauses
2605 for (OMPClause *C : S.clauses()) {
2606 // Currently only order, simdlen and safelen clauses are supported
2607 if (!(isa<OMPSimdlenClause>(C) || isa<OMPSafelenClause>(C) ||
2608 isa<OMPOrderClause>(C) || isa<OMPAlignedClause>(C)))
2609 return false;
2610 }
2611
2612 // Check if we have a statement with the ordered directive.
2613 // Visit the statement hierarchy to find a compound statement
2614 // with a ordered directive in it.
2615 if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(S.getRawStmt())) {
2616 if (const Stmt *SyntacticalLoop = CanonLoop->getLoopStmt()) {
2617 for (const Stmt *SubStmt : SyntacticalLoop->children()) {
2618 if (!SubStmt)
2619 continue;
2620 if (const CompoundStmt *CS = dyn_cast<CompoundStmt>(SubStmt)) {
2621 for (const Stmt *CSSubStmt : CS->children()) {
2622 if (!CSSubStmt)
2623 continue;
2624 if (isa<OMPOrderedDirective>(CSSubStmt)) {
2625 return false;
2626 }
2627 }
2628 }
2629 }
2630 }
2631 }
2632 return true;
2633}
2634static llvm::MapVector<llvm::Value *, llvm::Value *>
2636 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars;
2637 for (const auto *Clause : S.getClausesOfKind<OMPAlignedClause>()) {
2638 llvm::APInt ClauseAlignment(64, 0);
2639 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
2640 auto *AlignmentCI =
2641 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
2642 ClauseAlignment = AlignmentCI->getValue();
2643 }
2644 for (const Expr *E : Clause->varlists()) {
2645 llvm::APInt Alignment(ClauseAlignment);
2646 if (Alignment == 0) {
2647 // OpenMP [2.8.1, Description]
2648 // If no optional parameter is specified, implementation-defined default
2649 // alignments for SIMD instructions on the target platforms are assumed.
2650 Alignment =
2651 CGF.getContext()
2653 E->getType()->getPointeeType()))
2654 .getQuantity();
2655 }
2656 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2657 "alignment is not power of 2");
2658 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
2659 AlignedVars[PtrValue] = CGF.Builder.getInt64(Alignment.getSExtValue());
2660 }
2661 }
2662 return AlignedVars;
2663}
2664
2666 bool UseOMPIRBuilder =
2667 CGM.getLangOpts().OpenMPIRBuilder && isSupportedByOpenMPIRBuilder(S);
2668 if (UseOMPIRBuilder) {
2669 auto &&CodeGenIRBuilder = [this, &S, UseOMPIRBuilder](CodeGenFunction &CGF,
2670 PrePostActionTy &) {
2671 // Use the OpenMPIRBuilder if enabled.
2672 if (UseOMPIRBuilder) {
2673 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars =
2674 GetAlignedMapping(S, CGF);
2675 // Emit the associated statement and get its loop representation.
2676 const Stmt *Inner = S.getRawStmt();
2677 llvm::CanonicalLoopInfo *CLI =
2679
2680 llvm::OpenMPIRBuilder &OMPBuilder =
2682 // Add SIMD specific metadata
2683 llvm::ConstantInt *Simdlen = nullptr;
2684 if (const auto *C = S.getSingleClause<OMPSimdlenClause>()) {
2685 RValue Len =
2686 this->EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
2687 /*ignoreResult=*/true);
2688 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
2689 Simdlen = Val;
2690 }
2691 llvm::ConstantInt *Safelen = nullptr;
2692 if (const auto *C = S.getSingleClause<OMPSafelenClause>()) {
2693 RValue Len =
2694 this->EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
2695 /*ignoreResult=*/true);
2696 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
2697 Safelen = Val;
2698 }
2699 llvm::omp::OrderKind Order = llvm::omp::OrderKind::OMP_ORDER_unknown;
2700 if (const auto *C = S.getSingleClause<OMPOrderClause>()) {
2701 if (C->getKind() == OpenMPOrderClauseKind ::OMPC_ORDER_concurrent) {
2702 Order = llvm::omp::OrderKind::OMP_ORDER_concurrent;
2703 }
2704 }
2705 // Add simd metadata to the collapsed loop. Do not generate
2706 // another loop for if clause. Support for if clause is done earlier.
2707 OMPBuilder.applySimd(CLI, AlignedVars,
2708 /*IfCond*/ nullptr, Order, Simdlen, Safelen);
2709 return;
2710 }
2711 };
2712 {
2713 auto LPCRegion =
2715 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2716 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd,
2717 CodeGenIRBuilder);
2718 }
2719 return;
2720 }
2721
2722 ParentLoopDirectiveForScanRegion ScanRegion(*this, S);
2723 OMPFirstScanLoop = true;
2724 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2725 emitOMPSimdRegion(CGF, S, Action);
2726 };
2727 {
2728 auto LPCRegion =
2730 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2731 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2732 }
2733 // Check for outer lastprivate conditional update.
2735}
2736
2738 // Emit the de-sugared statement.
2739 OMPTransformDirectiveScopeRAII TileScope(*this, &S);
2740 EmitStmt(S.getTransformedStmt());
2741}
2742
2744 bool UseOMPIRBuilder = CGM.getLangOpts().OpenMPIRBuilder;
2745
2746 if (UseOMPIRBuilder) {
2747 auto DL = SourceLocToDebugLoc(S.getBeginLoc());
2748 const Stmt *Inner = S.getRawStmt();
2749
2750 // Consume nested loop. Clear the entire remaining loop stack because a
2751 // fully unrolled loop is non-transformable. For partial unrolling the
2752 // generated outer loop is pushed back to the stack.
2753 llvm::CanonicalLoopInfo *CLI = EmitOMPCollapsedCanonicalLoopNest(Inner, 1);
2754 OMPLoopNestStack.clear();
2755
2756 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2757
2758 bool NeedsUnrolledCLI = ExpectedOMPLoopDepth >= 1;
2759 llvm::CanonicalLoopInfo *UnrolledCLI = nullptr;
2760
2761 if (S.hasClausesOfKind<OMPFullClause>()) {
2762 assert(ExpectedOMPLoopDepth == 0);
2763 OMPBuilder.unrollLoopFull(DL, CLI);
2764 } else if (auto *PartialClause = S.getSingleClause<OMPPartialClause>()) {
2765 uint64_t Factor = 0;
2766 if (Expr *FactorExpr = PartialClause->getFactor()) {
2767 Factor = FactorExpr->EvaluateKnownConstInt(getContext()).getZExtValue();
2768 assert(Factor >= 1 && "Only positive factors are valid");
2769 }
2770 OMPBuilder.unrollLoopPartial(DL, CLI, Factor,
2771 NeedsUnrolledCLI ? &UnrolledCLI : nullptr);
2772 } else {
2773 OMPBuilder.unrollLoopHeuristic(DL, CLI);
2774 }
2775
2776 assert((!NeedsUnrolledCLI || UnrolledCLI) &&
2777 "NeedsUnrolledCLI implies UnrolledCLI to be set");
2778 if (UnrolledCLI)
2779 OMPLoopNestStack.push_back(UnrolledCLI);
2780
2781 return;
2782 }
2783
2784 // This function is only called if the unrolled loop is not consumed by any
2785 // other loop-associated construct. Such a loop-associated construct will have
2786 // used the transformed AST.
2787
2788 // Set the unroll metadata for the next emitted loop.
2790
2791 if (S.hasClausesOfKind<OMPFullClause>()) {
2793 } else if (auto *PartialClause = S.getSingleClause<OMPPartialClause>()) {
2794 if (Expr *FactorExpr = PartialClause->getFactor()) {
2795 uint64_t Factor =
2796 FactorExpr->EvaluateKnownConstInt(getContext()).getZExtValue();
2797 assert(Factor >= 1 && "Only positive factors are valid");
2798 LoopStack.setUnrollCount(Factor);
2799 }
2800 }
2801
2802 EmitStmt(S.getAssociatedStmt());
2803}
2804
2805void CodeGenFunction::EmitOMPOuterLoop(
2806 bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
2807 CodeGenFunction::OMPPrivateScope &LoopScope,
2808 const CodeGenFunction::OMPLoopArguments &LoopArgs,
2809 const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
2810 const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
2812
2813 const Expr *IVExpr = S.getIterationVariable();
2814 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2815 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2816
2817 JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
2818
2819 // Start the loop with a block that tests the condition.
2820 llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond");
2821 EmitBlock(CondBlock);
2822 const SourceRange R = S.getSourceRange();
2823 OMPLoopNestStack.clear();
2826
2827 llvm::Value *BoolCondVal = nullptr;
2828 if (!DynamicOrOrdered) {
2829 // UB = min(UB, GlobalUB) or
2830 // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
2831 // 'distribute parallel for')
2832 EmitIgnoredExpr(LoopArgs.EUB);
2833 // IV = LB
2834 EmitIgnoredExpr(LoopArgs.Init);
2835 // IV < UB
2836 BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
2837 } else {
2838 BoolCondVal =
2839 RT.emitForNext(*this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
2840 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
2841 }
2842
2843 // If there are any cleanups between here and the loop-exit scope,
2844 // create a block to stage a loop exit along.
2845 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
2846 if (LoopScope.requiresCleanups())
2847 ExitBlock = createBasicBlock("omp.dispatch.cleanup");
2848
2849 llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body");
2850 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
2851 if (ExitBlock != LoopExit.getBlock()) {
2852 EmitBlock(ExitBlock);
2854 }
2855 EmitBlock(LoopBody);
2856
2857 // Emit "IV = LB" (in case of static schedule, we have already calculated new
2858 // LB for loop condition and emitted it above).
2859 if (DynamicOrOrdered)
2860 EmitIgnoredExpr(LoopArgs.Init);
2861
2862 // Create a block for the increment.
2863 JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
2864 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
2865
2867 *this, S,
2868 [&S, IsMonotonic](CodeGenFunction &CGF, PrePostActionTy &) {
2869 // Generate !llvm.loop.parallel metadata for loads and stores for loops
2870 // with dynamic/guided scheduling and without ordered clause.
2871 if (!isOpenMPSimdDirective(S.getDirectiveKind())) {
2872 CGF.LoopStack.setParallel(!IsMonotonic);
2873 if (const auto *C = S.getSingleClause<OMPOrderClause>())
2874 if (C->getKind() == OMPC_ORDER_concurrent)
2875 CGF.LoopStack.setParallel(/*Enable=*/true);
2876 } else {
2877 CGF.EmitOMPSimdInit(S);
2878 }
2879 },
2880 [&S, &LoopArgs, LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered,
2881 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
2882 SourceLocation Loc = S.getBeginLoc();
2883 // when 'distribute' is not combined with a 'for':
2884 // while (idx <= UB) { BODY; ++idx; }
2885 // when 'distribute' is combined with a 'for'
2886 // (e.g. 'distribute parallel for')
2887 // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
2888 CGF.EmitOMPInnerLoop(
2889 S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
2890 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
2891 CodeGenLoop(CGF, S, LoopExit);
2892 },
2893 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
2894 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
2895 });
2896 });
2897
2898 EmitBlock(Continue.getBlock());
2899 BreakContinueStack.pop_back();
2900 if (!DynamicOrOrdered) {
2901 // Emit "LB = LB + Stride", "UB = UB + Stride".
2902 EmitIgnoredExpr(LoopArgs.NextLB);
2903 EmitIgnoredExpr(LoopArgs.NextUB);
2904 }
2905
2906 EmitBranch(CondBlock);
2907 OMPLoopNestStack.clear();
2908 LoopStack.pop();
2909 // Emit the fall-through block.
2910 EmitBlock(LoopExit.getBlock());
2911
2912 // Tell the runtime we are done.
2913 auto &&CodeGen = [DynamicOrOrdered, &S, &LoopArgs](CodeGenFunction &CGF) {
2914 if (!DynamicOrOrdered)
2915 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2916 LoopArgs.DKind);
2917 };
2918 OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
2919}
2920
2921void CodeGenFunction::EmitOMPForOuterLoop(
2922 const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
2923 const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
2924 const OMPLoopArguments &LoopArgs,
2925 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
2927
2928 // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
2929 const bool DynamicOrOrdered = Ordered || RT.isDynamic(ScheduleKind.Schedule);
2930
2931 assert((Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule,
2932 LoopArgs.Chunk != nullptr)) &&
2933 "static non-chunked schedule does not need outer loop");
2934
2935 // Emit outer loop.
2936 //
2937 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2938 // When schedule(dynamic,chunk_size) is specified, the iterations are
2939 // distributed to threads in the team in chunks as the threads request them.
2940 // Each thread executes a chunk of iterations, then requests another chunk,
2941 // until no chunks remain to be distributed. Each chunk contains chunk_size
2942 // iterations, except for the last chunk to be distributed, which may have
2943 // fewer iterations. When no chunk_size is specified, it defaults to 1.
2944 //
2945 // When schedule(guided,chunk_size) is specified, the iterations are assigned
2946 // to threads in the team in chunks as the executing threads request them.
2947 // Each thread executes a chunk of iterations, then requests another chunk,
2948 // until no chunks remain to be assigned. For a chunk_size of 1, the size of
2949 // each chunk is proportional to the number of unassigned iterations divided
2950 // by the number of threads in the team, decreasing to 1. For a chunk_size
2951 // with value k (greater than 1), the size of each chunk is determined in the
2952 // same way, with the restriction that the chunks do not contain fewer than k
2953 // iterations (except for the last chunk to be assigned, which may have fewer
2954 // than k iterations).
2955 //
2956 // When schedule(auto) is specified, the decision regarding scheduling is
2957 // delegated to the compiler and/or runtime system. The programmer gives the
2958 // implementation the freedom to choose any possible mapping of iterations to
2959 // threads in the team.
2960 //
2961 // When schedule(runtime) is specified, the decision regarding scheduling is
2962 // deferred until run time, and the schedule and chunk size are taken from the
2963 // run-sched-var ICV. If the ICV is set to auto, the schedule is
2964 // implementation defined
2965 //
2966 // while(__kmpc_dispatch_next(&LB, &UB)) {
2967 // idx = LB;
2968 // while (idx <= UB) { BODY; ++idx;
2969 // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
2970 // } // inner loop
2971 // }
2972 //
2973 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2974 // When schedule(static, chunk_size) is specified, iterations are divided into
2975 // chunks of size chunk_size, and the chunks are assigned to the threads in
2976 // the team in a round-robin fashion in the order of the thread number.
2977 //
2978 // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
2979 // while (idx <= UB) { BODY; ++idx; } // inner loop
2980 // LB = LB + ST;
2981 // UB = UB + ST;
2982 // }
2983 //
2984
2985 const Expr *IVExpr = S.getIterationVariable();
2986 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2987 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2988
2989 if (DynamicOrOrdered) {
2990 const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
2991 CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
2992 llvm::Value *LBVal = DispatchBounds.first;
2993 llvm::Value *UBVal = DispatchBounds.second;
2994 CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
2995 LoopArgs.Chunk};
2996 RT.emitForDispatchInit(*this, S.getBeginLoc(), ScheduleKind, IVSize,
2997 IVSigned, Ordered, DipatchRTInputValues);
2998 } else {
3000 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
3001 LoopArgs.ST, LoopArgs.Chunk);
3002 RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
3003 ScheduleKind, StaticInit);
3004 }
3005
3006 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
3007 const unsigned IVSize,
3008 const bool IVSigned) {
3009 if (Ordered) {
3010 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
3011 IVSigned);
3012 }
3013 };
3014
3015 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
3016 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
3017 OuterLoopArgs.IncExpr = S.getInc();
3018 OuterLoopArgs.Init = S.getInit();
3019 OuterLoopArgs.Cond = S.getCond();
3020 OuterLoopArgs.NextLB = S.getNextLowerBound();
3021 OuterLoopArgs.NextUB = S.getNextUpperBound();
3022 OuterLoopArgs.DKind = LoopArgs.DKind;
3023 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
3024 emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
3025}
3026
3028 const unsigned IVSize, const bool IVSigned) {}
3029
3030void CodeGenFunction::EmitOMPDistributeOuterLoop(
3031 OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
3032 OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
3033 const CodeGenLoopTy &CodeGenLoopContent) {
3034
3036
3037 // Emit outer loop.
3038 // Same behavior as a OMPForOuterLoop, except that schedule cannot be
3039 // dynamic
3040 //
3041
3042 const Expr *IVExpr = S.getIterationVariable();
3043 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3044 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3045
3047 IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
3048 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
3049 RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, StaticInit);
3050
3051 // for combined 'distribute' and 'for' the increment expression of distribute
3052 // is stored in DistInc. For 'distribute' alone, it is in Inc.
3053 Expr *IncExpr;
3054 if (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()))
3055 IncExpr = S.getDistInc();
3056 else
3057 IncExpr = S.getInc();
3058
3059 // this routine is shared by 'omp distribute parallel for' and
3060 // 'omp distribute': select the right EUB expression depending on the
3061 // directive
3062 OMPLoopArguments OuterLoopArgs;
3063 OuterLoopArgs.LB = LoopArgs.LB;
3064 OuterLoopArgs.UB = LoopArgs.UB;
3065 OuterLoopArgs.ST = LoopArgs.ST;
3066 OuterLoopArgs.IL = LoopArgs.IL;
3067 OuterLoopArgs.Chunk = LoopArgs.Chunk;
3068 OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3069 ? S.getCombinedEnsureUpperBound()
3070 : S.getEnsureUpperBound();
3071 OuterLoopArgs.IncExpr = IncExpr;
3072 OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3073 ? S.getCombinedInit()
3074 : S.getInit();
3075 OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3076 ? S.getCombinedCond()
3077 : S.getCond();
3078 OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3079 ? S.getCombinedNextLowerBound()
3080 : S.getNextLowerBound();
3081 OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3082 ? S.getCombinedNextUpperBound()
3083 : S.getNextUpperBound();
3084 OuterLoopArgs.DKind = OMPD_distribute;
3085
3086 EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
3087 LoopScope, OuterLoopArgs, CodeGenLoopContent,
3089}
3090
3091static std::pair<LValue, LValue>
3093 const OMPExecutableDirective &S) {
3094 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
3095 LValue LB =
3096 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
3097 LValue UB =
3098 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
3099
3100 // When composing 'distribute' with 'for' (e.g. as in 'distribute
3101 // parallel for') we need to use the 'distribute'
3102 // chunk lower and upper bounds rather than the whole loop iteration
3103 // space. These are parameters to the outlined function for 'parallel'
3104 // and we copy the bounds of the previous schedule into the
3105 // the current ones.
3106 LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
3107 LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
3108 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
3109 PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());
3110 PrevLBVal = CGF.EmitScalarConversion(
3111 PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
3114 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
3115 PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());
3116 PrevUBVal = CGF.EmitScalarConversion(
3117 PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
3120
3121 CGF.EmitStoreOfScalar(PrevLBVal, LB);
3122 CGF.EmitStoreOfScalar(PrevUBVal, UB);
3123
3124 return {LB, UB};
3125}
3126
3127/// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
3128/// we need to use the LB and UB expressions generated by the worksharing
3129/// code generation support, whereas in non combined situations we would
3130/// just emit 0 and the LastIteration expression
3131/// This function is necessary due to the difference of the LB and UB
3132/// types for the RT emission routines for 'for_static_init' and
3133/// 'for_dispatch_init'
3134static std::pair<llvm::Value *, llvm::Value *>
3136 const OMPExecutableDirective &S,
3137 Address LB, Address UB) {
3138 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
3139 const Expr *IVExpr = LS.getIterationVariable();
3140 // when implementing a dynamic schedule for a 'for' combined with a
3141 // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
3142 // is not normalized as each team only executes its own assigned
3143 // distribute chunk
3144 QualType IteratorTy = IVExpr->getType();
3145 llvm::Value *LBVal =
3146 CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
3147 llvm::Value *UBVal =
3148 CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
3149 return {LBVal, UBVal};
3150}
3151
3155 const auto &Dir = cast<OMPLoopDirective>(S);
3156 LValue LB =
3157 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
3158 llvm::Value *LBCast =
3159 CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(LB.getAddress(CGF)),
3160 CGF.SizeTy, /*isSigned=*/false);
3161 CapturedVars.push_back(LBCast);
3162 LValue UB =
3163 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
3164
3165 llvm::Value *UBCast =
3166 CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(UB.getAddress(CGF)),
3167 CGF.SizeTy, /*isSigned=*/false);
3168 CapturedVars.push_back(UBCast);
3169}
3170
3171static void
3173 const OMPLoopDirective &S,
3174 CodeGenFunction::JumpDest LoopExit) {
3175 auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
3176 PrePostActionTy &Action) {
3177 Action.Enter(CGF);
3178 bool HasCancel = false;
3179 if (!isOpenMPSimdDirective(S.getDirectiveKind())) {
3180 if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
3181 HasCancel = D->hasCancel();
3182 else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
3183 HasCancel = D->hasCancel();
3184 else if (const auto *D =
3185 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
3186 HasCancel = D->hasCancel();
3187 }
3188 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
3189 HasCancel);
3190 CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(),
3193 };
3194
3196 CGF, S,
3197 isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for,
3198 CGInlinedWorksharingLoop,
3200}
3201
3204 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3206 S.getDistInc());
3207 };
3208 OMPLexicalScope Scope(*this, S, OMPD_parallel);
3209 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3210}
3211
3214 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3216 S.getDistInc());
3217 };
3218 OMPLexicalScope Scope(*this, S, OMPD_parallel);
3219 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3220}
3221
3223 const OMPDistributeSimdDirective &S) {
3224 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3226 };
3227 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3228 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
3229}
3230
3232 CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
3233 // Emit SPMD target parallel for region as a standalone region.
3234 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3235 emitOMPSimdRegion(CGF, S, Action);
3236 };
3237 llvm::Function *Fn;
3238 llvm::Constant *Addr;
3239 // Emit target region as a standalone region.
3241 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
3242 assert(Fn && Addr && "Target device function emission failed.");
3243}
3244
3246 const OMPTargetSimdDirective &S) {
3247 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3248 emitOMPSimdRegion(CGF, S, Action);
3249 };
3250 emitCommonOMPTargetDirective(*this, S, CodeGen);
3251}
3252
3253namespace {
3254struct ScheduleKindModifiersTy {
3258 ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
3261 : Kind(Kind), M1(M1), M2(M2) {}
3262};
3263} // namespace
3264
3266 const OMPLoopDirective &S, Expr *EUB,
3267 const CodeGenLoopBoundsTy &CodeGenLoopBounds,
3268 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
3269 // Emit the loop iteration variable.
3270 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3271 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3272 EmitVarDecl(*IVDecl);
3273
3274 // Emit the iterations count variable.
3275 // If it is not a variable, Sema decided to calculate iterations count on each
3276 // iteration (e.g., it is foldable into a constant).
3277 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3278 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3279 // Emit calculation of the iterations count.
3280 EmitIgnoredExpr(S.getCalcLastIteration());
3281 }
3282
3284
3285 bool HasLastprivateClause;
3286 // Check pre-condition.
3287 {
3288 OMPLoopScope PreInitScope(*this, S);
3289 // Skip the entire loop if we don't meet the precondition.
3290 // If the condition constant folds and can be elided, avoid emitting the
3291 // whole loop.
3292 bool CondConstant;
3293 llvm::BasicBlock *ContBlock = nullptr;
3294 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
3295 if (!CondConstant)
3296 return false;
3297 } else {
3298 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
3299 ContBlock = createBasicBlock("omp.precond.end");
3300 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
3301 getProfileCount(&S));
3302 EmitBlock(ThenBlock);
3304 }
3305
3306 RunCleanupsScope DoacrossCleanupScope(*this);
3307 bool Ordered = false;
3308 if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
3309 if (OrderedClause->getNumForLoops())
3310 RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations());
3311 else
3312 Ordered = true;
3313 }
3314
3315 llvm::DenseSet<const Expr *> EmittedFinals;
3316 emitAlignedClause(*this, S);
3317 bool HasLinears = EmitOMPLinearClauseInit(S);
3318 // Emit helper vars inits.
3319
3320 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
3321 LValue LB = Bounds.first;
3322 LValue UB = Bounds.second;
3323 LValue ST =
3324 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
3325 LValue IL =
3326 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
3327
3328 // Emit 'then' code.
3329 {
3330 OMPPrivateScope LoopScope(*this);
3331 if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
3332 // Emit implicit barrier to synchronize threads and avoid data races on
3333 // initialization of firstprivate variables and post-update of
3334 // lastprivate variables.
3336 *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
3337 /*ForceSimpleCall=*/true);
3338 }
3339 EmitOMPPrivateClause(S, LoopScope);
3341 *this, S, EmitLValue(S.getIterationVariable()));
3342 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
3343 EmitOMPReductionClauseInit(S, LoopScope);
3344 EmitOMPPrivateLoopCounters(S, LoopScope);
3345 EmitOMPLinearClause(S, LoopScope);
3346 (void)LoopScope.Privatize();
3347 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
3349
3350 // Detect the loop schedule kind and chunk.
3351 const Expr *ChunkExpr = nullptr;
3352 OpenMPScheduleTy ScheduleKind;
3353 if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
3354 ScheduleKind.Schedule = C->getScheduleKind();
3355 ScheduleKind.M1 = C->getFirstScheduleModifier();
3356 ScheduleKind.M2 = C->getSecondScheduleModifier();
3357 ChunkExpr = C->getChunkSize();
3358 } else {
3359 // Default behaviour for schedule clause.
3361 *this, S, ScheduleKind.Schedule, ChunkExpr);
3362 }
3363 bool HasChunkSizeOne = false;
3364 llvm::Value *Chunk = nullptr;
3365 if (ChunkExpr) {
3366 Chunk = EmitScalarExpr(ChunkExpr);
3367 Chunk = EmitScalarConversion(Chunk, ChunkExpr->getType(),
3368 S.getIterationVariable()->getType(),
3369 S.getBeginLoc());
3371 if (ChunkExpr->EvaluateAsInt(Result, getContext())) {
3372 llvm::APSInt EvaluatedChunk = Result.Val.getInt();
3373 HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
3374 }
3375 }
3376 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3377 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3378 // OpenMP 4.5, 2.7.1 Loop Construct, Description.
3379 // If the static schedule kind is specified or if the ordered clause is
3380 // specified, and if no monotonic modifier is specified, the effect will
3381 // be as if the monotonic modifier was specified.
3382 bool StaticChunkedOne =
3383 RT.isStaticChunked(ScheduleKind.Schedule,
3384 /* Chunked */ Chunk != nullptr) &&
3385 HasChunkSizeOne &&
3386 isOpenMPLoopBoundSharingDirective(S.getDirectiveKind());
3387 bool IsMonotonic =
3388 Ordered ||
3389 (ScheduleKind.Schedule == OMPC_SCHEDULE_static &&
3390 !(ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_nonmonotonic ||
3391 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_nonmonotonic)) ||
3392 ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
3393 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
3394 if ((RT.isStaticNonchunked(ScheduleKind.Schedule,
3395 /* Chunked */ Chunk != nullptr) ||
3396 StaticChunkedOne) &&
3397 !Ordered) {
3398 JumpDest LoopExit =
3401 *this, S,
3402 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3403 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
3404 CGF.EmitOMPSimdInit(S);
3405 } else if (const auto *C = S.getSingleClause<OMPOrderClause>()) {
3406 if (C->getKind() == OMPC_ORDER_concurrent)
3407 CGF.LoopStack.setParallel(/*Enable=*/true);
3408 }
3409 },
3410 [IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk,
3411 &S, ScheduleKind, LoopExit,
3412 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
3413 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
3414 // When no chunk_size is specified, the iteration space is divided
3415 // into chunks that are approximately equal in size, and at most
3416 // one chunk is distributed to each thread. Note that the size of
3417 // the chunks is unspecified in this case.
3419 IVSize, IVSigned, Ordered, IL.getAddress(CGF),
3420 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF),
3421 StaticChunkedOne ? Chunk : nullptr);
3422 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
3423 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind,
3424 StaticInit);
3425 // UB = min(UB, GlobalUB);
3426 if (!StaticChunkedOne)
3427 CGF.EmitIgnoredExpr(S.getEnsureUpperBound());
3428 // IV = LB;
3429 CGF.EmitIgnoredExpr(S.getInit());
3430 // For unchunked static schedule generate:
3431 //
3432 // while (idx <= UB) {
3433 // BODY;
3434 // ++idx;
3435 // }
3436 //
3437 // For static schedule with chunk one:
3438 //
3439 // while (IV <= PrevUB) {
3440 // BODY;
3441 // IV += ST;
3442 // }
3443 CGF.EmitOMPInnerLoop(
3444 S, LoopScope.requiresCleanups(),
3445 StaticChunkedOne ? S.getCombinedParForInDistCond()
3446 : S.getCond(),
3447 StaticChunkedOne ? S.getDistInc() : S.getInc(),
3448 [&S, LoopExit](CodeGenFunction &CGF) {
3449 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit);
3450 },
3451 [](CodeGenFunction &) {});
3452 });
3453 EmitBlock(LoopExit.getBlock());
3454 // Tell the runtime we are done.
3455 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
3456 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
3457 OMPD_for);
3458 };
3459 OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
3460 } else {
3461 // Emit the outer loop, which requests its work chunk [LB..UB] from
3462 // runtime and runs the inner loop to process it.
3463 OMPLoopArguments LoopArguments(
3464 LB.getAddress(*this), UB.getAddress(*this), ST.getAddress(*this),
3465 IL.getAddress(*this), Chunk, EUB);
3466 LoopArguments.DKind = OMPD_for;
3467 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
3468 LoopArguments, CGDispatchBounds);
3469 }
3470 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
3471 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
3472 return CGF.Builder.CreateIsNotNull(
3473 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3474 });
3475 }
3477 S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind())
3478 ? /*Parallel and Simd*/ OMPD_parallel_for_simd
3479 : /*Parallel only*/ OMPD_parallel);
3480 // Emit post-update of the reduction variables if IsLastIter != 0.
3482 *this, S, [IL, &S](CodeGenFunction &CGF) {
3483 return CGF.Builder.CreateIsNotNull(
3484 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3485 });
3486 // Emit final copy of the lastprivate variables if IsLastIter != 0.
3487 if (HasLastprivateClause)
3489 S, isOpenMPSimdDirective(S.getDirectiveKind()),
3490 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
3491 LoopScope.restoreMap();
3492 EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) {
3493 return CGF.Builder.CreateIsNotNull(
3494 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3495 });
3496 }
3497 DoacrossCleanupScope.ForceCleanup();
3498 // We're now done with the loop, so jump to the continuation block.
3499 if (ContBlock) {
3500 EmitBranch(ContBlock);
3501 EmitBlock(ContBlock, /*IsFinished=*/true);
3502 }
3503 }
3504 return HasLastprivateClause;
3505}
3506
3507/// The following two functions generate expressions for the loop lower
3508/// and upper bounds in case of static and dynamic (dispatch) schedule
3509/// of the associated 'for' or 'distribute' loop.
3510static std::pair<LValue, LValue>
3512 const auto &LS = cast<OMPLoopDirective>(S);
3513 LValue LB =
3514 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
3515 LValue UB =
3516 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
3517 return {LB, UB};
3518}
3519
3520/// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
3521/// consider the lower and upper bound expressions generated by the
3522/// worksharing loop support, but we use 0 and the iteration space size as
3523/// constants
3524static std::pair<llvm::Value *, llvm::Value *>
3526 Address LB, Address UB) {
3527 const auto &LS = cast<OMPLoopDirective>(S);
3528 const Expr *IVExpr = LS.getIterationVariable();
3529 const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
3530 llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
3531 llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
3532 return {LBVal, UBVal};
3533}
3534
3535/// Emits internal temp array declarations for the directive with inscan
3536/// reductions.
3537/// The code is the following:
3538/// \code
3539/// size num_iters = <num_iters>;
3540/// <type> buffer[num_iters];
3541/// \endcode
3543 CodeGenFunction &CGF, const OMPLoopDirective &S,
3544 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen) {
3545 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
3546 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
3549 SmallVector<const Expr *, 4> ReductionOps;
3550 SmallVector<const Expr *, 4> CopyArrayTemps;
3551 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
3552 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
3553 "Only inscan reductions are expected.");
3554 Shareds.append(C->varlist_begin(), C->varlist_end());
3555 Privates.append(C->privates().begin(), C->privates().end());
3556 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
3557 CopyArrayTemps.append(C->copy_array_temps().begin(),
3558 C->copy_array_temps().end());
3559 }
3560 {
3561 // Emit buffers for each reduction variables.
3562 // ReductionCodeGen is required to emit correctly the code for array
3563 // reductions.
3564 ReductionCodeGen RedCG(Shareds, Shareds, Privates, ReductionOps);
3565 unsigned Count = 0;
3566 auto *ITA = CopyArrayTemps.begin();
3567 for (const Expr *IRef : Privates) {
3568 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
3569 // Emit variably modified arrays, used for arrays/array sections
3570 // reductions.
3571 if (PrivateVD->getType()->isVariablyModifiedType()) {
3572 RedCG.emitSharedOrigLValue(CGF, Count);
3573 RedCG.emitAggregateType(CGF, Count);
3574 }
3575 CodeGenFunction::OpaqueValueMapping DimMapping(
3576 CGF,
3577 cast<OpaqueValueExpr>(
3578 cast<VariableArrayType>((*ITA)->getType()->getAsArrayTypeUnsafe())
3579 ->getSizeExpr()),
3580 RValue::get(OMPScanNumIterations));
3581 // Emit temp buffer.
3582 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(*ITA)->getDecl()));
3583 ++ITA;
3584 ++Count;
3585 }
3586 }
3587}
3588
3589/// Copies final inscan reductions values to the original variables.
3590/// The code is the following:
3591/// \code
3592/// <orig_var> = buffer[num_iters-1];
3593/// \endcode
3595 CodeGenFunction &CGF, const OMPLoopDirective &S,
3596 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen) {
3597 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
3598 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
3604 SmallVector<const Expr *, 4> CopyArrayElems;
3605 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
3606 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
3607 "Only inscan reductions are expected.");
3608 Shareds.append(C->varlist_begin(), C->varlist_end());
3609 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
3610 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
3611 Privates.append(C->privates().begin(), C->privates().end());
3612 CopyOps.append(C->copy_ops().begin(), C->copy_ops().end());
3613 CopyArrayElems.append(C->copy_array_elems().begin(),
3614 C->copy_array_elems().end());
3615 }
3616 // Create temp var and copy LHS value to this temp value.
3617 // LHS = TMP[LastIter];
3618 llvm::Value *OMPLast = CGF.Builder.CreateNSWSub(
3619 OMPScanNumIterations,
3620 llvm::ConstantInt::get(CGF.SizeTy, 1, /*isSigned=*/false));
3621 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
3622 const Expr *PrivateExpr = Privates[I];
3623 const Expr *OrigExpr = Shareds[I];
3624 const Expr *CopyArrayElem = CopyArrayElems[I];
3625 CodeGenFunction::OpaqueValueMapping IdxMapping(
3626 CGF,
3627 cast<OpaqueValueExpr>(
3628 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3629 RValue::get(OMPLast));
3630 LValue DestLVal = CGF.EmitLValue(OrigExpr);
3631 LValue SrcLVal = CGF.EmitLValue(CopyArrayElem);
3632 CGF.EmitOMPCopy(PrivateExpr->getType(), DestLVal.getAddress(CGF),
3633 SrcLVal.getAddress(CGF),
3634 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
3635 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
3636 CopyOps[I]);
3637 }
3638}
3639
3640/// Emits the code for the directive with inscan reductions.
3641/// The code is the following:
3642/// \code
3643/// #pragma omp ...
3644/// for (i: 0..<num_iters>) {
3645/// <input phase>;
3646/// buffer[i] = red;
3647/// }
3648/// #pragma omp master // in parallel region
3649/// for (int k = 0; k != ceil(log2(num_iters)); ++k)
3650/// for (size cnt = last_iter; cnt >= pow(2, k); --k)
3651/// buffer[i] op= buffer[i-pow(2,k)];
3652/// #pragma omp barrier // in parallel region
3653/// #pragma omp ...
3654/// for (0..<num_iters>) {
3655/// red = InclusiveScan ? buffer[i] : buffer[i-1];
3656/// <scan phase>;
3657/// }
3658/// \endcode
3660 CodeGenFunction &CGF, const OMPLoopDirective &S,
3661 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen,
3662 llvm::function_ref<void(CodeGenFunction &)> FirstGen,
3663 llvm::function_ref<void(CodeGenFunction &)> SecondGen) {
3664 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
3665 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
3667 SmallVector<const Expr *, 4> ReductionOps;
3670 SmallVector<const Expr *, 4> CopyArrayElems;
3671 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
3672 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
3673 "Only inscan reductions are expected.");
3674 Privates.append(C->privates().begin(), C->privates().end());
3675 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
3676 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
3677 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
3678 CopyArrayElems.append(C->copy_array_elems().begin(),
3679 C->copy_array_elems().end());
3680 }
3681 CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S);
3682 {
3683 // Emit loop with input phase:
3684 // #pragma omp ...
3685 // for (i: 0..<num_iters>) {
3686 // <input phase>;
3687 // buffer[i] = red;
3688 // }
3689 CGF.OMPFirstScanLoop = true;
3690 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
3691 FirstGen(CGF);
3692 }
3693 // #pragma omp barrier // in parallel region
3694 auto &&CodeGen = [&S, OMPScanNumIterations, &LHSs, &RHSs, &CopyArrayElems,
3695 &ReductionOps,
3696 &Privates](CodeGenFunction &CGF, PrePostActionTy &Action) {
3697 Action.Enter(CGF);
3698 // Emit prefix reduction:
3699 // #pragma omp master // in parallel region
3700 // for (int k = 0; k <= ceil(log2(n)); ++k)
3701 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock();
3702 llvm::BasicBlock *LoopBB = CGF.createBasicBlock("omp.outer.log.scan.body");
3703 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.outer.log.scan.exit");
3704 llvm::Function *F =
3705 CGF.CGM.getIntrinsic(llvm::Intrinsic::log2, CGF.DoubleTy);
3706 llvm::Value *Arg =
3707 CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy);
3708 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg);
3709 F = CGF.CGM.getIntrinsic(llvm::Intrinsic::ceil, CGF.DoubleTy);
3710 LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal);
3711 LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy);
3712 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub(
3713 OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1));
3714 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getBeginLoc());
3715 CGF.EmitBlock(LoopBB);
3716 auto *Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2);
3717 // size pow2k = 1;
3718 auto *Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3719 Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB);
3720 Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB);
3721 // for (size i = n - 1; i >= 2 ^ k; --i)
3722 // tmp[i] op= tmp[i-pow2k];
3723 llvm::BasicBlock *InnerLoopBB =
3724 CGF.createBasicBlock("omp.inner.log.scan.body");
3725 llvm::BasicBlock *InnerExitBB =
3726 CGF.createBasicBlock("omp.inner.log.scan.exit");
3727 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K);
3728 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3729 CGF.EmitBlock(InnerLoopBB);
3730 auto *IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3731 IVal->addIncoming(NMin1, LoopBB);
3732 {
3733 CodeGenFunction::OMPPrivateScope PrivScope(CGF);
3734 auto *ILHS = LHSs.begin();
3735 auto *IRHS = RHSs.begin();
3736 for (const Expr *CopyArrayElem : CopyArrayElems) {
3737 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
3738 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
3739 Address LHSAddr = Address::invalid();
3740 {
3741 CodeGenFunction::OpaqueValueMapping IdxMapping(
3742 CGF,
3743 cast<OpaqueValueExpr>(
3744 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3745 RValue::get(IVal));
3746 LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF);
3747 }
3748 PrivScope.addPrivate(LHSVD, LHSAddr);
3749 Address RHSAddr = Address::invalid();
3750 {
3751 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K);
3752 CodeGenFunction::OpaqueValueMapping IdxMapping(
3753 CGF,
3754 cast<OpaqueValueExpr>(
3755 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3756 RValue::get(OffsetIVal));
3757 RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF);
3758 }
3759 PrivScope.addPrivate(RHSVD, RHSAddr);
3760 ++ILHS;
3761 ++IRHS;
3762 }
3763 PrivScope.Privatize();
3764 CGF.CGM.getOpenMPRuntime().emitReduction(
3765 CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
3766 {/*WithNowait=*/true, /*SimpleReduction=*/true, OMPD_unknown});
3767 }
3768 llvm::Value *NextIVal =
3769 CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1));
3770 IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock());
3771 CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K);
3772 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3773 CGF.EmitBlock(InnerExitBB);
3774 llvm::Value *Next =
3775 CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1));
3776 Counter->addIncoming(Next, CGF.Builder.GetInsertBlock());
3777 // pow2k <<= 1;
3778 llvm::Value *NextPow2K =
3779 CGF.Builder.CreateShl(Pow2K, 1, "", /*HasNUW=*/true);
3780 Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock());
3781 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(Next, LogVal);
3782 CGF.Builder.CreateCondBr(Cmp, LoopBB, ExitBB);
3783 auto DL1 = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getEndLoc());
3784 CGF.EmitBlock(ExitBB);
3785 };
3786 if (isOpenMPParallelDirective(S.getDirectiveKind())) {
3787 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());
3788 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
3789 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
3790 /*ForceSimpleCall=*/true);
3791 } else {
3792 RegionCodeGenTy RCG(CodeGen);
3793 RCG(CGF);
3794 }
3795
3796 CGF.OMPFirstScanLoop = false;
3797 SecondGen(CGF);
3798}
3799
3801 const OMPLoopDirective &S,
3802 bool HasCancel) {
3803 bool HasLastprivates;
3804 if (llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
3805 [](const OMPReductionClause *C) {
3806 return C->getModifier() == OMPC_REDUCTION_inscan;
3807 })) {
3808 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
3809 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
3810 OMPLoopScope LoopScope(CGF, S);
3811 return CGF.EmitScalarExpr(S.getNumIterations());
3812 };
3813 const auto &&FirstGen = [&S, HasCancel](CodeGenFunction &CGF) {
3814 CodeGenFunction::OMPCancelStackRAII CancelRegion(
3815 CGF, S.getDirectiveKind(), HasCancel);
3816 (void)CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
3819 // Emit an implicit barrier at the end.
3820 CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getBeginLoc(),
3821 OMPD_for);
3822 };
3823 const auto &&SecondGen = [&S, HasCancel,
3824 &HasLastprivates](CodeGenFunction &CGF) {
3825 CodeGenFunction::OMPCancelStackRAII CancelRegion(
3826 CGF, S.getDirectiveKind(), HasCancel);
3827 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
3830 };
3831 if (!isOpenMPParallelDirective(S.getDirectiveKind()))
3832 emitScanBasedDirectiveDecls(CGF, S, NumIteratorsGen);
3833 emitScanBasedDirective(CGF, S, NumIteratorsGen, FirstGen, SecondGen);
3834 if (!isOpenMPParallelDirective(S.getDirectiveKind()))
3835 emitScanBasedDirectiveFinals(CGF, S, NumIteratorsGen);
3836 } else {
3837 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
3838 HasCancel);
3839 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
3842 }
3843 return HasLastprivates;
3844}
3845
3847 if (S.hasCancel())
3848 return false;
3849 for (OMPClause *C : S.clauses()) {
3850 if (isa<OMPNowaitClause>(C))
3851 continue;
3852
3853 if (auto *SC = dyn_cast<OMPScheduleClause>(C)) {
3854 if (SC->getFirstScheduleModifier() != OMPC_SCHEDULE_MODIFIER_unknown)
3855 return false;
3856 if (SC->getSecondScheduleModifier() != OMPC_SCHEDULE_MODIFIER_unknown)
3857 return false;
3858 switch (SC->getScheduleKind()) {
3859 case OMPC_SCHEDULE_auto:
3860 case OMPC_SCHEDULE_dynamic:
3861 case OMPC_SCHEDULE_runtime:
3862 case OMPC_SCHEDULE_guided:
3863 case OMPC_SCHEDULE_static:
3864 continue;
3866 return false;
3867 }
3868 }
3869
3870 return false;
3871 }
3872
3873 return true;
3874}
3875
3876static llvm::omp::ScheduleKind
3878 switch (ScheduleClauseKind) {
3880 return llvm::omp::OMP_SCHEDULE_Default;
3881 case OMPC_SCHEDULE_auto:
3882 return llvm::omp::OMP_SCHEDULE_Auto;
3883 case OMPC_SCHEDULE_dynamic:
3884 return llvm::omp::OMP_SCHEDULE_Dynamic;
3885 case OMPC_SCHEDULE_guided:
3886 return llvm::omp::OMP_SCHEDULE_Guided;
3887 case OMPC_SCHEDULE_runtime:
3888 return llvm::omp::OMP_SCHEDULE_Runtime;
3889 case OMPC_SCHEDULE_static:
3890 return llvm::omp::OMP_SCHEDULE_Static;
3891 }
3892 llvm_unreachable("Unhandled schedule kind");
3893}
3894
3896 bool HasLastprivates = false;
3897 bool UseOMPIRBuilder =
3898 CGM.getLangOpts().OpenMPIRBuilder && isSupportedByOpenMPIRBuilder(S);
3899 auto &&CodeGen = [this, &S, &HasLastprivates,
3900 UseOMPIRBuilder](CodeGenFunction &CGF, PrePostActionTy &) {
3901 // Use the OpenMPIRBuilder if enabled.
3902 if (UseOMPIRBuilder) {
3903 bool NeedsBarrier = !S.getSingleClause<OMPNowaitClause>();
3904
3905 llvm::omp::ScheduleKind SchedKind = llvm::omp::OMP_SCHEDULE_Default;
3906 llvm::Value *ChunkSize = nullptr;
3907 if (auto *SchedClause = S.getSingleClause<OMPScheduleClause>()) {
3908 SchedKind =
3909 convertClauseKindToSchedKind(SchedClause->getScheduleKind());
3910 if (const Expr *ChunkSizeExpr = SchedClause->getChunkSize())
3911 ChunkSize = EmitScalarExpr(ChunkSizeExpr);
3912 }
3913
3914 // Emit the associated statement and get its loop representation.
3915 const Stmt *Inner = S.getRawStmt();
3916 llvm::CanonicalLoopInfo *CLI =
3918
3919 llvm::OpenMPIRBuilder &OMPBuilder =
3921 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
3922 AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
3923 OMPBuilder.applyWorkshareLoop(
3924 Builder.getCurrentDebugLocation(), CLI, AllocaIP, NeedsBarrier,
3925 SchedKind, ChunkSize, /*HasSimdModifier=*/false,
3926 /*HasMonotonicModifier=*/false, /*HasNonmonotonicModifier=*/false,
3927 /*HasOrderedClause=*/false);
3928 return;
3929 }
3930
3931 HasLastprivates = emitWorksharingDirective(CGF, S, S.hasCancel());
3932 };
3933 {
3934 auto LPCRegion =
3936 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3937 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
3938 S.hasCancel());
3939 }
3940
3941 if (!UseOMPIRBuilder) {
3942 // Emit an implicit barrier at the end.
3943 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
3944 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
3945 }
3946 // Check for outer lastprivate conditional update.
3948}
3949
3951 bool HasLastprivates = false;
3952 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
3953 PrePostActionTy &) {
3954 HasLastprivates = emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
3955 };
3956 {
3957 auto LPCRegion =
3959 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3960 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
3961 }
3962
3963 // Emit an implicit barrier at the end.
3964 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
3965 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
3966 // Check for outer lastprivate conditional update.
3968}
3969
3971 const Twine &Name,
3972 llvm::Value *Init = nullptr) {
3973 LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
3974 if (Init)
3975 CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
3976 return LVal;
3977}
3978
3979void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
3980 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
3981 const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
3982 bool HasLastprivates = false;
3983 auto &&CodeGen = [&S, CapturedStmt, CS,
3984 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
3985 const ASTContext &C = CGF.getContext();
3986 QualType KmpInt32Ty =
3987 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
3988 // Emit helper vars inits.
3989 LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
3990 CGF.Builder.getInt32(0));
3991 llvm::ConstantInt *GlobalUBVal = CS != nullptr
3992 ? CGF.Builder.getInt32(CS->size() - 1)
3993 : CGF.Builder.getInt32(0);
3994 LValue UB =
3995 createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
3996 LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
3997 CGF.Builder.getInt32(1));
3998 LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
3999 CGF.Builder.getInt32(0));
4000 // Loop counter.
4001 LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
4002 OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
4003 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
4004 OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
4005 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
4006 // Generate condition for loop.
4008 C, &IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_PRValue, OK_Ordinary,
4009 S.getBeginLoc(), FPOptionsOverride());
4010 // Increment for loop counter.
4012 C, &IVRefExpr, UO_PreInc, KmpInt32Ty, VK_PRValue, OK_Ordinary,
4013 S.getBeginLoc(), true, FPOptionsOverride());
4014 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
4015 // Iterate through all sections and emit a switch construct:
4016 // switch (IV) {
4017 // case 0:
4018 // <SectionStmt[0]>;
4019 // break;
4020 // ...
4021 // case <NumSection> - 1:
4022 // <SectionStmt[<NumSection> - 1]>;
4023 // break;
4024 // }
4025 // .omp.sections.exit:
4026 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
4027 llvm::SwitchInst *SwitchStmt =
4028 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
4029 ExitBB, CS == nullptr ? 1 : CS->size());
4030 if (CS) {
4031 unsigned CaseNumber = 0;
4032 for (const Stmt *SubStmt : CS->children()) {
4033 auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
4034 CGF.EmitBlock(CaseBB);
4035 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
4036 CGF.EmitStmt(SubStmt);
4037 CGF.EmitBranch(ExitBB);
4038 ++CaseNumber;
4039 }
4040 } else {
4041 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");
4042 CGF.EmitBlock(CaseBB);
4043 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
4044 CGF.EmitStmt(CapturedStmt);
4045 CGF.EmitBranch(ExitBB);
4046 }
4047 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
4048 };
4049
4050 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
4051 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
4052 // Emit implicit barrier to synchronize threads and avoid data races on
4053 // initialization of firstprivate variables and post-update of lastprivate
4054 // variables.
4055 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
4056 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
4057 /*ForceSimpleCall=*/true);
4058 }
4059 CGF.EmitOMPPrivateClause(S, LoopScope);
4060 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(CGF, S, IV);
4061 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4062 CGF.EmitOMPReductionClauseInit(S, LoopScope);
4063 (void)LoopScope.Privatize();
4064 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
4065 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4066
4067 // Emit static non-chunked loop.
4068 OpenMPScheduleTy ScheduleKind;
4069 ScheduleKind.Schedule = OMPC_SCHEDULE_static;
4071 /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(CGF),
4072 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF));
4073 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
4074 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit);
4075 // UB = min(UB, GlobalUB);
4076 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
4077 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
4078 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
4079 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
4080 // IV = LB;
4081 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
4082 // while (idx <= UB) { BODY; ++idx; }
4083 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, Cond, Inc, BodyGen,
4084 [](CodeGenFunction &) {});
4085 // Tell the runtime we are done.
4086 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
4087 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
4088 OMPD_sections);
4089 };
4090 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
4091 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4092 // Emit post-update of the reduction variables if IsLastIter != 0.
4093 emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) {
4094 return CGF.Builder.CreateIsNotNull(
4095 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
4096 });
4097
4098 // Emit final copy of the lastprivate variables if IsLastIter != 0.
4099 if (HasLastprivates)
4101 S, /*NoFinals=*/false,
4102 CGF.Builder.CreateIsNotNull(
4103 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())));
4104 };
4105
4106 bool HasCancel = false;
4107 if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
4108 HasCancel = OSD->hasCancel();
4109 else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
4110 HasCancel = OPSD->hasCancel();
4111 OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
4112 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
4113 HasCancel);
4114 // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
4115 // clause. Otherwise the barrier will be generated by the codegen for the
4116 // directive.
4117 if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
4118 // Emit implicit barrier to synchronize threads and avoid data races on
4119 // initialization of firstprivate variables.
4120 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
4121 OMPD_unknown);
4122 }
4123}
4124
4126 if (CGM.getLangOpts().OpenMPIRBuilder) {
4127 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4128 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4129 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4130
4131 auto FiniCB = [this](InsertPointTy IP) {
4133 };
4134
4135 const CapturedStmt *ICS = S.getInnermostCapturedStmt();
4136 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4137 const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
4139 if (CS) {
4140 for (const Stmt *SubStmt : CS->children()) {
4141 auto SectionCB = [this, SubStmt](InsertPointTy AllocaIP,
4142 InsertPointTy CodeGenIP) {
4144 *this, SubStmt, AllocaIP, CodeGenIP, "section");
4145 };
4146 SectionCBVector.push_back(SectionCB);
4147 }
4148 } else {
4149 auto SectionCB = [this, CapturedStmt](InsertPointTy AllocaIP,
4150 InsertPointTy CodeGenIP) {
4152 *this, CapturedStmt, AllocaIP, CodeGenIP, "section");
4153 };
4154 SectionCBVector.push_back(SectionCB);
4155 }
4156
4157 // Privatization callback that performs appropriate action for
4158 // shared/private/firstprivate/lastprivate/copyin/... variables.
4159 //
4160 // TODO: This defaults to shared right now.
4161 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
4162 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
4163 // The next line is appropriate only for variables (Val) with the
4164 // data-sharing attribute "shared".
4165 ReplVal = &Val;
4166
4167 return CodeGenIP;
4168 };
4169
4170 CGCapturedStmtInfo CGSI(*ICS, CR_OpenMP);
4171 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
4172 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
4173 AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
4174 Builder.restoreIP(OMPBuilder.createSections(
4175 Builder, AllocaIP, SectionCBVector, PrivCB, FiniCB, S.hasCancel(),
4176 S.getSingleClause<OMPNowaitClause>()));
4177 return;
4178 }
4179 {
4180 auto LPCRegion =
4182 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4183 EmitSections(S);
4184 }
4185 // Emit an implicit barrier at the end.
4186 if (!S.getSingleClause<OMPNowaitClause>()) {
4187 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
4188 OMPD_sections);
4189 }
4190 // Check for outer lastprivate conditional update.
4192}
4193
4195 if (CGM.getLangOpts().OpenMPIRBuilder) {
4196 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4197 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4198
4199 const Stmt *SectionRegionBodyStmt = S.getAssociatedStmt();
4200 auto FiniCB = [this](InsertPointTy IP) {
4202 };
4203
4204 auto BodyGenCB = [SectionRegionBodyStmt, this](InsertPointTy AllocaIP,
4205 InsertPointTy CodeGenIP) {
4207 *this, SectionRegionBodyStmt, AllocaIP, CodeGenIP, "section");
4208 };
4209
4210 LexicalScope Scope(*this, S.getSourceRange());
4211 EmitStopPoint(&S);
4212 Builder.restoreIP(OMPBuilder.createSection(Builder, BodyGenCB, FiniCB));
4213
4214 return;
4215 }
4216 LexicalScope Scope(*this, S.getSourceRange());
4217 EmitStopPoint(&S);
4218 EmitStmt(S.getAssociatedStmt());
4219}
4220
4222 llvm::SmallVector<const Expr *, 8> CopyprivateVars;
4226 // Check if there are any 'copyprivate' clauses associated with this
4227 // 'single' construct.
4228 // Build a list of copyprivate variables along with helper expressions
4229 // (<source>, <destination>, <destination>=<source> expressions)
4230 for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
4231 CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
4232 DestExprs.append(C->destination_exprs().begin(),
4233 C->destination_exprs().end());
4234 SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
4235 AssignmentOps.append(C->assignment_ops().begin(),
4236 C->assignment_ops().end());
4237 }
4238 // Emit code for 'single' region along with 'copyprivate' clauses
4239 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4240 Action.Enter(CGF);
4241 OMPPrivateScope SingleScope(CGF);
4244 (void)SingleScope.Privatize();
4245 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4246 };
4247 {
4248 auto LPCRegion =
4250 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4251 CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getBeginLoc(),
4252 CopyprivateVars, DestExprs,
4253 SrcExprs, AssignmentOps);
4254 }
4255 // Emit an implicit barrier at the end (to avoid data race on firstprivate
4256 // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
4257 if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
4259 *this, S.getBeginLoc(),
4260 S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
4261 }
4262 // Check for outer lastprivate conditional update.
4264}
4265
4267 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4268 Action.Enter(CGF);
4269 CGF.EmitStmt(S.getRawStmt());
4270 };
4271 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());
4272}
4273
4275 if (CGM.getLangOpts().OpenMPIRBuilder) {
4276 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4277 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4278
4279 const Stmt *MasterRegionBodyStmt = S.getAssociatedStmt();
4280
4281 auto FiniCB = [this](InsertPointTy IP) {
4283 };
4284
4285 auto BodyGenCB = [MasterRegionBodyStmt, this](InsertPointTy AllocaIP,
4286 InsertPointTy CodeGenIP) {
4288 *this, MasterRegionBodyStmt, AllocaIP, CodeGenIP, "master");
4289 };
4290
4291 LexicalScope Scope(*this, S.getSourceRange());
4292 EmitStopPoint(&S);
4293 Builder.restoreIP(OMPBuilder.createMaster(Builder, BodyGenCB, FiniCB));
4294
4295 return;
4296 }
4297 LexicalScope Scope(*this, S.getSourceRange());
4298 EmitStopPoint(&S);
4299 emitMaster(*this, S);
4300}
4301
4303 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4304 Action.Enter(CGF);
4305 CGF.EmitStmt(S.getRawStmt());
4306 };
4307 Expr *Filter = nullptr;
4308 if (const auto *FilterClause = S.getSingleClause<OMPFilterClause>())
4309 Filter = FilterClause->getThreadID();
4310 CGF.CGM.getOpenMPRuntime().emitMaskedRegion(CGF, CodeGen, S.getBeginLoc(),
4311 Filter);
4312}
4313
4315 if (CGM.getLangOpts().OpenMPIRBuilder) {
4316 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4317 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4318
4319 const Stmt *MaskedRegionBodyStmt = S.getAssociatedStmt();
4320 const Expr *Filter = nullptr;
4321 if (const auto *FilterClause = S.getSingleClause<OMPFilterClause>())
4322 Filter = FilterClause->getThreadID();
4323 llvm::Value *FilterVal = Filter
4324 ? EmitScalarExpr(Filter, CGM.Int32Ty)
4325 : llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/0);
4326
4327 auto FiniCB = [this](InsertPointTy IP) {
4329 };
4330
4331 auto BodyGenCB = [MaskedRegionBodyStmt, this](InsertPointTy AllocaIP,
4332 InsertPointTy CodeGenIP) {
4334 *this, MaskedRegionBodyStmt, AllocaIP, CodeGenIP, "masked");
4335 };
4336
4337 LexicalScope Scope(*this, S.getSourceRange());
4338 EmitStopPoint(&S);
4339 Builder.restoreIP(
4340 OMPBuilder.createMasked(Builder, BodyGenCB, FiniCB, FilterVal));
4341
4342 return;
4343 }
4344 LexicalScope Scope(*this, S.getSourceRange());
4345 EmitStopPoint(&S);
4346 emitMasked(*this, S);
4347}
4348
4350 if (CGM.getLangOpts().OpenMPIRBuilder) {
4351 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4352 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4353
4354 const Stmt *CriticalRegionBodyStmt = S.getAssociatedStmt();
4355 const Expr *Hint = nullptr;
4356 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
4357 Hint = HintClause->getHint();
4358
4359 // TODO: This is slightly different from what's currently being done in
4360 // clang. Fix the Int32Ty to IntPtrTy (pointer width size) when everything
4361 // about typing is final.
4362 llvm::Value *HintInst = nullptr;
4363 if (Hint)
4364 HintInst =
4365 Builder.CreateIntCast(EmitScalarExpr(Hint), CGM.Int32Ty, false);
4366
4367 auto FiniCB = [this](InsertPointTy IP) {
4369 };
4370
4371 auto BodyGenCB = [CriticalRegionBodyStmt, this](InsertPointTy AllocaIP,
4372 InsertPointTy CodeGenIP) {
4374 *this, CriticalRegionBodyStmt, AllocaIP, CodeGenIP, "critical");
4375 };
4376
4377 LexicalScope Scope(*this, S.getSourceRange());
4378 EmitStopPoint(&S);
4379 Builder.restoreIP(OMPBuilder.createCritical(
4380 Builder, BodyGenCB, FiniCB, S.getDirectiveName().getAsString(),
4381 HintInst));
4382
4383 return;
4384 }
4385
4386 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4387 Action.Enter(CGF);
4388 CGF.EmitStmt(S.getAssociatedStmt());
4389 };
4390 const Expr *Hint = nullptr;
4391 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
4392 Hint = HintClause->getHint();
4393 LexicalScope Scope(*this, S.getSourceRange());
4394 EmitStopPoint(&S);
4396 S.getDirectiveName().getAsString(),
4397 CodeGen, S.getBeginLoc(), Hint);
4398}
4399
4401 const OMPParallelForDirective &S) {
4402 // Emit directive as a combined directive that consists of two implicit
4403 // directives: 'parallel' with 'for' directive.
4404 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4405 Action.Enter(CGF);
4406 emitOMPCopyinClause(CGF, S);
4407 (void)emitWorksharingDirective(CGF, S, S.hasCancel());
4408 };
4409 {
4410 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
4411 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
4412 CGCapturedStmtInfo CGSI(CR_OpenMP);
4413 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
4414 OMPLoopScope LoopScope(CGF, S);
4415 return CGF.EmitScalarExpr(S.getNumIterations());
4416 };
4417 bool IsInscan = llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
4418 [](const OMPReductionClause *C) {
4419 return C->getModifier() == OMPC_REDUCTION_inscan;
4420 });
4421 if (IsInscan)
4422 emitScanBasedDirectiveDecls(*this, S, NumIteratorsGen);
4423 auto LPCRegion =
4425 emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
4427 if (IsInscan)
4428 emitScanBasedDirectiveFinals(*this, S, NumIteratorsGen);
4429 }
4430 // Check for outer lastprivate conditional update.
4432}
4433
4435 const OMPParallelForSimdDirective &S) {
4436 // Emit directive as a combined directive that consists of two implicit
4437 // directives: 'parallel' with 'for' directive.
4438 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4439 Action.Enter(CGF);
4440 emitOMPCopyinClause(CGF, S);
4441 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
4442 };
4443 {
4444 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
4445 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
4446 CGCapturedStmtInfo CGSI(CR_OpenMP);
4447 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
4448 OMPLoopScope LoopScope(CGF, S);
4449 return CGF.EmitScalarExpr(S.getNumIterations());
4450 };
4451 bool IsInscan = llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
4452 [](const OMPReductionClause *C) {
4453 return C->getModifier() == OMPC_REDUCTION_inscan;
4454 });
4455 if (IsInscan)
4456 emitScanBasedDirectiveDecls(*this, S, NumIteratorsGen);
4457 auto LPCRegion =
4459 emitCommonOMPParallelDirective(*this, S, OMPD_for_simd, CodeGen,
4461 if (IsInscan)
4462 emitScanBasedDirectiveFinals(*this, S, NumIteratorsGen);
4463 }
4464 // Check for outer lastprivate conditional update.
4466}
4467
4469 const OMPParallelMasterDirective &S) {
4470 // Emit directive as a combined directive that consists of two implicit
4471 // directives: 'parallel' with 'master' directive.
4472 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4473 Action.Enter(CGF);
4474 OMPPrivateScope PrivateScope(CGF);
4475 emitOMPCopyinClause(CGF, S);
4476 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4477 CGF.EmitOMPPrivateClause(S, PrivateScope);
4478 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4479 (void)PrivateScope.Privatize();
4480 emitMaster(CGF, S);
4481 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4482 };
4483 {
4484 auto LPCRegion =
4486 emitCommonOMPParallelDirective(*this, S, OMPD_master, CodeGen,
4489 [](CodeGenFunction &) { return nullptr; });
4490 }
4491 // Check for outer lastprivate conditional update.
4493}
4494
4496 const OMPParallelMaskedDirective &S) {
4497 // Emit directive as a combined directive that consists of two implicit
4498 // directives: 'parallel' with 'masked' directive.
4499 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4500 Action.Enter(CGF);
4501 OMPPrivateScope PrivateScope(CGF);
4502 emitOMPCopyinClause(CGF, S);
4503 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4504 CGF.EmitOMPPrivateClause(S, PrivateScope);
4505 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4506 (void)PrivateScope.Privatize();
4507 emitMasked(CGF, S);
4508 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4509 };
4510 {
4511 auto LPCRegion =
4513 emitCommonOMPParallelDirective(*this, S, OMPD_masked, CodeGen,
4516 [](CodeGenFunction &) { return nullptr; });
4517 }
4518 // Check for outer lastprivate conditional update.
4520}
4521
4524 // Emit directive as a combined directive that consists of two implicit
4525 // directives: 'parallel' with 'sections' directive.
4526 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4527 Action.Enter(CGF);
4528 emitOMPCopyinClause(CGF, S);
4529 CGF.EmitSections(S);
4530 };
4531 {
4532 auto LPCRegion =
4534 emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
4536 }
4537 // Check for outer lastprivate conditional update.
4539}
4540
4541namespace {
4542/// Get the list of variables declared in the context of the untied tasks.
4543class CheckVarsEscapingUntiedTaskDeclContext final
4544 : public ConstStmtVisitor<CheckVarsEscapingUntiedTaskDeclContext> {
4546
4547public:
4548 explicit CheckVarsEscapingUntiedTaskDeclContext() = default;
4549 virtual ~CheckVarsEscapingUntiedTaskDeclContext() = default;
4550 void VisitDeclStmt(const DeclStmt *S) {
4551 if (!S)
4552 return;
4553 // Need to privatize only local vars, static locals can be processed as is.
4554 for (const Decl *D : S->decls()) {
4555 if (const auto *VD = dyn_cast_or_null<VarDecl>(D))
4556 if (VD->hasLocalStorage())
4557 PrivateDecls.push_back(VD);
4558 }
4559 }
4560 void VisitOMPExecutableDirective(const OMPExecutableDirective *) {}
4561 void VisitCapturedStmt(const CapturedStmt *) {}
4562 void VisitLambdaExpr(const LambdaExpr *) {}
4563 void VisitBlockExpr(const BlockExpr *) {}
4564 void VisitStmt(const Stmt *S) {
4565 if (!S)
4566 return;
4567 for (const Stmt *Child : S->children())
4568 if (Child)
4569 Visit(Child);
4570 }
4571
4572 /// Swaps list of vars with the provided one.
4573 ArrayRef<const VarDecl *> getPrivateDecls() const { return PrivateDecls; }
4574};
4575} // anonymous namespace
4576
4579
4580 // First look for 'omp_all_memory' and add this first.
4581 bool OmpAllMemory = false;
4582 if (llvm::any_of(
4583 S.getClausesOfKind<OMPDependClause>(), [](const OMPDependClause *C) {
4584 return C->getDependencyKind() == OMPC_DEPEND_outallmemory ||
4585 C->getDependencyKind() == OMPC_DEPEND_inoutallmemory;
4586 })) {
4587 OmpAllMemory = true;
4588 // Since both OMPC_DEPEND_outallmemory and OMPC_DEPEND_inoutallmemory are
4589 // equivalent to the runtime, always use OMPC_DEPEND_outallmemory to
4590 // simplify.
4592 Data.Dependences.emplace_back(OMPC_DEPEND_outallmemory,
4593 /*IteratorExpr=*/nullptr);
4594 // Add a nullptr Expr to simplify the codegen in emitDependData.
4595 DD.DepExprs.push_back(nullptr);
4596 }
4597 // Add remaining dependences skipping any 'out' or 'inout' if they are
4598 // overridden by 'omp_all_memory'.
4599 for (const auto *C : S.getClausesOfKind<OMPDependClause>()) {
4600 OpenMPDependClauseKind Kind = C->getDependencyKind();
4601 if (Kind == OMPC_DEPEND_outallmemory || Kind == OMPC_DEPEND_inoutallmemory)
4602 continue;
4603 if (OmpAllMemory && (Kind == OMPC_DEPEND_out || Kind == OMPC_DEPEND_inout))
4604 continue;
4606 Data.Dependences.emplace_back(C->getDependencyKind(), C->getModifier());
4607 DD.DepExprs.append(C->varlist_begin(), C->varlist_end());
4608 }
4609}
4610
4612 const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
4613 const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
4615 // Emit outlined function for task construct.
4616 const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
4617 auto I = CS->getCapturedDecl()->param_begin();
4618 auto PartId = std::next(I);
4619 auto TaskT = std::next(I, 4);
4620 // Check if the task is final
4621 if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
4622 // If the condition constant folds and can be elided, try to avoid emitting
4623 // the condition and the dead arm of the if/else.
4624 const Expr *Cond = Clause->getCondition();
4625 bool CondConstant;
4626 if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
4627 Data.Final.setInt(CondConstant);
4628 else
4629 Data.Final.setPointer(EvaluateExprAsBool(Cond));
4630 } else {
4631 // By default the task is not final.
4632 Data.Final.setInt(/*IntVal=*/false);
4633 }
4634 // Check if the task has 'priority' clause.
4635 if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
4636 const Expr *Prio = Clause->getPriority();
4637 Data.Priority.setInt(/*IntVal=*/true);
4638 Data.Priority.setPointer(EmitScalarConversion(
4639 EmitScalarExpr(Prio), Prio->getType(),
4640 getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
4641 Prio->getExprLoc()));
4642 }
4643 // The first function argument for tasks is a thread id, the second one is a
4644 // part id (0 for tied tasks, >=0 for untied task).
4645 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
4646 // Get list of private variables.
4647 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
4648 auto IRef = C->varlist_begin();
4649 for (const Expr *IInit : C->private_copies()) {
4650 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4651 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4652 Data.PrivateVars.push_back(*IRef);
4653 Data.PrivateCopies.push_back(IInit);
4654 }
4655 ++IRef;
4656 }
4657 }
4658 EmittedAsPrivate.clear();
4659 // Get list of firstprivate variables.
4660 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
4661 auto IRef = C->varlist_begin();
4662 auto IElemInitRef = C->inits().begin();
4663 for (const Expr *IInit : C->private_copies()) {
4664 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4665 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4666 Data.FirstprivateVars.push_back(*IRef);
4667 Data.FirstprivateCopies.push_back(IInit);
4668 Data.FirstprivateInits.push_back(*IElemInitRef);
4669 }
4670 ++IRef;
4671 ++IElemInitRef;
4672 }
4673 }
4674 // Get list of lastprivate variables (for taskloops).
4675 llvm::MapVector<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
4676 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
4677 auto IRef = C->varlist_begin();
4678 auto ID = C->destination_exprs().begin();
4679 for (const Expr *IInit : C->private_copies()) {
4680 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4681 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4682 Data.LastprivateVars.push_back(*IRef);
4683 Data.LastprivateCopies.push_back(IInit);
4684 }
4685 LastprivateDstsOrigs.insert(
4686 std::make_pair(cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
4687 cast<DeclRefExpr>(*IRef)));
4688 ++IRef;
4689 ++ID;
4690 }
4691 }
4694 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
4695 Data.ReductionVars.append(C->varlist_begin(), C->varlist_end());
4696 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
4697 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
4698 Data.ReductionOps.append(C->reduction_ops().begin(),
4699 C->reduction_ops().end());
4700 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
4701 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
4702 }
4704 *this, S.getBeginLoc(), LHSs, RHSs, Data);
4705 // Build list of dependences.
4707 // Get list of local vars for untied tasks.
4708 if (!Data.Tied) {
4709 CheckVarsEscapingUntiedTaskDeclContext Checker;
4710 Checker.Visit(S.getInnermostCapturedStmt()->getCapturedStmt());
4711 Data.PrivateLocals.append(Checker.getPrivateDecls().begin(),
4712 Checker.getPrivateDecls().end());
4713 }
4714 auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
4715 CapturedRegion](CodeGenFunction &CGF,
4716 PrePostActionTy &Action) {
4717 llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
4718 std::pair<Address, Address>>
4719 UntiedLocalVars;
4720 // Set proper addresses for generated private copies.
4721 OMPPrivateScope Scope(CGF);
4722 // Generate debug info for variables present in shared clause.
4723 if (auto *DI = CGF.getDebugInfo()) {
4724 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields =
4725 CGF.CapturedStmtInfo->getCaptureFields();
4726 llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue();
4727 if (CaptureFields.size() && ContextValue) {
4728 unsigned CharWidth = CGF.getContext().getCharWidth();
4729 // The shared variables are packed together as members of structure.
4730 // So the address of each shared variable can be computed by adding
4731 // offset of it (within record) to the base address of record. For each
4732 // shared variable, debug intrinsic llvm.dbg.declare is generated with
4733 // appropriate expressions (DIExpression).
4734 // Ex:
4735 // %12 = load %struct.anon*, %struct.anon** %__context.addr.i
4736 // call void @llvm.dbg.declare(metadata %struct.anon* %12,
4737 // metadata !svar1,
4738 // metadata !DIExpression(DW_OP_deref))
4739 // call void @llvm.dbg.declare(metadata %struct.anon* %12,
4740 // metadata !svar2,
4741 // metadata !DIExpression(DW_OP_plus_uconst, 8, DW_OP_deref))
4742 for (auto It = CaptureFields.begin(); It != CaptureFields.end(); ++It) {
4743 const VarDecl *SharedVar = It->first;
4744 RecordDecl *CaptureRecord = It->second->getParent();
4745 const ASTRecordLayout &Layout =
4746 CGF.getContext().getASTRecordLayout(CaptureRecord);
4747 unsigned Offset =
4748 Layout.getFieldOffset(It->second->getFieldIndex()) / CharWidth;
4749 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
4750 (void)DI->EmitDeclareOfAutoVariable(SharedVar, ContextValue,
4751 CGF.Builder, false);
4752 llvm::Instruction &Last = CGF.Builder.GetInsertBlock()->back();
4753 // Get the call dbg.declare instruction we just created and update
4754 // its DIExpression to add offset to base address.
4755 if (auto DDI = dyn_cast<llvm::DbgVariableIntrinsic>(&Last)) {
4757 // Add offset to the base address if non zero.
4758 if (Offset) {
4759 Ops.push_back(llvm::dwarf::DW_OP_plus_uconst);
4760 Ops.push_back(Offset);
4761 }
4762 Ops.push_back(llvm::dwarf::DW_OP_deref);
4763 auto &Ctx = DDI->getContext();
4764 llvm::DIExpression *DIExpr = llvm::DIExpression::get(Ctx, Ops);
4765 Last.setOperand(2, llvm::MetadataAsValue::get(Ctx, DIExpr));
4766 }
4767 }
4768 }
4769 }
4771 if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
4772 !Data.LastprivateVars.empty() || !Data.PrivateLocals.empty()) {
4773 enum { PrivatesParam = 2, CopyFnParam = 3 };
4774 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
4775 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
4776 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
4777 CS->getCapturedDecl()->getParam(PrivatesParam)));
4778 // Map privates.
4782 CallArgs.push_back(PrivatesPtr);
4783 ParamTypes.push_back(PrivatesPtr->getType());
4784 for (const Expr *E : Data.PrivateVars) {
4785 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4786 Address PrivatePtr = CGF.CreateMemTemp(
4787 CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
4788 PrivatePtrs.emplace_back(VD, PrivatePtr);
4789 CallArgs.push_back(PrivatePtr.getPointer());
4790 ParamTypes.push_back(PrivatePtr.getType());
4791 }
4792 for (const Expr *E : Data.FirstprivateVars) {
4793 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4794 Address PrivatePtr =
4795 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
4796 ".firstpriv.ptr.addr");
4797 PrivatePtrs.emplace_back(VD, PrivatePtr);
4798 FirstprivatePtrs.emplace_back(VD, PrivatePtr);
4799 CallArgs.push_back(PrivatePtr.getPointer());
4800 ParamTypes.push_back(PrivatePtr.getType());
4801 }
4802 for (const Expr *E : Data.LastprivateVars) {
4803 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4804 Address PrivatePtr =
4805 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
4806 ".lastpriv.ptr.addr");
4807 PrivatePtrs.emplace_back(VD, PrivatePtr);
4808 CallArgs.push_back(PrivatePtr.getPointer());
4809 ParamTypes.push_back(PrivatePtr.getType());
4810 }
4811 for (const VarDecl *VD : Data.PrivateLocals) {
4813 if (VD->getType()->isLValueReferenceType())
4814 Ty = CGF.getContext().getPointerType(Ty);
4815 if (isAllocatableDecl(VD))
4816 Ty = CGF.getContext().getPointerType(Ty);
4817 Address PrivatePtr = CGF.CreateMemTemp(
4818 CGF.getContext().getPointerType(Ty), ".local.ptr.addr");
4819 auto Result = UntiedLocalVars.insert(
4820 std::make_pair(VD, std::make_pair(PrivatePtr, Address::invalid())));
4821 // If key exists update in place.
4822 if (Result.second == false)
4823 *Result.first = std::make_pair(
4824 VD, std::make_pair(PrivatePtr, Address::invalid()));
4825 CallArgs.push_back(PrivatePtr.getPointer());
4826 ParamTypes.push_back(PrivatePtr.getType());
4827 }
4828 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
4829 ParamTypes, /*isVarArg=*/false);
4830 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
4831 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
4832 for (const auto &Pair : LastprivateDstsOrigs) {
4833 const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
4834 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD),
4835 /*RefersToEnclosingVariableOrCapture=*/
4836 CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr,
4837 Pair.second->getType(), VK_LValue,
4838 Pair.second->getExprLoc());
4839 Scope.addPrivate(Pair.first, CGF.EmitLValue(&DRE).getAddress(CGF));
4840 }
4841 for (const auto &Pair : PrivatePtrs) {
4842 Address Replacement = Address(
4843 CGF.Builder.CreateLoad(Pair.second),
4844 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
4845 CGF.getContext().getDeclAlign(Pair.first));
4846 Scope.addPrivate(Pair.first, Replacement);
4847 if (auto *DI = CGF.getDebugInfo())
4848 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
4849 (void)DI->EmitDeclareOfAutoVariable(
4850 Pair.first, Pair.second.getPointer(), CGF.Builder,
4851 /*UsePointerValue*/ true);
4852 }
4853 // Adjust mapping for internal locals by mapping actual memory instead of
4854 // a pointer to this memory.
4855 for (auto &Pair : UntiedLocalVars) {
4856 QualType VDType = Pair.first->getType().getNonReferenceType();
4857 if (Pair.first->getType()->isLValueReferenceType())
4858 VDType = CGF.getContext().getPointerType(VDType);
4859 if (isAllocatableDecl(Pair.first)) {
4860 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
4861 Address Replacement(
4862 Ptr,
4863 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(VDType)),
4864 CGF.getPointerAlign());
4865 Pair.second.first = Replacement;
4866 Ptr = CGF.Builder.CreateLoad(Replacement);
4867 Replacement = Address(Ptr, CGF.ConvertTypeForMem(VDType),
4868 CGF.getContext().getDeclAlign(Pair.first));
4869 Pair.second.second = Replacement;
4870 } else {
4871 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
4872 Address Replacement(Ptr, CGF.ConvertTypeForMem(VDType),
4873 CGF.getContext().getDeclAlign(Pair.first));
4874 Pair.second.first = Replacement;
4875 }
4876 }
4877 }
4878 if (Data.Reductions) {
4879 OMPPrivateScope FirstprivateScope(CGF);
4880 for (const auto &Pair : FirstprivatePtrs) {
4881 Address Replacement(
4882 CGF.Builder.CreateLoad(Pair.second),
4883 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
4884 CGF.getContext().getDeclAlign(Pair.first));
4885 FirstprivateScope.addPrivate(Pair.first, Replacement);
4886 }
4887 (void)FirstprivateScope.Privatize();
4888 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
4889 ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionVars,
4890 Data.ReductionCopies, Data.ReductionOps);
4891 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
4892 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
4893 for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
4894 RedCG.emitSharedOrigLValue(CGF, Cnt);
4895 RedCG.emitAggregateType(CGF, Cnt);
4896 // FIXME: This must removed once the runtime library is fixed.
4897 // Emit required threadprivate variables for
4898 // initializer/combiner/finalizer.
4899 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
4900 RedCG, Cnt);
4901 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
4902 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
4903 Replacement =
4904 Address(CGF.EmitScalarConversion(
4905 Replacement.getPointer(), CGF.getContext().VoidPtrTy,
4906 CGF.getContext().getPointerType(
4907 Data.ReductionCopies[Cnt]->getType()),
4908 Data.ReductionCopies[Cnt]->getExprLoc()),
4909 CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()),
4910 Replacement.getAlignment());
4911 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
4912 Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
4913 }
4914 }
4915 // Privatize all private variables except for in_reduction items.
4916 (void)Scope.Privatize();
4920 SmallVector<const Expr *, 4> TaskgroupDescriptors;
4921 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
4922 auto IPriv = C->privates().begin();
4923 auto IRed = C->reduction_ops().begin();
4924 auto ITD = C->taskgroup_descriptors().begin();
4925 for (const Expr *Ref : C->varlists()) {
4926 InRedVars.emplace_back(Ref);
4927 InRedPrivs.emplace_back(*IPriv);
4928 InRedOps.emplace_back(*IRed);
4929 TaskgroupDescriptors.emplace_back(*ITD);
4930 std::advance(IPriv, 1);
4931 std::advance(IRed, 1);
4932 std::advance(ITD, 1);
4933 }
4934 }
4935 // Privatize in_reduction items here, because taskgroup descriptors must be
4936 // privatized earlier.
4937 OMPPrivateScope InRedScope(CGF);
4938 if (!InRedVars.empty()) {
4939 ReductionCodeGen RedCG(InRedVars, InRedVars, InRedPrivs, InRedOps);
4940 for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
4941 RedCG.emitSharedOrigLValue(CGF, Cnt);
4942 RedCG.emitAggregateType(CGF, Cnt);
4943 // The taskgroup descriptor variable is always implicit firstprivate and
4944 // privatized already during processing of the firstprivates.
4945 // FIXME: This must removed once the runtime library is fixed.
4946 // Emit required threadprivate variables for
4947 // initializer/combiner/finalizer.
4948 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
4949 RedCG, Cnt);
4950 llvm::Value *ReductionsPtr;
4951 if (const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
4952 ReductionsPtr = CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr),
4953 TRExpr->getExprLoc());
4954 } else {
4955 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4956 }
4957 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
4958 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
4959 Replacement = Address(
4960 CGF.EmitScalarConversion(
4961 Replacement.getPointer(), CGF.getContext().VoidPtrTy,
4962 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
4963 InRedPrivs[Cnt]->getExprLoc()),
4964 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),
4965 Replacement.getAlignment());
4966 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
4967 InRedScope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
4968 }
4969 }
4970 (void)InRedScope.Privatize();
4971
4973 UntiedLocalVars);
4974 Action.Enter(CGF);
4975 BodyGen(CGF);
4976 };
4977 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
4978 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied,
4979 Data.NumberOfParts);
4980 OMPLexicalScope Scope(*this, S, std::nullopt,
4981 !isOpenMPParallelDirective(S.getDirectiveKind()) &&
4982 !isOpenMPSimdDirective(S.getDirectiveKind()));
4983 TaskGen(*this, OutlinedFn, Data);
4984}
4985
4986static ImplicitParamDecl *
4988 QualType Ty, CapturedDecl *CD,
4989 SourceLocation Loc) {
4990 auto *OrigVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
4992 auto *OrigRef = DeclRefExpr::Create(
4994 /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
4995 auto *PrivateVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
4997 auto *PrivateRef = DeclRefExpr::Create(
4998 C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD,
4999 /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
5000 QualType ElemType = C.getBaseElementType(Ty);
5001 auto *InitVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, ElemType,
5003 auto *InitRef = DeclRefExpr::Create(
5005 /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue);
5006 PrivateVD->setInitStyle(VarDecl::CInit);
5007 PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue,
5008 InitRef, /*BasePath=*/nullptr,
5010 Data.FirstprivateVars.emplace_back(OrigRef);
5011 Data.FirstprivateCopies.emplace_back(PrivateRef);
5012 Data.FirstprivateInits.emplace_back(InitRef);
5013 return OrigVD;
5014}
5015
5017 const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,
5018 OMPTargetDataInfo &InputInfo) {
5019 // Emit outlined function for task construct.
5020 const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
5021 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
5023 auto I = CS->getCapturedDecl()->param_begin();
5024 auto PartId = std::next(I);
5025 auto TaskT = std::next(I, 4);
5027 // The task is not final.
5028 Data.Final.setInt(/*IntVal=*/false);
5029 // Get list of firstprivate variables.
5030 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
5031 auto IRef = C->varlist_begin();
5032 auto IElemInitRef = C->inits().begin();
5033 for (auto *IInit : C->private_copies()) {
5034 Data.FirstprivateVars.push_back(*IRef);
5035 Data.FirstprivateCopies.push_back(IInit);
5036 Data.FirstprivateInits.push_back(*IElemInitRef);
5037 ++IRef;
5038 ++IElemInitRef;
5039 }
5040 }
5043 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
5044 Data.ReductionVars.append(C->varlist_begin(), C->varlist_end());
5045 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
5046 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
5047 Data.ReductionOps.append(C->reduction_ops().begin(),
5048 C->reduction_ops().end());
5049 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
5050 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
5051 }
5052 OMPPrivateScope TargetScope(*this);
5053 VarDecl *BPVD = nullptr;
5054 VarDecl *PVD = nullptr;
5055 VarDecl *SVD = nullptr;
5056 VarDecl *MVD = nullptr;
5057 if (InputInfo.NumberOfTargetItems > 0) {
5058 auto *CD = CapturedDecl::Create(
5059 getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0);
5060 llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);
5061 QualType BaseAndPointerAndMapperType = getContext().getConstantArrayType(
5062 getContext().VoidPtrTy, ArrSize, nullptr, ArraySizeModifier::Normal,
5063 /*IndexTypeQuals=*/0);
5065 getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5067 getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5069 getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1),
5070 ArrSize, nullptr, ArraySizeModifier::Normal,
5071 /*IndexTypeQuals=*/0);
5072 SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD,
5073 S.getBeginLoc());
5074 TargetScope.addPrivate(BPVD, InputInfo.BasePointersArray);
5075 TargetScope.addPrivate(PVD, InputInfo.PointersArray);
5076 TargetScope.addPrivate(SVD, InputInfo.SizesArray);
5077 // If there is no user-defined mapper, the mapper array will be nullptr. In
5078 // this case, we don't need to privatize it.
5079 if (!isa_and_nonnull<llvm::ConstantPointerNull>(
5080 InputInfo.MappersArray.getPointer())) {
5082 getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5083 TargetScope.addPrivate(MVD, InputInfo.MappersArray);
5084 }
5085 }
5086 (void)TargetScope.Privatize();
5088 auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD, MVD,
5089 &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
5090 // Set proper addresses for generated private copies.
5091 OMPPrivateScope Scope(CGF);
5092 if (!Data.FirstprivateVars.empty()) {
5093 enum { PrivatesParam = 2, CopyFnParam = 3 };
5094 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
5095 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
5096 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
5097 CS->getCapturedDecl()->getParam(PrivatesParam)));
5098 // Map privates.
5102 CallArgs.push_back(PrivatesPtr);
5103 ParamTypes.push_back(PrivatesPtr->getType());
5104 for (const Expr *E : Data.FirstprivateVars) {
5105 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5106 Address PrivatePtr =
5107 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
5108 ".firstpriv.ptr.addr");
5109 PrivatePtrs.emplace_back(VD, PrivatePtr);
5110 CallArgs.push_back(PrivatePtr.getPointer());
5111 ParamTypes.push_back(PrivatePtr.getType());
5112 }
5113 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
5114 ParamTypes, /*isVarArg=*/false);
5115 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
5116 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
5117 for (const auto &Pair : PrivatePtrs) {
5118 Address Replacement(
5119 CGF.Builder.CreateLoad(Pair.second),
5120 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
5121 CGF.getContext().getDeclAlign(Pair.first));
5122 Scope.addPrivate(Pair.first, Replacement);
5123 }
5124 }
5125 CGF.processInReduction(S, Data, CGF, CS, Scope);
5126 if (InputInfo.NumberOfTargetItems > 0) {
5127 InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
5128 CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0);
5129 InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
5130 CGF.GetAddrOfLocalVar(PVD), /*Index=*/0);
5131 InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
5132 CGF.GetAddrOfLocalVar(SVD), /*Index=*/0);
5133 // If MVD is nullptr, the mapper array is not privatized
5134 if (MVD)
5135 InputInfo.MappersArray = CGF.Builder.CreateConstArrayGEP(
5136 CGF.GetAddrOfLocalVar(MVD), /*Index=*/0);
5137 }
5138
5139 Action.Enter(CGF);
5140 OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);
5141 auto *TL = S.getSingleClause<OMPThreadLimitClause>();
5142 if (CGF.CGM.getLangOpts().OpenMP >= 51 &&
5143 needsTaskBasedThreadLimit(S.getDirectiveKind()) && TL) {
5144 // Emit __kmpc_set_thread_limit() to set the thread_limit for the task
5145 // enclosing this target region. This will indirectly set the thread_limit
5146 // for every applicable construct within target region.
5147 CGF.CGM.getOpenMPRuntime().emitThreadLimitClause(
5148 CGF, TL->getThreadLimit(), S.getBeginLoc());
5149 }
5150 BodyGen(CGF);
5151 };
5152 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
5153 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true,
5154 Data.NumberOfParts);
5155 llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
5156 IntegerLiteral IfCond(getContext(), TrueOrFalse,
5157 getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
5158 SourceLocation());
5159 CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn,
5160 SharedsTy, CapturedStruct, &IfCond, Data);
5161}
5162
5165 CodeGenFunction &CGF,
5166 const CapturedStmt *CS,
5167 OMPPrivateScope &Scope) {
5168 if (Data.Reductions) {
5169 OpenMPDirectiveKind CapturedRegion = S.getDirectiveKind();
5170 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
5171 ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionVars,
5172 Data.ReductionCopies, Data.ReductionOps);
5173 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
5175 for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
5176 RedCG.emitSharedOrigLValue(CGF, Cnt);
5177 RedCG.emitAggregateType(CGF, Cnt);
5178 // FIXME: This must removed once the runtime library is fixed.
5179 // Emit required threadprivate variables for
5180 // initializer/combiner/finalizer.
5181 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
5182 RedCG, Cnt);
5184 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
5185 Replacement =
5187 Replacement.getPointer(), CGF.getContext().VoidPtrTy,
5189 Data.ReductionCopies[Cnt]->getType()),
5190 Data.ReductionCopies[Cnt]->getExprLoc()),
5191 CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()),
5192 Replacement.getAlignment());
5193 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
5194 Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
5195 }
5196 }
5197 (void)Scope.Privatize();
5201 SmallVector<const Expr *, 4> TaskgroupDescriptors;
5202 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
5203 auto IPriv = C->privates().begin();
5204 auto IRed = C->reduction_ops().begin();
5205 auto ITD = C->taskgroup_descriptors().begin();
5206 for (const Expr *Ref : C->varlists()) {
5207 InRedVars.emplace_back(Ref);
5208 InRedPrivs.emplace_back(*IPriv);
5209 InRedOps.emplace_back(*IRed);
5210 TaskgroupDescriptors.emplace_back(*ITD);
5211 std::advance(IPriv, 1);
5212 std::advance(IRed, 1);
5213 std::advance(ITD, 1);
5214 }
5215 }
5216 OMPPrivateScope InRedScope(CGF);
5217 if (!InRedVars.empty()) {
5218 ReductionCodeGen RedCG(InRedVars, InRedVars, InRedPrivs, InRedOps);
5219 for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
5220 RedCG.emitSharedOrigLValue(CGF, Cnt);
5221 RedCG.emitAggregateType(CGF, Cnt);
5222 // FIXME: This must removed once the runtime library is fixed.
5223 // Emit required threadprivate variables for
5224 // initializer/combiner/finalizer.
5225 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
5226 RedCG, Cnt);
5227 llvm::Value *ReductionsPtr;
5228 if (const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
5229 ReductionsPtr =
5230 CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr), TRExpr->getExprLoc());
5231 } else {
5232 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5233 }
5235 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
5236 Replacement = Address(
5238 Replacement.getPointer(), CGF.getContext().VoidPtrTy,
5239 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
5240 InRedPrivs[Cnt]->getExprLoc()),
5241 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),
5242 Replacement.getAlignment());
5243 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
5244 InRedScope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
5245 }
5246 }
5247 (void)InRedScope.Privatize();
5248}
5249
5251 // Emit outlined function for task construct.
5252 const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
5253 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
5255 const Expr *IfCond = nullptr;
5256 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
5257 if (C->getNameModifier() == OMPD_unknown ||
5258 C->getNameModifier() == OMPD_task) {
5259 IfCond = C->getCondition();
5260 break;
5261 }
5262 }
5263
5265 // Check if we should emit tied or untied task.
5266 Data.Tied = !S.getSingleClause<OMPUntiedClause>();
5267 auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
5268 CGF.EmitStmt(CS->getCapturedStmt());
5269 };
5270 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
5271 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
5272 const OMPTaskDataTy &Data) {
5273 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn,
5274 SharedsTy, CapturedStruct, IfCond,
5275 Data);
5276 };
5277 auto LPCRegion =
5279 EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data);
5280}
5281
5283 const OMPTaskyieldDirective &S) {
5284 CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getBeginLoc());
5285}
5286
5288 const OMPMessageClause *MC = S.getSingleClause<OMPMessageClause>();
5289 Expr *ME = MC ? MC->getMessageString() : nullptr;
5290 const OMPSeverityClause *SC = S.getSingleClause<OMPSeverityClause>();
5291 bool IsFatal = false;
5292 if (!SC || SC->getSeverityKind() == OMPC_SEVERITY_fatal)
5293 IsFatal = true;
5294 CGM.getOpenMPRuntime().emitErrorCall(*this, S.getBeginLoc(), ME, IsFatal);
5295}
5296
5298 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_barrier);
5299}
5300
5303 // Build list of dependences
5305 Data.HasNowaitClause = S.hasClausesOfKind<OMPNowaitClause>();
5306 CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc(), Data);
5307}
5308
5310 return T.clauses().empty();
5311}
5312
5314 const OMPTaskgroupDirective &S) {
5315 OMPLexicalScope Scope(*this, S, OMPD_unknown);
5316 if (CGM.getLangOpts().OpenMPIRBuilder && isSupportedByOpenMPIRBuilder(S)) {
5317 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
5318 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5319 InsertPointTy AllocaIP(AllocaInsertPt->getParent(),
5320 AllocaInsertPt->getIterator());
5321
5322 auto BodyGenCB = [&, this](InsertPointTy AllocaIP,
5323 InsertPointTy CodeGenIP) {
5324 Builder.restoreIP(CodeGenIP);
5325 EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
5326 };
5327 CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;
5328 if (!CapturedStmtInfo)
5329 CapturedStmtInfo = &CapStmtInfo;
5330 Builder.restoreIP(OMPBuilder.createTaskgroup(Builder, AllocaIP, BodyGenCB));
5331 return;
5332 }
5333 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5334 Action.Enter(CGF);
5335 if (const Expr *E = S.getReductionRef()) {
5339 for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
5340 Data.ReductionVars.append(C->varlist_begin(), C->varlist_end());
5341 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
5342 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
5343 Data.ReductionOps.append(C->reduction_ops().begin(),
5344 C->reduction_ops().end());
5345 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
5346 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
5347 }
5348 llvm::Value *ReductionDesc =
5349 CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(),
5350 LHSs, RHSs, Data);
5351 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5352 CGF.EmitVarDecl(*VD);
5353 CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),
5354 /*Volatile=*/false, E->getType());
5355 }
5356 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
5357 };
5358 CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getBeginLoc());
5359}
5360
5362 llvm::AtomicOrdering AO = S.getSingleClause<OMPFlushClause>()
5363 ? llvm::AtomicOrdering::NotAtomic
5364 : llvm::AtomicOrdering::AcquireRelease;
5366 *this,
5367 [&S]() -> ArrayRef<const Expr *> {
5368 if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>())
5369 return llvm::ArrayRef(FlushClause->varlist_begin(),
5370 FlushClause->varlist_end());
5371 return std::nullopt;
5372 }(),
5373 S.getBeginLoc(), AO);
5374}
5375
5377 const auto *DO = S.getSingleClause<OMPDepobjClause>();
5378 LValue DOLVal = EmitLValue(DO->getDepobj());
5379 if (const auto *DC = S.getSingleClause<OMPDependClause>()) {
5380 OMPTaskDataTy::DependData Dependencies(DC->getDependencyKind(),
5381 DC->getModifier());
5382 Dependencies.DepExprs.append(DC->varlist_begin(), DC->varlist_end());
5384 *this, Dependencies, DC->getBeginLoc());
5385 EmitStoreOfScalar(DepAddr.getPointer(), DOLVal);
5386 return;
5387 }
5388 if (const auto *DC = S.getSingleClause<OMPDestroyClause>()) {
5389 CGM.getOpenMPRuntime().emitDestroyClause(*this, DOLVal, DC->getBeginLoc());
5390 return;
5391 }
5392 if (const auto *UC = S.getSingleClause<OMPUpdateClause>()) {
5394 *this, DOLVal, UC->getDependencyKind(), UC->getBeginLoc());
5395 return;
5396 }
5397}
5398
5401 return;
5403 bool IsInclusive = S.hasClausesOfKind<OMPInclusiveClause>();
5408 SmallVector<const Expr *, 4> ReductionOps;
5410 SmallVector<const Expr *, 4> CopyArrayTemps;
5411 SmallVector<const Expr *, 4> CopyArrayElems;
5412 for (const auto *C : ParentDir.getClausesOfKind<OMPReductionClause>()) {
5413 if (C->getModifier() != OMPC_REDUCTION_inscan)
5414 continue;
5415 Shareds.append(C->varlist_begin(), C->varlist_end());
5416 Privates.append(C->privates().begin(), C->privates().end());
5417 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
5418 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
5419 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
5420 CopyOps.append(C->copy_ops().begin(), C->copy_ops().end());
5421 CopyArrayTemps.append(C->copy_array_temps().begin(),
5422 C->copy_array_temps().end());
5423 CopyArrayElems.append(C->copy_array_elems().begin(),
5424 C->copy_array_elems().end());
5425 }
5426 if (ParentDir.getDirectiveKind() == OMPD_simd ||
5427 (getLangOpts().OpenMPSimd &&
5429 // For simd directive and simd-based directives in simd only mode, use the
5430 // following codegen:
5431 // int x = 0;
5432 // #pragma omp simd reduction(inscan, +: x)
5433 // for (..) {
5434 // <first part>
5435 // #pragma omp scan inclusive(x)
5436 // <second part>
5437 // }
5438 // is transformed to:
5439 // int x = 0;
5440 // for (..) {
5441 // int x_priv = 0;
5442 // <first part>
5443 // x = x_priv + x;
5444 // x_priv = x;
5445 // <second part>
5446 // }
5447 // and
5448 // int x = 0;
5449 // #pragma omp simd reduction(inscan, +: x)
5450 // for (..) {
5451 // <first part>
5452 // #pragma omp scan exclusive(x)
5453 // <second part>
5454 // }
5455 // to
5456 // int x = 0;
5457 // for (..) {
5458 // int x_priv = 0;
5459 // <second part>
5460 // int temp = x;
5461 // x = x_priv + x;
5462 // x_priv = temp;
5463 // <first part>
5464 // }
5465 llvm::BasicBlock *OMPScanReduce = createBasicBlock("omp.inscan.reduce");
5466 EmitBranch(IsInclusive
5467 ? OMPScanReduce
5468 : BreakContinueStack.back().ContinueBlock.getBlock());
5470 {
5471 // New scope for correct construction/destruction of temp variables for
5472 // exclusive scan.
5473 LexicalScope Scope(*this, S.getSourceRange());
5475 EmitBlock(OMPScanReduce);
5476 if (!IsInclusive) {
5477 // Create temp var and copy LHS value to this temp value.
5478 // TMP = LHS;
5479 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5480 const Expr *PrivateExpr = Privates[I];
5481 const Expr *TempExpr = CopyArrayTemps[I];
5483 *cast<VarDecl>(cast<DeclRefExpr>(TempExpr)->getDecl()));
5484 LValue DestLVal = EmitLValue(TempExpr);
5485 LValue SrcLVal = EmitLValue(LHSs[I]);
5486 EmitOMPCopy(PrivateExpr->getType(), DestLVal.getAddress(*this),
5487 SrcLVal.getAddress(*this),
5488 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5489 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
5490 CopyOps[I]);
5491 }
5492 }
5494 *this, ParentDir.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
5495 {/*WithNowait=*/true, /*SimpleReduction=*/true, OMPD_simd});
5496 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5497 const Expr *PrivateExpr = Privates[I];
5498 LValue DestLVal;
5499 LValue SrcLVal;
5500 if (IsInclusive) {
5501 DestLVal = EmitLValue(RHSs[I]);
5502 SrcLVal = EmitLValue(LHSs[I]);
5503 } else {
5504 const Expr *TempExpr = CopyArrayTemps[I];
5505 DestLVal = EmitLValue(RHSs[I]);
5506 SrcLVal = EmitLValue(TempExpr);
5507 }
5508 EmitOMPCopy(PrivateExpr->getType(), DestLVal.getAddress(*this),
5509 SrcLVal.getAddress(*this),
5510 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5511 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
5512 CopyOps[I]);
5513 }
5514 }
5516 OMPScanExitBlock = IsInclusive
5517 ? BreakContinueStack.back().ContinueBlock.getBlock()
5518 : OMPScanReduce;
5520 return;
5521 }
5522 if (!IsInclusive) {
5523 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
5525 }
5526 if (OMPFirstScanLoop) {
5527 // Emit buffer[i] = red; at the end of the input phase.
5528 const auto *IVExpr = cast<OMPLoopDirective>(ParentDir)
5529 .getIterationVariable()
5531 LValue IdxLVal = EmitLValue(IVExpr);
5532 llvm::Value *IdxVal = EmitLoadOfScalar(IdxLVal, IVExpr->getExprLoc());
5533 IdxVal = Builder.CreateIntCast(IdxVal, SizeTy, /*isSigned=*/false);
5534 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5535 const Expr *PrivateExpr = Privates[I];
5536 const Expr *OrigExpr = Shareds[I];
5537 const Expr *CopyArrayElem = CopyArrayElems[I];
5538 OpaqueValueMapping IdxMapping(
5539 *this,
5540 cast<OpaqueValueExpr>(
5541 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
5542 RValue::get(IdxVal));
5543 LValue DestLVal = EmitLValue(CopyArrayElem);
5544 LValue SrcLVal = EmitLValue(OrigExpr);
5545 EmitOMPCopy(PrivateExpr->getType(), DestLVal.getAddress(*this),
5546 SrcLVal.getAddress(*this),
5547 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5548 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
5549 CopyOps[I]);
5550 }
5551 }
5552 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
5553 if (IsInclusive) {
5555 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
5556 }
5558 if (!OMPFirstScanLoop) {
5559 // Emit red = buffer[i]; at the entrance to the scan phase.
5560 const auto *IVExpr = cast<OMPLoopDirective>(ParentDir)
5561 .getIterationVariable()
5563 LValue IdxLVal = EmitLValue(IVExpr);
5564 llvm::Value *IdxVal = EmitLoadOfScalar(IdxLVal, IVExpr->getExprLoc());
5565 IdxVal = Builder.CreateIntCast(IdxVal, SizeTy, /*isSigned=*/false);
5566 llvm::BasicBlock *ExclusiveExitBB = nullptr;
5567 if (!IsInclusive) {
5568 llvm::BasicBlock *ContBB = createBasicBlock("omp.exclusive.dec");
5569 ExclusiveExitBB = createBasicBlock("omp.exclusive.copy.exit");
5570 llvm::Value *Cmp = Builder.CreateIsNull(IdxVal);
5571 Builder.CreateCondBr(Cmp, ExclusiveExitBB, ContBB);
5572 EmitBlock(ContBB);
5573 // Use idx - 1 iteration for exclusive scan.
5574 IdxVal = Builder.CreateNUWSub(IdxVal, llvm::ConstantInt::get(SizeTy, 1));
5575 }
5576 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5577 const Expr *PrivateExpr = Privates[I];
5578 const Expr *OrigExpr = Shareds[I];
5579 const Expr *CopyArrayElem = CopyArrayElems[I];
5580 OpaqueValueMapping IdxMapping(
5581 *this,
5582 cast<OpaqueValueExpr>(
5583 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
5584 RValue::get(IdxVal));
5585 LValue SrcLVal = EmitLValue(CopyArrayElem);
5586 LValue DestLVal = EmitLValue(OrigExpr);
5587 EmitOMPCopy(PrivateExpr->getType(), DestLVal.getAddress(*this),
5588 SrcLVal.getAddress(*this),
5589 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5590 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
5591 CopyOps[I]);
5592 }
5593 if (!IsInclusive) {
5594 EmitBlock(ExclusiveExitBB);
5595 }
5596 }
5600}
5601
5603 const CodeGenLoopTy &CodeGenLoop,
5604 Expr *IncExpr) {
5605 // Emit the loop iteration variable.
5606 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
5607 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
5608 EmitVarDecl(*IVDecl);
5609
5610 // Emit the iterations count variable.
5611 // If it is not a variable, Sema decided to calculate iterations count on each
5612 // iteration (e.g., it is foldable into a constant).
5613 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
5614 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
5615 // Emit calculation of the iterations count.
5616 EmitIgnoredExpr(S.getCalcLastIteration());
5617 }
5618
5620
5621 bool HasLastprivateClause = false;
5622 // Check pre-condition.
5623 {
5624 OMPLoopScope PreInitScope(*this, S);
5625 // Skip the entire loop if we don't meet the precondition.
5626 // If the condition constant folds and can be elided, avoid emitting the
5627 // whole loop.
5628 bool CondConstant;
5629 llvm::BasicBlock *ContBlock = nullptr;
5630 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
5631 if (!CondConstant)
5632 return;
5633 } else {
5634 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
5635 ContBlock = createBasicBlock("omp.precond.end");
5636 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
5637 getProfileCount(&S));
5638 EmitBlock(ThenBlock);
5640 }
5641
5642 emitAlignedClause(*this, S);
5643 // Emit 'then' code.
5644 {
5645 // Emit helper vars inits.
5646
5648 *this, cast<DeclRefExpr>(
5649 (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
5650 ? S.getCombinedLowerBoundVariable()
5651 : S.getLowerBoundVariable())));
5653 *this, cast<DeclRefExpr>(
5654 (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
5655 ? S.getCombinedUpperBoundVariable()
5656 : S.getUpperBoundVariable())));
5657 LValue ST =
5658 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
5659 LValue IL =
5660 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
5661
5662 OMPPrivateScope LoopScope(*this);
5663 if (EmitOMPFirstprivateClause(S, LoopScope)) {
5664 // Emit implicit barrier to synchronize threads and avoid data races
5665 // on initialization of firstprivate variables and post-update of
5666 // lastprivate variables.
5668 *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
5669 /*ForceSimpleCall=*/true);
5670 }
5671 EmitOMPPrivateClause(S, LoopScope);
5672 if (isOpenMPSimdDirective(S.getDirectiveKind()) &&
5673 !isOpenMPParallelDirective(S.getDirectiveKind()) &&
5674 !isOpenMPTeamsDirective(S.getDirectiveKind()))
5675 EmitOMPReductionClauseInit(S, LoopScope);
5676 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
5677 EmitOMPPrivateLoopCounters(S, LoopScope);
5678 (void)LoopScope.Privatize();
5679 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
5681
5682 // Detect the distribute schedule kind and chunk.
5683 llvm::Value *Chunk = nullptr;
5685 if (const auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
5686 ScheduleKind = C->getDistScheduleKind();
5687 if (const Expr *Ch = C->getChunkSize()) {
5688 Chunk = EmitScalarExpr(Ch);
5689 Chunk = EmitScalarConversion(Chunk, Ch->getType(),
5690 S.getIterationVariable()->getType(),
5691 S.getBeginLoc());
5692 }
5693 } else {
5694 // Default behaviour for dist_schedule clause.
5696 *this, S, ScheduleKind, Chunk);
5697 }
5698 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
5699 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
5700
5701 // OpenMP [2.10.8, distribute Construct, Description]
5702 // If dist_schedule is specified, kind must be static. If specified,
5703 // iterations are divided into chunks of size chunk_size, chunks are
5704 // assigned to the teams of the league in a round-robin fashion in the
5705 // order of the team number. When no chunk_size is specified, the
5706 // iteration space is divided into chunks that are approximately equal
5707 // in size, and at most one chunk is distributed to each team of the
5708 // league. The size of the chunks is unspecified in this case.
5709 bool StaticChunked =
5710 RT.isStaticChunked(ScheduleKind, /* Chunked */ Chunk != nullptr) &&
5711 isOpenMPLoopBoundSharingDirective(S.getDirectiveKind());
5712 if (RT.isStaticNonchunked(ScheduleKind,
5713 /* Chunked */ Chunk != nullptr) ||
5714 StaticChunked) {
5716 IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(*this),
5717 LB.getAddress(*this), UB.getAddress(*this), ST.getAddress(*this),
5718 StaticChunked ? Chunk : nullptr);
5719 RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind,
5720 StaticInit);
5721 JumpDest LoopExit =
5723 // UB = min(UB, GlobalUB);
5725 ? S.getCombinedEnsureUpperBound()
5726 : S.getEnsureUpperBound());
5727 // IV = LB;
5729 ? S.getCombinedInit()
5730 : S.getInit());
5731
5732 const Expr *Cond =
5733 isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
5734 ? S.getCombinedCond()
5735 : S.getCond();
5736
5737 if (StaticChunked)
5738 Cond = S.getCombinedDistCond();
5739
5740 // For static unchunked schedules generate:
5741 //
5742 // 1. For distribute alone, codegen
5743 // while (idx <= UB) {
5744 // BODY;
5745 // ++idx;
5746 // }
5747 //
5748 // 2. When combined with 'for' (e.g. as in 'distribute parallel for')
5749 // while (idx <= UB) {
5750 // <CodeGen rest of pragma>(LB, UB);
5751 // idx += ST;
5752 // }
5753 //
5754 // For static chunk one schedule generate:
5755 //
5756 // while (IV <= GlobalUB) {
5757 // <CodeGen rest of pragma>(LB, UB);
5758 // LB += ST;
5759 // UB += ST;
5760 // UB = min(UB, GlobalUB);
5761 // IV = LB;
5762 // }
5763 //
5765 *this, S,
5766 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
5767 if (isOpenMPSimdDirective(S.getDirectiveKind()))
5768 CGF.EmitOMPSimdInit(S);
5769 },
5770 [&S, &LoopScope, Cond, IncExpr, LoopExit, &CodeGenLoop,
5771 StaticChunked](CodeGenFunction &CGF, PrePostActionTy &) {
5772 CGF.EmitOMPInnerLoop(
5773 S, LoopScope.requiresCleanups(), Cond, IncExpr,
5774 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
5775 CodeGenLoop(CGF, S, LoopExit);
5776 },
5777 [&S, StaticChunked](CodeGenFunction &CGF) {
5778 if (StaticChunked) {
5779 CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());
5780 CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());
5781 CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());
5782 CGF.EmitIgnoredExpr(S.getCombinedInit());
5783 }
5784 });
5785 });
5786 EmitBlock(LoopExit.getBlock());
5787 // Tell the runtime we are done.
5788 RT.emitForStaticFinish(*this, S.getEndLoc(), OMPD_distribute);
5789 } else {
5790 // Emit the outer loop, which requests its work chunk [LB..UB] from
5791 // runtime and runs the inner loop to process it.
5792 const OMPLoopArguments LoopArguments = {
5793 LB.getAddress(*this), UB.getAddress(*this), ST.getAddress(*this),
5794 IL.getAddress(*this), Chunk};
5795 EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
5796 CodeGenLoop);
5797 }
5798 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
5799 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
5800 return CGF.Builder.CreateIsNotNull(
5801 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
5802 });
5803 }
5804 if (isOpenMPSimdDirective(S.getDirectiveKind()) &&
5805 !isOpenMPParallelDirective(S.getDirectiveKind()) &&
5806 !isOpenMPTeamsDirective(S.getDirectiveKind())) {
5807 EmitOMPReductionClauseFinal(S, OMPD_simd);
5808 // Emit post-update of the reduction variables if IsLastIter != 0.
5810 *this, S, [IL, &S](CodeGenFunction &CGF) {
5811 return CGF.Builder.CreateIsNotNull(
5812 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
5813 });
5814 }
5815 // Emit final copy of the lastprivate variables if IsLastIter != 0.
5816 if (HasLastprivateClause) {
5818 S, /*NoFinals=*/false,
5819 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
5820 }
5821 }
5822
5823 // We're now done with the loop, so jump to the continuation block.
5824 if (ContBlock) {
5825 EmitBranch(ContBlock);
5826 EmitBlock(ContBlock, true);
5827 }
5828 }
5829}
5830
5832 const OMPDistributeDirective &S) {
5833 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
5835 };
5836 OMPLexicalScope Scope(*this, S, OMPD_unknown);
5837 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
5838}
5839
5841 const CapturedStmt *S,
5842 SourceLocation Loc) {
5843 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
5844 CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;
5845 CGF.CapturedStmtInfo = &CapStmtInfo;
5846 llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S, Loc);
5847 Fn->setDoesNotRecurse();
5848 return Fn;
5849}
5850
5851template <typename T>
5852static void emitRestoreIP(CodeGenFunction &CGF, const T *C,
5853 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP,
5854 llvm::OpenMPIRBuilder &OMPBuilder) {
5855
5856 unsigned NumLoops = C->getNumLoops();
5858 /*DestWidth=*/64, /*Signed=*/1);
5860 for (unsigned I = 0; I < NumLoops; I++) {
5861 const Expr *CounterVal = C->getLoopData(I);
5862 assert(CounterVal);
5863 llvm::Value *StoreValue = CGF.EmitScalarConversion(
5864 CGF.EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty,
5865 CounterVal->getExprLoc());
5866 StoreValues.emplace_back(StoreValue);
5867 }
5868 OMPDoacrossKind<T> ODK;
5869 bool IsDependSource = ODK.isSource(C);
5870 CGF.Builder.restoreIP(
5871 OMPBuilder.createOrderedDepend(CGF.Builder, AllocaIP, NumLoops,
5872 StoreValues, ".cnt.addr", IsDependSource));
5873}
5874
5876 if (CGM.getLangOpts().OpenMPIRBuilder) {
5877 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
5878 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5879
5880 if (S.hasClausesOfKind<OMPDependClause>() ||
5881 S.hasClausesOfKind<OMPDoacrossClause>()) {
5882 // The ordered directive with depend clause.
5883 assert(!S.hasAssociatedStmt() && "No associated statement must be in "
5884 "ordered depend|doacross construct.");
5885 InsertPointTy AllocaIP(AllocaInsertPt->getParent(),
5886 AllocaInsertPt->getIterator());
5887 for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
5888 emitRestoreIP(*this, DC, AllocaIP, OMPBuilder);
5889 for (const auto *DC : S.getClausesOfKind<OMPDoacrossClause>())
5890 emitRestoreIP(*this, DC, AllocaIP, OMPBuilder);
5891 } else {
5892 // The ordered directive with threads or simd clause, or without clause.
5893 // Without clause, it behaves as if the threads clause is specified.
5894 const auto *C = S.getSingleClause<OMPSIMDClause>();
5895
5896 auto FiniCB = [this](InsertPointTy IP) {
5898 };
5899
5900 auto BodyGenCB = [&S, C, this](InsertPointTy AllocaIP,
5901 InsertPointTy CodeGenIP) {
5902 Builder.restoreIP(CodeGenIP);
5903
5904 const CapturedStmt *CS = S.getInnermostCapturedStmt();
5905 if (C) {
5906 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
5907 Builder, /*CreateBranch=*/false, ".ordered.after");
5909 GenerateOpenMPCapturedVars(*CS, CapturedVars);
5910 llvm::Function *OutlinedFn =
5911 emitOutlinedOrderedFunction(CGM, CS, S.getBeginLoc());
5912 assert(S.getBeginLoc().isValid() &&
5913 "Outlined function call location must be valid.");
5914 ApplyDebugLocation::CreateDefaultArtificial(*this, S.getBeginLoc());
5915 OMPBuilderCBHelpers::EmitCaptureStmt(*this, CodeGenIP, *FiniBB,
5916 OutlinedFn, CapturedVars);
5917 } else {
5919 *this, CS->getCapturedStmt(), AllocaIP, CodeGenIP, "ordered");
5920 }
5921 };
5922
5923 OMPLexicalScope Scope(*this, S, OMPD_unknown);
5924 Builder.restoreIP(
5925 OMPBuilder.createOrderedThreadsSimd(Builder, BodyGenCB, FiniCB, !C));
5926 }
5927 return;
5928 }
5929
5930 if (S.hasClausesOfKind<OMPDependClause>()) {
5931 assert(!S.hasAssociatedStmt() &&
5932 "No associated statement must be in ordered depend construct.");
5933 for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
5935 return;
5936 }
5937 if (S.hasClausesOfKind<OMPDoacrossClause>()) {
5938 assert(!S.hasAssociatedStmt() &&
5939 "No associated statement must be in ordered doacross construct.");
5940 for (const auto *DC : S.getClausesOfKind<OMPDoacrossClause>())
5942 return;
5943 }
5944 const auto *C = S.getSingleClause<OMPSIMDClause>();
5945 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
5946 PrePostActionTy &Action) {
5947 const CapturedStmt *CS = S.getInnermostCapturedStmt();
5948 if (C) {
5950 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
5951 llvm::Function *OutlinedFn =
5952 emitOutlinedOrderedFunction(CGM, CS, S.getBeginLoc());
5953 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
5954 OutlinedFn, CapturedVars);
5955 } else {
5956 Action.Enter(CGF);
5957 CGF.EmitStmt(CS->getCapturedStmt());
5958 }
5959 };
5960 OMPLexicalScope Scope(*this, S, OMPD_unknown);
5961 CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getBeginLoc(), !C);
5962}
5963
5964static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val,
5965 QualType SrcType, QualType DestType,
5966 SourceLocation Loc) {
5967 assert(CGF.hasScalarEvaluationKind(DestType) &&
5968 "DestType must have scalar evaluation kind.");
5969 assert(!Val.isAggregate() && "Must be a scalar or complex.");
5970 return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
5971 DestType, Loc)
5973 Val.getComplexVal(), SrcType, DestType, Loc);
5974}
5975
5978 QualType DestType, SourceLocation Loc) {
5979 assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
5980 "DestType must have complex evaluation kind.");
5982 if (Val.isScalar()) {
5983 // Convert the input element to the element type of the complex.
5984 QualType DestElementType =
5985 DestType->castAs<ComplexType>()->getElementType();
5986 llvm::Value *ScalarVal = CGF.EmitScalarConversion(
5987 Val.getScalarVal(), SrcType, DestElementType, Loc);
5988 ComplexVal = CodeGenFunction::ComplexPairTy(
5989 ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
5990 } else {
5991 assert(Val.isComplex() && "Must be a scalar or complex.");
5992 QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
5993 QualType DestElementType =
5994 DestType->castAs<ComplexType>()->getElementType();
5995 ComplexVal.first = CGF.EmitScalarConversion(
5996 Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
5997 ComplexVal.second = CGF.EmitScalarConversion(
5998 Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
5999 }
6000 return ComplexVal;
6001}
6002
6003static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO,
6004 LValue LVal, RValue RVal) {
6005 if (LVal.isGlobalReg())
6006 CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
6007 else
6008 CGF.EmitAtomicStore(RVal, LVal, AO, LVal.isVolatile(), /*isInit=*/false);
6009}
6010
6012 llvm::AtomicOrdering AO, LValue LVal,
6013 SourceLocation Loc) {
6014 if (LVal.isGlobalReg())
6015 return CGF.EmitLoadOfLValue(LVal, Loc);
6016 return CGF.EmitAtomicLoad(
6017 LVal, Loc, llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO),
6018 LVal.isVolatile());
6019}
6020
6022 QualType RValTy, SourceLocation Loc) {
6023 switch (getEvaluationKind(LVal.getType())) {
6024 case TEK_Scalar:
6026 *this, RVal, RValTy, LVal.getType(), Loc)),
6027 LVal);
6028 break;
6029 case TEK_Complex:
6031 convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,
6032 /*isInit=*/false);
6033 break;
6034 case TEK_Aggregate:
6035 llvm_unreachable("Must be a scalar or complex.");
6036 }
6037}
6038
6039static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO,
6040 const Expr *X, const Expr *V,
6041 SourceLocation Loc) {
6042 // v = x;
6043 assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
6044 assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
6045 LValue XLValue = CGF.EmitLValue(X);
6046 LValue VLValue = CGF.EmitLValue(V);
6047 RValue Res = emitSimpleAtomicLoad(CGF, AO, XLValue, Loc);
6048 // OpenMP, 2.17.7, atomic Construct
6049 // If the read or capture clause is specified and the acquire, acq_rel, or
6050 // seq_cst clause is specified then the strong flush on exit from the atomic
6051 // operation is also an acquire flush.
6052 switch (AO) {
6053 case llvm::AtomicOrdering::Acquire:
6054 case llvm::AtomicOrdering::AcquireRelease:
6055 case llvm::AtomicOrdering::SequentiallyConsistent:
6056 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, std::nullopt, Loc,
6057 llvm::AtomicOrdering::Acquire);
6058 break;
6059 case llvm::AtomicOrdering::Monotonic:
6060 case llvm::AtomicOrdering::Release:
6061 break;
6062 case llvm::AtomicOrdering::NotAtomic:
6063 case llvm::AtomicOrdering::Unordered:
6064 llvm_unreachable("Unexpected ordering.");
6065 }
6066 CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
6068}
6069
6071 llvm::AtomicOrdering AO, const Expr *X,
6072 const Expr *E, SourceLocation Loc) {
6073 // x = expr;
6074 assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
6075 emitSimpleAtomicStore(CGF, AO, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
6077 // OpenMP, 2.17.7, atomic Construct
6078 // If the write, update, or capture clause is specified and the release,
6079 // acq_rel, or seq_cst clause is specified then the strong flush on entry to
6080 // the atomic operation is also a release flush.
6081 switch (AO) {
6082 case llvm::AtomicOrdering::Release:
6083 case llvm::AtomicOrdering::AcquireRelease:
6084 case llvm::AtomicOrdering::SequentiallyConsistent:
6085 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, std::nullopt, Loc,
6086 llvm::AtomicOrdering::Release);
6087 break;
6088 case llvm::AtomicOrdering::Acquire:
6089 case llvm::AtomicOrdering::Monotonic:
6090 break;
6091 case llvm::AtomicOrdering::NotAtomic:
6092 case llvm::AtomicOrdering::Unordered:
6093 llvm_unreachable("Unexpected ordering.");
6094 }
6095}
6096
6097static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
6098 RValue Update,
6100 llvm::AtomicOrdering AO,
6101 bool IsXLHSInRHSPart) {
6102 ASTContext &Context = CGF.getContext();
6103 // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
6104 // expression is simple and atomic is allowed for the given type for the
6105 // target platform.
6106 if (BO == BO_Comma || !Update.isScalar() || !X.isSimple() ||
6107 (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
6108 (Update.getScalarVal()->getType() !=
6109 X.getAddress(CGF).getElementType())) ||
6111 Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
6112 return std::make_pair(false, RValue::get(nullptr));
6113
6114 auto &&CheckAtomicSupport = [&CGF](llvm::Type *T, BinaryOperatorKind BO) {
6115 if (T->isIntegerTy())
6116 return true;
6117
6118 if (T->isFloatingPointTy() && (BO == BO_Add || BO == BO_Sub))
6119 return llvm::isPowerOf2_64(CGF.CGM.getDataLayout().getTypeStoreSize(T));
6120
6121 return false;
6122 };
6123
6124 if (!CheckAtomicSupport(Update.getScalarVal()->getType(), BO) ||
6125 !CheckAtomicSupport(X.getAddress(CGF).getElementType(), BO))
6126 return std::make_pair(false, RValue::get(nullptr));
6127
6128 bool IsInteger = X.getAddress(CGF).getElementType()->isIntegerTy();
6129 llvm::AtomicRMWInst::BinOp RMWOp;
6130 switch (BO) {
6131 case BO_Add:
6132 RMWOp = IsInteger ? llvm::AtomicRMWInst::Add : llvm::AtomicRMWInst::FAdd;
6133 break;
6134 case BO_Sub:
6135 if (!IsXLHSInRHSPart)
6136 return std::make_pair(false, RValue::get(nullptr));
6137 RMWOp = IsInteger ? llvm::AtomicRMWInst::Sub : llvm::AtomicRMWInst::FSub;
6138 break;
6139 case BO_And:
6140 RMWOp = llvm::AtomicRMWInst::And;
6141 break;
6142 case BO_Or:
6143 RMWOp = llvm::AtomicRMWInst::Or;
6144 break;
6145 case BO_Xor:
6146 RMWOp = llvm::AtomicRMWInst::Xor;
6147 break;
6148 case BO_LT:
6149 if (IsInteger)
6150 RMWOp = X.getType()->hasSignedIntegerRepresentation()
6151 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
6152 : llvm::AtomicRMWInst::Max)
6153 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
6154 : llvm::AtomicRMWInst::UMax);
6155 else
6156 RMWOp = IsXLHSInRHSPart ? llvm::AtomicRMWInst::FMin
6157 : llvm::AtomicRMWInst::FMax;
6158 break;
6159 case BO_GT:
6160 if (IsInteger)
6161 RMWOp = X.getType()->hasSignedIntegerRepresentation()
6162 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
6163 : llvm::AtomicRMWInst::Min)
6164 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
6165 : llvm::AtomicRMWInst::UMin);
6166 else
6167 RMWOp = IsXLHSInRHSPart ? llvm::AtomicRMWInst::FMax
6168 : llvm::AtomicRMWInst::FMin;
6169 break;
6170 case BO_Assign:
6171 RMWOp = llvm::AtomicRMWInst::Xchg;
6172 break;
6173 case BO_Mul:
6174 case BO_Div:
6175 case BO_Rem:
6176 case BO_Shl:
6177 case BO_Shr:
6178 case BO_LAnd:
6179 case BO_LOr:
6180 return std::make_pair(false, RValue::get(nullptr));
6181 case BO_PtrMemD:
6182 case BO_PtrMemI:
6183 case BO_LE:
6184 case BO_GE:
6185 case BO_EQ:
6186 case BO_NE:
6187 case BO_Cmp:
6188 case BO_AddAssign:
6189 case BO_SubAssign:
6190 case BO_AndAssign:
6191 case BO_OrAssign:
6192 case BO_XorAssign:
6193 case BO_MulAssign:
6194 case BO_DivAssign:
6195 case BO_RemAssign:
6196 case BO_ShlAssign:
6197 case BO_ShrAssign:
6198 case BO_Comma:
6199 llvm_unreachable("Unsupported atomic update operation");
6200 }
6201 llvm::Value *UpdateVal = Update.getScalarVal();
6202 if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
6203 if (IsInteger)
6204 UpdateVal = CGF.Builder.CreateIntCast(
6205 IC, X.getAddress(CGF).getElementType(),
6206 X.getType()->hasSignedIntegerRepresentation());
6207 else
6208 UpdateVal = CGF.Builder.CreateCast(llvm::Instruction::CastOps::UIToFP, IC,
6209 X.getAddress(CGF).getElementType());
6210 }
6211 llvm::Value *Res =
6212 CGF.Builder.CreateAtomicRMW(RMWOp, X.getAddress(CGF), UpdateVal, AO);
6213 return std::make_pair(true, RValue::get(Res));
6214}
6215
6217 LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
6218 llvm::AtomicOrdering AO, SourceLocation Loc,
6219 const llvm::function_ref<RValue(RValue)> CommonGen) {
6220 // Update expressions are allowed to have the following forms:
6221 // x binop= expr; -> xrval + expr;
6222 // x++, ++x -> xrval + 1;
6223 // x--, --x -> xrval - 1;
6224 // x = x binop expr; -> xrval binop expr
6225 // x = expr Op x; - > expr binop xrval;
6226 auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);
6227 if (!Res.first) {
6228 if (X.isGlobalReg()) {
6229 // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
6230 // 'xrval'.
6231 EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
6232 } else {
6233 // Perform compare-and-swap procedure.
6234 EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
6235 }
6236 }
6237 return Res;
6238}
6239
6241 llvm::AtomicOrdering AO, const Expr *X,
6242 const Expr *E, const Expr *UE,
6243 bool IsXLHSInRHSPart, SourceLocation Loc) {
6244 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
6245 "Update expr in 'atomic update' must be a binary operator.");
6246 const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
6247 // Update expressions are allowed to have the following forms:
6248 // x binop= expr; -> xrval + expr;
6249 // x++, ++x -> xrval + 1;
6250 // x--, --x -> xrval - 1;
6251 // x = x binop expr; -> xrval binop expr
6252 // x = expr Op x; - > expr binop xrval;
6253 assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
6254 LValue XLValue = CGF.EmitLValue(X);
6255 RValue ExprRValue = CGF.EmitAnyExpr(E);
6256 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
6257 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
6258 const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
6259 const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
6260 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {
6261 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
6262 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
6263 return CGF.EmitAnyExpr(UE);
6264 };
6266 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
6268 // OpenMP, 2.17.7, atomic Construct
6269 // If the write, update, or capture clause is specified and the release,
6270 // acq_rel, or seq_cst clause is specified then the strong flush on entry to
6271 // the atomic operation is also a release flush.
6272 switch (AO) {
6273 case llvm::AtomicOrdering::Release:
6274 case llvm::AtomicOrdering::AcquireRelease:
6275 case llvm::AtomicOrdering::SequentiallyConsistent:
6276 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, std::nullopt, Loc,
6277 llvm::AtomicOrdering::Release);
6278 break;
6279 case llvm::AtomicOrdering::Acquire:
6280 case llvm::AtomicOrdering::Monotonic:
6281 break;
6282 case llvm::AtomicOrdering::NotAtomic:
6283 case llvm::AtomicOrdering::Unordered:
6284 llvm_unreachable("Unexpected ordering.");
6285 }
6286}
6287
6289 QualType SourceType, QualType ResType,
6290 SourceLocation Loc) {
6291 switch (CGF.getEvaluationKind(ResType)) {
6292 case TEK_Scalar:
6293 return RValue::get(
6294 convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
6295 case TEK_Complex: {
6296 auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
6297 return RValue::getComplex(Res.first, Res.second);
6298 }
6299 case TEK_Aggregate:
6300 break;
6301 }
6302 llvm_unreachable("Must be a scalar or complex.");
6303}
6304
6306 llvm::AtomicOrdering AO,
6307 bool IsPostfixUpdate, const Expr *V,
6308 const Expr *X, const Expr *E,
6309 const Expr *UE, bool IsXLHSInRHSPart,
6310 SourceLocation Loc) {
6311 assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
6312 assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
6313 RValue NewVVal;
6314 LValue VLValue = CGF.EmitLValue(V);
6315 LValue XLValue = CGF.EmitLValue(X);
6316 RValue ExprRValue = CGF.EmitAnyExpr(E);
6317 QualType NewVValType;
6318 if (UE) {
6319 // 'x' is updated with some additional value.
6320 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
6321 "Update expr in 'atomic capture' must be a binary operator.");
6322 const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
6323 // Update expressions are allowed to have the following forms:
6324 // x binop= expr; -> xrval + expr;
6325 // x++, ++x -> xrval + 1;
6326 // x--, --x -> xrval - 1;
6327 // x = x binop expr; -> xrval binop expr
6328 // x = expr Op x; - > expr binop xrval;
6329 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
6330 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
6331 const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
6332 NewVValType = XRValExpr->getType();
6333 const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
6334 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
6335 IsPostfixUpdate](RValue XRValue) {
6336 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
6337 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
6338 RValue Res = CGF.EmitAnyExpr(UE);
6339 NewVVal = IsPostfixUpdate ? XRValue : Res;
6340 return Res;
6341 };
6342 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
6343 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
6345 if (Res.first) {
6346 // 'atomicrmw' instruction was generated.
6347 if (IsPostfixUpdate) {
6348 // Use old value from 'atomicrmw'.
6349 NewVVal = Res.second;
6350 } else {
6351 // 'atomicrmw' does not provide new value, so evaluate it using old
6352 // value of 'x'.
6353 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
6354 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
6355 NewVVal = CGF.EmitAnyExpr(UE);
6356 }
6357 }
6358 } else {
6359 // 'x' is simply rewritten with some 'expr'.
6360 NewVValType = X->getType().getNonReferenceType();
6361 ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
6362 X->getType().getNonReferenceType(), Loc);
6363 auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) {
6364 NewVVal = XRValue;
6365 return ExprRValue;
6366 };
6367 // Try to perform atomicrmw xchg, otherwise simple exchange.
6368 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
6369 XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
6370 Loc, Gen);
6372 if (Res.first) {
6373 // 'atomicrmw' instruction was generated.
6374 NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
6375 }
6376 }
6377 // Emit post-update store to 'v' of old/new 'x' value.
6378 CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
6380 // OpenMP 5.1 removes the required flush for capture clause.
6381 if (CGF.CGM.getLangOpts().OpenMP < 51) {
6382 // OpenMP, 2.17.7, atomic Construct
6383 // If the write, update, or capture clause is specified and the release,
6384 // acq_rel, or seq_cst clause is specified then the strong flush on entry to
6385 // the atomic operation is also a release flush.
6386 // If the read or capture clause is specified and the acquire, acq_rel, or
6387 // seq_cst clause is specified then the strong flush on exit from the atomic
6388 // operation is also an acquire flush.
6389 switch (AO) {
6390 case llvm::AtomicOrdering::Release:
6391 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, std::nullopt, Loc,
6392 llvm::AtomicOrdering::Release);
6393 break;
6394 case llvm::AtomicOrdering::Acquire:
6395 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, std::nullopt, Loc,
6396 llvm::AtomicOrdering::Acquire);
6397 break;
6398 case llvm::AtomicOrdering::AcquireRelease:
6399 case llvm::AtomicOrdering::SequentiallyConsistent:
6401 CGF, std::nullopt, Loc, llvm::AtomicOrdering::AcquireRelease);
6402 break;
6403 case llvm::AtomicOrdering::Monotonic:
6404 break;
6405 case llvm::AtomicOrdering::NotAtomic:
6406 case llvm::AtomicOrdering::Unordered:
6407 llvm_unreachable("Unexpected ordering.");
6408 }
6409 }
6410}
6411
6413 CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO,
6414 const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *D,
6415 const Expr *CE, bool IsXBinopExpr, bool IsPostfixUpdate, bool IsFailOnly,
6416 SourceLocation Loc) {
6417 llvm::OpenMPIRBuilder &OMPBuilder =
6419
6420 OMPAtomicCompareOp Op;
6421 assert(isa<BinaryOperator>(CE) && "CE is not a BinaryOperator");
6422 switch (cast<BinaryOperator>(CE)->getOpcode()) {
6423 case BO_EQ:
6424 Op = OMPAtomicCompareOp::EQ;
6425 break;
6426 case BO_LT:
6427 Op = OMPAtomicCompareOp::MIN;
6428 break;
6429 case BO_GT:
6430 Op = OMPAtomicCompareOp::MAX;
6431 break;
6432 default:
6433 llvm_unreachable("unsupported atomic compare binary operator");
6434 }
6435
6436 LValue XLVal = CGF.EmitLValue(X);
6437 Address XAddr = XLVal.getAddress(CGF);
6438
6439 auto EmitRValueWithCastIfNeeded = [&CGF, Loc](const Expr *X, const Expr *E) {
6440 if (X->getType() == E->getType())
6441 return CGF.EmitScalarExpr(E);
6442 const Expr *NewE = E->IgnoreImplicitAsWritten();
6443 llvm::Value *V = CGF.EmitScalarExpr(NewE);
6444 if (NewE->getType() == X->getType())
6445 return V;
6446 return CGF.EmitScalarConversion(V, NewE->getType(), X->getType(), Loc);
6447 };
6448
6449 llvm::Value *EVal = EmitRValueWithCastIfNeeded(X, E);
6450 llvm::Value *DVal = D ? EmitRValueWithCastIfNeeded(X, D) : nullptr;
6451 if (auto *CI = dyn_cast<llvm::ConstantInt>(EVal))
6452 EVal = CGF.Builder.CreateIntCast(
6453 CI, XLVal.getAddress(CGF).getElementType(),
6455 if (DVal)
6456 if (auto *CI = dyn_cast<llvm::ConstantInt>(DVal))
6457 DVal = CGF.Builder.CreateIntCast(
6458 CI, XLVal.getAddress(CGF).getElementType(),
6460
6461 llvm::OpenMPIRBuilder::AtomicOpValue XOpVal{
6462 XAddr.getPointer(), XAddr.getElementType(),
6463 X->getType()->hasSignedIntegerRepresentation(),
6464 X->getType().isVolatileQualified()};
6465 llvm::OpenMPIRBuilder::AtomicOpValue VOpVal, ROpVal;
6466 if (V) {
6467 LValue LV = CGF.EmitLValue(V);
6468 Address Addr = LV.getAddress(CGF);
6469 VOpVal = {Addr.getPointer(), Addr.getElementType(),
6470 V->getType()->hasSignedIntegerRepresentation(),
6471 V->getType().isVolatileQualified()};
6472 }
6473 if (R) {
6474 LValue LV = CGF.EmitLValue(R);
6475 Address Addr = LV.getAddress(CGF);
6476 ROpVal = {Addr.getPointer(), Addr.getElementType(),
6479 }
6480
6481 if (FailAO == llvm::AtomicOrdering::NotAtomic) {
6482 // fail clause was not mentionend on the
6483 // "#pragma omp atomic compare" construct.
6484 CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare(
6485 CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
6486 IsPostfixUpdate, IsFailOnly));
6487 } else
6488 CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare(
6489 CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
6490 IsPostfixUpdate, IsFailOnly, FailAO));
6491}
6492
6494 llvm::AtomicOrdering AO,
6495 llvm::AtomicOrdering FailAO, bool IsPostfixUpdate,
6496 const Expr *X, const Expr *V, const Expr *R,
6497 const Expr *E, const Expr *UE, const Expr *D,
6498 const Expr *CE, bool IsXLHSInRHSPart,
6499 bool IsFailOnly, SourceLocation Loc) {
6500 switch (Kind) {
6501 case OMPC_read:
6502 emitOMPAtomicReadExpr(CGF, AO, X, V, Loc);
6503 break;
6504 case OMPC_write:
6505 emitOMPAtomicWriteExpr(CGF, AO, X, E, Loc);
6506 break;
6507 case OMPC_unknown:
6508 case OMPC_update:
6509 emitOMPAtomicUpdateExpr(CGF, AO, X, E, UE, IsXLHSInRHSPart, Loc);
6510 break;
6511 case OMPC_capture:
6512 emitOMPAtomicCaptureExpr(CGF, AO, IsPostfixUpdate, V, X, E, UE,
6513 IsXLHSInRHSPart, Loc);
6514 break;
6515 case OMPC_compare: {
6516 emitOMPAtomicCompareExpr(CGF, AO, FailAO, X, V, R, E, D, CE,
6517 IsXLHSInRHSPart, IsPostfixUpdate, IsFailOnly, Loc);
6518 break;
6519 }
6520 default:
6521 llvm_unreachable("Clause is not allowed in 'omp atomic'.");
6522 }
6523}
6524
6526 llvm::AtomicOrdering AO = llvm::AtomicOrdering::Monotonic;
6527 // Fail Memory Clause Ordering.
6528 llvm::AtomicOrdering FailAO = llvm::AtomicOrdering::NotAtomic;
6529 bool MemOrderingSpecified = false;
6530 if (S.getSingleClause<OMPSeqCstClause>()) {
6531 AO = llvm::AtomicOrdering::SequentiallyConsistent;
6532 MemOrderingSpecified = true;
6533 } else if (S.getSingleClause<OMPAcqRelClause>()) {
6534 AO = llvm::AtomicOrdering::AcquireRelease;
6535 MemOrderingSpecified = true;
6536 } else if (S.getSingleClause<OMPAcquireClause>()) {
6537 AO = llvm::AtomicOrdering::Acquire;
6538 MemOrderingSpecified = true;
6539 } else if (S.getSingleClause<OMPReleaseClause>()) {
6540 AO = llvm::AtomicOrdering::Release;
6541 MemOrderingSpecified = true;
6542 } else if (S.getSingleClause<OMPRelaxedClause>()) {
6543 AO = llvm::AtomicOrdering::Monotonic;
6544 MemOrderingSpecified = true;
6545 }
6546 llvm::SmallSet<OpenMPClauseKind, 2> KindsEncountered;
6547 OpenMPClauseKind Kind = OMPC_unknown;
6548 for (const OMPClause *C : S.clauses()) {
6549 // Find first clause (skip seq_cst|acq_rel|aqcuire|release|relaxed clause,
6550 // if it is first).
6551 OpenMPClauseKind K = C->getClauseKind();
6552 // TBD
6553 if (K == OMPC_weak)
6554 return;
6555 if (K == OMPC_seq_cst || K == OMPC_acq_rel || K == OMPC_acquire ||
6556 K == OMPC_release || K == OMPC_relaxed || K == OMPC_hint)
6557 continue;
6558 Kind = K;
6559 KindsEncountered.insert(K);
6560 }
6561 // We just need to correct Kind here. No need to set a bool saying it is
6562 // actually compare capture because we can tell from whether V and R are
6563 // nullptr.
6564 if (KindsEncountered.contains(OMPC_compare) &&
6565 KindsEncountered.contains(OMPC_capture))
6566 Kind = OMPC_compare;
6567 if (!MemOrderingSpecified) {
6568 llvm::AtomicOrdering DefaultOrder =
6570 if (DefaultOrder == llvm::AtomicOrdering::Monotonic ||
6571 DefaultOrder == llvm::AtomicOrdering::SequentiallyConsistent ||
6572 (DefaultOrder == llvm::AtomicOrdering::AcquireRelease &&
6573 Kind == OMPC_capture)) {
6574 AO = DefaultOrder;
6575 } else if (DefaultOrder == llvm::AtomicOrdering::AcquireRelease) {
6576 if (Kind == OMPC_unknown || Kind == OMPC_update || Kind == OMPC_write) {
6577 AO = llvm::AtomicOrdering::Release;
6578 } else if (Kind == OMPC_read) {
6579 assert(Kind == OMPC_read && "Unexpected atomic kind.");
6580 AO = llvm::AtomicOrdering::Acquire;
6581 }
6582 }
6583 }
6584
6585 if (KindsEncountered.contains(OMPC_compare) &&
6586 KindsEncountered.contains(OMPC_fail)) {
6587 Kind = OMPC_compare;
6588 const auto *FailClause = S.getSingleClause<OMPFailClause>();
6589 if (FailClause) {
6590 OpenMPClauseKind FailParameter = FailClause->getFailParameter();
6591 if (FailParameter == llvm::omp::OMPC_relaxed)
6592 FailAO = llvm::AtomicOrdering::Monotonic;
6593 else if (FailParameter == llvm::omp::OMPC_acquire)
6594 FailAO = llvm::AtomicOrdering::Acquire;
6595 else if (FailParameter == llvm::omp::OMPC_seq_cst)
6596 FailAO = llvm::AtomicOrdering::SequentiallyConsistent;
6597 }
6598 }
6599
6600 LexicalScope Scope(*this, S.getSourceRange());
6601 EmitStopPoint(S.getAssociatedStmt());
6602 emitOMPAtomicExpr(*this, Kind, AO, FailAO, S.isPostfixUpdate(), S.getX(),
6603 S.getV(), S.getR(), S.getExpr(), S.getUpdateExpr(),
6604 S.getD(), S.getCondExpr(), S.isXLHSInRHSPart(),
6605 S.isFailOnly(), S.getBeginLoc());
6606}
6607
6609 const OMPExecutableDirective &S,
6610 const RegionCodeGenTy &CodeGen) {
6611 assert(isOpenMPTargetExecutionDirective(S.getDirectiveKind()));
6612 CodeGenModule &CGM = CGF.CGM;
6613
6614 // On device emit this construct as inlined code.
6615 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
6616 OMPLexicalScope Scope(CGF, S, OMPD_target);
6618 CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) {
6619 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
6620 });
6621 return;
6622 }
6623
6625 llvm::Function *Fn = nullptr;
6626 llvm::Constant *FnID = nullptr;
6627
6628 const Expr *IfCond = nullptr;
6629 // Check for the at most one if clause associated with the target region.
6630 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
6631 if (C->getNameModifier() == OMPD_unknown ||
6632 C->getNameModifier() == OMPD_target) {
6633 IfCond = C->getCondition();
6634 break;
6635 }
6636 }
6637
6638 // Check if we have any device clause associated with the directive.
6639 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device(
6640 nullptr, OMPC_DEVICE_unknown);
6641 if (auto *C = S.getSingleClause<OMPDeviceClause>())
6642 Device.setPointerAndInt(C->getDevice(), C->getModifier());
6643
6644 // Check if we have an if clause whose conditional always evaluates to false
6645 // or if we do not have any targets specified. If so the target region is not
6646 // an offload entry point.
6647 bool IsOffloadEntry = true;
6648 if (IfCond) {
6649 bool Val;
6650 if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
6651 IsOffloadEntry = false;
6652 }
6653 if (CGM.getLangOpts().OMPTargetTriples.empty())
6654 IsOffloadEntry = false;
6655
6656 if (CGM.getLangOpts().OpenMPOffloadMandatory && !IsOffloadEntry) {
6657 unsigned DiagID = CGM.getDiags().getCustomDiagID(
6659 "No offloading entry generated while offloading is mandatory.");
6660 CGM.getDiags().Report(DiagID);
6661 }
6662
6663 assert(CGF.CurFuncDecl && "No parent declaration for target region!");
6664 StringRef ParentName;
6665 // In case we have Ctors/Dtors we use the complete type variant to produce
6666 // the mangling of the device outlined kernel.
6667 if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
6668 ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));
6669 else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
6670 ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));
6671 else
6672 ParentName =
6673 CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));
6674
6675 // Emit target region as a standalone region.
6676 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
6677 IsOffloadEntry, CodeGen);
6678 OMPLexicalScope Scope(CGF, S, OMPD_task);
6679 auto &&SizeEmitter =
6680 [IsOffloadEntry](CodeGenFunction &CGF,
6681 const OMPLoopDirective &D) -> llvm::Value * {
6682 if (IsOffloadEntry) {
6683 OMPLoopScope(CGF, D);
6684 // Emit calculation of the iterations count.
6685 llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations());
6686 NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty,
6687 /*isSigned=*/false);
6688 return NumIterations;
6689 }
6690 return nullptr;
6691 };
6692 CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device,
6693 SizeEmitter);
6694}
6695
6697 PrePostActionTy &Action) {
6698 Action.Enter(CGF);
6699 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
6700 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
6701 CGF.EmitOMPPrivateClause(S, PrivateScope);
6702 (void)PrivateScope.Privatize();
6703 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
6705
6706 CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
6707 CGF.EnsureInsertPoint();
6708}
6709
6711 StringRef ParentName,
6712 const OMPTargetDirective &S) {
6713 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6714 emitTargetRegion(CGF, S, Action);
6715 };
6716 llvm::Function *Fn;
6717 llvm::Constant *Addr;
6718 // Emit target region as a standalone region.
6720 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
6721 assert(Fn && Addr && "Target device function emission failed.");
6722}
6723
6725 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6726 emitTargetRegion(CGF, S, Action);
6727 };
6728 emitCommonOMPTargetDirective(*this, S, CodeGen);
6729}
6730
6732 const OMPExecutableDirective &S,
6733 OpenMPDirectiveKind InnermostKind,
6734 const RegionCodeGenTy &CodeGen) {
6735 const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
6736 llvm::Function *OutlinedFn =
6738 CGF, S, *CS->getCapturedDecl()->param_begin(), InnermostKind,
6739 CodeGen);
6740
6741 const auto *NT = S.getSingleClause<OMPNumTeamsClause>();
6742 const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
6743 if (NT || TL) {
6744 const Expr *NumTeams = NT ? NT->getNumTeams() : nullptr;
6745 const Expr *ThreadLimit = TL ? TL->getThreadLimit() : nullptr;
6746
6747 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
6748 S.getBeginLoc());
6749 }
6750
6751 OMPTeamsScope Scope(CGF, S);
6753 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
6754 CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn,
6755 CapturedVars);
6756}
6757
6759 // Emit teams region as a standalone region.
6760 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6761 Action.Enter(CGF);
6762 OMPPrivateScope PrivateScope(CGF);
6763 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
6764 CGF.EmitOMPPrivateClause(S, PrivateScope);
6765 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
6766 (void)PrivateScope.Privatize();
6767 CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
6768 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
6769 };
6770 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
6772 [](CodeGenFunction &) { return nullptr; });
6773}
6774
6776 const OMPTargetTeamsDirective &S) {
6777 auto *CS = S.getCapturedStmt(OMPD_teams);
6778 Action.Enter(CGF);
6779 // Emit teams region as a standalone region.
6780 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
6781 Action.Enter(CGF);
6782 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
6783 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
6784 CGF.EmitOMPPrivateClause(S, PrivateScope);
6785 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
6786 (void)PrivateScope.Privatize();
6787 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
6789 CGF.EmitStmt(CS->getCapturedStmt());
6790 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
6791 };
6792 emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
6794 [](CodeGenFunction &) { return nullptr; });
6795}
6796
6798 CodeGenModule &CGM, StringRef ParentName,
6799 const OMPTargetTeamsDirective &S) {
6800 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6801 emitTargetTeamsRegion(CGF, Action, S);
6802 };
6803 llvm::Function *Fn;
6804 llvm::Constant *Addr;
6805 // Emit target region as a standalone region.
6807 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
6808 assert(Fn && Addr && "Target device function emission failed.");
6809}
6810
6812 const OMPTargetTeamsDirective &S) {
6813 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6814 emitTargetTeamsRegion(CGF, Action, S);
6815 };
6816 emitCommonOMPTargetDirective(*this, S, CodeGen);
6817}
6818
6819static void
6822 Action.Enter(CGF);
6823 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
6825 };
6826
6827 // Emit teams region as a standalone region.
6828 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
6829 PrePostActionTy &Action) {
6830 Action.Enter(CGF);
6831 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
6832 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
6833 (void)PrivateScope.Privatize();
6834 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
6835 CodeGenDistribute);
6836 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
6837 };
6838 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
6840 [](CodeGenFunction &) { return nullptr; });
6841}
6842
6844 CodeGenModule &CGM, StringRef ParentName,
6846 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6847 emitTargetTeamsDistributeRegion(CGF, Action, S);
6848 };
6849 llvm::Function *Fn;
6850 llvm::Constant *Addr;
6851 // Emit target region as a standalone region.
6853 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
6854 assert(Fn && Addr && "Target device function emission failed.");
6855}
6856
6859 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6860 emitTargetTeamsDistributeRegion(CGF, Action, S);
6861 };
6862 emitCommonOMPTargetDirective(*this, S, CodeGen);
6863}
6864
6866 CodeGenFunction &CGF, PrePostActionTy &Action,
6868 Action.Enter(CGF);
6869 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
6871 };
6872
6873 // Emit teams region as a standalone region.
6874 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
6875 PrePostActionTy &Action) {
6876 Action.Enter(CGF);
6877 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
6878 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
6879 (void)PrivateScope.Privatize();
6880 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
6881 CodeGenDistribute);
6882 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
6883 };
6884 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);
6886 [](CodeGenFunction &) { return nullptr; });
6887}
6888
6890 CodeGenModule &CGM, StringRef ParentName,
6892 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6894 };
6895 llvm::Function *Fn;
6896 llvm::Constant *Addr;
6897 // Emit target region as a standalone region.
6899 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
6900 assert(Fn && Addr && "Target device function emission failed.");
6901}
6902
6905 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6907 };
6908 emitCommonOMPTargetDirective(*this, S, CodeGen);
6909}
6910
6912 const OMPTeamsDistributeDirective &S) {
6913
6914 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
6916 };
6917
6918 // Emit teams region as a standalone region.
6919 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
6920 PrePostActionTy &Action) {
6921 Action.Enter(CGF);
6922 OMPPrivateScope PrivateScope(CGF);
6923 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
6924 (void)PrivateScope.Privatize();
6925 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
6926 CodeGenDistribute);
6927 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
6928 };
6929 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
6931 [](CodeGenFunction &) { return nullptr; });
6932}
6933
6936 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
6938 };
6939
6940 // Emit teams region as a standalone region.
6941 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
6942 PrePostActionTy &Action) {
6943 Action.Enter(CGF);
6944 OMPPrivateScope PrivateScope(CGF);
6945 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
6946 (void)PrivateScope.Privatize();
6947 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
6948 CodeGenDistribute);
6949 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
6950 };
6951 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen);
6953 [](CodeGenFunction &) { return nullptr; });
6954}
6955
6958 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
6960 S.getDistInc());
6961 };
6962
6963 // Emit teams region as a standalone region.
6964 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
6965 PrePostActionTy &Action) {
6966 Action.Enter(CGF);
6967 OMPPrivateScope PrivateScope(CGF);
6968 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
6969 (void)PrivateScope.Privatize();
6970 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
6971 CodeGenDistribute);
6972 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
6973 };
6974 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
6976 [](CodeGenFunction &) { return nullptr; });
6977}
6978
6981 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
6983 S.getDistInc());
6984 };
6985
6986 // Emit teams region as a standalone region.
6987 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
6988 PrePostActionTy &Action) {
6989 Action.Enter(CGF);
6990 OMPPrivateScope PrivateScope(CGF);
6991 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
6992 (void)PrivateScope.Privatize();
6993 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
6994 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
6995 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
6996 };
6997 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for_simd,
6998 CodeGen);
7000 [](CodeGenFunction &) { return nullptr; });
7001}
7002
7004 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
7005 llvm::Value *Device = nullptr;
7006 llvm::Value *NumDependences = nullptr;
7007 llvm::Value *DependenceList = nullptr;
7008
7009 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
7010 Device = EmitScalarExpr(C->getDevice());
7011
7012 // Build list and emit dependences
7015 if (!Data.Dependences.empty()) {
7016 Address DependenciesArray = Address::invalid();
7017 std::tie(NumDependences, DependenciesArray) =
7018 CGM.getOpenMPRuntime().emitDependClause(*this, Data.Dependences,
7019 S.getBeginLoc());
7020 DependenceList = DependenciesArray.getPointer();
7021 }
7022 Data.HasNowaitClause = S.hasClausesOfKind<OMPNowaitClause>();
7023
7024 assert(!(Data.HasNowaitClause && !(S.getSingleClause<OMPInitClause>() ||
7025 S.getSingleClause<OMPDestroyClause>() ||
7026 S.getSingleClause<OMPUseClause>())) &&
7027 "OMPNowaitClause clause is used separately in OMPInteropDirective.");
7028
7029 auto ItOMPInitClause = S.getClausesOfKind<OMPInitClause>();
7030 if (!ItOMPInitClause.empty()) {
7031 // Look at the multiple init clauses
7032 for (const OMPInitClause *C : ItOMPInitClause) {
7033 llvm::Value *InteropvarPtr =
7034 EmitLValue(C->getInteropVar()).getPointer(*this);
7035 llvm::omp::OMPInteropType InteropType =
7036 llvm::omp::OMPInteropType::Unknown;
7037 if (C->getIsTarget()) {
7038 InteropType = llvm::omp::OMPInteropType::Target;
7039 } else {
7040 assert(C->getIsTargetSync() &&
7041 "Expected interop-type target/targetsync");
7042 InteropType = llvm::omp::OMPInteropType::TargetSync;
7043 }
7044 OMPBuilder.createOMPInteropInit(Builder, InteropvarPtr, InteropType,
7045 Device, NumDependences, DependenceList,
7046 Data.HasNowaitClause);
7047 }
7048 }
7049 auto ItOMPDestroyClause = S.getClausesOfKind<OMPDestroyClause>();
7050 if (!ItOMPDestroyClause.empty()) {
7051 // Look at the multiple destroy clauses
7052 for (const OMPDestroyClause *C : ItOMPDestroyClause) {
7053 llvm::Value *InteropvarPtr =
7054 EmitLValue(C->getInteropVar()).getPointer(*this);
7055 OMPBuilder.createOMPInteropDestroy(Builder, InteropvarPtr, Device,
7056 NumDependences, DependenceList,
7057 Data.HasNowaitClause);
7058 }
7059 }
7060 auto ItOMPUseClause = S.getClausesOfKind<OMPUseClause>();
7061 if (!ItOMPUseClause.empty()) {
7062 // Look at the multiple use clauses
7063 for (const OMPUseClause *C : ItOMPUseClause) {
7064 llvm::Value *InteropvarPtr =
7065 EmitLValue(C->getInteropVar()).getPointer(*this);
7066 OMPBuilder.createOMPInteropUse(Builder, InteropvarPtr, Device,
7067 NumDependences, DependenceList,
7068 Data.HasNowaitClause);
7069 }
7070 }
7071}
7072
7075 PrePostActionTy &Action) {
7076 Action.Enter(CGF);
7077 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7079 S.getDistInc());
7080 };
7081
7082 // Emit teams region as a standalone region.
7083 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7084 PrePostActionTy &Action) {
7085 Action.Enter(CGF);
7086 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7087 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7088 (void)PrivateScope.Privatize();
7090 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
7091 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7092 };
7093
7094 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
7095 CodeGenTeams);
7097 [](CodeGenFunction &) { return nullptr; });
7098}
7099
7101 CodeGenModule &CGM, StringRef ParentName,
7103 // Emit SPMD target teams distribute parallel for region as a standalone
7104 // region.
7105 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7107 };
7108 llvm::Function *Fn;
7109 llvm::Constant *Addr;
7110 // Emit target region as a standalone region.
7112 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
7113 assert(Fn && Addr && "Target device function emission failed.");
7114}
7115
7118 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7120 };
7121 emitCommonOMPTargetDirective(*this, S, CodeGen);
7122}
7123
7125 CodeGenFunction &CGF,
7127 PrePostActionTy &Action) {
7128 Action.Enter(CGF);
7129 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7131 S.getDistInc());
7132 };
7133
7134 // Emit teams region as a standalone region.
7135 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7136 PrePostActionTy &Action) {
7137 Action.Enter(CGF);
7138 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7139 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7140 (void)PrivateScope.Privatize();
7142 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
7143 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7144 };
7145
7146 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd,
7147 CodeGenTeams);
7149 [](CodeGenFunction &) { return nullptr; });
7150}
7151
7153 CodeGenModule &CGM, StringRef ParentName,
7155 // Emit SPMD target teams distribute parallel for simd region as a standalone
7156 // region.
7157 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7159 };
7160 llvm::Function *Fn;
7161 llvm::Constant *Addr;
7162 // Emit target region as a standalone region.
7164 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
7165 assert(Fn && Addr && "Target device function emission failed.");
7166}
7167
7170 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7172 };
7173 emitCommonOMPTargetDirective(*this, S, CodeGen);
7174}
7175
7178 CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getBeginLoc(),
7179 S.getCancelRegion());
7180}
7181
7183 const Expr *IfCond = nullptr;
7184 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
7185 if (C->getNameModifier() == OMPD_unknown ||
7186 C->getNameModifier() == OMPD_cancel) {
7187 IfCond = C->getCondition();
7188 break;
7189 }
7190 }
7191 if (CGM.getLangOpts().OpenMPIRBuilder) {
7192 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
7193 // TODO: This check is necessary as we only generate `omp parallel` through
7194 // the OpenMPIRBuilder for now.
7195 if (S.getCancelRegion() == OMPD_parallel ||
7196 S.getCancelRegion() == OMPD_sections ||
7197 S.getCancelRegion() == OMPD_section) {
7198 llvm::Value *IfCondition = nullptr;
7199 if (IfCond)
7200 IfCondition = EmitScalarExpr(IfCond,
7201 /*IgnoreResultAssign=*/true);
7202 return Builder.restoreIP(
7203 OMPBuilder.createCancel(Builder, IfCondition, S.getCancelRegion()));
7204 }
7205 }
7206
7207 CGM.getOpenMPRuntime().emitCancelCall(*this, S.getBeginLoc(), IfCond,
7208 S.getCancelRegion());
7209}
7210
7211CodeGenFunction::JumpDest
7213 if (Kind == OMPD_parallel || Kind == OMPD_task ||
7214 Kind == OMPD_target_parallel || Kind == OMPD_taskloop ||
7215 Kind == OMPD_master_taskloop || Kind == OMPD_parallel_master_taskloop)
7216 return ReturnBlock;
7217 assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
7218 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
7219 Kind == OMPD_distribute_parallel_for ||
7220 Kind == OMPD_target_parallel_for ||
7221 Kind == OMPD_teams_distribute_parallel_for ||
7222 Kind == OMPD_target_teams_distribute_parallel_for);
7223 return OMPCancelStack.getExitBlock();
7224}
7225
7227 const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope,
7228 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
7229 CaptureDeviceAddrMap) {
7230 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
7231 for (const Expr *OrigVarIt : C.varlists()) {
7232 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(OrigVarIt)->getDecl());
7233 if (!Processed.insert(OrigVD).second)
7234 continue;
7235
7236 // In order to identify the right initializer we need to match the
7237 // declaration used by the mapping logic. In some cases we may get
7238 // OMPCapturedExprDecl that refers to the original declaration.
7239 const ValueDecl *MatchingVD = OrigVD;
7240 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
7241 // OMPCapturedExprDecl are used to privative fields of the current
7242 // structure.
7243 const auto *ME = cast<MemberExpr>(OED->getInit());
7244 assert(isa<CXXThisExpr>(ME->getBase()->IgnoreImpCasts()) &&
7245 "Base should be the current struct!");
7246 MatchingVD = ME->getMemberDecl();
7247 }
7248
7249 // If we don't have information about the current list item, move on to
7250 // the next one.
7251 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
7252 if (InitAddrIt == CaptureDeviceAddrMap.end())
7253 continue;
7254
7255 llvm::Type *Ty = ConvertTypeForMem(OrigVD->getType().getNonReferenceType());
7256
7257 // Return the address of the private variable.
7258 bool IsRegistered = PrivateScope.addPrivate(
7259 OrigVD,
7260 Address(InitAddrIt->second, Ty,
7261 getContext().getTypeAlignInChars(getContext().VoidPtrTy)));
7262 assert(IsRegistered && "firstprivate var already registered as private");
7263 // Silence the warning about unused variable.
7264 (void)IsRegistered;
7265 }
7266}
7267
7268static const VarDecl *getBaseDecl(const Expr *Ref) {
7269 const Expr *Base = Ref->IgnoreParenImpCasts();
7270 while (const auto *OASE = dyn_cast<OMPArraySectionExpr>(Base))
7271 Base = OASE->getBase()->IgnoreParenImpCasts();
7272 while (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Base))
7273 Base = ASE->getBase()->IgnoreParenImpCasts();
7274 return cast<VarDecl>(cast<DeclRefExpr>(Base)->getDecl());
7275}
7276
7278 const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope,
7279 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
7280 CaptureDeviceAddrMap) {
7281 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
7282 for (const Expr *Ref : C.varlists()) {
7283 const VarDecl *OrigVD = getBaseDecl(Ref);
7284 if (!Processed.insert(OrigVD).second)
7285 continue;
7286 // In order to identify the right initializer we need to match the
7287 // declaration used by the mapping logic. In some cases we may get
7288 // OMPCapturedExprDecl that refers to the original declaration.
7289 const ValueDecl *MatchingVD = OrigVD;
7290 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
7291 // OMPCapturedExprDecl are used to privative fields of the current
7292 // structure.
7293 const auto *ME = cast<MemberExpr>(OED->getInit());
7294 assert(isa<CXXThisExpr>(ME->getBase()) &&
7295 "Base should be the current struct!");
7296 MatchingVD = ME->getMemberDecl();
7297 }
7298
7299 // If we don't have information about the current list item, move on to
7300 // the next one.
7301 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
7302 if (InitAddrIt == CaptureDeviceAddrMap.end())
7303 continue;
7304
7305 llvm::Type *Ty = ConvertTypeForMem(OrigVD->getType().getNonReferenceType());
7306
7307 Address PrivAddr =
7308 Address(InitAddrIt->second, Ty,
7309 getContext().getTypeAlignInChars(getContext().VoidPtrTy));
7310 // For declrefs and variable length array need to load the pointer for
7311 // correct mapping, since the pointer to the data was passed to the runtime.
7312 if (isa<DeclRefExpr>(Ref->IgnoreParenImpCasts()) ||
7313 MatchingVD->getType()->isArrayType()) {
7315 OrigVD->getType().getNonReferenceType());
7316 PrivAddr =
7318 PtrTy->castAs<PointerType>());
7319 }
7320
7321 (void)PrivateScope.addPrivate(OrigVD, PrivAddr);
7322 }
7323}
7324
7325// Generate the instructions for '#pragma omp target data' directive.
7327 const OMPTargetDataDirective &S) {
7328 CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true,
7329 /*SeparateBeginEndCalls=*/true);
7330
7331 // Create a pre/post action to signal the privatization of the device pointer.
7332 // This action can be replaced by the OpenMP runtime code generation to
7333 // deactivate privatization.
7334 bool PrivatizeDevicePointers = false;
7335 class DevicePointerPrivActionTy : public PrePostActionTy {
7336 bool &PrivatizeDevicePointers;
7337
7338 public:
7339 explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
7340 : PrivatizeDevicePointers(PrivatizeDevicePointers) {}
7341 void Enter(CodeGenFunction &CGF) override {
7342 PrivatizeDevicePointers = true;
7343 }
7344 };
7345 DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
7346
7347 auto &&CodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
7348 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7349 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
7350 };
7351
7352 // Codegen that selects whether to generate the privatization code or not.
7353 auto &&PrivCodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
7354 RegionCodeGenTy RCG(InnermostCodeGen);
7355 PrivatizeDevicePointers = false;
7356
7357 // Call the pre-action to change the status of PrivatizeDevicePointers if
7358 // needed.
7359 Action.Enter(CGF);
7360
7361 if (PrivatizeDevicePointers) {
7362 OMPPrivateScope PrivateScope(CGF);
7363 // Emit all instances of the use_device_ptr clause.
7364 for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
7365 CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
7366 Info.CaptureDeviceAddrMap);
7367 for (const auto *C : S.getClausesOfKind<OMPUseDeviceAddrClause>())
7368 CGF.EmitOMPUseDeviceAddrClause(*C, PrivateScope,
7369 Info.CaptureDeviceAddrMap);
7370 (void)PrivateScope.Privatize();
7371 RCG(CGF);
7372 } else {
7373 // If we don't have target devices, don't bother emitting the data
7374 // mapping code.
7375 std::optional<OpenMPDirectiveKind> CaptureRegion;
7376 if (CGM.getLangOpts().OMPTargetTriples.empty()) {
7377 // Emit helper decls of the use_device_ptr/use_device_addr clauses.
7378 for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
7379 for (const Expr *E : C->varlists()) {
7380 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
7381 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
7382 CGF.EmitVarDecl(*OED);
7383 }
7384 for (const auto *C : S.getClausesOfKind<OMPUseDeviceAddrClause>())
7385 for (const Expr *E : C->varlists()) {
7386 const Decl *D = getBaseDecl(E);
7387 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
7388 CGF.EmitVarDecl(*OED);
7389 }
7390 } else {
7391 CaptureRegion = OMPD_unknown;
7392 }
7393
7394 OMPLexicalScope Scope(CGF, S, CaptureRegion);
7395 RCG(CGF);
7396 }
7397 };
7398
7399 // Forward the provided action to the privatization codegen.
7400 RegionCodeGenTy PrivRCG(PrivCodeGen);
7401 PrivRCG.setAction(Action);
7402
7403 // Notwithstanding the body of the region is emitted as inlined directive,
7404 // we don't use an inline scope as changes in the references inside the
7405 // region are expected to be visible outside, so we do not privative them.
7406 OMPLexicalScope Scope(CGF, S);
7407 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
7408 PrivRCG);
7409 };
7410
7411 RegionCodeGenTy RCG(CodeGen);
7412
7413 // If we don't have target devices, don't bother emitting the data mapping
7414 // code.
7415 if (CGM.getLangOpts().OMPTargetTriples.empty()) {
7416 RCG(*this);
7417 return;
7418 }
7419
7420 // Check if we have any if clause associated with the directive.
7421 const Expr *IfCond = nullptr;
7422 if (const auto *C = S.getSingleClause<OMPIfClause>())
7423 IfCond = C->getCondition();
7424
7425 // Check if we have any device clause associated with the directive.
7426 const Expr *Device = nullptr;
7427 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
7428 Device = C->getDevice();
7429
7430 // Set the action to signal privatization of device pointers.
7431 RCG.setAction(PrivAction);
7432
7433 // Emit region code.
7434 CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
7435 Info);
7436}
7437
7439 const OMPTargetEnterDataDirective &S) {
7440 // If we don't have target devices, don't bother emitting the data mapping
7441 // code.
7442 if (CGM.getLangOpts().OMPTargetTriples.empty())
7443 return;
7444
7445 // Check if we have any if clause associated with the directive.
7446 const Expr *IfCond = nullptr;
7447 if (const auto *C = S.getSingleClause<OMPIfClause>())
7448 IfCond = C->getCondition();
7449
7450 // Check if we have any device clause associated with the directive.
7451 const Expr *Device = nullptr;
7452 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
7453 Device = C->getDevice();
7454
7455 OMPLexicalScope Scope(*this, S, OMPD_task);
7457}
7458
7460 const OMPTargetExitDataDirective &S) {
7461 // If we don't have target devices, don't bother emitting the data mapping
7462 // code.
7463 if (CGM.getLangOpts().OMPTargetTriples.empty())
7464 return;
7465
7466 // Check if we have any if clause associated with the directive.
7467 const Expr *IfCond = nullptr;
7468 if (const auto *C = S.getSingleClause<OMPIfClause>())
7469 IfCond = C->getCondition();
7470
7471 // Check if we have any device clause associated with the directive.
7472 const Expr *Device = nullptr;
7473 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
7474 Device = C->getDevice();
7475
7476 OMPLexicalScope Scope(*this, S, OMPD_task);
7478}
7479
7482 PrePostActionTy &Action) {
7483 // Get the captured statement associated with the 'parallel' region.
7484 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
7485 Action.Enter(CGF);
7486 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
7487 Action.Enter(CGF);
7488 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7489 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
7490 CGF.EmitOMPPrivateClause(S, PrivateScope);
7491 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7492 (void)PrivateScope.Privatize();
7493 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
7495 // TODO: Add support for clauses.
7496 CGF.EmitStmt(CS->getCapturedStmt());
7497 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
7498 };
7499 emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
7502 [](CodeGenFunction &) { return nullptr; });
7503}
7504
7506 CodeGenModule &CGM, StringRef ParentName,
7507 const OMPTargetParallelDirective &S) {
7508 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7509 emitTargetParallelRegion(CGF, S, Action);
7510 };
7511 llvm::Function *Fn;
7512 llvm::Constant *Addr;
7513 // Emit target region as a standalone region.
7515 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
7516 assert(Fn && Addr && "Target device function emission failed.");
7517}
7518
7520 const OMPTargetParallelDirective &S) {
7521 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7522 emitTargetParallelRegion(CGF, S, Action);
7523 };
7524 emitCommonOMPTargetDirective(*this, S, CodeGen);
7525}
7526
7529 PrePostActionTy &Action) {
7530 Action.Enter(CGF);
7531 // Emit directive as a combined directive that consists of two implicit
7532 // directives: 'parallel' with 'for' directive.
7533 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7534 Action.Enter(CGF);
7535 CodeGenFunction::OMPCancelStackRAII CancelRegion(
7536 CGF, OMPD_target_parallel_for, S.hasCancel());
7537 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
7539 };
7540 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
7542}
7543
7545 CodeGenModule &CGM, StringRef ParentName,
7547 // Emit SPMD target parallel for region as a standalone region.
7548 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7549 emitTargetParallelForRegion(CGF, S, Action);
7550 };
7551 llvm::Function *Fn;
7552 llvm::Constant *Addr;
7553 // Emit target region as a standalone region.
7555 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
7556 assert(Fn && Addr && "Target device function emission failed.");
7557}
7558
7561 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7562 emitTargetParallelForRegion(CGF, S, Action);
7563 };
7564 emitCommonOMPTargetDirective(*this, S, CodeGen);
7565}
7566
7567static void
7570 PrePostActionTy &Action) {
7571 Action.Enter(CGF);
7572 // Emit directive as a combined directive that consists of two implicit
7573 // directives: 'parallel' with 'for' directive.
7574 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7575 Action.Enter(CGF);
7576 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
7578 };
7579 emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
7581}
7582
7584 CodeGenModule &CGM, StringRef ParentName,
7586 // Emit SPMD target parallel for region as a standalone region.
7587 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7588 emitTargetParallelForSimdRegion(CGF, S, Action);
7589 };
7590 llvm::Function *Fn;
7591 llvm::Constant *Addr;
7592 // Emit target region as a standalone region.
7594 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
7595 assert(Fn && Addr && "Target device function emission failed.");
7596}
7597
7600 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7601 emitTargetParallelForSimdRegion(CGF, S, Action);
7602 };
7603 emitCommonOMPTargetDirective(*this, S, CodeGen);
7604}
7605
7606/// Emit a helper variable and return corresponding lvalue.
7607static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
7608 const ImplicitParamDecl *PVD,
7609 CodeGenFunction::OMPPrivateScope &Privates) {
7610 const auto *VDecl = cast<VarDecl>(Helper->getDecl());
7611 Privates.addPrivate(VDecl, CGF.GetAddrOfLocalVar(PVD));
7612}
7613
7615 assert(isOpenMPTaskLoopDirective(S.getDirectiveKind()));
7616 // Emit outlined function for task construct.
7617 const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);
7618 Address CapturedStruct = Address::invalid();
7619 {
7620 OMPLexicalScope Scope(*this, S, OMPD_taskloop, /*EmitPreInitStmt=*/false);
7621 CapturedStruct = GenerateCapturedStmtArgument(*CS);
7622 }
7624 const Expr *IfCond = nullptr;
7625 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
7626 if (C->getNameModifier() == OMPD_unknown ||
7627 C->getNameModifier() == OMPD_taskloop) {
7628 IfCond = C->getCondition();
7629 break;
7630 }
7631 }
7632
7634 // Check if taskloop must be emitted without taskgroup.
7635 Data.Nogroup = S.getSingleClause<OMPNogroupClause>();
7636 // TODO: Check if we should emit tied or untied task.
7637 Data.Tied = true;
7638 // Set scheduling for taskloop
7639 if (const auto *Clause = S.getSingleClause<OMPGrainsizeClause>()) {
7640 // grainsize clause
7641 Data.Schedule.setInt(/*IntVal=*/false);
7642 Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
7643 } else if (const auto *Clause = S.getSingleClause<OMPNumTasksClause>()) {
7644 // num_tasks clause
7645 Data.Schedule.setInt(/*IntVal=*/true);
7646 Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
7647 }
7648
7649 auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
7650 // if (PreCond) {
7651 // for (IV in 0..LastIteration) BODY;
7652 // <Final counter/linear vars updates>;
7653 // }
7654 //
7655
7656 // Emit: if (PreCond) - begin.
7657 // If the condition constant folds and can be elided, avoid emitting the
7658 // whole loop.
7659 bool CondConstant;
7660 llvm::BasicBlock *ContBlock = nullptr;
7661 OMPLoopScope PreInitScope(CGF, S);
7662 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
7663 if (!CondConstant)
7664 return;
7665 } else {
7666 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
7667 ContBlock = CGF.createBasicBlock("taskloop.if.end");
7668 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
7669 CGF.getProfileCount(&S));
7670 CGF.EmitBlock(ThenBlock);
7671 CGF.incrementProfileCounter(&S);
7672 }
7673
7674 (void)CGF.EmitOMPLinearClauseInit(S);
7675
7676 OMPPrivateScope LoopScope(CGF);
7677 // Emit helper vars inits.
7678 enum { LowerBound = 5, UpperBound, Stride, LastIter };
7679 auto *I = CS->getCapturedDecl()->param_begin();
7680 auto *LBP = std::next(I, LowerBound);
7681 auto *UBP = std::next(I, UpperBound);
7682 auto *STP = std::next(I, Stride);
7683 auto *LIP = std::next(I, LastIter);
7684 mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
7685 LoopScope);
7686 mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
7687 LoopScope);
7688 mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
7689 mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
7690 LoopScope);
7691 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
7692 CGF.EmitOMPLinearClause(S, LoopScope);
7693 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
7694 (void)LoopScope.Privatize();
7695 // Emit the loop iteration variable.
7696 const Expr *IVExpr = S.getIterationVariable();
7697 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
7698 CGF.EmitVarDecl(*IVDecl);
7699 CGF.EmitIgnoredExpr(S.getInit());
7700
7701 // Emit the iterations count variable.
7702 // If it is not a variable, Sema decided to calculate iterations count on
7703 // each iteration (e.g., it is foldable into a constant).
7704 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
7705 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
7706 // Emit calculation of the iterations count.
7707 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
7708 }
7709
7710 {
7711 OMPLexicalScope Scope(CGF, S, OMPD_taskloop, /*EmitPreInitStmt=*/false);
7713 CGF, S,
7714 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7715 if (isOpenMPSimdDirective(S.getDirectiveKind()))
7716 CGF.EmitOMPSimdInit(S);
7717 },
7718 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
7719 CGF.EmitOMPInnerLoop(
7720 S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),
7721 [&S](CodeGenFunction &CGF) {
7722 emitOMPLoopBodyWithStopPoint(CGF, S,
7723 CodeGenFunction::JumpDest());
7724 },
7725 [](CodeGenFunction &) {});
7726 });
7727 }
7728 // Emit: if (PreCond) - end.
7729 if (ContBlock) {
7730 CGF.EmitBranch(ContBlock);
7731 CGF.EmitBlock(ContBlock, true);
7732 }
7733 // Emit final copy of the lastprivate variables if IsLastIter != 0.
7734 if (HasLastprivateClause) {
7735 CGF.EmitOMPLastprivateClauseFinal(
7736 S, isOpenMPSimdDirective(S.getDirectiveKind()),
7737 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
7738 CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
7739 (*LIP)->getType(), S.getBeginLoc())));
7740 }
7741 LoopScope.restoreMap();
7742 CGF.EmitOMPLinearClauseFinal(S, [LIP, &S](CodeGenFunction &CGF) {
7743 return CGF.Builder.CreateIsNotNull(
7744 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
7745 (*LIP)->getType(), S.getBeginLoc()));
7746 });
7747 };
7748 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
7749 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
7750 const OMPTaskDataTy &Data) {
7751 auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
7753 OMPLoopScope PreInitScope(CGF, S);
7754 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
7755 OutlinedFn, SharedsTy,
7756 CapturedStruct, IfCond, Data);
7757 };
7758 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
7759 CodeGen);
7760 };
7761 if (Data.Nogroup) {
7762 EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data);
7763 } else {
7765 *this,
7766 [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
7767 PrePostActionTy &Action) {
7768 Action.Enter(CGF);
7769 CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen,
7770 Data);
7771 },
7772 S.getBeginLoc());
7773 }
7774}
7775
7777 auto LPCRegion =
7780}
7781
7783 const OMPTaskLoopSimdDirective &S) {
7784 auto LPCRegion =
7786 OMPLexicalScope Scope(*this, S);
7788}
7789
7791 const OMPMasterTaskLoopDirective &S) {
7792 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7793 Action.Enter(CGF);
7795 };
7796 auto LPCRegion =
7798 OMPLexicalScope Scope(*this, S, std::nullopt, /*EmitPreInitStmt=*/false);
7799 CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
7800}
7801
7804 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7805 Action.Enter(CGF);
7807 };
7808 auto LPCRegion =
7810 OMPLexicalScope Scope(*this, S);
7811 CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
7812}
7813
7816 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7817 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
7818 PrePostActionTy &Action) {
7819 Action.Enter(CGF);
7821 };
7822 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
7823 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
7824 S.getBeginLoc());
7825 };
7826 auto LPCRegion =
7828 emitCommonOMPParallelDirective(*this, S, OMPD_master_taskloop, CodeGen,
7830}
7831
7834 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7835 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
7836 PrePostActionTy &Action) {
7837 Action.Enter(CGF);
7839 };
7840 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
7841 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
7842 S.getBeginLoc());
7843 };
7844 auto LPCRegion =
7846 emitCommonOMPParallelDirective(*this, S, OMPD_master_taskloop_simd, CodeGen,
7848}
7849
7850// Generate the instructions for '#pragma omp target update' directive.
7852 const OMPTargetUpdateDirective &S) {
7853 // If we don't have target devices, don't bother emitting the data mapping
7854 // code.
7855 if (CGM.getLangOpts().OMPTargetTriples.empty())
7856 return;
7857
7858 // Check if we have any if clause associated with the directive.
7859 const Expr *IfCond = nullptr;
7860 if (const auto *C = S.getSingleClause<OMPIfClause>())
7861 IfCond = C->getCondition();
7862
7863 // Check if we have any device clause associated with the directive.
7864 const Expr *Device = nullptr;
7865 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
7866 Device = C->getDevice();
7867
7868 OMPLexicalScope Scope(*this, S, OMPD_task);
7870}
7871
7873 const OMPGenericLoopDirective &S) {
7874 // Unimplemented, just inline the underlying statement for now.
7875 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7876 // Emit the loop iteration variable.
7877 const Stmt *CS =
7878 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt();
7879 const auto *ForS = dyn_cast<ForStmt>(CS);
7880 if (ForS && !isa<DeclStmt>(ForS->getInit())) {
7881 OMPPrivateScope LoopScope(CGF);
7882 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
7883 (void)LoopScope.Privatize();
7884 CGF.EmitStmt(CS);
7885 LoopScope.restoreMap();
7886 } else {
7887 CGF.EmitStmt(CS);
7888 }
7889 };
7890 OMPLexicalScope Scope(*this, S, OMPD_unknown);
7891 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_loop, CodeGen);
7892}
7893
7895 const OMPLoopDirective &S) {
7896 // Emit combined directive as if its consituent constructs are 'parallel'
7897 // and 'for'.
7898 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7899 Action.Enter(CGF);
7900 emitOMPCopyinClause(CGF, S);
7901 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
7902 };
7903 {
7904 auto LPCRegion =
7906 emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
7908 }
7909 // Check for outer lastprivate conditional update.
7911}
7912
7915 // To be consistent with current behavior of 'target teams loop', emit
7916 // 'teams loop' as if its constituent constructs are 'distribute,
7917 // 'parallel, and 'for'.
7918 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7920 S.getDistInc());
7921 };
7922
7923 // Emit teams region as a standalone region.
7924 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7925 PrePostActionTy &Action) {
7926 Action.Enter(CGF);
7927 OMPPrivateScope PrivateScope(CGF);
7928 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7929 (void)PrivateScope.Privatize();
7930 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
7931 CodeGenDistribute);
7932 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7933 };
7934 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
7936 [](CodeGenFunction &) { return nullptr; });
7937}
7938
7939static void
7942 PrePostActionTy &Action) {
7943 Action.Enter(CGF);
7944 // Emit 'teams loop' as if its constituent constructs are 'distribute,
7945 // 'parallel, and 'for'.
7946 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7948 S.getDistInc());
7949 };
7950
7951 // Emit teams region as a standalone region.
7952 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7953 PrePostActionTy &Action) {
7954 Action.Enter(CGF);
7955 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7956 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7957 (void)PrivateScope.Privatize();
7959 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
7960 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7961 };
7962
7963 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
7964 CodeGenTeams);
7966 [](CodeGenFunction &) { return nullptr; });
7967}
7968
7969/// Emit combined directive 'target teams loop' as if its constituent
7970/// constructs are 'target', 'teams', 'distribute', 'parallel', and 'for'.
7973 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7974 emitTargetTeamsGenericLoopRegion(CGF, S, Action);
7975 };
7976 emitCommonOMPTargetDirective(*this, S, CodeGen);
7977}
7978
7980 CodeGenModule &CGM, StringRef ParentName,
7982 // Emit SPMD target parallel loop region as a standalone region.
7983 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7984 emitTargetTeamsGenericLoopRegion(CGF, S, Action);
7985 };
7986 llvm::Function *Fn;
7987 llvm::Constant *Addr;
7988 // Emit target region as a standalone region.
7990 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
7991 assert(Fn && Addr &&
7992 "Target device function emission failed for 'target teams loop'.");
7993}
7994
7997 PrePostActionTy &Action) {
7998 Action.Enter(CGF);
7999 // Emit as 'parallel for'.
8000 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8001 Action.Enter(CGF);
8002 CodeGenFunction::OMPCancelStackRAII CancelRegion(
8003 CGF, OMPD_target_parallel_loop, /*hasCancel=*/false);
8004 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
8006 };
8007 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
8009}
8010
8012 CodeGenModule &CGM, StringRef ParentName,
8014 // Emit target parallel loop region as a standalone region.
8015 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8017 };
8018 llvm::Function *Fn;
8019 llvm::Constant *Addr;
8020 // Emit target region as a standalone region.
8022 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
8023 assert(Fn && Addr && "Target device function emission failed.");
8024}
8025
8026/// Emit combined directive 'target parallel loop' as if its constituent
8027/// constructs are 'target', 'parallel', and 'for'.
8030 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8032 };
8033 emitCommonOMPTargetDirective(*this, S, CodeGen);
8034}
8035
8037 const OMPExecutableDirective &D) {
8038 if (const auto *SD = dyn_cast<OMPScanDirective>(&D)) {
8040 return;
8041 }
8042 if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
8043 return;
8044 auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {
8045 OMPPrivateScope GlobalsScope(CGF);
8047 // Capture global firstprivates to avoid crash.
8048 for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
8049 for (const Expr *Ref : C->varlists()) {
8050 const auto *DRE = cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
8051 if (!DRE)
8052 continue;
8053 const auto *VD = dyn_cast<VarDecl>(DRE->getDecl());
8054 if (!VD || VD->hasLocalStorage())
8055 continue;
8056 if (!CGF.LocalDeclMap.count(VD)) {
8057 LValue GlobLVal = CGF.EmitLValue(Ref);
8058 GlobalsScope.addPrivate(VD, GlobLVal.getAddress(CGF));
8059 }
8060 }
8061 }
8062 }
8064 (void)GlobalsScope.Privatize();
8065 ParentLoopDirectiveForScanRegion ScanRegion(CGF, D);
8066 emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action);
8067 } else {
8068 if (const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
8069 for (const Expr *E : LD->counters()) {
8070 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
8071 if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) {
8072 LValue GlobLVal = CGF.EmitLValue(E);
8073 GlobalsScope.addPrivate(VD, GlobLVal.getAddress(CGF));
8074 }
8075 if (isa<OMPCapturedExprDecl>(VD)) {
8076 // Emit only those that were not explicitly referenced in clauses.
8077 if (!CGF.LocalDeclMap.count(VD))
8078 CGF.EmitVarDecl(*VD);
8079 }
8080 }
8081 for (const auto *C : D.getClausesOfKind<OMPOrderedClause>()) {
8082 if (!C->getNumForLoops())
8083 continue;
8084 for (unsigned I = LD->getLoopsNumber(),
8085 E = C->getLoopNumIterations().size();
8086 I < E; ++I) {
8087 if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
8088 cast<DeclRefExpr>(C->getLoopCounter(I))->getDecl())) {
8089 // Emit only those that were not explicitly referenced in clauses.
8090 if (!CGF.LocalDeclMap.count(VD))
8091 CGF.EmitVarDecl(*VD);
8092 }
8093 }
8094 }
8095 }
8096 (void)GlobalsScope.Privatize();
8098 }
8099 };
8100 if (D.getDirectiveKind() == OMPD_atomic ||
8101 D.getDirectiveKind() == OMPD_critical ||
8102 D.getDirectiveKind() == OMPD_section ||
8103 D.getDirectiveKind() == OMPD_master ||
8104 D.getDirectiveKind() == OMPD_masked ||
8105 D.getDirectiveKind() == OMPD_unroll) {
8107 } else {
8108 auto LPCRegion =
8110 OMPSimdLexicalScope Scope(*this, D);
8112 *this,
8114 : D.getDirectiveKind(),
8115 CodeGen);
8116 }
8117 // Check for outer lastprivate conditional update.
8119}
Defines the clang::ASTContext interface.
#define V(N, I)
Definition: ASTContext.h:3259
static bool isAllocatableDecl(const VarDecl *VD)
static const VarDecl * getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE)
static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, PrePostActionTy &Action)
static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, PrePostActionTy &Action)
static llvm::Function * emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S, SourceLocation Loc)
static const VarDecl * getBaseDecl(const Expr *Ref)
static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *V, SourceLocation Loc)
static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, bool IsPostfixUpdate, const Expr *V, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
static void emitScanBasedDirective(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen, llvm::function_ref< void(CodeGenFunction &)> FirstGen, llvm::function_ref< void(CodeGenFunction &)> SecondGen)
Emits the code for the directive with inscan reductions.
static bool isSupportedByOpenMPIRBuilder(const OMPSimdDirective &S)
static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO, LValue LVal, RValue RVal)
static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, QualType DstType, StringRef Name, LValue AddrLV)
static void emitDistributeParallelForDistributeInnerBoundParams(CodeGenFunction &CGF, const OMPExecutableDirective &S, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars)
static void emitScanBasedDirectiveFinals(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen)
Copies final inscan reductions values to the original variables.
static void checkForLastprivateConditionalUpdate(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static std::pair< LValue, LValue > emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
The following two functions generate expressions for the loop lower and upper bounds in case of stati...
static void emitTargetParallelForRegion(CodeGenFunction &CGF, const OMPTargetParallelForDirective &S, PrePostActionTy &Action)
static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper)
Emit a helper variable and return corresponding lvalue.
static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
static llvm::Value * convertToScalarValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
static std::pair< bool, RValue > emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, RValue Update, BinaryOperatorKind BO, llvm::AtomicOrdering AO, bool IsXLHSInRHSPart)
static std::pair< LValue, LValue > emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitTargetParallelRegion(CodeGenFunction &CGF, const OMPTargetParallelDirective &S, PrePostActionTy &Action)
static std::pair< llvm::Value *, llvm::Value * > emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
When dealing with dispatch schedules (e.g.
static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitRestoreIP(CodeGenFunction &CGF, const T *C, llvm::OpenMPIRBuilder::InsertPointTy AllocaIP, llvm::OpenMPIRBuilder &OMPBuilder)
static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, const RegionCodeGenTy &CodeGen)
static void emitSimdlenSafelenClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
static void emitAlignedClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters)
static bool emitWorksharingDirective(CodeGenFunction &CGF, const OMPLoopDirective &S, bool HasCancel)
static void emitTargetTeamsGenericLoopRegion(CodeGenFunction &CGF, const OMPTargetTeamsGenericLoopDirective &S, PrePostActionTy &Action)
static void emitPostUpdateForReductionClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc, const unsigned IVSize, const bool IVSigned)
static RValue emitSimpleAtomicLoad(CodeGenFunction &CGF, llvm::AtomicOrdering AO, LValue LVal, SourceLocation Loc)
static std::pair< llvm::Value *, llvm::Value * > emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
if the 'for' loop has a dispatch schedule (e.g.
static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *UE, const Expr *D, const Expr *CE, bool IsXLHSInRHSPart, bool IsFailOnly, SourceLocation Loc)
static CodeGenFunction::ComplexPairTy convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static ImplicitParamDecl * createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data, QualType Ty, CapturedDecl *CD, SourceLocation Loc)
static EmittedClosureTy emitCapturedStmtFunc(CodeGenFunction &ParentCGF, const CapturedStmt *S)
Emit a captured statement and return the function as well as its captured closure context.
static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
static llvm::MapVector< llvm::Value *, llvm::Value * > GetAlignedMapping(const OMPSimdDirective &S, CodeGenFunction &CGF)
static void emitOMPCopyinClause(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitTargetTeamsDistributeParallelForRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, PrePostActionTy &Action)
static llvm::CallInst * emitCapturedStmtCall(CodeGenFunction &ParentCGF, EmittedClosureTy Cap, llvm::ArrayRef< llvm::Value * > Args)
Emit a call to a previously captured closure.
static void emitMasked(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop, int MaxLevel, int Level=0)
static void emitEmptyBoundParameters(CodeGenFunction &, const OMPExecutableDirective &, llvm::SmallVectorImpl< llvm::Value * > &)
static void emitTargetParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetParallelForSimdDirective &S, PrePostActionTy &Action)
static void emitOMPAtomicCompareExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO, const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *D, const Expr *CE, bool IsXBinopExpr, bool IsPostfixUpdate, bool IsFailOnly, SourceLocation Loc)
std::pair< llvm::Function *, llvm::Value * > EmittedClosureTy
static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDirective &S)
static void buildDependences(const OMPExecutableDirective &S, OMPTaskDataTy &Data)
static RValue convertToType(CodeGenFunction &CGF, RValue Value, QualType SourceType, QualType ResType, SourceLocation Loc)
static void emitScanBasedDirectiveDecls(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen)
Emits internal temp array declarations for the directive with inscan reductions.
static void emitTargetTeamsDistributeParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForSimdDirective &S, PrePostActionTy &Action)
static void emitTargetTeamsDistributeSimdRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeSimdDirective &S)
static llvm::omp::ScheduleKind convertClauseKindToSchedKind(OpenMPScheduleClauseKind ScheduleClauseKind)
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
static void emitTargetParallelGenericLoopRegion(CodeGenFunction &CGF, const OMPTargetParallelGenericLoopDirective &S, PrePostActionTy &Action)
static QualType getCanonicalParamType(ASTContext &C, QualType T)
static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S, const RegionCodeGenTy &SimdInitGen, const RegionCodeGenTy &BodyCodeGen)
static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, const Twine &Name, llvm::Value *Init=nullptr)
static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *E, SourceLocation Loc)
static llvm::Function * emitOutlinedFunctionPrologue(CodeGenFunction &CGF, FunctionArgList &Args, llvm::MapVector< const Decl *, std::pair< const VarDecl *, Address > > &LocalAddrs, llvm::DenseMap< const Decl *, std::pair< const Expr *, llvm::Value * > > &VLASizes, llvm::Value *&CXXThisValue, const FunctionOptions &FO)
static void emitInnerParallelForWhenCombined(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
static void emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeDirective &S)
This file defines OpenMP nodes for declarative directives.
static const Decl * getCanonicalDecl(const Decl *D)
#define X(type, name)
Definition: Value.h:142
This file defines OpenMP AST classes for clauses.
Defines some OpenMP-specific enums and functions.
Defines the PrettyStackTraceEntry class, which is used to make crashes give more contextual informati...
This file defines OpenMP AST classes for executable directives and clauses.
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:182
SourceManager & getSourceManager()
Definition: ASTContext.h:700
TranslationUnitDecl * getTranslationUnitDecl() const
Definition: ASTContext.h:1068
QualType getRecordType(const RecordDecl *Decl) const
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
CanQualType VoidPtrTy
Definition: ASTContext.h:1113
IdentifierTable & Idents
Definition: ASTContext.h:639
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
QualType getUIntPtrType() const
Return a type compatible with "uintptr_t" (C99 7.18.1.4), as defined by the target.
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth,...
TypeSourceInfo * getTrivialTypeSourceInfo(QualType T, SourceLocation Loc=SourceLocation()) const
Allocate a TypeSourceInfo where all locations have been initialized to a given location,...
unsigned getOpenMPDefaultSimdAlign(QualType T) const
Get default simd alignment of the specified complete type in bits.
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
Definition: ASTContext.h:2315
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
CanQualType VoidTy
Definition: ASTContext.h:1086
QualType getFunctionType(QualType ResultTy, ArrayRef< QualType > Args, const FunctionProtoType::ExtProtoInfo &EPI) const
Return a normal function type with a typed argument list.
Definition: ASTContext.h:1553
const TargetInfo & getTargetInfo() const
Definition: ASTContext.h:752
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
ASTRecordLayout - This class contains layout information for one RecordDecl, which is a struct/union/...
Definition: RecordLayout.h:38
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
Definition: RecordLayout.h:200
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Definition: Type.h:3147
Represents an attribute applied to a statement.
Definition: Stmt.h:2078
ArrayRef< const Attr * > getAttrs() const
Definition: Stmt.h:2112
A builtin binary operation expression such as "x + y" or "x <= y".
Definition: Expr.h:3834
static BinaryOperator * Create(const ASTContext &C, Expr *lhs, Expr *rhs, Opcode opc, QualType ResTy, ExprValueKind VK, ExprObjectKind OK, SourceLocation opLoc, FPOptionsOverride FPFeatures)
Definition: Expr.cpp:4767
BlockExpr - Adaptor class for mixing a BlockDecl with expressions.
Definition: Expr.h:6167
Represents the body of a CapturedStmt, and serves as its DeclContext.
Definition: Decl.h:4651
unsigned getContextParamPosition() const
Definition: Decl.h:4722
bool isNothrow() const
Definition: Decl.cpp:5402
static CapturedDecl * Create(ASTContext &C, DeclContext *DC, unsigned NumParams)
Definition: Decl.cpp:5387
param_iterator param_end() const
Retrieve an iterator one past the last parameter decl.
Definition: Decl.h:4730
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
Definition: Decl.h:4728
Stmt * getBody() const override
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
Definition: Decl.cpp:5399
ImplicitParamDecl * getParam(unsigned i) const
Definition: Decl.h:4695
This captures a statement into a function.
Definition: Stmt.h:3755
CapturedDecl * getCapturedDecl()
Retrieve the outlined function declaration.
Definition: Stmt.cpp:1407
child_range children()
Definition: Stmt.cpp:1398
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Definition: Stmt.h:3876
Stmt * getCapturedStmt()
Retrieve the statement being captured.
Definition: Stmt.h:3859
capture_range captures()
Definition: Stmt.h:3893
Expr *const * const_capture_init_iterator
Const iterator that walks over the capture initialization arguments.
Definition: Stmt.h:3919
CharUnits - This is an opaque type for sizes expressed in character units.
Definition: CharUnits.h:38
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Definition: CharUnits.h:185
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
Definition: CharUnits.h:214
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
Definition: CharUnits.h:63
CharUnits alignTo(const CharUnits &Align) const
alignTo - Returns the next integer (mod 2**64) that is greater than or equal to this quantity and is ...
Definition: CharUnits.h:201
bool hasReducedDebugInfo() const
Check if type and variable info should be emitted.
An aligned address.
Definition: Address.h:29
static Address invalid()
Definition: Address.h:46
CharUnits getAlignment() const
Return the alignment of this pointer.
Definition: Address.h:78
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Definition: Address.h:62
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Definition: Address.h:100
Address withAlignment(CharUnits NewAlignment) const
Return address with different alignment, but same pointer and element type.
Definition: Address.h:93
llvm::Value * getPointer() const
Definition: Address.h:51
llvm::PointerType * getType() const
Return the type of the pointer value.
Definition: Address.h:57
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored.
Definition: CGValue.h:580
static ApplyDebugLocation CreateDefaultArtificial(CodeGenFunction &CGF, SourceLocation TemporaryLocation)
Apply TemporaryLocation if it is valid.
Definition: CGDebugInfo.h:870
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Definition: CGBuilder.h:156
llvm::AtomicRMWInst * CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Ordering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
Definition: CGBuilder.h:140
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Definition: CGBuilder.h:71
CGFunctionInfo - Class to encapsulate the information about a function definition.
Manages list of lastprivate conditional decls for the specified directive.
static LastprivateConditionalRAII disable(CodeGenFunction &CGF, const OMPExecutableDirective &S)
Manages list of nontemporal decls for the specified directive.
Struct that keeps all the relevant information that should be kept throughout a 'target data' region.
Manages list of nontemporal decls for the specified directive.
virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
Emit task region for the task directive.
virtual llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST)
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device)
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
virtual void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc)
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)...
virtual void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps)
Emits a single region.
virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal)
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF, const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const
Choose default schedule type and chunk value for the schedule clause.
virtual void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars)
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
virtual void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancellation point' construct.
virtual const VarDecl * translateParameter(const FieldDecl *FD, const VarDecl *NativeParam) const
Translates the native parameter of outlined function if this is required for target.
virtual llvm::Function * emitTeamsOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP teams directive D.
virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr * > NumIterations)
Emit initialization for doacross loop nesting support.
virtual void adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF, const OMPExecutableDirective &D) const
Adjust some parameters for the target-based directives, like addresses of the variables captured by r...
virtual void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, CGOpenMPRuntime::TargetDataInfo &Info)
Emit the target data mapping code associated with D.
virtual Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const
Gets the address of the native argument basing on the address of the target-specific parameter.
virtual void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc)
Emit a taskgroup region.
virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams,...
virtual llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data)
Emit a code for initialization of task reduction clause.
void emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, OpenMPDependClauseKind NewDepKind, SourceLocation Loc)
Updates the dependency kind in the specified depobj object.
virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD, SourceLocation Loc)
Gets the address of the global copy used for lastprivate conditional update, if any.
virtual void emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, Expr *ME, bool IsFatal)
Emit __kmpc_error call for error directive extern void __kmpc_error(ident_t *loc, int severity,...
virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc)
Emits code for a taskyield directive.
virtual void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc, llvm::AtomicOrdering AO)
Emit flush of the variables specified in 'omp flush' directive.
virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPTaskDataTy &Data)
Emit code for 'taskwait' directive.
virtual void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc)
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, int proc_bind) to generat...
virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false)
Emit an implicit/explicit barrier for OpenMP threads.
virtual void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values)
virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind)
Call the appropriate runtime routine to notify that we finished all the work with current loop.
void emitIfClause(CodeGenFunction &CGF, const Expr *Cond, const RegionCodeGenTy &ThenGen, const RegionCodeGenTy &ElseGen)
Emits code for OpenMP 'if' clause using specified CodeGen function.
Address emitDepobjDependClause(CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies, SourceLocation Loc)
Emits list of dependecies based on the provided data (array of dependence/expression pairs) for depob...
virtual llvm::Function * emitParallelOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP parallel directive D.
virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values)
Call the appropriate runtime routine to initialize it before start of loop.
llvm::AtomicOrdering getDefaultMemoryOrdering() const
Gets default memory ordering as specified in requires directive.
virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static non-chunked.
virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancel' construct.
virtual void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc)
Emits a master region.
virtual llvm::Function * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts)
Emits outlined function for the OpenMP task directive D.
void emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal, SourceLocation Loc)
Emits the code to destroy the dependency object provided in depobj directive.
virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Required to resolve existing problems in the runtime.
virtual void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C)
Emit code for doacross ordered directive with 'depend' clause.
virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF, const Expr *LHS)
Checks if the provided LVal is lastprivate conditional and emits the code to update the value of the ...
virtual void getDefaultDistScheduleAndChunk(CodeGenFunction &CGF, const OMPLoopDirective &S, OpenMPDistScheduleClauseKind &ScheduleKind, llvm::Value *&Chunk) const
Choose default schedule type and chunk value for the dist_schedule clause.
llvm::OpenMPIRBuilder & getOMPBuilder()
virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Emit outilined function for 'target' directive.
virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr)
Emits a critical region.
virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned)
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
virtual void checkAndEmitSharedLastprivateConditional(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::DenseSet< CanonicalDeclPtr< const VarDecl > > &IgnoredDecls)
Checks if the lastprivate conditional was updated in inner region and writes the value.
virtual void emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool HasCancel=false)
Emit code for the directive that does not require outlining.
virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond, llvm::Value *NumThreads)
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static chunked.
virtual void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Emit the target offloading code associated with D.
virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps, ReductionOptionsTy Options)
Emit a code for reduction clause.
std::pair< llvm::Value *, Address > emitDependClause(CodeGenFunction &CGF, ArrayRef< OMPTaskDataTy::DependData > Dependencies, SourceLocation Loc)
Emits list of dependecies based on the provided data (array of dependence/expression pairs).
virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const
Check if the specified ScheduleKind is dynamic.
Address emitLastprivateConditionalInit(CodeGenFunction &CGF, const VarDecl *VD)
Create specialized alloca to handle lastprivate conditionals.
virtual void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads)
Emit an ordered region.
virtual void emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, ArrayRef< llvm::Value * > Args=std::nullopt) const
Emits call of the outlined function with the provided arguments, translating these arguments to corre...
virtual void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, bool IsWorksharingReduction)
Emits the following code for reduction clause with task modifier:
virtual void emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter=nullptr)
Emits a masked region.
virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues)
Call the appropriate runtime routine to initialize it before start of loop.
virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S)
Emit the captured statement body.
virtual StringRef getHelperName() const
Get the name of the capture helper.
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
RAII for preserving necessary info during inlined region body codegen.
RAII for preserving necessary info during Outlined region body codegen.
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion, const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen, OMPTaskDataTy &Data)
void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S)
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
void EmitOMPParallelGenericLoopDirective(const OMPLoopDirective &S)
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount, Stmt::Likelihood LH=Stmt::LH_None, const Expr *ConditionalOp=nullptr)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target)
The given basic block lies in the current EH scope, but may be a target of a potentially scope-crossi...
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
void EmitOMPOrderedDirective(const OMPOrderedDirective &S)
void EmitOMPTargetDirective(const OMPTargetDirective &S)
bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
void EmitOMPReductionClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope, bool ForInscan=false)
Emit initial code for reduction variables.
void EmitOMPAtomicDirective(const OMPAtomicDirective &S)
void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S)
void EmitOMPParallelMasterTaskLoopDirective(const OMPParallelMasterTaskLoopDirective &S)
void EmitAutoVarDecl(const VarDecl &D)
EmitAutoVarDecl - Emit an auto variable declaration.
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
static bool hasScalarEvaluationKind(QualType T)
llvm::function_ref< std::pair< llvm::Value *, llvm::Value * >(CodeGenFunction &, const OMPExecutableDirective &S, Address LB, Address UB)> CodeGenDispatchBoundsTy
CGCapturedStmtInfo * CapturedStmtInfo
void EmitDecl(const Decl &D)
EmitDecl - Emit a declaration.
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Load a pointer with type PtrTy stored at address Ptr.
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
llvm::Value * emitArrayLength(const ArrayType *arrayType, QualType &baseType, Address &addr)
emitArrayLength - Compute the length of an array, even if it's a VLA, and drill down to the base elem...
VlaSizePair getVLASize(const VariableArrayType *vla)
Returns an LLVM value that corresponds to the size, in non-variably-sized elements,...
const OMPExecutableDirective * OMPParentLoopDirectiveForScan
Parent loop-based directive for scan directive.
void EmitOMPTeamsDistributeParallelForDirective(const OMPTeamsDistributeParallelForDirective &S)
void EmitOMPTaskDirective(const OMPTaskDirective &S)
void EmitOMPScanDirective(const OMPScanDirective &S)
void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S)
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
void EmitOMPMasterTaskLoopSimdDirective(const OMPMasterTaskLoopSimdDirective &S)
void EmitOMPUseDevicePtrClause(const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap)
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
void EmitOMPDistributeLoop(const OMPLoopDirective &S, const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr)
Emit code for the distribute loop-based directive.
void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S)
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO, const llvm::function_ref< RValue(RValue)> &UpdateOp, bool IsVolatile)
const LangOptions & getLangOpts() const
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S)
Emit device code for the target simd directive.
void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S)
void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S, OMPPrivateScope &LoopScope)
Emit initial code for loop counters of loop-based directives.
void EmitOMPDistributeParallelForDirective(const OMPDistributeParallelForDirective &S)
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D, bool NoFinals, llvm::Value *IsLastIterCond=nullptr)
Emit final copying of lastprivate values to original variables at the end of the worksharing or simd ...
void EmitOMPTeamsGenericLoopDirective(const OMPTeamsGenericLoopDirective &S)
void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S)
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit)
EmitExprAsInit - Emits the code necessary to initialize a location in memory with the given initializ...
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
void EmitOMPParallelDirective(const OMPParallelDirective &S)
void EmitOMPSimdFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
llvm::Type * ConvertTypeForMem(QualType T)
void EmitOMPDistributeParallelForSimdDirective(const OMPDistributeParallelForSimdDirective &S)
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
void EmitOMPForSimdDirective(const OMPForSimdDirective &S)
llvm::AssertingVH< llvm::Instruction > AllocaInsertPt
AllocaInsertPoint - This is an instruction in the entry block before which we prefer to insert alloca...
void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy)
Emit an aggregate assignment.
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value * > &CapturedVars)
JumpDest ReturnBlock
ReturnBlock - Unified return block.
const llvm::function_ref< void(CodeGenFunction &, llvm::Function *, const OMPTaskDataTy &)> TaskGenTy
static void EmitOMPTargetTeamsGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsGenericLoopDirective &S)
Emit device code for the target teams loop directive.
void EmitOMPFlushDirective(const OMPFlushDirective &S)
llvm::BasicBlock * OMPBeforeScanBlock
void EmitOMPCancelDirective(const OMPCancelDirective &S)
void EmitOMPGenericLoopDirective(const OMPGenericLoopDirective &S)
void EmitOMPTargetTeamsDistributeDirective(const OMPTargetTeamsDistributeDirective &S)
ConstantEmission tryEmitAsConstant(DeclRefExpr *refExpr)
void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S)
static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S)
void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy, SourceLocation Loc)
void EmitOMPInnerLoop(const OMPExecutableDirective &S, bool RequiresCleanup, const Expr *LoopCond, const Expr *IncExpr, const llvm::function_ref< void(CodeGenFunction &)> BodyGen, const llvm::function_ref< void(CodeGenFunction &)> PostIncGen)
Emit inner loop of the worksharing/simd construct.
llvm::DebugLoc SourceLocToDebugLoc(SourceLocation Location)
Converts Location to a DebugLoc, if debug information is enabled.
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
void EmitOMPDepobjDirective(const OMPDepobjDirective &S)
void EmitOMPSingleDirective(const OMPSingleDirective &S)
void EmitOMPTargetTeamsGenericLoopDirective(const OMPTargetTeamsGenericLoopDirective &S)
llvm::CanonicalLoopInfo * EmitOMPCollapsedCanonicalLoopNest(const Stmt *S, int Depth)
Emit the Stmt S and return its topmost canonical loop, if any.
void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D)
Emit simple code for OpenMP directives in Simd-only mode.
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
EmitAnyExpr - Emit code to compute the specified expression which can have any type.
void EmitOMPParallelForDirective(const OMPParallelForDirective &S)
void EmitOMPTeamsDirective(const OMPTeamsDirective &S)
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
void EmitOMPUnrollDirective(const OMPUnrollDirective &S)
void EmitOMPParallelMasterTaskLoopSimdDirective(const OMPParallelMasterTaskLoopSimdDirective &S)
void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S)
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
llvm::Value * EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified complex type to the specified destination type,...
bool isTrivialInitializer(const Expr *Init)
Determine whether the given initializer is trivial in the sense that it requires no code to be genera...
void EmitOMPTargetTeamsDistributeParallelForSimdDirective(const OMPTargetTeamsDistributeParallelForSimdDirective &S)
void EmitOMPTeamsDistributeParallelForSimdDirective(const OMPTeamsDistributeParallelForSimdDirective &S)
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block,...
llvm::Function * GenerateCapturedStmtFunction(const CapturedStmt &S)
void EmitOMPInteropDirective(const OMPInteropDirective &S)
void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind)
Emit final update of reduction values to original variables at the end of the directive.
llvm::BasicBlock * OMPScanDispatch
llvm::function_ref< std::pair< LValue, LValue >(CodeGenFunction &, const OMPExecutableDirective &S)> CodeGenLoopBoundsTy
void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty, SourceLocation Loc, SourceLocation AssumptionLoc, llvm::Value *Alignment, llvm::Value *OffsetValue=nullptr)
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S)
void EmitOMPParallelMaskedDirective(const OMPParallelMaskedDirective &S)
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
void EmitOMPSimdInit(const OMPLoopDirective &D)
Helpers for the OpenMP loop directives.
int ExpectedOMPLoopDepth
Number of nested loop to be consumed by the last surrounding loop-associated directive.
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S)
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
SmallVector< llvm::CanonicalLoopInfo *, 4 > OMPLoopNestStack
List of recently emitted OMPCanonicalLoops.
bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB, const CodeGenLoopBoundsTy &CodeGenLoopBounds, const CodeGenDispatchBoundsTy &CGDispatchBounds)
Emit code for the worksharing loop-based directive.
void EmitOMPLinearClause(const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope)
Emit initial code for linear clauses.
void EmitBranchThroughCleanup(JumpDest Dest)
EmitBranchThroughCleanup - Emit a branch from the current insert block through the normal cleanup han...
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant,...
void EmitAutoVarCleanups(const AutoVarEmission &emission)
void EmitOMPTileDirective(const OMPTileDirective &S)
bool EmitOMPLinearClauseInit(const OMPLoopDirective &D)
Emit initial code for linear variables.
llvm::DenseMap< const ValueDecl *, FieldDecl * > LambdaCaptureFields
void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit)
Helper for the OpenMP loop directives.
void EmitOMPLinearClauseFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
Emit final code for linear clauses.
void EmitOMPSectionsDirective(const OMPSectionsDirective &S)
llvm::BasicBlock * OMPScanExitBlock
static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Emit device code for the target teams distribute parallel for simd directive.
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
void EmitOMPUseDeviceAddrClause(const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap)
void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S)
void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S)
void EmitOMPSimdDirective(const OMPSimdDirective &S)
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
void EmitOMPCriticalDirective(const OMPCriticalDirective &S)
void EmitOMPForDirective(const OMPForDirective &S)
void EmitOMPMetaDirective(const OMPMetaDirective &S)
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S)
LValue InitCapturedStruct(const CapturedStmt &S)
void EmitOMPParallelMasterDirective(const OMPParallelMasterDirective &S)
void EmitOMPTargetTeamsDistributeSimdDirective(const OMPTargetTeamsDistributeSimdDirective &S)
void processInReduction(const OMPExecutableDirective &S, OMPTaskDataTy &Data, CodeGenFunction &CGF, const CapturedStmt *CS, OMPPrivateScope &Scope)
void EmitOMPMasterDirective(const OMPMasterDirective &S)
void EmitOMPMasterTaskLoopDirective(const OMPMasterTaskLoopDirective &S)
void EmitOMPTargetParallelGenericLoopDirective(const OMPTargetParallelGenericLoopDirective &S)
bool EmitOMPCopyinClause(const OMPExecutableDirective &D)
Emit code for copyin clause in D directive.
void EmitOMPMaskedDirective(const OMPMaskedDirective &S)
uint64_t getProfileCount(const Stmt *S)
Get the profiler's count for the given statement.
void EmitOMPPrivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S, SourceLocation Loc)
void EmitOMPTargetParallelForSimdDirective(const OMPTargetParallelForSimdDirective &S)
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
Address GenerateCapturedStmtArgument(const CapturedStmt &S)
static void EmitOMPTargetParallelGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelGenericLoopDirective &S)
Emit device code for the target parallel loop directive.
llvm::function_ref< void(CodeGenFunction &, SourceLocation, const unsigned, const bool)> CodeGenOrderedTy
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
llvm::BasicBlock * OMPAfterScanBlock
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S)
void EmitOMPErrorDirective(const OMPErrorDirective &S)
void EmitOMPSectionDirective(const OMPSectionDirective &S)
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
void EmitOMPBarrierDirective(const OMPBarrierDirective &S)
void EmitStopPoint(const Stmt *S)
EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
void EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S)
void EmitStmt(const Stmt *S, ArrayRef< const Attr * > Attrs=std::nullopt)
EmitStmt - Emit the code for the statement.
void EnsureInsertPoint()
EnsureInsertPoint - Ensure that an insertion point is defined so that emitted IR has a place to go.
llvm::LLVMContext & getLLVMContext()
void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
void EmitOMPTargetTeamsDistributeParallelForDirective(const OMPTargetTeamsDistributeParallelForDirective &S)
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler's counter for the given statement by StepV.
bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for lastprivate variables.
void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S)
llvm::function_ref< void(CodeGenFunction &, const OMPLoopDirective &, JumpDest)> CodeGenLoopTy
void EmitOMPCanonicalLoop(const OMPCanonicalLoop *S)
Emit an OMPCanonicalLoop using the OpenMPIRBuilder.
void EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S)
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
void EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S)
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
This class organizes the cross-function state that is used while generating LLVM code.
void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
llvm::Module & getModule() const
CharUnits GetTargetTypeStoreSize(llvm::Type *Ty) const
Return the store size, in character units, of the given LLVM type.
DiagnosticsEngine & getDiags() const
const LangOptions & getLangOpts() const
CharUnits getNaturalTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, bool forPointeeType=false)
const llvm::DataLayout & getDataLayout() const
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
llvm::Constant * GetAddrOfGlobal(GlobalDecl GD, ForDefinition_t IsForDefinition=NotForDefinition)
ASTContext & getContext() const
const CodeGenOptions & getCodeGenOpts() const
StringRef getMangledName(GlobalDecl GD)
llvm::Constant * getStaticLocalDeclAddress(const VarDecl *D)
llvm::ConstantInt * getSize(CharUnits numChars)
Emit the given number of characters as a value of type size_t.
void assignRegionCounters(GlobalDecl GD, llvm::Function *Fn)
Assign counters to regions and configure them for PGO of a given function.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
Definition: CGCall.cpp:1625
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
Definition: CGCall.cpp:674
llvm::Type * ConvertTypeForMem(QualType T, bool ForBitField=false)
ConvertTypeForMem - Convert type T into a llvm::Type.
FunctionArgList - Type for representing both the decl and type of parameters to a function.
Definition: CGCall.h:352
LValue - This represents an lvalue references.
Definition: CGValue.h:171
Address getAddress(CodeGenFunction &CGF) const
Definition: CGValue.h:350
bool isVolatile() const
Definition: CGValue.h:316
bool isGlobalReg() const
Definition: CGValue.h:270
llvm::Value * getPointer(CodeGenFunction &CGF) const
Definition: CGValue.h:346
QualType getType() const
Definition: CGValue.h:279
void setAddress(Address address)
Definition: CGValue.h:354
void setUnrollCount(unsigned C)
Set the unroll count for the next loop pushed.
Definition: CGLoopInfo.h:275
void setVectorizeWidth(unsigned W)
Set the vectorize width for the next loop pushed.
Definition: CGLoopInfo.h:265
void setParallel(bool Enable=true)
Set the next pushed loop as parallel.
Definition: CGLoopInfo.h:235
void setUnrollState(const LoopAttributes::LVEnableState &State)
Set the next pushed loop unroll state.
Definition: CGLoopInfo.h:250
void pop()
End the current loop.
Definition: CGLoopInfo.cpp:823
void push(llvm::BasicBlock *Header, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
Begin a new structured loop.
void setVectorizeEnable(bool Enable=true)
Set the next pushed loop 'vectorize.enable'.
Definition: CGLoopInfo.h:238
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
virtual void Enter(CodeGenFunction &CGF)
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
Definition: CGValue.h:39
bool isScalar() const
Definition: CGValue.h:54
static RValue get(llvm::Value *V)
Definition: CGValue.h:89
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
Definition: CGValue.h:96
bool isAggregate() const
Definition: CGValue.h:56
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
Definition: CGValue.h:61
bool isComplex() const
Definition: CGValue.h:55
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
Definition: CGValue.h:68
Class intended to support codegen of all kind of the reduction clauses.
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N)
Emits lvalue for the shared and original reduction item.
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
Complex values, per C99 6.2.5p11.
Definition: Type.h:2845
CompoundStmt - This represents a group of statements like { stmt stmt }.
Definition: Stmt.h:1604
ConstStmtVisitor - This class implements a simple visitor for Stmt subclasses.
Definition: StmtVisitor.h:195
DeclContext * getParent()
getParent - Returns the containing DeclContext.
Definition: DeclBase.h:2076
A reference to a declared variable, function, enum, etc.
Definition: Expr.h:1260
static DeclRefExpr * Create(const ASTContext &Context, NestedNameSpecifierLoc QualifierLoc, SourceLocation TemplateKWLoc, ValueDecl *D, bool RefersToEnclosingVariableOrCapture, SourceLocation NameLoc, QualType T, ExprValueKind VK, NamedDecl *FoundD=nullptr, const TemplateArgumentListInfo *TemplateArgs=nullptr, NonOdrUseReason NOUR=NOUR_None)
Definition: Expr.cpp:488
ValueDecl * getDecl()
Definition: Expr.h:1328
DeclStmt - Adaptor class for mixing declarations with statements and expressions.
Definition: Stmt.h:1495
decl_range decls()
Definition: Stmt.h:1543
Decl - This represents one declaration (or definition), e.g.
Definition: DeclBase.h:85
T * getAttr() const
Definition: DeclBase.h:578
SourceLocation getBodyRBrace() const
getBodyRBrace - Gets the right brace of the body, if a body exists.
Definition: DeclBase.cpp:1029
virtual bool hasBody() const
Returns true if this Decl represents a declaration for a body of code, such as a function or method d...
Definition: DeclBase.h:1093
SourceLocation getLocation() const
Definition: DeclBase.h:444
bool hasAttr() const
Definition: DeclBase.h:582
The name of a declaration.
SourceLocation getBeginLoc() const LLVM_READONLY
Definition: Decl.h:822
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Definition: Diagnostic.h:1547
unsigned getCustomDiagID(Level L, const char(&FormatString)[N])
Return an ID for a diagnostic with the specified format string and level.
Definition: Diagnostic.h:873
This represents one expression.
Definition: Expr.h:110
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer,...
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
Definition: Expr.cpp:3045
Expr * IgnoreImplicitAsWritten() LLVM_READONLY
Skip past any implicit AST nodes which might surround this expression until reaching a fixed point.
Definition: Expr.cpp:3037
Expr * IgnoreImpCasts() LLVM_READONLY
Skip past any implicit casts which might surround this expression until reaching a fixed point.
Definition: Expr.cpp:3025
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Definition: Expr.cpp:277
QualType getType() const
Definition: Expr.h:142
Represents difference between two FPOptions values.
Definition: LangOptions.h:870
Represents a member of a struct/union/class.
Definition: Decl.h:3025
Represents a function declaration or definition.
Definition: Decl.h:1959
static FunctionDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation NLoc, DeclarationName N, QualType T, TypeSourceInfo *TInfo, StorageClass SC, bool UsesFPIntrin=false, bool isInlineSpecified=false, bool hasWrittenPrototype=true, ConstexprSpecKind ConstexprKind=ConstexprSpecKind::Unspecified, Expr *TrailingRequiresClause=nullptr)
Definition: Decl.h:2135
GlobalDecl - represents a global declaration.
Definition: GlobalDecl.h:56
One of these records is kept for each identifier that is lexed.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
static ImplicitCastExpr * Create(const ASTContext &Context, QualType T, CastKind Kind, Expr *Operand, const CXXCastPath *BasePath, ExprValueKind Cat, FPOptionsOverride FPO)
Definition: Expr.cpp:2060
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
Definition: Decl.cpp:5338
A C++ lambda expression, which produces a function object (of unspecified type) that can be invoked l...
Definition: ExprCXX.h:1938
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
Definition: LangOptions.h:492
Represents a point when we exit a loop.
Definition: ProgramPoint.h:711
IdentifierInfo * getIdentifier() const
Get the identifier that names this declaration, if there is one.
Definition: Decl.h:270
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
Definition: Decl.h:276
A C++ nested-name-specifier augmented with source location information.
This represents 'acq_rel' clause in the '#pragma omp atomic|flush' directives.
This represents 'acquire' clause in the '#pragma omp atomic|flush' directives.
This represents clause 'aligned' in the '#pragma omp ...' directives.
This represents '#pragma omp atomic' directive.
Definition: StmtOpenMP.h:2963
This represents '#pragma omp barrier' directive.
Definition: StmtOpenMP.h:2641
This represents '#pragma omp cancel' directive.
Definition: StmtOpenMP.h:3671
This represents '#pragma omp cancellation point' directive.
Definition: StmtOpenMP.h:3613
Representation of an OpenMP canonical loop.
Definition: StmtOpenMP.h:142
static OMPClauseWithPreInit * get(OMPClause *C)
This is a basic class for representing single OpenMP clause.
Definition: OpenMPClause.h:55
This represents clause 'copyin' in the '#pragma omp ...' directives.
This represents clause 'copyprivate' in the '#pragma omp ...' directives.
This represents '#pragma omp critical' directive.
Definition: StmtOpenMP.h:2092
This represents implicit clause 'depend' for the '#pragma omp task' directive.
This represents implicit clause 'depobj' for the '#pragma omp depobj' directive.
This represents '#pragma omp depobj' directive.
Definition: StmtOpenMP.h:2857
This represents 'destroy' clause in the '#pragma omp depobj' directive or the '#pragma omp interop' d...
This represents 'device' clause in the '#pragma omp ...' directive.
This represents 'dist_schedule' clause in the '#pragma omp ...' directive.
This represents '#pragma omp distribute' directive.
Definition: StmtOpenMP.h:4441
This represents '#pragma omp distribute parallel for' composite directive.
Definition: StmtOpenMP.h:4564
This represents '#pragma omp distribute parallel for simd' composite directive.
Definition: StmtOpenMP.h:4660
This represents '#pragma omp distribute simd' composite directive.
Definition: StmtOpenMP.h:4725
This represents the 'doacross' clause for the '#pragma omp ordered' directive.
This represents '#pragma omp error' directive.
Definition: StmtOpenMP.h:6302
This is a basic class for representing single OpenMP executable directive.
Definition: StmtOpenMP.h:266
CapturedStmt * getInnermostCapturedStmt()
Get innermost captured statement for the construct.
Definition: StmtOpenMP.h:556
bool hasAssociatedStmt() const
Returns true if directive has associated statement.
Definition: StmtOpenMP.h:531
OpenMPDirectiveKind getDirectiveKind() const
Definition: StmtOpenMP.h:569
const Stmt * getAssociatedStmt() const
Returns statement associated with the directive.
Definition: StmtOpenMP.h:534
SourceLocation getBeginLoc() const
Returns starting location of directive kind.
Definition: StmtOpenMP.h:502
ArrayRef< OMPClause * > clauses() const
Definition: StmtOpenMP.h:586
SourceLocation getEndLoc() const
Returns ending location of directive.
Definition: StmtOpenMP.h:504
static const SpecificClause * getSingleClause(ArrayRef< OMPClause * > Clauses)
Gets a single clause of the specified kind associated with the current directive iff there is only on...
Definition: StmtOpenMP.h:477
static llvm::iterator_range< specific_clause_iterator< SpecificClause > > getClausesOfKind(ArrayRef< OMPClause * > Clauses)
Definition: StmtOpenMP.h:459
This represents 'fail' clause in the '#pragma omp atomic' directive.
OpenMPClauseKind getFailParameter() const
Gets the parameter (type memory-order-clause) in Fail clause.
This represents 'filter' clause in the '#pragma omp ...' directive.
This represents 'final' clause in the '#pragma omp ...' directive.
Definition: OpenMPClause.h:630
This represents clause 'firstprivate' in the '#pragma omp ...' directives.
This represents implicit clause 'flush' for the '#pragma omp flush' directive.
This represents '#pragma omp flush' directive.
Definition: StmtOpenMP.h:2805
This represents '#pragma omp for' directive.
Definition: StmtOpenMP.h:1649
This represents '#pragma omp for simd' directive.
Definition: StmtOpenMP.h:1740
Representation of the 'full' clause of the '#pragma omp unroll' directive.
Definition: OpenMPClause.h:879
This represents '#pragma omp loop' directive.
Definition: StmtOpenMP.h:5982
This represents 'grainsize' clause in the '#pragma omp ...' directive.
This represents 'hint' clause in the '#pragma omp ...' directive.
This represents 'if' clause in the '#pragma omp ...' directive.
Definition: OpenMPClause.h:527
This represents clause 'in_reduction' in the '#pragma omp task' directives.
This represents clause 'inclusive' in the '#pragma omp scan' directive.
This represents the 'init' clause in '#pragma omp ...' directives.
This represents '#pragma omp interop' directive.
Definition: StmtOpenMP.h:5774
This represents clause 'lastprivate' in the '#pragma omp ...' directives.
This represents clause 'linear' in the '#pragma omp ...' directives.
The base class for all loop-based directives, including loop transformation directives.
Definition: StmtOpenMP.h:698
static Stmt * tryToFindNextInnerLoop(Stmt *CurStmt, bool TryImperfectlyNestedLoops)
Try to find the next loop sub-statement in the specified statement CurStmt.
Definition: StmtOpenMP.cpp:78
static bool doForAllLoops(Stmt *CurStmt, bool TryImperfectlyNestedLoops, unsigned NumLoops, llvm::function_ref< bool(unsigned, Stmt *)> Callback, llvm::function_ref< void(OMPLoopTransformationDirective *)> OnTransformationCallback)
Calls the specified callback function for all the loops in CurStmt, from the outermost to the innermo...
Definition: StmtOpenMP.cpp:125
unsigned getLoopsNumber() const
Get number of collapsed loops.
Definition: StmtOpenMP.h:892
This is a common base class for loop directives ('omp simd', 'omp for', 'omp for simd' etc....
Definition: StmtOpenMP.h:1018
ArrayRef< Expr * > finals_conditions()
Definition: StmtOpenMP.h:1528
Expr * getPrevUpperBoundVariable() const
Definition: StmtOpenMP.h:1421
ArrayRef< Expr * > counters()
Definition: StmtOpenMP.h:1486
Expr * getNumIterations() const
Definition: StmtOpenMP.h:1408
ArrayRef< Expr * > finals()
Definition: StmtOpenMP.h:1510
Expr * getPrevLowerBoundVariable() const
Definition: StmtOpenMP.h:1416
Expr * getIterationVariable() const
Definition: StmtOpenMP.h:1333
Expr * getUpperBoundVariable() const
Definition: StmtOpenMP.h:1368
ArrayRef< Expr * > updates()
Definition: StmtOpenMP.h:1504
Expr * getLowerBoundVariable() const
Definition: StmtOpenMP.h:1360
ArrayRef< Expr * > private_counters()
Definition: StmtOpenMP.h:1492
This represents '#pragma omp masked' directive.
Definition: StmtOpenMP.h:5892
This represents '#pragma omp master' directive.
Definition: StmtOpenMP.h:2044
This represents '#pragma omp master taskloop' directive.
Definition: StmtOpenMP.h:3870
This represents '#pragma omp master taskloop simd' directive.
Definition: StmtOpenMP.h:4022
This represents 'message' clause in the '#pragma omp error' directive.
Expr * getMessageString() const
Returns message string of the clause.
This represents '#pragma omp metadirective' directive.
Definition: StmtOpenMP.h:5943
This represents 'nogroup' clause in the '#pragma omp ...' directive.
This represents 'nowait' clause in the '#pragma omp ...' directive.
This represents 'num_tasks' clause in the '#pragma omp ...' directive.
This represents 'num_teams' clause in the '#pragma omp ...' directive.
This represents 'num_threads' clause in the '#pragma omp ...' directive.
Definition: OpenMPClause.h:676
This represents 'order' clause in the '#pragma omp ...' directive.
This represents 'ordered' clause in the '#pragma omp ...' directive.
This represents '#pragma omp ordered' directive.
Definition: StmtOpenMP.h:2909
This represents '#pragma omp parallel' directive.
Definition: StmtOpenMP.h:627
This represents '#pragma omp parallel for' directive.
Definition: StmtOpenMP.h:2163
This represents '#pragma omp parallel for simd' directive.
Definition: StmtOpenMP.h:2260
This represents '#pragma omp parallel masked' directive.
Definition: StmtOpenMP.h:2388
This represents '#pragma omp parallel master' directive.
Definition: StmtOpenMP.h:2325
This represents '#pragma omp parallel master taskloop' directive.
Definition: StmtOpenMP.h:4153
This represents '#pragma omp parallel master taskloop simd' directive.
Definition: StmtOpenMP.h:4309
This represents '#pragma omp parallel sections' directive.
Definition: StmtOpenMP.h:2452
Representation of the 'partial' clause of the '#pragma omp unroll' directive.
Definition: OpenMPClause.h:907
This represents 'priority' clause in the '#pragma omp ...' directive.
This represents clause 'private' in the '#pragma omp ...' directives.
This represents 'proc_bind' clause in the '#pragma omp ...' directive.
This represents clause 'reduction' in the '#pragma omp ...' directives.
This represents 'relaxed' clause in the '#pragma omp atomic' directives.
This represents 'release' clause in the '#pragma omp atomic|flush' directives.
This represents 'simd' clause in the '#pragma omp ...' directive.
This represents 'safelen' clause in the '#pragma omp ...' directive.
Definition: OpenMPClause.h:721
This represents '#pragma omp scan' directive.
Definition: StmtOpenMP.h:5721
This represents 'schedule' clause in the '#pragma omp ...' directive.
This represents '#pragma omp section' directive.
Definition: StmtOpenMP.h:1880
This represents '#pragma omp sections' directive.
Definition: StmtOpenMP.h:1803
This represents 'seq_cst' clause in the '#pragma omp atomic' directive.
This represents 'severity' clause in the '#pragma omp error' directive.
OpenMPSeverityClauseKind getSeverityKind() const
Returns kind of the clause.
This represents '#pragma omp simd' directive.
Definition: StmtOpenMP.h:1585
This represents 'simdlen' clause in the '#pragma omp ...' directive.
Definition: OpenMPClause.h:756
This represents '#pragma omp single' directive.
Definition: StmtOpenMP.h:1993
This represents '#pragma omp target data' directive.
Definition: StmtOpenMP.h:3222
This represents '#pragma omp target' directive.
Definition: StmtOpenMP.h:3168
This represents '#pragma omp target enter data' directive.
Definition: StmtOpenMP.h:3276
This represents '#pragma omp target exit data' directive.
Definition: StmtOpenMP.h:3331
This represents '#pragma omp target parallel' directive.
Definition: StmtOpenMP.h:3385
This represents '#pragma omp target parallel for' directive.
Definition: StmtOpenMP.h:3465
This represents '#pragma omp target parallel for simd' directive.
Definition: StmtOpenMP.h:4791
This represents '#pragma omp target parallel loop' directive.
Definition: StmtOpenMP.h:6240
This represents '#pragma omp target simd' directive.
Definition: StmtOpenMP.h:4858
This represents '#pragma omp target teams' directive.
Definition: StmtOpenMP.h:5216
This represents '#pragma omp target teams distribute' combined directive.
Definition: StmtOpenMP.h:5272
This represents '#pragma omp target teams distribute parallel for' combined directive.
Definition: StmtOpenMP.h:5339
This represents '#pragma omp target teams distribute parallel for simd' combined directive.
Definition: StmtOpenMP.h:5437
This represents '#pragma omp target teams distribute simd' combined directive.
Definition: StmtOpenMP.h:5507
This represents '#pragma omp target teams loop' directive.
Definition: StmtOpenMP.h:6109
This represents '#pragma omp target update' directive.
Definition: StmtOpenMP.h:4508
This represents '#pragma omp task' directive.
Definition: StmtOpenMP.h:2533
This represents '#pragma omp taskloop' directive.
Definition: StmtOpenMP.h:3731
This represents '#pragma omp taskloop simd' directive.
Definition: StmtOpenMP.h:3804
This represents clause 'task_reduction' in the '#pragma omp taskgroup' directives.
This represents '#pragma omp taskgroup' directive.
Definition: StmtOpenMP.h:2738
This represents '#pragma omp taskwait' directive.
Definition: StmtOpenMP.h:2687
This represents '#pragma omp taskyield' directive.
Definition: StmtOpenMP.h:2595
This represents '#pragma omp teams' directive.
Definition: StmtOpenMP.h:3560
This represents '#pragma omp teams distribute' directive.
Definition: StmtOpenMP.h:4923
This represents '#pragma omp teams distribute parallel for' composite directive.
Definition: StmtOpenMP.h:5123
This represents '#pragma omp teams distribute parallel for simd' composite directive.
Definition: StmtOpenMP.h:5057
This represents '#pragma omp teams distribute simd' combined directive.
Definition: StmtOpenMP.h:4989
This represents '#pragma omp teams loop' directive.
Definition: StmtOpenMP.h:6044
This represents 'thread_limit' clause in the '#pragma omp ...' directive.
This represents the '#pragma omp tile' loop transformation directive.
Definition: StmtOpenMP.h:5565
This represents the '#pragma omp unroll' loop transformation directive.
Definition: StmtOpenMP.h:5647
This represents 'untied' clause in the '#pragma omp ...' directive.
This represents 'update' clause in the '#pragma omp atomic' directive.
This represents the 'use' clause in '#pragma omp ...' directives.
This represents clause 'use_device_addr' in the '#pragma omp ...' directives.
This represents clause 'use_device_ptr' in the '#pragma omp ...' directives.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class.
Definition: Expr.h:1168
static ParmVarDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, StorageClass S, Expr *DefArg)
Definition: Decl.cpp:2915
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition: Type.h:2898
If a crash happens while one of these objects are live, the message is printed out along with the spe...
A (possibly-)qualified type.
Definition: Type.h:737
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
Definition: Type.h:6985
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Definition: Type.h:7102
Represents a struct/union/class.
Definition: Decl.h:4133
field_range fields() const
Definition: Decl.h:4339
field_iterator field_begin() const
Definition: Decl.cpp:5035
Base for LValueReferenceType and RValueReferenceType.
Definition: Type.h:3009
Scope - A scope is a transient data structure that is used while parsing the program.
Definition: Scope.h:41
Encodes a location in the source.
A trivial tuple used to represent a source range.
SourceLocation getEnd() const
SourceLocation getBegin() const
Stmt - This represents one statement.
Definition: Stmt.h:84
child_range children()
Definition: Stmt.cpp:287
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top,...
Definition: Stmt.cpp:197
SourceLocation getBeginLoc() const LLVM_READONLY
Definition: Stmt.cpp:338
SwitchStmt - This represents a 'switch' stmt.
Definition: Stmt.h:2386
virtual bool hasBuiltinAtomic(uint64_t AtomicSizeInBits, uint64_t AlignmentInBits) const
Returns true if the given target supports lock-free atomic operations at the specified width and alig...
Definition: TargetInfo.h:816
bool isTLSSupported() const
Whether the target supports thread-local storage.
Definition: TargetInfo.h:1525
The base class of the type hierarchy.
Definition: Type.h:1606
bool isArrayType() const
Definition: Type.h:7220
bool isPointerType() const
Definition: Type.h:7154
const T * castAs() const
Member-template castAs<specific type>.
Definition: Type.h:7724
bool isReferenceType() const
Definition: Type.h:7166
bool isScalarType() const
Definition: Type.h:7538
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
Definition: Type.cpp:651
bool isLValueReferenceType() const
Definition: Type.h:7170
bool isAnyComplexType() const
Definition: Type.h:7252
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g....
Definition: Type.cpp:2123
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
Definition: Type.h:2438
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type.
Definition: Type.h:7710
bool isAnyPointerType() const
Definition: Type.h:7158
UnaryOperator - This represents the unary-expression's (except sizeof and alignof),...
Definition: Expr.h:2182
static UnaryOperator * Create(const ASTContext &C, Expr *input, Opcode opc, QualType type, ExprValueKind VK, ExprObjectKind OK, SourceLocation l, bool CanOverflow, FPOptionsOverride FPFeatures)
Definition: Expr.cpp:4824
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Definition: Decl.h:706
QualType getType() const
Definition: Decl.h:717
Represents a variable declaration or definition.
Definition: Decl.h:918
TLSKind getTLSKind() const
Definition: Decl.cpp:2165
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
Definition: Decl.cpp:2254
@ CInit
C-style initialization with assignment.
Definition: Decl.h:923
bool hasGlobalStorage() const
Returns true for all variables that do not have local storage.
Definition: Decl.h:1210
bool isStaticLocal() const
Returns true if a variable with function scope is a static local variable.
Definition: Decl.h:1192
const Expr * getInit() const
Definition: Decl.h:1352
bool hasLocalStorage() const
Returns true if a variable with function scope is a non-static local variable.
Definition: Decl.h:1168
@ TLS_None
Not a TLS variable.
Definition: Decl.h:938
Represents a C array with a specified size that is not an integer-constant-expression.
Definition: Type.h:3290
Expr * getSizeExpr() const
Definition: Type.h:3309
@ Decl
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
bool Inc(InterpState &S, CodePtr OpPC)
1) Pops a pointer from the stack 2) Load the value from the pointer 3) Writes the value increased by ...
Definition: Interp.h:589
CharSourceRange getSourceRange(const SourceRange &Range)
Returns the token CharSourceRange corresponding to Range.
Definition: FixIt.h:32
llvm::cl::opt< std::string > Filter
The JSON file list parser is used to communicate input to InstallAPI.
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
@ Tile
'tile' clause, allowed on 'loop' and Combined constructs.
@ Device
'device' clause, allowed on the 'update' construct.
bool needsTaskBasedThreadLimit(OpenMPDirectiveKind DKind)
Checks if the specified target directive, combined or not, needs task based thread_limit.
@ Ctor_Complete
Complete object ctor.
Definition: ABI.h:25
llvm::omp::Directive OpenMPDirectiveKind
OpenMP directives.
Definition: OpenMPKinds.h:24
@ OK_Ordinary
An ordinary object is located at an address in memory.
Definition: Specifiers.h:148
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
BinaryOperatorKind
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
Definition: OpenMPKinds.h:38
@ OMPC_SCHEDULE_MODIFIER_unknown
Definition: OpenMPKinds.h:39
llvm::omp::Clause OpenMPClauseKind
OpenMP clauses.
Definition: OpenMPKinds.h:27
@ CR_OpenMP
Definition: CapturedStmt.h:19
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
@ SC_Static
Definition: Specifiers.h:249
@ SC_None
Definition: Specifiers.h:247
OpenMPDistScheduleClauseKind
OpenMP attributes for 'dist_schedule' clause.
Definition: OpenMPKinds.h:103
@ OMPC_DIST_SCHEDULE_unknown
Definition: OpenMPKinds.h:106
bool isOpenMPTaskingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of tasking directives - task, taskloop,...
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
@ Result
The result type of a method or function.
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
OpenMPDependClauseKind
OpenMP attributes for 'depend' clause.
Definition: OpenMPKinds.h:54
@ Dtor_Complete
Complete object dtor.
Definition: ABI.h:35
bool isOpenMPLoopBoundSharingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of the composite or combined directives that need loop ...
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
@ VK_PRValue
A pr-value expression (in the C++11 taxonomy) produces a temporary value.
Definition: Specifiers.h:132
@ VK_LValue
An l-value expression is a reference to an object with independent storage.
Definition: Specifiers.h:136
void getOpenMPCaptureRegions(llvm::SmallVectorImpl< OpenMPDirectiveKind > &CaptureRegions, OpenMPDirectiveKind DKind)
Return the captured regions of an OpenMP directive.
if(T->getSizeExpr()) TRY_TO(TraverseStmt(T -> getSizeExpr()))
@ OMPC_DEVICE_unknown
Definition: OpenMPKinds.h:50
@ ThreadPrivateVar
Parameter for Thread private variable.
@ Other
Other implicit parameter.
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
Definition: OpenMPKinds.h:30
@ OMPC_SCHEDULE_unknown
Definition: OpenMPKinds.h:34
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
unsigned long uint64_t
YAML serialization mapping.
Definition: Dominators.h:30
Definition: Format.h:5304
struct with the values to be passed to the dispatch runtime function
llvm::Value * Chunk
Chunk size specified using 'schedule' clause (nullptr if chunk was not specified)
Struct with the values to be passed to the static runtime function.
QualType getType() const
Definition: CGCall.h:232
static Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc)
Returns address of the threadprivate variable for the current thread.
llvm::OpenMPIRBuilder::InsertPointTy InsertPointTy
static void EmitOMPOutlinedRegionBody(CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Twine RegionName)
Emit the body of an OMP region that will be outlined in OpenMPIRBuilder::finalize().
static Address getAddressOfLocalVariable(CodeGenFunction &CGF, const VarDecl *VD)
Gets the OpenMP-specific address of the local variable /p VD.
static void EmitCaptureStmt(CodeGenFunction &CGF, InsertPointTy CodeGenIP, llvm::BasicBlock &FiniBB, llvm::Function *Fn, ArrayRef< llvm::Value * > Args)
static std::string getNameWithSeparators(ArrayRef< StringRef > Parts, StringRef FirstSeparator=".", StringRef Separator=".")
Get the platform-specific name separator.
static void FinalizeOMPRegion(CodeGenFunction &CGF, InsertPointTy IP)
Emit the Finalization for an OMP region.
static void EmitOMPInlinedRegionBody(CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Twine RegionName)
Emit the body of an OMP region.
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
SmallVector< const Expr *, 4 > DepExprs
EvalResult is a struct with detailed info about an evaluated expression.
Definition: Expr.h:642
Extra information about a function prototype.
Definition: Type.h:4278
Scheduling data for loop-based OpenMP directives.
Definition: OpenMPKinds.h:179
OpenMPScheduleClauseModifier M2
Definition: OpenMPKinds.h:182
OpenMPScheduleClauseModifier M1
Definition: OpenMPKinds.h:181
OpenMPScheduleClauseKind Schedule
Definition: OpenMPKinds.h:180