clang 19.0.0git
CGStmtOpenMP.cpp
Go to the documentation of this file.
1//===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit OpenMP nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCleanup.h"
14#include "CGOpenMPRuntime.h"
15#include "CodeGenFunction.h"
16#include "CodeGenModule.h"
17#include "TargetInfo.h"
19#include "clang/AST/Attr.h"
22#include "clang/AST/Stmt.h"
28#include "llvm/ADT/SmallSet.h"
29#include "llvm/BinaryFormat/Dwarf.h"
30#include "llvm/Frontend/OpenMP/OMPConstants.h"
31#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
32#include "llvm/IR/Constants.h"
33#include "llvm/IR/DebugInfoMetadata.h"
34#include "llvm/IR/Instructions.h"
35#include "llvm/IR/IntrinsicInst.h"
36#include "llvm/IR/Metadata.h"
37#include "llvm/Support/AtomicOrdering.h"
38#include "llvm/Support/Debug.h"
39#include <optional>
40using namespace clang;
41using namespace CodeGen;
42using namespace llvm::omp;
43
44#define TTL_CODEGEN_TYPE "target-teams-loop-codegen"
45
46static const VarDecl *getBaseDecl(const Expr *Ref);
47
48namespace {
49/// Lexical scope for OpenMP executable constructs, that handles correct codegen
50/// for captured expressions.
51class OMPLexicalScope : public CodeGenFunction::LexicalScope {
52 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
53 for (const auto *C : S.clauses()) {
54 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
55 if (const auto *PreInit =
56 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
57 for (const auto *I : PreInit->decls()) {
58 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
59 CGF.EmitVarDecl(cast<VarDecl>(*I));
60 } else {
61 CodeGenFunction::AutoVarEmission Emission =
62 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
63 CGF.EmitAutoVarCleanups(Emission);
64 }
65 }
66 }
67 }
68 }
69 }
70 CodeGenFunction::OMPPrivateScope InlinedShareds;
71
72 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
73 return CGF.LambdaCaptureFields.lookup(VD) ||
74 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
75 (isa_and_nonnull<BlockDecl>(CGF.CurCodeDecl) &&
76 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
77 }
78
79public:
80 OMPLexicalScope(
82 const std::optional<OpenMPDirectiveKind> CapturedRegion = std::nullopt,
83 const bool EmitPreInitStmt = true)
84 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
85 InlinedShareds(CGF) {
86 if (EmitPreInitStmt)
87 emitPreInitStmt(CGF, S);
88 if (!CapturedRegion)
89 return;
90 assert(S.hasAssociatedStmt() &&
91 "Expected associated statement for inlined directive.");
92 const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
93 for (const auto &C : CS->captures()) {
94 if (C.capturesVariable() || C.capturesVariableByCopy()) {
95 auto *VD = C.getCapturedVar();
96 assert(VD == VD->getCanonicalDecl() &&
97 "Canonical decl must be captured.");
98 DeclRefExpr DRE(
99 CGF.getContext(), const_cast<VarDecl *>(VD),
100 isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
101 InlinedShareds.isGlobalVarCaptured(VD)),
102 VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
103 InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress());
104 }
105 }
106 (void)InlinedShareds.Privatize();
107 }
108};
109
110/// Lexical scope for OpenMP parallel construct, that handles correct codegen
111/// for captured expressions.
112class OMPParallelScope final : public OMPLexicalScope {
113 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
114 OpenMPDirectiveKind Kind = S.getDirectiveKind();
115 return !(isOpenMPTargetExecutionDirective(Kind) ||
118 }
119
120public:
121 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
122 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt,
123 EmitPreInitStmt(S)) {}
124};
125
126/// Lexical scope for OpenMP teams construct, that handles correct codegen
127/// for captured expressions.
128class OMPTeamsScope final : public OMPLexicalScope {
129 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
130 OpenMPDirectiveKind Kind = S.getDirectiveKind();
131 return !isOpenMPTargetExecutionDirective(Kind) &&
133 }
134
135public:
136 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
137 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt,
138 EmitPreInitStmt(S)) {}
139};
140
141/// Private scope for OpenMP loop-based directives, that supports capturing
142/// of used expression from loop statement.
143class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
144 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopBasedDirective &S) {
145 const Stmt *PreInits;
146 CodeGenFunction::OMPMapVars PreCondVars;
147 if (auto *LD = dyn_cast<OMPLoopDirective>(&S)) {
148 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
149 for (const auto *E : LD->counters()) {
150 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
151 EmittedAsPrivate.insert(VD->getCanonicalDecl());
152 (void)PreCondVars.setVarAddr(
153 CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
154 }
155 // Mark private vars as undefs.
156 for (const auto *C : LD->getClausesOfKind<OMPPrivateClause>()) {
157 for (const Expr *IRef : C->varlists()) {
158 const auto *OrigVD =
159 cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
160 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
161 QualType OrigVDTy = OrigVD->getType().getNonReferenceType();
162 (void)PreCondVars.setVarAddr(
163 CGF, OrigVD,
164 Address(llvm::UndefValue::get(CGF.ConvertTypeForMem(
165 CGF.getContext().getPointerType(OrigVDTy))),
166 CGF.ConvertTypeForMem(OrigVDTy),
167 CGF.getContext().getDeclAlign(OrigVD)));
168 }
169 }
170 }
171 (void)PreCondVars.apply(CGF);
172 // Emit init, __range and __end variables for C++ range loops.
174 LD->getInnermostCapturedStmt()->getCapturedStmt(),
175 /*TryImperfectlyNestedLoops=*/true, LD->getLoopsNumber(),
176 [&CGF](unsigned Cnt, const Stmt *CurStmt) {
177 if (const auto *CXXFor = dyn_cast<CXXForRangeStmt>(CurStmt)) {
178 if (const Stmt *Init = CXXFor->getInit())
179 CGF.EmitStmt(Init);
180 CGF.EmitStmt(CXXFor->getRangeStmt());
181 CGF.EmitStmt(CXXFor->getEndStmt());
182 }
183 return false;
184 });
185 PreInits = LD->getPreInits();
186 } else if (const auto *Tile = dyn_cast<OMPTileDirective>(&S)) {
187 PreInits = Tile->getPreInits();
188 } else if (const auto *Unroll = dyn_cast<OMPUnrollDirective>(&S)) {
189 PreInits = Unroll->getPreInits();
190 } else {
191 llvm_unreachable("Unknown loop-based directive kind.");
192 }
193 if (PreInits) {
194 // CompoundStmts and DeclStmts are used as lists of PreInit statements and
195 // declarations. Since declarations must be visible in the the following
196 // that they initialize, unpack the CompoundStmt they are nested in.
197 SmallVector<const Stmt *> PreInitStmts;
198 if (auto *PreInitCompound = dyn_cast<CompoundStmt>(PreInits))
199 llvm::append_range(PreInitStmts, PreInitCompound->body());
200 else
201 PreInitStmts.push_back(PreInits);
202
203 for (const Stmt *S : PreInitStmts) {
204 // EmitStmt skips any OMPCapturedExprDecls, but needs to be emitted
205 // here.
206 if (auto *PreInitDecl = dyn_cast<DeclStmt>(S)) {
207 for (Decl *I : PreInitDecl->decls())
208 CGF.EmitVarDecl(cast<VarDecl>(*I));
209 continue;
210 }
211 CGF.EmitStmt(S);
212 }
213 }
214 PreCondVars.restore(CGF);
215 }
216
217public:
218 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopBasedDirective &S)
219 : CodeGenFunction::RunCleanupsScope(CGF) {
220 emitPreInitStmt(CGF, S);
221 }
222};
223
224class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
225 CodeGenFunction::OMPPrivateScope InlinedShareds;
226
227 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
228 return CGF.LambdaCaptureFields.lookup(VD) ||
229 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
230 (isa_and_nonnull<BlockDecl>(CGF.CurCodeDecl) &&
231 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
232 }
233
234public:
235 OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
236 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
237 InlinedShareds(CGF) {
238 for (const auto *C : S.clauses()) {
239 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
240 if (const auto *PreInit =
241 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
242 for (const auto *I : PreInit->decls()) {
243 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
244 CGF.EmitVarDecl(cast<VarDecl>(*I));
245 } else {
246 CodeGenFunction::AutoVarEmission Emission =
247 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
248 CGF.EmitAutoVarCleanups(Emission);
249 }
250 }
251 }
252 } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
253 for (const Expr *E : UDP->varlists()) {
254 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
255 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
256 CGF.EmitVarDecl(*OED);
257 }
258 } else if (const auto *UDP = dyn_cast<OMPUseDeviceAddrClause>(C)) {
259 for (const Expr *E : UDP->varlists()) {
260 const Decl *D = getBaseDecl(E);
261 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
262 CGF.EmitVarDecl(*OED);
263 }
264 }
265 }
266 if (!isOpenMPSimdDirective(S.getDirectiveKind()))
267 CGF.EmitOMPPrivateClause(S, InlinedShareds);
268 if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
269 if (const Expr *E = TG->getReductionRef())
270 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
271 }
272 // Temp copy arrays for inscan reductions should not be emitted as they are
273 // not used in simd only mode.
275 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
276 if (C->getModifier() != OMPC_REDUCTION_inscan)
277 continue;
278 for (const Expr *E : C->copy_array_temps())
279 CopyArrayTemps.insert(cast<DeclRefExpr>(E)->getDecl());
280 }
281 const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
282 while (CS) {
283 for (auto &C : CS->captures()) {
284 if (C.capturesVariable() || C.capturesVariableByCopy()) {
285 auto *VD = C.getCapturedVar();
286 if (CopyArrayTemps.contains(VD))
287 continue;
288 assert(VD == VD->getCanonicalDecl() &&
289 "Canonical decl must be captured.");
290 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
291 isCapturedVar(CGF, VD) ||
292 (CGF.CapturedStmtInfo &&
293 InlinedShareds.isGlobalVarCaptured(VD)),
295 C.getLocation());
296 InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress());
297 }
298 }
299 CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
300 }
301 (void)InlinedShareds.Privatize();
302 }
303};
304
305} // namespace
306
308 const OMPExecutableDirective &S,
309 const RegionCodeGenTy &CodeGen);
310
311LValue CodeGenFunction::EmitOMPSharedLValue(const Expr *E) {
312 if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
313 if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
314 OrigVD = OrigVD->getCanonicalDecl();
315 bool IsCaptured =
316 LambdaCaptureFields.lookup(OrigVD) ||
318 (isa_and_nonnull<BlockDecl>(CurCodeDecl));
319 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), IsCaptured,
320 OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
321 return EmitLValue(&DRE);
322 }
323 }
324 return EmitLValue(E);
325}
326
327llvm::Value *CodeGenFunction::getTypeSize(QualType Ty) {
329 llvm::Value *Size = nullptr;
330 auto SizeInChars = C.getTypeSizeInChars(Ty);
331 if (SizeInChars.isZero()) {
332 // getTypeSizeInChars() returns 0 for a VLA.
333 while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) {
334 VlaSizePair VlaSize = getVLASize(VAT);
335 Ty = VlaSize.Type;
336 Size =
337 Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts) : VlaSize.NumElts;
338 }
339 SizeInChars = C.getTypeSizeInChars(Ty);
340 if (SizeInChars.isZero())
341 return llvm::ConstantInt::get(SizeTy, /*V=*/0);
342 return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
343 }
344 return CGM.getSize(SizeInChars);
345}
346
348 const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
349 const RecordDecl *RD = S.getCapturedRecordDecl();
350 auto CurField = RD->field_begin();
351 auto CurCap = S.captures().begin();
352 for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
353 E = S.capture_init_end();
354 I != E; ++I, ++CurField, ++CurCap) {
355 if (CurField->hasCapturedVLAType()) {
356 const VariableArrayType *VAT = CurField->getCapturedVLAType();
357 llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
358 CapturedVars.push_back(Val);
359 } else if (CurCap->capturesThis()) {
360 CapturedVars.push_back(CXXThisValue);
361 } else if (CurCap->capturesVariableByCopy()) {
362 llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
363
364 // If the field is not a pointer, we need to save the actual value
365 // and load it as a void pointer.
366 if (!CurField->getType()->isAnyPointerType()) {
367 ASTContext &Ctx = getContext();
368 Address DstAddr = CreateMemTemp(
369 Ctx.getUIntPtrType(),
370 Twine(CurCap->getCapturedVar()->getName(), ".casted"));
371 LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
372
373 llvm::Value *SrcAddrVal = EmitScalarConversion(
374 DstAddr.emitRawPointer(*this),
376 Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
377 LValue SrcLV =
378 MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
379
380 // Store the value using the source type pointer.
382
383 // Load the value using the destination type pointer.
384 CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
385 }
386 CapturedVars.push_back(CV);
387 } else {
388 assert(CurCap->capturesVariable() && "Expected capture by reference.");
389 CapturedVars.push_back(EmitLValue(*I).getAddress().emitRawPointer(*this));
390 }
391 }
392}
393
395 QualType DstType, StringRef Name,
396 LValue AddrLV) {
397 ASTContext &Ctx = CGF.getContext();
398
399 llvm::Value *CastedPtr = CGF.EmitScalarConversion(
400 AddrLV.getAddress().emitRawPointer(CGF), Ctx.getUIntPtrType(),
401 Ctx.getPointerType(DstType), Loc);
402 // FIXME: should the pointee type (DstType) be passed?
403 Address TmpAddr =
404 CGF.MakeNaturalAlignAddrLValue(CastedPtr, DstType).getAddress();
405 return TmpAddr;
406}
407
410 return C.getLValueReferenceType(
411 getCanonicalParamType(C, T.getNonReferenceType()),
412 /*SpelledAsLValue=*/false);
413 if (T->isPointerType())
414 return C.getPointerType(getCanonicalParamType(C, T->getPointeeType()));
415 if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
416 if (const auto *VLA = dyn_cast<VariableArrayType>(A))
417 return getCanonicalParamType(C, VLA->getElementType());
418 if (!A->isVariablyModifiedType())
419 return C.getCanonicalType(T);
420 }
421 return C.getCanonicalParamType(T);
422}
423
424namespace {
425/// Contains required data for proper outlined function codegen.
426struct FunctionOptions {
427 /// Captured statement for which the function is generated.
428 const CapturedStmt *S = nullptr;
429 /// true if cast to/from UIntPtr is required for variables captured by
430 /// value.
431 const bool UIntPtrCastRequired = true;
432 /// true if only casted arguments must be registered as local args or VLA
433 /// sizes.
434 const bool RegisterCastedArgsOnly = false;
435 /// Name of the generated function.
436 const StringRef FunctionName;
437 /// Location of the non-debug version of the outlined function.
439 explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
440 bool RegisterCastedArgsOnly, StringRef FunctionName,
442 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
443 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
444 FunctionName(FunctionName), Loc(Loc) {}
445};
446} // namespace
447
448static llvm::Function *emitOutlinedFunctionPrologue(
450 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
451 &LocalAddrs,
452 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
453 &VLASizes,
454 llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
455 const CapturedDecl *CD = FO.S->getCapturedDecl();
456 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
457 assert(CD->hasBody() && "missing CapturedDecl body");
458
459 CXXThisValue = nullptr;
460 // Build the argument list.
461 CodeGenModule &CGM = CGF.CGM;
462 ASTContext &Ctx = CGM.getContext();
463 FunctionArgList TargetArgs;
464 Args.append(CD->param_begin(),
465 std::next(CD->param_begin(), CD->getContextParamPosition()));
466 TargetArgs.append(
467 CD->param_begin(),
468 std::next(CD->param_begin(), CD->getContextParamPosition()));
469 auto I = FO.S->captures().begin();
470 FunctionDecl *DebugFunctionDecl = nullptr;
471 if (!FO.UIntPtrCastRequired) {
473 QualType FunctionTy = Ctx.getFunctionType(Ctx.VoidTy, std::nullopt, EPI);
474 DebugFunctionDecl = FunctionDecl::Create(
475 Ctx, Ctx.getTranslationUnitDecl(), FO.S->getBeginLoc(),
476 SourceLocation(), DeclarationName(), FunctionTy,
477 Ctx.getTrivialTypeSourceInfo(FunctionTy), SC_Static,
478 /*UsesFPIntrin=*/false, /*isInlineSpecified=*/false,
479 /*hasWrittenPrototype=*/false);
480 }
481 for (const FieldDecl *FD : RD->fields()) {
482 QualType ArgType = FD->getType();
483 IdentifierInfo *II = nullptr;
484 VarDecl *CapVar = nullptr;
485
486 // If this is a capture by copy and the type is not a pointer, the outlined
487 // function argument type should be uintptr and the value properly casted to
488 // uintptr. This is necessary given that the runtime library is only able to
489 // deal with pointers. We can pass in the same way the VLA type sizes to the
490 // outlined function.
491 if (FO.UIntPtrCastRequired &&
492 ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
493 I->capturesVariableArrayType()))
494 ArgType = Ctx.getUIntPtrType();
495
496 if (I->capturesVariable() || I->capturesVariableByCopy()) {
497 CapVar = I->getCapturedVar();
498 II = CapVar->getIdentifier();
499 } else if (I->capturesThis()) {
500 II = &Ctx.Idents.get("this");
501 } else {
502 assert(I->capturesVariableArrayType());
503 II = &Ctx.Idents.get("vla");
504 }
505 if (ArgType->isVariablyModifiedType())
506 ArgType = getCanonicalParamType(Ctx, ArgType);
507 VarDecl *Arg;
508 if (CapVar && (CapVar->getTLSKind() != clang::VarDecl::TLS_None)) {
509 Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
510 II, ArgType,
512 } else if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
514 Ctx, DebugFunctionDecl,
515 CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),
516 CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
517 /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
518 } else {
519 Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
520 II, ArgType, ImplicitParamKind::Other);
521 }
522 Args.emplace_back(Arg);
523 // Do not cast arguments if we emit function with non-original types.
524 TargetArgs.emplace_back(
525 FO.UIntPtrCastRequired
526 ? Arg
527 : CGM.getOpenMPRuntime().translateParameter(FD, Arg));
528 ++I;
529 }
530 Args.append(std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
531 CD->param_end());
532 TargetArgs.append(
533 std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
534 CD->param_end());
535
536 // Create the function declaration.
537 const CGFunctionInfo &FuncInfo =
539 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
540
541 auto *F =
542 llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
543 FO.FunctionName, &CGM.getModule());
544 CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
545 if (CD->isNothrow())
546 F->setDoesNotThrow();
547 F->setDoesNotRecurse();
548
549 // Always inline the outlined function if optimizations are enabled.
550 if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
551 F->removeFnAttr(llvm::Attribute::NoInline);
552 F->addFnAttr(llvm::Attribute::AlwaysInline);
553 }
554
555 // Generate the function.
556 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
557 FO.UIntPtrCastRequired ? FO.Loc : FO.S->getBeginLoc(),
558 FO.UIntPtrCastRequired ? FO.Loc
559 : CD->getBody()->getBeginLoc());
560 unsigned Cnt = CD->getContextParamPosition();
561 I = FO.S->captures().begin();
562 for (const FieldDecl *FD : RD->fields()) {
563 // Do not map arguments if we emit function with non-original types.
564 Address LocalAddr(Address::invalid());
565 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
566 LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
567 TargetArgs[Cnt]);
568 } else {
569 LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
570 }
571 // If we are capturing a pointer by copy we don't need to do anything, just
572 // use the value that we get from the arguments.
573 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
574 const VarDecl *CurVD = I->getCapturedVar();
575 if (!FO.RegisterCastedArgsOnly)
576 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
577 ++Cnt;
578 ++I;
579 continue;
580 }
581
582 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
584 if (FD->hasCapturedVLAType()) {
585 if (FO.UIntPtrCastRequired) {
586 ArgLVal = CGF.MakeAddrLValue(
587 castValueFromUintptr(CGF, I->getLocation(), FD->getType(),
588 Args[Cnt]->getName(), ArgLVal),
590 }
591 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
592 const VariableArrayType *VAT = FD->getCapturedVLAType();
593 VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg);
594 } else if (I->capturesVariable()) {
595 const VarDecl *Var = I->getCapturedVar();
596 QualType VarTy = Var->getType();
597 Address ArgAddr = ArgLVal.getAddress();
598 if (ArgLVal.getType()->isLValueReferenceType()) {
599 ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
600 } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) {
601 assert(ArgLVal.getType()->isPointerType());
602 ArgAddr = CGF.EmitLoadOfPointer(
603 ArgAddr, ArgLVal.getType()->castAs<PointerType>());
604 }
605 if (!FO.RegisterCastedArgsOnly) {
606 LocalAddrs.insert(
607 {Args[Cnt], {Var, ArgAddr.withAlignment(Ctx.getDeclAlign(Var))}});
608 }
609 } else if (I->capturesVariableByCopy()) {
610 assert(!FD->getType()->isAnyPointerType() &&
611 "Not expecting a captured pointer.");
612 const VarDecl *Var = I->getCapturedVar();
613 LocalAddrs.insert({Args[Cnt],
614 {Var, FO.UIntPtrCastRequired
616 CGF, I->getLocation(), FD->getType(),
617 Args[Cnt]->getName(), ArgLVal)
618 : ArgLVal.getAddress()}});
619 } else {
620 // If 'this' is captured, load it into CXXThisValue.
621 assert(I->capturesThis());
622 CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
623 LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress()}});
624 }
625 ++Cnt;
626 ++I;
627 }
628
629 return F;
630}
631
632llvm::Function *
635 assert(
637 "CapturedStmtInfo should be set when generating the captured function");
638 const CapturedDecl *CD = S.getCapturedDecl();
639 // Build the argument list.
640 bool NeedWrapperFunction =
642 FunctionArgList Args;
643 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
644 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
645 SmallString<256> Buffer;
646 llvm::raw_svector_ostream Out(Buffer);
648 if (NeedWrapperFunction)
649 Out << "_debug__";
650 FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
651 Out.str(), Loc);
652 llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs,
653 VLASizes, CXXThisValue, FO);
654 CodeGenFunction::OMPPrivateScope LocalScope(*this);
655 for (const auto &LocalAddrPair : LocalAddrs) {
656 if (LocalAddrPair.second.first) {
657 LocalScope.addPrivate(LocalAddrPair.second.first,
658 LocalAddrPair.second.second);
659 }
660 }
661 (void)LocalScope.Privatize();
662 for (const auto &VLASizePair : VLASizes)
663 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
665 CapturedStmtInfo->EmitBody(*this, CD->getBody());
666 (void)LocalScope.ForceCleanup();
668 if (!NeedWrapperFunction)
669 return F;
670
671 FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
672 /*RegisterCastedArgsOnly=*/true,
674 CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
675 WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
676 Args.clear();
677 LocalAddrs.clear();
678 VLASizes.clear();
679 llvm::Function *WrapperF =
680 emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
681 WrapperCGF.CXXThisValue, WrapperFO);
683 auto *PI = F->arg_begin();
684 for (const auto *Arg : Args) {
685 llvm::Value *CallArg;
686 auto I = LocalAddrs.find(Arg);
687 if (I != LocalAddrs.end()) {
688 LValue LV = WrapperCGF.MakeAddrLValue(
689 I->second.second,
690 I->second.first ? I->second.first->getType() : Arg->getType(),
692 if (LV.getType()->isAnyComplexType())
693 LV.setAddress(LV.getAddress().withElementType(PI->getType()));
694 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
695 } else {
696 auto EI = VLASizes.find(Arg);
697 if (EI != VLASizes.end()) {
698 CallArg = EI->second.second;
699 } else {
700 LValue LV =
701 WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
702 Arg->getType(), AlignmentSource::Decl);
703 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
704 }
705 }
706 CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
707 ++PI;
708 }
709 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, Loc, F, CallArgs);
710 WrapperCGF.FinishFunction();
711 return WrapperF;
712}
713
714//===----------------------------------------------------------------------===//
715// OpenMP Directive Emission
716//===----------------------------------------------------------------------===//
718 Address DestAddr, Address SrcAddr, QualType OriginalType,
719 const llvm::function_ref<void(Address, Address)> CopyGen) {
720 // Perform element-by-element initialization.
721 QualType ElementTy;
722
723 // Drill down to the base element type on both arrays.
724 const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
725 llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
726 SrcAddr = SrcAddr.withElementType(DestAddr.getElementType());
727
728 llvm::Value *SrcBegin = SrcAddr.emitRawPointer(*this);
729 llvm::Value *DestBegin = DestAddr.emitRawPointer(*this);
730 // Cast from pointer to array type to pointer to single element.
731 llvm::Value *DestEnd = Builder.CreateInBoundsGEP(DestAddr.getElementType(),
732 DestBegin, NumElements);
733
734 // The basic structure here is a while-do loop.
735 llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body");
736 llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done");
737 llvm::Value *IsEmpty =
738 Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
739 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
740
741 // Enter the loop body, making that address the current address.
742 llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
743 EmitBlock(BodyBB);
744
745 CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
746
747 llvm::PHINode *SrcElementPHI =
748 Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
749 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
750 Address SrcElementCurrent =
751 Address(SrcElementPHI, SrcAddr.getElementType(),
752 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
753
754 llvm::PHINode *DestElementPHI = Builder.CreatePHI(
755 DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
756 DestElementPHI->addIncoming(DestBegin, EntryBB);
757 Address DestElementCurrent =
758 Address(DestElementPHI, DestAddr.getElementType(),
759 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
760
761 // Emit copy.
762 CopyGen(DestElementCurrent, SrcElementCurrent);
763
764 // Shift the address forward by one element.
765 llvm::Value *DestElementNext =
766 Builder.CreateConstGEP1_32(DestAddr.getElementType(), DestElementPHI,
767 /*Idx0=*/1, "omp.arraycpy.dest.element");
768 llvm::Value *SrcElementNext =
769 Builder.CreateConstGEP1_32(SrcAddr.getElementType(), SrcElementPHI,
770 /*Idx0=*/1, "omp.arraycpy.src.element");
771 // Check whether we've reached the end.
772 llvm::Value *Done =
773 Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
774 Builder.CreateCondBr(Done, DoneBB, BodyBB);
775 DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
776 SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
777
778 // Done.
779 EmitBlock(DoneBB, /*IsFinished=*/true);
780}
781
782void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
783 Address SrcAddr, const VarDecl *DestVD,
784 const VarDecl *SrcVD, const Expr *Copy) {
785 if (OriginalType->isArrayType()) {
786 const auto *BO = dyn_cast<BinaryOperator>(Copy);
787 if (BO && BO->getOpcode() == BO_Assign) {
788 // Perform simple memcpy for simple copying.
789 LValue Dest = MakeAddrLValue(DestAddr, OriginalType);
790 LValue Src = MakeAddrLValue(SrcAddr, OriginalType);
791 EmitAggregateAssign(Dest, Src, OriginalType);
792 } else {
793 // For arrays with complex element types perform element by element
794 // copying.
796 DestAddr, SrcAddr, OriginalType,
797 [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
798 // Working with the single array element, so have to remap
799 // destination and source variables to corresponding array
800 // elements.
801 CodeGenFunction::OMPPrivateScope Remap(*this);
802 Remap.addPrivate(DestVD, DestElement);
803 Remap.addPrivate(SrcVD, SrcElement);
804 (void)Remap.Privatize();
806 });
807 }
808 } else {
809 // Remap pseudo source variable to private copy.
810 CodeGenFunction::OMPPrivateScope Remap(*this);
811 Remap.addPrivate(SrcVD, SrcAddr);
812 Remap.addPrivate(DestVD, DestAddr);
813 (void)Remap.Privatize();
814 // Emit copying of the whole variable.
816 }
817}
818
820 OMPPrivateScope &PrivateScope) {
821 if (!HaveInsertPoint())
822 return false;
823 bool DeviceConstTarget =
824 getLangOpts().OpenMPIsTargetDevice &&
825 isOpenMPTargetExecutionDirective(D.getDirectiveKind());
826 bool FirstprivateIsLastprivate = false;
827 llvm::DenseMap<const VarDecl *, OpenMPLastprivateModifier> Lastprivates;
828 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
829 for (const auto *D : C->varlists())
830 Lastprivates.try_emplace(
831 cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl(),
832 C->getKind());
833 }
834 llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
836 getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind());
837 // Force emission of the firstprivate copy if the directive does not emit
838 // outlined function, like omp for, omp simd, omp distribute etc.
839 bool MustEmitFirstprivateCopy =
840 CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
841 for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
842 const auto *IRef = C->varlist_begin();
843 const auto *InitsRef = C->inits().begin();
844 for (const Expr *IInit : C->private_copies()) {
845 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
846 bool ThisFirstprivateIsLastprivate =
847 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
848 const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
849 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
850 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
851 !FD->getType()->isReferenceType() &&
852 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
853 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
854 ++IRef;
855 ++InitsRef;
856 continue;
857 }
858 // Do not emit copy for firstprivate constant variables in target regions,
859 // captured by reference.
860 if (DeviceConstTarget && OrigVD->getType().isConstant(getContext()) &&
861 FD && FD->getType()->isReferenceType() &&
862 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
863 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
864 ++IRef;
865 ++InitsRef;
866 continue;
867 }
868 FirstprivateIsLastprivate =
869 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
870 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
871 const auto *VDInit =
872 cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
873 bool IsRegistered;
874 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
875 /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
876 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
877 LValue OriginalLVal;
878 if (!FD) {
879 // Check if the firstprivate variable is just a constant value.
880 ConstantEmission CE = tryEmitAsConstant(&DRE);
881 if (CE && !CE.isReference()) {
882 // Constant value, no need to create a copy.
883 ++IRef;
884 ++InitsRef;
885 continue;
886 }
887 if (CE && CE.isReference()) {
888 OriginalLVal = CE.getReferenceLValue(*this, &DRE);
889 } else {
890 assert(!CE && "Expected non-constant firstprivate.");
891 OriginalLVal = EmitLValue(&DRE);
892 }
893 } else {
894 OriginalLVal = EmitLValue(&DRE);
895 }
896 QualType Type = VD->getType();
897 if (Type->isArrayType()) {
898 // Emit VarDecl with copy init for arrays.
899 // Get the address of the original variable captured in current
900 // captured region.
901 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
902 const Expr *Init = VD->getInit();
903 if (!isa<CXXConstructExpr>(Init) || isTrivialInitializer(Init)) {
904 // Perform simple memcpy.
905 LValue Dest = MakeAddrLValue(Emission.getAllocatedAddress(), Type);
906 EmitAggregateAssign(Dest, OriginalLVal, Type);
907 } else {
909 Emission.getAllocatedAddress(), OriginalLVal.getAddress(), Type,
910 [this, VDInit, Init](Address DestElement, Address SrcElement) {
911 // Clean up any temporaries needed by the
912 // initialization.
913 RunCleanupsScope InitScope(*this);
914 // Emit initialization for single element.
915 setAddrOfLocalVar(VDInit, SrcElement);
916 EmitAnyExprToMem(Init, DestElement,
917 Init->getType().getQualifiers(),
918 /*IsInitializer*/ false);
919 LocalDeclMap.erase(VDInit);
920 });
921 }
922 EmitAutoVarCleanups(Emission);
923 IsRegistered =
924 PrivateScope.addPrivate(OrigVD, Emission.getAllocatedAddress());
925 } else {
926 Address OriginalAddr = OriginalLVal.getAddress();
927 // Emit private VarDecl with copy init.
928 // Remap temp VDInit variable to the address of the original
929 // variable (for proper handling of captured global variables).
930 setAddrOfLocalVar(VDInit, OriginalAddr);
931 EmitDecl(*VD);
932 LocalDeclMap.erase(VDInit);
933 Address VDAddr = GetAddrOfLocalVar(VD);
934 if (ThisFirstprivateIsLastprivate &&
935 Lastprivates[OrigVD->getCanonicalDecl()] ==
936 OMPC_LASTPRIVATE_conditional) {
937 // Create/init special variable for lastprivate conditionals.
938 llvm::Value *V =
939 EmitLoadOfScalar(MakeAddrLValue(VDAddr, (*IRef)->getType(),
941 (*IRef)->getExprLoc());
943 *this, OrigVD);
944 EmitStoreOfScalar(V, MakeAddrLValue(VDAddr, (*IRef)->getType(),
946 LocalDeclMap.erase(VD);
947 setAddrOfLocalVar(VD, VDAddr);
948 }
949 IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);
950 }
951 assert(IsRegistered &&
952 "firstprivate var already registered as private");
953 // Silence the warning about unused variable.
954 (void)IsRegistered;
955 }
956 ++IRef;
957 ++InitsRef;
958 }
959 }
960 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
961}
962
965 CodeGenFunction::OMPPrivateScope &PrivateScope) {
966 if (!HaveInsertPoint())
967 return;
968 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
969 for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
970 auto IRef = C->varlist_begin();
971 for (const Expr *IInit : C->private_copies()) {
972 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
973 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
974 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
975 EmitDecl(*VD);
976 // Emit private VarDecl with copy init.
977 bool IsRegistered =
978 PrivateScope.addPrivate(OrigVD, GetAddrOfLocalVar(VD));
979 assert(IsRegistered && "private var already registered as private");
980 // Silence the warning about unused variable.
981 (void)IsRegistered;
982 }
983 ++IRef;
984 }
985 }
986}
987
989 if (!HaveInsertPoint())
990 return false;
991 // threadprivate_var1 = master_threadprivate_var1;
992 // operator=(threadprivate_var2, master_threadprivate_var2);
993 // ...
994 // __kmpc_barrier(&loc, global_tid);
996 llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
997 for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
998 auto IRef = C->varlist_begin();
999 auto ISrcRef = C->source_exprs().begin();
1000 auto IDestRef = C->destination_exprs().begin();
1001 for (const Expr *AssignOp : C->assignment_ops()) {
1002 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1003 QualType Type = VD->getType();
1004 if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
1005 // Get the address of the master variable. If we are emitting code with
1006 // TLS support, the address is passed from the master as field in the
1007 // captured declaration.
1008 Address MasterAddr = Address::invalid();
1009 if (getLangOpts().OpenMPUseTLS &&
1010 getContext().getTargetInfo().isTLSSupported()) {
1011 assert(CapturedStmtInfo->lookup(VD) &&
1012 "Copyin threadprivates should have been captured!");
1013 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), true,
1014 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
1015 MasterAddr = EmitLValue(&DRE).getAddress();
1016 LocalDeclMap.erase(VD);
1017 } else {
1018 MasterAddr =
1020 : CGM.GetAddrOfGlobal(VD),
1022 getContext().getDeclAlign(VD));
1023 }
1024 // Get the address of the threadprivate variable.
1025 Address PrivateAddr = EmitLValue(*IRef).getAddress();
1026 if (CopiedVars.size() == 1) {
1027 // At first check if current thread is a master thread. If it is, no
1028 // need to copy data.
1029 CopyBegin = createBasicBlock("copyin.not.master");
1030 CopyEnd = createBasicBlock("copyin.not.master.end");
1031 // TODO: Avoid ptrtoint conversion.
1032 auto *MasterAddrInt = Builder.CreatePtrToInt(
1033 MasterAddr.emitRawPointer(*this), CGM.IntPtrTy);
1034 auto *PrivateAddrInt = Builder.CreatePtrToInt(
1035 PrivateAddr.emitRawPointer(*this), CGM.IntPtrTy);
1036 Builder.CreateCondBr(
1037 Builder.CreateICmpNE(MasterAddrInt, PrivateAddrInt), CopyBegin,
1038 CopyEnd);
1039 EmitBlock(CopyBegin);
1040 }
1041 const auto *SrcVD =
1042 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1043 const auto *DestVD =
1044 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1045 EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
1046 }
1047 ++IRef;
1048 ++ISrcRef;
1049 ++IDestRef;
1050 }
1051 }
1052 if (CopyEnd) {
1053 // Exit out of copying procedure for non-master thread.
1054 EmitBlock(CopyEnd, /*IsFinished=*/true);
1055 return true;
1056 }
1057 return false;
1058}
1059
1061 const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
1062 if (!HaveInsertPoint())
1063 return false;
1064 bool HasAtLeastOneLastprivate = false;
1066 if (isOpenMPSimdDirective(D.getDirectiveKind())) {
1067 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1068 for (const Expr *C : LoopDirective->counters()) {
1069 SIMDLCVs.insert(
1070 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
1071 }
1072 }
1073 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1074 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1075 HasAtLeastOneLastprivate = true;
1076 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
1077 !getLangOpts().OpenMPSimd)
1078 break;
1079 const auto *IRef = C->varlist_begin();
1080 const auto *IDestRef = C->destination_exprs().begin();
1081 for (const Expr *IInit : C->private_copies()) {
1082 // Keep the address of the original variable for future update at the end
1083 // of the loop.
1084 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1085 // Taskloops do not require additional initialization, it is done in
1086 // runtime support library.
1087 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
1088 const auto *DestVD =
1089 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1090 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1091 /*RefersToEnclosingVariableOrCapture=*/
1092 CapturedStmtInfo->lookup(OrigVD) != nullptr,
1093 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
1094 PrivateScope.addPrivate(DestVD, EmitLValue(&DRE).getAddress());
1095 // Check if the variable is also a firstprivate: in this case IInit is
1096 // not generated. Initialization of this variable will happen in codegen
1097 // for 'firstprivate' clause.
1098 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
1099 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
1100 Address VDAddr = Address::invalid();
1101 if (C->getKind() == OMPC_LASTPRIVATE_conditional) {
1103 *this, OrigVD);
1104 setAddrOfLocalVar(VD, VDAddr);
1105 } else {
1106 // Emit private VarDecl with copy init.
1107 EmitDecl(*VD);
1108 VDAddr = GetAddrOfLocalVar(VD);
1109 }
1110 bool IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);
1111 assert(IsRegistered &&
1112 "lastprivate var already registered as private");
1113 (void)IsRegistered;
1114 }
1115 }
1116 ++IRef;
1117 ++IDestRef;
1118 }
1119 }
1120 return HasAtLeastOneLastprivate;
1121}
1122
1124 const OMPExecutableDirective &D, bool NoFinals,
1125 llvm::Value *IsLastIterCond) {
1126 if (!HaveInsertPoint())
1127 return;
1128 // Emit following code:
1129 // if (<IsLastIterCond>) {
1130 // orig_var1 = private_orig_var1;
1131 // ...
1132 // orig_varn = private_orig_varn;
1133 // }
1134 llvm::BasicBlock *ThenBB = nullptr;
1135 llvm::BasicBlock *DoneBB = nullptr;
1136 if (IsLastIterCond) {
1137 // Emit implicit barrier if at least one lastprivate conditional is found
1138 // and this is not a simd mode.
1139 if (!getLangOpts().OpenMPSimd &&
1140 llvm::any_of(D.getClausesOfKind<OMPLastprivateClause>(),
1141 [](const OMPLastprivateClause *C) {
1142 return C->getKind() == OMPC_LASTPRIVATE_conditional;
1143 })) {
1145 OMPD_unknown,
1146 /*EmitChecks=*/false,
1147 /*ForceSimpleCall=*/true);
1148 }
1149 ThenBB = createBasicBlock(".omp.lastprivate.then");
1150 DoneBB = createBasicBlock(".omp.lastprivate.done");
1151 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1152 EmitBlock(ThenBB);
1153 }
1154 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1155 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1156 if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1157 auto IC = LoopDirective->counters().begin();
1158 for (const Expr *F : LoopDirective->finals()) {
1159 const auto *D =
1160 cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1161 if (NoFinals)
1162 AlreadyEmittedVars.insert(D);
1163 else
1164 LoopCountersAndUpdates[D] = F;
1165 ++IC;
1166 }
1167 }
1168 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1169 auto IRef = C->varlist_begin();
1170 auto ISrcRef = C->source_exprs().begin();
1171 auto IDestRef = C->destination_exprs().begin();
1172 for (const Expr *AssignOp : C->assignment_ops()) {
1173 const auto *PrivateVD =
1174 cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1175 QualType Type = PrivateVD->getType();
1176 const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1177 if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1178 // If lastprivate variable is a loop control variable for loop-based
1179 // directive, update its value before copyin back to original
1180 // variable.
1181 if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1182 EmitIgnoredExpr(FinalExpr);
1183 const auto *SrcVD =
1184 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1185 const auto *DestVD =
1186 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1187 // Get the address of the private variable.
1188 Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
1189 if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1190 PrivateAddr = Address(
1191 Builder.CreateLoad(PrivateAddr),
1192 CGM.getTypes().ConvertTypeForMem(RefTy->getPointeeType()),
1193 CGM.getNaturalTypeAlignment(RefTy->getPointeeType()));
1194 // Store the last value to the private copy in the last iteration.
1195 if (C->getKind() == OMPC_LASTPRIVATE_conditional)
1197 *this, MakeAddrLValue(PrivateAddr, (*IRef)->getType()), PrivateVD,
1198 (*IRef)->getExprLoc());
1199 // Get the address of the original variable.
1200 Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1201 EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1202 }
1203 ++IRef;
1204 ++ISrcRef;
1205 ++IDestRef;
1206 }
1207 if (const Expr *PostUpdate = C->getPostUpdateExpr())
1208 EmitIgnoredExpr(PostUpdate);
1209 }
1210 if (IsLastIterCond)
1211 EmitBlock(DoneBB, /*IsFinished=*/true);
1212}
1213
1216 CodeGenFunction::OMPPrivateScope &PrivateScope, bool ForInscan) {
1217 if (!HaveInsertPoint())
1218 return;
1221 SmallVector<const Expr *, 4> ReductionOps;
1227 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1228 if (ForInscan != (C->getModifier() == OMPC_REDUCTION_inscan))
1229 continue;
1230 Shareds.append(C->varlist_begin(), C->varlist_end());
1231 Privates.append(C->privates().begin(), C->privates().end());
1232 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1233 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1234 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1235 if (C->getModifier() == OMPC_REDUCTION_task) {
1236 Data.ReductionVars.append(C->privates().begin(), C->privates().end());
1237 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
1238 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
1239 Data.ReductionOps.append(C->reduction_ops().begin(),
1240 C->reduction_ops().end());
1241 TaskLHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1242 TaskRHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1243 }
1244 }
1245 ReductionCodeGen RedCG(Shareds, Shareds, Privates, ReductionOps);
1246 unsigned Count = 0;
1247 auto *ILHS = LHSs.begin();
1248 auto *IRHS = RHSs.begin();
1249 auto *IPriv = Privates.begin();
1250 for (const Expr *IRef : Shareds) {
1251 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1252 // Emit private VarDecl with reduction init.
1253 RedCG.emitSharedOrigLValue(*this, Count);
1254 RedCG.emitAggregateType(*this, Count);
1255 AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);
1256 RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
1257 RedCG.getSharedLValue(Count).getAddress(),
1258 [&Emission](CodeGenFunction &CGF) {
1259 CGF.EmitAutoVarInit(Emission);
1260 return true;
1261 });
1262 EmitAutoVarCleanups(Emission);
1263 Address BaseAddr = RedCG.adjustPrivateAddress(
1264 *this, Count, Emission.getAllocatedAddress());
1265 bool IsRegistered =
1266 PrivateScope.addPrivate(RedCG.getBaseDecl(Count), BaseAddr);
1267 assert(IsRegistered && "private var already registered as private");
1268 // Silence the warning about unused variable.
1269 (void)IsRegistered;
1270
1271 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1272 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1273 QualType Type = PrivateVD->getType();
1274 bool isaOMPArraySectionExpr = isa<ArraySectionExpr>(IRef);
1275 if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1276 // Store the address of the original variable associated with the LHS
1277 // implicit variable.
1278 PrivateScope.addPrivate(LHSVD, RedCG.getSharedLValue(Count).getAddress());
1279 PrivateScope.addPrivate(RHSVD, GetAddrOfLocalVar(PrivateVD));
1280 } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1281 isa<ArraySubscriptExpr>(IRef)) {
1282 // Store the address of the original variable associated with the LHS
1283 // implicit variable.
1284 PrivateScope.addPrivate(LHSVD, RedCG.getSharedLValue(Count).getAddress());
1285 PrivateScope.addPrivate(RHSVD,
1286 GetAddrOfLocalVar(PrivateVD).withElementType(
1287 ConvertTypeForMem(RHSVD->getType())));
1288 } else {
1289 QualType Type = PrivateVD->getType();
1290 bool IsArray = getContext().getAsArrayType(Type) != nullptr;
1291 Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress();
1292 // Store the address of the original variable associated with the LHS
1293 // implicit variable.
1294 if (IsArray) {
1295 OriginalAddr =
1296 OriginalAddr.withElementType(ConvertTypeForMem(LHSVD->getType()));
1297 }
1298 PrivateScope.addPrivate(LHSVD, OriginalAddr);
1299 PrivateScope.addPrivate(
1300 RHSVD, IsArray ? GetAddrOfLocalVar(PrivateVD).withElementType(
1301 ConvertTypeForMem(RHSVD->getType()))
1302 : GetAddrOfLocalVar(PrivateVD));
1303 }
1304 ++ILHS;
1305 ++IRHS;
1306 ++IPriv;
1307 ++Count;
1308 }
1309 if (!Data.ReductionVars.empty()) {
1310 Data.IsReductionWithTaskMod = true;
1311 Data.IsWorksharingReduction =
1312 isOpenMPWorksharingDirective(D.getDirectiveKind());
1313 llvm::Value *ReductionDesc = CGM.getOpenMPRuntime().emitTaskReductionInit(
1314 *this, D.getBeginLoc(), TaskLHSs, TaskRHSs, Data);
1315 const Expr *TaskRedRef = nullptr;
1316 switch (D.getDirectiveKind()) {
1317 case OMPD_parallel:
1318 TaskRedRef = cast<OMPParallelDirective>(D).getTaskReductionRefExpr();
1319 break;
1320 case OMPD_for:
1321 TaskRedRef = cast<OMPForDirective>(D).getTaskReductionRefExpr();
1322 break;
1323 case OMPD_sections:
1324 TaskRedRef = cast<OMPSectionsDirective>(D).getTaskReductionRefExpr();
1325 break;
1326 case OMPD_parallel_for:
1327 TaskRedRef = cast<OMPParallelForDirective>(D).getTaskReductionRefExpr();
1328 break;
1329 case OMPD_parallel_master:
1330 TaskRedRef =
1331 cast<OMPParallelMasterDirective>(D).getTaskReductionRefExpr();
1332 break;
1333 case OMPD_parallel_sections:
1334 TaskRedRef =
1335 cast<OMPParallelSectionsDirective>(D).getTaskReductionRefExpr();
1336 break;
1337 case OMPD_target_parallel:
1338 TaskRedRef =
1339 cast<OMPTargetParallelDirective>(D).getTaskReductionRefExpr();
1340 break;
1341 case OMPD_target_parallel_for:
1342 TaskRedRef =
1343 cast<OMPTargetParallelForDirective>(D).getTaskReductionRefExpr();
1344 break;
1345 case OMPD_distribute_parallel_for:
1346 TaskRedRef =
1347 cast<OMPDistributeParallelForDirective>(D).getTaskReductionRefExpr();
1348 break;
1349 case OMPD_teams_distribute_parallel_for:
1350 TaskRedRef = cast<OMPTeamsDistributeParallelForDirective>(D)
1351 .getTaskReductionRefExpr();
1352 break;
1353 case OMPD_target_teams_distribute_parallel_for:
1354 TaskRedRef = cast<OMPTargetTeamsDistributeParallelForDirective>(D)
1355 .getTaskReductionRefExpr();
1356 break;
1357 case OMPD_simd:
1358 case OMPD_for_simd:
1359 case OMPD_section:
1360 case OMPD_single:
1361 case OMPD_master:
1362 case OMPD_critical:
1363 case OMPD_parallel_for_simd:
1364 case OMPD_task:
1365 case OMPD_taskyield:
1366 case OMPD_error:
1367 case OMPD_barrier:
1368 case OMPD_taskwait:
1369 case OMPD_taskgroup:
1370 case OMPD_flush:
1371 case OMPD_depobj:
1372 case OMPD_scan:
1373 case OMPD_ordered:
1374 case OMPD_atomic:
1375 case OMPD_teams:
1376 case OMPD_target:
1377 case OMPD_cancellation_point:
1378 case OMPD_cancel:
1379 case OMPD_target_data:
1380 case OMPD_target_enter_data:
1381 case OMPD_target_exit_data:
1382 case OMPD_taskloop:
1383 case OMPD_taskloop_simd:
1384 case OMPD_master_taskloop:
1385 case OMPD_master_taskloop_simd:
1386 case OMPD_parallel_master_taskloop:
1387 case OMPD_parallel_master_taskloop_simd:
1388 case OMPD_distribute:
1389 case OMPD_target_update:
1390 case OMPD_distribute_parallel_for_simd:
1391 case OMPD_distribute_simd:
1392 case OMPD_target_parallel_for_simd:
1393 case OMPD_target_simd:
1394 case OMPD_teams_distribute:
1395 case OMPD_teams_distribute_simd:
1396 case OMPD_teams_distribute_parallel_for_simd:
1397 case OMPD_target_teams:
1398 case OMPD_target_teams_distribute:
1399 case OMPD_target_teams_distribute_parallel_for_simd:
1400 case OMPD_target_teams_distribute_simd:
1401 case OMPD_declare_target:
1402 case OMPD_end_declare_target:
1403 case OMPD_threadprivate:
1404 case OMPD_allocate:
1405 case OMPD_declare_reduction:
1406 case OMPD_declare_mapper:
1407 case OMPD_declare_simd:
1408 case OMPD_requires:
1409 case OMPD_declare_variant:
1410 case OMPD_begin_declare_variant:
1411 case OMPD_end_declare_variant:
1412 case OMPD_unknown:
1413 default:
1414 llvm_unreachable("Unexpected directive with task reductions.");
1415 }
1416
1417 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(TaskRedRef)->getDecl());
1418 EmitVarDecl(*VD);
1419 EmitStoreOfScalar(ReductionDesc, GetAddrOfLocalVar(VD),
1420 /*Volatile=*/false, TaskRedRef->getType());
1421 }
1422}
1423
1425 const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1426 if (!HaveInsertPoint())
1427 return;
1432 bool HasAtLeastOneReduction = false;
1433 bool IsReductionWithTaskMod = false;
1434 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1435 // Do not emit for inscan reductions.
1436 if (C->getModifier() == OMPC_REDUCTION_inscan)
1437 continue;
1438 HasAtLeastOneReduction = true;
1439 Privates.append(C->privates().begin(), C->privates().end());
1440 LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1441 RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1442 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1443 IsReductionWithTaskMod =
1444 IsReductionWithTaskMod || C->getModifier() == OMPC_REDUCTION_task;
1445 }
1446 if (HasAtLeastOneReduction) {
1447 if (IsReductionWithTaskMod) {
1449 *this, D.getBeginLoc(),
1450 isOpenMPWorksharingDirective(D.getDirectiveKind()));
1451 }
1452 bool TeamsLoopCanBeParallel = false;
1453 if (auto *TTLD = dyn_cast<OMPTargetTeamsGenericLoopDirective>(&D))
1454 TeamsLoopCanBeParallel = TTLD->canBeParallelFor();
1455 bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1456 isOpenMPParallelDirective(D.getDirectiveKind()) ||
1457 TeamsLoopCanBeParallel || ReductionKind == OMPD_simd;
1458 bool SimpleReduction = ReductionKind == OMPD_simd;
1459 // Emit nowait reduction if nowait clause is present or directive is a
1460 // parallel directive (it always has implicit barrier).
1462 *this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1463 {WithNowait, SimpleReduction, ReductionKind});
1464 }
1465}
1466
1469 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1470 if (!CGF.HaveInsertPoint())
1471 return;
1472 llvm::BasicBlock *DoneBB = nullptr;
1473 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1474 if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
1475 if (!DoneBB) {
1476 if (llvm::Value *Cond = CondGen(CGF)) {
1477 // If the first post-update expression is found, emit conditional
1478 // block if it was requested.
1479 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1480 DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1481 CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1482 CGF.EmitBlock(ThenBB);
1483 }
1484 }
1485 CGF.EmitIgnoredExpr(PostUpdate);
1486 }
1487 }
1488 if (DoneBB)
1489 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1490}
1491
1492namespace {
1493/// Codegen lambda for appending distribute lower and upper bounds to outlined
1494/// parallel function. This is necessary for combined constructs such as
1495/// 'distribute parallel for'
1496typedef llvm::function_ref<void(CodeGenFunction &,
1497 const OMPExecutableDirective &,
1499 CodeGenBoundParametersTy;
1500} // anonymous namespace
1501
1502static void
1504 const OMPExecutableDirective &S) {
1505 if (CGF.getLangOpts().OpenMP < 50)
1506 return;
1508 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
1509 for (const Expr *Ref : C->varlists()) {
1510 if (!Ref->getType()->isScalarType())
1511 continue;
1512 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1513 if (!DRE)
1514 continue;
1515 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1517 }
1518 }
1519 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
1520 for (const Expr *Ref : C->varlists()) {
1521 if (!Ref->getType()->isScalarType())
1522 continue;
1523 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1524 if (!DRE)
1525 continue;
1526 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1528 }
1529 }
1530 for (const auto *C : S.getClausesOfKind<OMPLinearClause>()) {
1531 for (const Expr *Ref : C->varlists()) {
1532 if (!Ref->getType()->isScalarType())
1533 continue;
1534 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1535 if (!DRE)
1536 continue;
1537 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1539 }
1540 }
1541 // Privates should ne analyzed since they are not captured at all.
1542 // Task reductions may be skipped - tasks are ignored.
1543 // Firstprivates do not return value but may be passed by reference - no need
1544 // to check for updated lastprivate conditional.
1545 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
1546 for (const Expr *Ref : C->varlists()) {
1547 if (!Ref->getType()->isScalarType())
1548 continue;
1549 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1550 if (!DRE)
1551 continue;
1552 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1553 }
1554 }
1556 CGF, S, PrivateDecls);
1557}
1558
1561 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1562 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1563 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1564 llvm::Value *NumThreads = nullptr;
1565 llvm::Function *OutlinedFn =
1567 CGF, S, *CS->getCapturedDecl()->param_begin(), InnermostKind,
1568 CodeGen);
1569 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
1570 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1571 NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1572 /*IgnoreResultAssign=*/true);
1574 CGF, NumThreads, NumThreadsClause->getBeginLoc());
1575 }
1576 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
1577 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1579 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1580 }
1581 const Expr *IfCond = nullptr;
1582 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1583 if (C->getNameModifier() == OMPD_unknown ||
1584 C->getNameModifier() == OMPD_parallel) {
1585 IfCond = C->getCondition();
1586 break;
1587 }
1588 }
1589
1590 OMPParallelScope Scope(CGF, S);
1592 // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
1593 // lower and upper bounds with the pragma 'for' chunking mechanism.
1594 // The following lambda takes care of appending the lower and upper bound
1595 // parameters when necessary
1596 CodeGenBoundParameters(CGF, S, CapturedVars);
1597 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
1598 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn,
1599 CapturedVars, IfCond, NumThreads);
1600}
1601
1602static bool isAllocatableDecl(const VarDecl *VD) {
1603 const VarDecl *CVD = VD->getCanonicalDecl();
1604 if (!CVD->hasAttr<OMPAllocateDeclAttr>())
1605 return false;
1606 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
1607 // Use the default allocation.
1608 return !((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||
1609 AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&
1610 !AA->getAllocator());
1611}
1612
1614 const OMPExecutableDirective &,
1616
1618 const OMPExecutableDirective &S) {
1619 bool Copyins = CGF.EmitOMPCopyinClause(S);
1620 if (Copyins) {
1621 // Emit implicit barrier to synchronize threads and avoid data races on
1622 // propagation master's thread values of threadprivate variables to local
1623 // instances of that variables of all other implicit threads.
1625 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
1626 /*ForceSimpleCall=*/true);
1627 }
1628}
1629
1631 CodeGenFunction &CGF, const VarDecl *VD) {
1632 CodeGenModule &CGM = CGF.CGM;
1633 auto &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1634
1635 if (!VD)
1636 return Address::invalid();
1637 const VarDecl *CVD = VD->getCanonicalDecl();
1638 if (!isAllocatableDecl(CVD))
1639 return Address::invalid();
1640 llvm::Value *Size;
1641 CharUnits Align = CGM.getContext().getDeclAlign(CVD);
1642 if (CVD->getType()->isVariablyModifiedType()) {
1643 Size = CGF.getTypeSize(CVD->getType());
1644 // Align the size: ((size + align - 1) / align) * align
1645 Size = CGF.Builder.CreateNUWAdd(
1646 Size, CGM.getSize(Align - CharUnits::fromQuantity(1)));
1647 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align));
1648 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align));
1649 } else {
1651 Size = CGM.getSize(Sz.alignTo(Align));
1652 }
1653
1654 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
1655 assert(AA->getAllocator() &&
1656 "Expected allocator expression for non-default allocator.");
1657 llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator());
1658 // According to the standard, the original allocator type is a enum (integer).
1659 // Convert to pointer type, if required.
1660 if (Allocator->getType()->isIntegerTy())
1661 Allocator = CGF.Builder.CreateIntToPtr(Allocator, CGM.VoidPtrTy);
1662 else if (Allocator->getType()->isPointerTy())
1663 Allocator = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Allocator,
1664 CGM.VoidPtrTy);
1665
1666 llvm::Value *Addr = OMPBuilder.createOMPAlloc(
1667 CGF.Builder, Size, Allocator,
1668 getNameWithSeparators({CVD->getName(), ".void.addr"}, ".", "."));
1669 llvm::CallInst *FreeCI =
1670 OMPBuilder.createOMPFree(CGF.Builder, Addr, Allocator);
1671
1672 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(NormalAndEHCleanup, FreeCI);
1674 Addr,
1676 getNameWithSeparators({CVD->getName(), ".addr"}, ".", "."));
1677 return Address(Addr, CGF.ConvertTypeForMem(CVD->getType()), Align);
1678}
1679
1681 CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr,
1683 CodeGenModule &CGM = CGF.CGM;
1684 if (CGM.getLangOpts().OpenMPUseTLS &&
1686 return VDAddr;
1687
1688 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1689
1690 llvm::Type *VarTy = VDAddr.getElementType();
1691 llvm::Value *Data =
1692 CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.Int8PtrTy);
1693 llvm::ConstantInt *Size = CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy));
1694 std::string Suffix = getNameWithSeparators({"cache", ""});
1695 llvm::Twine CacheName = Twine(CGM.getMangledName(VD)).concat(Suffix);
1696
1697 llvm::CallInst *ThreadPrivateCacheCall =
1698 OMPBuilder.createCachedThreadPrivate(CGF.Builder, Data, Size, CacheName);
1699
1700 return Address(ThreadPrivateCacheCall, CGM.Int8Ty, VDAddr.getAlignment());
1701}
1702
1704 ArrayRef<StringRef> Parts, StringRef FirstSeparator, StringRef Separator) {
1705 SmallString<128> Buffer;
1706 llvm::raw_svector_ostream OS(Buffer);
1707 StringRef Sep = FirstSeparator;
1708 for (StringRef Part : Parts) {
1709 OS << Sep << Part;
1710 Sep = Separator;
1711 }
1712 return OS.str().str();
1713}
1714
1716 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
1717 InsertPointTy CodeGenIP, Twine RegionName) {
1719 Builder.restoreIP(CodeGenIP);
1720 llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder, /*CreateBranch=*/false,
1721 "." + RegionName + ".after");
1722
1723 {
1724 OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);
1725 CGF.EmitStmt(RegionBodyStmt);
1726 }
1727
1728 if (Builder.saveIP().isSet())
1729 Builder.CreateBr(FiniBB);
1730}
1731
1733 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
1734 InsertPointTy CodeGenIP, Twine RegionName) {
1736 Builder.restoreIP(CodeGenIP);
1737 llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder, /*CreateBranch=*/false,
1738 "." + RegionName + ".after");
1739
1740 {
1741 OMPBuilderCBHelpers::OutlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);
1742 CGF.EmitStmt(RegionBodyStmt);
1743 }
1744
1745 if (Builder.saveIP().isSet())
1746 Builder.CreateBr(FiniBB);
1747}
1748
1750 if (CGM.getLangOpts().OpenMPIRBuilder) {
1751 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1752 // Check if we have any if clause associated with the directive.
1753 llvm::Value *IfCond = nullptr;
1754 if (const auto *C = S.getSingleClause<OMPIfClause>())
1755 IfCond = EmitScalarExpr(C->getCondition(),
1756 /*IgnoreResultAssign=*/true);
1757
1758 llvm::Value *NumThreads = nullptr;
1759 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>())
1760 NumThreads = EmitScalarExpr(NumThreadsClause->getNumThreads(),
1761 /*IgnoreResultAssign=*/true);
1762
1763 ProcBindKind ProcBind = OMP_PROC_BIND_default;
1764 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>())
1765 ProcBind = ProcBindClause->getProcBindKind();
1766
1767 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1768
1769 // The cleanup callback that finalizes all variables at the given location,
1770 // thus calls destructors etc.
1771 auto FiniCB = [this](InsertPointTy IP) {
1773 };
1774
1775 // Privatization callback that performs appropriate action for
1776 // shared/private/firstprivate/lastprivate/copyin/... variables.
1777 //
1778 // TODO: This defaults to shared right now.
1779 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
1780 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
1781 // The next line is appropriate only for variables (Val) with the
1782 // data-sharing attribute "shared".
1783 ReplVal = &Val;
1784
1785 return CodeGenIP;
1786 };
1787
1788 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1789 const Stmt *ParallelRegionBodyStmt = CS->getCapturedStmt();
1790
1791 auto BodyGenCB = [&, this](InsertPointTy AllocaIP,
1792 InsertPointTy CodeGenIP) {
1794 *this, ParallelRegionBodyStmt, AllocaIP, CodeGenIP, "parallel");
1795 };
1796
1797 CGCapturedStmtInfo CGSI(*CS, CR_OpenMP);
1798 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
1799 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
1800 AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
1801 Builder.restoreIP(
1802 OMPBuilder.createParallel(Builder, AllocaIP, BodyGenCB, PrivCB, FiniCB,
1803 IfCond, NumThreads, ProcBind, S.hasCancel()));
1804 return;
1805 }
1806
1807 // Emit parallel region as a standalone region.
1808 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1809 Action.Enter(CGF);
1810 OMPPrivateScope PrivateScope(CGF);
1811 emitOMPCopyinClause(CGF, S);
1812 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1813 CGF.EmitOMPPrivateClause(S, PrivateScope);
1814 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1815 (void)PrivateScope.Privatize();
1816 CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1817 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
1818 };
1819 {
1820 auto LPCRegion =
1822 emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
1825 [](CodeGenFunction &) { return nullptr; });
1826 }
1827 // Check for outer lastprivate conditional update.
1829}
1830
1832 EmitStmt(S.getIfStmt());
1833}
1834
1835namespace {
1836/// RAII to handle scopes for loop transformation directives.
1837class OMPTransformDirectiveScopeRAII {
1838 OMPLoopScope *Scope = nullptr;
1839 CodeGenFunction::CGCapturedStmtInfo *CGSI = nullptr;
1840 CodeGenFunction::CGCapturedStmtRAII *CapInfoRAII = nullptr;
1841
1842 OMPTransformDirectiveScopeRAII(const OMPTransformDirectiveScopeRAII &) =
1843 delete;
1844 OMPTransformDirectiveScopeRAII &
1845 operator=(const OMPTransformDirectiveScopeRAII &) = delete;
1846
1847public:
1848 OMPTransformDirectiveScopeRAII(CodeGenFunction &CGF, const Stmt *S) {
1849 if (const auto *Dir = dyn_cast<OMPLoopBasedDirective>(S)) {
1850 Scope = new OMPLoopScope(CGF, *Dir);
1851 CGSI = new CodeGenFunction::CGCapturedStmtInfo(CR_OpenMP);
1852 CapInfoRAII = new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI);
1853 }
1854 }
1855 ~OMPTransformDirectiveScopeRAII() {
1856 if (!Scope)
1857 return;
1858 delete CapInfoRAII;
1859 delete CGSI;
1860 delete Scope;
1861 }
1862};
1863} // namespace
1864
1865static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
1866 int MaxLevel, int Level = 0) {
1867 assert(Level < MaxLevel && "Too deep lookup during loop body codegen.");
1868 const Stmt *SimplifiedS = S->IgnoreContainers();
1869 if (const auto *CS = dyn_cast<CompoundStmt>(SimplifiedS)) {
1870 PrettyStackTraceLoc CrashInfo(
1871 CGF.getContext().getSourceManager(), CS->getLBracLoc(),
1872 "LLVM IR generation of compound statement ('{}')");
1873
1874 // Keep track of the current cleanup stack depth, including debug scopes.
1875 CodeGenFunction::LexicalScope Scope(CGF, S->getSourceRange());
1876 for (const Stmt *CurStmt : CS->body())
1877 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level);
1878 return;
1879 }
1880 if (SimplifiedS == NextLoop) {
1881 if (auto *Dir = dyn_cast<OMPLoopTransformationDirective>(SimplifiedS))
1882 SimplifiedS = Dir->getTransformedStmt();
1883 if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(SimplifiedS))
1884 SimplifiedS = CanonLoop->getLoopStmt();
1885 if (const auto *For = dyn_cast<ForStmt>(SimplifiedS)) {
1886 S = For->getBody();
1887 } else {
1888 assert(isa<CXXForRangeStmt>(SimplifiedS) &&
1889 "Expected canonical for loop or range-based for loop.");
1890 const auto *CXXFor = cast<CXXForRangeStmt>(SimplifiedS);
1891 CGF.EmitStmt(CXXFor->getLoopVarStmt());
1892 S = CXXFor->getBody();
1893 }
1894 if (Level + 1 < MaxLevel) {
1896 S, /*TryImperfectlyNestedLoops=*/true);
1897 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1);
1898 return;
1899 }
1900 }
1901 CGF.EmitStmt(S);
1902}
1903
1905 JumpDest LoopExit) {
1906 RunCleanupsScope BodyScope(*this);
1907 // Update counters values on current iteration.
1908 for (const Expr *UE : D.updates())
1909 EmitIgnoredExpr(UE);
1910 // Update the linear variables.
1911 // In distribute directives only loop counters may be marked as linear, no
1912 // need to generate the code for them.
1913 if (!isOpenMPDistributeDirective(D.getDirectiveKind())) {
1914 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1915 for (const Expr *UE : C->updates())
1916 EmitIgnoredExpr(UE);
1917 }
1918 }
1919
1920 // On a continue in the body, jump to the end.
1921 JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue");
1922 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1923 for (const Expr *E : D.finals_conditions()) {
1924 if (!E)
1925 continue;
1926 // Check that loop counter in non-rectangular nest fits into the iteration
1927 // space.
1928 llvm::BasicBlock *NextBB = createBasicBlock("omp.body.next");
1929 EmitBranchOnBoolExpr(E, NextBB, Continue.getBlock(),
1931 EmitBlock(NextBB);
1932 }
1933
1934 OMPPrivateScope InscanScope(*this);
1935 EmitOMPReductionClauseInit(D, InscanScope, /*ForInscan=*/true);
1936 bool IsInscanRegion = InscanScope.Privatize();
1937 if (IsInscanRegion) {
1938 // Need to remember the block before and after scan directive
1939 // to dispatch them correctly depending on the clause used in
1940 // this directive, inclusive or exclusive. For inclusive scan the natural
1941 // order of the blocks is used, for exclusive clause the blocks must be
1942 // executed in reverse order.
1943 OMPBeforeScanBlock = createBasicBlock("omp.before.scan.bb");
1944 OMPAfterScanBlock = createBasicBlock("omp.after.scan.bb");
1945 // No need to allocate inscan exit block, in simd mode it is selected in the
1946 // codegen for the scan directive.
1947 if (D.getDirectiveKind() != OMPD_simd && !getLangOpts().OpenMPSimd)
1948 OMPScanExitBlock = createBasicBlock("omp.exit.inscan.bb");
1949 OMPScanDispatch = createBasicBlock("omp.inscan.dispatch");
1952 }
1953
1954 // Emit loop variables for C++ range loops.
1955 const Stmt *Body =
1956 D.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers();
1957 // Emit loop body.
1958 emitBody(*this, Body,
1960 Body, /*TryImperfectlyNestedLoops=*/true),
1961 D.getLoopsNumber());
1962
1963 // Jump to the dispatcher at the end of the loop body.
1964 if (IsInscanRegion)
1966
1967 // The end (updates/cleanups).
1968 EmitBlock(Continue.getBlock());
1969 BreakContinueStack.pop_back();
1970}
1971
1972using EmittedClosureTy = std::pair<llvm::Function *, llvm::Value *>;
1973
1974/// Emit a captured statement and return the function as well as its captured
1975/// closure context.
1977 const CapturedStmt *S) {
1978 LValue CapStruct = ParentCGF.InitCapturedStruct(*S);
1979 CodeGenFunction CGF(ParentCGF.CGM, /*suppressNewContext=*/true);
1980 std::unique_ptr<CodeGenFunction::CGCapturedStmtInfo> CSI =
1981 std::make_unique<CodeGenFunction::CGCapturedStmtInfo>(*S);
1982 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, CSI.get());
1983 llvm::Function *F = CGF.GenerateCapturedStmtFunction(*S);
1984
1985 return {F, CapStruct.getPointer(ParentCGF)};
1986}
1987
1988/// Emit a call to a previously captured closure.
1989static llvm::CallInst *
1992 // Append the closure context to the argument.
1993 SmallVector<llvm::Value *> EffectiveArgs;
1994 EffectiveArgs.reserve(Args.size() + 1);
1995 llvm::append_range(EffectiveArgs, Args);
1996 EffectiveArgs.push_back(Cap.second);
1997
1998 return ParentCGF.Builder.CreateCall(Cap.first, EffectiveArgs);
1999}
2000
2001llvm::CanonicalLoopInfo *
2003 assert(Depth == 1 && "Nested loops with OpenMPIRBuilder not yet implemented");
2004
2005 // The caller is processing the loop-associated directive processing the \p
2006 // Depth loops nested in \p S. Put the previous pending loop-associated
2007 // directive to the stack. If the current loop-associated directive is a loop
2008 // transformation directive, it will push its generated loops onto the stack
2009 // such that together with the loops left here they form the combined loop
2010 // nest for the parent loop-associated directive.
2011 int ParentExpectedOMPLoopDepth = ExpectedOMPLoopDepth;
2012 ExpectedOMPLoopDepth = Depth;
2013
2014 EmitStmt(S);
2015 assert(OMPLoopNestStack.size() >= (size_t)Depth && "Found too few loops");
2016
2017 // The last added loop is the outermost one.
2018 llvm::CanonicalLoopInfo *Result = OMPLoopNestStack.back();
2019
2020 // Pop the \p Depth loops requested by the call from that stack and restore
2021 // the previous context.
2022 OMPLoopNestStack.pop_back_n(Depth);
2023 ExpectedOMPLoopDepth = ParentExpectedOMPLoopDepth;
2024
2025 return Result;
2026}
2027
2029 const Stmt *SyntacticalLoop = S->getLoopStmt();
2030 if (!getLangOpts().OpenMPIRBuilder) {
2031 // Ignore if OpenMPIRBuilder is not enabled.
2032 EmitStmt(SyntacticalLoop);
2033 return;
2034 }
2035
2036 LexicalScope ForScope(*this, S->getSourceRange());
2037
2038 // Emit init statements. The Distance/LoopVar funcs may reference variable
2039 // declarations they contain.
2040 const Stmt *BodyStmt;
2041 if (const auto *For = dyn_cast<ForStmt>(SyntacticalLoop)) {
2042 if (const Stmt *InitStmt = For->getInit())
2043 EmitStmt(InitStmt);
2044 BodyStmt = For->getBody();
2045 } else if (const auto *RangeFor =
2046 dyn_cast<CXXForRangeStmt>(SyntacticalLoop)) {
2047 if (const DeclStmt *RangeStmt = RangeFor->getRangeStmt())
2048 EmitStmt(RangeStmt);
2049 if (const DeclStmt *BeginStmt = RangeFor->getBeginStmt())
2050 EmitStmt(BeginStmt);
2051 if (const DeclStmt *EndStmt = RangeFor->getEndStmt())
2052 EmitStmt(EndStmt);
2053 if (const DeclStmt *LoopVarStmt = RangeFor->getLoopVarStmt())
2054 EmitStmt(LoopVarStmt);
2055 BodyStmt = RangeFor->getBody();
2056 } else
2057 llvm_unreachable("Expected for-stmt or range-based for-stmt");
2058
2059 // Emit closure for later use. By-value captures will be captured here.
2060 const CapturedStmt *DistanceFunc = S->getDistanceFunc();
2061 EmittedClosureTy DistanceClosure = emitCapturedStmtFunc(*this, DistanceFunc);
2062 const CapturedStmt *LoopVarFunc = S->getLoopVarFunc();
2063 EmittedClosureTy LoopVarClosure = emitCapturedStmtFunc(*this, LoopVarFunc);
2064
2065 // Call the distance function to get the number of iterations of the loop to
2066 // come.
2067 QualType LogicalTy = DistanceFunc->getCapturedDecl()
2068 ->getParam(0)
2069 ->getType()
2071 RawAddress CountAddr = CreateMemTemp(LogicalTy, ".count.addr");
2072 emitCapturedStmtCall(*this, DistanceClosure, {CountAddr.getPointer()});
2073 llvm::Value *DistVal = Builder.CreateLoad(CountAddr, ".count");
2074
2075 // Emit the loop structure.
2076 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2077 auto BodyGen = [&, this](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP,
2078 llvm::Value *IndVar) {
2079 Builder.restoreIP(CodeGenIP);
2080
2081 // Emit the loop body: Convert the logical iteration number to the loop
2082 // variable and emit the body.
2083 const DeclRefExpr *LoopVarRef = S->getLoopVarRef();
2084 LValue LCVal = EmitLValue(LoopVarRef);
2085 Address LoopVarAddress = LCVal.getAddress();
2086 emitCapturedStmtCall(*this, LoopVarClosure,
2087 {LoopVarAddress.emitRawPointer(*this), IndVar});
2088
2089 RunCleanupsScope BodyScope(*this);
2090 EmitStmt(BodyStmt);
2091 };
2092 llvm::CanonicalLoopInfo *CL =
2093 OMPBuilder.createCanonicalLoop(Builder, BodyGen, DistVal);
2094
2095 // Finish up the loop.
2096 Builder.restoreIP(CL->getAfterIP());
2097 ForScope.ForceCleanup();
2098
2099 // Remember the CanonicalLoopInfo for parent AST nodes consuming it.
2100 OMPLoopNestStack.push_back(CL);
2101}
2102
2104 const OMPExecutableDirective &S, bool RequiresCleanup, const Expr *LoopCond,
2105 const Expr *IncExpr,
2106 const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
2107 const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
2108 auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
2109
2110 // Start the loop with a block that tests the condition.
2111 auto CondBlock = createBasicBlock("omp.inner.for.cond");
2112 EmitBlock(CondBlock);
2113 const SourceRange R = S.getSourceRange();
2114
2115 // If attributes are attached, push to the basic block with them.
2116 const auto &OMPED = cast<OMPExecutableDirective>(S);
2117 const CapturedStmt *ICS = OMPED.getInnermostCapturedStmt();
2118 const Stmt *SS = ICS->getCapturedStmt();
2119 const AttributedStmt *AS = dyn_cast_or_null<AttributedStmt>(SS);
2120 OMPLoopNestStack.clear();
2121 if (AS)
2125 else
2128
2129 // If there are any cleanups between here and the loop-exit scope,
2130 // create a block to stage a loop exit along.
2131 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
2132 if (RequiresCleanup)
2133 ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
2134
2135 llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body");
2136
2137 // Emit condition.
2138 EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
2139 if (ExitBlock != LoopExit.getBlock()) {
2140 EmitBlock(ExitBlock);
2142 }
2143
2144 EmitBlock(LoopBody);
2146
2147 // Create a block for the increment.
2148 JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
2149 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
2150
2151 BodyGen(*this);
2152
2153 // Emit "IV = IV + 1" and a back-edge to the condition block.
2154 EmitBlock(Continue.getBlock());
2155 EmitIgnoredExpr(IncExpr);
2156 PostIncGen(*this);
2157 BreakContinueStack.pop_back();
2158 EmitBranch(CondBlock);
2159 LoopStack.pop();
2160 // Emit the fall-through block.
2161 EmitBlock(LoopExit.getBlock());
2162}
2163
2165 if (!HaveInsertPoint())
2166 return false;
2167 // Emit inits for the linear variables.
2168 bool HasLinears = false;
2169 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2170 for (const Expr *Init : C->inits()) {
2171 HasLinears = true;
2172 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
2173 if (const auto *Ref =
2174 dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
2175 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
2176 const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
2177 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
2178 CapturedStmtInfo->lookup(OrigVD) != nullptr,
2179 VD->getInit()->getType(), VK_LValue,
2180 VD->getInit()->getExprLoc());
2182 &DRE, VD,
2183 MakeAddrLValue(Emission.getAllocatedAddress(), VD->getType()),
2184 /*capturedByInit=*/false);
2185 EmitAutoVarCleanups(Emission);
2186 } else {
2187 EmitVarDecl(*VD);
2188 }
2189 }
2190 // Emit the linear steps for the linear clauses.
2191 // If a step is not constant, it is pre-calculated before the loop.
2192 if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
2193 if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
2194 EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
2195 // Emit calculation of the linear step.
2196 EmitIgnoredExpr(CS);
2197 }
2198 }
2199 return HasLinears;
2200}
2201
2203 const OMPLoopDirective &D,
2204 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
2205 if (!HaveInsertPoint())
2206 return;
2207 llvm::BasicBlock *DoneBB = nullptr;
2208 // Emit the final values of the linear variables.
2209 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2210 auto IC = C->varlist_begin();
2211 for (const Expr *F : C->finals()) {
2212 if (!DoneBB) {
2213 if (llvm::Value *Cond = CondGen(*this)) {
2214 // If the first post-update expression is found, emit conditional
2215 // block if it was requested.
2216 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu");
2217 DoneBB = createBasicBlock(".omp.linear.pu.done");
2218 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2219 EmitBlock(ThenBB);
2220 }
2221 }
2222 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
2223 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
2224 CapturedStmtInfo->lookup(OrigVD) != nullptr,
2225 (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
2226 Address OrigAddr = EmitLValue(&DRE).getAddress();
2227 CodeGenFunction::OMPPrivateScope VarScope(*this);
2228 VarScope.addPrivate(OrigVD, OrigAddr);
2229 (void)VarScope.Privatize();
2230 EmitIgnoredExpr(F);
2231 ++IC;
2232 }
2233 if (const Expr *PostUpdate = C->getPostUpdateExpr())
2234 EmitIgnoredExpr(PostUpdate);
2235 }
2236 if (DoneBB)
2237 EmitBlock(DoneBB, /*IsFinished=*/true);
2238}
2239
2241 const OMPExecutableDirective &D) {
2242 if (!CGF.HaveInsertPoint())
2243 return;
2244 for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
2245 llvm::APInt ClauseAlignment(64, 0);
2246 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
2247 auto *AlignmentCI =
2248 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
2249 ClauseAlignment = AlignmentCI->getValue();
2250 }
2251 for (const Expr *E : Clause->varlists()) {
2252 llvm::APInt Alignment(ClauseAlignment);
2253 if (Alignment == 0) {
2254 // OpenMP [2.8.1, Description]
2255 // If no optional parameter is specified, implementation-defined default
2256 // alignments for SIMD instructions on the target platforms are assumed.
2257 Alignment =
2258 CGF.getContext()
2260 E->getType()->getPointeeType()))
2261 .getQuantity();
2262 }
2263 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2264 "alignment is not power of 2");
2265 if (Alignment != 0) {
2266 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
2268 PtrValue, E, /*No second loc needed*/ SourceLocation(),
2269 llvm::ConstantInt::get(CGF.getLLVMContext(), Alignment));
2270 }
2271 }
2272 }
2273}
2274
2276 const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
2277 if (!HaveInsertPoint())
2278 return;
2279 auto I = S.private_counters().begin();
2280 for (const Expr *E : S.counters()) {
2281 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2282 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
2283 // Emit var without initialization.
2284 AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD);
2285 EmitAutoVarCleanups(VarEmission);
2286 LocalDeclMap.erase(PrivateVD);
2287 (void)LoopScope.addPrivate(VD, VarEmission.getAllocatedAddress());
2288 if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
2289 VD->hasGlobalStorage()) {
2290 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD),
2291 LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
2292 E->getType(), VK_LValue, E->getExprLoc());
2293 (void)LoopScope.addPrivate(PrivateVD, EmitLValue(&DRE).getAddress());
2294 } else {
2295 (void)LoopScope.addPrivate(PrivateVD, VarEmission.getAllocatedAddress());
2296 }
2297 ++I;
2298 }
2299 // Privatize extra loop counters used in loops for ordered(n) clauses.
2300 for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) {
2301 if (!C->getNumForLoops())
2302 continue;
2303 for (unsigned I = S.getLoopsNumber(), E = C->getLoopNumIterations().size();
2304 I < E; ++I) {
2305 const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I));
2306 const auto *VD = cast<VarDecl>(DRE->getDecl());
2307 // Override only those variables that can be captured to avoid re-emission
2308 // of the variables declared within the loops.
2309 if (DRE->refersToEnclosingVariableOrCapture()) {
2310 (void)LoopScope.addPrivate(
2311 VD, CreateMemTemp(DRE->getType(), VD->getName()));
2312 }
2313 }
2314 }
2315}
2316
2318 const Expr *Cond, llvm::BasicBlock *TrueBlock,
2319 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
2320 if (!CGF.HaveInsertPoint())
2321 return;
2322 {
2323 CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
2324 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
2325 (void)PreCondScope.Privatize();
2326 // Get initial values of real counters.
2327 for (const Expr *I : S.inits()) {
2328 CGF.EmitIgnoredExpr(I);
2329 }
2330 }
2331 // Create temp loop control variables with their init values to support
2332 // non-rectangular loops.
2333 CodeGenFunction::OMPMapVars PreCondVars;
2334 for (const Expr *E : S.dependent_counters()) {
2335 if (!E)
2336 continue;
2337 assert(!E->getType().getNonReferenceType()->isRecordType() &&
2338 "dependent counter must not be an iterator.");
2339 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2340 Address CounterAddr =
2342 (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr);
2343 }
2344 (void)PreCondVars.apply(CGF);
2345 for (const Expr *E : S.dependent_inits()) {
2346 if (!E)
2347 continue;
2348 CGF.EmitIgnoredExpr(E);
2349 }
2350 // Check that loop is executed at least one time.
2351 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
2352 PreCondVars.restore(CGF);
2353}
2354
2356 const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
2357 if (!HaveInsertPoint())
2358 return;
2360 if (isOpenMPSimdDirective(D.getDirectiveKind())) {
2361 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
2362 for (const Expr *C : LoopDirective->counters()) {
2363 SIMDLCVs.insert(
2364 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
2365 }
2366 }
2367 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2368 auto CurPrivate = C->privates().begin();
2369 for (const Expr *E : C->varlists()) {
2370 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2371 const auto *PrivateVD =
2372 cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
2373 if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
2374 // Emit private VarDecl with copy init.
2375 EmitVarDecl(*PrivateVD);
2376 bool IsRegistered =
2377 PrivateScope.addPrivate(VD, GetAddrOfLocalVar(PrivateVD));
2378 assert(IsRegistered && "linear var already registered as private");
2379 // Silence the warning about unused variable.
2380 (void)IsRegistered;
2381 } else {
2382 EmitVarDecl(*PrivateVD);
2383 }
2384 ++CurPrivate;
2385 }
2386 }
2387}
2388
2390 const OMPExecutableDirective &D) {
2391 if (!CGF.HaveInsertPoint())
2392 return;
2393 if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
2394 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
2395 /*ignoreResult=*/true);
2396 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
2397 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
2398 // In presence of finite 'safelen', it may be unsafe to mark all
2399 // the memory instructions parallel, because loop-carried
2400 // dependences of 'safelen' iterations are possible.
2401 CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>());
2402 } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
2403 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
2404 /*ignoreResult=*/true);
2405 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
2406 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
2407 // In presence of finite 'safelen', it may be unsafe to mark all
2408 // the memory instructions parallel, because loop-carried
2409 // dependences of 'safelen' iterations are possible.
2410 CGF.LoopStack.setParallel(/*Enable=*/false);
2411 }
2412}
2413
2415 // Walk clauses and process safelen/lastprivate.
2416 LoopStack.setParallel(/*Enable=*/true);
2419 if (const auto *C = D.getSingleClause<OMPOrderClause>())
2420 if (C->getKind() == OMPC_ORDER_concurrent)
2421 LoopStack.setParallel(/*Enable=*/true);
2422 if ((D.getDirectiveKind() == OMPD_simd ||
2423 (getLangOpts().OpenMPSimd &&
2424 isOpenMPSimdDirective(D.getDirectiveKind()))) &&
2425 llvm::any_of(D.getClausesOfKind<OMPReductionClause>(),
2426 [](const OMPReductionClause *C) {
2427 return C->getModifier() == OMPC_REDUCTION_inscan;
2428 }))
2429 // Disable parallel access in case of prefix sum.
2430 LoopStack.setParallel(/*Enable=*/false);
2431}
2432
2434 const OMPLoopDirective &D,
2435 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
2436 if (!HaveInsertPoint())
2437 return;
2438 llvm::BasicBlock *DoneBB = nullptr;
2439 auto IC = D.counters().begin();
2440 auto IPC = D.private_counters().begin();
2441 for (const Expr *F : D.finals()) {
2442 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
2443 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
2444 const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
2445 if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
2446 OrigVD->hasGlobalStorage() || CED) {
2447 if (!DoneBB) {
2448 if (llvm::Value *Cond = CondGen(*this)) {
2449 // If the first post-update expression is found, emit conditional
2450 // block if it was requested.
2451 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then");
2452 DoneBB = createBasicBlock(".omp.final.done");
2453 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2454 EmitBlock(ThenBB);
2455 }
2456 }
2457 Address OrigAddr = Address::invalid();
2458 if (CED) {
2459 OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();
2460 } else {
2461 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(PrivateVD),
2462 /*RefersToEnclosingVariableOrCapture=*/false,
2463 (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
2464 OrigAddr = EmitLValue(&DRE).getAddress();
2465 }
2466 OMPPrivateScope VarScope(*this);
2467 VarScope.addPrivate(OrigVD, OrigAddr);
2468 (void)VarScope.Privatize();
2469 EmitIgnoredExpr(F);
2470 }
2471 ++IC;
2472 ++IPC;
2473 }
2474 if (DoneBB)
2475 EmitBlock(DoneBB, /*IsFinished=*/true);
2476}
2477
2479 const OMPLoopDirective &S,
2480 CodeGenFunction::JumpDest LoopExit) {
2481 CGF.EmitOMPLoopBody(S, LoopExit);
2482 CGF.EmitStopPoint(&S);
2483}
2484
2485/// Emit a helper variable and return corresponding lvalue.
2487 const DeclRefExpr *Helper) {
2488 auto VDecl = cast<VarDecl>(Helper->getDecl());
2489 CGF.EmitVarDecl(*VDecl);
2490 return CGF.EmitLValue(Helper);
2491}
2492
2494 const RegionCodeGenTy &SimdInitGen,
2495 const RegionCodeGenTy &BodyCodeGen) {
2496 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](CodeGenFunction &CGF,
2497 PrePostActionTy &) {
2498 CGOpenMPRuntime::NontemporalDeclsRAII NontemporalsRegion(CGF.CGM, S);
2499 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
2500 SimdInitGen(CGF);
2501
2502 BodyCodeGen(CGF);
2503 };
2504 auto &&ElseGen = [&BodyCodeGen](CodeGenFunction &CGF, PrePostActionTy &) {
2505 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
2506 CGF.LoopStack.setVectorizeEnable(/*Enable=*/false);
2507
2508 BodyCodeGen(CGF);
2509 };
2510 const Expr *IfCond = nullptr;
2511 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
2512 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
2513 if (CGF.getLangOpts().OpenMP >= 50 &&
2514 (C->getNameModifier() == OMPD_unknown ||
2515 C->getNameModifier() == OMPD_simd)) {
2516 IfCond = C->getCondition();
2517 break;
2518 }
2519 }
2520 }
2521 if (IfCond) {
2522 CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, IfCond, ThenGen, ElseGen);
2523 } else {
2524 RegionCodeGenTy ThenRCG(ThenGen);
2525 ThenRCG(CGF);
2526 }
2527}
2528
2530 PrePostActionTy &Action) {
2531 Action.Enter(CGF);
2532 assert(isOpenMPSimdDirective(S.getDirectiveKind()) &&
2533 "Expected simd directive");
2534 OMPLoopScope PreInitScope(CGF, S);
2535 // if (PreCond) {
2536 // for (IV in 0..LastIteration) BODY;
2537 // <Final counter/linear vars updates>;
2538 // }
2539 //
2540 if (isOpenMPDistributeDirective(S.getDirectiveKind()) ||
2541 isOpenMPWorksharingDirective(S.getDirectiveKind()) ||
2542 isOpenMPTaskLoopDirective(S.getDirectiveKind())) {
2543 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));
2544 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));
2545 }
2546
2547 // Emit: if (PreCond) - begin.
2548 // If the condition constant folds and can be elided, avoid emitting the
2549 // whole loop.
2550 bool CondConstant;
2551 llvm::BasicBlock *ContBlock = nullptr;
2552 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2553 if (!CondConstant)
2554 return;
2555 } else {
2556 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");
2557 ContBlock = CGF.createBasicBlock("simd.if.end");
2558 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
2559 CGF.getProfileCount(&S));
2560 CGF.EmitBlock(ThenBlock);
2562 }
2563
2564 // Emit the loop iteration variable.
2565 const Expr *IVExpr = S.getIterationVariable();
2566 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
2567 CGF.EmitVarDecl(*IVDecl);
2568 CGF.EmitIgnoredExpr(S.getInit());
2569
2570 // Emit the iterations count variable.
2571 // If it is not a variable, Sema decided to calculate iterations count on
2572 // each iteration (e.g., it is foldable into a constant).
2573 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2574 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2575 // Emit calculation of the iterations count.
2576 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
2577 }
2578
2579 emitAlignedClause(CGF, S);
2580 (void)CGF.EmitOMPLinearClauseInit(S);
2581 {
2582 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2583 CGF.EmitOMPPrivateClause(S, LoopScope);
2584 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
2585 CGF.EmitOMPLinearClause(S, LoopScope);
2586 CGF.EmitOMPReductionClauseInit(S, LoopScope);
2588 CGF, S, CGF.EmitLValue(S.getIterationVariable()));
2589 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2590 (void)LoopScope.Privatize();
2591 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
2593
2595 CGF, S,
2596 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2597 CGF.EmitOMPSimdInit(S);
2598 },
2599 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
2600 CGF.EmitOMPInnerLoop(
2601 S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),
2602 [&S](CodeGenFunction &CGF) {
2603 emitOMPLoopBodyWithStopPoint(CGF, S,
2604 CodeGenFunction::JumpDest());
2605 },
2606 [](CodeGenFunction &) {});
2607 });
2608 CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; });
2609 // Emit final copy of the lastprivate variables at the end of loops.
2610 if (HasLastprivateClause)
2611 CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
2612 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
2614 [](CodeGenFunction &) { return nullptr; });
2615 LoopScope.restoreMap();
2616 CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; });
2617 }
2618 // Emit: if (PreCond) - end.
2619 if (ContBlock) {
2620 CGF.EmitBranch(ContBlock);
2621 CGF.EmitBlock(ContBlock, true);
2622 }
2623}
2624
2626 // Check for unsupported clauses
2627 for (OMPClause *C : S.clauses()) {
2628 // Currently only order, simdlen and safelen clauses are supported
2629 if (!(isa<OMPSimdlenClause>(C) || isa<OMPSafelenClause>(C) ||
2630 isa<OMPOrderClause>(C) || isa<OMPAlignedClause>(C)))
2631 return false;
2632 }
2633
2634 // Check if we have a statement with the ordered directive.
2635 // Visit the statement hierarchy to find a compound statement
2636 // with a ordered directive in it.
2637 if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(S.getRawStmt())) {
2638 if (const Stmt *SyntacticalLoop = CanonLoop->getLoopStmt()) {
2639 for (const Stmt *SubStmt : SyntacticalLoop->children()) {
2640 if (!SubStmt)
2641 continue;
2642 if (const CompoundStmt *CS = dyn_cast<CompoundStmt>(SubStmt)) {
2643 for (const Stmt *CSSubStmt : CS->children()) {
2644 if (!CSSubStmt)
2645 continue;
2646 if (isa<OMPOrderedDirective>(CSSubStmt)) {
2647 return false;
2648 }
2649 }
2650 }
2651 }
2652 }
2653 }
2654 return true;
2655}
2656static llvm::MapVector<llvm::Value *, llvm::Value *>
2658 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars;
2659 for (const auto *Clause : S.getClausesOfKind<OMPAlignedClause>()) {
2660 llvm::APInt ClauseAlignment(64, 0);
2661 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
2662 auto *AlignmentCI =
2663 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
2664 ClauseAlignment = AlignmentCI->getValue();
2665 }
2666 for (const Expr *E : Clause->varlists()) {
2667 llvm::APInt Alignment(ClauseAlignment);
2668 if (Alignment == 0) {
2669 // OpenMP [2.8.1, Description]
2670 // If no optional parameter is specified, implementation-defined default
2671 // alignments for SIMD instructions on the target platforms are assumed.
2672 Alignment =
2673 CGF.getContext()
2675 E->getType()->getPointeeType()))
2676 .getQuantity();
2677 }
2678 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2679 "alignment is not power of 2");
2680 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
2681 AlignedVars[PtrValue] = CGF.Builder.getInt64(Alignment.getSExtValue());
2682 }
2683 }
2684 return AlignedVars;
2685}
2686
2688 bool UseOMPIRBuilder =
2689 CGM.getLangOpts().OpenMPIRBuilder && isSupportedByOpenMPIRBuilder(S);
2690 if (UseOMPIRBuilder) {
2691 auto &&CodeGenIRBuilder = [this, &S, UseOMPIRBuilder](CodeGenFunction &CGF,
2692 PrePostActionTy &) {
2693 // Use the OpenMPIRBuilder if enabled.
2694 if (UseOMPIRBuilder) {
2695 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars =
2696 GetAlignedMapping(S, CGF);
2697 // Emit the associated statement and get its loop representation.
2698 const Stmt *Inner = S.getRawStmt();
2699 llvm::CanonicalLoopInfo *CLI =
2701
2702 llvm::OpenMPIRBuilder &OMPBuilder =
2704 // Add SIMD specific metadata
2705 llvm::ConstantInt *Simdlen = nullptr;
2706 if (const auto *C = S.getSingleClause<OMPSimdlenClause>()) {
2707 RValue Len =
2708 this->EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
2709 /*ignoreResult=*/true);
2710 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
2711 Simdlen = Val;
2712 }
2713 llvm::ConstantInt *Safelen = nullptr;
2714 if (const auto *C = S.getSingleClause<OMPSafelenClause>()) {
2715 RValue Len =
2716 this->EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
2717 /*ignoreResult=*/true);
2718 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
2719 Safelen = Val;
2720 }
2721 llvm::omp::OrderKind Order = llvm::omp::OrderKind::OMP_ORDER_unknown;
2722 if (const auto *C = S.getSingleClause<OMPOrderClause>()) {
2723 if (C->getKind() == OpenMPOrderClauseKind ::OMPC_ORDER_concurrent) {
2724 Order = llvm::omp::OrderKind::OMP_ORDER_concurrent;
2725 }
2726 }
2727 // Add simd metadata to the collapsed loop. Do not generate
2728 // another loop for if clause. Support for if clause is done earlier.
2729 OMPBuilder.applySimd(CLI, AlignedVars,
2730 /*IfCond*/ nullptr, Order, Simdlen, Safelen);
2731 return;
2732 }
2733 };
2734 {
2735 auto LPCRegion =
2737 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2738 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd,
2739 CodeGenIRBuilder);
2740 }
2741 return;
2742 }
2743
2744 ParentLoopDirectiveForScanRegion ScanRegion(*this, S);
2745 OMPFirstScanLoop = true;
2746 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2747 emitOMPSimdRegion(CGF, S, Action);
2748 };
2749 {
2750 auto LPCRegion =
2752 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2753 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2754 }
2755 // Check for outer lastprivate conditional update.
2757}
2758
2760 // Emit the de-sugared statement.
2761 OMPTransformDirectiveScopeRAII TileScope(*this, &S);
2762 EmitStmt(S.getTransformedStmt());
2763}
2764
2766 bool UseOMPIRBuilder = CGM.getLangOpts().OpenMPIRBuilder;
2767
2768 if (UseOMPIRBuilder) {
2769 auto DL = SourceLocToDebugLoc(S.getBeginLoc());
2770 const Stmt *Inner = S.getRawStmt();
2771
2772 // Consume nested loop. Clear the entire remaining loop stack because a
2773 // fully unrolled loop is non-transformable. For partial unrolling the
2774 // generated outer loop is pushed back to the stack.
2775 llvm::CanonicalLoopInfo *CLI = EmitOMPCollapsedCanonicalLoopNest(Inner, 1);
2776 OMPLoopNestStack.clear();
2777
2778 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2779
2780 bool NeedsUnrolledCLI = ExpectedOMPLoopDepth >= 1;
2781 llvm::CanonicalLoopInfo *UnrolledCLI = nullptr;
2782
2783 if (S.hasClausesOfKind<OMPFullClause>()) {
2784 assert(ExpectedOMPLoopDepth == 0);
2785 OMPBuilder.unrollLoopFull(DL, CLI);
2786 } else if (auto *PartialClause = S.getSingleClause<OMPPartialClause>()) {
2787 uint64_t Factor = 0;
2788 if (Expr *FactorExpr = PartialClause->getFactor()) {
2789 Factor = FactorExpr->EvaluateKnownConstInt(getContext()).getZExtValue();
2790 assert(Factor >= 1 && "Only positive factors are valid");
2791 }
2792 OMPBuilder.unrollLoopPartial(DL, CLI, Factor,
2793 NeedsUnrolledCLI ? &UnrolledCLI : nullptr);
2794 } else {
2795 OMPBuilder.unrollLoopHeuristic(DL, CLI);
2796 }
2797
2798 assert((!NeedsUnrolledCLI || UnrolledCLI) &&
2799 "NeedsUnrolledCLI implies UnrolledCLI to be set");
2800 if (UnrolledCLI)
2801 OMPLoopNestStack.push_back(UnrolledCLI);
2802
2803 return;
2804 }
2805
2806 // This function is only called if the unrolled loop is not consumed by any
2807 // other loop-associated construct. Such a loop-associated construct will have
2808 // used the transformed AST.
2809
2810 // Set the unroll metadata for the next emitted loop.
2812
2813 if (S.hasClausesOfKind<OMPFullClause>()) {
2815 } else if (auto *PartialClause = S.getSingleClause<OMPPartialClause>()) {
2816 if (Expr *FactorExpr = PartialClause->getFactor()) {
2817 uint64_t Factor =
2818 FactorExpr->EvaluateKnownConstInt(getContext()).getZExtValue();
2819 assert(Factor >= 1 && "Only positive factors are valid");
2820 LoopStack.setUnrollCount(Factor);
2821 }
2822 }
2823
2824 EmitStmt(S.getAssociatedStmt());
2825}
2826
2827void CodeGenFunction::EmitOMPOuterLoop(
2828 bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
2829 CodeGenFunction::OMPPrivateScope &LoopScope,
2830 const CodeGenFunction::OMPLoopArguments &LoopArgs,
2831 const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
2832 const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
2834
2835 const Expr *IVExpr = S.getIterationVariable();
2836 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2837 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2838
2839 JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
2840
2841 // Start the loop with a block that tests the condition.
2842 llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond");
2843 EmitBlock(CondBlock);
2844 const SourceRange R = S.getSourceRange();
2845 OMPLoopNestStack.clear();
2848
2849 llvm::Value *BoolCondVal = nullptr;
2850 if (!DynamicOrOrdered) {
2851 // UB = min(UB, GlobalUB) or
2852 // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
2853 // 'distribute parallel for')
2854 EmitIgnoredExpr(LoopArgs.EUB);
2855 // IV = LB
2856 EmitIgnoredExpr(LoopArgs.Init);
2857 // IV < UB
2858 BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
2859 } else {
2860 BoolCondVal =
2861 RT.emitForNext(*this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
2862 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
2863 }
2864
2865 // If there are any cleanups between here and the loop-exit scope,
2866 // create a block to stage a loop exit along.
2867 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
2868 if (LoopScope.requiresCleanups())
2869 ExitBlock = createBasicBlock("omp.dispatch.cleanup");
2870
2871 llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body");
2872 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
2873 if (ExitBlock != LoopExit.getBlock()) {
2874 EmitBlock(ExitBlock);
2876 }
2877 EmitBlock(LoopBody);
2878
2879 // Emit "IV = LB" (in case of static schedule, we have already calculated new
2880 // LB for loop condition and emitted it above).
2881 if (DynamicOrOrdered)
2882 EmitIgnoredExpr(LoopArgs.Init);
2883
2884 // Create a block for the increment.
2885 JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
2886 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
2887
2889 *this, S,
2890 [&S, IsMonotonic](CodeGenFunction &CGF, PrePostActionTy &) {
2891 // Generate !llvm.loop.parallel metadata for loads and stores for loops
2892 // with dynamic/guided scheduling and without ordered clause.
2893 if (!isOpenMPSimdDirective(S.getDirectiveKind())) {
2894 CGF.LoopStack.setParallel(!IsMonotonic);
2895 if (const auto *C = S.getSingleClause<OMPOrderClause>())
2896 if (C->getKind() == OMPC_ORDER_concurrent)
2897 CGF.LoopStack.setParallel(/*Enable=*/true);
2898 } else {
2899 CGF.EmitOMPSimdInit(S);
2900 }
2901 },
2902 [&S, &LoopArgs, LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered,
2903 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
2904 SourceLocation Loc = S.getBeginLoc();
2905 // when 'distribute' is not combined with a 'for':
2906 // while (idx <= UB) { BODY; ++idx; }
2907 // when 'distribute' is combined with a 'for'
2908 // (e.g. 'distribute parallel for')
2909 // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
2910 CGF.EmitOMPInnerLoop(
2911 S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
2912 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
2913 CodeGenLoop(CGF, S, LoopExit);
2914 },
2915 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
2916 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
2917 });
2918 });
2919
2920 EmitBlock(Continue.getBlock());
2921 BreakContinueStack.pop_back();
2922 if (!DynamicOrOrdered) {
2923 // Emit "LB = LB + Stride", "UB = UB + Stride".
2924 EmitIgnoredExpr(LoopArgs.NextLB);
2925 EmitIgnoredExpr(LoopArgs.NextUB);
2926 }
2927
2928 EmitBranch(CondBlock);
2929 OMPLoopNestStack.clear();
2930 LoopStack.pop();
2931 // Emit the fall-through block.
2932 EmitBlock(LoopExit.getBlock());
2933
2934 // Tell the runtime we are done.
2935 auto &&CodeGen = [DynamicOrOrdered, &S, &LoopArgs](CodeGenFunction &CGF) {
2936 if (!DynamicOrOrdered)
2937 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2938 LoopArgs.DKind);
2939 };
2940 OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
2941}
2942
2943void CodeGenFunction::EmitOMPForOuterLoop(
2944 const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
2945 const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
2946 const OMPLoopArguments &LoopArgs,
2947 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
2949
2950 // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
2951 const bool DynamicOrOrdered = Ordered || RT.isDynamic(ScheduleKind.Schedule);
2952
2953 assert((Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule,
2954 LoopArgs.Chunk != nullptr)) &&
2955 "static non-chunked schedule does not need outer loop");
2956
2957 // Emit outer loop.
2958 //
2959 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2960 // When schedule(dynamic,chunk_size) is specified, the iterations are
2961 // distributed to threads in the team in chunks as the threads request them.
2962 // Each thread executes a chunk of iterations, then requests another chunk,
2963 // until no chunks remain to be distributed. Each chunk contains chunk_size
2964 // iterations, except for the last chunk to be distributed, which may have
2965 // fewer iterations. When no chunk_size is specified, it defaults to 1.
2966 //
2967 // When schedule(guided,chunk_size) is specified, the iterations are assigned
2968 // to threads in the team in chunks as the executing threads request them.
2969 // Each thread executes a chunk of iterations, then requests another chunk,
2970 // until no chunks remain to be assigned. For a chunk_size of 1, the size of
2971 // each chunk is proportional to the number of unassigned iterations divided
2972 // by the number of threads in the team, decreasing to 1. For a chunk_size
2973 // with value k (greater than 1), the size of each chunk is determined in the
2974 // same way, with the restriction that the chunks do not contain fewer than k
2975 // iterations (except for the last chunk to be assigned, which may have fewer
2976 // than k iterations).
2977 //
2978 // When schedule(auto) is specified, the decision regarding scheduling is
2979 // delegated to the compiler and/or runtime system. The programmer gives the
2980 // implementation the freedom to choose any possible mapping of iterations to
2981 // threads in the team.
2982 //
2983 // When schedule(runtime) is specified, the decision regarding scheduling is
2984 // deferred until run time, and the schedule and chunk size are taken from the
2985 // run-sched-var ICV. If the ICV is set to auto, the schedule is
2986 // implementation defined
2987 //
2988 // __kmpc_dispatch_init();
2989 // while(__kmpc_dispatch_next(&LB, &UB)) {
2990 // idx = LB;
2991 // while (idx <= UB) { BODY; ++idx;
2992 // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
2993 // } // inner loop
2994 // }
2995 // __kmpc_dispatch_deinit();
2996 //
2997 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2998 // When schedule(static, chunk_size) is specified, iterations are divided into
2999 // chunks of size chunk_size, and the chunks are assigned to the threads in
3000 // the team in a round-robin fashion in the order of the thread number.
3001 //
3002 // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
3003 // while (idx <= UB) { BODY; ++idx; } // inner loop
3004 // LB = LB + ST;
3005 // UB = UB + ST;
3006 // }
3007 //
3008
3009 const Expr *IVExpr = S.getIterationVariable();
3010 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3011 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3012
3013 if (DynamicOrOrdered) {
3014 const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
3015 CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
3016 llvm::Value *LBVal = DispatchBounds.first;
3017 llvm::Value *UBVal = DispatchBounds.second;
3018 CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
3019 LoopArgs.Chunk};
3020 RT.emitForDispatchInit(*this, S.getBeginLoc(), ScheduleKind, IVSize,
3021 IVSigned, Ordered, DipatchRTInputValues);
3022 } else {
3024 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
3025 LoopArgs.ST, LoopArgs.Chunk);
3026 RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
3027 ScheduleKind, StaticInit);
3028 }
3029
3030 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
3031 const unsigned IVSize,
3032 const bool IVSigned) {
3033 if (Ordered) {
3035 IVSigned);
3036 }
3037 };
3038
3039 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
3040 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
3041 OuterLoopArgs.IncExpr = S.getInc();
3042 OuterLoopArgs.Init = S.getInit();
3043 OuterLoopArgs.Cond = S.getCond();
3044 OuterLoopArgs.NextLB = S.getNextLowerBound();
3045 OuterLoopArgs.NextUB = S.getNextUpperBound();
3046 OuterLoopArgs.DKind = LoopArgs.DKind;
3047 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
3048 emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
3049 if (DynamicOrOrdered) {
3050 RT.emitForDispatchDeinit(*this, S.getBeginLoc());
3051 }
3052}
3053
3055 const unsigned IVSize, const bool IVSigned) {}
3056
3057void CodeGenFunction::EmitOMPDistributeOuterLoop(
3058 OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
3059 OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
3060 const CodeGenLoopTy &CodeGenLoopContent) {
3061
3063
3064 // Emit outer loop.
3065 // Same behavior as a OMPForOuterLoop, except that schedule cannot be
3066 // dynamic
3067 //
3068
3069 const Expr *IVExpr = S.getIterationVariable();
3070 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3071 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3072
3074 IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
3075 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
3076 RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, StaticInit);
3077
3078 // for combined 'distribute' and 'for' the increment expression of distribute
3079 // is stored in DistInc. For 'distribute' alone, it is in Inc.
3080 Expr *IncExpr;
3081 if (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()))
3082 IncExpr = S.getDistInc();
3083 else
3084 IncExpr = S.getInc();
3085
3086 // this routine is shared by 'omp distribute parallel for' and
3087 // 'omp distribute': select the right EUB expression depending on the
3088 // directive
3089 OMPLoopArguments OuterLoopArgs;
3090 OuterLoopArgs.LB = LoopArgs.LB;
3091 OuterLoopArgs.UB = LoopArgs.UB;
3092 OuterLoopArgs.ST = LoopArgs.ST;
3093 OuterLoopArgs.IL = LoopArgs.IL;
3094 OuterLoopArgs.Chunk = LoopArgs.Chunk;
3095 OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3096 ? S.getCombinedEnsureUpperBound()
3097 : S.getEnsureUpperBound();
3098 OuterLoopArgs.IncExpr = IncExpr;
3099 OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3100 ? S.getCombinedInit()
3101 : S.getInit();
3102 OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3103 ? S.getCombinedCond()
3104 : S.getCond();
3105 OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3106 ? S.getCombinedNextLowerBound()
3107 : S.getNextLowerBound();
3108 OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3109 ? S.getCombinedNextUpperBound()
3110 : S.getNextUpperBound();
3111 OuterLoopArgs.DKind = OMPD_distribute;
3112
3113 EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
3114 LoopScope, OuterLoopArgs, CodeGenLoopContent,
3116}
3117
3118static std::pair<LValue, LValue>
3120 const OMPExecutableDirective &S) {
3121 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
3122 LValue LB =
3123 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
3124 LValue UB =
3125 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
3126
3127 // When composing 'distribute' with 'for' (e.g. as in 'distribute
3128 // parallel for') we need to use the 'distribute'
3129 // chunk lower and upper bounds rather than the whole loop iteration
3130 // space. These are parameters to the outlined function for 'parallel'
3131 // and we copy the bounds of the previous schedule into the
3132 // the current ones.
3133 LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
3134 LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
3135 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
3136 PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());
3137 PrevLBVal = CGF.EmitScalarConversion(
3138 PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
3141 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
3142 PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());
3143 PrevUBVal = CGF.EmitScalarConversion(
3144 PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
3147
3148 CGF.EmitStoreOfScalar(PrevLBVal, LB);
3149 CGF.EmitStoreOfScalar(PrevUBVal, UB);
3150
3151 return {LB, UB};
3152}
3153
3154/// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
3155/// we need to use the LB and UB expressions generated by the worksharing
3156/// code generation support, whereas in non combined situations we would
3157/// just emit 0 and the LastIteration expression
3158/// This function is necessary due to the difference of the LB and UB
3159/// types for the RT emission routines for 'for_static_init' and
3160/// 'for_dispatch_init'
3161static std::pair<llvm::Value *, llvm::Value *>
3163 const OMPExecutableDirective &S,
3164 Address LB, Address UB) {
3165 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
3166 const Expr *IVExpr = LS.getIterationVariable();
3167 // when implementing a dynamic schedule for a 'for' combined with a
3168 // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
3169 // is not normalized as each team only executes its own assigned
3170 // distribute chunk
3171 QualType IteratorTy = IVExpr->getType();
3172 llvm::Value *LBVal =
3173 CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
3174 llvm::Value *UBVal =
3175 CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
3176 return {LBVal, UBVal};
3177}
3178
3182 const auto &Dir = cast<OMPLoopDirective>(S);
3183 LValue LB =
3184 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
3185 llvm::Value *LBCast = CGF.Builder.CreateIntCast(
3186 CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
3187 CapturedVars.push_back(LBCast);
3188 LValue UB =
3189 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
3190
3191 llvm::Value *UBCast = CGF.Builder.CreateIntCast(
3192 CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
3193 CapturedVars.push_back(UBCast);
3194}
3195
3196static void
3198 const OMPLoopDirective &S,
3199 CodeGenFunction::JumpDest LoopExit) {
3200 auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
3201 PrePostActionTy &Action) {
3202 Action.Enter(CGF);
3203 bool HasCancel = false;
3204 if (!isOpenMPSimdDirective(S.getDirectiveKind())) {
3205 if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
3206 HasCancel = D->hasCancel();
3207 else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
3208 HasCancel = D->hasCancel();
3209 else if (const auto *D =
3210 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
3211 HasCancel = D->hasCancel();
3212 }
3213 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
3214 HasCancel);
3215 CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(),
3218 };
3219
3221 CGF, S,
3222 isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for,
3223 CGInlinedWorksharingLoop,
3225}
3226
3229 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3231 S.getDistInc());
3232 };
3233 OMPLexicalScope Scope(*this, S, OMPD_parallel);
3234 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3235}
3236
3239 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3241 S.getDistInc());
3242 };
3243 OMPLexicalScope Scope(*this, S, OMPD_parallel);
3244 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3245}
3246
3248 const OMPDistributeSimdDirective &S) {
3249 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3251 };
3252 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3253 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
3254}
3255
3257 CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
3258 // Emit SPMD target parallel for region as a standalone region.
3259 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3260 emitOMPSimdRegion(CGF, S, Action);
3261 };
3262 llvm::Function *Fn;
3263 llvm::Constant *Addr;
3264 // Emit target region as a standalone region.
3266 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
3267 assert(Fn && Addr && "Target device function emission failed.");
3268}
3269
3271 const OMPTargetSimdDirective &S) {
3272 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3273 emitOMPSimdRegion(CGF, S, Action);
3274 };
3275 emitCommonOMPTargetDirective(*this, S, CodeGen);
3276}
3277
3278namespace {
3279struct ScheduleKindModifiersTy {
3283 ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
3286 : Kind(Kind), M1(M1), M2(M2) {}
3287};
3288} // namespace
3289
3291 const OMPLoopDirective &S, Expr *EUB,
3292 const CodeGenLoopBoundsTy &CodeGenLoopBounds,
3293 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
3294 // Emit the loop iteration variable.
3295 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3296 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3297 EmitVarDecl(*IVDecl);
3298
3299 // Emit the iterations count variable.
3300 // If it is not a variable, Sema decided to calculate iterations count on each
3301 // iteration (e.g., it is foldable into a constant).
3302 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3303 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3304 // Emit calculation of the iterations count.
3305 EmitIgnoredExpr(S.getCalcLastIteration());
3306 }
3307
3309
3310 bool HasLastprivateClause;
3311 // Check pre-condition.
3312 {
3313 OMPLoopScope PreInitScope(*this, S);
3314 // Skip the entire loop if we don't meet the precondition.
3315 // If the condition constant folds and can be elided, avoid emitting the
3316 // whole loop.
3317 bool CondConstant;
3318 llvm::BasicBlock *ContBlock = nullptr;
3319 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
3320 if (!CondConstant)
3321 return false;
3322 } else {
3323 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
3324 ContBlock = createBasicBlock("omp.precond.end");
3325 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
3326 getProfileCount(&S));
3327 EmitBlock(ThenBlock);
3329 }
3330
3331 RunCleanupsScope DoacrossCleanupScope(*this);
3332 bool Ordered = false;
3333 if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
3334 if (OrderedClause->getNumForLoops())
3335 RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations());
3336 else
3337 Ordered = true;
3338 }
3339
3340 llvm::DenseSet<const Expr *> EmittedFinals;
3341 emitAlignedClause(*this, S);
3342 bool HasLinears = EmitOMPLinearClauseInit(S);
3343 // Emit helper vars inits.
3344
3345 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
3346 LValue LB = Bounds.first;
3347 LValue UB = Bounds.second;
3348 LValue ST =
3349 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
3350 LValue IL =
3351 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
3352
3353 // Emit 'then' code.
3354 {
3355 OMPPrivateScope LoopScope(*this);
3356 if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
3357 // Emit implicit barrier to synchronize threads and avoid data races on
3358 // initialization of firstprivate variables and post-update of
3359 // lastprivate variables.
3361 *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
3362 /*ForceSimpleCall=*/true);
3363 }
3364 EmitOMPPrivateClause(S, LoopScope);
3366 *this, S, EmitLValue(S.getIterationVariable()));
3367 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
3368 EmitOMPReductionClauseInit(S, LoopScope);
3369 EmitOMPPrivateLoopCounters(S, LoopScope);
3370 EmitOMPLinearClause(S, LoopScope);
3371 (void)LoopScope.Privatize();
3372 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
3374
3375 // Detect the loop schedule kind and chunk.
3376 const Expr *ChunkExpr = nullptr;
3377 OpenMPScheduleTy ScheduleKind;
3378 if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
3379 ScheduleKind.Schedule = C->getScheduleKind();
3380 ScheduleKind.M1 = C->getFirstScheduleModifier();
3381 ScheduleKind.M2 = C->getSecondScheduleModifier();
3382 ChunkExpr = C->getChunkSize();
3383 } else {
3384 // Default behaviour for schedule clause.
3386 *this, S, ScheduleKind.Schedule, ChunkExpr);
3387 }
3388 bool HasChunkSizeOne = false;
3389 llvm::Value *Chunk = nullptr;
3390 if (ChunkExpr) {
3391 Chunk = EmitScalarExpr(ChunkExpr);
3392 Chunk = EmitScalarConversion(Chunk, ChunkExpr->getType(),
3393 S.getIterationVariable()->getType(),
3394 S.getBeginLoc());
3396 if (ChunkExpr->EvaluateAsInt(Result, getContext())) {
3397 llvm::APSInt EvaluatedChunk = Result.Val.getInt();
3398 HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
3399 }
3400 }
3401 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3402 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3403 // OpenMP 4.5, 2.7.1 Loop Construct, Description.
3404 // If the static schedule kind is specified or if the ordered clause is
3405 // specified, and if no monotonic modifier is specified, the effect will
3406 // be as if the monotonic modifier was specified.
3407 bool StaticChunkedOne =
3408 RT.isStaticChunked(ScheduleKind.Schedule,
3409 /* Chunked */ Chunk != nullptr) &&
3410 HasChunkSizeOne &&
3411 isOpenMPLoopBoundSharingDirective(S.getDirectiveKind());
3412 bool IsMonotonic =
3413 Ordered ||
3414 (ScheduleKind.Schedule == OMPC_SCHEDULE_static &&
3415 !(ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_nonmonotonic ||
3416 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_nonmonotonic)) ||
3417 ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
3418 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
3419 if ((RT.isStaticNonchunked(ScheduleKind.Schedule,
3420 /* Chunked */ Chunk != nullptr) ||
3421 StaticChunkedOne) &&
3422 !Ordered) {
3423 JumpDest LoopExit =
3426 *this, S,
3427 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3428 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
3429 CGF.EmitOMPSimdInit(S);
3430 } else if (const auto *C = S.getSingleClause<OMPOrderClause>()) {
3431 if (C->getKind() == OMPC_ORDER_concurrent)
3432 CGF.LoopStack.setParallel(/*Enable=*/true);
3433 }
3434 },
3435 [IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk,
3436 &S, ScheduleKind, LoopExit,
3437 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
3438 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
3439 // When no chunk_size is specified, the iteration space is divided
3440 // into chunks that are approximately equal in size, and at most
3441 // one chunk is distributed to each thread. Note that the size of
3442 // the chunks is unspecified in this case.
3444 IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
3445 UB.getAddress(), ST.getAddress(),
3446 StaticChunkedOne ? Chunk : nullptr);
3447 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
3448 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind,
3449 StaticInit);
3450 // UB = min(UB, GlobalUB);
3451 if (!StaticChunkedOne)
3452 CGF.EmitIgnoredExpr(S.getEnsureUpperBound());
3453 // IV = LB;
3454 CGF.EmitIgnoredExpr(S.getInit());
3455 // For unchunked static schedule generate:
3456 //
3457 // while (idx <= UB) {
3458 // BODY;
3459 // ++idx;
3460 // }
3461 //
3462 // For static schedule with chunk one:
3463 //
3464 // while (IV <= PrevUB) {
3465 // BODY;
3466 // IV += ST;
3467 // }
3468 CGF.EmitOMPInnerLoop(
3469 S, LoopScope.requiresCleanups(),
3470 StaticChunkedOne ? S.getCombinedParForInDistCond()
3471 : S.getCond(),
3472 StaticChunkedOne ? S.getDistInc() : S.getInc(),
3473 [&S, LoopExit](CodeGenFunction &CGF) {
3474 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit);
3475 },
3476 [](CodeGenFunction &) {});
3477 });
3478 EmitBlock(LoopExit.getBlock());
3479 // Tell the runtime we are done.
3480 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
3481 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
3482 OMPD_for);
3483 };
3484 OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
3485 } else {
3486 // Emit the outer loop, which requests its work chunk [LB..UB] from
3487 // runtime and runs the inner loop to process it.
3488 OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),
3489 ST.getAddress(), IL.getAddress(), Chunk,
3490 EUB);
3491 LoopArguments.DKind = OMPD_for;
3492 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
3493 LoopArguments, CGDispatchBounds);
3494 }
3495 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
3496 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
3497 return CGF.Builder.CreateIsNotNull(
3498 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3499 });
3500 }
3502 S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind())
3503 ? /*Parallel and Simd*/ OMPD_parallel_for_simd
3504 : /*Parallel only*/ OMPD_parallel);
3505 // Emit post-update of the reduction variables if IsLastIter != 0.
3507 *this, S, [IL, &S](CodeGenFunction &CGF) {
3508 return CGF.Builder.CreateIsNotNull(
3509 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3510 });
3511 // Emit final copy of the lastprivate variables if IsLastIter != 0.
3512 if (HasLastprivateClause)
3514 S, isOpenMPSimdDirective(S.getDirectiveKind()),
3515 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
3516 LoopScope.restoreMap();
3517 EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) {
3518 return CGF.Builder.CreateIsNotNull(
3519 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3520 });
3521 }
3522 DoacrossCleanupScope.ForceCleanup();
3523 // We're now done with the loop, so jump to the continuation block.
3524 if (ContBlock) {
3525 EmitBranch(ContBlock);
3526 EmitBlock(ContBlock, /*IsFinished=*/true);
3527 }
3528 }
3529 return HasLastprivateClause;
3530}
3531
3532/// The following two functions generate expressions for the loop lower
3533/// and upper bounds in case of static and dynamic (dispatch) schedule
3534/// of the associated 'for' or 'distribute' loop.
3535static std::pair<LValue, LValue>
3537 const auto &LS = cast<OMPLoopDirective>(S);
3538 LValue LB =
3539 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
3540 LValue UB =
3541 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
3542 return {LB, UB};
3543}
3544
3545/// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
3546/// consider the lower and upper bound expressions generated by the
3547/// worksharing loop support, but we use 0 and the iteration space size as
3548/// constants
3549static std::pair<llvm::Value *, llvm::Value *>
3551 Address LB, Address UB) {
3552 const auto &LS = cast<OMPLoopDirective>(S);
3553 const Expr *IVExpr = LS.getIterationVariable();
3554 const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
3555 llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
3556 llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
3557 return {LBVal, UBVal};
3558}
3559
3560/// Emits internal temp array declarations for the directive with inscan
3561/// reductions.
3562/// The code is the following:
3563/// \code
3564/// size num_iters = <num_iters>;
3565/// <type> buffer[num_iters];
3566/// \endcode
3568 CodeGenFunction &CGF, const OMPLoopDirective &S,
3569 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen) {
3570 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
3571 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
3574 SmallVector<const Expr *, 4> ReductionOps;
3575 SmallVector<const Expr *, 4> CopyArrayTemps;
3576 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
3577 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
3578 "Only inscan reductions are expected.");
3579 Shareds.append(C->varlist_begin(), C->varlist_end());
3580 Privates.append(C->privates().begin(), C->privates().end());
3581 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
3582 CopyArrayTemps.append(C->copy_array_temps().begin(),
3583 C->copy_array_temps().end());
3584 }
3585 {
3586 // Emit buffers for each reduction variables.
3587 // ReductionCodeGen is required to emit correctly the code for array
3588 // reductions.
3589 ReductionCodeGen RedCG(Shareds, Shareds, Privates, ReductionOps);
3590 unsigned Count = 0;
3591 auto *ITA = CopyArrayTemps.begin();
3592 for (const Expr *IRef : Privates) {
3593 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
3594 // Emit variably modified arrays, used for arrays/array sections
3595 // reductions.
3596 if (PrivateVD->getType()->isVariablyModifiedType()) {
3597 RedCG.emitSharedOrigLValue(CGF, Count);
3598 RedCG.emitAggregateType(CGF, Count);
3599 }
3600 CodeGenFunction::OpaqueValueMapping DimMapping(
3601 CGF,
3602 cast<OpaqueValueExpr>(
3603 cast<VariableArrayType>((*ITA)->getType()->getAsArrayTypeUnsafe())
3604 ->getSizeExpr()),
3605 RValue::get(OMPScanNumIterations));
3606 // Emit temp buffer.
3607 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(*ITA)->getDecl()));
3608 ++ITA;
3609 ++Count;
3610 }
3611 }
3612}
3613
3614/// Copies final inscan reductions values to the original variables.
3615/// The code is the following:
3616/// \code
3617/// <orig_var> = buffer[num_iters-1];
3618/// \endcode
3620 CodeGenFunction &CGF, const OMPLoopDirective &S,
3621 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen) {
3622 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
3623 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
3629 SmallVector<const Expr *, 4> CopyArrayElems;
3630 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
3631 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
3632 "Only inscan reductions are expected.");
3633 Shareds.append(C->varlist_begin(), C->varlist_end());
3634 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
3635 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
3636 Privates.append(C->privates().begin(), C->privates().end());
3637 CopyOps.append(C->copy_ops().begin(), C->copy_ops().end());
3638 CopyArrayElems.append(C->copy_array_elems().begin(),
3639 C->copy_array_elems().end());
3640 }
3641 // Create temp var and copy LHS value to this temp value.
3642 // LHS = TMP[LastIter];
3643 llvm::Value *OMPLast = CGF.Builder.CreateNSWSub(
3644 OMPScanNumIterations,
3645 llvm::ConstantInt::get(CGF.SizeTy, 1, /*isSigned=*/false));
3646 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
3647 const Expr *PrivateExpr = Privates[I];
3648 const Expr *OrigExpr = Shareds[I];
3649 const Expr *CopyArrayElem = CopyArrayElems[I];
3650 CodeGenFunction::OpaqueValueMapping IdxMapping(
3651 CGF,
3652 cast<OpaqueValueExpr>(
3653 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3654 RValue::get(OMPLast));
3655 LValue DestLVal = CGF.EmitLValue(OrigExpr);
3656 LValue SrcLVal = CGF.EmitLValue(CopyArrayElem);
3657 CGF.EmitOMPCopy(
3658 PrivateExpr->getType(), DestLVal.getAddress(), SrcLVal.getAddress(),
3659 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
3660 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]);
3661 }
3662}
3663
3664/// Emits the code for the directive with inscan reductions.
3665/// The code is the following:
3666/// \code
3667/// #pragma omp ...
3668/// for (i: 0..<num_iters>) {
3669/// <input phase>;
3670/// buffer[i] = red;
3671/// }
3672/// #pragma omp master // in parallel region
3673/// for (int k = 0; k != ceil(log2(num_iters)); ++k)
3674/// for (size cnt = last_iter; cnt >= pow(2, k); --k)
3675/// buffer[i] op= buffer[i-pow(2,k)];
3676/// #pragma omp barrier // in parallel region
3677/// #pragma omp ...
3678/// for (0..<num_iters>) {
3679/// red = InclusiveScan ? buffer[i] : buffer[i-1];
3680/// <scan phase>;
3681/// }
3682/// \endcode
3684 CodeGenFunction &CGF, const OMPLoopDirective &S,
3685 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen,
3686 llvm::function_ref<void(CodeGenFunction &)> FirstGen,
3687 llvm::function_ref<void(CodeGenFunction &)> SecondGen) {
3688 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
3689 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
3691 SmallVector<const Expr *, 4> ReductionOps;
3694 SmallVector<const Expr *, 4> CopyArrayElems;
3695 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
3696 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
3697 "Only inscan reductions are expected.");
3698 Privates.append(C->privates().begin(), C->privates().end());
3699 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
3700 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
3701 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
3702 CopyArrayElems.append(C->copy_array_elems().begin(),
3703 C->copy_array_elems().end());
3704 }
3705 CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S);
3706 {
3707 // Emit loop with input phase:
3708 // #pragma omp ...
3709 // for (i: 0..<num_iters>) {
3710 // <input phase>;
3711 // buffer[i] = red;
3712 // }
3713 CGF.OMPFirstScanLoop = true;
3714 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
3715 FirstGen(CGF);
3716 }
3717 // #pragma omp barrier // in parallel region
3718 auto &&CodeGen = [&S, OMPScanNumIterations, &LHSs, &RHSs, &CopyArrayElems,
3719 &ReductionOps,
3720 &Privates](CodeGenFunction &CGF, PrePostActionTy &Action) {
3721 Action.Enter(CGF);
3722 // Emit prefix reduction:
3723 // #pragma omp master // in parallel region
3724 // for (int k = 0; k <= ceil(log2(n)); ++k)
3725 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock();
3726 llvm::BasicBlock *LoopBB = CGF.createBasicBlock("omp.outer.log.scan.body");
3727 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.outer.log.scan.exit");
3728 llvm::Function *F =
3729 CGF.CGM.getIntrinsic(llvm::Intrinsic::log2, CGF.DoubleTy);
3730 llvm::Value *Arg =
3731 CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy);
3732 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg);
3733 F = CGF.CGM.getIntrinsic(llvm::Intrinsic::ceil, CGF.DoubleTy);
3734 LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal);
3735 LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy);
3736 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub(
3737 OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1));
3738 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getBeginLoc());
3739 CGF.EmitBlock(LoopBB);
3740 auto *Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2);
3741 // size pow2k = 1;
3742 auto *Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3743 Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB);
3744 Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB);
3745 // for (size i = n - 1; i >= 2 ^ k; --i)
3746 // tmp[i] op= tmp[i-pow2k];
3747 llvm::BasicBlock *InnerLoopBB =
3748 CGF.createBasicBlock("omp.inner.log.scan.body");
3749 llvm::BasicBlock *InnerExitBB =
3750 CGF.createBasicBlock("omp.inner.log.scan.exit");
3751 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K);
3752 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3753 CGF.EmitBlock(InnerLoopBB);
3754 auto *IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3755 IVal->addIncoming(NMin1, LoopBB);
3756 {
3757 CodeGenFunction::OMPPrivateScope PrivScope(CGF);
3758 auto *ILHS = LHSs.begin();
3759 auto *IRHS = RHSs.begin();
3760 for (const Expr *CopyArrayElem : CopyArrayElems) {
3761 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
3762 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
3763 Address LHSAddr = Address::invalid();
3764 {
3765 CodeGenFunction::OpaqueValueMapping IdxMapping(
3766 CGF,
3767 cast<OpaqueValueExpr>(
3768 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3769 RValue::get(IVal));
3770 LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress();
3771 }
3772 PrivScope.addPrivate(LHSVD, LHSAddr);
3773 Address RHSAddr = Address::invalid();
3774 {
3775 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K);
3776 CodeGenFunction::OpaqueValueMapping IdxMapping(
3777 CGF,
3778 cast<OpaqueValueExpr>(
3779 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3780 RValue::get(OffsetIVal));
3781 RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress();
3782 }
3783 PrivScope.addPrivate(RHSVD, RHSAddr);
3784 ++ILHS;
3785 ++IRHS;
3786 }
3787 PrivScope.Privatize();
3788 CGF.CGM.getOpenMPRuntime().emitReduction(
3789 CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
3790 {/*WithNowait=*/true, /*SimpleReduction=*/true, OMPD_unknown});
3791 }
3792 llvm::Value *NextIVal =
3793 CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1));
3794 IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock());
3795 CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K);
3796 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3797 CGF.EmitBlock(InnerExitBB);
3798 llvm::Value *Next =
3799 CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1));
3800 Counter->addIncoming(Next, CGF.Builder.GetInsertBlock());
3801 // pow2k <<= 1;
3802 llvm::Value *NextPow2K =
3803 CGF.Builder.CreateShl(Pow2K, 1, "", /*HasNUW=*/true);
3804 Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock());
3805 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(Next, LogVal);
3806 CGF.Builder.CreateCondBr(Cmp, LoopBB, ExitBB);
3807 auto DL1 = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getEndLoc());
3808 CGF.EmitBlock(ExitBB);
3809 };
3810 if (isOpenMPParallelDirective(S.getDirectiveKind())) {
3811 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());
3812 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
3813 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
3814 /*ForceSimpleCall=*/true);
3815 } else {
3816 RegionCodeGenTy RCG(CodeGen);
3817 RCG(CGF);
3818 }
3819
3820 CGF.OMPFirstScanLoop = false;
3821 SecondGen(CGF);
3822}
3823
3825 const OMPLoopDirective &S,
3826 bool HasCancel) {
3827 bool HasLastprivates;
3828 if (llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
3829 [](const OMPReductionClause *C) {
3830 return C->getModifier() == OMPC_REDUCTION_inscan;
3831 })) {
3832 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
3833 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
3834 OMPLoopScope LoopScope(CGF, S);
3835 return CGF.EmitScalarExpr(S.getNumIterations());
3836 };
3837 const auto &&FirstGen = [&S, HasCancel](CodeGenFunction &CGF) {
3838 CodeGenFunction::OMPCancelStackRAII CancelRegion(
3839 CGF, S.getDirectiveKind(), HasCancel);
3840 (void)CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
3843 // Emit an implicit barrier at the end.
3844 CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getBeginLoc(),
3845 OMPD_for);
3846 };
3847 const auto &&SecondGen = [&S, HasCancel,
3848 &HasLastprivates](CodeGenFunction &CGF) {
3849 CodeGenFunction::OMPCancelStackRAII CancelRegion(
3850 CGF, S.getDirectiveKind(), HasCancel);
3851 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
3854 };
3855 if (!isOpenMPParallelDirective(S.getDirectiveKind()))
3856 emitScanBasedDirectiveDecls(CGF, S, NumIteratorsGen);
3857 emitScanBasedDirective(CGF, S, NumIteratorsGen, FirstGen, SecondGen);
3858 if (!isOpenMPParallelDirective(S.getDirectiveKind()))
3859 emitScanBasedDirectiveFinals(CGF, S, NumIteratorsGen);
3860 } else {
3861 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
3862 HasCancel);
3863 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
3866 }
3867 return HasLastprivates;
3868}
3869
3871 if (S.hasCancel())
3872 return false;
3873 for (OMPClause *C : S.clauses()) {
3874 if (isa<OMPNowaitClause>(C))
3875 continue;
3876
3877 if (auto *SC = dyn_cast<OMPScheduleClause>(C)) {
3878 if (SC->getFirstScheduleModifier() != OMPC_SCHEDULE_MODIFIER_unknown)
3879 return false;
3880 if (SC->getSecondScheduleModifier() != OMPC_SCHEDULE_MODIFIER_unknown)
3881 return false;
3882 switch (SC->getScheduleKind()) {
3883 case OMPC_SCHEDULE_auto:
3884 case OMPC_SCHEDULE_dynamic:
3885 case OMPC_SCHEDULE_runtime:
3886 case OMPC_SCHEDULE_guided:
3887 case OMPC_SCHEDULE_static:
3888 continue;
3890 return false;
3891 }
3892 }
3893
3894 return false;
3895 }
3896
3897 return true;
3898}
3899
3900static llvm::omp::ScheduleKind
3902 switch (ScheduleClauseKind) {
3904 return llvm::omp::OMP_SCHEDULE_Default;
3905 case OMPC_SCHEDULE_auto:
3906 return llvm::omp::OMP_SCHEDULE_Auto;
3907 case OMPC_SCHEDULE_dynamic:
3908 return llvm::omp::OMP_SCHEDULE_Dynamic;
3909 case OMPC_SCHEDULE_guided:
3910 return llvm::omp::OMP_SCHEDULE_Guided;
3911 case OMPC_SCHEDULE_runtime:
3912 return llvm::omp::OMP_SCHEDULE_Runtime;
3913 case OMPC_SCHEDULE_static:
3914 return llvm::omp::OMP_SCHEDULE_Static;
3915 }
3916 llvm_unreachable("Unhandled schedule kind");
3917}
3918
3920 bool HasLastprivates = false;
3921 bool UseOMPIRBuilder =
3922 CGM.getLangOpts().OpenMPIRBuilder && isSupportedByOpenMPIRBuilder(S);
3923 auto &&CodeGen = [this, &S, &HasLastprivates,
3924 UseOMPIRBuilder](CodeGenFunction &CGF, PrePostActionTy &) {
3925 // Use the OpenMPIRBuilder if enabled.
3926 if (UseOMPIRBuilder) {
3927 bool NeedsBarrier = !S.getSingleClause<OMPNowaitClause>();
3928
3929 llvm::omp::ScheduleKind SchedKind = llvm::omp::OMP_SCHEDULE_Default;
3930 llvm::Value *ChunkSize = nullptr;
3931 if (auto *SchedClause = S.getSingleClause<OMPScheduleClause>()) {
3932 SchedKind =
3933 convertClauseKindToSchedKind(SchedClause->getScheduleKind());
3934 if (const Expr *ChunkSizeExpr = SchedClause->getChunkSize())
3935 ChunkSize = EmitScalarExpr(ChunkSizeExpr);
3936 }
3937
3938 // Emit the associated statement and get its loop representation.
3939 const Stmt *Inner = S.getRawStmt();
3940 llvm::CanonicalLoopInfo *CLI =
3942
3943 llvm::OpenMPIRBuilder &OMPBuilder =
3945 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
3946 AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
3947 OMPBuilder.applyWorkshareLoop(
3948 Builder.getCurrentDebugLocation(), CLI, AllocaIP, NeedsBarrier,
3949 SchedKind, ChunkSize, /*HasSimdModifier=*/false,
3950 /*HasMonotonicModifier=*/false, /*HasNonmonotonicModifier=*/false,
3951 /*HasOrderedClause=*/false);
3952 return;
3953 }
3954
3955 HasLastprivates = emitWorksharingDirective(CGF, S, S.hasCancel());
3956 };
3957 {
3958 auto LPCRegion =
3960 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3961 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
3962 S.hasCancel());
3963 }
3964
3965 if (!UseOMPIRBuilder) {
3966 // Emit an implicit barrier at the end.
3967 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
3968 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
3969 }
3970 // Check for outer lastprivate conditional update.
3972}
3973
3975 bool HasLastprivates = false;
3976 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
3977 PrePostActionTy &) {
3978 HasLastprivates = emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
3979 };
3980 {
3981 auto LPCRegion =
3983 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3984 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
3985 }
3986
3987 // Emit an implicit barrier at the end.
3988 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
3989 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
3990 // Check for outer lastprivate conditional update.
3992}
3993
3995 const Twine &Name,
3996 llvm::Value *Init = nullptr) {
3997 LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
3998 if (Init)
3999 CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
4000 return LVal;
4001}
4002
4003void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
4004 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4005 const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
4006 bool HasLastprivates = false;
4007 auto &&CodeGen = [&S, CapturedStmt, CS,
4008 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
4009 const ASTContext &C = CGF.getContext();
4010 QualType KmpInt32Ty =
4011 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
4012 // Emit helper vars inits.
4013 LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
4014 CGF.Builder.getInt32(0));
4015 llvm::ConstantInt *GlobalUBVal = CS != nullptr
4016 ? CGF.Builder.getInt32(CS->size() - 1)
4017 : CGF.Builder.getInt32(0);
4018 LValue UB =
4019 createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
4020 LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
4021 CGF.Builder.getInt32(1));
4022 LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
4023 CGF.Builder.getInt32(0));
4024 // Loop counter.
4025 LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
4026 OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
4027 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
4028 OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
4029 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
4030 // Generate condition for loop.
4032 C, &IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_PRValue, OK_Ordinary,
4033 S.getBeginLoc(), FPOptionsOverride());
4034 // Increment for loop counter.
4036 C, &IVRefExpr, UO_PreInc, KmpInt32Ty, VK_PRValue, OK_Ordinary,
4037 S.getBeginLoc(), true, FPOptionsOverride());
4038 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
4039 // Iterate through all sections and emit a switch construct:
4040 // switch (IV) {
4041 // case 0:
4042 // <SectionStmt[0]>;
4043 // break;
4044 // ...
4045 // case <NumSection> - 1:
4046 // <SectionStmt[<NumSection> - 1]>;
4047 // break;
4048 // }
4049 // .omp.sections.exit:
4050 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
4051 llvm::SwitchInst *SwitchStmt =
4052 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
4053 ExitBB, CS == nullptr ? 1 : CS->size());
4054 if (CS) {
4055 unsigned CaseNumber = 0;
4056 for (const Stmt *SubStmt : CS->children()) {
4057 auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
4058 CGF.EmitBlock(CaseBB);
4059 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
4060 CGF.EmitStmt(SubStmt);
4061 CGF.EmitBranch(ExitBB);
4062 ++CaseNumber;
4063 }
4064 } else {
4065 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");
4066 CGF.EmitBlock(CaseBB);
4067 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
4068 CGF.EmitStmt(CapturedStmt);
4069 CGF.EmitBranch(ExitBB);
4070 }
4071 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
4072 };
4073
4074 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
4075 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
4076 // Emit implicit barrier to synchronize threads and avoid data races on
4077 // initialization of firstprivate variables and post-update of lastprivate
4078 // variables.
4079 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
4080 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
4081 /*ForceSimpleCall=*/true);
4082 }
4083 CGF.EmitOMPPrivateClause(S, LoopScope);
4084 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(CGF, S, IV);
4085 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4086 CGF.EmitOMPReductionClauseInit(S, LoopScope);
4087 (void)LoopScope.Privatize();
4088 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
4089 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4090
4091 // Emit static non-chunked loop.
4092 OpenMPScheduleTy ScheduleKind;
4093 ScheduleKind.Schedule = OMPC_SCHEDULE_static;
4095 /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(),
4096 LB.getAddress(), UB.getAddress(), ST.getAddress());
4097 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
4098 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit);
4099 // UB = min(UB, GlobalUB);
4100 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
4101 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
4102 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
4103 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
4104 // IV = LB;
4105 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
4106 // while (idx <= UB) { BODY; ++idx; }
4107 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, Cond, Inc, BodyGen,
4108 [](CodeGenFunction &) {});
4109 // Tell the runtime we are done.
4110 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
4111 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
4112 OMPD_sections);
4113 };
4114 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
4115 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4116 // Emit post-update of the reduction variables if IsLastIter != 0.
4117 emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) {
4118 return CGF.Builder.CreateIsNotNull(
4119 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
4120 });
4121
4122 // Emit final copy of the lastprivate variables if IsLastIter != 0.
4123 if (HasLastprivates)
4125 S, /*NoFinals=*/false,
4126 CGF.Builder.CreateIsNotNull(
4127 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())));
4128 };
4129
4130 bool HasCancel = false;
4131 if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
4132 HasCancel = OSD->hasCancel();
4133 else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
4134 HasCancel = OPSD->hasCancel();
4135 OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
4136 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
4137 HasCancel);
4138 // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
4139 // clause. Otherwise the barrier will be generated by the codegen for the
4140 // directive.
4141 if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
4142 // Emit implicit barrier to synchronize threads and avoid data races on
4143 // initialization of firstprivate variables.
4144 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
4145 OMPD_unknown);
4146 }
4147}
4148
4150 if (CGM.getLangOpts().OpenMPIRBuilder) {
4151 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4152 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4153 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4154
4155 auto FiniCB = [this](InsertPointTy IP) {
4157 };
4158
4159 const CapturedStmt *ICS = S.getInnermostCapturedStmt();
4160 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4161 const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
4163 if (CS) {
4164 for (const Stmt *SubStmt : CS->children()) {
4165 auto SectionCB = [this, SubStmt](InsertPointTy AllocaIP,
4166 InsertPointTy CodeGenIP) {
4168 *this, SubStmt, AllocaIP, CodeGenIP, "section");
4169 };
4170 SectionCBVector.push_back(SectionCB);
4171 }
4172 } else {
4173 auto SectionCB = [this, CapturedStmt](InsertPointTy AllocaIP,
4174 InsertPointTy CodeGenIP) {
4176 *this, CapturedStmt, AllocaIP, CodeGenIP, "section");
4177 };
4178 SectionCBVector.push_back(SectionCB);
4179 }
4180
4181 // Privatization callback that performs appropriate action for
4182 // shared/private/firstprivate/lastprivate/copyin/... variables.
4183 //
4184 // TODO: This defaults to shared right now.
4185 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
4186 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
4187 // The next line is appropriate only for variables (Val) with the
4188 // data-sharing attribute "shared".
4189 ReplVal = &Val;
4190
4191 return CodeGenIP;
4192 };
4193
4194 CGCapturedStmtInfo CGSI(*ICS, CR_OpenMP);
4195 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
4196 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
4197 AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
4198 Builder.restoreIP(OMPBuilder.createSections(
4199 Builder, AllocaIP, SectionCBVector, PrivCB, FiniCB, S.hasCancel(),
4200 S.getSingleClause<OMPNowaitClause>()));
4201 return;
4202 }
4203 {
4204 auto LPCRegion =
4206 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4207 EmitSections(S);
4208 }
4209 // Emit an implicit barrier at the end.
4210 if (!S.getSingleClause<OMPNowaitClause>()) {
4211 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
4212 OMPD_sections);
4213 }
4214 // Check for outer lastprivate conditional update.
4216}
4217
4219 if (CGM.getLangOpts().OpenMPIRBuilder) {
4220 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4221 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4222
4223 const Stmt *SectionRegionBodyStmt = S.getAssociatedStmt();
4224 auto FiniCB = [this](InsertPointTy IP) {
4226 };
4227
4228 auto BodyGenCB = [SectionRegionBodyStmt, this](InsertPointTy AllocaIP,
4229 InsertPointTy CodeGenIP) {
4231 *this, SectionRegionBodyStmt, AllocaIP, CodeGenIP, "section");
4232 };
4233
4234 LexicalScope Scope(*this, S.getSourceRange());
4235 EmitStopPoint(&S);
4236 Builder.restoreIP(OMPBuilder.createSection(Builder, BodyGenCB, FiniCB));
4237
4238 return;
4239 }
4240 LexicalScope Scope(*this, S.getSourceRange());
4241 EmitStopPoint(&S);
4242 EmitStmt(S.getAssociatedStmt());
4243}
4244
4246 llvm::SmallVector<const Expr *, 8> CopyprivateVars;
4250 // Check if there are any 'copyprivate' clauses associated with this
4251 // 'single' construct.
4252 // Build a list of copyprivate variables along with helper expressions
4253 // (<source>, <destination>, <destination>=<source> expressions)
4254 for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
4255 CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
4256 DestExprs.append(C->destination_exprs().begin(),
4257 C->destination_exprs().end());
4258 SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
4259 AssignmentOps.append(C->assignment_ops().begin(),
4260 C->assignment_ops().end());
4261 }
4262 // Emit code for 'single' region along with 'copyprivate' clauses
4263 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4264 Action.Enter(CGF);
4265 OMPPrivateScope SingleScope(CGF);
4268 (void)SingleScope.Privatize();
4269 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4270 };
4271 {
4272 auto LPCRegion =
4274 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4275 CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getBeginLoc(),
4276 CopyprivateVars, DestExprs,
4277 SrcExprs, AssignmentOps);
4278 }
4279 // Emit an implicit barrier at the end (to avoid data race on firstprivate
4280 // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
4281 if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
4283 *this, S.getBeginLoc(),
4284 S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
4285 }
4286 // Check for outer lastprivate conditional update.
4288}
4289
4291 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4292 Action.Enter(CGF);
4293 CGF.EmitStmt(S.getRawStmt());
4294 };
4295 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());
4296}
4297
4299 if (CGM.getLangOpts().OpenMPIRBuilder) {
4300 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4301 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4302
4303 const Stmt *MasterRegionBodyStmt = S.getAssociatedStmt();
4304
4305 auto FiniCB = [this](InsertPointTy IP) {
4307 };
4308
4309 auto BodyGenCB = [MasterRegionBodyStmt, this](InsertPointTy AllocaIP,
4310 InsertPointTy CodeGenIP) {
4312 *this, MasterRegionBodyStmt, AllocaIP, CodeGenIP, "master");
4313 };
4314
4315 LexicalScope Scope(*this, S.getSourceRange());
4316 EmitStopPoint(&S);
4317 Builder.restoreIP(OMPBuilder.createMaster(Builder, BodyGenCB, FiniCB));
4318
4319 return;
4320 }
4321 LexicalScope Scope(*this, S.getSourceRange());
4322 EmitStopPoint(&S);
4323 emitMaster(*this, S);
4324}
4325
4327 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4328 Action.Enter(CGF);
4329 CGF.EmitStmt(S.getRawStmt());
4330 };
4331 Expr *Filter = nullptr;
4332 if (const auto *FilterClause = S.getSingleClause<OMPFilterClause>())
4333 Filter = FilterClause->getThreadID();
4334 CGF.CGM.getOpenMPRuntime().emitMaskedRegion(CGF, CodeGen, S.getBeginLoc(),
4335 Filter);
4336}
4337
4339 if (CGM.getLangOpts().OpenMPIRBuilder) {
4340 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4341 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4342
4343 const Stmt *MaskedRegionBodyStmt = S.getAssociatedStmt();
4344 const Expr *Filter = nullptr;
4345 if (const auto *FilterClause = S.getSingleClause<OMPFilterClause>())
4346 Filter = FilterClause->getThreadID();
4347 llvm::Value *FilterVal = Filter
4348 ? EmitScalarExpr(Filter, CGM.Int32Ty)
4349 : llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/0);
4350
4351 auto FiniCB = [this](InsertPointTy IP) {
4353 };
4354
4355 auto BodyGenCB = [MaskedRegionBodyStmt, this](InsertPointTy AllocaIP,
4356 InsertPointTy CodeGenIP) {
4358 *this, MaskedRegionBodyStmt, AllocaIP, CodeGenIP, "masked");
4359 };
4360
4361 LexicalScope Scope(*this, S.getSourceRange());
4362 EmitStopPoint(&S);
4363 Builder.restoreIP(
4364 OMPBuilder.createMasked(Builder, BodyGenCB, FiniCB, FilterVal));
4365
4366 return;
4367 }
4368 LexicalScope Scope(*this, S.getSourceRange());
4369 EmitStopPoint(&S);
4370 emitMasked(*this, S);
4371}
4372
4374 if (CGM.getLangOpts().OpenMPIRBuilder) {
4375 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4376 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4377
4378 const Stmt *CriticalRegionBodyStmt = S.getAssociatedStmt();
4379 const Expr *Hint = nullptr;
4380 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
4381 Hint = HintClause->getHint();
4382
4383 // TODO: This is slightly different from what's currently being done in
4384 // clang. Fix the Int32Ty to IntPtrTy (pointer width size) when everything
4385 // about typing is final.
4386 llvm::Value *HintInst = nullptr;
4387 if (Hint)
4388 HintInst =
4389 Builder.CreateIntCast(EmitScalarExpr(Hint), CGM.Int32Ty, false);
4390
4391 auto FiniCB = [this](InsertPointTy IP) {
4393 };
4394
4395 auto BodyGenCB = [CriticalRegionBodyStmt, this](InsertPointTy AllocaIP,
4396 InsertPointTy CodeGenIP) {
4398 *this, CriticalRegionBodyStmt, AllocaIP, CodeGenIP, "critical");
4399 };
4400
4401 LexicalScope Scope(*this, S.getSourceRange());
4402 EmitStopPoint(&S);
4403 Builder.restoreIP(OMPBuilder.createCritical(
4404 Builder, BodyGenCB, FiniCB, S.getDirectiveName().getAsString(),
4405 HintInst));
4406
4407 return;
4408 }
4409
4410 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4411 Action.Enter(CGF);
4412 CGF.EmitStmt(S.getAssociatedStmt());
4413 };
4414 const Expr *Hint = nullptr;
4415 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
4416 Hint = HintClause->getHint();
4417 LexicalScope Scope(*this, S.getSourceRange());
4418 EmitStopPoint(&S);
4420 S.getDirectiveName().getAsString(),
4421 CodeGen, S.getBeginLoc(), Hint);
4422}
4423
4425 const OMPParallelForDirective &S) {
4426 // Emit directive as a combined directive that consists of two implicit
4427 // directives: 'parallel' with 'for' directive.
4428 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4429 Action.Enter(CGF);
4430 emitOMPCopyinClause(CGF, S);
4431 (void)emitWorksharingDirective(CGF, S, S.hasCancel());
4432 };
4433 {
4434 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
4435 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
4436 CGCapturedStmtInfo CGSI(CR_OpenMP);
4437 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
4438 OMPLoopScope LoopScope(CGF, S);
4439 return CGF.EmitScalarExpr(S.getNumIterations());
4440 };
4441 bool IsInscan = llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
4442 [](const OMPReductionClause *C) {
4443 return C->getModifier() == OMPC_REDUCTION_inscan;
4444 });
4445 if (IsInscan)
4446 emitScanBasedDirectiveDecls(*this, S, NumIteratorsGen);
4447 auto LPCRegion =
4449 emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
4451 if (IsInscan)
4452 emitScanBasedDirectiveFinals(*this, S, NumIteratorsGen);
4453 }
4454 // Check for outer lastprivate conditional update.
4456}
4457
4459 const OMPParallelForSimdDirective &S) {
4460 // Emit directive as a combined directive that consists of two implicit
4461 // directives: 'parallel' with 'for' directive.
4462 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4463 Action.Enter(CGF);
4464 emitOMPCopyinClause(CGF, S);
4465 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
4466 };
4467 {
4468 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
4469 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
4470 CGCapturedStmtInfo CGSI(CR_OpenMP);
4471 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
4472 OMPLoopScope LoopScope(CGF, S);
4473 return CGF.EmitScalarExpr(S.getNumIterations());
4474 };
4475 bool IsInscan = llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
4476 [](const OMPReductionClause *C) {
4477 return C->getModifier() == OMPC_REDUCTION_inscan;
4478 });
4479 if (IsInscan)
4480 emitScanBasedDirectiveDecls(*this, S, NumIteratorsGen);
4481 auto LPCRegion =
4483 emitCommonOMPParallelDirective(*this, S, OMPD_for_simd, CodeGen,
4485 if (IsInscan)
4486 emitScanBasedDirectiveFinals(*this, S, NumIteratorsGen);
4487 }
4488 // Check for outer lastprivate conditional update.
4490}
4491
4493 const OMPParallelMasterDirective &S) {
4494 // Emit directive as a combined directive that consists of two implicit
4495 // directives: 'parallel' with 'master' directive.
4496 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4497 Action.Enter(CGF);
4498 OMPPrivateScope PrivateScope(CGF);
4499 emitOMPCopyinClause(CGF, S);
4500 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4501 CGF.EmitOMPPrivateClause(S, PrivateScope);
4502 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4503 (void)PrivateScope.Privatize();
4504 emitMaster(CGF, S);
4505 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4506 };
4507 {
4508 auto LPCRegion =
4510 emitCommonOMPParallelDirective(*this, S, OMPD_master, CodeGen,
4513 [](CodeGenFunction &) { return nullptr; });
4514 }
4515 // Check for outer lastprivate conditional update.
4517}
4518
4520 const OMPParallelMaskedDirective &S) {
4521 // Emit directive as a combined directive that consists of two implicit
4522 // directives: 'parallel' with 'masked' directive.
4523 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4524 Action.Enter(CGF);
4525 OMPPrivateScope PrivateScope(CGF);
4526 emitOMPCopyinClause(CGF, S);
4527 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4528 CGF.EmitOMPPrivateClause(S, PrivateScope);
4529 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4530 (void)PrivateScope.Privatize();
4531 emitMasked(CGF, S);
4532 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4533 };
4534 {
4535 auto LPCRegion =
4537 emitCommonOMPParallelDirective(*this, S, OMPD_masked, CodeGen,
4540 [](CodeGenFunction &) { return nullptr; });
4541 }
4542 // Check for outer lastprivate conditional update.
4544}
4545
4548 // Emit directive as a combined directive that consists of two implicit
4549 // directives: 'parallel' with 'sections' directive.
4550 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4551 Action.Enter(CGF);
4552 emitOMPCopyinClause(CGF, S);
4553 CGF.EmitSections(S);
4554 };
4555 {
4556 auto LPCRegion =
4558 emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
4560 }
4561 // Check for outer lastprivate conditional update.
4563}
4564
4565namespace {
4566/// Get the list of variables declared in the context of the untied tasks.
4567class CheckVarsEscapingUntiedTaskDeclContext final
4568 : public ConstStmtVisitor<CheckVarsEscapingUntiedTaskDeclContext> {
4570
4571public:
4572 explicit CheckVarsEscapingUntiedTaskDeclContext() = default;
4573 virtual ~CheckVarsEscapingUntiedTaskDeclContext() = default;
4574 void VisitDeclStmt(const DeclStmt *S) {
4575 if (!S)
4576 return;
4577 // Need to privatize only local vars, static locals can be processed as is.
4578 for (const Decl *D : S->decls()) {
4579 if (const auto *VD = dyn_cast_or_null<VarDecl>(D))
4580 if (VD->hasLocalStorage())
4581 PrivateDecls.push_back(VD);
4582 }
4583 }
4584 void VisitOMPExecutableDirective(const OMPExecutableDirective *) {}
4585 void VisitCapturedStmt(const CapturedStmt *) {}
4586 void VisitLambdaExpr(const LambdaExpr *) {}
4587 void VisitBlockExpr(const BlockExpr *) {}
4588 void VisitStmt(const Stmt *S) {
4589 if (!S)
4590 return;
4591 for (const Stmt *Child : S->children())
4592 if (Child)
4593 Visit(Child);
4594 }
4595
4596 /// Swaps list of vars with the provided one.
4597 ArrayRef<const VarDecl *> getPrivateDecls() const { return PrivateDecls; }
4598};
4599} // anonymous namespace
4600
4603
4604 // First look for 'omp_all_memory' and add this first.
4605 bool OmpAllMemory = false;
4606 if (llvm::any_of(
4607 S.getClausesOfKind<OMPDependClause>(), [](const OMPDependClause *C) {
4608 return C->getDependencyKind() == OMPC_DEPEND_outallmemory ||
4609 C->getDependencyKind() == OMPC_DEPEND_inoutallmemory;
4610 })) {
4611 OmpAllMemory = true;
4612 // Since both OMPC_DEPEND_outallmemory and OMPC_DEPEND_inoutallmemory are
4613 // equivalent to the runtime, always use OMPC_DEPEND_outallmemory to
4614 // simplify.
4616 Data.Dependences.emplace_back(OMPC_DEPEND_outallmemory,
4617 /*IteratorExpr=*/nullptr);
4618 // Add a nullptr Expr to simplify the codegen in emitDependData.
4619 DD.DepExprs.push_back(nullptr);
4620 }
4621 // Add remaining dependences skipping any 'out' or 'inout' if they are
4622 // overridden by 'omp_all_memory'.
4623 for (const auto *C : S.getClausesOfKind<OMPDependClause>()) {
4624 OpenMPDependClauseKind Kind = C->getDependencyKind();
4625 if (Kind == OMPC_DEPEND_outallmemory || Kind == OMPC_DEPEND_inoutallmemory)
4626 continue;
4627 if (OmpAllMemory && (Kind == OMPC_DEPEND_out || Kind == OMPC_DEPEND_inout))
4628 continue;
4630 Data.Dependences.emplace_back(C->getDependencyKind(), C->getModifier());
4631 DD.DepExprs.append(C->varlist_begin(), C->varlist_end());
4632 }
4633}
4634
4636 const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
4637 const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
4639 // Emit outlined function for task construct.
4640 const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
4641 auto I = CS->getCapturedDecl()->param_begin();
4642 auto PartId = std::next(I);
4643 auto TaskT = std::next(I, 4);
4644 // Check if the task is final
4645 if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
4646 // If the condition constant folds and can be elided, try to avoid emitting
4647 // the condition and the dead arm of the if/else.
4648 const Expr *Cond = Clause->getCondition();
4649 bool CondConstant;
4650 if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
4651 Data.Final.setInt(CondConstant);
4652 else
4653 Data.Final.setPointer(EvaluateExprAsBool(Cond));
4654 } else {
4655 // By default the task is not final.
4656 Data.Final.setInt(/*IntVal=*/false);
4657 }
4658 // Check if the task has 'priority' clause.
4659 if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
4660 const Expr *Prio = Clause->getPriority();
4661 Data.Priority.setInt(/*IntVal=*/true);
4662 Data.Priority.setPointer(EmitScalarConversion(
4663 EmitScalarExpr(Prio), Prio->getType(),
4664 getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
4665 Prio->getExprLoc()));
4666 }
4667 // The first function argument for tasks is a thread id, the second one is a
4668 // part id (0 for tied tasks, >=0 for untied task).
4669 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
4670 // Get list of private variables.
4671 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
4672 auto IRef = C->varlist_begin();
4673 for (const Expr *IInit : C->private_copies()) {
4674 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4675 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4676 Data.PrivateVars.push_back(*IRef);
4677 Data.PrivateCopies.push_back(IInit);
4678 }
4679 ++IRef;
4680 }
4681 }
4682 EmittedAsPrivate.clear();
4683 // Get list of firstprivate variables.
4684 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
4685 auto IRef = C->varlist_begin();
4686 auto IElemInitRef = C->inits().begin();
4687 for (const Expr *IInit : C->private_copies()) {
4688 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4689 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4690 Data.FirstprivateVars.push_back(*IRef);
4691 Data.FirstprivateCopies.push_back(IInit);
4692 Data.FirstprivateInits.push_back(*IElemInitRef);
4693 }
4694 ++IRef;
4695 ++IElemInitRef;
4696 }
4697 }
4698 // Get list of lastprivate variables (for taskloops).
4699 llvm::MapVector<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
4700 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
4701 auto IRef = C->varlist_begin();
4702 auto ID = C->destination_exprs().begin();
4703 for (const Expr *IInit : C->private_copies()) {
4704 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4705 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4706 Data.LastprivateVars.push_back(*IRef);
4707 Data.LastprivateCopies.push_back(IInit);
4708 }
4709 LastprivateDstsOrigs.insert(
4710 std::make_pair(cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
4711 cast<DeclRefExpr>(*IRef)));
4712 ++IRef;
4713 ++ID;
4714 }
4715 }
4718 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
4719 Data.ReductionVars.append(C->varlist_begin(), C->varlist_end());
4720 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
4721 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
4722 Data.ReductionOps.append(C->reduction_ops().begin(),
4723 C->reduction_ops().end());
4724 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
4725 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
4726 }
4728 *this, S.getBeginLoc(), LHSs, RHSs, Data);
4729 // Build list of dependences.
4731 // Get list of local vars for untied tasks.
4732 if (!Data.Tied) {
4733 CheckVarsEscapingUntiedTaskDeclContext Checker;
4734 Checker.Visit(S.getInnermostCapturedStmt()->getCapturedStmt());
4735 Data.PrivateLocals.append(Checker.getPrivateDecls().begin(),
4736 Checker.getPrivateDecls().end());
4737 }
4738 auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
4739 CapturedRegion](CodeGenFunction &CGF,
4740 PrePostActionTy &Action) {
4741 llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
4742 std::pair<Address, Address>>
4743 UntiedLocalVars;
4744 // Set proper addresses for generated private copies.
4745 OMPPrivateScope Scope(CGF);
4746 // Generate debug info for variables present in shared clause.
4747 if (auto *DI = CGF.getDebugInfo()) {
4748 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields =
4749 CGF.CapturedStmtInfo->getCaptureFields();
4750 llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue();
4751 if (CaptureFields.size() && ContextValue) {
4752 unsigned CharWidth = CGF.getContext().getCharWidth();
4753 // The shared variables are packed together as members of structure.
4754 // So the address of each shared variable can be computed by adding
4755 // offset of it (within record) to the base address of record. For each
4756 // shared variable, debug intrinsic llvm.dbg.declare is generated with
4757 // appropriate expressions (DIExpression).
4758 // Ex:
4759 // %12 = load %struct.anon*, %struct.anon** %__context.addr.i
4760 // call void @llvm.dbg.declare(metadata %struct.anon* %12,
4761 // metadata !svar1,
4762 // metadata !DIExpression(DW_OP_deref))
4763 // call void @llvm.dbg.declare(metadata %struct.anon* %12,
4764 // metadata !svar2,
4765 // metadata !DIExpression(DW_OP_plus_uconst, 8, DW_OP_deref))
4766 for (auto It = CaptureFields.begin(); It != CaptureFields.end(); ++It) {
4767 const VarDecl *SharedVar = It->first;
4768 RecordDecl *CaptureRecord = It->second->getParent();
4769 const ASTRecordLayout &Layout =
4770 CGF.getContext().getASTRecordLayout(CaptureRecord);
4771 unsigned Offset =
4772 Layout.getFieldOffset(It->second->getFieldIndex()) / CharWidth;
4773 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
4774 (void)DI->EmitDeclareOfAutoVariable(SharedVar, ContextValue,
4775 CGF.Builder, false);
4776 // Get the call dbg.declare instruction we just created and update
4777 // its DIExpression to add offset to base address.
4778 auto UpdateExpr = [](llvm::LLVMContext &Ctx, auto *Declare,
4779 unsigned Offset) {
4781 // Add offset to the base address if non zero.
4782 if (Offset) {
4783 Ops.push_back(llvm::dwarf::DW_OP_plus_uconst);
4784 Ops.push_back(Offset);
4785 }
4786 Ops.push_back(llvm::dwarf::DW_OP_deref);
4787 Declare->setExpression(llvm::DIExpression::get(Ctx, Ops));
4788 };
4789 llvm::Instruction &Last = CGF.Builder.GetInsertBlock()->back();
4790 if (auto DDI = dyn_cast<llvm::DbgVariableIntrinsic>(&Last))
4791 UpdateExpr(DDI->getContext(), DDI, Offset);
4792 // If we're emitting using the new debug info format into a block
4793 // without a terminator, the record will be "trailing".
4794 assert(!Last.isTerminator() && "unexpected terminator");
4795 if (auto *Marker =
4796 CGF.Builder.GetInsertBlock()->getTrailingDbgRecords()) {
4797 for (llvm::DbgVariableRecord &DVR : llvm::reverse(
4798 llvm::filterDbgVars(Marker->getDbgRecordRange()))) {
4799 UpdateExpr(Last.getContext(), &DVR, Offset);
4800 break;
4801 }
4802 }
4803 }
4804 }
4805 }