clang 18.0.0git
CGStmtOpenMP.cpp
Go to the documentation of this file.
1//===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit OpenMP nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCleanup.h"
14#include "CGOpenMPRuntime.h"
15#include "CodeGenFunction.h"
16#include "CodeGenModule.h"
17#include "TargetInfo.h"
19#include "clang/AST/Attr.h"
22#include "clang/AST/Stmt.h"
27#include "llvm/ADT/SmallSet.h"
28#include "llvm/BinaryFormat/Dwarf.h"
29#include "llvm/Frontend/OpenMP/OMPConstants.h"
30#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
31#include "llvm/IR/Constants.h"
32#include "llvm/IR/DebugInfoMetadata.h"
33#include "llvm/IR/Instructions.h"
34#include "llvm/IR/IntrinsicInst.h"
35#include "llvm/IR/Metadata.h"
36#include "llvm/Support/AtomicOrdering.h"
37#include <optional>
38using namespace clang;
39using namespace CodeGen;
40using namespace llvm::omp;
41
42static const VarDecl *getBaseDecl(const Expr *Ref);
43
44namespace {
45/// Lexical scope for OpenMP executable constructs, that handles correct codegen
46/// for captured expressions.
47class OMPLexicalScope : public CodeGenFunction::LexicalScope {
48 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
49 for (const auto *C : S.clauses()) {
50 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
51 if (const auto *PreInit =
52 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
53 for (const auto *I : PreInit->decls()) {
54 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
55 CGF.EmitVarDecl(cast<VarDecl>(*I));
56 } else {
57 CodeGenFunction::AutoVarEmission Emission =
58 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
59 CGF.EmitAutoVarCleanups(Emission);
60 }
61 }
62 }
63 }
64 }
65 }
66 CodeGenFunction::OMPPrivateScope InlinedShareds;
67
68 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
69 return CGF.LambdaCaptureFields.lookup(VD) ||
70 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
71 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) &&
72 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
73 }
74
75public:
76 OMPLexicalScope(
78 const std::optional<OpenMPDirectiveKind> CapturedRegion = std::nullopt,
79 const bool EmitPreInitStmt = true)
80 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
81 InlinedShareds(CGF) {
82 if (EmitPreInitStmt)
83 emitPreInitStmt(CGF, S);
84 if (!CapturedRegion)
85 return;
86 assert(S.hasAssociatedStmt() &&
87 "Expected associated statement for inlined directive.");
88 const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
89 for (const auto &C : CS->captures()) {
90 if (C.capturesVariable() || C.capturesVariableByCopy()) {
91 auto *VD = C.getCapturedVar();
92 assert(VD == VD->getCanonicalDecl() &&
93 "Canonical decl must be captured.");
94 DeclRefExpr DRE(
95 CGF.getContext(), const_cast<VarDecl *>(VD),
96 isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
97 InlinedShareds.isGlobalVarCaptured(VD)),
98 VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
99 InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress(CGF));
100 }
101 }
102 (void)InlinedShareds.Privatize();
103 }
104};
105
106/// Lexical scope for OpenMP parallel construct, that handles correct codegen
107/// for captured expressions.
108class OMPParallelScope final : public OMPLexicalScope {
109 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
110 OpenMPDirectiveKind Kind = S.getDirectiveKind();
111 return !(isOpenMPTargetExecutionDirective(Kind) ||
114 }
115
116public:
117 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
118 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt,
119 EmitPreInitStmt(S)) {}
120};
121
122/// Lexical scope for OpenMP teams construct, that handles correct codegen
123/// for captured expressions.
124class OMPTeamsScope final : public OMPLexicalScope {
125 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
126 OpenMPDirectiveKind Kind = S.getDirectiveKind();
127 return !isOpenMPTargetExecutionDirective(Kind) &&
129 }
130
131public:
132 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
133 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt,
134 EmitPreInitStmt(S)) {}
135};
136
137/// Private scope for OpenMP loop-based directives, that supports capturing
138/// of used expression from loop statement.
139class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
140 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopBasedDirective &S) {
141 const DeclStmt *PreInits;
142 CodeGenFunction::OMPMapVars PreCondVars;
143 if (auto *LD = dyn_cast<OMPLoopDirective>(&S)) {
144 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
145 for (const auto *E : LD->counters()) {
146 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
147 EmittedAsPrivate.insert(VD->getCanonicalDecl());
148 (void)PreCondVars.setVarAddr(
149 CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
150 }
151 // Mark private vars as undefs.
152 for (const auto *C : LD->getClausesOfKind<OMPPrivateClause>()) {
153 for (const Expr *IRef : C->varlists()) {
154 const auto *OrigVD =
155 cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
156 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
157 QualType OrigVDTy = OrigVD->getType().getNonReferenceType();
158 (void)PreCondVars.setVarAddr(
159 CGF, OrigVD,
160 Address(llvm::UndefValue::get(CGF.ConvertTypeForMem(
161 CGF.getContext().getPointerType(OrigVDTy))),
162 CGF.ConvertTypeForMem(OrigVDTy),
163 CGF.getContext().getDeclAlign(OrigVD)));
164 }
165 }
166 }
167 (void)PreCondVars.apply(CGF);
168 // Emit init, __range and __end variables for C++ range loops.
170 LD->getInnermostCapturedStmt()->getCapturedStmt(),
171 /*TryImperfectlyNestedLoops=*/true, LD->getLoopsNumber(),
172 [&CGF](unsigned Cnt, const Stmt *CurStmt) {
173 if (const auto *CXXFor = dyn_cast<CXXForRangeStmt>(CurStmt)) {
174 if (const Stmt *Init = CXXFor->getInit())
175 CGF.EmitStmt(Init);
176 CGF.EmitStmt(CXXFor->getRangeStmt());
177 CGF.EmitStmt(CXXFor->getEndStmt());
178 }
179 return false;
180 });
181 PreInits = cast_or_null<DeclStmt>(LD->getPreInits());
182 } else if (const auto *Tile = dyn_cast<OMPTileDirective>(&S)) {
183 PreInits = cast_or_null<DeclStmt>(Tile->getPreInits());
184 } else if (const auto *Unroll = dyn_cast<OMPUnrollDirective>(&S)) {
185 PreInits = cast_or_null<DeclStmt>(Unroll->getPreInits());
186 } else {
187 llvm_unreachable("Unknown loop-based directive kind.");
188 }
189 if (PreInits) {
190 for (const auto *I : PreInits->decls())
191 CGF.EmitVarDecl(cast<VarDecl>(*I));
192 }
193 PreCondVars.restore(CGF);
194 }
195
196public:
197 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopBasedDirective &S)
198 : CodeGenFunction::RunCleanupsScope(CGF) {
199 emitPreInitStmt(CGF, S);
200 }
201};
202
203class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
204 CodeGenFunction::OMPPrivateScope InlinedShareds;
205
206 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
207 return CGF.LambdaCaptureFields.lookup(VD) ||
208 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
209 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) &&
210 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
211 }
212
213public:
214 OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
215 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
216 InlinedShareds(CGF) {
217 for (const auto *C : S.clauses()) {
218 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
219 if (const auto *PreInit =
220 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
221 for (const auto *I : PreInit->decls()) {
222 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
223 CGF.EmitVarDecl(cast<VarDecl>(*I));
224 } else {
225 CodeGenFunction::AutoVarEmission Emission =
226 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
227 CGF.EmitAutoVarCleanups(Emission);
228 }
229 }
230 }
231 } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
232 for (const Expr *E : UDP->varlists()) {
233 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
234 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
235 CGF.EmitVarDecl(*OED);
236 }
237 } else if (const auto *UDP = dyn_cast<OMPUseDeviceAddrClause>(C)) {
238 for (const Expr *E : UDP->varlists()) {
239 const Decl *D = getBaseDecl(E);
240 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
241 CGF.EmitVarDecl(*OED);
242 }
243 }
244 }
245 if (!isOpenMPSimdDirective(S.getDirectiveKind()))
246 CGF.EmitOMPPrivateClause(S, InlinedShareds);
247 if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
248 if (const Expr *E = TG->getReductionRef())
249 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
250 }
251 // Temp copy arrays for inscan reductions should not be emitted as they are
252 // not used in simd only mode.
254 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
255 if (C->getModifier() != OMPC_REDUCTION_inscan)
256 continue;
257 for (const Expr *E : C->copy_array_temps())
258 CopyArrayTemps.insert(cast<DeclRefExpr>(E)->getDecl());
259 }
260 const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
261 while (CS) {
262 for (auto &C : CS->captures()) {
263 if (C.capturesVariable() || C.capturesVariableByCopy()) {
264 auto *VD = C.getCapturedVar();
265 if (CopyArrayTemps.contains(VD))
266 continue;
267 assert(VD == VD->getCanonicalDecl() &&
268 "Canonical decl must be captured.");
269 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
270 isCapturedVar(CGF, VD) ||
271 (CGF.CapturedStmtInfo &&
272 InlinedShareds.isGlobalVarCaptured(VD)),
274 C.getLocation());
275 InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress(CGF));
276 }
277 }
278 CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
279 }
280 (void)InlinedShareds.Privatize();
281 }
282};
283
284} // namespace
285
287 const OMPExecutableDirective &S,
288 const RegionCodeGenTy &CodeGen);
289
290LValue CodeGenFunction::EmitOMPSharedLValue(const Expr *E) {
291 if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
292 if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
293 OrigVD = OrigVD->getCanonicalDecl();
294 bool IsCaptured =
295 LambdaCaptureFields.lookup(OrigVD) ||
297 (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
298 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), IsCaptured,
299 OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
300 return EmitLValue(&DRE);
301 }
302 }
303 return EmitLValue(E);
304}
305
306llvm::Value *CodeGenFunction::getTypeSize(QualType Ty) {
308 llvm::Value *Size = nullptr;
309 auto SizeInChars = C.getTypeSizeInChars(Ty);
310 if (SizeInChars.isZero()) {
311 // getTypeSizeInChars() returns 0 for a VLA.
312 while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) {
313 VlaSizePair VlaSize = getVLASize(VAT);
314 Ty = VlaSize.Type;
315 Size =
316 Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts) : VlaSize.NumElts;
317 }
318 SizeInChars = C.getTypeSizeInChars(Ty);
319 if (SizeInChars.isZero())
320 return llvm::ConstantInt::get(SizeTy, /*V=*/0);
321 return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
322 }
323 return CGM.getSize(SizeInChars);
324}
325
327 const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
328 const RecordDecl *RD = S.getCapturedRecordDecl();
329 auto CurField = RD->field_begin();
330 auto CurCap = S.captures().begin();
331 for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
332 E = S.capture_init_end();
333 I != E; ++I, ++CurField, ++CurCap) {
334 if (CurField->hasCapturedVLAType()) {
335 const VariableArrayType *VAT = CurField->getCapturedVLAType();
336 llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
337 CapturedVars.push_back(Val);
338 } else if (CurCap->capturesThis()) {
339 CapturedVars.push_back(CXXThisValue);
340 } else if (CurCap->capturesVariableByCopy()) {
341 llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
342
343 // If the field is not a pointer, we need to save the actual value
344 // and load it as a void pointer.
345 if (!CurField->getType()->isAnyPointerType()) {
346 ASTContext &Ctx = getContext();
347 Address DstAddr = CreateMemTemp(
348 Ctx.getUIntPtrType(),
349 Twine(CurCap->getCapturedVar()->getName(), ".casted"));
350 LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
351
352 llvm::Value *SrcAddrVal = EmitScalarConversion(
353 DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
354 Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
355 LValue SrcLV =
356 MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
357
358 // Store the value using the source type pointer.
360
361 // Load the value using the destination type pointer.
362 CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
363 }
364 CapturedVars.push_back(CV);
365 } else {
366 assert(CurCap->capturesVariable() && "Expected capture by reference.");
367 CapturedVars.push_back(EmitLValue(*I).getAddress(*this).getPointer());
368 }
369 }
370}
371
373 QualType DstType, StringRef Name,
374 LValue AddrLV) {
375 ASTContext &Ctx = CGF.getContext();
376
377 llvm::Value *CastedPtr = CGF.EmitScalarConversion(
378 AddrLV.getAddress(CGF).getPointer(), Ctx.getUIntPtrType(),
379 Ctx.getPointerType(DstType), Loc);
380 Address TmpAddr =
381 CGF.MakeNaturalAlignAddrLValue(CastedPtr, DstType).getAddress(CGF);
382 return TmpAddr;
383}
384
386 if (T->isLValueReferenceType())
387 return C.getLValueReferenceType(
389 /*SpelledAsLValue=*/false);
390 if (T->isPointerType())
391 return C.getPointerType(getCanonicalParamType(C, T->getPointeeType()));
392 if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
393 if (const auto *VLA = dyn_cast<VariableArrayType>(A))
394 return getCanonicalParamType(C, VLA->getElementType());
395 if (!A->isVariablyModifiedType())
396 return C.getCanonicalType(T);
397 }
398 return C.getCanonicalParamType(T);
399}
400
401namespace {
402/// Contains required data for proper outlined function codegen.
403struct FunctionOptions {
404 /// Captured statement for which the function is generated.
405 const CapturedStmt *S = nullptr;
406 /// true if cast to/from UIntPtr is required for variables captured by
407 /// value.
408 const bool UIntPtrCastRequired = true;
409 /// true if only casted arguments must be registered as local args or VLA
410 /// sizes.
411 const bool RegisterCastedArgsOnly = false;
412 /// Name of the generated function.
413 const StringRef FunctionName;
414 /// Location of the non-debug version of the outlined function.
415 SourceLocation Loc;
416 explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
417 bool RegisterCastedArgsOnly, StringRef FunctionName,
418 SourceLocation Loc)
419 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
420 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
421 FunctionName(FunctionName), Loc(Loc) {}
422};
423} // namespace
424
425static llvm::Function *emitOutlinedFunctionPrologue(
427 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
428 &LocalAddrs,
429 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
430 &VLASizes,
431 llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
432 const CapturedDecl *CD = FO.S->getCapturedDecl();
433 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
434 assert(CD->hasBody() && "missing CapturedDecl body");
435
436 CXXThisValue = nullptr;
437 // Build the argument list.
438 CodeGenModule &CGM = CGF.CGM;
439 ASTContext &Ctx = CGM.getContext();
440 FunctionArgList TargetArgs;
441 Args.append(CD->param_begin(),
442 std::next(CD->param_begin(), CD->getContextParamPosition()));
443 TargetArgs.append(
444 CD->param_begin(),
445 std::next(CD->param_begin(), CD->getContextParamPosition()));
446 auto I = FO.S->captures().begin();
447 FunctionDecl *DebugFunctionDecl = nullptr;
448 if (!FO.UIntPtrCastRequired) {
450 QualType FunctionTy = Ctx.getFunctionType(Ctx.VoidTy, std::nullopt, EPI);
451 DebugFunctionDecl = FunctionDecl::Create(
452 Ctx, Ctx.getTranslationUnitDecl(), FO.S->getBeginLoc(),
453 SourceLocation(), DeclarationName(), FunctionTy,
454 Ctx.getTrivialTypeSourceInfo(FunctionTy), SC_Static,
455 /*UsesFPIntrin=*/false, /*isInlineSpecified=*/false,
456 /*hasWrittenPrototype=*/false);
457 }
458 for (const FieldDecl *FD : RD->fields()) {
459 QualType ArgType = FD->getType();
460 IdentifierInfo *II = nullptr;
461 VarDecl *CapVar = nullptr;
462
463 // If this is a capture by copy and the type is not a pointer, the outlined
464 // function argument type should be uintptr and the value properly casted to
465 // uintptr. This is necessary given that the runtime library is only able to
466 // deal with pointers. We can pass in the same way the VLA type sizes to the
467 // outlined function.
468 if (FO.UIntPtrCastRequired &&
469 ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
470 I->capturesVariableArrayType()))
471 ArgType = Ctx.getUIntPtrType();
472
473 if (I->capturesVariable() || I->capturesVariableByCopy()) {
474 CapVar = I->getCapturedVar();
475 II = CapVar->getIdentifier();
476 } else if (I->capturesThis()) {
477 II = &Ctx.Idents.get("this");
478 } else {
479 assert(I->capturesVariableArrayType());
480 II = &Ctx.Idents.get("vla");
481 }
482 if (ArgType->isVariablyModifiedType())
483 ArgType = getCanonicalParamType(Ctx, ArgType);
484 VarDecl *Arg;
485 if (CapVar && (CapVar->getTLSKind() != clang::VarDecl::TLS_None)) {
486 Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
487 II, ArgType,
489 } else if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
491 Ctx, DebugFunctionDecl,
492 CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),
493 CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
494 /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
495 } else {
496 Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
497 II, ArgType, ImplicitParamDecl::Other);
498 }
499 Args.emplace_back(Arg);
500 // Do not cast arguments if we emit function with non-original types.
501 TargetArgs.emplace_back(
502 FO.UIntPtrCastRequired
503 ? Arg
504 : CGM.getOpenMPRuntime().translateParameter(FD, Arg));
505 ++I;
506 }
507 Args.append(std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
508 CD->param_end());
509 TargetArgs.append(
510 std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
511 CD->param_end());
512
513 // Create the function declaration.
514 const CGFunctionInfo &FuncInfo =
516 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
517
518 auto *F =
519 llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
520 FO.FunctionName, &CGM.getModule());
521 CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
522 if (CD->isNothrow())
523 F->setDoesNotThrow();
524 F->setDoesNotRecurse();
525
526 // Always inline the outlined function if optimizations are enabled.
527 if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
528 F->removeFnAttr(llvm::Attribute::NoInline);
529 F->addFnAttr(llvm::Attribute::AlwaysInline);
530 }
531
532 // Generate the function.
533 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
534 FO.UIntPtrCastRequired ? FO.Loc : FO.S->getBeginLoc(),
535 FO.UIntPtrCastRequired ? FO.Loc
536 : CD->getBody()->getBeginLoc());
537 unsigned Cnt = CD->getContextParamPosition();
538 I = FO.S->captures().begin();
539 for (const FieldDecl *FD : RD->fields()) {
540 // Do not map arguments if we emit function with non-original types.
541 Address LocalAddr(Address::invalid());
542 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
543 LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
544 TargetArgs[Cnt]);
545 } else {
546 LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
547 }
548 // If we are capturing a pointer by copy we don't need to do anything, just
549 // use the value that we get from the arguments.
550 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
551 const VarDecl *CurVD = I->getCapturedVar();
552 if (!FO.RegisterCastedArgsOnly)
553 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
554 ++Cnt;
555 ++I;
556 continue;
557 }
558
559 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
561 if (FD->hasCapturedVLAType()) {
562 if (FO.UIntPtrCastRequired) {
563 ArgLVal = CGF.MakeAddrLValue(
564 castValueFromUintptr(CGF, I->getLocation(), FD->getType(),
565 Args[Cnt]->getName(), ArgLVal),
567 }
568 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
569 const VariableArrayType *VAT = FD->getCapturedVLAType();
570 VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg);
571 } else if (I->capturesVariable()) {
572 const VarDecl *Var = I->getCapturedVar();
573 QualType VarTy = Var->getType();
574 Address ArgAddr = ArgLVal.getAddress(CGF);
575 if (ArgLVal.getType()->isLValueReferenceType()) {
576 ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
577 } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) {
578 assert(ArgLVal.getType()->isPointerType());
579 ArgAddr = CGF.EmitLoadOfPointer(
580 ArgAddr, ArgLVal.getType()->castAs<PointerType>());
581 }
582 if (!FO.RegisterCastedArgsOnly) {
583 LocalAddrs.insert(
584 {Args[Cnt], {Var, ArgAddr.withAlignment(Ctx.getDeclAlign(Var))}});
585 }
586 } else if (I->capturesVariableByCopy()) {
587 assert(!FD->getType()->isAnyPointerType() &&
588 "Not expecting a captured pointer.");
589 const VarDecl *Var = I->getCapturedVar();
590 LocalAddrs.insert({Args[Cnt],
591 {Var, FO.UIntPtrCastRequired
593 CGF, I->getLocation(), FD->getType(),
594 Args[Cnt]->getName(), ArgLVal)
595 : ArgLVal.getAddress(CGF)}});
596 } else {
597 // If 'this' is captured, load it into CXXThisValue.
598 assert(I->capturesThis());
599 CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
600 LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress(CGF)}});
601 }
602 ++Cnt;
603 ++I;
604 }
605
606 return F;
607}
608
609llvm::Function *
611 SourceLocation Loc) {
612 assert(
614 "CapturedStmtInfo should be set when generating the captured function");
615 const CapturedDecl *CD = S.getCapturedDecl();
616 // Build the argument list.
617 bool NeedWrapperFunction =
619 FunctionArgList Args;
620 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
621 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
622 SmallString<256> Buffer;
623 llvm::raw_svector_ostream Out(Buffer);
625 if (NeedWrapperFunction)
626 Out << "_debug__";
627 FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
628 Out.str(), Loc);
629 llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs,
630 VLASizes, CXXThisValue, FO);
631 CodeGenFunction::OMPPrivateScope LocalScope(*this);
632 for (const auto &LocalAddrPair : LocalAddrs) {
633 if (LocalAddrPair.second.first) {
634 LocalScope.addPrivate(LocalAddrPair.second.first,
635 LocalAddrPair.second.second);
636 }
637 }
638 (void)LocalScope.Privatize();
639 for (const auto &VLASizePair : VLASizes)
640 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
642 CapturedStmtInfo->EmitBody(*this, CD->getBody());
643 (void)LocalScope.ForceCleanup();
645 if (!NeedWrapperFunction)
646 return F;
647
648 FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
649 /*RegisterCastedArgsOnly=*/true,
651 CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
652 WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
653 Args.clear();
654 LocalAddrs.clear();
655 VLASizes.clear();
656 llvm::Function *WrapperF =
657 emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
658 WrapperCGF.CXXThisValue, WrapperFO);
660 auto *PI = F->arg_begin();
661 for (const auto *Arg : Args) {
662 llvm::Value *CallArg;
663 auto I = LocalAddrs.find(Arg);
664 if (I != LocalAddrs.end()) {
665 LValue LV = WrapperCGF.MakeAddrLValue(
666 I->second.second,
667 I->second.first ? I->second.first->getType() : Arg->getType(),
669 if (LV.getType()->isAnyComplexType())
670 LV.setAddress(WrapperCGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
671 LV.getAddress(WrapperCGF),
672 PI->getType()->getPointerTo(
673 LV.getAddress(WrapperCGF).getAddressSpace()),
674 PI->getType()));
675 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
676 } else {
677 auto EI = VLASizes.find(Arg);
678 if (EI != VLASizes.end()) {
679 CallArg = EI->second.second;
680 } else {
681 LValue LV =
682 WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
683 Arg->getType(), AlignmentSource::Decl);
684 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
685 }
686 }
687 CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
688 ++PI;
689 }
690 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, Loc, F, CallArgs);
691 WrapperCGF.FinishFunction();
692 return WrapperF;
693}
694
695//===----------------------------------------------------------------------===//
696// OpenMP Directive Emission
697//===----------------------------------------------------------------------===//
699 Address DestAddr, Address SrcAddr, QualType OriginalType,
700 const llvm::function_ref<void(Address, Address)> CopyGen) {
701 // Perform element-by-element initialization.
702 QualType ElementTy;
703
704 // Drill down to the base element type on both arrays.
705 const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
706 llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
707 SrcAddr = SrcAddr.withElementType(DestAddr.getElementType());
708
709 llvm::Value *SrcBegin = SrcAddr.getPointer();
710 llvm::Value *DestBegin = DestAddr.getPointer();
711 // Cast from pointer to array type to pointer to single element.
712 llvm::Value *DestEnd = Builder.CreateInBoundsGEP(DestAddr.getElementType(),
713 DestBegin, NumElements);
714
715 // The basic structure here is a while-do loop.
716 llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body");
717 llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done");
718 llvm::Value *IsEmpty =
719 Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
720 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
721
722 // Enter the loop body, making that address the current address.
723 llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
724 EmitBlock(BodyBB);
725
726 CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
727
728 llvm::PHINode *SrcElementPHI =
729 Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
730 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
731 Address SrcElementCurrent =
732 Address(SrcElementPHI, SrcAddr.getElementType(),
733 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
734
735 llvm::PHINode *DestElementPHI = Builder.CreatePHI(
736 DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
737 DestElementPHI->addIncoming(DestBegin, EntryBB);
738 Address DestElementCurrent =
739 Address(DestElementPHI, DestAddr.getElementType(),
740 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
741
742 // Emit copy.
743 CopyGen(DestElementCurrent, SrcElementCurrent);
744
745 // Shift the address forward by one element.
746 llvm::Value *DestElementNext =
747 Builder.CreateConstGEP1_32(DestAddr.getElementType(), DestElementPHI,
748 /*Idx0=*/1, "omp.arraycpy.dest.element");
749 llvm::Value *SrcElementNext =
750 Builder.CreateConstGEP1_32(SrcAddr.getElementType(), SrcElementPHI,
751 /*Idx0=*/1, "omp.arraycpy.src.element");
752 // Check whether we've reached the end.
753 llvm::Value *Done =
754 Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
755 Builder.CreateCondBr(Done, DoneBB, BodyBB);
756 DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
757 SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
758
759 // Done.
760 EmitBlock(DoneBB, /*IsFinished=*/true);
761}
762
763void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
764 Address SrcAddr, const VarDecl *DestVD,
765 const VarDecl *SrcVD, const Expr *Copy) {
766 if (OriginalType->isArrayType()) {
767 const auto *BO = dyn_cast<BinaryOperator>(Copy);
768 if (BO && BO->getOpcode() == BO_Assign) {
769 // Perform simple memcpy for simple copying.
770 LValue Dest = MakeAddrLValue(DestAddr, OriginalType);
771 LValue Src = MakeAddrLValue(SrcAddr, OriginalType);
772 EmitAggregateAssign(Dest, Src, OriginalType);
773 } else {
774 // For arrays with complex element types perform element by element
775 // copying.
777 DestAddr, SrcAddr, OriginalType,
778 [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
779 // Working with the single array element, so have to remap
780 // destination and source variables to corresponding array
781 // elements.
782 CodeGenFunction::OMPPrivateScope Remap(*this);
783 Remap.addPrivate(DestVD, DestElement);
784 Remap.addPrivate(SrcVD, SrcElement);
785 (void)Remap.Privatize();
787 });
788 }
789 } else {
790 // Remap pseudo source variable to private copy.
791 CodeGenFunction::OMPPrivateScope Remap(*this);
792 Remap.addPrivate(SrcVD, SrcAddr);
793 Remap.addPrivate(DestVD, DestAddr);
794 (void)Remap.Privatize();
795 // Emit copying of the whole variable.
797 }
798}
799
801 OMPPrivateScope &PrivateScope) {
802 if (!HaveInsertPoint())
803 return false;
804 bool DeviceConstTarget =
805 getLangOpts().OpenMPIsTargetDevice &&
807 bool FirstprivateIsLastprivate = false;
808 llvm::DenseMap<const VarDecl *, OpenMPLastprivateModifier> Lastprivates;
809 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
810 for (const auto *D : C->varlists())
811 Lastprivates.try_emplace(
812 cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl(),
813 C->getKind());
814 }
815 llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
817 getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind());
818 // Force emission of the firstprivate copy if the directive does not emit
819 // outlined function, like omp for, omp simd, omp distribute etc.
820 bool MustEmitFirstprivateCopy =
821 CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
822 for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
823 const auto *IRef = C->varlist_begin();
824 const auto *InitsRef = C->inits().begin();
825 for (const Expr *IInit : C->private_copies()) {
826 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
827 bool ThisFirstprivateIsLastprivate =
828 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
829 const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
830 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
831 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
832 !FD->getType()->isReferenceType() &&
833 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
834 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
835 ++IRef;
836 ++InitsRef;
837 continue;
838 }
839 // Do not emit copy for firstprivate constant variables in target regions,
840 // captured by reference.
841 if (DeviceConstTarget && OrigVD->getType().isConstant(getContext()) &&
842 FD && FD->getType()->isReferenceType() &&
843 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
844 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
845 ++IRef;
846 ++InitsRef;
847 continue;
848 }
849 FirstprivateIsLastprivate =
850 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
851 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
852 const auto *VDInit =
853 cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
854 bool IsRegistered;
855 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
856 /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
857 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
858 LValue OriginalLVal;
859 if (!FD) {
860 // Check if the firstprivate variable is just a constant value.
861 ConstantEmission CE = tryEmitAsConstant(&DRE);
862 if (CE && !CE.isReference()) {
863 // Constant value, no need to create a copy.
864 ++IRef;
865 ++InitsRef;
866 continue;
867 }
868 if (CE && CE.isReference()) {
869 OriginalLVal = CE.getReferenceLValue(*this, &DRE);
870 } else {
871 assert(!CE && "Expected non-constant firstprivate.");
872 OriginalLVal = EmitLValue(&DRE);
873 }
874 } else {
875 OriginalLVal = EmitLValue(&DRE);
876 }
877 QualType Type = VD->getType();
878 if (Type->isArrayType()) {
879 // Emit VarDecl with copy init for arrays.
880 // Get the address of the original variable captured in current
881 // captured region.
882 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
883 const Expr *Init = VD->getInit();
884 if (!isa<CXXConstructExpr>(Init) || isTrivialInitializer(Init)) {
885 // Perform simple memcpy.
886 LValue Dest = MakeAddrLValue(Emission.getAllocatedAddress(), Type);
887 EmitAggregateAssign(Dest, OriginalLVal, Type);
888 } else {
890 Emission.getAllocatedAddress(), OriginalLVal.getAddress(*this),
891 Type,
892 [this, VDInit, Init](Address DestElement, Address SrcElement) {
893 // Clean up any temporaries needed by the
894 // initialization.
895 RunCleanupsScope InitScope(*this);
896 // Emit initialization for single element.
897 setAddrOfLocalVar(VDInit, SrcElement);
898 EmitAnyExprToMem(Init, DestElement,
899 Init->getType().getQualifiers(),
900 /*IsInitializer*/ false);
901 LocalDeclMap.erase(VDInit);
902 });
903 }
904 EmitAutoVarCleanups(Emission);
905 IsRegistered =
906 PrivateScope.addPrivate(OrigVD, Emission.getAllocatedAddress());
907 } else {
908 Address OriginalAddr = OriginalLVal.getAddress(*this);
909 // Emit private VarDecl with copy init.
910 // Remap temp VDInit variable to the address of the original
911 // variable (for proper handling of captured global variables).
912 setAddrOfLocalVar(VDInit, OriginalAddr);
913 EmitDecl(*VD);
914 LocalDeclMap.erase(VDInit);
915 Address VDAddr = GetAddrOfLocalVar(VD);
916 if (ThisFirstprivateIsLastprivate &&
917 Lastprivates[OrigVD->getCanonicalDecl()] ==
918 OMPC_LASTPRIVATE_conditional) {
919 // Create/init special variable for lastprivate conditionals.
920 llvm::Value *V =
921 EmitLoadOfScalar(MakeAddrLValue(VDAddr, (*IRef)->getType(),
923 (*IRef)->getExprLoc());
925 *this, OrigVD);
926 EmitStoreOfScalar(V, MakeAddrLValue(VDAddr, (*IRef)->getType(),
928 LocalDeclMap.erase(VD);
929 setAddrOfLocalVar(VD, VDAddr);
930 }
931 IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);
932 }
933 assert(IsRegistered &&
934 "firstprivate var already registered as private");
935 // Silence the warning about unused variable.
936 (void)IsRegistered;
937 }
938 ++IRef;
939 ++InitsRef;
940 }
941 }
942 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
943}
944
946 const OMPExecutableDirective &D,
947 CodeGenFunction::OMPPrivateScope &PrivateScope) {
948 if (!HaveInsertPoint())
949 return;
950 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
951 for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
952 auto IRef = C->varlist_begin();
953 for (const Expr *IInit : C->private_copies()) {
954 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
955 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
956 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
957 EmitDecl(*VD);
958 // Emit private VarDecl with copy init.
959 bool IsRegistered =
960 PrivateScope.addPrivate(OrigVD, GetAddrOfLocalVar(VD));
961 assert(IsRegistered && "private var already registered as private");
962 // Silence the warning about unused variable.
963 (void)IsRegistered;
964 }
965 ++IRef;
966 }
967 }
968}
969
971 if (!HaveInsertPoint())
972 return false;
973 // threadprivate_var1 = master_threadprivate_var1;
974 // operator=(threadprivate_var2, master_threadprivate_var2);
975 // ...
976 // __kmpc_barrier(&loc, global_tid);
978 llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
979 for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
980 auto IRef = C->varlist_begin();
981 auto ISrcRef = C->source_exprs().begin();
982 auto IDestRef = C->destination_exprs().begin();
983 for (const Expr *AssignOp : C->assignment_ops()) {
984 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
985 QualType Type = VD->getType();
986 if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
987 // Get the address of the master variable. If we are emitting code with
988 // TLS support, the address is passed from the master as field in the
989 // captured declaration.
990 Address MasterAddr = Address::invalid();
991 if (getLangOpts().OpenMPUseTLS &&
992 getContext().getTargetInfo().isTLSSupported()) {
993 assert(CapturedStmtInfo->lookup(VD) &&
994 "Copyin threadprivates should have been captured!");
995 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), true,
996 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
997 MasterAddr = EmitLValue(&DRE).getAddress(*this);
998 LocalDeclMap.erase(VD);
999 } else {
1000 MasterAddr =
1002 : CGM.GetAddrOfGlobal(VD),
1004 getContext().getDeclAlign(VD));
1005 }
1006 // Get the address of the threadprivate variable.
1007 Address PrivateAddr = EmitLValue(*IRef).getAddress(*this);
1008 if (CopiedVars.size() == 1) {
1009 // At first check if current thread is a master thread. If it is, no
1010 // need to copy data.
1011 CopyBegin = createBasicBlock("copyin.not.master");
1012 CopyEnd = createBasicBlock("copyin.not.master.end");
1013 // TODO: Avoid ptrtoint conversion.
1014 auto *MasterAddrInt =
1015 Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy);
1016 auto *PrivateAddrInt =
1017 Builder.CreatePtrToInt(PrivateAddr.getPointer(), CGM.IntPtrTy);
1018 Builder.CreateCondBr(
1019 Builder.CreateICmpNE(MasterAddrInt, PrivateAddrInt), CopyBegin,
1020 CopyEnd);
1021 EmitBlock(CopyBegin);
1022 }
1023 const auto *SrcVD =
1024 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1025 const auto *DestVD =
1026 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1027 EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
1028 }
1029 ++IRef;
1030 ++ISrcRef;
1031 ++IDestRef;
1032 }
1033 }
1034 if (CopyEnd) {
1035 // Exit out of copying procedure for non-master thread.
1036 EmitBlock(CopyEnd, /*IsFinished=*/true);
1037 return true;
1038 }
1039 return false;
1040}
1041
1043 const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
1044 if (!HaveInsertPoint())
1045 return false;
1046 bool HasAtLeastOneLastprivate = false;
1049 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1050 for (const Expr *C : LoopDirective->counters()) {
1051 SIMDLCVs.insert(
1052 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
1053 }
1054 }
1055 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1056 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1057 HasAtLeastOneLastprivate = true;
1059 !getLangOpts().OpenMPSimd)
1060 break;
1061 const auto *IRef = C->varlist_begin();
1062 const auto *IDestRef = C->destination_exprs().begin();
1063 for (const Expr *IInit : C->private_copies()) {
1064 // Keep the address of the original variable for future update at the end
1065 // of the loop.
1066 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1067 // Taskloops do not require additional initialization, it is done in
1068 // runtime support library.
1069 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
1070 const auto *DestVD =
1071 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1072 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1073 /*RefersToEnclosingVariableOrCapture=*/
1074 CapturedStmtInfo->lookup(OrigVD) != nullptr,
1075 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
1076 PrivateScope.addPrivate(DestVD, EmitLValue(&DRE).getAddress(*this));
1077 // Check if the variable is also a firstprivate: in this case IInit is
1078 // not generated. Initialization of this variable will happen in codegen
1079 // for 'firstprivate' clause.
1080 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
1081 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
1082 Address VDAddr = Address::invalid();
1083 if (C->getKind() == OMPC_LASTPRIVATE_conditional) {
1085 *this, OrigVD);
1086 setAddrOfLocalVar(VD, VDAddr);
1087 } else {
1088 // Emit private VarDecl with copy init.
1089 EmitDecl(*VD);
1090 VDAddr = GetAddrOfLocalVar(VD);
1091 }
1092 bool IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);
1093 assert(IsRegistered &&
1094 "lastprivate var already registered as private");
1095 (void)IsRegistered;
1096 }
1097 }
1098 ++IRef;
1099 ++IDestRef;
1100 }
1101 }
1102 return HasAtLeastOneLastprivate;
1103}
1104
1106 const OMPExecutableDirective &D, bool NoFinals,
1107 llvm::Value *IsLastIterCond) {
1108 if (!HaveInsertPoint())
1109 return;
1110 // Emit following code:
1111 // if (<IsLastIterCond>) {
1112 // orig_var1 = private_orig_var1;
1113 // ...
1114 // orig_varn = private_orig_varn;
1115 // }
1116 llvm::BasicBlock *ThenBB = nullptr;
1117 llvm::BasicBlock *DoneBB = nullptr;
1118 if (IsLastIterCond) {
1119 // Emit implicit barrier if at least one lastprivate conditional is found
1120 // and this is not a simd mode.
1121 if (!getLangOpts().OpenMPSimd &&
1122 llvm::any_of(D.getClausesOfKind<OMPLastprivateClause>(),
1123 [](const OMPLastprivateClause *C) {
1124 return C->getKind() == OMPC_LASTPRIVATE_conditional;
1125 })) {
1127 OMPD_unknown,
1128 /*EmitChecks=*/false,
1129 /*ForceSimpleCall=*/true);
1130 }
1131 ThenBB = createBasicBlock(".omp.lastprivate.then");
1132 DoneBB = createBasicBlock(".omp.lastprivate.done");
1133 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1134 EmitBlock(ThenBB);
1135 }
1136 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1137 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1138 if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1139 auto IC = LoopDirective->counters().begin();
1140 for (const Expr *F : LoopDirective->finals()) {
1141 const auto *D =
1142 cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1143 if (NoFinals)
1144 AlreadyEmittedVars.insert(D);
1145 else
1146 LoopCountersAndUpdates[D] = F;
1147 ++IC;
1148 }
1149 }
1150 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1151 auto IRef = C->varlist_begin();
1152 auto ISrcRef = C->source_exprs().begin();
1153 auto IDestRef = C->destination_exprs().begin();
1154 for (const Expr *AssignOp : C->assignment_ops()) {
1155 const auto *PrivateVD =
1156 cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1157 QualType Type = PrivateVD->getType();
1158 const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1159 if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1160 // If lastprivate variable is a loop control variable for loop-based
1161 // directive, update its value before copyin back to original
1162 // variable.
1163 if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1164 EmitIgnoredExpr(FinalExpr);
1165 const auto *SrcVD =
1166 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1167 const auto *DestVD =
1168 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1169 // Get the address of the private variable.
1170 Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
1171 if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1172 PrivateAddr = Address(
1173 Builder.CreateLoad(PrivateAddr),
1174 CGM.getTypes().ConvertTypeForMem(RefTy->getPointeeType()),
1175 CGM.getNaturalTypeAlignment(RefTy->getPointeeType()));
1176 // Store the last value to the private copy in the last iteration.
1177 if (C->getKind() == OMPC_LASTPRIVATE_conditional)
1179 *this, MakeAddrLValue(PrivateAddr, (*IRef)->getType()), PrivateVD,
1180 (*IRef)->getExprLoc());
1181 // Get the address of the original variable.
1182 Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1183 EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1184 }
1185 ++IRef;
1186 ++ISrcRef;
1187 ++IDestRef;
1188 }
1189 if (const Expr *PostUpdate = C->getPostUpdateExpr())
1190 EmitIgnoredExpr(PostUpdate);
1191 }
1192 if (IsLastIterCond)
1193 EmitBlock(DoneBB, /*IsFinished=*/true);
1194}
1195
1197 const OMPExecutableDirective &D,
1198 CodeGenFunction::OMPPrivateScope &PrivateScope, bool ForInscan) {
1199 if (!HaveInsertPoint())
1200 return;
1203 SmallVector<const Expr *, 4> ReductionOps;
1209 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1210 if (ForInscan != (C->getModifier() == OMPC_REDUCTION_inscan))
1211 continue;
1212 Shareds.append(C->varlist_begin(), C->varlist_end());
1213 Privates.append(C->privates().begin(), C->privates().end());
1214 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1215 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1216 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1217 if (C->getModifier() == OMPC_REDUCTION_task) {
1218 Data.ReductionVars.append(C->privates().begin(), C->privates().end());
1219 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
1220 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
1221 Data.ReductionOps.append(C->reduction_ops().begin(),
1222 C->reduction_ops().end());
1223 TaskLHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1224 TaskRHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1225 }
1226 }
1227 ReductionCodeGen RedCG(Shareds, Shareds, Privates, ReductionOps);
1228 unsigned Count = 0;
1229 auto *ILHS = LHSs.begin();
1230 auto *IRHS = RHSs.begin();
1231 auto *IPriv = Privates.begin();
1232 for (const Expr *IRef : Shareds) {
1233 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1234 // Emit private VarDecl with reduction init.
1235 RedCG.emitSharedOrigLValue(*this, Count);
1236 RedCG.emitAggregateType(*this, Count);
1237 AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);
1238 RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
1239 RedCG.getSharedLValue(Count).getAddress(*this),
1240 [&Emission](CodeGenFunction &CGF) {
1241 CGF.EmitAutoVarInit(Emission);
1242 return true;
1243 });
1244 EmitAutoVarCleanups(Emission);
1245 Address BaseAddr = RedCG.adjustPrivateAddress(
1246 *this, Count, Emission.getAllocatedAddress());
1247 bool IsRegistered =
1248 PrivateScope.addPrivate(RedCG.getBaseDecl(Count), BaseAddr);
1249 assert(IsRegistered && "private var already registered as private");
1250 // Silence the warning about unused variable.
1251 (void)IsRegistered;
1252
1253 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1254 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1255 QualType Type = PrivateVD->getType();
1256 bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1257 if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1258 // Store the address of the original variable associated with the LHS
1259 // implicit variable.
1260 PrivateScope.addPrivate(LHSVD,
1261 RedCG.getSharedLValue(Count).getAddress(*this));
1262 PrivateScope.addPrivate(RHSVD, GetAddrOfLocalVar(PrivateVD));
1263 } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1264 isa<ArraySubscriptExpr>(IRef)) {
1265 // Store the address of the original variable associated with the LHS
1266 // implicit variable.
1267 PrivateScope.addPrivate(LHSVD,
1268 RedCG.getSharedLValue(Count).getAddress(*this));
1269 PrivateScope.addPrivate(RHSVD,
1270 GetAddrOfLocalVar(PrivateVD).withElementType(
1271 ConvertTypeForMem(RHSVD->getType())));
1272 } else {
1273 QualType Type = PrivateVD->getType();
1274 bool IsArray = getContext().getAsArrayType(Type) != nullptr;
1275 Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress(*this);
1276 // Store the address of the original variable associated with the LHS
1277 // implicit variable.
1278 if (IsArray) {
1279 OriginalAddr =
1280 OriginalAddr.withElementType(ConvertTypeForMem(LHSVD->getType()));
1281 }
1282 PrivateScope.addPrivate(LHSVD, OriginalAddr);
1283 PrivateScope.addPrivate(
1284 RHSVD, IsArray ? GetAddrOfLocalVar(PrivateVD).withElementType(
1285 ConvertTypeForMem(RHSVD->getType()))
1286 : GetAddrOfLocalVar(PrivateVD));
1287 }
1288 ++ILHS;
1289 ++IRHS;
1290 ++IPriv;
1291 ++Count;
1292 }
1293 if (!Data.ReductionVars.empty()) {
1294 Data.IsReductionWithTaskMod = true;
1295 Data.IsWorksharingReduction =
1297 llvm::Value *ReductionDesc = CGM.getOpenMPRuntime().emitTaskReductionInit(
1298 *this, D.getBeginLoc(), TaskLHSs, TaskRHSs, Data);
1299 const Expr *TaskRedRef = nullptr;
1300 switch (D.getDirectiveKind()) {
1301 case OMPD_parallel:
1302 TaskRedRef = cast<OMPParallelDirective>(D).getTaskReductionRefExpr();
1303 break;
1304 case OMPD_for:
1305 TaskRedRef = cast<OMPForDirective>(D).getTaskReductionRefExpr();
1306 break;
1307 case OMPD_sections:
1308 TaskRedRef = cast<OMPSectionsDirective>(D).getTaskReductionRefExpr();
1309 break;
1310 case OMPD_parallel_for:
1311 TaskRedRef = cast<OMPParallelForDirective>(D).getTaskReductionRefExpr();
1312 break;
1313 case OMPD_parallel_master:
1314 TaskRedRef =
1315 cast<OMPParallelMasterDirective>(D).getTaskReductionRefExpr();
1316 break;
1317 case OMPD_parallel_sections:
1318 TaskRedRef =
1319 cast<OMPParallelSectionsDirective>(D).getTaskReductionRefExpr();
1320 break;
1321 case OMPD_target_parallel:
1322 TaskRedRef =
1323 cast<OMPTargetParallelDirective>(D).getTaskReductionRefExpr();
1324 break;
1325 case OMPD_target_parallel_for:
1326 TaskRedRef =
1327 cast<OMPTargetParallelForDirective>(D).getTaskReductionRefExpr();
1328 break;
1329 case OMPD_distribute_parallel_for:
1330 TaskRedRef =
1331 cast<OMPDistributeParallelForDirective>(D).getTaskReductionRefExpr();
1332 break;
1333 case OMPD_teams_distribute_parallel_for:
1334 TaskRedRef = cast<OMPTeamsDistributeParallelForDirective>(D)
1335 .getTaskReductionRefExpr();
1336 break;
1337 case OMPD_target_teams_distribute_parallel_for:
1338 TaskRedRef = cast<OMPTargetTeamsDistributeParallelForDirective>(D)
1339 .getTaskReductionRefExpr();
1340 break;
1341 case OMPD_simd:
1342 case OMPD_for_simd:
1343 case OMPD_section:
1344 case OMPD_single:
1345 case OMPD_master:
1346 case OMPD_critical:
1347 case OMPD_parallel_for_simd:
1348 case OMPD_task:
1349 case OMPD_taskyield:
1350 case OMPD_error:
1351 case OMPD_barrier:
1352 case OMPD_taskwait:
1353 case OMPD_taskgroup:
1354 case OMPD_flush:
1355 case OMPD_depobj:
1356 case OMPD_scan:
1357 case OMPD_ordered:
1358 case OMPD_atomic:
1359 case OMPD_teams:
1360 case OMPD_target:
1361 case OMPD_cancellation_point:
1362 case OMPD_cancel:
1363 case OMPD_target_data:
1364 case OMPD_target_enter_data:
1365 case OMPD_target_exit_data:
1366 case OMPD_taskloop:
1367 case OMPD_taskloop_simd:
1368 case OMPD_master_taskloop:
1369 case OMPD_master_taskloop_simd:
1370 case OMPD_parallel_master_taskloop:
1371 case OMPD_parallel_master_taskloop_simd:
1372 case OMPD_distribute:
1373 case OMPD_target_update:
1374 case OMPD_distribute_parallel_for_simd:
1375 case OMPD_distribute_simd:
1376 case OMPD_target_parallel_for_simd:
1377 case OMPD_target_simd:
1378 case OMPD_teams_distribute:
1379 case OMPD_teams_distribute_simd:
1380 case OMPD_teams_distribute_parallel_for_simd:
1381 case OMPD_target_teams:
1382 case OMPD_target_teams_distribute:
1383 case OMPD_target_teams_distribute_parallel_for_simd:
1384 case OMPD_target_teams_distribute_simd:
1385 case OMPD_declare_target:
1386 case OMPD_end_declare_target:
1387 case OMPD_threadprivate:
1388 case OMPD_allocate:
1389 case OMPD_declare_reduction:
1390 case OMPD_declare_mapper:
1391 case OMPD_declare_simd:
1392 case OMPD_requires:
1393 case OMPD_declare_variant:
1394 case OMPD_begin_declare_variant:
1395 case OMPD_end_declare_variant:
1396 case OMPD_unknown:
1397 default:
1398 llvm_unreachable("Enexpected directive with task reductions.");
1399 }
1400
1401 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(TaskRedRef)->getDecl());
1402 EmitVarDecl(*VD);
1403 EmitStoreOfScalar(ReductionDesc, GetAddrOfLocalVar(VD),
1404 /*Volatile=*/false, TaskRedRef->getType());
1405 }
1406}
1407
1409 const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1410 if (!HaveInsertPoint())
1411 return;
1416 bool HasAtLeastOneReduction = false;
1417 bool IsReductionWithTaskMod = false;
1418 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1419 // Do not emit for inscan reductions.
1420 if (C->getModifier() == OMPC_REDUCTION_inscan)
1421 continue;
1422 HasAtLeastOneReduction = true;
1423 Privates.append(C->privates().begin(), C->privates().end());
1424 LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1425 RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1426 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1427 IsReductionWithTaskMod =
1428 IsReductionWithTaskMod || C->getModifier() == OMPC_REDUCTION_task;
1429 }
1430 if (HasAtLeastOneReduction) {
1431 if (IsReductionWithTaskMod) {
1433 *this, D.getBeginLoc(),
1435 }
1436 bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1438 ReductionKind == OMPD_simd;
1439 bool SimpleReduction = ReductionKind == OMPD_simd;
1440 // Emit nowait reduction if nowait clause is present or directive is a
1441 // parallel directive (it always has implicit barrier).
1443 *this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1444 {WithNowait, SimpleReduction, ReductionKind});
1445 }
1446}
1447
1450 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1451 if (!CGF.HaveInsertPoint())
1452 return;
1453 llvm::BasicBlock *DoneBB = nullptr;
1454 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1455 if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
1456 if (!DoneBB) {
1457 if (llvm::Value *Cond = CondGen(CGF)) {
1458 // If the first post-update expression is found, emit conditional
1459 // block if it was requested.
1460 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1461 DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1462 CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1463 CGF.EmitBlock(ThenBB);
1464 }
1465 }
1466 CGF.EmitIgnoredExpr(PostUpdate);
1467 }
1468 }
1469 if (DoneBB)
1470 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1471}
1472
1473namespace {
1474/// Codegen lambda for appending distribute lower and upper bounds to outlined
1475/// parallel function. This is necessary for combined constructs such as
1476/// 'distribute parallel for'
1477typedef llvm::function_ref<void(CodeGenFunction &,
1478 const OMPExecutableDirective &,
1480 CodeGenBoundParametersTy;
1481} // anonymous namespace
1482
1483static void
1485 const OMPExecutableDirective &S) {
1486 if (CGF.getLangOpts().OpenMP < 50)
1487 return;
1489 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
1490 for (const Expr *Ref : C->varlists()) {
1491 if (!Ref->getType()->isScalarType())
1492 continue;
1493 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1494 if (!DRE)
1495 continue;
1496 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1498 }
1499 }
1500 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
1501 for (const Expr *Ref : C->varlists()) {
1502 if (!Ref->getType()->isScalarType())
1503 continue;
1504 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1505 if (!DRE)
1506 continue;
1507 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1509 }
1510 }
1511 for (const auto *C : S.getClausesOfKind<OMPLinearClause>()) {
1512 for (const Expr *Ref : C->varlists()) {
1513 if (!Ref->getType()->isScalarType())
1514 continue;
1515 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1516 if (!DRE)
1517 continue;
1518 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1520 }
1521 }
1522 // Privates should ne analyzed since they are not captured at all.
1523 // Task reductions may be skipped - tasks are ignored.
1524 // Firstprivates do not return value but may be passed by reference - no need
1525 // to check for updated lastprivate conditional.
1526 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
1527 for (const Expr *Ref : C->varlists()) {
1528 if (!Ref->getType()->isScalarType())
1529 continue;
1530 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1531 if (!DRE)
1532 continue;
1533 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1534 }
1535 }
1537 CGF, S, PrivateDecls);
1538}
1539
1542 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1543 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1544 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1545 llvm::Value *NumThreads = nullptr;
1546 llvm::Function *OutlinedFn =
1548 CGF, S, *CS->getCapturedDecl()->param_begin(), InnermostKind,
1549 CodeGen);
1550 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
1551 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1552 NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1553 /*IgnoreResultAssign=*/true);
1555 CGF, NumThreads, NumThreadsClause->getBeginLoc());
1556 }
1557 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
1558 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1560 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1561 }
1562 const Expr *IfCond = nullptr;
1563 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1564 if (C->getNameModifier() == OMPD_unknown ||
1565 C->getNameModifier() == OMPD_parallel) {
1566 IfCond = C->getCondition();
1567 break;
1568 }
1569 }
1570
1571 OMPParallelScope Scope(CGF, S);
1573 // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
1574 // lower and upper bounds with the pragma 'for' chunking mechanism.
1575 // The following lambda takes care of appending the lower and upper bound
1576 // parameters when necessary
1577 CodeGenBoundParameters(CGF, S, CapturedVars);
1578 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
1579 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn,
1580 CapturedVars, IfCond, NumThreads);
1581}
1582
1583static bool isAllocatableDecl(const VarDecl *VD) {
1584 const VarDecl *CVD = VD->getCanonicalDecl();
1585 if (!CVD->hasAttr<OMPAllocateDeclAttr>())
1586 return false;
1587 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
1588 // Use the default allocation.
1589 return !((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||
1590 AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&
1591 !AA->getAllocator());
1592}
1593
1595 const OMPExecutableDirective &,
1597
1599 const OMPExecutableDirective &S) {
1600 bool Copyins = CGF.EmitOMPCopyinClause(S);
1601 if (Copyins) {
1602 // Emit implicit barrier to synchronize threads and avoid data races on
1603 // propagation master's thread values of threadprivate variables to local
1604 // instances of that variables of all other implicit threads.
1606 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
1607 /*ForceSimpleCall=*/true);
1608 }
1609}
1610
1612 CodeGenFunction &CGF, const VarDecl *VD) {
1613 CodeGenModule &CGM = CGF.CGM;
1614 auto &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1615
1616 if (!VD)
1617 return Address::invalid();
1618 const VarDecl *CVD = VD->getCanonicalDecl();
1619 if (!isAllocatableDecl(CVD))
1620 return Address::invalid();
1621 llvm::Value *Size;
1622 CharUnits Align = CGM.getContext().getDeclAlign(CVD);
1623 if (CVD->getType()->isVariablyModifiedType()) {
1624 Size = CGF.getTypeSize(CVD->getType());
1625 // Align the size: ((size + align - 1) / align) * align
1626 Size = CGF.Builder.CreateNUWAdd(
1627 Size, CGM.getSize(Align - CharUnits::fromQuantity(1)));
1628 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align));
1629 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align));
1630 } else {
1632 Size = CGM.getSize(Sz.alignTo(Align));
1633 }
1634
1635 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
1636 assert(AA->getAllocator() &&
1637 "Expected allocator expression for non-default allocator.");
1638 llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator());
1639 // According to the standard, the original allocator type is a enum (integer).
1640 // Convert to pointer type, if required.
1641 if (Allocator->getType()->isIntegerTy())
1642 Allocator = CGF.Builder.CreateIntToPtr(Allocator, CGM.VoidPtrTy);
1643 else if (Allocator->getType()->isPointerTy())
1644 Allocator = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Allocator,
1645 CGM.VoidPtrTy);
1646
1647 llvm::Value *Addr = OMPBuilder.createOMPAlloc(
1648 CGF.Builder, Size, Allocator,
1649 getNameWithSeparators({CVD->getName(), ".void.addr"}, ".", "."));
1650 llvm::CallInst *FreeCI =
1651 OMPBuilder.createOMPFree(CGF.Builder, Addr, Allocator);
1652
1653 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(NormalAndEHCleanup, FreeCI);
1655 Addr,
1657 getNameWithSeparators({CVD->getName(), ".addr"}, ".", "."));
1658 return Address(Addr, CGF.ConvertTypeForMem(CVD->getType()), Align);
1659}
1660
1662 CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr,
1663 SourceLocation Loc) {
1664 CodeGenModule &CGM = CGF.CGM;
1665 if (CGM.getLangOpts().OpenMPUseTLS &&
1667 return VDAddr;
1668
1669 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1670
1671 llvm::Type *VarTy = VDAddr.getElementType();
1672 llvm::Value *Data =
1673 CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy);
1674 llvm::ConstantInt *Size = CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy));
1675 std::string Suffix = getNameWithSeparators({"cache", ""});
1676 llvm::Twine CacheName = Twine(CGM.getMangledName(VD)).concat(Suffix);
1677
1678 llvm::CallInst *ThreadPrivateCacheCall =
1679 OMPBuilder.createCachedThreadPrivate(CGF.Builder, Data, Size, CacheName);
1680
1681 return Address(ThreadPrivateCacheCall, CGM.Int8Ty, VDAddr.getAlignment());
1682}
1683
1685 ArrayRef<StringRef> Parts, StringRef FirstSeparator, StringRef Separator) {
1686 SmallString<128> Buffer;
1687 llvm::raw_svector_ostream OS(Buffer);
1688 StringRef Sep = FirstSeparator;
1689 for (StringRef Part : Parts) {
1690 OS << Sep << Part;
1691 Sep = Separator;
1692 }
1693 return OS.str().str();
1694}
1695
1697 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
1698 InsertPointTy CodeGenIP, Twine RegionName) {
1700 Builder.restoreIP(CodeGenIP);
1701 llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder, /*CreateBranch=*/false,
1702 "." + RegionName + ".after");
1703
1704 {
1705 OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);
1706 CGF.EmitStmt(RegionBodyStmt);
1707 }
1708
1709 if (Builder.saveIP().isSet())
1710 Builder.CreateBr(FiniBB);
1711}
1712
1714 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
1715 InsertPointTy CodeGenIP, Twine RegionName) {
1717 Builder.restoreIP(CodeGenIP);
1718 llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder, /*CreateBranch=*/false,
1719 "." + RegionName + ".after");
1720
1721 {
1722 OMPBuilderCBHelpers::OutlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);
1723 CGF.EmitStmt(RegionBodyStmt);
1724 }
1725
1726 if (Builder.saveIP().isSet())
1727 Builder.CreateBr(FiniBB);
1728}
1729
1731 if (CGM.getLangOpts().OpenMPIRBuilder) {
1732 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1733 // Check if we have any if clause associated with the directive.
1734 llvm::Value *IfCond = nullptr;
1735 if (const auto *C = S.getSingleClause<OMPIfClause>())
1736 IfCond = EmitScalarExpr(C->getCondition(),
1737 /*IgnoreResultAssign=*/true);
1738
1739 llvm::Value *NumThreads = nullptr;
1740 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>())
1741 NumThreads = EmitScalarExpr(NumThreadsClause->getNumThreads(),
1742 /*IgnoreResultAssign=*/true);
1743
1744 ProcBindKind ProcBind = OMP_PROC_BIND_default;
1745 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>())
1746 ProcBind = ProcBindClause->getProcBindKind();
1747
1748 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1749
1750 // The cleanup callback that finalizes all variabels at the given location,
1751 // thus calls destructors etc.
1752 auto FiniCB = [this](InsertPointTy IP) {
1754 };
1755
1756 // Privatization callback that performs appropriate action for
1757 // shared/private/firstprivate/lastprivate/copyin/... variables.
1758 //
1759 // TODO: This defaults to shared right now.
1760 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
1761 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
1762 // The next line is appropriate only for variables (Val) with the
1763 // data-sharing attribute "shared".
1764 ReplVal = &Val;
1765
1766 return CodeGenIP;
1767 };
1768
1769 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1770 const Stmt *ParallelRegionBodyStmt = CS->getCapturedStmt();
1771
1772 auto BodyGenCB = [&, this](InsertPointTy AllocaIP,
1773 InsertPointTy CodeGenIP) {
1775 *this, ParallelRegionBodyStmt, AllocaIP, CodeGenIP, "parallel");
1776 };
1777
1778 CGCapturedStmtInfo CGSI(*CS, CR_OpenMP);
1779 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
1780 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
1781 AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
1782 Builder.restoreIP(
1783 OMPBuilder.createParallel(Builder, AllocaIP, BodyGenCB, PrivCB, FiniCB,
1784 IfCond, NumThreads, ProcBind, S.hasCancel()));
1785 return;
1786 }
1787
1788 // Emit parallel region as a standalone region.
1789 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1790 Action.Enter(CGF);
1791 OMPPrivateScope PrivateScope(CGF);
1792 emitOMPCopyinClause(CGF, S);
1793 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1794 CGF.EmitOMPPrivateClause(S, PrivateScope);
1795 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1796 (void)PrivateScope.Privatize();
1797 CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1798 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
1799 };
1800 {
1801 auto LPCRegion =
1803 emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
1806 [](CodeGenFunction &) { return nullptr; });
1807 }
1808 // Check for outer lastprivate conditional update.
1810}
1811
1813 EmitStmt(S.getIfStmt());
1814}
1815
1816namespace {
1817/// RAII to handle scopes for loop transformation directives.
1818class OMPTransformDirectiveScopeRAII {
1819 OMPLoopScope *Scope = nullptr;
1820 CodeGenFunction::CGCapturedStmtInfo *CGSI = nullptr;
1821 CodeGenFunction::CGCapturedStmtRAII *CapInfoRAII = nullptr;
1822
1823 OMPTransformDirectiveScopeRAII(const OMPTransformDirectiveScopeRAII &) =
1824 delete;
1825 OMPTransformDirectiveScopeRAII &
1826 operator=(const OMPTransformDirectiveScopeRAII &) = delete;
1827
1828public:
1829 OMPTransformDirectiveScopeRAII(CodeGenFunction &CGF, const Stmt *S) {
1830 if (const auto *Dir = dyn_cast<OMPLoopBasedDirective>(S)) {
1831 Scope = new OMPLoopScope(CGF, *Dir);
1832 CGSI = new CodeGenFunction::CGCapturedStmtInfo(CR_OpenMP);
1833 CapInfoRAII = new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI);
1834 }
1835 }
1836 ~OMPTransformDirectiveScopeRAII() {
1837 if (!Scope)
1838 return;
1839 delete CapInfoRAII;
1840 delete CGSI;
1841 delete Scope;
1842 }
1843};
1844} // namespace
1845
1846static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
1847 int MaxLevel, int Level = 0) {
1848 assert(Level < MaxLevel && "Too deep lookup during loop body codegen.");
1849 const Stmt *SimplifiedS = S->IgnoreContainers();
1850 if (const auto *CS = dyn_cast<CompoundStmt>(SimplifiedS)) {
1851 PrettyStackTraceLoc CrashInfo(
1852 CGF.getContext().getSourceManager(), CS->getLBracLoc(),
1853 "LLVM IR generation of compound statement ('{}')");
1854
1855 // Keep track of the current cleanup stack depth, including debug scopes.
1856 CodeGenFunction::LexicalScope Scope(CGF, S->getSourceRange());
1857 for (const Stmt *CurStmt : CS->body())
1858 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level);
1859 return;
1860 }
1861 if (SimplifiedS == NextLoop) {
1862 if (auto *Dir = dyn_cast<OMPLoopTransformationDirective>(SimplifiedS))
1863 SimplifiedS = Dir->getTransformedStmt();
1864 if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(SimplifiedS))
1865 SimplifiedS = CanonLoop->getLoopStmt();
1866 if (const auto *For = dyn_cast<ForStmt>(SimplifiedS)) {
1867 S = For->getBody();
1868 } else {
1869 assert(isa<CXXForRangeStmt>(SimplifiedS) &&
1870 "Expected canonical for loop or range-based for loop.");
1871 const auto *CXXFor = cast<CXXForRangeStmt>(SimplifiedS);
1872 CGF.EmitStmt(CXXFor->getLoopVarStmt());
1873 S = CXXFor->getBody();
1874 }
1875 if (Level + 1 < MaxLevel) {
1877 S, /*TryImperfectlyNestedLoops=*/true);
1878 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1);
1879 return;
1880 }
1881 }
1882 CGF.EmitStmt(S);
1883}
1884
1886 JumpDest LoopExit) {
1887 RunCleanupsScope BodyScope(*this);
1888 // Update counters values on current iteration.
1889 for (const Expr *UE : D.updates())
1890 EmitIgnoredExpr(UE);
1891 // Update the linear variables.
1892 // In distribute directives only loop counters may be marked as linear, no
1893 // need to generate the code for them.
1895 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1896 for (const Expr *UE : C->updates())
1897 EmitIgnoredExpr(UE);
1898 }
1899 }
1900
1901 // On a continue in the body, jump to the end.
1902 JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue");
1903 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1904 for (const Expr *E : D.finals_conditions()) {
1905 if (!E)
1906 continue;
1907 // Check that loop counter in non-rectangular nest fits into the iteration
1908 // space.
1909 llvm::BasicBlock *NextBB = createBasicBlock("omp.body.next");
1910 EmitBranchOnBoolExpr(E, NextBB, Continue.getBlock(),
1912 EmitBlock(NextBB);
1913 }
1914
1915 OMPPrivateScope InscanScope(*this);
1916 EmitOMPReductionClauseInit(D, InscanScope, /*ForInscan=*/true);
1917 bool IsInscanRegion = InscanScope.Privatize();
1918 if (IsInscanRegion) {
1919 // Need to remember the block before and after scan directive
1920 // to dispatch them correctly depending on the clause used in
1921 // this directive, inclusive or exclusive. For inclusive scan the natural
1922 // order of the blocks is used, for exclusive clause the blocks must be
1923 // executed in reverse order.
1924 OMPBeforeScanBlock = createBasicBlock("omp.before.scan.bb");
1925 OMPAfterScanBlock = createBasicBlock("omp.after.scan.bb");
1926 // No need to allocate inscan exit block, in simd mode it is selected in the
1927 // codegen for the scan directive.
1928 if (D.getDirectiveKind() != OMPD_simd && !getLangOpts().OpenMPSimd)
1929 OMPScanExitBlock = createBasicBlock("omp.exit.inscan.bb");
1930 OMPScanDispatch = createBasicBlock("omp.inscan.dispatch");
1933 }
1934
1935 // Emit loop variables for C++ range loops.
1936 const Stmt *Body =
1938 // Emit loop body.
1939 emitBody(*this, Body,
1941 Body, /*TryImperfectlyNestedLoops=*/true),
1942 D.getLoopsNumber());
1943
1944 // Jump to the dispatcher at the end of the loop body.
1945 if (IsInscanRegion)
1947
1948 // The end (updates/cleanups).
1949 EmitBlock(Continue.getBlock());
1950 BreakContinueStack.pop_back();
1951}
1952
1953using EmittedClosureTy = std::pair<llvm::Function *, llvm::Value *>;
1954
1955/// Emit a captured statement and return the function as well as its captured
1956/// closure context.
1958 const CapturedStmt *S) {
1959 LValue CapStruct = ParentCGF.InitCapturedStruct(*S);
1960 CodeGenFunction CGF(ParentCGF.CGM, /*suppressNewContext=*/true);
1961 std::unique_ptr<CodeGenFunction::CGCapturedStmtInfo> CSI =
1962 std::make_unique<CodeGenFunction::CGCapturedStmtInfo>(*S);
1963 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, CSI.get());
1964 llvm::Function *F = CGF.GenerateCapturedStmtFunction(*S);
1965
1966 return {F, CapStruct.getPointer(ParentCGF)};
1967}
1968
1969/// Emit a call to a previously captured closure.
1970static llvm::CallInst *
1973 // Append the closure context to the argument.
1974 SmallVector<llvm::Value *> EffectiveArgs;
1975 EffectiveArgs.reserve(Args.size() + 1);
1976 llvm::append_range(EffectiveArgs, Args);
1977 EffectiveArgs.push_back(Cap.second);
1978
1979 return ParentCGF.Builder.CreateCall(Cap.first, EffectiveArgs);
1980}
1981
1982llvm::CanonicalLoopInfo *
1984 assert(Depth == 1 && "Nested loops with OpenMPIRBuilder not yet implemented");
1985
1986 // The caller is processing the loop-associated directive processing the \p
1987 // Depth loops nested in \p S. Put the previous pending loop-associated
1988 // directive to the stack. If the current loop-associated directive is a loop
1989 // transformation directive, it will push its generated loops onto the stack
1990 // such that together with the loops left here they form the combined loop
1991 // nest for the parent loop-associated directive.
1992 int ParentExpectedOMPLoopDepth = ExpectedOMPLoopDepth;
1993 ExpectedOMPLoopDepth = Depth;
1994
1995 EmitStmt(S);
1996 assert(OMPLoopNestStack.size() >= (size_t)Depth && "Found too few loops");
1997
1998 // The last added loop is the outermost one.
1999 llvm::CanonicalLoopInfo *Result = OMPLoopNestStack.back();
2000
2001 // Pop the \p Depth loops requested by the call from that stack and restore
2002 // the previous context.
2003 OMPLoopNestStack.pop_back_n(Depth);
2004 ExpectedOMPLoopDepth = ParentExpectedOMPLoopDepth;
2005
2006 return Result;
2007}
2008
2010 const Stmt *SyntacticalLoop = S->getLoopStmt();
2011 if (!getLangOpts().OpenMPIRBuilder) {
2012 // Ignore if OpenMPIRBuilder is not enabled.
2013 EmitStmt(SyntacticalLoop);
2014 return;
2015 }
2016
2017 LexicalScope ForScope(*this, S->getSourceRange());
2018
2019 // Emit init statements. The Distance/LoopVar funcs may reference variable
2020 // declarations they contain.
2021 const Stmt *BodyStmt;
2022 if (const auto *For = dyn_cast<ForStmt>(SyntacticalLoop)) {
2023 if (const Stmt *InitStmt = For->getInit())
2024 EmitStmt(InitStmt);
2025 BodyStmt = For->getBody();
2026 } else if (const auto *RangeFor =
2027 dyn_cast<CXXForRangeStmt>(SyntacticalLoop)) {
2028 if (const DeclStmt *RangeStmt = RangeFor->getRangeStmt())
2029 EmitStmt(RangeStmt);
2030 if (const DeclStmt *BeginStmt = RangeFor->getBeginStmt())
2031 EmitStmt(BeginStmt);
2032 if (const DeclStmt *EndStmt = RangeFor->getEndStmt())
2033 EmitStmt(EndStmt);
2034 if (const DeclStmt *LoopVarStmt = RangeFor->getLoopVarStmt())
2035 EmitStmt(LoopVarStmt);
2036 BodyStmt = RangeFor->getBody();
2037 } else
2038 llvm_unreachable("Expected for-stmt or range-based for-stmt");
2039
2040 // Emit closure for later use. By-value captures will be captured here.
2041 const CapturedStmt *DistanceFunc = S->getDistanceFunc();
2042 EmittedClosureTy DistanceClosure = emitCapturedStmtFunc(*this, DistanceFunc);
2043 const CapturedStmt *LoopVarFunc = S->getLoopVarFunc();
2044 EmittedClosureTy LoopVarClosure = emitCapturedStmtFunc(*this, LoopVarFunc);
2045
2046 // Call the distance function to get the number of iterations of the loop to
2047 // come.
2048 QualType LogicalTy = DistanceFunc->getCapturedDecl()
2049 ->getParam(0)
2050 ->getType()
2052 Address CountAddr = CreateMemTemp(LogicalTy, ".count.addr");
2053 emitCapturedStmtCall(*this, DistanceClosure, {CountAddr.getPointer()});
2054 llvm::Value *DistVal = Builder.CreateLoad(CountAddr, ".count");
2055
2056 // Emit the loop structure.
2057 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2058 auto BodyGen = [&, this](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP,
2059 llvm::Value *IndVar) {
2060 Builder.restoreIP(CodeGenIP);
2061
2062 // Emit the loop body: Convert the logical iteration number to the loop
2063 // variable and emit the body.
2064 const DeclRefExpr *LoopVarRef = S->getLoopVarRef();
2065 LValue LCVal = EmitLValue(LoopVarRef);
2066 Address LoopVarAddress = LCVal.getAddress(*this);
2067 emitCapturedStmtCall(*this, LoopVarClosure,
2068 {LoopVarAddress.getPointer(), IndVar});
2069
2070 RunCleanupsScope BodyScope(*this);
2071 EmitStmt(BodyStmt);
2072 };
2073 llvm::CanonicalLoopInfo *CL =
2074 OMPBuilder.createCanonicalLoop(Builder, BodyGen, DistVal);
2075
2076 // Finish up the loop.
2077 Builder.restoreIP(CL->getAfterIP());
2078 ForScope.ForceCleanup();
2079
2080 // Remember the CanonicalLoopInfo for parent AST nodes consuming it.
2081 OMPLoopNestStack.push_back(CL);
2082}
2083
2085 const OMPExecutableDirective &S, bool RequiresCleanup, const Expr *LoopCond,
2086 const Expr *IncExpr,
2087 const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
2088 const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
2089 auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
2090
2091 // Start the loop with a block that tests the condition.
2092 auto CondBlock = createBasicBlock("omp.inner.for.cond");
2093 EmitBlock(CondBlock);
2094 const SourceRange R = S.getSourceRange();
2095
2096 // If attributes are attached, push to the basic block with them.
2097 const auto &OMPED = cast<OMPExecutableDirective>(S);
2098 const CapturedStmt *ICS = OMPED.getInnermostCapturedStmt();
2099 const Stmt *SS = ICS->getCapturedStmt();
2100 const AttributedStmt *AS = dyn_cast_or_null<AttributedStmt>(SS);
2101 OMPLoopNestStack.clear();
2102 if (AS)
2106 else
2109
2110 // If there are any cleanups between here and the loop-exit scope,
2111 // create a block to stage a loop exit along.
2112 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
2113 if (RequiresCleanup)
2114 ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
2115
2116 llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body");
2117
2118 // Emit condition.
2119 EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
2120 if (ExitBlock != LoopExit.getBlock()) {
2121 EmitBlock(ExitBlock);
2123 }
2124
2125 EmitBlock(LoopBody);
2127
2128 // Create a block for the increment.
2129 JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
2130 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
2131
2132 BodyGen(*this);
2133
2134 // Emit "IV = IV + 1" and a back-edge to the condition block.
2135 EmitBlock(Continue.getBlock());
2136 EmitIgnoredExpr(IncExpr);
2137 PostIncGen(*this);
2138 BreakContinueStack.pop_back();
2139 EmitBranch(CondBlock);
2140 LoopStack.pop();
2141 // Emit the fall-through block.
2142 EmitBlock(LoopExit.getBlock());
2143}
2144
2146 if (!HaveInsertPoint())
2147 return false;
2148 // Emit inits for the linear variables.
2149 bool HasLinears = false;
2150 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2151 for (const Expr *Init : C->inits()) {
2152 HasLinears = true;
2153 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
2154 if (const auto *Ref =
2155 dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
2156 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
2157 const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
2158 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
2159 CapturedStmtInfo->lookup(OrigVD) != nullptr,
2160 VD->getInit()->getType(), VK_LValue,
2161 VD->getInit()->getExprLoc());
2163 &DRE, VD,
2164 MakeAddrLValue(Emission.getAllocatedAddress(), VD->getType()),
2165 /*capturedByInit=*/false);
2166 EmitAutoVarCleanups(Emission);
2167 } else {
2168 EmitVarDecl(*VD);
2169 }
2170 }
2171 // Emit the linear steps for the linear clauses.
2172 // If a step is not constant, it is pre-calculated before the loop.
2173 if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
2174 if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
2175 EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
2176 // Emit calculation of the linear step.
2177 EmitIgnoredExpr(CS);
2178 }
2179 }
2180 return HasLinears;
2181}
2182
2184 const OMPLoopDirective &D,
2185 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
2186 if (!HaveInsertPoint())
2187 return;
2188 llvm::BasicBlock *DoneBB = nullptr;
2189 // Emit the final values of the linear variables.
2190 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2191 auto IC = C->varlist_begin();
2192 for (const Expr *F : C->finals()) {
2193 if (!DoneBB) {
2194 if (llvm::Value *Cond = CondGen(*this)) {
2195 // If the first post-update expression is found, emit conditional
2196 // block if it was requested.
2197 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu");
2198 DoneBB = createBasicBlock(".omp.linear.pu.done");
2199 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2200 EmitBlock(ThenBB);
2201 }
2202 }
2203 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
2204 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
2205 CapturedStmtInfo->lookup(OrigVD) != nullptr,
2206 (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
2207 Address OrigAddr = EmitLValue(&DRE).getAddress(*this);
2208 CodeGenFunction::OMPPrivateScope VarScope(*this);
2209 VarScope.addPrivate(OrigVD, OrigAddr);
2210 (void)VarScope.Privatize();
2211 EmitIgnoredExpr(F);
2212 ++IC;
2213 }
2214 if (const Expr *PostUpdate = C->getPostUpdateExpr())
2215 EmitIgnoredExpr(PostUpdate);
2216 }
2217 if (DoneBB)
2218 EmitBlock(DoneBB, /*IsFinished=*/true);
2219}
2220
2222 const OMPExecutableDirective &D) {
2223 if (!CGF.HaveInsertPoint())
2224 return;
2225 for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
2226 llvm::APInt ClauseAlignment(64, 0);
2227 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
2228 auto *AlignmentCI =
2229 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
2230 ClauseAlignment = AlignmentCI->getValue();
2231 }
2232 for (const Expr *E : Clause->varlists()) {
2233 llvm::APInt Alignment(ClauseAlignment);
2234 if (Alignment == 0) {
2235 // OpenMP [2.8.1, Description]
2236 // If no optional parameter is specified, implementation-defined default
2237 // alignments for SIMD instructions on the target platforms are assumed.
2238 Alignment =
2239 CGF.getContext()
2241 E->getType()->getPointeeType()))
2242 .getQuantity();
2243 }
2244 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2245 "alignment is not power of 2");
2246 if (Alignment != 0) {
2247 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
2249 PtrValue, E, /*No second loc needed*/ SourceLocation(),
2250 llvm::ConstantInt::get(CGF.getLLVMContext(), Alignment));
2251 }
2252 }
2253 }
2254}
2255
2257 const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
2258 if (!HaveInsertPoint())
2259 return;
2260 auto I = S.private_counters().begin();
2261 for (const Expr *E : S.counters()) {
2262 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2263 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
2264 // Emit var without initialization.
2265 AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD);
2266 EmitAutoVarCleanups(VarEmission);
2267 LocalDeclMap.erase(PrivateVD);
2268 (void)LoopScope.addPrivate(VD, VarEmission.getAllocatedAddress());
2269 if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
2270 VD->hasGlobalStorage()) {
2271 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD),
2272 LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
2273 E->getType(), VK_LValue, E->getExprLoc());
2274 (void)LoopScope.addPrivate(PrivateVD, EmitLValue(&DRE).getAddress(*this));
2275 } else {
2276 (void)LoopScope.addPrivate(PrivateVD, VarEmission.getAllocatedAddress());
2277 }
2278 ++I;
2279 }
2280 // Privatize extra loop counters used in loops for ordered(n) clauses.
2281 for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) {
2282 if (!C->getNumForLoops())
2283 continue;
2284 for (unsigned I = S.getLoopsNumber(), E = C->getLoopNumIterations().size();
2285 I < E; ++I) {
2286 const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I));
2287 const auto *VD = cast<VarDecl>(DRE->getDecl());
2288 // Override only those variables that can be captured to avoid re-emission
2289 // of the variables declared within the loops.
2290 if (DRE->refersToEnclosingVariableOrCapture()) {
2291 (void)LoopScope.addPrivate(
2292 VD, CreateMemTemp(DRE->getType(), VD->getName()));
2293 }
2294 }
2295 }
2296}
2297
2299 const Expr *Cond, llvm::BasicBlock *TrueBlock,
2300 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
2301 if (!CGF.HaveInsertPoint())
2302 return;
2303 {
2304 CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
2305 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
2306 (void)PreCondScope.Privatize();
2307 // Get initial values of real counters.
2308 for (const Expr *I : S.inits()) {
2309 CGF.EmitIgnoredExpr(I);
2310 }
2311 }
2312 // Create temp loop control variables with their init values to support
2313 // non-rectangular loops.
2314 CodeGenFunction::OMPMapVars PreCondVars;
2315 for (const Expr *E : S.dependent_counters()) {
2316 if (!E)
2317 continue;
2318 assert(!E->getType().getNonReferenceType()->isRecordType() &&
2319 "dependent counter must not be an iterator.");
2320 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2321 Address CounterAddr =
2323 (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr);
2324 }
2325 (void)PreCondVars.apply(CGF);
2326 for (const Expr *E : S.dependent_inits()) {
2327 if (!E)
2328 continue;
2329 CGF.EmitIgnoredExpr(E);
2330 }
2331 // Check that loop is executed at least one time.
2332 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
2333 PreCondVars.restore(CGF);
2334}
2335
2337 const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
2338 if (!HaveInsertPoint())
2339 return;
2342 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
2343 for (const Expr *C : LoopDirective->counters()) {
2344 SIMDLCVs.insert(
2345 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
2346 }
2347 }
2348 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2349 auto CurPrivate = C->privates().begin();
2350 for (const Expr *E : C->varlists()) {
2351 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2352 const auto *PrivateVD =
2353 cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
2354 if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
2355 // Emit private VarDecl with copy init.
2356 EmitVarDecl(*PrivateVD);
2357 bool IsRegistered =
2358 PrivateScope.addPrivate(VD, GetAddrOfLocalVar(PrivateVD));
2359 assert(IsRegistered && "linear var already registered as private");
2360 // Silence the warning about unused variable.
2361 (void)IsRegistered;
2362 } else {
2363 EmitVarDecl(*PrivateVD);
2364 }
2365 ++CurPrivate;
2366 }
2367 }
2368}
2369
2371 const OMPExecutableDirective &D) {
2372 if (!CGF.HaveInsertPoint())
2373 return;
2374 if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
2375 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
2376 /*ignoreResult=*/true);
2377 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
2378 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
2379 // In presence of finite 'safelen', it may be unsafe to mark all
2380 // the memory instructions parallel, because loop-carried
2381 // dependences of 'safelen' iterations are possible.
2383 } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
2384 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
2385 /*ignoreResult=*/true);
2386 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
2387 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
2388 // In presence of finite 'safelen', it may be unsafe to mark all
2389 // the memory instructions parallel, because loop-carried
2390 // dependences of 'safelen' iterations are possible.
2391 CGF.LoopStack.setParallel(/*Enable=*/false);
2392 }
2393}
2394
2396 // Walk clauses and process safelen/lastprivate.
2397 LoopStack.setParallel(/*Enable=*/true);
2399 emitSimdlenSafelenClause(*this, D);
2400 if (const auto *C = D.getSingleClause<OMPOrderClause>())
2401 if (C->getKind() == OMPC_ORDER_concurrent)
2402 LoopStack.setParallel(/*Enable=*/true);
2403 if ((D.getDirectiveKind() == OMPD_simd ||
2404 (getLangOpts().OpenMPSimd &&
2406 llvm::any_of(D.getClausesOfKind<OMPReductionClause>(),
2407 [](const OMPReductionClause *C) {
2408 return C->getModifier() == OMPC_REDUCTION_inscan;
2409 }))
2410 // Disable parallel access in case of prefix sum.
2411 LoopStack.setParallel(/*Enable=*/false);
2412}
2413
2415 const OMPLoopDirective &D,
2416 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
2417 if (!HaveInsertPoint())
2418 return;
2419 llvm::BasicBlock *DoneBB = nullptr;
2420 auto IC = D.counters().begin();
2421 auto IPC = D.private_counters().begin();
2422 for (const Expr *F : D.finals()) {
2423 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
2424 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
2425 const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
2426 if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
2427 OrigVD->hasGlobalStorage() || CED) {
2428 if (!DoneBB) {
2429 if (llvm::Value *Cond = CondGen(*this)) {
2430 // If the first post-update expression is found, emit conditional
2431 // block if it was requested.
2432 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then");
2433 DoneBB = createBasicBlock(".omp.final.done");
2434 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2435 EmitBlock(ThenBB);
2436 }
2437 }
2438 Address OrigAddr = Address::invalid();
2439 if (CED) {
2440 OrigAddr =
2441 EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress(*this);
2442 } else {
2443 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(PrivateVD),
2444 /*RefersToEnclosingVariableOrCapture=*/false,
2445 (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
2446 OrigAddr = EmitLValue(&DRE).getAddress(*this);
2447 }
2448 OMPPrivateScope VarScope(*this);
2449 VarScope.addPrivate(OrigVD, OrigAddr);
2450 (void)VarScope.Privatize();
2451 EmitIgnoredExpr(F);
2452 }
2453 ++IC;
2454 ++IPC;
2455 }
2456 if (DoneBB)
2457 EmitBlock(DoneBB, /*IsFinished=*/true);
2458}
2459
2461 const OMPLoopDirective &S,
2462 CodeGenFunction::JumpDest LoopExit) {
2463 CGF.EmitOMPLoopBody(S, LoopExit);
2464 CGF.EmitStopPoint(&S);
2465}
2466
2467/// Emit a helper variable and return corresponding lvalue.
2469 const DeclRefExpr *Helper) {
2470 auto VDecl = cast<VarDecl>(Helper->getDecl());
2471 CGF.EmitVarDecl(*VDecl);
2472 return CGF.EmitLValue(Helper);
2473}
2474
2476 const RegionCodeGenTy &SimdInitGen,
2477 const RegionCodeGenTy &BodyCodeGen) {
2478 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](CodeGenFunction &CGF,
2479 PrePostActionTy &) {
2480 CGOpenMPRuntime::NontemporalDeclsRAII NontemporalsRegion(CGF.CGM, S);
2481 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
2482 SimdInitGen(CGF);
2483
2484 BodyCodeGen(CGF);
2485 };
2486 auto &&ElseGen = [&BodyCodeGen](CodeGenFunction &CGF, PrePostActionTy &) {
2487 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
2488 CGF.LoopStack.setVectorizeEnable(/*Enable=*/false);
2489
2490 BodyCodeGen(CGF);
2491 };
2492 const Expr *IfCond = nullptr;
2493 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
2494 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
2495 if (CGF.getLangOpts().OpenMP >= 50 &&
2496 (C->getNameModifier() == OMPD_unknown ||
2497 C->getNameModifier() == OMPD_simd)) {
2498 IfCond = C->getCondition();
2499 break;
2500 }
2501 }
2502 }
2503 if (IfCond) {
2504 CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, IfCond, ThenGen, ElseGen);
2505 } else {
2506 RegionCodeGenTy ThenRCG(ThenGen);
2507 ThenRCG(CGF);
2508 }
2509}
2510
2512 PrePostActionTy &Action) {
2513 Action.Enter(CGF);
2514 assert(isOpenMPSimdDirective(S.getDirectiveKind()) &&
2515 "Expected simd directive");
2516 OMPLoopScope PreInitScope(CGF, S);
2517 // if (PreCond) {
2518 // for (IV in 0..LastIteration) BODY;
2519 // <Final counter/linear vars updates>;
2520 // }
2521 //
2522 if (isOpenMPDistributeDirective(S.getDirectiveKind()) ||
2523 isOpenMPWorksharingDirective(S.getDirectiveKind()) ||
2524 isOpenMPTaskLoopDirective(S.getDirectiveKind())) {
2525 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));
2526 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));
2527 }
2528
2529 // Emit: if (PreCond) - begin.
2530 // If the condition constant folds and can be elided, avoid emitting the
2531 // whole loop.
2532 bool CondConstant;
2533 llvm::BasicBlock *ContBlock = nullptr;
2534 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2535 if (!CondConstant)
2536 return;
2537 } else {
2538 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");
2539 ContBlock = CGF.createBasicBlock("simd.if.end");
2540 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
2541 CGF.getProfileCount(&S));
2542 CGF.EmitBlock(ThenBlock);
2544 }
2545
2546 // Emit the loop iteration variable.
2547 const Expr *IVExpr = S.getIterationVariable();
2548 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
2549 CGF.EmitVarDecl(*IVDecl);
2550 CGF.EmitIgnoredExpr(S.getInit());
2551
2552 // Emit the iterations count variable.
2553 // If it is not a variable, Sema decided to calculate iterations count on
2554 // each iteration (e.g., it is foldable into a constant).
2555 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2556 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2557 // Emit calculation of the iterations count.
2558 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
2559 }
2560
2561 emitAlignedClause(CGF, S);
2562 (void)CGF.EmitOMPLinearClauseInit(S);
2563 {
2564 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2565 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
2566 CGF.EmitOMPLinearClause(S, LoopScope);
2567 CGF.EmitOMPPrivateClause(S, LoopScope);
2568 CGF.EmitOMPReductionClauseInit(S, LoopScope);
2570 CGF, S, CGF.EmitLValue(S.getIterationVariable()));
2571 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2572 (void)LoopScope.Privatize();
2573 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
2575
2577 CGF, S,
2578 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2579 CGF.EmitOMPSimdInit(S);
2580 },
2581 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
2582 CGF.EmitOMPInnerLoop(
2583 S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),
2584 [&S](CodeGenFunction &CGF) {
2585 emitOMPLoopBodyWithStopPoint(CGF, S,
2586 CodeGenFunction::JumpDest());
2587 },
2588 [](CodeGenFunction &) {});
2589 });
2590 CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; });
2591 // Emit final copy of the lastprivate variables at the end of loops.
2592 if (HasLastprivateClause)
2593 CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
2594 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
2596 [](CodeGenFunction &) { return nullptr; });
2597 LoopScope.restoreMap();
2598 CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; });
2599 }
2600 // Emit: if (PreCond) - end.
2601 if (ContBlock) {
2602 CGF.EmitBranch(ContBlock);
2603 CGF.EmitBlock(ContBlock, true);
2604 }
2605}
2606
2608 // Check for unsupported clauses
2609 for (OMPClause *C : S.clauses()) {
2610 // Currently only order, simdlen and safelen clauses are supported
2611 if (!(isa<OMPSimdlenClause>(C) || isa<OMPSafelenClause>(C) ||
2612 isa<OMPOrderClause>(C) || isa<OMPAlignedClause>(C)))
2613 return false;
2614 }
2615
2616 // Check if we have a statement with the ordered directive.
2617 // Visit the statement hierarchy to find a compound statement
2618 // with a ordered directive in it.
2619 if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(S.getRawStmt())) {
2620 if (const Stmt *SyntacticalLoop = CanonLoop->getLoopStmt()) {
2621 for (const Stmt *SubStmt : SyntacticalLoop->children()) {
2622 if (!SubStmt)
2623 continue;
2624 if (const CompoundStmt *CS = dyn_cast<CompoundStmt>(SubStmt)) {
2625 for (const Stmt *CSSubStmt : CS->children()) {
2626 if (!CSSubStmt)
2627 continue;
2628 if (isa<OMPOrderedDirective>(CSSubStmt)) {
2629 return false;
2630 }
2631 }
2632 }
2633 }
2634 }
2635 }
2636 return true;
2637}
2638static llvm::MapVector<llvm::Value *, llvm::Value *>
2640 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars;
2641 for (const auto *Clause : S.getClausesOfKind<OMPAlignedClause>()) {
2642 llvm::APInt ClauseAlignment(64, 0);
2643 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
2644 auto *AlignmentCI =
2645 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
2646 ClauseAlignment = AlignmentCI->getValue();
2647 }
2648 for (const Expr *E : Clause->varlists()) {
2649 llvm::APInt Alignment(ClauseAlignment);
2650 if (Alignment == 0) {
2651 // OpenMP [2.8.1, Description]
2652 // If no optional parameter is specified, implementation-defined default
2653 // alignments for SIMD instructions on the target platforms are assumed.
2654 Alignment =
2655 CGF.getContext()
2657 E->getType()->getPointeeType()))
2658 .getQuantity();
2659 }
2660 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2661 "alignment is not power of 2");
2662 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
2663 AlignedVars[PtrValue] = CGF.Builder.getInt64(Alignment.getSExtValue());
2664 }
2665 }
2666 return AlignedVars;
2667}
2668
2670 bool UseOMPIRBuilder =
2671 CGM.getLangOpts().OpenMPIRBuilder && isSupportedByOpenMPIRBuilder(S);
2672 if (UseOMPIRBuilder) {
2673 auto &&CodeGenIRBuilder = [this, &S, UseOMPIRBuilder](CodeGenFunction &CGF,
2674 PrePostActionTy &) {
2675 // Use the OpenMPIRBuilder if enabled.
2676 if (UseOMPIRBuilder) {
2677 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars =
2678 GetAlignedMapping(S, CGF);
2679 // Emit the associated statement and get its loop representation.
2680 const Stmt *Inner = S.getRawStmt();
2681 llvm::CanonicalLoopInfo *CLI =
2683
2684 llvm::OpenMPIRBuilder &OMPBuilder =
2686 // Add SIMD specific metadata
2687 llvm::ConstantInt *Simdlen = nullptr;
2688 if (const auto *C = S.getSingleClause<OMPSimdlenClause>()) {
2689 RValue Len =
2690 this->EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
2691 /*ignoreResult=*/true);
2692 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
2693 Simdlen = Val;
2694 }
2695 llvm::ConstantInt *Safelen = nullptr;
2696 if (const auto *C = S.getSingleClause<OMPSafelenClause>()) {
2697 RValue Len =
2698 this->EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
2699 /*ignoreResult=*/true);
2700 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
2701 Safelen = Val;
2702 }
2703 llvm::omp::OrderKind Order = llvm::omp::OrderKind::OMP_ORDER_unknown;
2704 if (const auto *C = S.getSingleClause<OMPOrderClause>()) {
2705 if (C->getKind() == OpenMPOrderClauseKind ::OMPC_ORDER_concurrent) {
2706 Order = llvm::omp::OrderKind::OMP_ORDER_concurrent;
2707 }
2708 }
2709 // Add simd metadata to the collapsed loop. Do not generate
2710 // another loop for if clause. Support for if clause is done earlier.
2711 OMPBuilder.applySimd(CLI, AlignedVars,
2712 /*IfCond*/ nullptr, Order, Simdlen, Safelen);
2713 return;
2714 }
2715 };
2716 {
2717 auto LPCRegion =
2719 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2720 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd,
2721 CodeGenIRBuilder);
2722 }
2723 return;
2724 }
2725
2726 ParentLoopDirectiveForScanRegion ScanRegion(*this, S);
2727 OMPFirstScanLoop = true;
2728 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2729 emitOMPSimdRegion(CGF, S, Action);
2730 };
2731 {
2732 auto LPCRegion =
2734 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2735 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2736 }
2737 // Check for outer lastprivate conditional update.
2739}
2740
2742 // Emit the de-sugared statement.
2743 OMPTransformDirectiveScopeRAII TileScope(*this, &S);
2744 EmitStmt(S.getTransformedStmt());
2745}
2746
2748 bool UseOMPIRBuilder = CGM.getLangOpts().OpenMPIRBuilder;
2749
2750 if (UseOMPIRBuilder) {
2751 auto DL = SourceLocToDebugLoc(S.getBeginLoc());
2752 const Stmt *Inner = S.getRawStmt();
2753
2754 // Consume nested loop. Clear the entire remaining loop stack because a
2755 // fully unrolled loop is non-transformable. For partial unrolling the
2756 // generated outer loop is pushed back to the stack.
2757 llvm::CanonicalLoopInfo *CLI = EmitOMPCollapsedCanonicalLoopNest(Inner, 1);
2758 OMPLoopNestStack.clear();
2759
2760 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2761
2762 bool NeedsUnrolledCLI = ExpectedOMPLoopDepth >= 1;
2763 llvm::CanonicalLoopInfo *UnrolledCLI = nullptr;
2764
2765 if (S.hasClausesOfKind<OMPFullClause>()) {
2766 assert(ExpectedOMPLoopDepth == 0);
2767 OMPBuilder.unrollLoopFull(DL, CLI);
2768 } else if (auto *PartialClause = S.getSingleClause<OMPPartialClause>()) {
2769 uint64_t Factor = 0;
2770 if (Expr *FactorExpr = PartialClause->getFactor()) {
2771 Factor = FactorExpr->EvaluateKnownConstInt(getContext()).getZExtValue();
2772 assert(Factor >= 1 && "Only positive factors are valid");
2773 }
2774 OMPBuilder.unrollLoopPartial(DL, CLI, Factor,
2775 NeedsUnrolledCLI ? &UnrolledCLI : nullptr);
2776 } else {
2777 OMPBuilder.unrollLoopHeuristic(DL, CLI);
2778 }
2779
2780 assert((!NeedsUnrolledCLI || UnrolledCLI) &&
2781 "NeedsUnrolledCLI implies UnrolledCLI to be set");
2782 if (UnrolledCLI)
2783 OMPLoopNestStack.push_back(UnrolledCLI);
2784
2785 return;
2786 }
2787
2788 // This function is only called if the unrolled loop is not consumed by any
2789 // other loop-associated construct. Such a loop-associated construct will have
2790 // used the transformed AST.
2791
2792 // Set the unroll metadata for the next emitted loop.
2794
2795 if (S.hasClausesOfKind<OMPFullClause>()) {
2797 } else if (auto *PartialClause = S.getSingleClause<OMPPartialClause>()) {
2798 if (Expr *FactorExpr = PartialClause->getFactor()) {
2799 uint64_t Factor =
2800 FactorExpr->EvaluateKnownConstInt(getContext()).getZExtValue();
2801 assert(Factor >= 1 && "Only positive factors are valid");
2802 LoopStack.setUnrollCount(Factor);
2803 }
2804 }
2805
2806 EmitStmt(S.getAssociatedStmt());
2807}
2808
2809void CodeGenFunction::EmitOMPOuterLoop(
2810 bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
2811 CodeGenFunction::OMPPrivateScope &LoopScope,
2812 const CodeGenFunction::OMPLoopArguments &LoopArgs,
2813 const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
2814 const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
2816
2817 const Expr *IVExpr = S.getIterationVariable();
2818 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2819 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2820
2821 JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
2822
2823 // Start the loop with a block that tests the condition.
2824 llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond");
2825 EmitBlock(CondBlock);
2826 const SourceRange R = S.getSourceRange();
2827 OMPLoopNestStack.clear();
2830
2831 llvm::Value *BoolCondVal = nullptr;
2832 if (!DynamicOrOrdered) {
2833 // UB = min(UB, GlobalUB) or
2834 // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
2835 // 'distribute parallel for')
2836 EmitIgnoredExpr(LoopArgs.EUB);
2837 // IV = LB
2838 EmitIgnoredExpr(LoopArgs.Init);
2839 // IV < UB
2840 BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
2841 } else {
2842 BoolCondVal =
2843 RT.emitForNext(*this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
2844 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
2845 }
2846
2847 // If there are any cleanups between here and the loop-exit scope,
2848 // create a block to stage a loop exit along.
2849 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
2850 if (LoopScope.requiresCleanups())
2851 ExitBlock = createBasicBlock("omp.dispatch.cleanup");
2852
2853 llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body");
2854 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
2855 if (ExitBlock != LoopExit.getBlock()) {
2856 EmitBlock(ExitBlock);
2858 }
2859 EmitBlock(LoopBody);
2860
2861 // Emit "IV = LB" (in case of static schedule, we have already calculated new
2862 // LB for loop condition and emitted it above).
2863 if (DynamicOrOrdered)
2864 EmitIgnoredExpr(LoopArgs.Init);
2865
2866 // Create a block for the increment.
2867 JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
2868 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
2869
2871 *this, S,
2872 [&S, IsMonotonic](CodeGenFunction &CGF, PrePostActionTy &) {
2873 // Generate !llvm.loop.parallel metadata for loads and stores for loops
2874 // with dynamic/guided scheduling and without ordered clause.
2875 if (!isOpenMPSimdDirective(S.getDirectiveKind())) {
2876 CGF.LoopStack.setParallel(!IsMonotonic);
2877 if (const auto *C = S.getSingleClause<OMPOrderClause>())
2878 if (C->getKind() == OMPC_ORDER_concurrent)
2879 CGF.LoopStack.setParallel(/*Enable=*/true);
2880 } else {
2881 CGF.EmitOMPSimdInit(S);
2882 }
2883 },
2884 [&S, &LoopArgs, LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered,
2885 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
2886 SourceLocation Loc = S.getBeginLoc();
2887 // when 'distribute' is not combined with a 'for':
2888 // while (idx <= UB) { BODY; ++idx; }
2889 // when 'distribute' is combined with a 'for'
2890 // (e.g. 'distribute parallel for')
2891 // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
2892 CGF.EmitOMPInnerLoop(
2893 S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
2894 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
2895 CodeGenLoop(CGF, S, LoopExit);
2896 },
2897 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
2898 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
2899 });
2900 });
2901
2902 EmitBlock(Continue.getBlock());
2903 BreakContinueStack.pop_back();
2904 if (!DynamicOrOrdered) {
2905 // Emit "LB = LB + Stride", "UB = UB + Stride".
2906 EmitIgnoredExpr(LoopArgs.NextLB);
2907 EmitIgnoredExpr(LoopArgs.NextUB);
2908 }
2909
2910 EmitBranch(CondBlock);
2911 OMPLoopNestStack.clear();
2912 LoopStack.pop();
2913 // Emit the fall-through block.
2914 EmitBlock(LoopExit.getBlock());
2915
2916 // Tell the runtime we are done.
2917 auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) {
2918 if (!DynamicOrOrdered)
2919 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2920 S.getDirectiveKind());
2921 };
2922 OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
2923}
2924
2925void CodeGenFunction::EmitOMPForOuterLoop(
2926 const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
2927 const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
2928 const OMPLoopArguments &LoopArgs,
2929 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
2931
2932 // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
2933 const bool DynamicOrOrdered = Ordered || RT.isDynamic(ScheduleKind.Schedule);
2934
2935 assert((Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule,
2936 LoopArgs.Chunk != nullptr)) &&
2937 "static non-chunked schedule does not need outer loop");
2938
2939 // Emit outer loop.
2940 //
2941 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2942 // When schedule(dynamic,chunk_size) is specified, the iterations are
2943 // distributed to threads in the team in chunks as the threads request them.
2944 // Each thread executes a chunk of iterations, then requests another chunk,
2945 // until no chunks remain to be distributed. Each chunk contains chunk_size
2946 // iterations, except for the last chunk to be distributed, which may have
2947 // fewer iterations. When no chunk_size is specified, it defaults to 1.
2948 //
2949 // When schedule(guided,chunk_size) is specified, the iterations are assigned
2950 // to threads in the team in chunks as the executing threads request them.
2951 // Each thread executes a chunk of iterations, then requests another chunk,
2952 // until no chunks remain to be assigned. For a chunk_size of 1, the size of
2953 // each chunk is proportional to the number of unassigned iterations divided
2954 // by the number of threads in the team, decreasing to 1. For a chunk_size
2955 // with value k (greater than 1), the size of each chunk is determined in the
2956 // same way, with the restriction that the chunks do not contain fewer than k
2957 // iterations (except for the last chunk to be assigned, which may have fewer
2958 // than k iterations).
2959 //
2960 // When schedule(auto) is specified, the decision regarding scheduling is
2961 // delegated to the compiler and/or runtime system. The programmer gives the
2962 // implementation the freedom to choose any possible mapping of iterations to
2963 // threads in the team.
2964 //
2965 // When schedule(runtime) is specified, the decision regarding scheduling is
2966 // deferred until run time, and the schedule and chunk size are taken from the
2967 // run-sched-var ICV. If the ICV is set to auto, the schedule is
2968 // implementation defined
2969 //
2970 // while(__kmpc_dispatch_next(&LB, &UB)) {
2971 // idx = LB;
2972 // while (idx <= UB) { BODY; ++idx;
2973 // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
2974 // } // inner loop
2975 // }
2976 //
2977 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2978 // When schedule(static, chunk_size) is specified, iterations are divided into
2979 // chunks of size chunk_size, and the chunks are assigned to the threads in
2980 // the team in a round-robin fashion in the order of the thread number.
2981 //
2982 // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
2983 // while (idx <= UB) { BODY; ++idx; } // inner loop
2984 // LB = LB + ST;
2985 // UB = UB + ST;
2986 // }
2987 //
2988
2989 const Expr *IVExpr = S.getIterationVariable();
2990 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2991 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2992
2993 if (DynamicOrOrdered) {
2994 const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
2995 CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
2996 llvm::Value *LBVal = DispatchBounds.first;
2997 llvm::Value *UBVal = DispatchBounds.second;
2998 CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
2999 LoopArgs.Chunk};
3000 RT.emitForDispatchInit(*this, S.getBeginLoc(), ScheduleKind, IVSize,
3001 IVSigned, Ordered, DipatchRTInputValues);
3002 } else {
3004 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
3005 LoopArgs.ST, LoopArgs.Chunk);
3006 RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
3007 ScheduleKind, StaticInit);
3008 }
3009
3010 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
3011 const unsigned IVSize,
3012 const bool IVSigned) {
3013 if (Ordered) {
3014 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
3015 IVSigned);
3016 }
3017 };
3018
3019 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
3020 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
3021 OuterLoopArgs.IncExpr = S.getInc();
3022 OuterLoopArgs.Init = S.getInit();
3023 OuterLoopArgs.Cond = S.getCond();
3024 OuterLoopArgs.NextLB = S.getNextLowerBound();
3025 OuterLoopArgs.NextUB = S.getNextUpperBound();
3026 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
3027 emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
3028}
3029
3031 const unsigned IVSize, const bool IVSigned) {}
3032
3033void CodeGenFunction::EmitOMPDistributeOuterLoop(
3034 OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
3035 OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
3036 const CodeGenLoopTy &CodeGenLoopContent) {
3037
3039
3040 // Emit outer loop.
3041 // Same behavior as a OMPForOuterLoop, except that schedule cannot be
3042 // dynamic
3043 //
3044
3045 const Expr *IVExpr = S.getIterationVariable();
3046 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3047 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3048
3050 IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
3051 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
3052 RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, StaticInit);
3053
3054 // for combined 'distribute' and 'for' the increment expression of distribute
3055 // is stored in DistInc. For 'distribute' alone, it is in Inc.
3056 Expr *IncExpr;
3057 if (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()))
3058 IncExpr = S.getDistInc();
3059 else
3060 IncExpr = S.getInc();
3061
3062 // this routine is shared by 'omp distribute parallel for' and
3063 // 'omp distribute': select the right EUB expression depending on the
3064 // directive
3065 OMPLoopArguments OuterLoopArgs;
3066 OuterLoopArgs.LB = LoopArgs.LB;
3067 OuterLoopArgs.UB = LoopArgs.UB;
3068 OuterLoopArgs.ST = LoopArgs.ST;
3069 OuterLoopArgs.IL = LoopArgs.IL;
3070 OuterLoopArgs.Chunk = LoopArgs.Chunk;
3071 OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3072 ? S.getCombinedEnsureUpperBound()
3073 : S.getEnsureUpperBound();
3074 OuterLoopArgs.IncExpr = IncExpr;
3075 OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3076 ? S.getCombinedInit()
3077 : S.getInit();
3078 OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3079 ? S.getCombinedCond()
3080 : S.getCond();
3081 OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3082 ? S.getCombinedNextLowerBound()
3083 : S.getNextLowerBound();
3084 OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3085 ? S.getCombinedNextUpperBound()
3086 : S.getNextUpperBound();
3087
3088 EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
3089 LoopScope, OuterLoopArgs, CodeGenLoopContent,
3091}
3092
3093static std::pair<LValue, LValue>
3095 const OMPExecutableDirective &S) {
3096 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
3097 LValue LB =
3098 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
3099 LValue UB =
3100 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
3101
3102 // When composing 'distribute' with 'for' (e.g. as in 'distribute
3103 // parallel for') we need to use the 'distribute'
3104 // chunk lower and upper bounds rather than the whole loop iteration
3105 // space. These are parameters to the outlined function for 'parallel'
3106 // and we copy the bounds of the previous schedule into the
3107 // the current ones.
3108 LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
3109 LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
3110 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
3111 PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());
3112 PrevLBVal = CGF.EmitScalarConversion(
3113 PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
3116 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
3117 PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());
3118 PrevUBVal = CGF.EmitScalarConversion(
3119 PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
3122
3123 CGF.EmitStoreOfScalar(PrevLBVal, LB);
3124 CGF.EmitStoreOfScalar(PrevUBVal, UB);
3125
3126 return {LB, UB};
3127}
3128
3129/// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
3130/// we need to use the LB and UB expressions generated by the worksharing
3131/// code generation support, whereas in non combined situations we would
3132/// just emit 0 and the LastIteration expression
3133/// This function is necessary due to the difference of the LB and UB
3134/// types for the RT emission routines for 'for_static_init' and
3135/// 'for_dispatch_init'
3136static std::pair<llvm::Value *, llvm::Value *>
3138 const OMPExecutableDirective &S,
3139 Address LB, Address UB) {
3140 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
3141 const Expr *IVExpr = LS.getIterationVariable();
3142 // when implementing a dynamic schedule for a 'for' combined with a
3143 // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
3144 // is not normalized as each team only executes its own assigned
3145 // distribute chunk
3146 QualType IteratorTy = IVExpr->getType();
3147 llvm::Value *LBVal =
3148 CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
3149 llvm::Value *UBVal =
3150 CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
3151 return {LBVal, UBVal};
3152}
3153
3157 const auto &Dir = cast<OMPLoopDirective>(S);
3158 LValue LB =
3159 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
3160 llvm::Value *LBCast =
3161 CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(LB.getAddress(CGF)),
3162 CGF.SizeTy, /*isSigned=*/false);
3163 CapturedVars.push_back(LBCast);
3164 LValue UB =
3165 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
3166
3167 llvm::Value *UBCast =
3168 CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(UB.getAddress(CGF)),
3169 CGF.SizeTy, /*isSigned=*/false);
3170 CapturedVars.push_back(UBCast);
3171}
3172
3173static void
3175 const OMPLoopDirective &S,
3176 CodeGenFunction::JumpDest LoopExit) {
3177 auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
3178 PrePostActionTy &Action) {
3179 Action.Enter(CGF);
3180 bool HasCancel = false;
3181 if (!isOpenMPSimdDirective(S.getDirectiveKind())) {
3182 if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
3183 HasCancel = D->hasCancel();
3184 else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
3185 HasCancel = D->hasCancel();
3186 else if (const auto *D =
3187 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
3188 HasCancel = D->hasCancel();
3189 }
3190 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
3191 HasCancel);
3192 CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(),
3195 };
3196
3198 CGF, S,
3199 isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for,
3200 CGInlinedWorksharingLoop,
3202}
3203
3206 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3208 S.getDistInc());
3209 };
3210 OMPLexicalScope Scope(*this, S, OMPD_parallel);
3211 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3212}
3213
3216 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3218 S.getDistInc());
3219 };
3220 OMPLexicalScope Scope(*this, S, OMPD_parallel);
3221 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3222}
3223
3225 const OMPDistributeSimdDirective &S) {
3226 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3228 };
3229 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3230 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
3231}
3232
3234 CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
3235 // Emit SPMD target parallel for region as a standalone region.
3236 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3237 emitOMPSimdRegion(CGF, S, Action);
3238 };
3239 llvm::Function *Fn;
3240 llvm::Constant *Addr;
3241 // Emit target region as a standalone region.
3243 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
3244 assert(Fn && Addr && "Target device function emission failed.");
3245}
3246
3248 const OMPTargetSimdDirective &S) {
3249 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3250 emitOMPSimdRegion(CGF, S, Action);
3251 };
3252 emitCommonOMPTargetDirective(*this, S, CodeGen);
3253}
3254
3255namespace {
3256struct ScheduleKindModifiersTy {
3260 ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
3263 : Kind(Kind), M1(M1), M2(M2) {}
3264};
3265} // namespace
3266
3268 const OMPLoopDirective &S, Expr *EUB,
3269 const CodeGenLoopBoundsTy &CodeGenLoopBounds,
3270 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
3271 // Emit the loop iteration variable.
3272 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3273 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3274 EmitVarDecl(*IVDecl);
3275
3276 // Emit the iterations count variable.
3277 // If it is not a variable, Sema decided to calculate iterations count on each
3278 // iteration (e.g., it is foldable into a constant).
3279 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3280 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3281 // Emit calculation of the iterations count.
3282 EmitIgnoredExpr(S.getCalcLastIteration());
3283 }
3284
3286
3287 bool HasLastprivateClause;
3288 // Check pre-condition.
3289 {
3290 OMPLoopScope PreInitScope(*this, S);
3291 // Skip the entire loop if we don't meet the precondition.
3292 // If the condition constant folds and can be elided, avoid emitting the
3293 // whole loop.
3294 bool CondConstant;
3295 llvm::BasicBlock *ContBlock = nullptr;
3296 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
3297 if (!CondConstant)
3298 return false;
3299 } else {
3300 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
3301 ContBlock = createBasicBlock("omp.precond.end");
3302 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
3303 getProfileCount(&S));
3304 EmitBlock(ThenBlock);
3306 }
3307
3308 RunCleanupsScope DoacrossCleanupScope(*this);
3309 bool Ordered = false;
3310 if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
3311 if (OrderedClause->getNumForLoops())
3312 RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations());
3313 else
3314 Ordered = true;
3315 }
3316
3317 llvm::DenseSet<const Expr *> EmittedFinals;
3318 emitAlignedClause(*this, S);
3319 bool HasLinears = EmitOMPLinearClauseInit(S);
3320 // Emit helper vars inits.
3321
3322 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
3323 LValue LB = Bounds.first;
3324 LValue UB = Bounds.second;
3325 LValue ST =
3326 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
3327 LValue IL =
3328 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
3329
3330 // Emit 'then' code.
3331 {
3332 OMPPrivateScope LoopScope(*this);
3333 if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
3334 // Emit implicit barrier to synchronize threads and avoid data races on
3335 // initialization of firstprivate variables and post-update of
3336 // lastprivate variables.
3338 *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
3339 /*ForceSimpleCall=*/true);
3340 }
3341 EmitOMPPrivateClause(S, LoopScope);
3343 *this, S, EmitLValue(S.getIterationVariable()));
3344 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
3345 EmitOMPReductionClauseInit(S, LoopScope);
3346 EmitOMPPrivateLoopCounters(S, LoopScope);
3347 EmitOMPLinearClause(S, LoopScope);
3348 (void)LoopScope.Privatize();
3349 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
3351
3352 // Detect the loop schedule kind and chunk.
3353 const Expr *ChunkExpr = nullptr;
3354 OpenMPScheduleTy ScheduleKind;
3355 if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
3356 ScheduleKind.Schedule = C->getScheduleKind();
3357 ScheduleKind.M1 = C->getFirstScheduleModifier();
3358 ScheduleKind.M2 = C->getSecondScheduleModifier();
3359 ChunkExpr = C->getChunkSize();
3360 } else {
3361 // Default behaviour for schedule clause.
3363 *this, S, ScheduleKind.Schedule, ChunkExpr);
3364 }
3365 bool HasChunkSizeOne = false;
3366 llvm::Value *Chunk = nullptr;
3367 if (ChunkExpr) {
3368 Chunk = EmitScalarExpr(ChunkExpr);
3369 Chunk = EmitScalarConversion(Chunk, ChunkExpr->getType(),
3370 S.getIterationVariable()->getType(),
3371 S.getBeginLoc());
3373 if (ChunkExpr->EvaluateAsInt(Result, getContext())) {
3374 llvm::APSInt EvaluatedChunk = Result.Val.getInt();
3375 HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
3376 }
3377 }
3378 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3379 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3380 // OpenMP 4.5, 2.7.1 Loop Construct, Description.
3381 // If the static schedule kind is specified or if the ordered clause is
3382 // specified, and if no monotonic modifier is specified, the effect will
3383 // be as if the monotonic modifier was specified.
3384 bool StaticChunkedOne =
3385 RT.isStaticChunked(ScheduleKind.Schedule,
3386 /* Chunked */ Chunk != nullptr) &&
3387 HasChunkSizeOne &&
3388 isOpenMPLoopBoundSharingDirective(S.getDirectiveKind());
3389 bool IsMonotonic =
3390 Ordered ||
3391 (ScheduleKind.Schedule == OMPC_SCHEDULE_static &&
3392 !(ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_nonmonotonic ||
3393 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_nonmonotonic)) ||
3394 ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
3395 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
3396 if ((RT.isStaticNonchunked(ScheduleKind.Schedule,
3397 /* Chunked */ Chunk != nullptr) ||
3398 StaticChunkedOne) &&
3399 !Ordered) {
3400 JumpDest LoopExit =
3403 *this, S,
3404 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3405 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
3406 CGF.EmitOMPSimdInit(S);
3407 } else if (const auto *C = S.getSingleClause<OMPOrderClause>()) {
3408 if (C->getKind() == OMPC_ORDER_concurrent)
3409 CGF.LoopStack.setParallel(/*Enable=*/true);
3410 }
3411 },
3412 [IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk,
3413 &S, ScheduleKind, LoopExit,
3414 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
3415 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
3416 // When no chunk_size is specified, the iteration space is divided
3417 // into chunks that are approximately equal in size, and at most
3418 // one chunk is distributed to each thread. Note that the size of
3419 // the chunks is unspecified in this case.
3421 IVSize, IVSigned, Ordered, IL.getAddress(CGF),
3422 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF),
3423 StaticChunkedOne ? Chunk : nullptr);
3424 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
3425 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind,
3426 StaticInit);
3427 // UB = min(UB, GlobalUB);
3428 if (!StaticChunkedOne)
3429 CGF.EmitIgnoredExpr(S.getEnsureUpperBound());
3430 // IV = LB;
3431 CGF.EmitIgnoredExpr(S.getInit());
3432 // For unchunked static schedule generate:
3433 //
3434 // while (idx <= UB) {
3435 // BODY;
3436 // ++idx;
3437 // }
3438 //
3439 // For static schedule with chunk one:
3440 //
3441 // while (IV <= PrevUB) {
3442 // BODY;
3443 // IV += ST;
3444 // }
3445 CGF.EmitOMPInnerLoop(
3446 S, LoopScope.requiresCleanups(),
3447 StaticChunkedOne ? S.getCombinedParForInDistCond()
3448 : S.getCond(),
3449 StaticChunkedOne ? S.getDistInc() : S.getInc(),
3450 [&S, LoopExit](CodeGenFunction &CGF) {
3451 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit);
3452 },
3453 [](CodeGenFunction &) {});
3454 });
3455 EmitBlock(LoopExit.getBlock());
3456 // Tell the runtime we are done.
3457 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
3458 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
3459 S.getDirectiveKind());
3460 };
3461 OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
3462 } else {
3463 // Emit the outer loop, which requests its work chunk [LB..UB] from
3464 // runtime and runs the inner loop to process it.
3465 const OMPLoopArguments LoopArguments(
3466 LB.getAddress(*this), UB.getAddress(*this), ST.getAddress(*this),
3467 IL.getAddress(*this), Chunk, EUB);
3468 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
3469 LoopArguments, CGDispatchBounds);
3470 }
3471 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
3472 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
3473 return CGF.Builder.CreateIsNotNull(
3474 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3475 });
3476 }
3478 S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind())
3479 ? /*Parallel and Simd*/ OMPD_parallel_for_simd
3480 : /*Parallel only*/ OMPD_parallel);
3481 // Emit post-update of the reduction variables if IsLastIter != 0.
3483 *this, S, [IL, &S](CodeGenFunction &CGF) {
3484 return CGF.Builder.CreateIsNotNull(
3485 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3486 });
3487 // Emit final copy of the lastprivate variables if IsLastIter != 0.
3488 if (HasLastprivateClause)
3490 S, isOpenMPSimdDirective(S.getDirectiveKind()),
3491 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
3492 LoopScope.restoreMap();
3493 EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) {
3494 return CGF.Builder.CreateIsNotNull(
3495 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3496 });
3497 }
3498 DoacrossCleanupScope.ForceCleanup();
3499 // We're now done with the loop, so jump to the continuation block.
3500 if (ContBlock) {
3501 EmitBranch(ContBlock);
3502 EmitBlock(ContBlock, /*IsFinished=*/true);
3503 }
3504 }
3505 return HasLastprivateClause;
3506}
3507
3508/// The following two functions generate expressions for the loop lower
3509/// and upper bounds in case of static and dynamic (dispatch) schedule
3510/// of the associated 'for' or 'distribute' loop.
3511static std::pair<LValue, LValue>
3513 const auto &LS = cast<OMPLoopDirective>(S);
3514 LValue LB =
3515 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
3516 LValue UB =
3517 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
3518 return {LB, UB};
3519}
3520
3521/// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
3522/// consider the lower and upper bound expressions generated by the
3523/// worksharing loop support, but we use 0 and the iteration space size as
3524/// constants
3525static std::pair<llvm::Value *, llvm::Value *>
3527 Address LB, Address UB) {
3528 const auto &LS = cast<OMPLoopDirective>(S);
3529 const Expr *IVExpr = LS.getIterationVariable();
3530 const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
3531 llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
3532 llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
3533 return {LBVal, UBVal};
3534}
3535
3536/// Emits internal temp array declarations for the directive with inscan
3537/// reductions.
3538/// The code is the following:
3539/// \code
3540/// size num_iters = <num_iters>;
3541/// <type> buffer[num_iters];
3542/// \endcode
3544 CodeGenFunction &CGF, const OMPLoopDirective &S,
3545 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen) {
3546 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
3547 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
3550 SmallVector<const Expr *, 4> ReductionOps;
3551 SmallVector<const Expr *, 4> CopyArrayTemps;
3552 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
3553 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
3554 "Only inscan reductions are expected.");
3555 Shareds.append(C->varlist_begin(), C->varlist_end());
3556 Privates.append(C->privates().begin(), C->privates().end());
3557 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
3558 CopyArrayTemps.append(C->copy_array_temps().begin(),
3559 C->copy_array_temps().end());
3560 }
3561 {
3562 // Emit buffers for each reduction variables.
3563 // ReductionCodeGen is required to emit correctly the code for array
3564 // reductions.
3565 ReductionCodeGen RedCG(Shareds, Shareds, Privates, ReductionOps);
3566 unsigned Count = 0;
3567 auto *ITA = CopyArrayTemps.begin();
3568 for (const Expr *IRef : Privates) {
3569 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
3570 // Emit variably modified arrays, used for arrays/array sections
3571 // reductions.
3572 if (PrivateVD->getType()->isVariablyModifiedType()) {
3573 RedCG.emitSharedOrigLValue(CGF, Count);
3574 RedCG.emitAggregateType(CGF, Count);
3575 }
3576 CodeGenFunction::OpaqueValueMapping DimMapping(
3577 CGF,
3578 cast<OpaqueValueExpr>(
3579 cast<VariableArrayType>((*ITA)->getType()->getAsArrayTypeUnsafe())
3580 ->getSizeExpr()),
3581 RValue::get(OMPScanNumIterations));
3582 // Emit temp buffer.
3583 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(*ITA)->getDecl()));
3584 ++ITA;
3585 ++Count;
3586 }
3587 }
3588}
3589
3590/// Copies final inscan reductions values to the original variables.
3591/// The code is the following:
3592/// \code
3593/// <orig_var> = buffer[num_iters-1];
3594/// \endcode
3596 CodeGenFunction &CGF, const OMPLoopDirective &S,
3597 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen) {
3598 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
3599 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
3605 SmallVector<const Expr *, 4> CopyArrayElems;
3606 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
3607 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
3608 "Only inscan reductions are expected.");
3609 Shareds.append(C->varlist_begin(), C->varlist_end());
3610 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
3611 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
3612 Privates.append(C->privates().begin(), C->privates().end());
3613 CopyOps.append(C->copy_ops().begin(), C->copy_ops().end());
3614 CopyArrayElems.append(C->copy_array_elems().begin(),
3615 C->copy_array_elems().end());
3616 }
3617 // Create temp var and copy LHS value to this temp value.
3618 // LHS = TMP[LastIter];
3619 llvm::Value *OMPLast = CGF.Builder.CreateNSWSub(
3620 OMPScanNumIterations,
3621 llvm::ConstantInt::get(CGF.SizeTy, 1, /*isSigned=*/false));
3622 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
3623 const Expr *PrivateExpr = Privates[I];
3624 const Expr *OrigExpr = Shareds[I];
3625 const Expr *CopyArrayElem = CopyArrayElems[I];
3626 CodeGenFunction::OpaqueValueMapping IdxMapping(
3627 CGF,
3628 cast<OpaqueValueExpr>(
3629 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3630 RValue::get(OMPLast));
3631 LValue DestLVal = CGF.EmitLValue(OrigExpr);
3632 LValue SrcLVal = CGF.EmitLValue(CopyArrayElem);
3633 CGF.EmitOMPCopy(PrivateExpr->getType(), DestLVal.getAddress(CGF),
3634 SrcLVal.getAddress(CGF),
3635 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
3636 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
3637 CopyOps[I]);
3638 }
3639}
3640
3641/// Emits the code for the directive with inscan reductions.
3642/// The code is the following:
3643/// \code
3644/// #pragma omp ...
3645/// for (i: 0..<num_iters>) {
3646/// <input phase>;
3647/// buffer[i] = red;
3648/// }
3649/// #pragma omp master // in parallel region
3650/// for (int k = 0; k != ceil(log2(num_iters)); ++k)
3651/// for (size cnt = last_iter; cnt >= pow(2, k); --k)
3652/// buffer[i] op= buffer[i-pow(2,k)];
3653/// #pragma omp barrier // in parallel region
3654/// #pragma omp ...
3655/// for (0..<num_iters>) {
3656/// red = InclusiveScan ? buffer[i] : buffer[i-1];
3657/// <scan phase>;
3658/// }
3659/// \endcode
3661 CodeGenFunction &CGF, const OMPLoopDirective &S,
3662 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen,
3663 llvm::function_ref<void(CodeGenFunction &)> FirstGen,
3664 llvm::function_ref<void(CodeGenFunction &)> SecondGen) {
3665 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
3666 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
3668 SmallVector<const Expr *, 4> ReductionOps;
3671 SmallVector<const Expr *, 4> CopyArrayElems;
3672 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
3673 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
3674 "Only inscan reductions are expected.");
3675 Privates.append(C->privates().begin(), C->privates().end());
3676 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
3677 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
3678 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
3679 CopyArrayElems.append(C->copy_array_elems().begin(),
3680 C->copy_array_elems().end());
3681 }
3682 CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S);
3683 {
3684 // Emit loop with input phase:
3685 // #pragma omp ...
3686 // for (i: 0..<num_iters>) {
3687 // <input phase>;
3688 // buffer[i] = red;
3689 // }
3690 CGF.OMPFirstScanLoop = true;
3691 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
3692 FirstGen(CGF);
3693 }
3694 // #pragma omp barrier // in parallel region
3695 auto &&CodeGen = [&S, OMPScanNumIterations, &LHSs, &RHSs, &CopyArrayElems,
3696 &ReductionOps,
3697 &Privates](CodeGenFunction &CGF, PrePostActionTy &Action) {
3698 Action.Enter(CGF);
3699 // Emit prefix reduction:
3700 // #pragma omp master // in parallel region
3701 // for (int k = 0; k <= ceil(log2(n)); ++k)
3702 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock();
3703 llvm::BasicBlock *LoopBB = CGF.createBasicBlock("omp.outer.log.scan.body");
3704 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.outer.log.scan.exit");
3705 llvm::Function *F =
3706 CGF.CGM.getIntrinsic(llvm::Intrinsic::log2, CGF.DoubleTy);
3707 llvm::Value *Arg =
3708 CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy);
3709 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg);
3710 F = CGF.CGM.getIntrinsic(llvm::Intrinsic::ceil, CGF.DoubleTy);
3711 LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal);
3712 LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy);
3713 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub(
3714 OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1));
3715 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getBeginLoc());
3716 CGF.EmitBlock(LoopBB);
3717 auto *Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2);
3718 // size pow2k = 1;
3719 auto *Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3720 Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB);
3721 Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB);
3722 // for (size i = n - 1; i >= 2 ^ k; --i)
3723 // tmp[i] op= tmp[i-pow2k];
3724 llvm::BasicBlock *InnerLoopBB =
3725 CGF.createBasicBlock("omp.inner.log.scan.body");
3726 llvm::BasicBlock *InnerExitBB =
3727 CGF.createBasicBlock("omp.inner.log.scan.exit");
3728 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K);
3729 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3730 CGF.EmitBlock(InnerLoopBB);
3731 auto *IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3732 IVal->addIncoming(NMin1, LoopBB);
3733 {
3734 CodeGenFunction::OMPPrivateScope PrivScope(CGF);
3735 auto *ILHS = LHSs.begin();
3736 auto *IRHS = RHSs.begin();
3737 for (const Expr *CopyArrayElem : CopyArrayElems) {
3738 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
3739 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
3740 Address LHSAddr = Address::invalid();
3741 {
3742 CodeGenFunction::OpaqueValueMapping IdxMapping(
3743 CGF,
3744 cast<OpaqueValueExpr>(
3745 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3746 RValue::get(IVal));
3747 LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF);
3748 }
3749 PrivScope.addPrivate(LHSVD, LHSAddr);
3750 Address RHSAddr = Address::invalid();
3751 {
3752 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K);
3753 CodeGenFunction::OpaqueValueMapping IdxMapping(
3754 CGF,
3755 cast<OpaqueValueExpr>(
3756 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3757 RValue::get(OffsetIVal));
3758 RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF);
3759 }
3760 PrivScope.addPrivate(RHSVD, RHSAddr);
3761 ++ILHS;
3762 ++IRHS;
3763 }
3764 PrivScope.Privatize();
3765 CGF.CGM.getOpenMPRuntime().emitReduction(
3766 CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
3767 {/*WithNowait=*/true, /*SimpleReduction=*/true, OMPD_unknown});
3768 }
3769 llvm::Value *NextIVal =
3770 CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1));
3771 IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock());
3772 CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K);
3773 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3774 CGF.EmitBlock(InnerExitBB);
3775 llvm::Value *Next =
3776 CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1));
3777 Counter->addIncoming(Next, CGF.Builder.GetInsertBlock());
3778 // pow2k <<= 1;
3779 llvm::Value *NextPow2K =
3780 CGF.Builder.CreateShl(Pow2K, 1, "", /*HasNUW=*/true);
3781 Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock());
3782 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(Next, LogVal);
3783 CGF.Builder.CreateCondBr(Cmp, LoopBB, ExitBB);
3784 auto DL1 = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getEndLoc());
3785 CGF.EmitBlock(ExitBB);
3786 };
3787 if (isOpenMPParallelDirective(S.getDirectiveKind())) {
3788 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());
3789 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
3790 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
3791 /*ForceSimpleCall=*/true);
3792 } else {
3793 RegionCodeGenTy RCG(CodeGen);
3794 RCG(CGF);
3795 }
3796
3797 CGF.OMPFirstScanLoop = false;
3798 SecondGen(CGF);
3799}
3800
3802 const OMPLoopDirective &S,
3803 bool HasCancel) {
3804 bool HasLastprivates;
3805 if (llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
3806 [](const OMPReductionClause *C) {
3807 return C->getModifier() == OMPC_REDUCTION_inscan;
3808 })) {
3809 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
3810 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
3811 OMPLoopScope LoopScope(CGF, S);
3812 return CGF.EmitScalarExpr(S.getNumIterations());
3813 };
3814 const auto &&FirstGen = [&S, HasCancel](CodeGenFunction &CGF) {
3815 CodeGenFunction::OMPCancelStackRAII CancelRegion(
3816 CGF, S.getDirectiveKind(), HasCancel);
3817 (void)CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
3820 // Emit an implicit barrier at the end.
3821 CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getBeginLoc(),
3822 OMPD_for);
3823 };
3824 const auto &&SecondGen = [&S, HasCancel,
3825 &HasLastprivates](CodeGenFunction &CGF) {
3826 CodeGenFunction::OMPCancelStackRAII CancelRegion(
3827 CGF, S.getDirectiveKind(), HasCancel);
3828 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
3831 };
3832 if (!isOpenMPParallelDirective(S.getDirectiveKind()))
3833 emitScanBasedDirectiveDecls(CGF, S, NumIteratorsGen);
3834 emitScanBasedDirective(CGF, S, NumIteratorsGen, FirstGen, SecondGen);
3835 if (!isOpenMPParallelDirective(S.getDirectiveKind()))
3836 emitScanBasedDirectiveFinals(CGF, S, NumIteratorsGen);
3837 } else {
3838 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
3839 HasCancel);
3840 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
3843 }
3844 return HasLastprivates;
3845}
3846
3848 if (S.hasCancel())
3849 return false;
3850 for (OMPClause *C : S.clauses()) {
3851 if (isa<OMPNowaitClause>(C))
3852 continue;
3853
3854 if (auto *SC = dyn_cast<OMPScheduleClause>(C)) {
3855 if (SC->getFirstScheduleModifier() != OMPC_SCHEDULE_MODIFIER_unknown)
3856 return false;
3857 if (SC->getSecondScheduleModifier() != OMPC_SCHEDULE_MODIFIER_unknown)
3858 return false;
3859 switch (SC->getScheduleKind()) {
3860 case OMPC_SCHEDULE_auto:
3861 case OMPC_SCHEDULE_dynamic:
3862 case OMPC_SCHEDULE_runtime:
3863 case OMPC_SCHEDULE_guided:
3864 case OMPC_SCHEDULE_static:
3865 continue;
3867 return false;
3868 }
3869 }
3870
3871 return false;
3872 }
3873
3874 return true;
3875}
3876
3877static llvm::omp::ScheduleKind
3879 switch (ScheduleClauseKind) {
3881 return llvm::omp::OMP_SCHEDULE_Default;
3882 case OMPC_SCHEDULE_auto:
3883 return llvm::omp::OMP_SCHEDULE_Auto;
3884 case OMPC_SCHEDULE_dynamic:
3885 return llvm::omp::OMP_SCHEDULE_Dynamic;
3886 case OMPC_SCHEDULE_guided:
3887 return llvm::omp::OMP_SCHEDULE_Guided;
3888 case OMPC_SCHEDULE_runtime:
3889 return llvm::omp::OMP_SCHEDULE_Runtime;
3890 case OMPC_SCHEDULE_static:
3891 return llvm::omp::OMP_SCHEDULE_Static;
3892 }
3893 llvm_unreachable("Unhandled schedule kind");
3894}
3895
3897 bool HasLastprivates = false;
3898 bool UseOMPIRBuilder =
3899 CGM.getLangOpts().OpenMPIRBuilder && isSupportedByOpenMPIRBuilder(S);
3900 auto &&CodeGen = [this, &S, &HasLastprivates,
3901 UseOMPIRBuilder](CodeGenFunction &CGF, PrePostActionTy &) {
3902 // Use the OpenMPIRBuilder if enabled.
3903 if (UseOMPIRBuilder) {
3904 bool NeedsBarrier = !S.getSingleClause<OMPNowaitClause>();
3905
3906 llvm::omp::ScheduleKind SchedKind = llvm::omp::OMP_SCHEDULE_Default;
3907 llvm::Value *ChunkSize = nullptr;
3908 if (auto *SchedClause = S.getSingleClause<OMPScheduleClause>()) {
3909 SchedKind =
3910 convertClauseKindToSchedKind(SchedClause->getScheduleKind());
3911 if (const Expr *ChunkSizeExpr = SchedClause->getChunkSize())
3912 ChunkSize = EmitScalarExpr(ChunkSizeExpr);
3913 }
3914
3915 // Emit the associated statement and get its loop representation.
3916 const Stmt *Inner = S.getRawStmt();
3917 llvm::CanonicalLoopInfo *CLI =
3919
3920 llvm::OpenMPIRBuilder &OMPBuilder =
3922 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
3923 AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
3924 OMPBuilder.applyWorkshareLoop(
3925 Builder.getCurrentDebugLocation(), CLI, AllocaIP, NeedsBarrier,
3926 SchedKind, ChunkSize, /*HasSimdModifier=*/false,
3927 /*HasMonotonicModifier=*/false, /*HasNonmonotonicModifier=*/false,
3928 /*HasOrderedClause=*/false);
3929 return;
3930 }
3931
3932 HasLastprivates = emitWorksharingDirective(CGF, S, S.hasCancel());
3933 };
3934 {
3935 auto LPCRegion =
3937 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3938 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
3939 S.hasCancel());
3940 }
3941
3942 if (!UseOMPIRBuilder) {
3943 // Emit an implicit barrier at the end.
3944 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
3945 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
3946 }
3947 // Check for outer lastprivate conditional update.
3949}
3950
3952 bool HasLastprivates = false;
3953 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
3954 PrePostActionTy &) {
3955 HasLastprivates = emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
3956 };
3957 {
3958 auto LPCRegion =
3960 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3961 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
3962 }
3963
3964 // Emit an implicit barrier at the end.
3965 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
3966 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
3967 // Check for outer lastprivate conditional update.
3969}
3970
3972 const Twine &Name,
3973 llvm::Value *Init = nullptr) {
3974 LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
3975 if (Init)
3976 CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
3977 return LVal;
3978}
3979
3980void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
3981 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
3982 const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
3983 bool HasLastprivates = false;
3984 auto &&CodeGen = [&S, CapturedStmt, CS,
3985 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
3986 const ASTContext &C = CGF.getContext();
3987 QualType KmpInt32Ty =
3988 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
3989 // Emit helper vars inits.
3990 LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
3991 CGF.Builder.getInt32(0));
3992 llvm::ConstantInt *GlobalUBVal = CS != nullptr
3993 ? CGF.Builder.getInt32(CS->size() - 1)
3994 : CGF.Builder.getInt32(0);
3995 LValue UB =
3996 createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
3997 LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
3998 CGF.Builder.getInt32(1));
3999 LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
4000 CGF.Builder.getInt32(0));
4001 // Loop counter.
4002 LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
4003 OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
4004 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
4005 OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
4006 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
4007 // Generate condition for loop.
4009 C, &IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_PRValue, OK_Ordinary,
4010 S.getBeginLoc(), FPOptionsOverride());
4011 // Increment for loop counter.
4013 C, &IVRefExpr, UO_PreInc, KmpInt32Ty, VK_PRValue, OK_Ordinary,
4014 S.getBeginLoc(), true, FPOptionsOverride());
4015 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
4016 // Iterate through all sections and emit a switch construct:
4017 // switch (IV) {
4018 // case 0:
4019 // <SectionStmt[0]>;
4020 // break;
4021 // ...
4022 // case <NumSection> - 1:
4023 // <SectionStmt[<NumSection> - 1]>;
4024 // break;
4025 // }
4026 // .omp.sections.exit:
4027 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
4028 llvm::SwitchInst *SwitchStmt =
4029 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
4030 ExitBB, CS == nullptr ? 1 : CS->size());
4031 if (CS) {
4032 unsigned CaseNumber = 0;
4033 for (const Stmt *SubStmt : CS->children()) {
4034 auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
4035 CGF.EmitBlock(CaseBB);
4036 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
4037 CGF.EmitStmt(SubStmt);
4038 CGF.EmitBranch(ExitBB);
4039 ++CaseNumber;
4040 }
4041 } else {
4042 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");
4043 CGF.EmitBlock(CaseBB);
4044 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
4045 CGF.EmitStmt(CapturedStmt);
4046 CGF.EmitBranch(ExitBB);
4047 }
4048 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
4049 };
4050
4051 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
4052 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
4053 // Emit implicit barrier to synchronize threads and avoid data races on
4054 // initialization of firstprivate variables and post-update of lastprivate
4055 // variables.
4056 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
4057 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
4058 /*ForceSimpleCall=*/true);
4059 }
4060 CGF.EmitOMPPrivateClause(S, LoopScope);
4061 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(CGF, S, IV);
4062 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4063 CGF.EmitOMPReductionClauseInit(S, LoopScope);
4064 (void)LoopScope.Privatize();
4065 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
4066 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4067
4068 // Emit static non-chunked loop.
4069 OpenMPScheduleTy ScheduleKind;
4070 ScheduleKind.Schedule = OMPC_SCHEDULE_static;
4072 /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(CGF),
4073 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF));
4074 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
4075 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit);
4076 // UB = min(UB, GlobalUB);
4077 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
4078 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
4079 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
4080 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
4081 // IV = LB;
4082 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
4083 // while (idx <= UB) { BODY; ++idx; }
4084 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, Cond, Inc, BodyGen,
4085 [](CodeGenFunction &) {});
4086 // Tell the runtime we are done.
4087 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
4088 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
4089 S.getDirectiveKind());
4090 };
4091 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
4092 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4093 // Emit post-update of the reduction variables if IsLastIter != 0.
4094 emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) {
4095 return CGF.Builder.CreateIsNotNull(
4096 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
4097 });
4098
4099 // Emit final copy of the lastprivate variables if IsLastIter != 0.
4100 if (HasLastprivates)
4102 S, /*NoFinals=*/false,
4103 CGF.Builder.CreateIsNotNull(
4104 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())));
4105 };
4106
4107 bool HasCancel = false;
4108 if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
4109 HasCancel = OSD->hasCancel();
4110 else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
4111 HasCancel = OPSD->hasCancel();
4112 OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
4113 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
4114 HasCancel);
4115 // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
4116 // clause. Otherwise the barrier will be generated by the codegen for the
4117 // directive.
4118 if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
4119 // Emit implicit barrier to synchronize threads and avoid data races on
4120 // initialization of firstprivate variables.
4121 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
4122 OMPD_unknown);
4123 }
4124}
4125
4127 if (CGM.getLangOpts().OpenMPIRBuilder) {
4128 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4129 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4130 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4131
4132 auto FiniCB = [this](InsertPointTy IP) {
4134 };
4135
4136 const CapturedStmt *ICS = S.getInnermostCapturedStmt();
4137 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4138 const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
4140 if (CS) {
4141 for (const Stmt *SubStmt : CS->children()) {
4142 auto SectionCB = [this, SubStmt](InsertPointTy AllocaIP,
4143 InsertPointTy CodeGenIP) {
4145 *this, SubStmt, AllocaIP, CodeGenIP, "section");
4146 };
4147 SectionCBVector.push_back(SectionCB);
4148 }
4149 } else {
4150 auto SectionCB = [this, CapturedStmt](InsertPointTy AllocaIP,
4151 InsertPointTy CodeGenIP) {
4153 *this, CapturedStmt, AllocaIP, CodeGenIP, "section");
4154 };
4155 SectionCBVector.push_back(SectionCB);
4156 }
4157
4158 // Privatization callback that performs appropriate action for
4159 // shared/private/firstprivate/lastprivate/copyin/... variables.
4160 //
4161 // TODO: This defaults to shared right now.
4162 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
4163 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
4164 // The next line is appropriate only for variables (Val) with the
4165 // data-sharing attribute "shared".
4166 ReplVal = &Val;
4167
4168 return CodeGenIP;
4169 };
4170
4171 CGCapturedStmtInfo CGSI(*ICS, CR_OpenMP);
4172 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
4173 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
4174 AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
4175 Builder.restoreIP(OMPBuilder.createSections(
4176 Builder, AllocaIP, SectionCBVector, PrivCB, FiniCB, S.hasCancel(),
4177 S.getSingleClause<OMPNowaitClause>()));
4178 return;
4179 }
4180 {
4181 auto LPCRegion =
4183 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4184 EmitSections(S);
4185 }
4186 // Emit an implicit barrier at the end.
4187 if (!S.getSingleClause<OMPNowaitClause>()) {
4188 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
4189 OMPD_sections);
4190 }
4191 // Check for outer lastprivate conditional update.
4193}
4194
4196 if (CGM.getLangOpts().OpenMPIRBuilder) {
4197 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4198 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4199
4200 const Stmt *SectionRegionBodyStmt = S.getAssociatedStmt();
4201 auto FiniCB = [this](InsertPointTy IP) {
4203 };
4204
4205 auto BodyGenCB = [SectionRegionBodyStmt, this](InsertPointTy AllocaIP,
4206 InsertPointTy CodeGenIP) {
4208 *this, SectionRegionBodyStmt, AllocaIP, CodeGenIP, "section");
4209 };
4210
4211 LexicalScope Scope(*this, S.getSourceRange());
4212 EmitStopPoint(&S);
4213 Builder.restoreIP(OMPBuilder.createSection(Builder, BodyGenCB, FiniCB));
4214
4215 return;
4216 }
4217 LexicalScope Scope(*this, S.getSourceRange());
4218 EmitStopPoint(&S);
4219 EmitStmt(S.getAssociatedStmt());
4220}
4221
4223 llvm::SmallVector<const Expr *, 8> CopyprivateVars;
4227 // Check if there are any 'copyprivate' clauses associated with this
4228 // 'single' construct.
4229 // Build a list of copyprivate variables along with helper expressions
4230 // (<source>, <destination>, <destination>=<source> expressions)
4231 for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
4232 CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
4233 DestExprs.append(C->destination_exprs().begin(),
4234 C->destination_exprs().end());
4235 SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
4236 AssignmentOps.append(C->assignment_ops().begin(),
4237 C->assignment_ops().end());
4238 }
4239 // Emit code for 'single' region along with 'copyprivate' clauses
4240 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4241 Action.Enter(CGF);
4242 OMPPrivateScope SingleScope(CGF);
4243 (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
4244 CGF.EmitOMPPrivateClause(S, SingleScope);
4245 (void)SingleScope.Privatize();
4246 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4247 };
4248 {
4249 auto LPCRegion =
4251 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4252 CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getBeginLoc(),
4253 CopyprivateVars, DestExprs,
4254 SrcExprs, AssignmentOps);
4255 }
4256 // Emit an implicit barrier at the end (to avoid data race on firstprivate
4257 // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
4258 if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
4260 *this, S.getBeginLoc(),
4261 S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
4262 }
4263 // Check for outer lastprivate conditional update.
4265}
4266
4268 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4269 Action.Enter(CGF);
4270 CGF.EmitStmt(S.getRawStmt());
4271 };
4272 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());
4273}
4274
4276 if (CGM.getLangOpts().OpenMPIRBuilder) {
4277 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4278 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4279
4280 const Stmt *MasterRegionBodyStmt = S.getAssociatedStmt();
4281
4282 auto FiniCB = [this](InsertPointTy IP) {
4284 };
4285
4286 auto BodyGenCB = [MasterRegionBodyStmt, this](InsertPointTy AllocaIP,
4287 InsertPointTy CodeGenIP) {
4289 *this, MasterRegionBodyStmt, AllocaIP, CodeGenIP, "master");
4290 };
4291
4292 LexicalScope Scope(*this, S.getSourceRange());
4293 EmitStopPoint(&S);
4294 Builder.restoreIP(OMPBuilder.createMaster(Builder, BodyGenCB, FiniCB));
4295
4296 return;
4297 }
4298 LexicalScope Scope(*this, S.getSourceRange());
4299 EmitStopPoint(&S);
4300 emitMaster(*this, S);
4301}
4302
4304 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4305 Action.Enter(CGF);
4306 CGF.EmitStmt(S.getRawStmt());
4307 };
4308 Expr *Filter = nullptr;
4309 if (const auto *FilterClause = S.getSingleClause<OMPFilterClause>())
4310 Filter = FilterClause->getThreadID();
4311 CGF.CGM.getOpenMPRuntime().emitMaskedRegion(CGF, CodeGen, S.getBeginLoc(),
4312 Filter);
4313}
4314
4316 if (CGM.getLangOpts().OpenMPIRBuilder) {
4317 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4318 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4319
4320 const Stmt *MaskedRegionBodyStmt = S.getAssociatedStmt();
4321 const Expr *Filter = nullptr;
4322 if (const auto *FilterClause = S.getSingleClause<OMPFilterClause>())
4323 Filter = FilterClause->getThreadID();
4324 llvm::Value *FilterVal = Filter
4325 ? EmitScalarExpr(Filter, CGM.Int32Ty)
4326 : llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/0);
4327
4328 auto FiniCB = [this](InsertPointTy IP) {
4330 };
4331
4332 auto BodyGenCB = [MaskedRegionBodyStmt, this](InsertPointTy AllocaIP,
4333 InsertPointTy CodeGenIP) {
4335 *this, MaskedRegionBodyStmt, AllocaIP, CodeGenIP, "masked");
4336 };
4337
4338 LexicalScope Scope(*this, S.getSourceRange());
4339 EmitStopPoint(&S);
4340 Builder.restoreIP(
4341 OMPBuilder.createMasked(Builder, BodyGenCB, FiniCB, FilterVal));
4342
4343 return;
4344 }
4345 LexicalScope Scope(*this, S.getSourceRange());
4346 EmitStopPoint(&S);
4347 emitMasked(*this, S);
4348}
4349
4351 if (CGM.getLangOpts().OpenMPIRBuilder) {
4352 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4353 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4354
4355 const Stmt *CriticalRegionBodyStmt = S.getAssociatedStmt();
4356 const Expr *Hint = nullptr;
4357 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
4358 Hint = HintClause->getHint();
4359
4360 // TODO: This is slightly different from what's currently being done in
4361 // clang. Fix the Int32Ty to IntPtrTy (pointer width size) when everything
4362 // about typing is final.
4363 llvm::Value *HintInst = nullptr;
4364 if (Hint)
4365 HintInst =
4366 Builder.CreateIntCast(EmitScalarExpr(Hint), CGM.Int32Ty, false);
4367
4368 auto FiniCB = [this](InsertPointTy IP) {
4370 };
4371
4372 auto BodyGenCB = [CriticalRegionBodyStmt, this](InsertPointTy AllocaIP,
4373 InsertPointTy CodeGenIP) {
4375 *this, CriticalRegionBodyStmt, AllocaIP, CodeGenIP, "critical");
4376 };
4377
4378 LexicalScope Scope(*this, S.getSourceRange());
4379 EmitStopPoint(&S);
4380 Builder.restoreIP(OMPBuilder.createCritical(
4381 Builder, BodyGenCB, FiniCB, S.getDirectiveName().getAsString(),
4382 HintInst));
4383
4384 return;
4385 }
4386
4387 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4388 Action.Enter(CGF);
4389 CGF.EmitStmt(S.getAssociatedStmt());
4390 };
4391 const Expr *Hint = nullptr;
4392 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
4393 Hint = HintClause->getHint();
4394 LexicalScope Scope(*this, S.getSourceRange());
4395 EmitStopPoint(&S);
4397 S.getDirectiveName().getAsString(),
4398 CodeGen, S.getBeginLoc(), Hint);
4399}
4400
4402 const OMPParallelForDirective &S) {
4403 // Emit directive as a combined directive that consists of two implicit
4404 // directives: 'parallel' with 'for' directive.
4405 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4406 Action.Enter(CGF);
4407 emitOMPCopyinClause(CGF, S);
4408 (void)emitWorksharingDirective(CGF, S, S.hasCancel());
4409 };
4410 {
4411 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
4412 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
4413 CGCapturedStmtInfo CGSI(CR_OpenMP);
4414 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
4415 OMPLoopScope LoopScope(CGF, S);
4416 return CGF.EmitScalarExpr(S.getNumIterations());
4417 };
4418 bool IsInscan = llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
4419 [](const OMPReductionClause *C) {
4420 return C->getModifier() == OMPC_REDUCTION_inscan;
4421 });
4422 if (IsInscan)
4423 emitScanBasedDirectiveDecls(*this, S, NumIteratorsGen);
4424 auto LPCRegion =
4426 emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
4428 if (IsInscan)
4429 emitScanBasedDirectiveFinals(*this, S, NumIteratorsGen);
4430 }
4431 // Check for outer lastprivate conditional update.
4433}
4434
4436 const OMPParallelForSimdDirective &S) {
4437 // Emit directive as a combined directive that consists of two implicit
4438 // directives: 'parallel' with 'for' directive.
4439 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4440 Action.Enter(CGF);
4441 emitOMPCopyinClause(CGF, S);
4442 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
4443 };
4444 {
4445 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
4446 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
4447 CGCapturedStmtInfo CGSI(CR_OpenMP);
4448 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
4449 OMPLoopScope LoopScope(CGF, S);
4450 return CGF.EmitScalarExpr(S.getNumIterations());
4451 };
4452 bool IsInscan = llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
4453 [](const OMPReductionClause *C) {
4454 return C->getModifier() == OMPC_REDUCTION_inscan;
4455 });
4456 if (IsInscan)
4457 emitScanBasedDirectiveDecls(*this, S, NumIteratorsGen);
4458 auto LPCRegion =
4460 emitCommonOMPParallelDirective(*this, S, OMPD_for_simd, CodeGen,
4462 if (IsInscan)
4463 emitScanBasedDirectiveFinals(*this, S, NumIteratorsGen);
4464 }
4465 // Check for outer lastprivate conditional update.
4467}
4468
4470 const OMPParallelMasterDirective &S) {
4471 // Emit directive as a combined directive that consists of two implicit
4472 // directives: 'parallel' with 'master' directive.
4473 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4474 Action.Enter(CGF);
4475 OMPPrivateScope PrivateScope(CGF);
4476 emitOMPCopyinClause(CGF, S);
4477 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4478 CGF.EmitOMPPrivateClause(S, PrivateScope);
4479 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4480 (void)PrivateScope.Privatize();
4481 emitMaster(CGF, S);
4482 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4483 };
4484 {
4485 auto LPCRegion =
4487 emitCommonOMPParallelDirective(*this, S, OMPD_master, CodeGen,
4490 [](CodeGenFunction &) { return nullptr; });
4491 }
4492 // Check for outer lastprivate conditional update.
4494}
4495
4497 const OMPParallelMaskedDirective &S) {
4498 // Emit directive as a combined directive that consists of two implicit
4499 // directives: 'parallel' with 'masked' directive.
4500 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4501 Action.Enter(CGF);
4502 OMPPrivateScope PrivateScope(CGF);
4503 emitOMPCopyinClause(CGF, S);
4504 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4505 CGF.EmitOMPPrivateClause(S, PrivateScope);
4506 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4507 (void)PrivateScope.Privatize();
4508 emitMasked(CGF, S);
4509 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4510 };
4511 {
4512 auto LPCRegion =
4514 emitCommonOMPParallelDirective(*this, S, OMPD_masked, CodeGen,
4517 [](CodeGenFunction &) { return nullptr; });
4518 }
4519 // Check for outer lastprivate conditional update.
4521}
4522
4525 // Emit directive as a combined directive that consists of two implicit
4526 // directives: 'parallel' with 'sections' directive.
4527 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4528 Action.Enter(CGF);
4529 emitOMPCopyinClause(CGF, S);
4530 CGF.EmitSections(S);
4531 };
4532 {
4533 auto LPCRegion =
4535 emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
4537 }
4538 // Check for outer lastprivate conditional update.
4540}
4541
4542namespace {
4543/// Get the list of variables declared in the context of the untied tasks.
4544class CheckVarsEscapingUntiedTaskDeclContext final
4545 : public ConstStmtVisitor<CheckVarsEscapingUntiedTaskDeclContext> {
4547
4548public:
4549 explicit CheckVarsEscapingUntiedTaskDeclContext() = default;
4550 virtual ~CheckVarsEscapingUntiedTaskDeclContext() = default;
4551 void VisitDeclStmt(const DeclStmt *S) {
4552 if (!S)
4553 return;
4554 // Need to privatize only local vars, static locals can be processed as is.
4555 for (const Decl *D : S->decls()) {
4556 if (const auto *VD = dyn_cast_or_null<VarDecl>(D))
4557 if (VD->hasLocalStorage())
4558 PrivateDecls.push_back(VD);
4559 }
4560 }
4561 void VisitOMPExecutableDirective(const OMPExecutableDirective *) {}
4562 void VisitCapturedStmt(const CapturedStmt *) {}
4563 void VisitLambdaExpr(const LambdaExpr *) {}
4564 void VisitBlockExpr(const BlockExpr *) {}
4565 void VisitStmt(const Stmt *S) {
4566 if (!S)
4567 return;
4568 for (const Stmt *Child : S->children())
4569 if (Child)
4570 Visit(Child);
4571 }
4572
4573 /// Swaps list of vars with the provided one.
4574 ArrayRef<const VarDecl *> getPrivateDecls() const { return PrivateDecls; }
4575};
4576} // anonymous namespace
4577
4580
4581 // First look for 'omp_all_memory' and add this first.
4582 bool OmpAllMemory = false;
4583 if (llvm::any_of(
4584 S.getClausesOfKind<OMPDependClause>(), [](const OMPDependClause *C) {
4585 return C->getDependencyKind() == OMPC_DEPEND_outallmemory ||
4586 C->getDependencyKind() == OMPC_DEPEND_inoutallmemory;
4587 })) {
4588 OmpAllMemory = true;
4589 // Since both OMPC_DEPEND_outallmemory and OMPC_DEPEND_inoutallmemory are
4590 // equivalent to the runtime, always use OMPC_DEPEND_outallmemory to
4591 // simplify.
4593 Data.Dependences.emplace_back(OMPC_DEPEND_outallmemory,
4594 /*IteratorExpr=*/nullptr);
4595 // Add a nullptr Expr to simplify the codegen in emitDependData.
4596 DD.DepExprs.push_back(nullptr);
4597 }
4598 // Add remaining dependences skipping any 'out' or 'inout' if they are
4599 // overridden by 'omp_all_memory'.
4600 for (const auto *C : S.getClausesOfKind<OMPDependClause>()) {
4601 OpenMPDependClauseKind Kind = C->getDependencyKind();
4602 if (Kind == OMPC_DEPEND_outallmemory || Kind == OMPC_DEPEND_inoutallmemory)
4603 continue;
4604 if (OmpAllMemory && (Kind == OMPC_DEPEND_out || Kind == OMPC_DEPEND_inout))
4605 continue;
4607 Data.Dependences.emplace_back(C->getDependencyKind(), C->getModifier());
4608 DD.DepExprs.append(C->varlist_begin(), C->varlist_end());
4609 }
4610}
4611
4613 const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
4614 const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
4616 // Emit outlined function for task construct.
4617 const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
4618 auto I = CS->getCapturedDecl()->param_begin();
4619 auto PartId = std::next(I);
4620 auto TaskT = std::next(I, 4);
4621 // Check if the task is final
4622 if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
4623 // If the condition constant folds and can be elided, try to avoid emitting
4624 // the condition and the dead arm of the if/else.
4625 const Expr *Cond = Clause->getCondition();
4626 bool CondConstant;
4627 if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
4628 Data.Final.setInt(CondConstant);
4629 else
4630 Data.Final.setPointer(EvaluateExprAsBool(Cond));
4631 } else {
4632 // By default the task is not final.
4633 Data.Final.setInt(/*IntVal=*/false);
4634 }
4635 // Check if the task has 'priority' clause.
4636 if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
4637 const Expr *Prio = Clause->getPriority();
4638 Data.Priority.setInt(/*IntVal=*/true);
4639 Data.Priority.setPointer(EmitScalarConversion(
4640 EmitScalarExpr(Prio), Prio->getType(),
4641 getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
4642 Prio->getExprLoc()));
4643 }
4644 // The first function argument for tasks is a thread id, the second one is a
4645 // part id (0 for tied tasks, >=0 for untied task).
4646 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
4647 // Get list of private variables.
4648 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
4649 auto IRef = C->varlist_begin();
4650 for (const Expr *IInit : C->private_copies()) {
4651 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4652 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4653 Data.PrivateVars.push_back(*IRef);
4654 Data.PrivateCopies.push_back(IInit);
4655 }
4656 ++IRef;
4657 }
4658 }
4659 EmittedAsPrivate.clear();
4660 // Get list of firstprivate variables.
4661 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
4662 auto IRef = C->varlist_begin();
4663 auto IElemInitRef = C->inits().begin();
4664 for (const Expr *IInit : C->private_copies()) {
4665 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4666 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4667 Data.FirstprivateVars.push_back(*IRef);
4668 Data.FirstprivateCopies.push_back(IInit);
4669 Data.FirstprivateInits.push_back(*IElemInitRef);
4670 }
4671 ++IRef;
4672 ++IElemInitRef;
4673 }
4674 }
4675 // Get list of lastprivate variables (for taskloops).
4676 llvm::MapVector<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
4677 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
4678 auto IRef = C->varlist_begin();
4679 auto ID = C->destination_exprs().begin();
4680 for (const Expr *IInit : C->private_copies()) {
4681 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4682 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4683 Data.LastprivateVars.push_back(*IRef);
4684 Data.LastprivateCopies.push_back(IInit);
4685 }
4686 LastprivateDstsOrigs.insert(
4687 std::make_pair(cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
4688 cast<DeclRefExpr>(*IRef)));
4689 ++IRef;
4690 ++ID;
4691 }
4692 }
4695 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
4696 Data.ReductionVars.append(C->varlist_begin(), C->varlist_end());
4697 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
4698 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
4699 Data.ReductionOps.append(C->reduction_ops().begin(),
4700 C->reduction_ops().end());
4701 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
4702 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
4703 }
4705 *this, S.getBeginLoc(), LHSs, RHSs, Data);
4706 // Build list of dependences.
4708 // Get list of local vars for untied tasks.
4709 if (!Data.Tied) {
4710 CheckVarsEscapingUntiedTaskDeclContext Checker;
4711 Checker.Visit(S.getInnermostCapturedStmt()->getCapturedStmt());
4712 Data.PrivateLocals.append(Checker.getPrivateDecls().begin(),
4713 Checker.getPrivateDecls().end());
4714 }
4715 auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
4716 CapturedRegion](CodeGenFunction &CGF,
4717 PrePostActionTy &Action) {
4718 llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
4719 std::pair<Address, Address>>
4720 UntiedLocalVars;
4721 // Set proper addresses for generated private copies.
4722 OMPPrivateScope Scope(CGF);
4723 // Generate debug info for variables present in shared clause.
4724 if (auto *DI = CGF.getDebugInfo()) {
4725 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields =
4726 CGF.CapturedStmtInfo->getCaptureFields();
4727 llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue();
4728 if (CaptureFields.size() && ContextValue) {
4729 unsigned CharWidth = CGF.getContext().getCharWidth();
4730 // The shared variables are packed together as members of structure.
4731 // So the address of each shared variable can be computed by adding
4732 // offset of it (within record) to the base address of record. For each
4733 // shared variable, debug intrinsic llvm.dbg.declare is generated with
4734 // appropriate expressions (DIExpression).
4735 // Ex:
4736 // %12 = load %struct.anon*, %struct.anon** %__context.addr.i
4737 // call void @llvm.dbg.declare(metadata %struct.anon* %12,
4738 // metadata !svar1,
4739 // metadata !DIExpression(DW_OP_deref))
4740 // call void @llvm.dbg.declare(metadata %struct.anon* %12,
4741 // metadata !svar2,
4742 // metadata !DIExpression(DW_OP_plus_uconst, 8, DW_OP_deref))
4743 for (auto It = CaptureFields.begin(); It != CaptureFields.end(); ++It) {
4744 const VarDecl *SharedVar = It->first;
4745 RecordDecl *CaptureRecord = It->second->getParent();
4746 const ASTRecordLayout &Layout =
4747 CGF.getContext().getASTRecordLayout(CaptureRecord);
4748 unsigned Offset =
4749 Layout.getFieldOffset(It->second->getFieldIndex()) / CharWidth;
4750 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
4751 (void)DI->EmitDeclareOfAutoVariable(SharedVar, ContextValue,
4752 CGF.Builder, false);
4753 llvm::Instruction &Last = CGF.Builder.GetInsertBlock()->back();
4754 // Get the call dbg.declare instruction we just created and update
4755 // its DIExpression to add offset to base address.
4756 if (auto DDI = dyn_cast<llvm::DbgVariableIntrinsic>(&Last)) {
4758 // Add offset to the base address if non zero.
4759 if (Offset) {
4760 Ops.push_back(llvm::dwarf::DW_OP_plus_uconst);
4761 Ops.push_back(Offset);
4762 }
4763 Ops.push_back(llvm::dwarf::DW_OP_deref);
4764 auto &Ctx = DDI->getContext();
4765 llvm::DIExpression *DIExpr = llvm::DIExpression::get(Ctx, Ops);
4766 Last.setOperand(2, llvm::MetadataAsValue::get(Ctx, DIExpr));
4767 }
4768 }
4769 }
4770 }
4772 if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
4773 !Data.LastprivateVars.empty() || !Data.PrivateLocals.empty()) {
4774 enum { PrivatesParam = 2, CopyFnParam = 3 };
4775 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
4776 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
4777 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
4778 CS->getCapturedDecl()->getParam(PrivatesParam)));
4779 // Map privates.
4783 CallArgs.push_back(PrivatesPtr);
4784 ParamTypes.push_back(PrivatesPtr->getType());
4785 for (const Expr *E : Data.PrivateVars) {
4786 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4787 Address PrivatePtr = CGF.CreateMemTemp(
4788 CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
4789 PrivatePtrs.emplace_back(VD, PrivatePtr);
4790 CallArgs.push_back(PrivatePtr.getPointer());
4791 ParamTypes.push_back(PrivatePtr.getType());
4792 }
4793 for (const Expr *E : Data.FirstprivateVars) {
4794 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4795 Address PrivatePtr =
4796 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
4797 ".firstpriv.ptr.addr");
4798 PrivatePtrs.emplace_back(VD, PrivatePtr);
4799 FirstprivatePtrs.emplace_back(VD, PrivatePtr);
4800 CallArgs.push_back(PrivatePtr.getPointer());
4801 ParamTypes.push_back(PrivatePtr.getType());
4802 }
4803 for (const Expr *E : Data.LastprivateVars) {
4804 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4805 Address PrivatePtr =
4806 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
4807 ".lastpriv.ptr.addr");
4808 PrivatePtrs.emplace_back(VD, PrivatePtr);
4809 CallArgs.push_back(PrivatePtr.getPointer());
4810 ParamTypes.push_back(PrivatePtr.getType());
4811 }
4812 for (const VarDecl *VD : Data.PrivateLocals) {
4814 if (VD->getType()->isLValueReferenceType())
4815 Ty = CGF.getContext().getPointerType(Ty);
4816 if (isAllocatableDecl(VD))
4817 Ty = CGF.getContext().getPointerType(Ty);
4818 Address PrivatePtr = CGF.CreateMemTemp(
4819 CGF.getContext().getPointerType(Ty), ".local.ptr.addr");
4820 auto Result = UntiedLocalVars.insert(
4821 std::make_pair(VD, std::make_pair(PrivatePtr, Address::invalid())));
4822 // If key exists update in place.
4823 if (Result.second == false)
4824 *Result.first = std::make_pair(
4825 VD, std::make_pair(PrivatePtr, Address::invalid()));
4826 CallArgs.push_back(PrivatePtr.getPointer());
4827 ParamTypes.push_back(PrivatePtr.getType());
4828 }
4829 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
4830 ParamTypes, /*isVarArg=*/false);
4831 CopyFn = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4832 CopyFn, CopyFnTy->getPointerTo());
4833 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
4834 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
4835 for (const auto &Pair : LastprivateDstsOrigs) {
4836 const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
4837 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD),
4838 /*RefersToEnclosingVariableOrCapture=*/
4839 CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr,
4840 Pair.second->getType(), VK_LValue,
4841 Pair.second->getExprLoc());
4842 Scope.addPrivate(Pair.first, CGF.EmitLValue(&DRE).getAddress(CGF));
4843 }
4844 for (const auto &Pair : PrivatePtrs) {
4845 Address Replacement = Address(
4846 CGF.Builder.CreateLoad(Pair.second),
4847 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
4848 CGF.getContext().getDeclAlign(Pair.first));
4849 Scope.addPrivate(Pair.first, Replacement);
4850 if (auto *DI = CGF.getDebugInfo())
4851 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
4852 (void)DI->EmitDeclareOfAutoVariable(
4853 Pair.first, Pair.second.getPointer(), CGF.Builder,
4854 /*UsePointerValue*/ true);
4855 }
4856 // Adjust mapping for internal locals by mapping actual memory instead of
4857 // a pointer to this memory.
4858 for (auto &Pair : UntiedLocalVars) {
4859 QualType VDType = Pair.first->getType().getNonReferenceType();
4860 if (Pair.first->getType()->isLValueReferenceType())
4861 VDType = CGF.getContext().getPointerType(VDType);
4862 if (isAllocatableDecl(Pair.first)) {
4863 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
4864 Address Replacement(
4865 Ptr,
4866 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(VDType)),
4