clang 17.0.0git
CGStmtOpenMP.cpp
Go to the documentation of this file.
1//===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit OpenMP nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCleanup.h"
14#include "CGOpenMPRuntime.h"
15#include "CodeGenFunction.h"
16#include "CodeGenModule.h"
17#include "TargetInfo.h"
19#include "clang/AST/Attr.h"
22#include "clang/AST/Stmt.h"
27#include "llvm/ADT/SmallSet.h"
28#include "llvm/BinaryFormat/Dwarf.h"
29#include "llvm/Frontend/OpenMP/OMPConstants.h"
30#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
31#include "llvm/IR/Constants.h"
32#include "llvm/IR/DebugInfoMetadata.h"
33#include "llvm/IR/Instructions.h"
34#include "llvm/IR/IntrinsicInst.h"
35#include "llvm/IR/Metadata.h"
36#include "llvm/Support/AtomicOrdering.h"
37#include <optional>
38using namespace clang;
39using namespace CodeGen;
40using namespace llvm::omp;
41
42static const VarDecl *getBaseDecl(const Expr *Ref);
43
44namespace {
45/// Lexical scope for OpenMP executable constructs, that handles correct codegen
46/// for captured expressions.
47class OMPLexicalScope : public CodeGenFunction::LexicalScope {
48 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
49 for (const auto *C : S.clauses()) {
50 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
51 if (const auto *PreInit =
52 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
53 for (const auto *I : PreInit->decls()) {
54 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
55 CGF.EmitVarDecl(cast<VarDecl>(*I));
56 } else {
57 CodeGenFunction::AutoVarEmission Emission =
58 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
59 CGF.EmitAutoVarCleanups(Emission);
60 }
61 }
62 }
63 }
64 }
65 }
66 CodeGenFunction::OMPPrivateScope InlinedShareds;
67
68 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
69 return CGF.LambdaCaptureFields.lookup(VD) ||
70 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
71 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) &&
72 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
73 }
74
75public:
76 OMPLexicalScope(
78 const std::optional<OpenMPDirectiveKind> CapturedRegion = std::nullopt,
79 const bool EmitPreInitStmt = true)
80 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
81 InlinedShareds(CGF) {
82 if (EmitPreInitStmt)
83 emitPreInitStmt(CGF, S);
84 if (!CapturedRegion)
85 return;
86 assert(S.hasAssociatedStmt() &&
87 "Expected associated statement for inlined directive.");
88 const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
89 for (const auto &C : CS->captures()) {
90 if (C.capturesVariable() || C.capturesVariableByCopy()) {
91 auto *VD = C.getCapturedVar();
92 assert(VD == VD->getCanonicalDecl() &&
93 "Canonical decl must be captured.");
94 DeclRefExpr DRE(
95 CGF.getContext(), const_cast<VarDecl *>(VD),
96 isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
97 InlinedShareds.isGlobalVarCaptured(VD)),
98 VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
99 InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress(CGF));
100 }
101 }
102 (void)InlinedShareds.Privatize();
103 }
104};
105
106/// Lexical scope for OpenMP parallel construct, that handles correct codegen
107/// for captured expressions.
108class OMPParallelScope final : public OMPLexicalScope {
109 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
110 OpenMPDirectiveKind Kind = S.getDirectiveKind();
111 return !(isOpenMPTargetExecutionDirective(Kind) ||
114 }
115
116public:
117 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
118 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt,
119 EmitPreInitStmt(S)) {}
120};
121
122/// Lexical scope for OpenMP teams construct, that handles correct codegen
123/// for captured expressions.
124class OMPTeamsScope final : public OMPLexicalScope {
125 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
126 OpenMPDirectiveKind Kind = S.getDirectiveKind();
127 return !isOpenMPTargetExecutionDirective(Kind) &&
129 }
130
131public:
132 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
133 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt,
134 EmitPreInitStmt(S)) {}
135};
136
137/// Private scope for OpenMP loop-based directives, that supports capturing
138/// of used expression from loop statement.
139class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
140 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopBasedDirective &S) {
141 const DeclStmt *PreInits;
142 CodeGenFunction::OMPMapVars PreCondVars;
143 if (auto *LD = dyn_cast<OMPLoopDirective>(&S)) {
144 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
145 for (const auto *E : LD->counters()) {
146 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
147 EmittedAsPrivate.insert(VD->getCanonicalDecl());
148 (void)PreCondVars.setVarAddr(
149 CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
150 }
151 // Mark private vars as undefs.
152 for (const auto *C : LD->getClausesOfKind<OMPPrivateClause>()) {
153 for (const Expr *IRef : C->varlists()) {
154 const auto *OrigVD =
155 cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
156 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
157 QualType OrigVDTy = OrigVD->getType().getNonReferenceType();
158 (void)PreCondVars.setVarAddr(
159 CGF, OrigVD,
160 Address(llvm::UndefValue::get(CGF.ConvertTypeForMem(
161 CGF.getContext().getPointerType(OrigVDTy))),
162 CGF.ConvertTypeForMem(OrigVDTy),
163 CGF.getContext().getDeclAlign(OrigVD)));
164 }
165 }
166 }
167 (void)PreCondVars.apply(CGF);
168 // Emit init, __range and __end variables for C++ range loops.
170 LD->getInnermostCapturedStmt()->getCapturedStmt(),
171 /*TryImperfectlyNestedLoops=*/true, LD->getLoopsNumber(),
172 [&CGF](unsigned Cnt, const Stmt *CurStmt) {
173 if (const auto *CXXFor = dyn_cast<CXXForRangeStmt>(CurStmt)) {
174 if (const Stmt *Init = CXXFor->getInit())
175 CGF.EmitStmt(Init);
176 CGF.EmitStmt(CXXFor->getRangeStmt());
177 CGF.EmitStmt(CXXFor->getEndStmt());
178 }
179 return false;
180 });
181 PreInits = cast_or_null<DeclStmt>(LD->getPreInits());
182 } else if (const auto *Tile = dyn_cast<OMPTileDirective>(&S)) {
183 PreInits = cast_or_null<DeclStmt>(Tile->getPreInits());
184 } else if (const auto *Unroll = dyn_cast<OMPUnrollDirective>(&S)) {
185 PreInits = cast_or_null<DeclStmt>(Unroll->getPreInits());
186 } else {
187 llvm_unreachable("Unknown loop-based directive kind.");
188 }
189 if (PreInits) {
190 for (const auto *I : PreInits->decls())
191 CGF.EmitVarDecl(cast<VarDecl>(*I));
192 }
193 PreCondVars.restore(CGF);
194 }
195
196public:
197 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopBasedDirective &S)
198 : CodeGenFunction::RunCleanupsScope(CGF) {
199 emitPreInitStmt(CGF, S);
200 }
201};
202
203class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
204 CodeGenFunction::OMPPrivateScope InlinedShareds;
205
206 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
207 return CGF.LambdaCaptureFields.lookup(VD) ||
208 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
209 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) &&
210 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
211 }
212
213public:
214 OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
215 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
216 InlinedShareds(CGF) {
217 for (const auto *C : S.clauses()) {
218 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
219 if (const auto *PreInit =
220 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
221 for (const auto *I : PreInit->decls()) {
222 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
223 CGF.EmitVarDecl(cast<VarDecl>(*I));
224 } else {
225 CodeGenFunction::AutoVarEmission Emission =
226 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
227 CGF.EmitAutoVarCleanups(Emission);
228 }
229 }
230 }
231 } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
232 for (const Expr *E : UDP->varlists()) {
233 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
234 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
235 CGF.EmitVarDecl(*OED);
236 }
237 } else if (const auto *UDP = dyn_cast<OMPUseDeviceAddrClause>(C)) {
238 for (const Expr *E : UDP->varlists()) {
239 const Decl *D = getBaseDecl(E);
240 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
241 CGF.EmitVarDecl(*OED);
242 }
243 }
244 }
245 if (!isOpenMPSimdDirective(S.getDirectiveKind()))
246 CGF.EmitOMPPrivateClause(S, InlinedShareds);
247 if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
248 if (const Expr *E = TG->getReductionRef())
249 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
250 }
251 // Temp copy arrays for inscan reductions should not be emitted as they are
252 // not used in simd only mode.
254 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
255 if (C->getModifier() != OMPC_REDUCTION_inscan)
256 continue;
257 for (const Expr *E : C->copy_array_temps())
258 CopyArrayTemps.insert(cast<DeclRefExpr>(E)->getDecl());
259 }
260 const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
261 while (CS) {
262 for (auto &C : CS->captures()) {
263 if (C.capturesVariable() || C.capturesVariableByCopy()) {
264 auto *VD = C.getCapturedVar();
265 if (CopyArrayTemps.contains(VD))
266 continue;
267 assert(VD == VD->getCanonicalDecl() &&
268 "Canonical decl must be captured.");
269 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
270 isCapturedVar(CGF, VD) ||
271 (CGF.CapturedStmtInfo &&
272 InlinedShareds.isGlobalVarCaptured(VD)),
274 C.getLocation());
275 InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress(CGF));
276 }
277 }
278 CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
279 }
280 (void)InlinedShareds.Privatize();
281 }
282};
283
284} // namespace
285
287 const OMPExecutableDirective &S,
288 const RegionCodeGenTy &CodeGen);
289
290LValue CodeGenFunction::EmitOMPSharedLValue(const Expr *E) {
291 if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
292 if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
293 OrigVD = OrigVD->getCanonicalDecl();
294 bool IsCaptured =
295 LambdaCaptureFields.lookup(OrigVD) ||
297 (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
298 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), IsCaptured,
299 OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
300 return EmitLValue(&DRE);
301 }
302 }
303 return EmitLValue(E);
304}
305
306llvm::Value *CodeGenFunction::getTypeSize(QualType Ty) {
308 llvm::Value *Size = nullptr;
309 auto SizeInChars = C.getTypeSizeInChars(Ty);
310 if (SizeInChars.isZero()) {
311 // getTypeSizeInChars() returns 0 for a VLA.
312 while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) {
313 VlaSizePair VlaSize = getVLASize(VAT);
314 Ty = VlaSize.Type;
315 Size =
316 Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts) : VlaSize.NumElts;
317 }
318 SizeInChars = C.getTypeSizeInChars(Ty);
319 if (SizeInChars.isZero())
320 return llvm::ConstantInt::get(SizeTy, /*V=*/0);
321 return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
322 }
323 return CGM.getSize(SizeInChars);
324}
325
327 const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
328 const RecordDecl *RD = S.getCapturedRecordDecl();
329 auto CurField = RD->field_begin();
330 auto CurCap = S.captures().begin();
331 for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
332 E = S.capture_init_end();
333 I != E; ++I, ++CurField, ++CurCap) {
334 if (CurField->hasCapturedVLAType()) {
335 const VariableArrayType *VAT = CurField->getCapturedVLAType();
336 llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
337 CapturedVars.push_back(Val);
338 } else if (CurCap->capturesThis()) {
339 CapturedVars.push_back(CXXThisValue);
340 } else if (CurCap->capturesVariableByCopy()) {
341 llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
342
343 // If the field is not a pointer, we need to save the actual value
344 // and load it as a void pointer.
345 if (!CurField->getType()->isAnyPointerType()) {
346 ASTContext &Ctx = getContext();
347 Address DstAddr = CreateMemTemp(
348 Ctx.getUIntPtrType(),
349 Twine(CurCap->getCapturedVar()->getName(), ".casted"));
350 LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
351
352 llvm::Value *SrcAddrVal = EmitScalarConversion(
353 DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
354 Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
355 LValue SrcLV =
356 MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
357
358 // Store the value using the source type pointer.
360
361 // Load the value using the destination type pointer.
362 CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
363 }
364 CapturedVars.push_back(CV);
365 } else {
366 assert(CurCap->capturesVariable() && "Expected capture by reference.");
367 CapturedVars.push_back(EmitLValue(*I).getAddress(*this).getPointer());
368 }
369 }
370}
371
373 QualType DstType, StringRef Name,
374 LValue AddrLV) {
375 ASTContext &Ctx = CGF.getContext();
376
377 llvm::Value *CastedPtr = CGF.EmitScalarConversion(
378 AddrLV.getAddress(CGF).getPointer(), Ctx.getUIntPtrType(),
379 Ctx.getPointerType(DstType), Loc);
380 Address TmpAddr =
381 CGF.MakeNaturalAlignAddrLValue(CastedPtr, DstType).getAddress(CGF);
382 return TmpAddr;
383}
384
386 if (T->isLValueReferenceType())
387 return C.getLValueReferenceType(
389 /*SpelledAsLValue=*/false);
390 if (T->isPointerType())
391 return C.getPointerType(getCanonicalParamType(C, T->getPointeeType()));
392 if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
393 if (const auto *VLA = dyn_cast<VariableArrayType>(A))
394 return getCanonicalParamType(C, VLA->getElementType());
395 if (!A->isVariablyModifiedType())
396 return C.getCanonicalType(T);
397 }
398 return C.getCanonicalParamType(T);
399}
400
401namespace {
402/// Contains required data for proper outlined function codegen.
403struct FunctionOptions {
404 /// Captured statement for which the function is generated.
405 const CapturedStmt *S = nullptr;
406 /// true if cast to/from UIntPtr is required for variables captured by
407 /// value.
408 const bool UIntPtrCastRequired = true;
409 /// true if only casted arguments must be registered as local args or VLA
410 /// sizes.
411 const bool RegisterCastedArgsOnly = false;
412 /// Name of the generated function.
413 const StringRef FunctionName;
414 /// Location of the non-debug version of the outlined function.
415 SourceLocation Loc;
416 explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
417 bool RegisterCastedArgsOnly, StringRef FunctionName,
418 SourceLocation Loc)
419 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
420 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
421 FunctionName(FunctionName), Loc(Loc) {}
422};
423} // namespace
424
425static llvm::Function *emitOutlinedFunctionPrologue(
427 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
428 &LocalAddrs,
429 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
430 &VLASizes,
431 llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
432 const CapturedDecl *CD = FO.S->getCapturedDecl();
433 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
434 assert(CD->hasBody() && "missing CapturedDecl body");
435
436 CXXThisValue = nullptr;
437 // Build the argument list.
438 CodeGenModule &CGM = CGF.CGM;
439 ASTContext &Ctx = CGM.getContext();
440 FunctionArgList TargetArgs;
441 Args.append(CD->param_begin(),
442 std::next(CD->param_begin(), CD->getContextParamPosition()));
443 TargetArgs.append(
444 CD->param_begin(),
445 std::next(CD->param_begin(), CD->getContextParamPosition()));
446 auto I = FO.S->captures().begin();
447 FunctionDecl *DebugFunctionDecl = nullptr;
448 if (!FO.UIntPtrCastRequired) {
450 QualType FunctionTy = Ctx.getFunctionType(Ctx.VoidTy, std::nullopt, EPI);
451 DebugFunctionDecl = FunctionDecl::Create(
452 Ctx, Ctx.getTranslationUnitDecl(), FO.S->getBeginLoc(),
453 SourceLocation(), DeclarationName(), FunctionTy,
454 Ctx.getTrivialTypeSourceInfo(FunctionTy), SC_Static,
455 /*UsesFPIntrin=*/false, /*isInlineSpecified=*/false,
456 /*hasWrittenPrototype=*/false);
457 }
458 for (const FieldDecl *FD : RD->fields()) {
459 QualType ArgType = FD->getType();
460 IdentifierInfo *II = nullptr;
461 VarDecl *CapVar = nullptr;
462
463 // If this is a capture by copy and the type is not a pointer, the outlined
464 // function argument type should be uintptr and the value properly casted to
465 // uintptr. This is necessary given that the runtime library is only able to
466 // deal with pointers. We can pass in the same way the VLA type sizes to the
467 // outlined function.
468 if (FO.UIntPtrCastRequired &&
469 ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
470 I->capturesVariableArrayType()))
471 ArgType = Ctx.getUIntPtrType();
472
473 if (I->capturesVariable() || I->capturesVariableByCopy()) {
474 CapVar = I->getCapturedVar();
475 II = CapVar->getIdentifier();
476 } else if (I->capturesThis()) {
477 II = &Ctx.Idents.get("this");
478 } else {
479 assert(I->capturesVariableArrayType());
480 II = &Ctx.Idents.get("vla");
481 }
482 if (ArgType->isVariablyModifiedType())
483 ArgType = getCanonicalParamType(Ctx, ArgType);
484 VarDecl *Arg;
485 if (CapVar && (CapVar->getTLSKind() != clang::VarDecl::TLS_None)) {
486 Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
487 II, ArgType,
489 } else if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
491 Ctx, DebugFunctionDecl,
492 CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),
493 CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
494 /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
495 } else {
496 Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
497 II, ArgType, ImplicitParamDecl::Other);
498 }
499 Args.emplace_back(Arg);
500 // Do not cast arguments if we emit function with non-original types.
501 TargetArgs.emplace_back(
502 FO.UIntPtrCastRequired
503 ? Arg
504 : CGM.getOpenMPRuntime().translateParameter(FD, Arg));
505 ++I;
506 }
507 Args.append(std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
508 CD->param_end());
509 TargetArgs.append(
510 std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
511 CD->param_end());
512
513 // Create the function declaration.
514 const CGFunctionInfo &FuncInfo =
516 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
517
518 auto *F =
519 llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
520 FO.FunctionName, &CGM.getModule());
521 CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
522 if (CD->isNothrow())
523 F->setDoesNotThrow();
524 F->setDoesNotRecurse();
525
526 // Always inline the outlined function if optimizations are enabled.
527 if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
528 F->removeFnAttr(llvm::Attribute::NoInline);
529 F->addFnAttr(llvm::Attribute::AlwaysInline);
530 }
531
532 // Generate the function.
533 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
534 FO.UIntPtrCastRequired ? FO.Loc : FO.S->getBeginLoc(),
535 FO.UIntPtrCastRequired ? FO.Loc
536 : CD->getBody()->getBeginLoc());
537 unsigned Cnt = CD->getContextParamPosition();
538 I = FO.S->captures().begin();
539 for (const FieldDecl *FD : RD->fields()) {
540 // Do not map arguments if we emit function with non-original types.
541 Address LocalAddr(Address::invalid());
542 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
543 LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
544 TargetArgs[Cnt]);
545 } else {
546 LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
547 }
548 // If we are capturing a pointer by copy we don't need to do anything, just
549 // use the value that we get from the arguments.
550 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
551 const VarDecl *CurVD = I->getCapturedVar();
552 if (!FO.RegisterCastedArgsOnly)
553 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
554 ++Cnt;
555 ++I;
556 continue;
557 }
558
559 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
561 if (FD->hasCapturedVLAType()) {
562 if (FO.UIntPtrCastRequired) {
563 ArgLVal = CGF.MakeAddrLValue(
564 castValueFromUintptr(CGF, I->getLocation(), FD->getType(),
565 Args[Cnt]->getName(), ArgLVal),
567 }
568 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
569 const VariableArrayType *VAT = FD->getCapturedVLAType();
570 VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg);
571 } else if (I->capturesVariable()) {
572 const VarDecl *Var = I->getCapturedVar();
573 QualType VarTy = Var->getType();
574 Address ArgAddr = ArgLVal.getAddress(CGF);
575 if (ArgLVal.getType()->isLValueReferenceType()) {
576 ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
577 } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) {
578 assert(ArgLVal.getType()->isPointerType());
579 ArgAddr = CGF.EmitLoadOfPointer(
580 ArgAddr, ArgLVal.getType()->castAs<PointerType>());
581 }
582 if (!FO.RegisterCastedArgsOnly) {
583 LocalAddrs.insert(
584 {Args[Cnt], {Var, ArgAddr.withAlignment(Ctx.getDeclAlign(Var))}});
585 }
586 } else if (I->capturesVariableByCopy()) {
587 assert(!FD->getType()->isAnyPointerType() &&
588 "Not expecting a captured pointer.");
589 const VarDecl *Var = I->getCapturedVar();
590 LocalAddrs.insert({Args[Cnt],
591 {Var, FO.UIntPtrCastRequired
593 CGF, I->getLocation(), FD->getType(),
594 Args[Cnt]->getName(), ArgLVal)
595 : ArgLVal.getAddress(CGF)}});
596 } else {
597 // If 'this' is captured, load it into CXXThisValue.
598 assert(I->capturesThis());
599 CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
600 LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress(CGF)}});
601 }
602 ++Cnt;
603 ++I;
604 }
605
606 return F;
607}
608
609llvm::Function *
611 SourceLocation Loc) {
612 assert(
614 "CapturedStmtInfo should be set when generating the captured function");
615 const CapturedDecl *CD = S.getCapturedDecl();
616 // Build the argument list.
617 bool NeedWrapperFunction =
619 FunctionArgList Args;
620 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
621 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
622 SmallString<256> Buffer;
623 llvm::raw_svector_ostream Out(Buffer);
625 if (NeedWrapperFunction)
626 Out << "_debug__";
627 FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
628 Out.str(), Loc);
629 llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs,
630 VLASizes, CXXThisValue, FO);
631 CodeGenFunction::OMPPrivateScope LocalScope(*this);
632 for (const auto &LocalAddrPair : LocalAddrs) {
633 if (LocalAddrPair.second.first) {
634 LocalScope.addPrivate(LocalAddrPair.second.first,
635 LocalAddrPair.second.second);
636 }
637 }
638 (void)LocalScope.Privatize();
639 for (const auto &VLASizePair : VLASizes)
640 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
642 CapturedStmtInfo->EmitBody(*this, CD->getBody());
643 (void)LocalScope.ForceCleanup();
645 if (!NeedWrapperFunction)
646 return F;
647
648 FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
649 /*RegisterCastedArgsOnly=*/true,
651 CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
652 WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
653 Args.clear();
654 LocalAddrs.clear();
655 VLASizes.clear();
656 llvm::Function *WrapperF =
657 emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
658 WrapperCGF.CXXThisValue, WrapperFO);
660 auto *PI = F->arg_begin();
661 for (const auto *Arg : Args) {
662 llvm::Value *CallArg;
663 auto I = LocalAddrs.find(Arg);
664 if (I != LocalAddrs.end()) {
665 LValue LV = WrapperCGF.MakeAddrLValue(
666 I->second.second,
667 I->second.first ? I->second.first->getType() : Arg->getType(),
669 if (LV.getType()->isAnyComplexType())
670 LV.setAddress(WrapperCGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
671 LV.getAddress(WrapperCGF),
672 PI->getType()->getPointerTo(
673 LV.getAddress(WrapperCGF).getAddressSpace()),
674 PI->getType()));
675 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
676 } else {
677 auto EI = VLASizes.find(Arg);
678 if (EI != VLASizes.end()) {
679 CallArg = EI->second.second;
680 } else {
681 LValue LV =
682 WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
683 Arg->getType(), AlignmentSource::Decl);
684 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
685 }
686 }
687 CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
688 ++PI;
689 }
690 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, Loc, F, CallArgs);
691 WrapperCGF.FinishFunction();
692 return WrapperF;
693}
694
695//===----------------------------------------------------------------------===//
696// OpenMP Directive Emission
697//===----------------------------------------------------------------------===//
699 Address DestAddr, Address SrcAddr, QualType OriginalType,
700 const llvm::function_ref<void(Address, Address)> CopyGen) {
701 // Perform element-by-element initialization.
702 QualType ElementTy;
703
704 // Drill down to the base element type on both arrays.
705 const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
706 llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
707 SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
708
709 llvm::Value *SrcBegin = SrcAddr.getPointer();
710 llvm::Value *DestBegin = DestAddr.getPointer();
711 // Cast from pointer to array type to pointer to single element.
712 llvm::Value *DestEnd = Builder.CreateInBoundsGEP(DestAddr.getElementType(),
713 DestBegin, NumElements);
714
715 // The basic structure here is a while-do loop.
716 llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body");
717 llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done");
718 llvm::Value *IsEmpty =
719 Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
720 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
721
722 // Enter the loop body, making that address the current address.
723 llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
724 EmitBlock(BodyBB);
725
726 CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
727
728 llvm::PHINode *SrcElementPHI =
729 Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
730 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
731 Address SrcElementCurrent =
732 Address(SrcElementPHI, SrcAddr.getElementType(),
733 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
734
735 llvm::PHINode *DestElementPHI = Builder.CreatePHI(
736 DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
737 DestElementPHI->addIncoming(DestBegin, EntryBB);
738 Address DestElementCurrent =
739 Address(DestElementPHI, DestAddr.getElementType(),
740 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
741
742 // Emit copy.
743 CopyGen(DestElementCurrent, SrcElementCurrent);
744
745 // Shift the address forward by one element.
746 llvm::Value *DestElementNext =
747 Builder.CreateConstGEP1_32(DestAddr.getElementType(), DestElementPHI,
748 /*Idx0=*/1, "omp.arraycpy.dest.element");
749 llvm::Value *SrcElementNext =
750 Builder.CreateConstGEP1_32(SrcAddr.getElementType(), SrcElementPHI,
751 /*Idx0=*/1, "omp.arraycpy.src.element");
752 // Check whether we've reached the end.
753 llvm::Value *Done =
754 Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
755 Builder.CreateCondBr(Done, DoneBB, BodyBB);
756 DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
757 SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
758
759 // Done.
760 EmitBlock(DoneBB, /*IsFinished=*/true);
761}
762
763void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
764 Address SrcAddr, const VarDecl *DestVD,
765 const VarDecl *SrcVD, const Expr *Copy) {
766 if (OriginalType->isArrayType()) {
767 const auto *BO = dyn_cast<BinaryOperator>(Copy);
768 if (BO && BO->getOpcode() == BO_Assign) {
769 // Perform simple memcpy for simple copying.
770 LValue Dest = MakeAddrLValue(DestAddr, OriginalType);
771 LValue Src = MakeAddrLValue(SrcAddr, OriginalType);
772 EmitAggregateAssign(Dest, Src, OriginalType);
773 } else {
774 // For arrays with complex element types perform element by element
775 // copying.
777 DestAddr, SrcAddr, OriginalType,
778 [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
779 // Working with the single array element, so have to remap
780 // destination and source variables to corresponding array
781 // elements.
782 CodeGenFunction::OMPPrivateScope Remap(*this);
783 Remap.addPrivate(DestVD, DestElement);
784 Remap.addPrivate(SrcVD, SrcElement);
785 (void)Remap.Privatize();
786 EmitIgnoredExpr(Copy);
787 });
788 }
789 } else {
790 // Remap pseudo source variable to private copy.
791 CodeGenFunction::OMPPrivateScope Remap(*this);
792 Remap.addPrivate(SrcVD, SrcAddr);
793 Remap.addPrivate(DestVD, DestAddr);
794 (void)Remap.Privatize();
795 // Emit copying of the whole variable.
796 EmitIgnoredExpr(Copy);
797 }
798}
799
801 OMPPrivateScope &PrivateScope) {
802 if (!HaveInsertPoint())
803 return false;
804 bool DeviceConstTarget =
805 getLangOpts().OpenMPIsDevice &&
807 bool FirstprivateIsLastprivate = false;
808 llvm::DenseMap<const VarDecl *, OpenMPLastprivateModifier> Lastprivates;
809 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
810 for (const auto *D : C->varlists())
811 Lastprivates.try_emplace(
812 cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl(),
813 C->getKind());
814 }
815 llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
817 getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind());
818 // Force emission of the firstprivate copy if the directive does not emit
819 // outlined function, like omp for, omp simd, omp distribute etc.
820 bool MustEmitFirstprivateCopy =
821 CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
822 for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
823 const auto *IRef = C->varlist_begin();
824 const auto *InitsRef = C->inits().begin();
825 for (const Expr *IInit : C->private_copies()) {
826 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
827 bool ThisFirstprivateIsLastprivate =
828 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
829 const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
830 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
831 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
832 !FD->getType()->isReferenceType() &&
833 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
834 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
835 ++IRef;
836 ++InitsRef;
837 continue;
838 }
839 // Do not emit copy for firstprivate constant variables in target regions,
840 // captured by reference.
841 if (DeviceConstTarget && OrigVD->getType().isConstant(getContext()) &&
842 FD && FD->getType()->isReferenceType() &&
843 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
844 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
845 ++IRef;
846 ++InitsRef;
847 continue;
848 }
849 FirstprivateIsLastprivate =
850 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
851 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
852 const auto *VDInit =
853 cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
854 bool IsRegistered;
855 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
856 /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
857 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
858 LValue OriginalLVal;
859 if (!FD) {
860 // Check if the firstprivate variable is just a constant value.
861 ConstantEmission CE = tryEmitAsConstant(&DRE);
862 if (CE && !CE.isReference()) {
863 // Constant value, no need to create a copy.
864 ++IRef;
865 ++InitsRef;
866 continue;
867 }
868 if (CE && CE.isReference()) {
869 OriginalLVal = CE.getReferenceLValue(*this, &DRE);
870 } else {
871 assert(!CE && "Expected non-constant firstprivate.");
872 OriginalLVal = EmitLValue(&DRE);
873 }
874 } else {
875 OriginalLVal = EmitLValue(&DRE);
876 }
877 QualType Type = VD->getType();
878 if (Type->isArrayType()) {
879 // Emit VarDecl with copy init for arrays.
880 // Get the address of the original variable captured in current
881 // captured region.
882 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
883 const Expr *Init = VD->getInit();
884 if (!isa<CXXConstructExpr>(Init) || isTrivialInitializer(Init)) {
885 // Perform simple memcpy.
886 LValue Dest = MakeAddrLValue(Emission.getAllocatedAddress(), Type);
887 EmitAggregateAssign(Dest, OriginalLVal, Type);
888 } else {
890 Emission.getAllocatedAddress(), OriginalLVal.getAddress(*this),
891 Type,
892 [this, VDInit, Init](Address DestElement, Address SrcElement) {
893 // Clean up any temporaries needed by the
894 // initialization.
895 RunCleanupsScope InitScope(*this);
896 // Emit initialization for single element.
897 setAddrOfLocalVar(VDInit, SrcElement);
898 EmitAnyExprToMem(Init, DestElement,
899 Init->getType().getQualifiers(),
900 /*IsInitializer*/ false);
901 LocalDeclMap.erase(VDInit);
902 });
903 }
904 EmitAutoVarCleanups(Emission);
905 IsRegistered =
906 PrivateScope.addPrivate(OrigVD, Emission.getAllocatedAddress());
907 } else {
908 Address OriginalAddr = OriginalLVal.getAddress(*this);
909 // Emit private VarDecl with copy init.
910 // Remap temp VDInit variable to the address of the original
911 // variable (for proper handling of captured global variables).
912 setAddrOfLocalVar(VDInit, OriginalAddr);
913 EmitDecl(*VD);
914 LocalDeclMap.erase(VDInit);
915 Address VDAddr = GetAddrOfLocalVar(VD);
916 if (ThisFirstprivateIsLastprivate &&
917 Lastprivates[OrigVD->getCanonicalDecl()] ==
918 OMPC_LASTPRIVATE_conditional) {
919 // Create/init special variable for lastprivate conditionals.
920 llvm::Value *V =
921 EmitLoadOfScalar(MakeAddrLValue(VDAddr, (*IRef)->getType(),
923 (*IRef)->getExprLoc());
925 *this, OrigVD);
926 EmitStoreOfScalar(V, MakeAddrLValue(VDAddr, (*IRef)->getType(),
928 LocalDeclMap.erase(VD);
929 setAddrOfLocalVar(VD, VDAddr);
930 }
931 IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);
932 }
933 assert(IsRegistered &&
934 "firstprivate var already registered as private");
935 // Silence the warning about unused variable.
936 (void)IsRegistered;
937 }
938 ++IRef;
939 ++InitsRef;
940 }
941 }
942 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
943}
944
946 const OMPExecutableDirective &D,
947 CodeGenFunction::OMPPrivateScope &PrivateScope) {
948 if (!HaveInsertPoint())
949 return;
950 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
951 for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
952 auto IRef = C->varlist_begin();
953 for (const Expr *IInit : C->private_copies()) {
954 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
955 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
956 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
957 EmitDecl(*VD);
958 // Emit private VarDecl with copy init.
959 bool IsRegistered =
960 PrivateScope.addPrivate(OrigVD, GetAddrOfLocalVar(VD));
961 assert(IsRegistered && "private var already registered as private");
962 // Silence the warning about unused variable.
963 (void)IsRegistered;
964 }
965 ++IRef;
966 }
967 }
968}
969
971 if (!HaveInsertPoint())
972 return false;
973 // threadprivate_var1 = master_threadprivate_var1;
974 // operator=(threadprivate_var2, master_threadprivate_var2);
975 // ...
976 // __kmpc_barrier(&loc, global_tid);
978 llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
979 for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
980 auto IRef = C->varlist_begin();
981 auto ISrcRef = C->source_exprs().begin();
982 auto IDestRef = C->destination_exprs().begin();
983 for (const Expr *AssignOp : C->assignment_ops()) {
984 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
985 QualType Type = VD->getType();
986 if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
987 // Get the address of the master variable. If we are emitting code with
988 // TLS support, the address is passed from the master as field in the
989 // captured declaration.
990 Address MasterAddr = Address::invalid();
991 if (getLangOpts().OpenMPUseTLS &&
992 getContext().getTargetInfo().isTLSSupported()) {
993 assert(CapturedStmtInfo->lookup(VD) &&
994 "Copyin threadprivates should have been captured!");
995 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), true,
996 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
997 MasterAddr = EmitLValue(&DRE).getAddress(*this);
998 LocalDeclMap.erase(VD);
999 } else {
1000 MasterAddr =
1002 : CGM.GetAddrOfGlobal(VD),
1004 getContext().getDeclAlign(VD));
1005 }
1006 // Get the address of the threadprivate variable.
1007 Address PrivateAddr = EmitLValue(*IRef).getAddress(*this);
1008 if (CopiedVars.size() == 1) {
1009 // At first check if current thread is a master thread. If it is, no
1010 // need to copy data.
1011 CopyBegin = createBasicBlock("copyin.not.master");
1012 CopyEnd = createBasicBlock("copyin.not.master.end");
1013 // TODO: Avoid ptrtoint conversion.
1014 auto *MasterAddrInt =
1015 Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy);
1016 auto *PrivateAddrInt =
1017 Builder.CreatePtrToInt(PrivateAddr.getPointer(), CGM.IntPtrTy);
1018 Builder.CreateCondBr(
1019 Builder.CreateICmpNE(MasterAddrInt, PrivateAddrInt), CopyBegin,
1020 CopyEnd);
1021 EmitBlock(CopyBegin);
1022 }
1023 const auto *SrcVD =
1024 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1025 const auto *DestVD =
1026 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1027 EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
1028 }
1029 ++IRef;
1030 ++ISrcRef;
1031 ++IDestRef;
1032 }
1033 }
1034 if (CopyEnd) {
1035 // Exit out of copying procedure for non-master thread.
1036 EmitBlock(CopyEnd, /*IsFinished=*/true);
1037 return true;
1038 }
1039 return false;
1040}
1041
1043 const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
1044 if (!HaveInsertPoint())
1045 return false;
1046 bool HasAtLeastOneLastprivate = false;
1049 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1050 for (const Expr *C : LoopDirective->counters()) {
1051 SIMDLCVs.insert(
1052 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
1053 }
1054 }
1055 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1056 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1057 HasAtLeastOneLastprivate = true;
1059 !getLangOpts().OpenMPSimd)
1060 break;
1061 const auto *IRef = C->varlist_begin();
1062 const auto *IDestRef = C->destination_exprs().begin();
1063 for (const Expr *IInit : C->private_copies()) {
1064 // Keep the address of the original variable for future update at the end
1065 // of the loop.
1066 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1067 // Taskloops do not require additional initialization, it is done in
1068 // runtime support library.
1069 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
1070 const auto *DestVD =
1071 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1072 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1073 /*RefersToEnclosingVariableOrCapture=*/
1074 CapturedStmtInfo->lookup(OrigVD) != nullptr,
1075 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
1076 PrivateScope.addPrivate(DestVD, EmitLValue(&DRE).getAddress(*this));
1077 // Check if the variable is also a firstprivate: in this case IInit is
1078 // not generated. Initialization of this variable will happen in codegen
1079 // for 'firstprivate' clause.
1080 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
1081 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
1082 Address VDAddr = Address::invalid();
1083 if (C->getKind() == OMPC_LASTPRIVATE_conditional) {
1085 *this, OrigVD);
1086 setAddrOfLocalVar(VD, VDAddr);
1087 } else {
1088 // Emit private VarDecl with copy init.
1089 EmitDecl(*VD);
1090 VDAddr = GetAddrOfLocalVar(VD);
1091 }
1092 bool IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);
1093 assert(IsRegistered &&
1094 "lastprivate var already registered as private");
1095 (void)IsRegistered;
1096 }
1097 }
1098 ++IRef;
1099 ++IDestRef;
1100 }
1101 }
1102 return HasAtLeastOneLastprivate;
1103}
1104
1106 const OMPExecutableDirective &D, bool NoFinals,
1107 llvm::Value *IsLastIterCond) {
1108 if (!HaveInsertPoint())
1109 return;
1110 // Emit following code:
1111 // if (<IsLastIterCond>) {
1112 // orig_var1 = private_orig_var1;
1113 // ...
1114 // orig_varn = private_orig_varn;
1115 // }
1116 llvm::BasicBlock *ThenBB = nullptr;
1117 llvm::BasicBlock *DoneBB = nullptr;
1118 if (IsLastIterCond) {
1119 // Emit implicit barrier if at least one lastprivate conditional is found
1120 // and this is not a simd mode.
1121 if (!getLangOpts().OpenMPSimd &&
1122 llvm::any_of(D.getClausesOfKind<OMPLastprivateClause>(),
1123 [](const OMPLastprivateClause *C) {
1124 return C->getKind() == OMPC_LASTPRIVATE_conditional;
1125 })) {
1127 OMPD_unknown,
1128 /*EmitChecks=*/false,
1129 /*ForceSimpleCall=*/true);
1130 }
1131 ThenBB = createBasicBlock(".omp.lastprivate.then");
1132 DoneBB = createBasicBlock(".omp.lastprivate.done");
1133 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1134 EmitBlock(ThenBB);
1135 }
1136 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1137 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1138 if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1139 auto IC = LoopDirective->counters().begin();
1140 for (const Expr *F : LoopDirective->finals()) {
1141 const auto *D =
1142 cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1143 if (NoFinals)
1144 AlreadyEmittedVars.insert(D);
1145 else
1146 LoopCountersAndUpdates[D] = F;
1147 ++IC;
1148 }
1149 }
1150 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1151 auto IRef = C->varlist_begin();
1152 auto ISrcRef = C->source_exprs().begin();
1153 auto IDestRef = C->destination_exprs().begin();
1154 for (const Expr *AssignOp : C->assignment_ops()) {
1155 const auto *PrivateVD =
1156 cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1157 QualType Type = PrivateVD->getType();
1158 const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1159 if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1160 // If lastprivate variable is a loop control variable for loop-based
1161 // directive, update its value before copyin back to original
1162 // variable.
1163 if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1164 EmitIgnoredExpr(FinalExpr);
1165 const auto *SrcVD =
1166 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1167 const auto *DestVD =
1168 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1169 // Get the address of the private variable.
1170 Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
1171 if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1172 PrivateAddr = Address(
1173 Builder.CreateLoad(PrivateAddr),
1174 CGM.getTypes().ConvertTypeForMem(RefTy->getPointeeType()),
1175 CGM.getNaturalTypeAlignment(RefTy->getPointeeType()));
1176 // Store the last value to the private copy in the last iteration.
1177 if (C->getKind() == OMPC_LASTPRIVATE_conditional)
1179 *this, MakeAddrLValue(PrivateAddr, (*IRef)->getType()), PrivateVD,
1180 (*IRef)->getExprLoc());
1181 // Get the address of the original variable.
1182 Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1183 EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1184 }
1185 ++IRef;
1186 ++ISrcRef;
1187 ++IDestRef;
1188 }
1189 if (const Expr *PostUpdate = C->getPostUpdateExpr())
1190 EmitIgnoredExpr(PostUpdate);
1191 }
1192 if (IsLastIterCond)
1193 EmitBlock(DoneBB, /*IsFinished=*/true);
1194}
1195
1197 const OMPExecutableDirective &D,
1198 CodeGenFunction::OMPPrivateScope &PrivateScope, bool ForInscan) {
1199 if (!HaveInsertPoint())
1200 return;
1203 SmallVector<const Expr *, 4> ReductionOps;
1209 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1210 if (ForInscan != (C->getModifier() == OMPC_REDUCTION_inscan))
1211 continue;
1212 Shareds.append(C->varlist_begin(), C->varlist_end());
1213 Privates.append(C->privates().begin(), C->privates().end());
1214 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1215 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1216 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1217 if (C->getModifier() == OMPC_REDUCTION_task) {
1218 Data.ReductionVars.append(C->privates().begin(), C->privates().end());
1219 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
1220 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
1221 Data.ReductionOps.append(C->reduction_ops().begin(),
1222 C->reduction_ops().end());
1223 TaskLHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1224 TaskRHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1225 }
1226 }
1227 ReductionCodeGen RedCG(Shareds, Shareds, Privates, ReductionOps);
1228 unsigned Count = 0;
1229 auto *ILHS = LHSs.begin();
1230 auto *IRHS = RHSs.begin();
1231 auto *IPriv = Privates.begin();
1232 for (const Expr *IRef : Shareds) {
1233 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1234 // Emit private VarDecl with reduction init.
1235 RedCG.emitSharedOrigLValue(*this, Count);
1236 RedCG.emitAggregateType(*this, Count);
1237 AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);
1238 RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
1239 RedCG.getSharedLValue(Count).getAddress(*this),
1240 [&Emission](CodeGenFunction &CGF) {
1241 CGF.EmitAutoVarInit(Emission);
1242 return true;
1243 });
1244 EmitAutoVarCleanups(Emission);
1245 Address BaseAddr = RedCG.adjustPrivateAddress(
1246 *this, Count, Emission.getAllocatedAddress());
1247 bool IsRegistered =
1248 PrivateScope.addPrivate(RedCG.getBaseDecl(Count), BaseAddr);
1249 assert(IsRegistered && "private var already registered as private");
1250 // Silence the warning about unused variable.
1251 (void)IsRegistered;
1252
1253 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1254 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1255 QualType Type = PrivateVD->getType();
1256 bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1257 if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1258 // Store the address of the original variable associated with the LHS
1259 // implicit variable.
1260 PrivateScope.addPrivate(LHSVD,
1261 RedCG.getSharedLValue(Count).getAddress(*this));
1262 PrivateScope.addPrivate(RHSVD, GetAddrOfLocalVar(PrivateVD));
1263 } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1264 isa<ArraySubscriptExpr>(IRef)) {
1265 // Store the address of the original variable associated with the LHS
1266 // implicit variable.
1267 PrivateScope.addPrivate(LHSVD,
1268 RedCG.getSharedLValue(Count).getAddress(*this));
1269 PrivateScope.addPrivate(RHSVD, Builder.CreateElementBitCast(
1270 GetAddrOfLocalVar(PrivateVD),
1271 ConvertTypeForMem(RHSVD->getType()),
1272 "rhs.begin"));
1273 } else {
1274 QualType Type = PrivateVD->getType();
1275 bool IsArray = getContext().getAsArrayType(Type) != nullptr;
1276 Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress(*this);
1277 // Store the address of the original variable associated with the LHS
1278 // implicit variable.
1279 if (IsArray) {
1280 OriginalAddr = Builder.CreateElementBitCast(
1281 OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin");
1282 }
1283 PrivateScope.addPrivate(LHSVD, OriginalAddr);
1284 PrivateScope.addPrivate(
1285 RHSVD, IsArray ? Builder.CreateElementBitCast(
1286 GetAddrOfLocalVar(PrivateVD),
1287 ConvertTypeForMem(RHSVD->getType()), "rhs.begin")
1288 : GetAddrOfLocalVar(PrivateVD));
1289 }
1290 ++ILHS;
1291 ++IRHS;
1292 ++IPriv;
1293 ++Count;
1294 }
1295 if (!Data.ReductionVars.empty()) {
1296 Data.IsReductionWithTaskMod = true;
1297 Data.IsWorksharingReduction =
1299 llvm::Value *ReductionDesc = CGM.getOpenMPRuntime().emitTaskReductionInit(
1300 *this, D.getBeginLoc(), TaskLHSs, TaskRHSs, Data);
1301 const Expr *TaskRedRef = nullptr;
1302 switch (D.getDirectiveKind()) {
1303 case OMPD_parallel:
1304 TaskRedRef = cast<OMPParallelDirective>(D).getTaskReductionRefExpr();
1305 break;
1306 case OMPD_for:
1307 TaskRedRef = cast<OMPForDirective>(D).getTaskReductionRefExpr();
1308 break;
1309 case OMPD_sections:
1310 TaskRedRef = cast<OMPSectionsDirective>(D).getTaskReductionRefExpr();
1311 break;
1312 case OMPD_parallel_for:
1313 TaskRedRef = cast<OMPParallelForDirective>(D).getTaskReductionRefExpr();
1314 break;
1315 case OMPD_parallel_master:
1316 TaskRedRef =
1317 cast<OMPParallelMasterDirective>(D).getTaskReductionRefExpr();
1318 break;
1319 case OMPD_parallel_sections:
1320 TaskRedRef =
1321 cast<OMPParallelSectionsDirective>(D).getTaskReductionRefExpr();
1322 break;
1323 case OMPD_target_parallel:
1324 TaskRedRef =
1325 cast<OMPTargetParallelDirective>(D).getTaskReductionRefExpr();
1326 break;
1327 case OMPD_target_parallel_for:
1328 TaskRedRef =
1329 cast<OMPTargetParallelForDirective>(D).getTaskReductionRefExpr();
1330 break;
1331 case OMPD_distribute_parallel_for:
1332 TaskRedRef =
1333 cast<OMPDistributeParallelForDirective>(D).getTaskReductionRefExpr();
1334 break;
1335 case OMPD_teams_distribute_parallel_for:
1336 TaskRedRef = cast<OMPTeamsDistributeParallelForDirective>(D)
1337 .getTaskReductionRefExpr();
1338 break;
1339 case OMPD_target_teams_distribute_parallel_for:
1340 TaskRedRef = cast<OMPTargetTeamsDistributeParallelForDirective>(D)
1341 .getTaskReductionRefExpr();
1342 break;
1343 case OMPD_simd:
1344 case OMPD_for_simd:
1345 case OMPD_section:
1346 case OMPD_single:
1347 case OMPD_master:
1348 case OMPD_critical:
1349 case OMPD_parallel_for_simd:
1350 case OMPD_task:
1351 case OMPD_taskyield:
1352 case OMPD_error:
1353 case OMPD_barrier:
1354 case OMPD_taskwait:
1355 case OMPD_taskgroup:
1356 case OMPD_flush:
1357 case OMPD_depobj:
1358 case OMPD_scan:
1359 case OMPD_ordered:
1360 case OMPD_atomic:
1361 case OMPD_teams:
1362 case OMPD_target:
1363 case OMPD_cancellation_point:
1364 case OMPD_cancel:
1365 case OMPD_target_data:
1366 case OMPD_target_enter_data:
1367 case OMPD_target_exit_data:
1368 case OMPD_taskloop:
1369 case OMPD_taskloop_simd:
1370 case OMPD_master_taskloop:
1371 case OMPD_master_taskloop_simd:
1372 case OMPD_parallel_master_taskloop:
1373 case OMPD_parallel_master_taskloop_simd:
1374 case OMPD_distribute:
1375 case OMPD_target_update:
1376 case OMPD_distribute_parallel_for_simd:
1377 case OMPD_distribute_simd:
1378 case OMPD_target_parallel_for_simd:
1379 case OMPD_target_simd:
1380 case OMPD_teams_distribute:
1381 case OMPD_teams_distribute_simd:
1382 case OMPD_teams_distribute_parallel_for_simd:
1383 case OMPD_target_teams:
1384 case OMPD_target_teams_distribute:
1385 case OMPD_target_teams_distribute_parallel_for_simd:
1386 case OMPD_target_teams_distribute_simd:
1387 case OMPD_declare_target:
1388 case OMPD_end_declare_target:
1389 case OMPD_threadprivate:
1390 case OMPD_allocate:
1391 case OMPD_declare_reduction:
1392 case OMPD_declare_mapper:
1393 case OMPD_declare_simd:
1394 case OMPD_requires:
1395 case OMPD_declare_variant:
1396 case OMPD_begin_declare_variant:
1397 case OMPD_end_declare_variant:
1398 case OMPD_unknown:
1399 default:
1400 llvm_unreachable("Enexpected directive with task reductions.");
1401 }
1402
1403 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(TaskRedRef)->getDecl());
1404 EmitVarDecl(*VD);
1405 EmitStoreOfScalar(ReductionDesc, GetAddrOfLocalVar(VD),
1406 /*Volatile=*/false, TaskRedRef->getType());
1407 }
1408}
1409
1411 const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1412 if (!HaveInsertPoint())
1413 return;
1418 bool HasAtLeastOneReduction = false;
1419 bool IsReductionWithTaskMod = false;
1420 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1421 // Do not emit for inscan reductions.
1422 if (C->getModifier() == OMPC_REDUCTION_inscan)
1423 continue;
1424 HasAtLeastOneReduction = true;
1425 Privates.append(C->privates().begin(), C->privates().end());
1426 LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1427 RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1428 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1429 IsReductionWithTaskMod =
1430 IsReductionWithTaskMod || C->getModifier() == OMPC_REDUCTION_task;
1431 }
1432 if (HasAtLeastOneReduction) {
1433 if (IsReductionWithTaskMod) {
1435 *this, D.getBeginLoc(),
1437 }
1438 bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1440 ReductionKind == OMPD_simd;
1441 bool SimpleReduction = ReductionKind == OMPD_simd;
1442 // Emit nowait reduction if nowait clause is present or directive is a
1443 // parallel directive (it always has implicit barrier).
1445 *this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1446 {WithNowait, SimpleReduction, ReductionKind});
1447 }
1448}
1449
1452 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1453 if (!CGF.HaveInsertPoint())
1454 return;
1455 llvm::BasicBlock *DoneBB = nullptr;
1456 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1457 if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
1458 if (!DoneBB) {
1459 if (llvm::Value *Cond = CondGen(CGF)) {
1460 // If the first post-update expression is found, emit conditional
1461 // block if it was requested.
1462 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1463 DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1464 CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1465 CGF.EmitBlock(ThenBB);
1466 }
1467 }
1468 CGF.EmitIgnoredExpr(PostUpdate);
1469 }
1470 }
1471 if (DoneBB)
1472 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1473}
1474
1475namespace {
1476/// Codegen lambda for appending distribute lower and upper bounds to outlined
1477/// parallel function. This is necessary for combined constructs such as
1478/// 'distribute parallel for'
1479typedef llvm::function_ref<void(CodeGenFunction &,
1480 const OMPExecutableDirective &,
1482 CodeGenBoundParametersTy;
1483} // anonymous namespace
1484
1485static void
1487 const OMPExecutableDirective &S) {
1488 if (CGF.getLangOpts().OpenMP < 50)
1489 return;
1491 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
1492 for (const Expr *Ref : C->varlists()) {
1493 if (!Ref->getType()->isScalarType())
1494 continue;
1495 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1496 if (!DRE)
1497 continue;
1498 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1500 }
1501 }
1502 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
1503 for (const Expr *Ref : C->varlists()) {
1504 if (!Ref->getType()->isScalarType())
1505 continue;
1506 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1507 if (!DRE)
1508 continue;
1509 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1511 }
1512 }
1513 for (const auto *C : S.getClausesOfKind<OMPLinearClause>()) {
1514 for (const Expr *Ref : C->varlists()) {
1515 if (!Ref->getType()->isScalarType())
1516 continue;
1517 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1518 if (!DRE)
1519 continue;
1520 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1522 }
1523 }
1524 // Privates should ne analyzed since they are not captured at all.
1525 // Task reductions may be skipped - tasks are ignored.
1526 // Firstprivates do not return value but may be passed by reference - no need
1527 // to check for updated lastprivate conditional.
1528 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
1529 for (const Expr *Ref : C->varlists()) {
1530 if (!Ref->getType()->isScalarType())
1531 continue;
1532 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1533 if (!DRE)
1534 continue;
1535 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1536 }
1537 }
1539 CGF, S, PrivateDecls);
1540}
1541
1544 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1545 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1546 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1547 llvm::Value *NumThreads = nullptr;
1548 llvm::Function *OutlinedFn =
1550 S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
1551 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
1552 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1553 NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1554 /*IgnoreResultAssign=*/true);
1556 CGF, NumThreads, NumThreadsClause->getBeginLoc());
1557 }
1558 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
1559 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1561 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1562 }
1563 const Expr *IfCond = nullptr;
1564 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1565 if (C->getNameModifier() == OMPD_unknown ||
1566 C->getNameModifier() == OMPD_parallel) {
1567 IfCond = C->getCondition();
1568 break;
1569 }
1570 }
1571
1572 OMPParallelScope Scope(CGF, S);
1574 // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
1575 // lower and upper bounds with the pragma 'for' chunking mechanism.
1576 // The following lambda takes care of appending the lower and upper bound
1577 // parameters when necessary
1578 CodeGenBoundParameters(CGF, S, CapturedVars);
1579 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
1580 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn,
1581 CapturedVars, IfCond, NumThreads);
1582}
1583
1584static bool isAllocatableDecl(const VarDecl *VD) {
1585 const VarDecl *CVD = VD->getCanonicalDecl();
1586 if (!CVD->hasAttr<OMPAllocateDeclAttr>())
1587 return false;
1588 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
1589 // Use the default allocation.
1590 return !((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||
1591 AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&
1592 !AA->getAllocator());
1593}
1594
1596 const OMPExecutableDirective &,
1598
1600 const OMPExecutableDirective &S) {
1601 bool Copyins = CGF.EmitOMPCopyinClause(S);
1602 if (Copyins) {
1603 // Emit implicit barrier to synchronize threads and avoid data races on
1604 // propagation master's thread values of threadprivate variables to local
1605 // instances of that variables of all other implicit threads.
1607 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
1608 /*ForceSimpleCall=*/true);
1609 }
1610}
1611
1613 CodeGenFunction &CGF, const VarDecl *VD) {
1614 CodeGenModule &CGM = CGF.CGM;
1615 auto &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1616
1617 if (!VD)
1618 return Address::invalid();
1619 const VarDecl *CVD = VD->getCanonicalDecl();
1620 if (!isAllocatableDecl(CVD))
1621 return Address::invalid();
1622 llvm::Value *Size;
1623 CharUnits Align = CGM.getContext().getDeclAlign(CVD);
1624 if (CVD->getType()->isVariablyModifiedType()) {
1625 Size = CGF.getTypeSize(CVD->getType());
1626 // Align the size: ((size + align - 1) / align) * align
1627 Size = CGF.Builder.CreateNUWAdd(
1628 Size, CGM.getSize(Align - CharUnits::fromQuantity(1)));
1629 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align));
1630 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align));
1631 } else {
1633 Size = CGM.getSize(Sz.alignTo(Align));
1634 }
1635
1636 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
1637 assert(AA->getAllocator() &&
1638 "Expected allocator expression for non-default allocator.");
1639 llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator());
1640 // According to the standard, the original allocator type is a enum (integer).
1641 // Convert to pointer type, if required.
1642 if (Allocator->getType()->isIntegerTy())
1643 Allocator = CGF.Builder.CreateIntToPtr(Allocator, CGM.VoidPtrTy);
1644 else if (Allocator->getType()->isPointerTy())
1645 Allocator = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Allocator,
1646 CGM.VoidPtrTy);
1647
1648 llvm::Value *Addr = OMPBuilder.createOMPAlloc(
1649 CGF.Builder, Size, Allocator,
1650 getNameWithSeparators({CVD->getName(), ".void.addr"}, ".", "."));
1651 llvm::CallInst *FreeCI =
1652 OMPBuilder.createOMPFree(CGF.Builder, Addr, Allocator);
1653
1654 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(NormalAndEHCleanup, FreeCI);
1656 Addr,
1658 getNameWithSeparators({CVD->getName(), ".addr"}, ".", "."));
1659 return Address(Addr, CGF.ConvertTypeForMem(CVD->getType()), Align);
1660}
1661
1663 CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr,
1664 SourceLocation Loc) {
1665 CodeGenModule &CGM = CGF.CGM;
1666 if (CGM.getLangOpts().OpenMPUseTLS &&
1668 return VDAddr;
1669
1670 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1671
1672 llvm::Type *VarTy = VDAddr.getElementType();
1673 llvm::Value *Data =
1674 CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy);
1675 llvm::ConstantInt *Size = CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy));
1676 std::string Suffix = getNameWithSeparators({"cache", ""});
1677 llvm::Twine CacheName = Twine(CGM.getMangledName(VD)).concat(Suffix);
1678
1679 llvm::CallInst *ThreadPrivateCacheCall =
1680 OMPBuilder.createCachedThreadPrivate(CGF.Builder, Data, Size, CacheName);
1681
1682 return Address(ThreadPrivateCacheCall, CGM.Int8Ty, VDAddr.getAlignment());
1683}
1684
1686 ArrayRef<StringRef> Parts, StringRef FirstSeparator, StringRef Separator) {
1687 SmallString<128> Buffer;
1688 llvm::raw_svector_ostream OS(Buffer);
1689 StringRef Sep = FirstSeparator;
1690 for (StringRef Part : Parts) {
1691 OS << Sep << Part;
1692 Sep = Separator;
1693 }
1694 return OS.str().str();
1695}
1696
1698 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
1699 InsertPointTy CodeGenIP, Twine RegionName) {
1701 Builder.restoreIP(CodeGenIP);
1702 llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder, /*CreateBranch=*/false,
1703 "." + RegionName + ".after");
1704
1705 {
1706 OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);
1707 CGF.EmitStmt(RegionBodyStmt);
1708 }
1709
1710 if (Builder.saveIP().isSet())
1711 Builder.CreateBr(FiniBB);
1712}
1713
1715 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
1716 InsertPointTy CodeGenIP, Twine RegionName) {
1718 Builder.restoreIP(CodeGenIP);
1719 llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder, /*CreateBranch=*/false,
1720 "." + RegionName + ".after");
1721
1722 {
1723 OMPBuilderCBHelpers::OutlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);
1724 CGF.EmitStmt(RegionBodyStmt);
1725 }
1726
1727 if (Builder.saveIP().isSet())
1728 Builder.CreateBr(FiniBB);
1729}
1730
1732 if (CGM.getLangOpts().OpenMPIRBuilder) {
1733 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1734 // Check if we have any if clause associated with the directive.
1735 llvm::Value *IfCond = nullptr;
1736 if (const auto *C = S.getSingleClause<OMPIfClause>())
1737 IfCond = EmitScalarExpr(C->getCondition(),
1738 /*IgnoreResultAssign=*/true);
1739
1740 llvm::Value *NumThreads = nullptr;
1741 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>())
1742 NumThreads = EmitScalarExpr(NumThreadsClause->getNumThreads(),
1743 /*IgnoreResultAssign=*/true);
1744
1745 ProcBindKind ProcBind = OMP_PROC_BIND_default;
1746 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>())
1747 ProcBind = ProcBindClause->getProcBindKind();
1748
1749 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1750
1751 // The cleanup callback that finalizes all variabels at the given location,
1752 // thus calls destructors etc.
1753 auto FiniCB = [this](InsertPointTy IP) {
1755 };
1756
1757 // Privatization callback that performs appropriate action for
1758 // shared/private/firstprivate/lastprivate/copyin/... variables.
1759 //
1760 // TODO: This defaults to shared right now.
1761 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
1762 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
1763 // The next line is appropriate only for variables (Val) with the
1764 // data-sharing attribute "shared".
1765 ReplVal = &Val;
1766
1767 return CodeGenIP;
1768 };
1769
1770 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1771 const Stmt *ParallelRegionBodyStmt = CS->getCapturedStmt();
1772
1773 auto BodyGenCB = [&, this](InsertPointTy AllocaIP,
1774 InsertPointTy CodeGenIP) {
1776 *this, ParallelRegionBodyStmt, AllocaIP, CodeGenIP, "parallel");
1777 };
1778
1779 CGCapturedStmtInfo CGSI(*CS, CR_OpenMP);
1780 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
1781 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
1782 AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
1783 Builder.restoreIP(
1784 OMPBuilder.createParallel(Builder, AllocaIP, BodyGenCB, PrivCB, FiniCB,
1785 IfCond, NumThreads, ProcBind, S.hasCancel()));
1786 return;
1787 }
1788
1789 // Emit parallel region as a standalone region.
1790 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1791 Action.Enter(CGF);
1792 OMPPrivateScope PrivateScope(CGF);
1793 emitOMPCopyinClause(CGF, S);
1794 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1795 CGF.EmitOMPPrivateClause(S, PrivateScope);
1796 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1797 (void)PrivateScope.Privatize();
1798 CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1799 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
1800 };
1801 {
1802 auto LPCRegion =
1804 emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
1807 [](CodeGenFunction &) { return nullptr; });
1808 }
1809 // Check for outer lastprivate conditional update.
1811}
1812
1814 EmitStmt(S.getIfStmt());
1815}
1816
1817namespace {
1818/// RAII to handle scopes for loop transformation directives.
1819class OMPTransformDirectiveScopeRAII {
1820 OMPLoopScope *Scope = nullptr;
1821 CodeGenFunction::CGCapturedStmtInfo *CGSI = nullptr;
1822 CodeGenFunction::CGCapturedStmtRAII *CapInfoRAII = nullptr;
1823
1824public:
1825 OMPTransformDirectiveScopeRAII(CodeGenFunction &CGF, const Stmt *S) {
1826 if (const auto *Dir = dyn_cast<OMPLoopBasedDirective>(S)) {
1827 Scope = new OMPLoopScope(CGF, *Dir);
1828 CGSI = new CodeGenFunction::CGCapturedStmtInfo(CR_OpenMP);
1829 CapInfoRAII = new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI);
1830 }
1831 }
1832 ~OMPTransformDirectiveScopeRAII() {
1833 if (!Scope)
1834 return;
1835 delete CapInfoRAII;
1836 delete CGSI;
1837 delete Scope;
1838 }
1839};
1840} // namespace
1841
1842static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
1843 int MaxLevel, int Level = 0) {
1844 assert(Level < MaxLevel && "Too deep lookup during loop body codegen.");
1845 const Stmt *SimplifiedS = S->IgnoreContainers();
1846 if (const auto *CS = dyn_cast<CompoundStmt>(SimplifiedS)) {
1847 PrettyStackTraceLoc CrashInfo(
1848 CGF.getContext().getSourceManager(), CS->getLBracLoc(),
1849 "LLVM IR generation of compound statement ('{}')");
1850
1851 // Keep track of the current cleanup stack depth, including debug scopes.
1852 CodeGenFunction::LexicalScope Scope(CGF, S->getSourceRange());
1853 for (const Stmt *CurStmt : CS->body())
1854 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level);
1855 return;
1856 }
1857 if (SimplifiedS == NextLoop) {
1858 if (auto *Dir = dyn_cast<OMPLoopTransformationDirective>(SimplifiedS))
1859 SimplifiedS = Dir->getTransformedStmt();
1860 if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(SimplifiedS))
1861 SimplifiedS = CanonLoop->getLoopStmt();
1862 if (const auto *For = dyn_cast<ForStmt>(SimplifiedS)) {
1863 S = For->getBody();
1864 } else {
1865 assert(isa<CXXForRangeStmt>(SimplifiedS) &&
1866 "Expected canonical for loop or range-based for loop.");
1867 const auto *CXXFor = cast<CXXForRangeStmt>(SimplifiedS);
1868 CGF.EmitStmt(CXXFor->getLoopVarStmt());
1869 S = CXXFor->getBody();
1870 }
1871 if (Level + 1 < MaxLevel) {
1873 S, /*TryImperfectlyNestedLoops=*/true);
1874 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1);
1875 return;
1876 }
1877 }
1878 CGF.EmitStmt(S);
1879}
1880
1882 JumpDest LoopExit) {
1883 RunCleanupsScope BodyScope(*this);
1884 // Update counters values on current iteration.
1885 for (const Expr *UE : D.updates())
1886 EmitIgnoredExpr(UE);
1887 // Update the linear variables.
1888 // In distribute directives only loop counters may be marked as linear, no
1889 // need to generate the code for them.
1891 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1892 for (const Expr *UE : C->updates())
1893 EmitIgnoredExpr(UE);
1894 }
1895 }
1896
1897 // On a continue in the body, jump to the end.
1898 JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue");
1899 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1900 for (const Expr *E : D.finals_conditions()) {
1901 if (!E)
1902 continue;
1903 // Check that loop counter in non-rectangular nest fits into the iteration
1904 // space.
1905 llvm::BasicBlock *NextBB = createBasicBlock("omp.body.next");
1906 EmitBranchOnBoolExpr(E, NextBB, Continue.getBlock(),
1908 EmitBlock(NextBB);
1909 }
1910
1911 OMPPrivateScope InscanScope(*this);
1912 EmitOMPReductionClauseInit(D, InscanScope, /*ForInscan=*/true);
1913 bool IsInscanRegion = InscanScope.Privatize();
1914 if (IsInscanRegion) {
1915 // Need to remember the block before and after scan directive
1916 // to dispatch them correctly depending on the clause used in
1917 // this directive, inclusive or exclusive. For inclusive scan the natural
1918 // order of the blocks is used, for exclusive clause the blocks must be
1919 // executed in reverse order.
1920 OMPBeforeScanBlock = createBasicBlock("omp.before.scan.bb");
1921 OMPAfterScanBlock = createBasicBlock("omp.after.scan.bb");
1922 // No need to allocate inscan exit block, in simd mode it is selected in the
1923 // codegen for the scan directive.
1924 if (D.getDirectiveKind() != OMPD_simd && !getLangOpts().OpenMPSimd)
1925 OMPScanExitBlock = createBasicBlock("omp.exit.inscan.bb");
1926 OMPScanDispatch = createBasicBlock("omp.inscan.dispatch");
1929 }
1930
1931 // Emit loop variables for C++ range loops.
1932 const Stmt *Body =
1934 // Emit loop body.
1935 emitBody(*this, Body,
1937 Body, /*TryImperfectlyNestedLoops=*/true),
1938 D.getLoopsNumber());
1939
1940 // Jump to the dispatcher at the end of the loop body.
1941 if (IsInscanRegion)
1943
1944 // The end (updates/cleanups).
1945 EmitBlock(Continue.getBlock());
1946 BreakContinueStack.pop_back();
1947}
1948
1949using EmittedClosureTy = std::pair<llvm::Function *, llvm::Value *>;
1950
1951/// Emit a captured statement and return the function as well as its captured
1952/// closure context.
1954 const CapturedStmt *S) {
1955 LValue CapStruct = ParentCGF.InitCapturedStruct(*S);
1956 CodeGenFunction CGF(ParentCGF.CGM, /*suppressNewContext=*/true);
1957 std::unique_ptr<CodeGenFunction::CGCapturedStmtInfo> CSI =
1958 std::make_unique<CodeGenFunction::CGCapturedStmtInfo>(*S);
1959 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, CSI.get());
1960 llvm::Function *F = CGF.GenerateCapturedStmtFunction(*S);
1961
1962 return {F, CapStruct.getPointer(ParentCGF)};
1963}
1964
1965/// Emit a call to a previously captured closure.
1966static llvm::CallInst *
1969 // Append the closure context to the argument.
1970 SmallVector<llvm::Value *> EffectiveArgs;
1971 EffectiveArgs.reserve(Args.size() + 1);
1972 llvm::append_range(EffectiveArgs, Args);
1973 EffectiveArgs.push_back(Cap.second);
1974
1975 return ParentCGF.Builder.CreateCall(Cap.first, EffectiveArgs);
1976}
1977
1978llvm::CanonicalLoopInfo *
1980 assert(Depth == 1 && "Nested loops with OpenMPIRBuilder not yet implemented");
1981
1982 // The caller is processing the loop-associated directive processing the \p
1983 // Depth loops nested in \p S. Put the previous pending loop-associated
1984 // directive to the stack. If the current loop-associated directive is a loop
1985 // transformation directive, it will push its generated loops onto the stack
1986 // such that together with the loops left here they form the combined loop
1987 // nest for the parent loop-associated directive.
1988 int ParentExpectedOMPLoopDepth = ExpectedOMPLoopDepth;
1989 ExpectedOMPLoopDepth = Depth;
1990
1991 EmitStmt(S);
1992 assert(OMPLoopNestStack.size() >= (size_t)Depth && "Found too few loops");
1993
1994 // The last added loop is the outermost one.
1995 llvm::CanonicalLoopInfo *Result = OMPLoopNestStack.back();
1996
1997 // Pop the \p Depth loops requested by the call from that stack and restore
1998 // the previous context.
1999 OMPLoopNestStack.pop_back_n(Depth);
2000 ExpectedOMPLoopDepth = ParentExpectedOMPLoopDepth;
2001
2002 return Result;
2003}
2004
2006 const Stmt *SyntacticalLoop = S->getLoopStmt();
2007 if (!getLangOpts().OpenMPIRBuilder) {
2008 // Ignore if OpenMPIRBuilder is not enabled.
2009 EmitStmt(SyntacticalLoop);
2010 return;
2011 }
2012
2013 LexicalScope ForScope(*this, S->getSourceRange());
2014
2015 // Emit init statements. The Distance/LoopVar funcs may reference variable
2016 // declarations they contain.
2017 const Stmt *BodyStmt;
2018 if (const auto *For = dyn_cast<ForStmt>(SyntacticalLoop)) {
2019 if (const Stmt *InitStmt = For->getInit())
2020 EmitStmt(InitStmt);
2021 BodyStmt = For->getBody();
2022 } else if (const auto *RangeFor =
2023 dyn_cast<CXXForRangeStmt>(SyntacticalLoop)) {
2024 if (const DeclStmt *RangeStmt = RangeFor->getRangeStmt())
2025 EmitStmt(RangeStmt);
2026 if (const DeclStmt *BeginStmt = RangeFor->getBeginStmt())
2027 EmitStmt(BeginStmt);
2028 if (const DeclStmt *EndStmt = RangeFor->getEndStmt())
2029 EmitStmt(EndStmt);
2030 if (const DeclStmt *LoopVarStmt = RangeFor->getLoopVarStmt())
2031 EmitStmt(LoopVarStmt);
2032 BodyStmt = RangeFor->getBody();
2033 } else
2034 llvm_unreachable("Expected for-stmt or range-based for-stmt");
2035
2036 // Emit closure for later use. By-value captures will be captured here.
2037 const CapturedStmt *DistanceFunc = S->getDistanceFunc();
2038 EmittedClosureTy DistanceClosure = emitCapturedStmtFunc(*this, DistanceFunc);
2039 const CapturedStmt *LoopVarFunc = S->getLoopVarFunc();
2040 EmittedClosureTy LoopVarClosure = emitCapturedStmtFunc(*this, LoopVarFunc);
2041
2042 // Call the distance function to get the number of iterations of the loop to
2043 // come.
2044 QualType LogicalTy = DistanceFunc->getCapturedDecl()
2045 ->getParam(0)
2046 ->getType()
2048 Address CountAddr = CreateMemTemp(LogicalTy, ".count.addr");
2049 emitCapturedStmtCall(*this, DistanceClosure, {CountAddr.getPointer()});
2050 llvm::Value *DistVal = Builder.CreateLoad(CountAddr, ".count");
2051
2052 // Emit the loop structure.
2053 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2054 auto BodyGen = [&, this](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP,
2055 llvm::Value *IndVar) {
2056 Builder.restoreIP(CodeGenIP);
2057
2058 // Emit the loop body: Convert the logical iteration number to the loop
2059 // variable and emit the body.
2060 const DeclRefExpr *LoopVarRef = S->getLoopVarRef();
2061 LValue LCVal = EmitLValue(LoopVarRef);
2062 Address LoopVarAddress = LCVal.getAddress(*this);
2063 emitCapturedStmtCall(*this, LoopVarClosure,
2064 {LoopVarAddress.getPointer(), IndVar});
2065
2066 RunCleanupsScope BodyScope(*this);
2067 EmitStmt(BodyStmt);
2068 };
2069 llvm::CanonicalLoopInfo *CL =
2070 OMPBuilder.createCanonicalLoop(Builder, BodyGen, DistVal);
2071
2072 // Finish up the loop.
2073 Builder.restoreIP(CL->getAfterIP());
2074 ForScope.ForceCleanup();
2075
2076 // Remember the CanonicalLoopInfo for parent AST nodes consuming it.
2077 OMPLoopNestStack.push_back(CL);
2078}
2079
2081 const OMPExecutableDirective &S, bool RequiresCleanup, const Expr *LoopCond,
2082 const Expr *IncExpr,
2083 const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
2084 const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
2085 auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
2086
2087 // Start the loop with a block that tests the condition.
2088 auto CondBlock = createBasicBlock("omp.inner.for.cond");
2089 EmitBlock(CondBlock);
2090 const SourceRange R = S.getSourceRange();
2091
2092 // If attributes are attached, push to the basic block with them.
2093 const auto &OMPED = cast<OMPExecutableDirective>(S);
2094 const CapturedStmt *ICS = OMPED.getInnermostCapturedStmt();
2095 const Stmt *SS = ICS->getCapturedStmt();
2096 const AttributedStmt *AS = dyn_cast_or_null<AttributedStmt>(SS);
2097 OMPLoopNestStack.clear();
2098 if (AS)
2102 else
2105
2106 // If there are any cleanups between here and the loop-exit scope,
2107 // create a block to stage a loop exit along.
2108 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
2109 if (RequiresCleanup)
2110 ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
2111
2112 llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body");
2113
2114 // Emit condition.
2115 EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
2116 if (ExitBlock != LoopExit.getBlock()) {
2117 EmitBlock(ExitBlock);
2119 }
2120
2121 EmitBlock(LoopBody);
2123
2124 // Create a block for the increment.
2125 JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
2126 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
2127
2128 BodyGen(*this);
2129
2130 // Emit "IV = IV + 1" and a back-edge to the condition block.
2131 EmitBlock(Continue.getBlock());
2132 EmitIgnoredExpr(IncExpr);
2133 PostIncGen(*this);
2134 BreakContinueStack.pop_back();
2135 EmitBranch(CondBlock);
2136 LoopStack.pop();
2137 // Emit the fall-through block.
2138 EmitBlock(LoopExit.getBlock());
2139}
2140
2142 if (!HaveInsertPoint())
2143 return false;
2144 // Emit inits for the linear variables.
2145 bool HasLinears = false;
2146 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2147 for (const Expr *Init : C->inits()) {
2148 HasLinears = true;
2149 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
2150 if (const auto *Ref =
2151 dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
2152 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
2153 const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
2154 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
2155 CapturedStmtInfo->lookup(OrigVD) != nullptr,
2156 VD->getInit()->getType(), VK_LValue,
2157 VD->getInit()->getExprLoc());
2159 &DRE, VD,
2160 MakeAddrLValue(Emission.getAllocatedAddress(), VD->getType()),
2161 /*capturedByInit=*/false);
2162 EmitAutoVarCleanups(Emission);
2163 } else {
2164 EmitVarDecl(*VD);
2165 }
2166 }
2167 // Emit the linear steps for the linear clauses.
2168 // If a step is not constant, it is pre-calculated before the loop.
2169 if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
2170 if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
2171 EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
2172 // Emit calculation of the linear step.
2173 EmitIgnoredExpr(CS);
2174 }
2175 }
2176 return HasLinears;
2177}
2178
2180 const OMPLoopDirective &D,
2181 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
2182 if (!HaveInsertPoint())
2183 return;
2184 llvm::BasicBlock *DoneBB = nullptr;
2185 // Emit the final values of the linear variables.
2186 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2187 auto IC = C->varlist_begin();
2188 for (const Expr *F : C->finals()) {
2189 if (!DoneBB) {
2190 if (llvm::Value *Cond = CondGen(*this)) {
2191 // If the first post-update expression is found, emit conditional
2192 // block if it was requested.
2193 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu");
2194 DoneBB = createBasicBlock(".omp.linear.pu.done");
2195 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2196 EmitBlock(ThenBB);
2197 }
2198 }
2199 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
2200 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
2201 CapturedStmtInfo->lookup(OrigVD) != nullptr,
2202 (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
2203 Address OrigAddr = EmitLValue(&DRE).getAddress(*this);
2204 CodeGenFunction::OMPPrivateScope VarScope(*this);
2205 VarScope.addPrivate(OrigVD, OrigAddr);
2206 (void)VarScope.Privatize();
2207 EmitIgnoredExpr(F);
2208 ++IC;
2209 }
2210 if (const Expr *PostUpdate = C->getPostUpdateExpr())
2211 EmitIgnoredExpr(PostUpdate);
2212 }
2213 if (DoneBB)
2214 EmitBlock(DoneBB, /*IsFinished=*/true);
2215}
2216
2218 const OMPExecutableDirective &D) {
2219 if (!CGF.HaveInsertPoint())
2220 return;
2221 for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
2222 llvm::APInt ClauseAlignment(64, 0);
2223 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
2224 auto *AlignmentCI =
2225 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
2226 ClauseAlignment = AlignmentCI->getValue();
2227 }
2228 for (const Expr *E : Clause->varlists()) {
2229 llvm::APInt Alignment(ClauseAlignment);
2230 if (Alignment == 0) {
2231 // OpenMP [2.8.1, Description]
2232 // If no optional parameter is specified, implementation-defined default
2233 // alignments for SIMD instructions on the target platforms are assumed.
2234 Alignment =
2235 CGF.getContext()
2237 E->getType()->getPointeeType()))
2238 .getQuantity();
2239 }
2240 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2241 "alignment is not power of 2");
2242 if (Alignment != 0) {
2243 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
2245 PtrValue, E, /*No second loc needed*/ SourceLocation(),
2246 llvm::ConstantInt::get(CGF.getLLVMContext(), Alignment));
2247 }
2248 }
2249 }
2250}
2251
2253 const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
2254 if (!HaveInsertPoint())
2255 return;
2256 auto I = S.private_counters().begin();
2257 for (const Expr *E : S.counters()) {
2258 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2259 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
2260 // Emit var without initialization.
2261 AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD);
2262 EmitAutoVarCleanups(VarEmission);
2263 LocalDeclMap.erase(PrivateVD);
2264 (void)LoopScope.addPrivate(VD, VarEmission.getAllocatedAddress());
2265 if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
2266 VD->hasGlobalStorage()) {
2267 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD),
2268 LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
2269 E->getType(), VK_LValue, E->getExprLoc());
2270 (void)LoopScope.addPrivate(PrivateVD, EmitLValue(&DRE).getAddress(*this));
2271 } else {
2272 (void)LoopScope.addPrivate(PrivateVD, VarEmission.getAllocatedAddress());
2273 }
2274 ++I;
2275 }
2276 // Privatize extra loop counters used in loops for ordered(n) clauses.
2277 for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) {
2278 if (!C->getNumForLoops())
2279 continue;
2280 for (unsigned I = S.getLoopsNumber(), E = C->getLoopNumIterations().size();
2281 I < E; ++I) {
2282 const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I));
2283 const auto *VD = cast<VarDecl>(DRE->getDecl());
2284 // Override only those variables that can be captured to avoid re-emission
2285 // of the variables declared within the loops.
2286 if (DRE->refersToEnclosingVariableOrCapture()) {
2287 (void)LoopScope.addPrivate(
2288 VD, CreateMemTemp(DRE->getType(), VD->getName()));
2289 }
2290 }
2291 }
2292}
2293
2295 const Expr *Cond, llvm::BasicBlock *TrueBlock,
2296 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
2297 if (!CGF.HaveInsertPoint())
2298 return;
2299 {
2300 CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
2301 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
2302 (void)PreCondScope.Privatize();
2303 // Get initial values of real counters.
2304 for (const Expr *I : S.inits()) {
2305 CGF.EmitIgnoredExpr(I);
2306 }
2307 }
2308 // Create temp loop control variables with their init values to support
2309 // non-rectangular loops.
2310 CodeGenFunction::OMPMapVars PreCondVars;
2311 for (const Expr *E : S.dependent_counters()) {
2312 if (!E)
2313 continue;
2314 assert(!E->getType().getNonReferenceType()->isRecordType() &&
2315 "dependent counter must not be an iterator.");
2316 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2317 Address CounterAddr =
2319 (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr);
2320 }
2321 (void)PreCondVars.apply(CGF);
2322 for (const Expr *E : S.dependent_inits()) {
2323 if (!E)
2324 continue;
2325 CGF.EmitIgnoredExpr(E);
2326 }
2327 // Check that loop is executed at least one time.
2328 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
2329 PreCondVars.restore(CGF);
2330}
2331
2333 const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
2334 if (!HaveInsertPoint())
2335 return;
2338 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
2339 for (const Expr *C : LoopDirective->counters()) {
2340 SIMDLCVs.insert(
2341 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
2342 }
2343 }
2344 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2345 auto CurPrivate = C->privates().begin();
2346 for (const Expr *E : C->varlists()) {
2347 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2348 const auto *PrivateVD =
2349 cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
2350 if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
2351 // Emit private VarDecl with copy init.
2352 EmitVarDecl(*PrivateVD);
2353 bool IsRegistered =
2354 PrivateScope.addPrivate(VD, GetAddrOfLocalVar(PrivateVD));
2355 assert(IsRegistered && "linear var already registered as private");
2356 // Silence the warning about unused variable.
2357 (void)IsRegistered;
2358 } else {
2359 EmitVarDecl(*PrivateVD);
2360 }
2361 ++CurPrivate;
2362 }
2363 }
2364}
2365
2367 const OMPExecutableDirective &D) {
2368 if (!CGF.HaveInsertPoint())
2369 return;
2370 if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
2371 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
2372 /*ignoreResult=*/true);
2373 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
2374 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
2375 // In presence of finite 'safelen', it may be unsafe to mark all
2376 // the memory instructions parallel, because loop-carried
2377 // dependences of 'safelen' iterations are possible.
2379 } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
2380 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
2381 /*ignoreResult=*/true);
2382 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
2383 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
2384 // In presence of finite 'safelen', it may be unsafe to mark all
2385 // the memory instructions parallel, because loop-carried
2386 // dependences of 'safelen' iterations are possible.
2387 CGF.LoopStack.setParallel(/*Enable=*/false);
2388 }
2389}
2390
2392 // Walk clauses and process safelen/lastprivate.
2393 LoopStack.setParallel(/*Enable=*/true);
2395 emitSimdlenSafelenClause(*this, D);
2396 if (const auto *C = D.getSingleClause<OMPOrderClause>())
2397 if (C->getKind() == OMPC_ORDER_concurrent)
2398 LoopStack.setParallel(/*Enable=*/true);
2399 if ((D.getDirectiveKind() == OMPD_simd ||
2400 (getLangOpts().OpenMPSimd &&
2402 llvm::any_of(D.getClausesOfKind<OMPReductionClause>(),
2403 [](const OMPReductionClause *C) {
2404 return C->getModifier() == OMPC_REDUCTION_inscan;
2405 }))
2406 // Disable parallel access in case of prefix sum.
2407 LoopStack.setParallel(/*Enable=*/false);
2408}
2409
2411 const OMPLoopDirective &D,
2412 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
2413 if (!HaveInsertPoint())
2414 return;
2415 llvm::BasicBlock *DoneBB = nullptr;
2416 auto IC = D.counters().begin();
2417 auto IPC = D.private_counters().begin();
2418 for (const Expr *F : D.finals()) {
2419 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
2420 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
2421 const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
2422 if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
2423 OrigVD->hasGlobalStorage() || CED) {
2424 if (!DoneBB) {
2425 if (llvm::Value *Cond = CondGen(*this)) {
2426 // If the first post-update expression is found, emit conditional
2427 // block if it was requested.
2428 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then");
2429 DoneBB = createBasicBlock(".omp.final.done");
2430 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2431 EmitBlock(ThenBB);
2432 }
2433 }
2434 Address OrigAddr = Address::invalid();
2435 if (CED) {
2436 OrigAddr =
2437 EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress(*this);
2438 } else {
2439 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(PrivateVD),
2440 /*RefersToEnclosingVariableOrCapture=*/false,
2441 (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
2442 OrigAddr = EmitLValue(&DRE).getAddress(*this);
2443 }
2444 OMPPrivateScope VarScope(*this);
2445 VarScope.addPrivate(OrigVD, OrigAddr);
2446 (void)VarScope.Privatize();
2447 EmitIgnoredExpr(F);
2448 }
2449 ++IC;
2450 ++IPC;
2451 }
2452 if (DoneBB)
2453 EmitBlock(DoneBB, /*IsFinished=*/true);
2454}
2455
2457 const OMPLoopDirective &S,
2458 CodeGenFunction::JumpDest LoopExit) {
2459 CGF.EmitOMPLoopBody(S, LoopExit);
2460 CGF.EmitStopPoint(&S);
2461}
2462
2463/// Emit a helper variable and return corresponding lvalue.
2465 const DeclRefExpr *Helper) {
2466 auto VDecl = cast<VarDecl>(Helper->getDecl());
2467 CGF.EmitVarDecl(*VDecl);
2468 return CGF.EmitLValue(Helper);
2469}
2470
2472 const RegionCodeGenTy &SimdInitGen,
2473 const RegionCodeGenTy &BodyCodeGen) {
2474 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](CodeGenFunction &CGF,
2475 PrePostActionTy &) {
2476 CGOpenMPRuntime::NontemporalDeclsRAII NontemporalsRegion(CGF.CGM, S);
2477 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
2478 SimdInitGen(CGF);
2479
2480 BodyCodeGen(CGF);
2481 };
2482 auto &&ElseGen = [&BodyCodeGen](CodeGenFunction &CGF, PrePostActionTy &) {
2483 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
2484 CGF.LoopStack.setVectorizeEnable(/*Enable=*/false);
2485
2486 BodyCodeGen(CGF);
2487 };
2488 const Expr *IfCond = nullptr;
2489 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
2490 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
2491 if (CGF.getLangOpts().OpenMP >= 50 &&
2492 (C->getNameModifier() == OMPD_unknown ||
2493 C->getNameModifier() == OMPD_simd)) {
2494 IfCond = C->getCondition();
2495 break;
2496 }
2497 }
2498 }
2499 if (IfCond) {
2500 CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, IfCond, ThenGen, ElseGen);
2501 } else {
2502 RegionCodeGenTy ThenRCG(ThenGen);
2503 ThenRCG(CGF);
2504 }
2505}
2506
2508 PrePostActionTy &Action) {
2509 Action.Enter(CGF);
2510 assert(isOpenMPSimdDirective(S.getDirectiveKind()) &&
2511 "Expected simd directive");
2512 OMPLoopScope PreInitScope(CGF, S);
2513 // if (PreCond) {
2514 // for (IV in 0..LastIteration) BODY;
2515 // <Final counter/linear vars updates>;
2516 // }
2517 //
2518 if (isOpenMPDistributeDirective(S.getDirectiveKind()) ||
2519 isOpenMPWorksharingDirective(S.getDirectiveKind()) ||
2520 isOpenMPTaskLoopDirective(S.getDirectiveKind())) {
2521 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));
2522 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));
2523 }
2524
2525 // Emit: if (PreCond) - begin.
2526 // If the condition constant folds and can be elided, avoid emitting the
2527 // whole loop.
2528 bool CondConstant;
2529 llvm::BasicBlock *ContBlock = nullptr;
2530 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2531 if (!CondConstant)
2532 return;
2533 } else {
2534 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");
2535 ContBlock = CGF.createBasicBlock("simd.if.end");
2536 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
2537 CGF.getProfileCount(&S));
2538 CGF.EmitBlock(ThenBlock);
2540 }
2541
2542 // Emit the loop iteration variable.
2543 const Expr *IVExpr = S.getIterationVariable();
2544 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
2545 CGF.EmitVarDecl(*IVDecl);
2546 CGF.EmitIgnoredExpr(S.getInit());
2547
2548 // Emit the iterations count variable.
2549 // If it is not a variable, Sema decided to calculate iterations count on
2550 // each iteration (e.g., it is foldable into a constant).
2551 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2552 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2553 // Emit calculation of the iterations count.
2554 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
2555 }
2556
2557 emitAlignedClause(CGF, S);
2558 (void)CGF.EmitOMPLinearClauseInit(S);
2559 {
2560 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2561 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
2562 CGF.EmitOMPLinearClause(S, LoopScope);
2563 CGF.EmitOMPPrivateClause(S, LoopScope);
2564 CGF.EmitOMPReductionClauseInit(S, LoopScope);
2566 CGF, S, CGF.EmitLValue(S.getIterationVariable()));
2567 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2568 (void)LoopScope.Privatize();
2569 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
2571
2573 CGF, S,
2574 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2575 CGF.EmitOMPSimdInit(S);
2576 },
2577 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
2578 CGF.EmitOMPInnerLoop(
2579 S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),
2580 [&S](CodeGenFunction &CGF) {
2581 emitOMPLoopBodyWithStopPoint(CGF, S,
2582 CodeGenFunction::JumpDest());
2583 },
2584 [](CodeGenFunction &) {});
2585 });
2586 CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; });
2587 // Emit final copy of the lastprivate variables at the end of loops.
2588 if (HasLastprivateClause)
2589 CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
2590 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
2592 [](CodeGenFunction &) { return nullptr; });
2593 LoopScope.restoreMap();
2594 CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; });
2595 }
2596 // Emit: if (PreCond) - end.
2597 if (ContBlock) {
2598 CGF.EmitBranch(ContBlock);
2599 CGF.EmitBlock(ContBlock, true);
2600 }
2601}
2602
2604 // Check for unsupported clauses
2605 for (OMPClause *C : S.clauses()) {
2606 // Currently only order, simdlen and safelen clauses are supported
2607 if (!(isa<OMPSimdlenClause>(C) || isa<OMPSafelenClause>(C) ||
2608 isa<OMPOrderClause>(C) || isa<OMPAlignedClause>(C)))
2609 return false;
2610 }
2611
2612 // Check if we have a statement with the ordered directive.
2613 // Visit the statement hierarchy to find a compound statement
2614 // with a ordered directive in it.
2615 if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(S.getRawStmt())) {
2616 if (const Stmt *SyntacticalLoop = CanonLoop->getLoopStmt()) {
2617 for (const Stmt *SubStmt : SyntacticalLoop->children()) {
2618 if (!SubStmt)
2619 continue;
2620 if (const CompoundStmt *CS = dyn_cast<CompoundStmt>(SubStmt)) {
2621 for (const Stmt *CSSubStmt : CS->children()) {
2622 if (!CSSubStmt)
2623 continue;
2624 if (isa<OMPOrderedDirective>(CSSubStmt)) {
2625 return false;
2626 }
2627 }
2628 }
2629 }
2630 }
2631 }
2632 return true;
2633}
2634static llvm::MapVector<llvm::Value *, llvm::Value *>
2636 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars;
2637 for (const auto *Clause : S.getClausesOfKind<OMPAlignedClause>()) {
2638 llvm::APInt ClauseAlignment(64, 0);
2639 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
2640 auto *AlignmentCI =
2641 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
2642 ClauseAlignment = AlignmentCI->getValue();
2643 }
2644 for (const Expr *E : Clause->varlists()) {
2645 llvm::APInt Alignment(ClauseAlignment);
2646 if (Alignment == 0) {
2647 // OpenMP [2.8.1, Description]
2648 // If no optional parameter is specified, implementation-defined default
2649 // alignments for SIMD instructions on the target platforms are assumed.
2650 Alignment =
2651 CGF.getContext()
2653 E->getType()->getPointeeType()))
2654 .getQuantity();
2655 }
2656 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2657 "alignment is not power of 2");
2658 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
2659 AlignedVars[PtrValue] = CGF.Builder.getInt64(Alignment.getSExtValue());
2660 }
2661 }
2662 return AlignedVars;
2663}
2664
2666 bool UseOMPIRBuilder =
2667 CGM.getLangOpts().OpenMPIRBuilder && isSupportedByOpenMPIRBuilder(S);
2668 if (UseOMPIRBuilder) {
2669 auto &&CodeGenIRBuilder = [this, &S, UseOMPIRBuilder](CodeGenFunction &CGF,
2670 PrePostActionTy &) {
2671 // Use the OpenMPIRBuilder if enabled.
2672 if (UseOMPIRBuilder) {
2673 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars =
2674 GetAlignedMapping(S, CGF);
2675 // Emit the associated statement and get its loop representation.
2676 const Stmt *Inner = S.getRawStmt();
2677 llvm::CanonicalLoopInfo *CLI =
2679
2680 llvm::OpenMPIRBuilder &OMPBuilder =
2682 // Add SIMD specific metadata
2683 llvm::ConstantInt *Simdlen = nullptr;
2684 if (const auto *C = S.getSingleClause<OMPSimdlenClause>()) {
2685 RValue Len =
2686 this->EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
2687 /*ignoreResult=*/true);
2688 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
2689 Simdlen = Val;
2690 }
2691 llvm::ConstantInt *Safelen = nullptr;
2692 if (const auto *C = S.getSingleClause<OMPSafelenClause>()) {
2693 RValue Len =
2694 this->EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
2695 /*ignoreResult=*/true);
2696 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
2697 Safelen = Val;
2698 }
2699 llvm::omp::OrderKind Order = llvm::omp::OrderKind::OMP_ORDER_unknown;
2700 if (const auto *C = S.getSingleClause<OMPOrderClause>()) {
2701 if (C->getKind() == OpenMPOrderClauseKind ::OMPC_ORDER_concurrent) {
2702 Order = llvm::omp::OrderKind::OMP_ORDER_concurrent;
2703 }
2704 }
2705 // Add simd metadata to the collapsed loop. Do not generate
2706 // another loop for if clause. Support for if clause is done earlier.
2707 OMPBuilder.applySimd(CLI, AlignedVars,
2708 /*IfCond*/ nullptr, Order, Simdlen, Safelen);
2709 return;
2710 }
2711 };
2712 {
2713 auto LPCRegion =
2715 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2716 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd,
2717 CodeGenIRBuilder);
2718 }
2719 return;
2720 }
2721
2722 ParentLoopDirectiveForScanRegion ScanRegion(*this, S);
2723 OMPFirstScanLoop = true;
2724 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2725 emitOMPSimdRegion(CGF, S, Action);
2726 };
2727 {
2728 auto LPCRegion =
2730 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2731 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2732 }
2733 // Check for outer lastprivate conditional update.
2735}
2736
2738 // Emit the de-sugared statement.
2739 OMPTransformDirectiveScopeRAII TileScope(*this, &S);
2740 EmitStmt(S.getTransformedStmt());
2741}
2742
2744 bool UseOMPIRBuilder = CGM.getLangOpts().OpenMPIRBuilder;
2745
2746 if (UseOMPIRBuilder) {
2747 auto DL = SourceLocToDebugLoc(S.getBeginLoc());
2748 const Stmt *Inner = S.getRawStmt();
2749
2750 // Consume nested loop. Clear the entire remaining loop stack because a
2751 // fully unrolled loop is non-transformable. For partial unrolling the
2752 // generated outer loop is pushed back to the stack.
2753 llvm::CanonicalLoopInfo *CLI = EmitOMPCollapsedCanonicalLoopNest(Inner, 1);
2754 OMPLoopNestStack.clear();
2755
2756 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2757
2758 bool NeedsUnrolledCLI = ExpectedOMPLoopDepth >= 1;
2759 llvm::CanonicalLoopInfo *UnrolledCLI = nullptr;
2760
2761 if (S.hasClausesOfKind<OMPFullClause>()) {
2762 assert(ExpectedOMPLoopDepth == 0);
2763 OMPBuilder.unrollLoopFull(DL, CLI);
2764 } else if (auto *PartialClause = S.getSingleClause<OMPPartialClause>()) {
2765 uint64_t Factor = 0;
2766 if (Expr *FactorExpr = PartialClause->getFactor()) {
2767 Factor = FactorExpr->EvaluateKnownConstInt(getContext()).getZExtValue();
2768 assert(Factor >= 1 && "Only positive factors are valid");
2769 }
2770 OMPBuilder.unrollLoopPartial(DL, CLI, Factor,
2771 NeedsUnrolledCLI ? &UnrolledCLI : nullptr);
2772 } else {
2773 OMPBuilder.unrollLoopHeuristic(DL, CLI);
2774 }
2775
2776 assert((!NeedsUnrolledCLI || UnrolledCLI) &&
2777 "NeedsUnrolledCLI implies UnrolledCLI to be set");
2778 if (UnrolledCLI)
2779 OMPLoopNestStack.push_back(UnrolledCLI);
2780
2781 return;
2782 }
2783
2784 // This function is only called if the unrolled loop is not consumed by any
2785 // other loop-associated construct. Such a loop-associated construct will have
2786 // used the transformed AST.
2787
2788 // Set the unroll metadata for the next emitted loop.
2790
2791 if (S.hasClausesOfKind<OMPFullClause>()) {
2793 } else if (auto *PartialClause = S.getSingleClause<OMPPartialClause>()) {
2794 if (Expr *FactorExpr = PartialClause->getFactor()) {
2795 uint64_t Factor =
2796 FactorExpr->EvaluateKnownConstInt(getContext()).getZExtValue();
2797 assert(Factor >= 1 && "Only positive factors are valid");
2798 LoopStack.setUnrollCount(Factor);
2799 }
2800 }
2801
2802 EmitStmt(S.getAssociatedStmt());
2803}
2804
2805void CodeGenFunction::EmitOMPOuterLoop(
2806 bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
2807 CodeGenFunction::OMPPrivateScope &LoopScope,
2808 const CodeGenFunction::OMPLoopArguments &LoopArgs,
2809 const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
2810 const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
2812
2813 const Expr *IVExpr = S.getIterationVariable();
2814 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2815 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2816
2817 JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
2818
2819 // Start the loop with a block that tests the condition.
2820 llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond");
2821 EmitBlock(CondBlock);
2822 const SourceRange R = S.getSourceRange();
2823 OMPLoopNestStack.clear();
2826
2827 llvm::Value *BoolCondVal = nullptr;
2828 if (!DynamicOrOrdered) {
2829 // UB = min(UB, GlobalUB) or
2830 // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
2831 // 'distribute parallel for')
2832 EmitIgnoredExpr(LoopArgs.EUB);
2833 // IV = LB
2834 EmitIgnoredExpr(LoopArgs.Init);
2835 // IV < UB
2836 BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
2837 } else {
2838 BoolCondVal =
2839 RT.emitForNext(*this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
2840 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
2841 }
2842
2843 // If there are any cleanups between here and the loop-exit scope,
2844 // create a block to stage a loop exit along.
2845 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
2846 if (LoopScope.requiresCleanups())
2847 ExitBlock = createBasicBlock("omp.dispatch.cleanup");
2848
2849 llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body");
2850 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
2851 if (ExitBlock != LoopExit.getBlock()) {
2852 EmitBlock(ExitBlock);
2854 }
2855 EmitBlock(LoopBody);
2856
2857 // Emit "IV = LB" (in case of static schedule, we have already calculated new
2858 // LB for loop condition and emitted it above).
2859 if (DynamicOrOrdered)
2860 EmitIgnoredExpr(LoopArgs.Init);
2861
2862 // Create a block for the increment.
2863 JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
2864 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
2865
2867 *this, S,
2868 [&S, IsMonotonic](CodeGenFunction &CGF, PrePostActionTy &) {
2869 // Generate !llvm.loop.parallel metadata for loads and stores for loops
2870 // with dynamic/guided scheduling and without ordered clause.
2871 if (!isOpenMPSimdDirective(S.getDirectiveKind())) {
2872 CGF.LoopStack.setParallel(!IsMonotonic);
2873 if (const auto *C = S.getSingleClause<OMPOrderClause>())
2874 if (C->getKind() == OMPC_ORDER_concurrent)
2875 CGF.LoopStack.setParallel(/*Enable=*/true);
2876 } else {
2877 CGF.EmitOMPSimdInit(S);
2878 }
2879 },
2880 [&S, &LoopArgs, LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered,
2881 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
2882 SourceLocation Loc = S.getBeginLoc();
2883 // when 'distribute' is not combined with a 'for':
2884 // while (idx <= UB) { BODY; ++idx; }
2885 // when 'distribute' is combined with a 'for'
2886 // (e.g. 'distribute parallel for')
2887 // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
2888 CGF.EmitOMPInnerLoop(
2889 S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
2890 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
2891 CodeGenLoop(CGF, S, LoopExit);
2892 },
2893 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
2894 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
2895 });
2896 });
2897
2898 EmitBlock(Continue.getBlock());
2899 BreakContinueStack.pop_back();
2900 if (!DynamicOrOrdered) {
2901 // Emit "LB = LB + Stride", "UB = UB + Stride".
2902 EmitIgnoredExpr(LoopArgs.NextLB);
2903 EmitIgnoredExpr(LoopArgs.NextUB);
2904 }
2905
2906 EmitBranch(CondBlock);
2907 OMPLoopNestStack.clear();
2908 LoopStack.pop();
2909 // Emit the fall-through block.
2910 EmitBlock(LoopExit.getBlock());
2911
2912 // Tell the runtime we are done.
2913 auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) {
2914 if (!DynamicOrOrdered)
2915 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2916 S.getDirectiveKind());
2917 };
2918 OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
2919}
2920
2921void CodeGenFunction::EmitOMPForOuterLoop(
2922 const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
2923 const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
2924 const OMPLoopArguments &LoopArgs,
2925 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
2927
2928 // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
2929 const bool DynamicOrOrdered = Ordered || RT.isDynamic(ScheduleKind.Schedule);
2930
2931 assert((Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule,
2932 LoopArgs.Chunk != nullptr)) &&
2933 "static non-chunked schedule does not need outer loop");
2934
2935 // Emit outer loop.
2936 //
2937 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2938 // When schedule(dynamic,chunk_size) is specified, the iterations are
2939 // distributed to threads in the team in chunks as the threads request them.
2940 // Each thread executes a chunk of iterations, then requests another chunk,
2941 // until no chunks remain to be distributed. Each chunk contains chunk_size
2942 // iterations, except for the last chunk to be distributed, which may have
2943 // fewer iterations. When no chunk_size is specified, it defaults to 1.
2944 //
2945 // When schedule(guided,chunk_size) is specified, the iterations are assigned
2946 // to threads in the team in chunks as the executing threads request them.
2947 // Each thread executes a chunk of iterations, then requests another chunk,
2948 // until no chunks remain to be assigned. For a chunk_size of 1, the size of
2949 // each chunk is proportional to the number of unassigned iterations divided
2950 // by the number of threads in the team, decreasing to 1. For a chunk_size
2951 // with value k (greater than 1), the size of each chunk is determined in the
2952 // same way, with the restriction that the chunks do not contain fewer than k
2953 // iterations (except for the last chunk to be assigned, which may have fewer
2954 // than k iterations).
2955 //
2956 // When schedule(auto) is specified, the decision regarding scheduling is
2957 // delegated to the compiler and/or runtime system. The programmer gives the
2958 // implementation the freedom to choose any possible mapping of iterations to
2959 // threads in the team.
2960 //
2961 // When schedule(runtime) is specified, the decision regarding scheduling is
2962 // deferred until run time, and the schedule and chunk size are taken from the
2963 // run-sched-var ICV. If the ICV is set to auto, the schedule is
2964 // implementation defined
2965 //
2966 // while(__kmpc_dispatch_next(&LB, &UB)) {
2967 // idx = LB;
2968 // while (idx <= UB) { BODY; ++idx;
2969 // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
2970 // } // inner loop
2971 // }
2972 //
2973 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2974 // When schedule(static, chunk_size) is specified, iterations are divided into
2975 // chunks of size chunk_size, and the chunks are assigned to the threads in
2976 // the team in a round-robin fashion in the order of the thread number.
2977 //
2978 // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
2979 // while (idx <= UB) { BODY; ++idx; } // inner loop
2980 // LB = LB + ST;
2981 // UB = UB + ST;
2982 // }
2983 //
2984
2985 const Expr *IVExpr = S.getIterationVariable();
2986 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2987 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2988
2989 if (DynamicOrOrdered) {
2990 const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
2991 CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
2992 llvm::Value *LBVal = DispatchBounds.first;
2993 llvm::Value *UBVal = DispatchBounds.second;
2994 CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
2995 LoopArgs.Chunk};
2996 RT.emitForDispatchInit(*this, S.getBeginLoc(), ScheduleKind, IVSize,
2997 IVSigned, Ordered, DipatchRTInputValues);
2998 } else {
3000 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
3001 LoopArgs.ST, LoopArgs.Chunk);
3002 RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
3003 ScheduleKind, StaticInit);
3004 }
3005
3006 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
3007 const unsigned IVSize,
3008 const bool IVSigned) {
3009 if (Ordered) {
3010 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
3011 IVSigned);
3012 }
3013 };
3014
3015 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
3016 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
3017 OuterLoopArgs.IncExpr = S.getInc();
3018 OuterLoopArgs.Init = S.getInit();
3019 OuterLoopArgs.Cond = S.getCond();
3020 OuterLoopArgs.NextLB = S.getNextLowerBound();
3021 OuterLoopArgs.NextUB = S.getNextUpperBound();
3022 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
3023 emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
3024}
3025
3027 const unsigned IVSize, const bool IVSigned) {}
3028
3029void CodeGenFunction::EmitOMPDistributeOuterLoop(
3030 OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
3031 OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
3032 const CodeGenLoopTy &CodeGenLoopContent) {
3033
3035
3036 // Emit outer loop.
3037 // Same behavior as a OMPForOuterLoop, except that schedule cannot be
3038 // dynamic
3039 //
3040
3041 const Expr *IVExpr = S.getIterationVariable();
3042 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3043 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3044
3046 IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
3047 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
3048 RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, StaticInit);
3049
3050 // for combined 'distribute' and 'for' the increment expression of distribute
3051 // is stored in DistInc. For 'distribute' alone, it is in Inc.
3052 Expr *IncExpr;
3053 if (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()))
3054 IncExpr = S.getDistInc();
3055 else
3056 IncExpr = S.getInc();
3057
3058 // this routine is shared by 'omp distribute parallel for' and
3059 // 'omp distribute': select the right EUB expression depending on the
3060 // directive
3061 OMPLoopArguments OuterLoopArgs;
3062 OuterLoopArgs.LB = LoopArgs.LB;
3063 OuterLoopArgs.UB = LoopArgs.UB;
3064 OuterLoopArgs.ST = LoopArgs.ST;
3065 OuterLoopArgs.IL = LoopArgs.IL;
3066 OuterLoopArgs.Chunk = LoopArgs.Chunk;
3067 OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3068 ? S.getCombinedEnsureUpperBound()
3069 : S.getEnsureUpperBound();
3070 OuterLoopArgs.IncExpr = IncExpr;
3071 OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3072 ? S.getCombinedInit()
3073 : S.getInit();
3074 OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3075 ? S.getCombinedCond()
3076 : S.getCond();
3077 OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3078 ? S.getCombinedNextLowerBound()
3079 : S.getNextLowerBound();
3080 OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3081 ? S.getCombinedNextUpperBound()
3082 : S.getNextUpperBound();
3083
3084 EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
3085 LoopScope, OuterLoopArgs, CodeGenLoopContent,
3087}
3088
3089static std::pair<LValue, LValue>
3091 const OMPExecutableDirective &S) {
3092 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
3093 LValue LB =
3094 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
3095 LValue UB =
3096 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
3097
3098 // When composing 'distribute' with 'for' (e.g. as in 'distribute
3099 // parallel for') we need to use the 'distribute'
3100 // chunk lower and upper bounds rather than the whole loop iteration
3101 // space. These are parameters to the outlined function for 'parallel'
3102 // and we copy the bounds of the previous schedule into the
3103 // the current ones.
3104 LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
3105 LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
3106 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
3107 PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());
3108 PrevLBVal = CGF.EmitScalarConversion(
3109 PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
3112 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
3113 PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());
3114 PrevUBVal = CGF.EmitScalarConversion(
3115 PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
3118
3119 CGF.EmitStoreOfScalar(PrevLBVal, LB);
3120 CGF.EmitStoreOfScalar(PrevUBVal, UB);
3121
3122 return {LB, UB};
3123}
3124
3125/// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
3126/// we need to use the LB and UB expressions generated by the worksharing
3127/// code generation support, whereas in non combined situations we would
3128/// just emit 0 and the LastIteration expression
3129/// This function is necessary due to the difference of the LB and UB
3130/// types for the RT emission routines for 'for_static_init' and
3131/// 'for_dispatch_init'
3132static std::pair<llvm::Value *, llvm::Value *>
3134 const OMPExecutableDirective &S,
3135 Address LB, Address UB) {
3136 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
3137 const Expr *IVExpr = LS.getIterationVariable();
3138 // when implementing a dynamic schedule for a 'for' combined with a
3139 // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
3140 // is not normalized as each team only executes its own assigned
3141 // distribute chunk
3142 QualType IteratorTy = IVExpr->getType();
3143 llvm::Value *LBVal =
3144 CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
3145 llvm::Value *UBVal =
3146 CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
3147 return {LBVal, UBVal};
3148}
3149
3153 const auto &Dir = cast<OMPLoopDirective>(S);
3154 LValue LB =
3155 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
3156 llvm::Value *LBCast =
3157 CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(LB.getAddress(CGF)),
3158 CGF.SizeTy, /*isSigned=*/false);
3159 CapturedVars.push_back(LBCast);
3160 LValue UB =
3161 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
3162
3163 llvm::Value *UBCast =
3164 CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(UB.getAddress(CGF)),
3165 CGF.SizeTy, /*isSigned=*/false);
3166 CapturedVars.push_back(UBCast);
3167}
3168
3169static void
3171 const OMPLoopDirective &S,
3172 CodeGenFunction::JumpDest LoopExit) {
3173 auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
3174 PrePostActionTy &Action) {
3175 Action.Enter(CGF);
3176 bool HasCancel = false;
3177 if (!isOpenMPSimdDirective(S.getDirectiveKind())) {
3178 if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
3179 HasCancel = D->hasCancel();
3180 else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
3181 HasCancel = D->hasCancel();
3182 else if (const auto *D =
3183 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
3184 HasCancel = D->hasCancel();
3185 }
3186 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
3187 HasCancel);
3188 CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(),
3191 };
3192
3194 CGF, S,
3195 isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for,
3196 CGInlinedWorksharingLoop,
3198}
3199
3202 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3204 S.getDistInc());
3205 };
3206 OMPLexicalScope Scope(*this, S, OMPD_parallel);
3207 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3208}
3209
3212 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3214 S.getDistInc());
3215 };
3216 OMPLexicalScope Scope(*this, S, OMPD_parallel);
3217 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3218}
3219
3221 const OMPDistributeSimdDirective &S) {
3222 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3224 };
3225 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3226 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
3227}
3228
3230 CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
3231 // Emit SPMD target parallel for region as a standalone region.
3232 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3233 emitOMPSimdRegion(CGF, S, Action);
3234 };
3235 llvm::Function *Fn;
3236 llvm::Constant *Addr;
3237 // Emit target region as a standalone region.
3239 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
3240 assert(Fn && Addr && "Target device function emission failed.");
3241}
3242
3244 const OMPTargetSimdDirective &S) {
3245 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3246 emitOMPSimdRegion(CGF, S, Action);
3247 };
3248 emitCommonOMPTargetDirective(*this, S, CodeGen);
3249}
3250
3251namespace {
3252struct ScheduleKindModifiersTy {
3256 ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
3259 : Kind(Kind), M1(M1), M2(M2) {}
3260};
3261} // namespace
3262
3264 const OMPLoopDirective &S, Expr *EUB,
3265 const CodeGenLoopBoundsTy &CodeGenLoopBounds,
3266 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
3267 // Emit the loop iteration variable.
3268 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3269 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3270 EmitVarDecl(*IVDecl);
3271
3272 // Emit the iterations count variable.
3273 // If it is not a variable, Sema decided to calculate iterations count on each
3274 // iteration (e.g., it is foldable into a constant).
3275 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3276 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3277 // Emit calculation of the iterations count.
3278 EmitIgnoredExpr(S.getCalcLastIteration());
3279 }
3280
3282
3283 bool HasLastprivateClause;
3284 // Check pre-condition.
3285 {
3286 OMPLoopScope PreInitScope(*this, S);
3287 // Skip the entire loop if we don't meet the precondition.
3288 // If the condition constant folds and can be elided, avoid emitting the
3289 // whole loop.
3290 bool CondConstant;
3291 llvm::BasicBlock *ContBlock = nullptr;
3292 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
3293 if (!CondConstant)
3294 return false;
3295 } else {
3296 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
3297 ContBlock = createBasicBlock("omp.precond.end");
3298 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
3299 getProfileCount(&S));
3300 EmitBlock(ThenBlock);
3302 }
3303
3304 RunCleanupsScope DoacrossCleanupScope(*this);
3305 bool Ordered = false;
3306 if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
3307 if (OrderedClause->getNumForLoops())
3308 RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations());
3309 else
3310 Ordered = true;
3311 }
3312
3313 llvm::DenseSet<const Expr *> EmittedFinals;
3314 emitAlignedClause(*this, S);
3315 bool HasLinears = EmitOMPLinearClauseInit(S);
3316 // Emit helper vars inits.
3317
3318 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
3319 LValue LB = Bounds.first;
3320 LValue UB = Bounds.second;
3321 LValue ST =
3322 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
3323 LValue IL =
3324 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
3325
3326 // Emit 'then' code.
3327 {
3328 OMPPrivateScope LoopScope(*this);
3329 if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
3330 // Emit implicit barrier to synchronize threads and avoid data races on
3331 // initialization of firstprivate variables and post-update of
3332 // lastprivate variables.
3334 *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
3335 /*ForceSimpleCall=*/true);
3336 }
3337 EmitOMPPrivateClause(S, LoopScope);
3339 *this, S, EmitLValue(S.getIterationVariable()));
3340 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
3341 EmitOMPReductionClauseInit(S, LoopScope);
3342 EmitOMPPrivateLoopCounters(S, LoopScope);
3343 EmitOMPLinearClause(S, LoopScope);
3344 (void)LoopScope.Privatize();
3345 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
3347
3348 // Detect the loop schedule kind and chunk.
3349 const Expr *ChunkExpr = nullptr;
3350 OpenMPScheduleTy ScheduleKind;
3351 if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
3352 ScheduleKind.Schedule = C->getScheduleKind();
3353 ScheduleKind.M1 = C->getFirstScheduleModifier();
3354 ScheduleKind.M2 = C->getSecondScheduleModifier();
3355 ChunkExpr = C->getChunkSize();
3356 } else {
3357 // Default behaviour for schedule clause.
3359 *this, S, ScheduleKind.Schedule, ChunkExpr);
3360 }
3361 bool HasChunkSizeOne = false;
3362 llvm::Value *Chunk = nullptr;
3363 if (ChunkExpr) {
3364 Chunk = EmitScalarExpr(ChunkExpr);
3365 Chunk = EmitScalarConversion(Chunk, ChunkExpr->getType(),
3366 S.getIterationVariable()->getType(),
3367 S.getBeginLoc());
3369 if (ChunkExpr->EvaluateAsInt(Result, getContext())) {
3370 llvm::APSInt EvaluatedChunk = Result.Val.getInt();
3371 HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
3372 }
3373 }
3374 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3375 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3376 // OpenMP 4.5, 2.7.1 Loop Construct, Description.
3377 // If the static schedule kind is specified or if the ordered clause is
3378 // specified, and if no monotonic modifier is specified, the effect will
3379 // be as if the monotonic modifier was specified.
3380 bool StaticChunkedOne =
3381 RT.isStaticChunked(ScheduleKind.Schedule,
3382 /* Chunked */ Chunk != nullptr) &&
3383 HasChunkSizeOne &&
3384 isOpenMPLoopBoundSharingDirective(S.getDirectiveKind());
3385 bool IsMonotonic =
3386 Ordered ||
3387 (ScheduleKind.Schedule == OMPC_SCHEDULE_static &&
3388 !(ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_nonmonotonic ||
3389 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_nonmonotonic)) ||
3390 ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
3391 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
3392 if ((RT.isStaticNonchunked(ScheduleKind.Schedule,
3393 /* Chunked */ Chunk != nullptr) ||
3394 StaticChunkedOne) &&
3395 !Ordered) {
3396 JumpDest LoopExit =
3399 *this, S,
3400 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3401 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
3402 CGF.EmitOMPSimdInit(S);
3403 } else if (const auto *C = S.getSingleClause<OMPOrderClause>()) {
3404 if (C->getKind() == OMPC_ORDER_concurrent)
3405 CGF.LoopStack.setParallel(/*Enable=*/true);
3406 }
3407 },
3408 [IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk,
3409 &S, ScheduleKind, LoopExit,
3410 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
3411 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
3412 // When no chunk_size is specified, the iteration space is divided
3413 // into chunks that are approximately equal in size, and at most
3414 // one chunk is distributed to each thread. Note that the size of
3415 // the chunks is unspecified in this case.
3417 IVSize, IVSigned, Ordered, IL.getAddress(CGF),
3418 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF),
3419 StaticChunkedOne ? Chunk : nullptr);
3420 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
3421 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind,
3422 StaticInit);
3423 // UB = min(UB, GlobalUB);
3424 if (!StaticChunkedOne)
3425 CGF.EmitIgnoredExpr(S.getEnsureUpperBound());
3426 // IV = LB;
3427 CGF.EmitIgnoredExpr(S.getInit());
3428 // For unchunked static schedule generate:
3429 //
3430 // while (idx <= UB) {
3431 // BODY;
3432 // ++idx;
3433 // }
3434 //
3435 // For static schedule with chunk one:
3436 //
3437 // while (IV <= PrevUB) {
3438 // BODY;
3439 // IV += ST;
3440 // }
3441 CGF.EmitOMPInnerLoop(
3442 S, LoopScope.requiresCleanups(),
3443 StaticChunkedOne ? S.getCombinedParForInDistCond()
3444 : S.getCond(),
3445 StaticChunkedOne ? S.getDistInc() : S.getInc(),
3446 [&S, LoopExit](CodeGenFunction &CGF) {
3447 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit);
3448 },
3449 [](CodeGenFunction &) {});
3450 });
3451 EmitBlock(LoopExit.getBlock());
3452 // Tell the runtime we are done.
3453 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
3454 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
3455 S.getDirectiveKind());
3456 };
3457 OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
3458 } else {
3459 // Emit the outer loop, which requests its work chunk [LB..UB] from
3460 // runtime and runs the inner loop to process it.
3461 const OMPLoopArguments LoopArguments(
3462 LB.getAddress(*this), UB.getAddress(*this), ST.getAddress(*this),
3463 IL.getAddress(*this), Chunk, EUB);
3464 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
3465 LoopArguments, CGDispatchBounds);
3466 }
3467 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
3468 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
3469 return CGF.Builder.CreateIsNotNull(
3470 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3471 });
3472 }
3474 S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind())
3475 ? /*Parallel and Simd*/ OMPD_parallel_for_simd
3476 : /*Parallel only*/ OMPD_parallel);
3477 // Emit post-update of the reduction variables if IsLastIter != 0.
3479 *this, S, [IL, &S](CodeGenFunction &CGF) {
3480 return CGF.Builder.CreateIsNotNull(
3481 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3482 });
3483 // Emit final copy of the lastprivate variables if IsLastIter != 0.
3484 if (HasLastprivateClause)
3486 S, isOpenMPSimdDirective(S.getDirectiveKind()),
3487 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
3488 LoopScope.restoreMap();
3489 EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) {
3490 return CGF.Builder.CreateIsNotNull(
3491 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3492 });
3493 }
3494 DoacrossCleanupScope.ForceCleanup();
3495 // We're now done with the loop, so jump to the continuation block.
3496 if (ContBlock) {
3497 EmitBranch(ContBlock);
3498 EmitBlock(ContBlock, /*IsFinished=*/true);
3499 }
3500 }
3501 return HasLastprivateClause;
3502}
3503
3504/// The following two functions generate expressions for the loop lower
3505/// and upper bounds in case of static and dynamic (dispatch) schedule
3506/// of the associated 'for' or 'distribute' loop.
3507static std::pair<LValue, LValue>
3509 const auto &LS = cast<OMPLoopDirective>(S);
3510 LValue LB =
3511 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
3512 LValue UB =
3513 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
3514 return {LB, UB};
3515}
3516
3517/// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
3518/// consider the lower and upper bound expressions generated by the
3519/// worksharing loop support, but we use 0 and the iteration space size as
3520/// constants
3521static std::pair<llvm::Value *, llvm::Value *>
3523 Address LB, Address UB) {
3524 const auto &LS = cast<OMPLoopDirective>(S);
3525 const Expr *IVExpr = LS.getIterationVariable();
3526 const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
3527 llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
3528 llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
3529 return {LBVal, UBVal};
3530}
3531
3532/// Emits internal temp array declarations for the directive with inscan
3533/// reductions.
3534/// The code is the following:
3535/// \code
3536/// size num_iters = <num_iters>;
3537/// <type> buffer[num_iters];
3538/// \endcode
3540 CodeGenFunction &CGF, const OMPLoopDirective &S,
3541 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen) {
3542 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
3543 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
3546 SmallVector<const Expr *, 4> ReductionOps;
3547 SmallVector<const Expr *, 4> CopyArrayTemps;
3548 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
3549 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
3550 "Only inscan reductions are expected.");
3551 Shareds.append(C->varlist_begin(), C->varlist_end());
3552 Privates.append(C->privates().begin(), C->privates().end());
3553 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
3554 CopyArrayTemps.append(C->copy_array_temps().begin(),
3555 C->copy_array_temps().end());
3556 }
3557 {
3558 // Emit buffers for each reduction variables.
3559 // ReductionCodeGen is required to emit correctly the code for array
3560 // reductions.
3561 ReductionCodeGen RedCG(Shareds, Shareds, Privates, ReductionOps);
3562 unsigned Count = 0;
3563 auto *ITA = CopyArrayTemps.begin();
3564 for (const Expr *IRef : Privates) {
3565 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
3566 // Emit variably modified arrays, used for arrays/array sections
3567 // reductions.
3568 if (PrivateVD->getType()->isVariablyModifiedType()) {
3569 RedCG.emitSharedOrigLValue(CGF, Count);
3570 RedCG.emitAggregateType(CGF, Count);
3571 }
3572 CodeGenFunction::OpaqueValueMapping DimMapping(
3573 CGF,
3574 cast<OpaqueValueExpr>(
3575 cast<VariableArrayType>((*ITA)->getType()->getAsArrayTypeUnsafe())
3576 ->getSizeExpr()),
3577 RValue::get(OMPScanNumIterations));
3578 // Emit temp buffer.
3579 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(*ITA)->getDecl()));
3580 ++ITA;
3581 ++Count;
3582 }
3583 }
3584}
3585
3586/// Copies final inscan reductions values to the original variables.
3587/// The code is the following:
3588/// \code
3589/// <orig_var> = buffer[num_iters-1];
3590/// \endcode
3592 CodeGenFunction &CGF, const OMPLoopDirective &S,
3593 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen) {
3594 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
3595 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
3601 SmallVector<const Expr *, 4> CopyArrayElems;
3602 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
3603 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
3604 "Only inscan reductions are expected.");
3605 Shareds.append(C->varlist_begin(), C->varlist_end());
3606 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
3607 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
3608 Privates.append(C->privates().begin(), C->privates().end());
3609 CopyOps.append(C->copy_ops().begin(), C->copy_ops().end());
3610 CopyArrayElems.append(C->copy_array_elems().begin(),
3611 C->copy_array_elems().end());
3612 }
3613 // Create temp var and copy LHS value to this temp value.
3614 // LHS = TMP[LastIter];
3615 llvm::Value *OMPLast = CGF.Builder.CreateNSWSub(
3616 OMPScanNumIterations,
3617 llvm::ConstantInt::get(CGF.SizeTy, 1, /*isSigned=*/false));
3618 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
3619 const Expr *PrivateExpr = Privates[I];
3620 const Expr *OrigExpr = Shareds[I];
3621 const Expr *CopyArrayElem = CopyArrayElems[I];
3622 CodeGenFunction::OpaqueValueMapping IdxMapping(
3623 CGF,
3624 cast<OpaqueValueExpr>(
3625 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3626 RValue::get(OMPLast));
3627 LValue DestLVal = CGF.EmitLValue(OrigExpr);
3628 LValue SrcLVal = CGF.EmitLValue(CopyArrayElem);
3629 CGF.EmitOMPCopy(PrivateExpr->getType(), DestLVal.getAddress(CGF),
3630 SrcLVal.getAddress(CGF),
3631 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
3632 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
3633 CopyOps[I]);
3634 }
3635}
3636
3637/// Emits the code for the directive with inscan reductions.
3638/// The code is the following:
3639/// \code
3640/// #pragma omp ...
3641/// for (i: 0..<num_iters>) {
3642/// <input phase>;
3643/// buffer[i] = red;
3644/// }
3645/// #pragma omp master // in parallel region
3646/// for (int k = 0; k != ceil(log2(num_iters)); ++k)
3647/// for (size cnt = last_iter; cnt >= pow(2, k); --k)
3648/// buffer[i] op= buffer[i-pow(2,k)];
3649/// #pragma omp barrier // in parallel region
3650/// #pragma omp ...
3651/// for (0..<num_iters>) {
3652/// red = InclusiveScan ? buffer[i] : buffer[i-1];
3653/// <scan phase>;
3654/// }
3655/// \endcode
3657 CodeGenFunction &CGF, const OMPLoopDirective &S,
3658 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen,
3659 llvm::function_ref<void(CodeGenFunction &)> FirstGen,
3660 llvm::function_ref<void(CodeGenFunction &)> SecondGen) {
3661 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
3662 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
3664 SmallVector<const Expr *, 4> ReductionOps;
3667 SmallVector<const Expr *, 4> CopyArrayElems;
3668 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
3669 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
3670 "Only inscan reductions are expected.");
3671 Privates.append(C->privates().begin(), C->privates().end());
3672 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
3673 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
3674 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
3675 CopyArrayElems.append(C->copy_array_elems().begin(),
3676 C->copy_array_elems().end());
3677 }
3678 CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S);
3679 {
3680 // Emit loop with input phase:
3681 // #pragma omp ...
3682 // for (i: 0..<num_iters>) {
3683 // <input phase>;
3684 // buffer[i] = red;
3685 // }
3686 CGF.OMPFirstScanLoop = true;
3687 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
3688 FirstGen(CGF);
3689 }
3690 // #pragma omp barrier // in parallel region
3691 auto &&CodeGen = [&S, OMPScanNumIterations, &LHSs, &RHSs, &CopyArrayElems,
3692 &ReductionOps,
3693 &Privates](CodeGenFunction &CGF, PrePostActionTy &Action) {
3694 Action.Enter(CGF);
3695 // Emit prefix reduction:
3696 // #pragma omp master // in parallel region
3697 // for (int k = 0; k <= ceil(log2(n)); ++k)
3698 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock();
3699 llvm::BasicBlock *LoopBB = CGF.createBasicBlock("omp.outer.log.scan.body");
3700 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.outer.log.scan.exit");
3701 llvm::Function *F =
3702 CGF.CGM.getIntrinsic(llvm::Intrinsic::log2, CGF.DoubleTy);
3703 llvm::Value *Arg =
3704 CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy);
3705 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg);
3706 F = CGF.CGM.getIntrinsic(llvm::Intrinsic::ceil, CGF.DoubleTy);
3707 LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal);
3708 LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy);
3709 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub(
3710 OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1));
3711 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getBeginLoc());
3712 CGF.EmitBlock(LoopBB);
3713 auto *Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2);
3714 // size pow2k = 1;
3715 auto *Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3716 Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB);
3717 Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB);
3718 // for (size i = n - 1; i >= 2 ^ k; --i)
3719 // tmp[i] op= tmp[i-pow2k];
3720 llvm::BasicBlock *InnerLoopBB =
3721 CGF.createBasicBlock("omp.inner.log.scan.body");
3722 llvm::BasicBlock *InnerExitBB =
3723 CGF.createBasicBlock("omp.inner.log.scan.exit");
3724 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K);
3725 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3726 CGF.EmitBlock(InnerLoopBB);
3727 auto *IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3728 IVal->addIncoming(NMin1, LoopBB);
3729 {
3730 CodeGenFunction::OMPPrivateScope PrivScope(CGF);
3731 auto *ILHS = LHSs.begin();
3732 auto *IRHS = RHSs.begin();
3733 for (const Expr *CopyArrayElem : CopyArrayElems) {
3734 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
3735 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
3736 Address LHSAddr = Address::invalid();
3737 {
3738 CodeGenFunction::OpaqueValueMapping IdxMapping(
3739 CGF,
3740 cast<OpaqueValueExpr>(
3741 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3742 RValue::get(IVal));
3743 LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF);
3744 }
3745 PrivScope.addPrivate(LHSVD, LHSAddr);
3746 Address RHSAddr = Address::invalid();
3747 {
3748 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K);
3749 CodeGenFunction::OpaqueValueMapping IdxMapping(
3750 CGF,
3751 cast<OpaqueValueExpr>(
3752 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3753 RValue::get(OffsetIVal));
3754 RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF);
3755 }
3756 PrivScope.addPrivate(RHSVD, RHSAddr);
3757 ++ILHS;
3758 ++IRHS;
3759 }
3760 PrivScope.Privatize();
3761 CGF.CGM.getOpenMPRuntime().emitReduction(
3762 CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
3763 {/*WithNowait=*/true, /*SimpleReduction=*/true, OMPD_unknown});
3764 }
3765 llvm::Value *NextIVal =
3766 CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1));
3767 IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock());
3768 CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K);
3769 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3770 CGF.EmitBlock(InnerExitBB);
3771 llvm::Value *Next =
3772 CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1));
3773 Counter->addIncoming(Next, CGF.Builder.GetInsertBlock());
3774 // pow2k <<= 1;
3775 llvm::Value *NextPow2K =
3776 CGF.Builder.CreateShl(Pow2K, 1, "", /*HasNUW=*/true);
3777 Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock());
3778 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(Next, LogVal);
3779 CGF.Builder.CreateCondBr(Cmp, LoopBB, ExitBB);
3780 auto DL1 = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getEndLoc());
3781 CGF.EmitBlock(ExitBB);
3782 };
3783 if (isOpenMPParallelDirective(S.getDirectiveKind())) {
3784 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());
3785 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
3786 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
3787 /*ForceSimpleCall=*/true);
3788 } else {
3789 RegionCodeGenTy RCG(CodeGen);
3790 RCG(CGF);
3791 }
3792
3793 CGF.OMPFirstScanLoop = false;
3794 SecondGen(CGF);
3795}
3796
3798 const OMPLoopDirective &S,
3799 bool HasCancel) {
3800 bool HasLastprivates;
3801 if (llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
3802 [](const OMPReductionClause *C) {
3803 return C->getModifier() == OMPC_REDUCTION_inscan;
3804 })) {
3805 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
3806 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
3807 OMPLoopScope LoopScope(CGF, S);
3808 return CGF.EmitScalarExpr(S.getNumIterations());
3809 };
3810 const auto &&FirstGen = [&S, HasCancel](CodeGenFunction &CGF) {
3811 CodeGenFunction::OMPCancelStackRAII CancelRegion(
3812 CGF, S.getDirectiveKind(), HasCancel);
3813 (void)CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
3816 // Emit an implicit barrier at the end.
3817 CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getBeginLoc(),
3818 OMPD_for);
3819 };
3820 const auto &&SecondGen = [&S, HasCancel,
3821 &HasLastprivates](CodeGenFunction &CGF) {
3822 CodeGenFunction::OMPCancelStackRAII CancelRegion(
3823 CGF, S.getDirectiveKind(), HasCancel);
3824 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
3827 };
3828 if (!isOpenMPParallelDirective(S.getDirectiveKind()))
3829 emitScanBasedDirectiveDecls(CGF, S, NumIteratorsGen);
3830 emitScanBasedDirective(CGF, S, NumIteratorsGen, FirstGen, SecondGen);
3831 if (!isOpenMPParallelDirective(S.getDirectiveKind()))
3832 emitScanBasedDirectiveFinals(CGF, S, NumIteratorsGen);
3833 } else {
3834 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
3835 HasCancel);
3836 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
3839 }
3840 return HasLastprivates;
3841}
3842
3844 if (S.hasCancel())
3845 return false;
3846 for (OMPClause *C : S.clauses()) {
3847 if (isa<OMPNowaitClause>(C))
3848 continue;
3849
3850 if (auto *SC = dyn_cast<OMPScheduleClause>(C)) {
3851 if (SC->getFirstScheduleModifier() != OMPC_SCHEDULE_MODIFIER_unknown)
3852 return false;
3853 if (SC->getSecondScheduleModifier() != OMPC_SCHEDULE_MODIFIER_unknown)
3854 return false;
3855 switch (SC->getScheduleKind()) {
3856 case OMPC_SCHEDULE_auto:
3857 case OMPC_SCHEDULE_dynamic:
3858 case OMPC_SCHEDULE_runtime:
3859 case OMPC_SCHEDULE_guided:
3860 case OMPC_SCHEDULE_static:
3861 continue;
3863 return false;
3864 }
3865 }
3866
3867 return false;
3868 }
3869
3870 return true;
3871}
3872
3873static llvm::omp::ScheduleKind
3875 switch (ScheduleClauseKind) {
3877 return llvm::omp::OMP_SCHEDULE_Default;
3878 case OMPC_SCHEDULE_auto:
3879 return llvm::omp::OMP_SCHEDULE_Auto;
3880 case OMPC_SCHEDULE_dynamic:
3881 return llvm::omp::OMP_SCHEDULE_Dynamic;
3882 case OMPC_SCHEDULE_guided:
3883 return llvm::omp::OMP_SCHEDULE_Guided;
3884 case OMPC_SCHEDULE_runtime:
3885 return llvm::omp::OMP_SCHEDULE_Runtime;
3886 case OMPC_SCHEDULE_static:
3887 return llvm::omp::OMP_SCHEDULE_Static;
3888 }
3889 llvm_unreachable("Unhandled schedule kind");
3890}
3891
3893 bool HasLastprivates = false;
3894 bool UseOMPIRBuilder =
3895 CGM.getLangOpts().OpenMPIRBuilder && isSupportedByOpenMPIRBuilder(S);
3896 auto &&CodeGen = [this, &S, &HasLastprivates,
3897 UseOMPIRBuilder](CodeGenFunction &CGF, PrePostActionTy &) {
3898 // Use the OpenMPIRBuilder if enabled.
3899 if (UseOMPIRBuilder) {
3900 bool NeedsBarrier = !S.getSingleClause<OMPNowaitClause>();
3901
3902 llvm::omp::ScheduleKind SchedKind = llvm::omp::OMP_SCHEDULE_Default;
3903 llvm::Value *ChunkSize = nullptr;
3904 if (auto *SchedClause = S.getSingleClause<OMPScheduleClause>()) {
3905 SchedKind =
3906 convertClauseKindToSchedKind(SchedClause->getScheduleKind());
3907 if (const Expr *ChunkSizeExpr = SchedClause->getChunkSize())
3908 ChunkSize = EmitScalarExpr(ChunkSizeExpr);
3909 }
3910
3911 // Emit the associated statement and get its loop representation.
3912 const Stmt *Inner = S.getRawStmt();
3913 llvm::CanonicalLoopInfo *CLI =
3915
3916 llvm::OpenMPIRBuilder &OMPBuilder =
3918 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
3919 AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
3920 OMPBuilder.applyWorkshareLoop(
3921 Builder.getCurrentDebugLocation(), CLI, AllocaIP, NeedsBarrier,
3922 SchedKind, ChunkSize, /*HasSimdModifier=*/false,
3923 /*HasMonotonicModifier=*/false, /*HasNonmonotonicModifier=*/false,
3924 /*HasOrderedClause=*/false);
3925 return;
3926 }
3927
3928 HasLastprivates = emitWorksharingDirective(CGF, S, S.hasCancel());
3929 };
3930 {
3931 auto LPCRegion =
3933 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3934 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
3935 S.hasCancel());
3936 }
3937
3938 if (!UseOMPIRBuilder) {
3939 // Emit an implicit barrier at the end.
3940 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
3941 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
3942 }
3943 // Check for outer lastprivate conditional update.
3945}
3946
3948 bool HasLastprivates = false;
3949 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
3950 PrePostActionTy &) {
3951 HasLastprivates = emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
3952 };
3953 {
3954 auto LPCRegion =
3956 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3957 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
3958 }
3959
3960 // Emit an implicit barrier at the end.
3961 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
3962 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
3963 // Check for outer lastprivate conditional update.
3965}
3966
3968 const Twine &Name,
3969 llvm::Value *Init = nullptr) {
3970 LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
3971 if (Init)
3972 CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
3973 return LVal;
3974}
3975
3976void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
3977 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
3978 const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
3979 bool HasLastprivates = false;
3980 auto &&CodeGen = [&S, CapturedStmt, CS,
3981 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
3982 const ASTContext &C = CGF.getContext();
3983 QualType KmpInt32Ty =
3984 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
3985 // Emit helper vars inits.
3986 LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
3987 CGF.Builder.getInt32(0));
3988 llvm::ConstantInt *GlobalUBVal = CS != nullptr
3989 ? CGF.Builder.getInt32(CS->size() - 1)
3990 : CGF.Builder.getInt32(0);
3991 LValue UB =
3992 createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
3993 LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
3994 CGF.Builder.getInt32(1));
3995 LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
3996 CGF.Builder.getInt32(0));
3997 // Loop counter.
3998 LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
3999 OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
4000 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
4001 OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
4002 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
4003 // Generate condition for loop.
4005 C, &IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_PRValue, OK_Ordinary,
4006 S.getBeginLoc(), FPOptionsOverride());
4007 // Increment for loop counter.
4009 C, &IVRefExpr, UO_PreInc, KmpInt32Ty, VK_PRValue, OK_Ordinary,
4010 S.getBeginLoc(), true, FPOptionsOverride());
4011 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
4012 // Iterate through all sections and emit a switch construct:
4013 // switch (IV) {
4014 // case 0:
4015 // <SectionStmt[0]>;
4016 // break;
4017 // ...
4018 // case <NumSection> - 1:
4019 // <SectionStmt[<NumSection> - 1]>;
4020 // break;
4021 // }
4022 // .omp.sections.exit:
4023 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
4024 llvm::SwitchInst *SwitchStmt =
4025 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
4026 ExitBB, CS == nullptr ? 1 : CS->size());
4027 if (CS) {
4028 unsigned CaseNumber = 0;
4029 for (const Stmt *SubStmt : CS->children()) {
4030 auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
4031 CGF.EmitBlock(CaseBB);
4032 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
4033 CGF.EmitStmt(SubStmt);
4034 CGF.EmitBranch(ExitBB);
4035 ++CaseNumber;
4036 }
4037 } else {
4038 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");
4039 CGF.EmitBlock(CaseBB);
4040 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
4041 CGF.EmitStmt(CapturedStmt);
4042 CGF.EmitBranch(ExitBB);
4043 }
4044 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
4045 };
4046
4047 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
4048 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
4049 // Emit implicit barrier to synchronize threads and avoid data races on
4050 // initialization of firstprivate variables and post-update of lastprivate
4051 // variables.
4052 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
4053 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
4054 /*ForceSimpleCall=*/true);
4055 }
4056 CGF.EmitOMPPrivateClause(S, LoopScope);
4057 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(CGF, S, IV);
4058 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4059 CGF.EmitOMPReductionClauseInit(S, LoopScope);
4060 (void)LoopScope.Privatize();
4061 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
4062 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4063
4064 // Emit static non-chunked loop.
4065 OpenMPScheduleTy ScheduleKind;
4066 ScheduleKind.Schedule = OMPC_SCHEDULE_static;
4068 /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(CGF),
4069 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF));
4070 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
4071 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit);
4072 // UB = min(UB, GlobalUB);
4073 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
4074 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
4075 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
4076 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
4077 // IV = LB;
4078 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
4079 // while (idx <= UB) { BODY; ++idx; }
4080 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, Cond, Inc, BodyGen,
4081 [](CodeGenFunction &) {});
4082 // Tell the runtime we are done.
4083 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
4084 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
4085 S.getDirectiveKind());
4086 };
4087 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
4088 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4089 // Emit post-update of the reduction variables if IsLastIter != 0.
4090 emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) {
4091 return CGF.Builder.CreateIsNotNull(
4092 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
4093 });
4094
4095 // Emit final copy of the lastprivate variables if IsLastIter != 0.
4096 if (HasLastprivates)
4098 S, /*NoFinals=*/false,
4099 CGF.Builder.CreateIsNotNull(
4100 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())));
4101 };
4102
4103 bool HasCancel = false;
4104 if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
4105 HasCancel = OSD->hasCancel();
4106 else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
4107 HasCancel = OPSD->hasCancel();
4108 OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
4109 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
4110 HasCancel);
4111 // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
4112 // clause. Otherwise the barrier will be generated by the codegen for the
4113 // directive.
4114 if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
4115 // Emit implicit barrier to synchronize threads and avoid data races on
4116 // initialization of firstprivate variables.
4117 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
4118 OMPD_unknown);
4119 }
4120}
4121
4123 if (CGM.getLangOpts().OpenMPIRBuilder) {
4124 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4125 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4126 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4127
4128 auto FiniCB = [this](InsertPointTy IP) {
4130 };
4131
4132 const CapturedStmt *ICS = S.getInnermostCapturedStmt();
4133 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4134 const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
4136 if (CS) {
4137 for (const Stmt *SubStmt : CS->children()) {
4138 auto SectionCB = [this, SubStmt](InsertPointTy AllocaIP,
4139 InsertPointTy CodeGenIP) {
4141 *this, SubStmt, AllocaIP, CodeGenIP, "section");
4142 };
4143 SectionCBVector.push_back(SectionCB);
4144 }
4145 } else {
4146 auto SectionCB = [this, CapturedStmt](InsertPointTy AllocaIP,
4147 InsertPointTy CodeGenIP) {
4149 *this, CapturedStmt, AllocaIP, CodeGenIP, "section");
4150 };
4151 SectionCBVector.push_back(SectionCB);
4152 }
4153
4154 // Privatization callback that performs appropriate action for
4155 // shared/private/firstprivate/lastprivate/copyin/... variables.
4156 //
4157 // TODO: This defaults to shared right now.
4158 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
4159 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
4160 // The next line is appropriate only for variables (Val) with the
4161 // data-sharing attribute "shared".
4162 ReplVal = &Val;
4163
4164 return CodeGenIP;
4165 };
4166
4167 CGCapturedStmtInfo CGSI(*ICS, CR_OpenMP);
4168 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
4169 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
4170 AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
4171 Builder.restoreIP(OMPBuilder.createSections(
4172 Builder, AllocaIP, SectionCBVector, PrivCB, FiniCB, S.hasCancel(),
4173 S.getSingleClause<OMPNowaitClause>()));
4174 return;
4175 }
4176 {
4177 auto LPCRegion =
4179 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4180 EmitSections(S);
4181 }
4182 // Emit an implicit barrier at the end.
4183 if (!S.getSingleClause<OMPNowaitClause>()) {
4184 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
4185 OMPD_sections);
4186 }
4187 // Check for outer lastprivate conditional update.
4189}
4190
4192 if (CGM.getLangOpts().OpenMPIRBuilder) {
4193 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4194 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4195
4196 const Stmt *SectionRegionBodyStmt = S.getAssociatedStmt();
4197 auto FiniCB = [this](InsertPointTy IP) {
4199 };
4200
4201 auto BodyGenCB = [SectionRegionBodyStmt, this](InsertPointTy AllocaIP,
4202 InsertPointTy CodeGenIP) {
4204 *this, SectionRegionBodyStmt, AllocaIP, CodeGenIP, "section");
4205 };
4206
4207 LexicalScope Scope(*this, S.getSourceRange());
4208 EmitStopPoint(&S);
4209 Builder.restoreIP(OMPBuilder.createSection(Builder, BodyGenCB, FiniCB));
4210
4211 return;
4212 }
4213 LexicalScope Scope(*this, S.getSourceRange());
4214 EmitStopPoint(&S);
4215 EmitStmt(S.getAssociatedStmt());
4216}
4217
4219 llvm::SmallVector<const Expr *, 8> CopyprivateVars;
4223 // Check if there are any 'copyprivate' clauses associated with this
4224 // 'single' construct.
4225 // Build a list of copyprivate variables along with helper expressions
4226 // (<source>, <destination>, <destination>=<source> expressions)
4227 for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
4228 CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
4229 DestExprs.append(C->destination_exprs().begin(),
4230 C->destination_exprs().end());
4231 SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
4232 AssignmentOps.append(C->assignment_ops().begin(),
4233 C->assignment_ops().end());
4234 }
4235 // Emit code for 'single' region along with 'copyprivate' clauses
4236 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4237 Action.Enter(CGF);
4238 OMPPrivateScope SingleScope(CGF);
4239 (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
4240 CGF.EmitOMPPrivateClause(S, SingleScope);
4241 (void)SingleScope.Privatize();
4242 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4243 };
4244 {
4245 auto LPCRegion =
4247 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4248 CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getBeginLoc(),
4249 CopyprivateVars, DestExprs,
4250 SrcExprs, AssignmentOps);
4251 }
4252 // Emit an implicit barrier at the end (to avoid data race on firstprivate
4253 // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
4254 if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
4256 *this, S.getBeginLoc(),
4257 S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
4258 }
4259 // Check for outer lastprivate conditional update.
4261}
4262
4264 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4265 Action.Enter(CGF);
4266 CGF.EmitStmt(S.getRawStmt());
4267 };
4268 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());
4269}
4270
4272 if (CGM.getLangOpts().OpenMPIRBuilder) {
4273 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4274 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4275
4276 const Stmt *MasterRegionBodyStmt = S.getAssociatedStmt();
4277
4278 auto FiniCB = [this](InsertPointTy IP) {
4280 };
4281
4282 auto BodyGenCB = [MasterRegionBodyStmt, this](InsertPointTy AllocaIP,
4283 InsertPointTy CodeGenIP) {
4285 *this, MasterRegionBodyStmt, AllocaIP, CodeGenIP, "master");
4286 };
4287
4288 LexicalScope Scope(*this, S.getSourceRange());
4289 EmitStopPoint(&S);
4290 Builder.restoreIP(OMPBuilder.createMaster(Builder, BodyGenCB, FiniCB));
4291
4292 return;
4293 }
4294 LexicalScope Scope(*this, S.getSourceRange());
4295 EmitStopPoint(&S);
4296 emitMaster(*this, S);
4297}
4298
4300 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4301 Action.Enter(CGF);
4302 CGF.EmitStmt(S.getRawStmt());
4303 };
4304 Expr *Filter = nullptr;
4305 if (const auto *FilterClause = S.getSingleClause<OMPFilterClause>())
4306 Filter = FilterClause->getThreadID();
4307 CGF.CGM.getOpenMPRuntime().emitMaskedRegion(CGF, CodeGen, S.getBeginLoc(),
4308 Filter);
4309}
4310
4312 if (CGM.getLangOpts().OpenMPIRBuilder) {
4313 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4314 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4315
4316 const Stmt *MaskedRegionBodyStmt = S.getAssociatedStmt();
4317 const Expr *Filter = nullptr;
4318 if (const auto *FilterClause = S.getSingleClause<OMPFilterClause>())
4319 Filter = FilterClause->getThreadID();
4320 llvm::Value *FilterVal = Filter
4321 ? EmitScalarExpr(Filter, CGM.Int32Ty)
4322 : llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/0);
4323
4324 auto FiniCB = [this](InsertPointTy IP) {
4326 };
4327
4328 auto BodyGenCB = [MaskedRegionBodyStmt, this](InsertPointTy AllocaIP,
4329 InsertPointTy CodeGenIP) {
4331 *this, MaskedRegionBodyStmt, AllocaIP, CodeGenIP, "masked");
4332 };
4333
4334 LexicalScope Scope(*this, S.getSourceRange());
4335 EmitStopPoint(&S);
4336 Builder.restoreIP(
4337 OMPBuilder.createMasked(Builder, BodyGenCB, FiniCB, FilterVal));
4338
4339 return;
4340 }
4341 LexicalScope Scope(*this, S.getSourceRange());
4342 EmitStopPoint(&S);
4343 emitMasked(*this, S);
4344}
4345
4347 if (CGM.getLangOpts().OpenMPIRBuilder) {
4348 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4349 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4350
4351 const Stmt *CriticalRegionBodyStmt = S.getAssociatedStmt();
4352 const Expr *Hint = nullptr;
4353 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
4354 Hint = HintClause->getHint();
4355
4356 // TODO: This is slightly different from what's currently being done in
4357 // clang. Fix the Int32Ty to IntPtrTy (pointer width size) when everything
4358 // about typing is final.
4359 llvm::Value *HintInst = nullptr;
4360 if (Hint)
4361 HintInst =
4362 Builder.CreateIntCast(EmitScalarExpr(Hint), CGM.Int32Ty, false);
4363
4364 auto FiniCB = [this](InsertPointTy IP) {
4366 };
4367
4368 auto BodyGenCB = [CriticalRegionBodyStmt, this](InsertPointTy AllocaIP,
4369 InsertPointTy CodeGenIP) {
4371 *this, CriticalRegionBodyStmt, AllocaIP, CodeGenIP, "critical");
4372 };
4373
4374 LexicalScope Scope(*this, S.getSourceRange());
4375 EmitStopPoint(&S);
4376 Builder.restoreIP(OMPBuilder.createCritical(
4377 Builder, BodyGenCB, FiniCB, S.getDirectiveName().getAsString(),
4378 HintInst));
4379
4380 return;
4381 }
4382
4383 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4384 Action.Enter(CGF);
4385 CGF.EmitStmt(S.getAssociatedStmt());
4386 };
4387 const Expr *Hint = nullptr;
4388 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
4389 Hint = HintClause->getHint();
4390 LexicalScope Scope(*this, S.getSourceRange());
4391 EmitStopPoint(&S);
4393 S.getDirectiveName().getAsString(),
4394 CodeGen, S.getBeginLoc(), Hint);
4395}
4396
4398 const OMPParallelForDirective &S) {
4399 // Emit directive as a combined directive that consists of two implicit
4400 // directives: 'parallel' with 'for' directive.
4401 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4402 Action.Enter(CGF);
4403 emitOMPCopyinClause(CGF, S);
4404 (void)emitWorksharingDirective(CGF, S, S.hasCancel());
4405 };
4406 {
4407 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
4408 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
4409 CGCapturedStmtInfo CGSI(CR_OpenMP);
4410 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
4411 OMPLoopScope LoopScope(CGF, S);
4412 return CGF.EmitScalarExpr(S.getNumIterations());
4413 };
4414 bool IsInscan = llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
4415 [](const OMPReductionClause *C) {
4416 return C->getModifier() == OMPC_REDUCTION_inscan;
4417 });
4418 if (IsInscan)
4419 emitScanBasedDirectiveDecls(*this, S, NumIteratorsGen);
4420 auto LPCRegion =
4422 emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
4424 if (IsInscan)
4425 emitScanBasedDirectiveFinals(*this, S, NumIteratorsGen);
4426 }
4427 // Check for outer lastprivate conditional update.
4429}
4430
4432 const OMPParallelForSimdDirective &S) {
4433 // Emit directive as a combined directive that consists of two implicit
4434 // directives: 'parallel' with 'for' directive.
4435 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4436 Action.Enter(CGF);
4437 emitOMPCopyinClause(CGF, S);
4438 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
4439 };
4440 {
4441 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
4442 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
4443 CGCapturedStmtInfo CGSI(CR_OpenMP);
4444 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
4445 OMPLoopScope LoopScope(CGF, S);
4446 return CGF.EmitScalarExpr(S.getNumIterations());
4447 };
4448 bool IsInscan = llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
4449 [](const OMPReductionClause *C) {
4450 return C->getModifier() == OMPC_REDUCTION_inscan;
4451 });
4452 if (IsInscan)
4453 emitScanBasedDirectiveDecls(*this, S, NumIteratorsGen);
4454 auto LPCRegion =
4456 emitCommonOMPParallelDirective(*this, S, OMPD_for_simd, CodeGen,
4458 if (IsInscan)
4459 emitScanBasedDirectiveFinals(*this, S, NumIteratorsGen);
4460 }
4461 // Check for outer lastprivate conditional update.
4463}
4464
4466 const OMPParallelMasterDirective &S) {
4467 // Emit directive as a combined directive that consists of two implicit
4468 // directives: 'parallel' with 'master' directive.
4469 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4470 Action.Enter(CGF);
4471 OMPPrivateScope PrivateScope(CGF);
4472 emitOMPCopyinClause(CGF, S);
4473 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4474 CGF.EmitOMPPrivateClause(S, PrivateScope);
4475 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4476 (void)PrivateScope.Privatize();
4477 emitMaster(CGF, S);
4478 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4479 };
4480 {
4481 auto LPCRegion =
4483 emitCommonOMPParallelDirective(*this, S, OMPD_master, CodeGen,
4486 [](CodeGenFunction &) { return nullptr; });
4487 }
4488 // Check for outer lastprivate conditional update.
4490}
4491
4494 // Emit directive as a combined directive that consists of two implicit
4495 // directives: 'parallel' with 'sections' directive.
4496 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4497 Action.Enter(CGF);
4498 emitOMPCopyinClause(CGF, S);
4499 CGF.EmitSections(S);
4500 };
4501 {
4502 auto LPCRegion =
4504 emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
4506 }
4507 // Check for outer lastprivate conditional update.
4509}
4510
4511namespace {
4512/// Get the list of variables declared in the context of the untied tasks.
4513class CheckVarsEscapingUntiedTaskDeclContext final
4514 : public ConstStmtVisitor<CheckVarsEscapingUntiedTaskDeclContext> {
4516
4517public:
4518 explicit CheckVarsEscapingUntiedTaskDeclContext() = default;
4519 virtual ~CheckVarsEscapingUntiedTaskDeclContext() = default;
4520 void VisitDeclStmt(const DeclStmt *S) {
4521 if (!S)
4522 return;
4523 // Need to privatize only local vars, static locals can be processed as is.
4524 for (const Decl *D : S->decls()) {
4525 if (const auto *VD = dyn_cast_or_null<VarDecl>(D))
4526 if (VD->hasLocalStorage())
4527 PrivateDecls.push_back(VD);
4528 }
4529 }
4530 void VisitOMPExecutableDirective(const OMPExecutableDirective *) {}
4531 void VisitCapturedStmt(const CapturedStmt *) {}
4532 void VisitLambdaExpr(const LambdaExpr *) {}
4533 void VisitBlockExpr(const BlockExpr *) {}
4534 void VisitStmt(const Stmt *S) {
4535 if (!S)
4536 return;
4537 for (const Stmt *Child : S->children())
4538 if (Child)
4539 Visit(Child);
4540 }
4541
4542 /// Swaps list of vars with the provided one.
4543 ArrayRef<const VarDecl *> getPrivateDecls() const { return PrivateDecls; }
4544};
4545} // anonymous namespace
4546
4549
4550 // First look for 'omp_all_memory' and add this first.
4551 bool OmpAllMemory = false;
4552 if (llvm::any_of(
4553 S.getClausesOfKind<OMPDependClause>(), [](const OMPDependClause *C) {
4554 return C->getDependencyKind() == OMPC_DEPEND_outallmemory ||
4555 C->getDependencyKind() == OMPC_DEPEND_inoutallmemory;
4556 })) {
4557 OmpAllMemory = true;
4558 // Since both OMPC_DEPEND_outallmemory and OMPC_DEPEND_inoutallmemory are
4559 // equivalent to the runtime, always use OMPC_DEPEND_outallmemory to
4560 // simplify.
4562 Data.Dependences.emplace_back(OMPC_DEPEND_outallmemory,
4563 /*IteratorExpr=*/nullptr);
4564 // Add a nullptr Expr to simplify the codegen in emitDependData.
4565 DD.DepExprs.push_back(nullptr);
4566 }
4567 // Add remaining dependences skipping any 'out' or 'inout' if they are
4568 // overridden by 'omp_all_memory'.
4569 for (const auto *C : S.getClausesOfKind<OMPDependClause>()) {
4570 OpenMPDependClauseKind Kind = C->getDependencyKind();
4571 if (Kind == OMPC_DEPEND_outallmemory || Kind == OMPC_DEPEND_inoutallmemory)
4572 continue;
4573 if (OmpAllMemory && (Kind == OMPC_DEPEND_out || Kind == OMPC_DEPEND_inout))
4574 continue;
4576 Data.Dependences.emplace_back(C->getDependencyKind(), C->getModifier());
4577 DD.DepExprs.append(C->varlist_begin(), C->varlist_end());
4578 }
4579}
4580
4582 const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
4583 const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
4585 // Emit outlined function for task construct.
4586 const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
4587 auto I = CS->getCapturedDecl()->param_begin();
4588 auto PartId = std::next(I);
4589 auto TaskT = std::next(I, 4);
4590 // Check if the task is final
4591 if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
4592 // If the condition constant folds and can be elided, try to avoid emitting
4593 // the condition and the dead arm of the if/else.
4594 const Expr *Cond = Clause->getCondition();
4595 bool CondConstant;
4596 if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
4597 Data.Final.setInt(CondConstant);
4598 else
4599 Data.Final.setPointer(EvaluateExprAsBool(Cond));
4600 } else {
4601 // By default the task is not final.
4602 Data.Final.setInt(/*IntVal=*/false);
4603 }
4604 // Check if the task has 'priority' clause.
4605 if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
4606 const Expr *Prio = Clause->getPriority();
4607 Data.Priority.setInt(/*IntVal=*/true);
4608 Data.Priority.setPointer(EmitScalarConversion(
4609 EmitScalarExpr(Prio), Prio->getType(),
4610 getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
4611 Prio->getExprLoc()));
4612 }
4613 // The first function argument for tasks is a thread id, the second one is a
4614 // part id (0 for tied tasks, >=0 for untied task).
4615 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
4616 // Get list of private variables.
4617 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
4618 auto IRef = C->varlist_begin();
4619 for (const Expr *IInit : C->private_copies()) {
4620 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4621 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4622 Data.PrivateVars.push_back(*IRef);
4623 Data.PrivateCopies.push_back(IInit);
4624 }
4625 ++IRef;
4626 }
4627 }
4628 EmittedAsPrivate.clear();
4629 // Get list of firstprivate variables.
4630 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
4631 auto IRef = C->varlist_begin();
4632 auto IElemInitRef = C->inits().begin();
4633 for (const Expr *IInit : C->private_copies()) {
4634 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4635 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4636 Data.FirstprivateVars.push_back(*IRef);
4637 Data.FirstprivateCopies.push_back(IInit);
4638 Data.FirstprivateInits.push_back(*IElemInitRef);
4639 }
4640 ++IRef;
4641 ++IElemInitRef;
4642 }
4643 }
4644 // Get list of lastprivate variables (for taskloops).
4645 llvm::MapVector<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
4646 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
4647 auto IRef = C->varlist_begin();
4648 auto ID = C->destination_exprs().begin();
4649 for (const Expr *IInit : C->private_copies()) {
4650 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4651 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4652 Data.LastprivateVars.push_back(*IRef);
4653 Data.LastprivateCopies.push_back(IInit);
4654 }
4655 LastprivateDstsOrigs.insert(
4656 std::make_pair(cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
4657 cast<DeclRefExpr>(*IRef)));
4658 ++IRef;
4659 ++ID;
4660 }
4661 }
4664 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
4665 Data.ReductionVars.append(C->varlist_begin(), C->varlist_end());
4666 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
4667 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
4668 Data.ReductionOps.append(C->reduction_ops().begin(),
4669 C->reduction_ops().end());
4670 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
4671 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
4672 }
4674 *this, S.getBeginLoc(), LHSs, RHSs, Data);
4675 // Build list of dependences.
4677 // Get list of local vars for untied tasks.
4678 if (!Data.Tied) {
4679 CheckVarsEscapingUntiedTaskDeclContext Checker;
4680 Checker.Visit(S.getInnermostCapturedStmt()->getCapturedStmt());
4681 Data.PrivateLocals.append(Checker.getPrivateDecls().begin(),
4682 Checker.getPrivateDecls().end());
4683 }
4684 auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
4685 CapturedRegion](CodeGenFunction &CGF,
4686 PrePostActionTy &Action) {
4687 llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
4688 std::pair<Address, Address>>
4689 UntiedLocalVars;
4690 // Set proper addresses for generated private copies.
4691 OMPPrivateScope Scope(CGF);
4692 // Generate debug info for variables present in shared clause.
4693 if (auto *DI = CGF.getDebugInfo()) {
4694 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields =
4695 CGF.CapturedStmtInfo->getCaptureFields();
4696 llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue();
4697 if (CaptureFields.size() && ContextValue) {
4698 unsigned CharWidth = CGF.getContext().getCharWidth();
4699 // The shared variables are packed together as members of structure.
4700 // So the address of each shared variable can be computed by adding
4701 // offset of it (within record) to the base address of record. For each
4702 // shared variable, debug intrinsic llvm.dbg.declare is generated with
4703 // appropriate expressions (DIExpression).
4704 // Ex:
4705 // %12 = load %struct.anon*, %struct.anon** %__context.addr.i
4706 // call void @llvm.dbg.declare(metadata %struct.anon* %12,
4707 // metadata !svar1,
4708 // metadata !DIExpression(DW_OP_deref))
4709 // call void @llvm.dbg.declare(metadata %struct.anon* %12,
4710 // metadata !svar2,
4711 // metadata !DIExpression(DW_OP_plus_uconst, 8, DW_OP_deref))
4712 for (auto It = CaptureFields.begin(); It != CaptureFields.end(); ++It) {
4713 const VarDecl *SharedVar = It->first;
4714 RecordDecl *CaptureRecord = It->second->getParent();
4715 const ASTRecordLayout &Layout =
4716 CGF.getContext().getASTRecordLayout(CaptureRecord);
4717 unsigned Offset =
4718 Layout.getFieldOffset(It->second->getFieldIndex()) / CharWidth;
4719 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
4720 (void)DI->EmitDeclareOfAutoVariable(SharedVar, ContextValue,
4721 CGF.Builder, false);
4722 llvm::Instruction &Last = CGF.Builder.GetInsertBlock()->back();
4723 // Get the call dbg.declare instruction we just created and update
4724 // its DIExpression to add offset to base address.
4725 if (auto DDI = dyn_cast<llvm::DbgVariableIntrinsic>(&Last)) {
4727 // Add offset to the base address if non zero.
4728 if (Offset) {
4729 Ops.push_back(llvm::dwarf::DW_OP_plus_uconst);
4730 Ops.push_back(Offset);
4731 }
4732 Ops.push_back(llvm::dwarf::DW_OP_deref);
4733 auto &Ctx = DDI->getContext();
4734 llvm::DIExpression *DIExpr = llvm::DIExpression::get(Ctx, Ops);
4735 Last.setOperand(2, llvm::MetadataAsValue::get(Ctx, DIExpr));
4736 }
4737 }
4738 }
4739 }
4741 if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
4742 !Data.LastprivateVars.empty() || !Data.PrivateLocals.empty()) {
4743 enum { PrivatesParam = 2, CopyFnParam = 3 };
4744 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
4745 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
4746 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
4747 CS->getCapturedDecl()->getParam(PrivatesParam)));
4748 // Map privates.
4752 CallArgs.push_back(PrivatesPtr);
4753 ParamTypes.push_back(PrivatesPtr->getType());
4754 for (const Expr *E : Data.PrivateVars) {
4755 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4756 Address PrivatePtr = CGF.CreateMemTemp(
4757 CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
4758 PrivatePtrs.emplace_back(VD, PrivatePtr);
4759 CallArgs.push_back(PrivatePtr.getPointer());
4760 ParamTypes.push_back(PrivatePtr.getType());
4761 }
4762 for (const Expr *E : Data.FirstprivateVars) {
4763 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4764 Address PrivatePtr =
4765 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
4766 ".firstpriv.ptr.addr");
4767 PrivatePtrs.emplace_back(VD, PrivatePtr);
4768 FirstprivatePtrs.emplace_back(VD, PrivatePtr);
4769 CallArgs.push_back(PrivatePtr.getPointer());
4770 ParamTypes.push_back(PrivatePtr.getType());
4771 }
4772 for (const Expr *E : Data.LastprivateVars) {
4773 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4774 Address PrivatePtr =
4775 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
4776 ".lastpriv.ptr.addr");
4777 PrivatePtrs.emplace_back(VD, PrivatePtr);
4778 CallArgs.push_back(PrivatePtr.getPointer());
4779 ParamTypes.push_back(PrivatePtr.getType());
4780 }
4781 for (const VarDecl *VD : Data.PrivateLocals) {
4783 if (VD->getType()->isLValueReferenceType())
4784 Ty = CGF.getContext().getPointerType(Ty);
4785 if (isAllocatableDecl(VD))
4786 Ty = CGF.getContext().getPointerType(Ty);
4787 Address PrivatePtr = CGF.CreateMemTemp(
4788 CGF.getContext().getPointerType(Ty), ".local.ptr.addr");
4789 auto Result = UntiedLocalVars.insert(
4790 std::make_pair(VD, std::make_pair(PrivatePtr, Address::invalid())));
4791 // If key exists update in place.
4792 if (Result.second == false)
4793 *Result.first = std::make_pair(
4794 VD, std::make_pair(PrivatePtr, Address::invalid()));
4795 CallArgs.push_back(PrivatePtr.getPointer());
4796 ParamTypes.push_back(PrivatePtr.getType());
4797 }
4798 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
4799 ParamTypes, /*isVarArg=*/false);
4800 CopyFn = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4801 CopyFn, CopyFnTy->getPointerTo());
4802 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
4803 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
4804 for (const auto &Pair : LastprivateDstsOrigs) {
4805 const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
4806 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD),
4807 /*RefersToEnclosingVariableOrCapture=*/
4808 CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr,
4809 Pair.second->getType(), VK_LValue,
4810 Pair.second->getExprLoc());
4811 Scope.addPrivate(Pair.first, CGF.EmitLValue(&DRE).getAddress(CGF));
4812 }
4813 for (const auto &Pair : PrivatePtrs) {
4814 Address Replacement = Address(
4815 CGF.Builder.CreateLoad(Pair.second),
4816 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
4817 CGF.getContext().getDeclAlign(Pair.first));
4818 Scope.addPrivate(Pair.first, Replacement);
4819 if (auto *DI = CGF.getDebugInfo())
4820 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
4821 (void)DI->EmitDeclareOfAutoVariable(
4822 Pair.first, Pair.second.getPointer(), CGF.Builder,
4823 /*UsePointerValue*/ true);
4824 }
4825 // Adjust mapping for internal locals by mapping actual memory instead of
4826 // a pointer to this memory.
4827 for (auto &Pair : UntiedLocalVars) {
4828 QualType VDType = Pair.first->getType().getNonReferenceType();
4829 if (isAllocatableDecl(Pair.first)) {
4830 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
4831 Address Replacement(
4832 Ptr,
4833 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(VDType)),
4834 CGF.getPointerAlign());
4835 Pair.second.first = Replacement;
4836 Ptr = CGF.Builder.CreateLoad(Replacement);
4837 Replacement = Address(Ptr, CGF.ConvertTypeForMem(VDType),
4838 CGF.getContext().getDeclAlign(Pair.first));
4839 Pair.second.second = Replacement;
4840 } else {
4841 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
4842 Address Replacement(Ptr, CGF.ConvertTypeForMem(VDType),
4843 CGF.getContext().getDeclAlign(Pair.first));
4844 Pair.second.first = Replacement;
4845 }
4846 }
4847 }
4848 if (Data.Reductions) {
4849 OMPPrivateScope FirstprivateScope(CGF);
4850 for (const auto &Pair : FirstprivatePtrs) {
4851 Address Replacement(
4852 CGF.Builder.CreateLoad(Pair.second),
4853 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
4854 CGF.getContext().getDeclAlign(Pair.first));
4855 FirstprivateScope.addPrivate(Pair.first, Replacement);
4856 }
4857 (void)FirstprivateScope.Privatize();
4858 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
4859 ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionVars,
4860 Data.ReductionCopies, Data.ReductionOps);
4861 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
4862 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
4863 for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
4864 RedCG.emitSharedOrigLValue(CGF, Cnt);
4865 RedCG.emitAggregateType(CGF, Cnt);
4866 // FIXME: This must removed once the runtime library is fixed.
4867 // Emit required threadprivate variables for
4868 // initializer/combiner/finalizer.
4869 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
4870