clang  9.0.0svn
CGStmtOpenMP.cpp
Go to the documentation of this file.
1 //===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This contains code to emit OpenMP nodes as LLVM code.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CGCleanup.h"
14 #include "CGOpenMPRuntime.h"
15 #include "CodeGenFunction.h"
16 #include "CodeGenModule.h"
17 #include "TargetInfo.h"
18 #include "clang/AST/Stmt.h"
19 #include "clang/AST/StmtOpenMP.h"
20 #include "clang/AST/DeclOpenMP.h"
21 using namespace clang;
22 using namespace CodeGen;
23 
24 namespace {
25 /// Lexical scope for OpenMP executable constructs, that handles correct codegen
26 /// for captured expressions.
27 class OMPLexicalScope : public CodeGenFunction::LexicalScope {
28  void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
29  for (const auto *C : S.clauses()) {
30  if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
31  if (const auto *PreInit =
32  cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
33  for (const auto *I : PreInit->decls()) {
34  if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
35  CGF.EmitVarDecl(cast<VarDecl>(*I));
36  } else {
38  CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
39  CGF.EmitAutoVarCleanups(Emission);
40  }
41  }
42  }
43  }
44  }
45  }
46  CodeGenFunction::OMPPrivateScope InlinedShareds;
47 
48  static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
49  return CGF.LambdaCaptureFields.lookup(VD) ||
50  (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
51  (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl));
52  }
53 
54 public:
55  OMPLexicalScope(
57  const llvm::Optional<OpenMPDirectiveKind> CapturedRegion = llvm::None,
58  const bool EmitPreInitStmt = true)
59  : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
60  InlinedShareds(CGF) {
61  if (EmitPreInitStmt)
62  emitPreInitStmt(CGF, S);
63  if (!CapturedRegion.hasValue())
64  return;
65  assert(S.hasAssociatedStmt() &&
66  "Expected associated statement for inlined directive.");
67  const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
68  for (const auto &C : CS->captures()) {
69  if (C.capturesVariable() || C.capturesVariableByCopy()) {
70  auto *VD = C.getCapturedVar();
71  assert(VD == VD->getCanonicalDecl() &&
72  "Canonical decl must be captured.");
73  DeclRefExpr DRE(
74  CGF.getContext(), const_cast<VarDecl *>(VD),
75  isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
76  InlinedShareds.isGlobalVarCaptured(VD)),
77  VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
78  InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
79  return CGF.EmitLValue(&DRE).getAddress();
80  });
81  }
82  }
83  (void)InlinedShareds.Privatize();
84  }
85 };
86 
87 /// Lexical scope for OpenMP parallel construct, that handles correct codegen
88 /// for captured expressions.
89 class OMPParallelScope final : public OMPLexicalScope {
90  bool EmitPreInitStmt(const OMPExecutableDirective &S) {
92  return !(isOpenMPTargetExecutionDirective(Kind) ||
95  }
96 
97 public:
98  OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
99  : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
100  EmitPreInitStmt(S)) {}
101 };
102 
103 /// Lexical scope for OpenMP teams construct, that handles correct codegen
104 /// for captured expressions.
105 class OMPTeamsScope final : public OMPLexicalScope {
106  bool EmitPreInitStmt(const OMPExecutableDirective &S) {
108  return !isOpenMPTargetExecutionDirective(Kind) &&
110  }
111 
112 public:
113  OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
114  : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
115  EmitPreInitStmt(S)) {}
116 };
117 
118 /// Private scope for OpenMP loop-based directives, that supports capturing
119 /// of used expression from loop statement.
120 class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
121  void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) {
122  CodeGenFunction::OMPMapVars PreCondVars;
123  for (const auto *E : S.counters()) {
124  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
125  (void)PreCondVars.setVarAddr(
126  CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
127  }
128  (void)PreCondVars.apply(CGF);
129  if (const auto *PreInits = cast_or_null<DeclStmt>(S.getPreInits())) {
130  for (const auto *I : PreInits->decls())
131  CGF.EmitVarDecl(cast<VarDecl>(*I));
132  }
133  PreCondVars.restore(CGF);
134  }
135 
136 public:
137  OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S)
138  : CodeGenFunction::RunCleanupsScope(CGF) {
139  emitPreInitStmt(CGF, S);
140  }
141 };
142 
143 class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
144  CodeGenFunction::OMPPrivateScope InlinedShareds;
145 
146  static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
147  return CGF.LambdaCaptureFields.lookup(VD) ||
148  (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
149  (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) &&
150  cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
151  }
152 
153 public:
154  OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
155  : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
156  InlinedShareds(CGF) {
157  for (const auto *C : S.clauses()) {
158  if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
159  if (const auto *PreInit =
160  cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
161  for (const auto *I : PreInit->decls()) {
162  if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
163  CGF.EmitVarDecl(cast<VarDecl>(*I));
164  } else {
166  CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
167  CGF.EmitAutoVarCleanups(Emission);
168  }
169  }
170  }
171  } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
172  for (const Expr *E : UDP->varlists()) {
173  const Decl *D = cast<DeclRefExpr>(E)->getDecl();
174  if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
175  CGF.EmitVarDecl(*OED);
176  }
177  }
178  }
180  CGF.EmitOMPPrivateClause(S, InlinedShareds);
181  if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
182  if (const Expr *E = TG->getReductionRef())
183  CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
184  }
185  const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
186  while (CS) {
187  for (auto &C : CS->captures()) {
188  if (C.capturesVariable() || C.capturesVariableByCopy()) {
189  auto *VD = C.getCapturedVar();
190  assert(VD == VD->getCanonicalDecl() &&
191  "Canonical decl must be captured.");
192  DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
193  isCapturedVar(CGF, VD) ||
194  (CGF.CapturedStmtInfo &&
195  InlinedShareds.isGlobalVarCaptured(VD)),
197  C.getLocation());
198  InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
199  return CGF.EmitLValue(&DRE).getAddress();
200  });
201  }
202  }
203  CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
204  }
205  (void)InlinedShareds.Privatize();
206  }
207 };
208 
209 } // namespace
210 
212  const OMPExecutableDirective &S,
213  const RegionCodeGenTy &CodeGen);
214 
216  if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
217  if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
218  OrigVD = OrigVD->getCanonicalDecl();
219  bool IsCaptured =
220  LambdaCaptureFields.lookup(OrigVD) ||
221  (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
222  (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
223  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), IsCaptured,
224  OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
225  return EmitLValue(&DRE);
226  }
227  }
228  return EmitLValue(E);
229 }
230 
232  ASTContext &C = getContext();
233  llvm::Value *Size = nullptr;
234  auto SizeInChars = C.getTypeSizeInChars(Ty);
235  if (SizeInChars.isZero()) {
236  // getTypeSizeInChars() returns 0 for a VLA.
237  while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) {
238  VlaSizePair VlaSize = getVLASize(VAT);
239  Ty = VlaSize.Type;
240  Size = Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts)
241  : VlaSize.NumElts;
242  }
243  SizeInChars = C.getTypeSizeInChars(Ty);
244  if (SizeInChars.isZero())
245  return llvm::ConstantInt::get(SizeTy, /*V=*/0);
246  return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
247  }
248  return CGM.getSize(SizeInChars);
249 }
250 
252  const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
253  const RecordDecl *RD = S.getCapturedRecordDecl();
254  auto CurField = RD->field_begin();
255  auto CurCap = S.captures().begin();
257  E = S.capture_init_end();
258  I != E; ++I, ++CurField, ++CurCap) {
259  if (CurField->hasCapturedVLAType()) {
260  const VariableArrayType *VAT = CurField->getCapturedVLAType();
261  llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
262  CapturedVars.push_back(Val);
263  } else if (CurCap->capturesThis()) {
264  CapturedVars.push_back(CXXThisValue);
265  } else if (CurCap->capturesVariableByCopy()) {
266  llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
267 
268  // If the field is not a pointer, we need to save the actual value
269  // and load it as a void pointer.
270  if (!CurField->getType()->isAnyPointerType()) {
271  ASTContext &Ctx = getContext();
272  Address DstAddr = CreateMemTemp(
273  Ctx.getUIntPtrType(),
274  Twine(CurCap->getCapturedVar()->getName(), ".casted"));
275  LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
276 
277  llvm::Value *SrcAddrVal = EmitScalarConversion(
278  DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
279  Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
280  LValue SrcLV =
281  MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
282 
283  // Store the value using the source type pointer.
284  EmitStoreThroughLValue(RValue::get(CV), SrcLV);
285 
286  // Load the value using the destination type pointer.
287  CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
288  }
289  CapturedVars.push_back(CV);
290  } else {
291  assert(CurCap->capturesVariable() && "Expected capture by reference.");
292  CapturedVars.push_back(EmitLValue(*I).getAddress().getPointer());
293  }
294  }
295 }
296 
298  QualType DstType, StringRef Name,
299  LValue AddrLV) {
300  ASTContext &Ctx = CGF.getContext();
301 
302  llvm::Value *CastedPtr = CGF.EmitScalarConversion(
303  AddrLV.getAddress().getPointer(), Ctx.getUIntPtrType(),
304  Ctx.getPointerType(DstType), Loc);
305  Address TmpAddr =
306  CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType))
307  .getAddress();
308  return TmpAddr;
309 }
310 
312  if (T->isLValueReferenceType())
313  return C.getLValueReferenceType(
315  /*SpelledAsLValue=*/false);
316  if (T->isPointerType())
318  if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
319  if (const auto *VLA = dyn_cast<VariableArrayType>(A))
320  return getCanonicalParamType(C, VLA->getElementType());
321  if (!A->isVariablyModifiedType())
322  return C.getCanonicalType(T);
323  }
324  return C.getCanonicalParamType(T);
325 }
326 
327 namespace {
328  /// Contains required data for proper outlined function codegen.
329  struct FunctionOptions {
330  /// Captured statement for which the function is generated.
331  const CapturedStmt *S = nullptr;
332  /// true if cast to/from UIntPtr is required for variables captured by
333  /// value.
334  const bool UIntPtrCastRequired = true;
335  /// true if only casted arguments must be registered as local args or VLA
336  /// sizes.
337  const bool RegisterCastedArgsOnly = false;
338  /// Name of the generated function.
339  const StringRef FunctionName;
340  explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
341  bool RegisterCastedArgsOnly,
342  StringRef FunctionName)
343  : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
344  RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
345  FunctionName(FunctionName) {}
346  };
347 }
348 
349 static llvm::Function *emitOutlinedFunctionPrologue(
350  CodeGenFunction &CGF, FunctionArgList &Args,
351  llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
352  &LocalAddrs,
353  llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
354  &VLASizes,
355  llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
356  const CapturedDecl *CD = FO.S->getCapturedDecl();
357  const RecordDecl *RD = FO.S->getCapturedRecordDecl();
358  assert(CD->hasBody() && "missing CapturedDecl body");
359 
360  CXXThisValue = nullptr;
361  // Build the argument list.
362  CodeGenModule &CGM = CGF.CGM;
363  ASTContext &Ctx = CGM.getContext();
364  FunctionArgList TargetArgs;
365  Args.append(CD->param_begin(),
366  std::next(CD->param_begin(), CD->getContextParamPosition()));
367  TargetArgs.append(
368  CD->param_begin(),
369  std::next(CD->param_begin(), CD->getContextParamPosition()));
370  auto I = FO.S->captures().begin();
371  FunctionDecl *DebugFunctionDecl = nullptr;
372  if (!FO.UIntPtrCastRequired) {
374  QualType FunctionTy = Ctx.getFunctionType(Ctx.VoidTy, llvm::None, EPI);
375  DebugFunctionDecl = FunctionDecl::Create(
376  Ctx, Ctx.getTranslationUnitDecl(), FO.S->getBeginLoc(),
377  SourceLocation(), DeclarationName(), FunctionTy,
378  Ctx.getTrivialTypeSourceInfo(FunctionTy), SC_Static,
379  /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/false);
380  }
381  for (const FieldDecl *FD : RD->fields()) {
382  QualType ArgType = FD->getType();
383  IdentifierInfo *II = nullptr;
384  VarDecl *CapVar = nullptr;
385 
386  // If this is a capture by copy and the type is not a pointer, the outlined
387  // function argument type should be uintptr and the value properly casted to
388  // uintptr. This is necessary given that the runtime library is only able to
389  // deal with pointers. We can pass in the same way the VLA type sizes to the
390  // outlined function.
391  if (FO.UIntPtrCastRequired &&
392  ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
393  I->capturesVariableArrayType()))
394  ArgType = Ctx.getUIntPtrType();
395 
396  if (I->capturesVariable() || I->capturesVariableByCopy()) {
397  CapVar = I->getCapturedVar();
398  II = CapVar->getIdentifier();
399  } else if (I->capturesThis()) {
400  II = &Ctx.Idents.get("this");
401  } else {
402  assert(I->capturesVariableArrayType());
403  II = &Ctx.Idents.get("vla");
404  }
405  if (ArgType->isVariablyModifiedType())
406  ArgType = getCanonicalParamType(Ctx, ArgType);
407  VarDecl *Arg;
408  if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
409  Arg = ParmVarDecl::Create(
410  Ctx, DebugFunctionDecl,
411  CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),
412  CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
413  /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
414  } else {
415  Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
416  II, ArgType, ImplicitParamDecl::Other);
417  }
418  Args.emplace_back(Arg);
419  // Do not cast arguments if we emit function with non-original types.
420  TargetArgs.emplace_back(
421  FO.UIntPtrCastRequired
422  ? Arg
423  : CGM.getOpenMPRuntime().translateParameter(FD, Arg));
424  ++I;
425  }
426  Args.append(
427  std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
428  CD->param_end());
429  TargetArgs.append(
430  std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
431  CD->param_end());
432 
433  // Create the function declaration.
434  const CGFunctionInfo &FuncInfo =
435  CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, TargetArgs);
436  llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
437 
438  auto *F =
440  FO.FunctionName, &CGM.getModule());
441  CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
442  if (CD->isNothrow())
443  F->setDoesNotThrow();
444  F->setDoesNotRecurse();
445 
446  // Generate the function.
447  CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
448  FO.S->getBeginLoc(), CD->getBody()->getBeginLoc());
449  unsigned Cnt = CD->getContextParamPosition();
450  I = FO.S->captures().begin();
451  for (const FieldDecl *FD : RD->fields()) {
452  // Do not map arguments if we emit function with non-original types.
453  Address LocalAddr(Address::invalid());
454  if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
455  LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
456  TargetArgs[Cnt]);
457  } else {
458  LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
459  }
460  // If we are capturing a pointer by copy we don't need to do anything, just
461  // use the value that we get from the arguments.
462  if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
463  const VarDecl *CurVD = I->getCapturedVar();
464  if (!FO.RegisterCastedArgsOnly)
465  LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
466  ++Cnt;
467  ++I;
468  continue;
469  }
470 
471  LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
473  if (FD->hasCapturedVLAType()) {
474  if (FO.UIntPtrCastRequired) {
475  ArgLVal = CGF.MakeAddrLValue(
476  castValueFromUintptr(CGF, I->getLocation(), FD->getType(),
477  Args[Cnt]->getName(), ArgLVal),
479  }
480  llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
481  const VariableArrayType *VAT = FD->getCapturedVLAType();
482  VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg);
483  } else if (I->capturesVariable()) {
484  const VarDecl *Var = I->getCapturedVar();
485  QualType VarTy = Var->getType();
486  Address ArgAddr = ArgLVal.getAddress();
487  if (ArgLVal.getType()->isLValueReferenceType()) {
488  ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
489  } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) {
490  assert(ArgLVal.getType()->isPointerType());
491  ArgAddr = CGF.EmitLoadOfPointer(
492  ArgAddr, ArgLVal.getType()->castAs<PointerType>());
493  }
494  if (!FO.RegisterCastedArgsOnly) {
495  LocalAddrs.insert(
496  {Args[Cnt],
497  {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}});
498  }
499  } else if (I->capturesVariableByCopy()) {
500  assert(!FD->getType()->isAnyPointerType() &&
501  "Not expecting a captured pointer.");
502  const VarDecl *Var = I->getCapturedVar();
503  LocalAddrs.insert({Args[Cnt],
504  {Var, FO.UIntPtrCastRequired
506  CGF, I->getLocation(), FD->getType(),
507  Args[Cnt]->getName(), ArgLVal)
508  : ArgLVal.getAddress()}});
509  } else {
510  // If 'this' is captured, load it into CXXThisValue.
511  assert(I->capturesThis());
512  CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
513  LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress()}});
514  }
515  ++Cnt;
516  ++I;
517  }
518 
519  return F;
520 }
521 
522 llvm::Function *
524  assert(
525  CapturedStmtInfo &&
526  "CapturedStmtInfo should be set when generating the captured function");
527  const CapturedDecl *CD = S.getCapturedDecl();
528  // Build the argument list.
529  bool NeedWrapperFunction =
530  getDebugInfo() &&
531  CGM.getCodeGenOpts().getDebugInfo() >= codegenoptions::LimitedDebugInfo;
532  FunctionArgList Args;
533  llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
534  llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
535  SmallString<256> Buffer;
536  llvm::raw_svector_ostream Out(Buffer);
537  Out << CapturedStmtInfo->getHelperName();
538  if (NeedWrapperFunction)
539  Out << "_debug__";
540  FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
541  Out.str());
542  llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs,
543  VLASizes, CXXThisValue, FO);
544  CodeGenFunction::OMPPrivateScope LocalScope(*this);
545  for (const auto &LocalAddrPair : LocalAddrs) {
546  if (LocalAddrPair.second.first) {
547  LocalScope.addPrivate(LocalAddrPair.second.first, [&LocalAddrPair]() {
548  return LocalAddrPair.second.second;
549  });
550  }
551  }
552  (void)LocalScope.Privatize();
553  for (const auto &VLASizePair : VLASizes)
554  VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
555  PGO.assignRegionCounters(GlobalDecl(CD), F);
556  CapturedStmtInfo->EmitBody(*this, CD->getBody());
557  (void)LocalScope.ForceCleanup();
558  FinishFunction(CD->getBodyRBrace());
559  if (!NeedWrapperFunction)
560  return F;
561 
562  FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
563  /*RegisterCastedArgsOnly=*/true,
564  CapturedStmtInfo->getHelperName());
565  CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
566  WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
567  Args.clear();
568  LocalAddrs.clear();
569  VLASizes.clear();
570  llvm::Function *WrapperF =
571  emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
572  WrapperCGF.CXXThisValue, WrapperFO);
574  for (const auto *Arg : Args) {
576  auto I = LocalAddrs.find(Arg);
577  if (I != LocalAddrs.end()) {
578  LValue LV = WrapperCGF.MakeAddrLValue(
579  I->second.second,
580  I->second.first ? I->second.first->getType() : Arg->getType(),
582  CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
583  } else {
584  auto EI = VLASizes.find(Arg);
585  if (EI != VLASizes.end()) {
586  CallArg = EI->second.second;
587  } else {
588  LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
589  Arg->getType(),
591  CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
592  }
593  }
594  CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
595  }
596  CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.getBeginLoc(),
597  F, CallArgs);
598  WrapperCGF.FinishFunction();
599  return WrapperF;
600 }
601 
602 //===----------------------------------------------------------------------===//
603 // OpenMP Directive Emission
604 //===----------------------------------------------------------------------===//
606  Address DestAddr, Address SrcAddr, QualType OriginalType,
607  const llvm::function_ref<void(Address, Address)> CopyGen) {
608  // Perform element-by-element initialization.
609  QualType ElementTy;
610 
611  // Drill down to the base element type on both arrays.
612  const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
613  llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
614  SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
615 
616  llvm::Value *SrcBegin = SrcAddr.getPointer();
617  llvm::Value *DestBegin = DestAddr.getPointer();
618  // Cast from pointer to array type to pointer to single element.
619  llvm::Value *DestEnd = Builder.CreateGEP(DestBegin, NumElements);
620  // The basic structure here is a while-do loop.
621  llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body");
622  llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done");
623  llvm::Value *IsEmpty =
624  Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
625  Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
626 
627  // Enter the loop body, making that address the current address.
628  llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
629  EmitBlock(BodyBB);
630 
631  CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
632 
633  llvm::PHINode *SrcElementPHI =
634  Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
635  SrcElementPHI->addIncoming(SrcBegin, EntryBB);
636  Address SrcElementCurrent =
637  Address(SrcElementPHI,
638  SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
639 
640  llvm::PHINode *DestElementPHI =
641  Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
642  DestElementPHI->addIncoming(DestBegin, EntryBB);
643  Address DestElementCurrent =
644  Address(DestElementPHI,
645  DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
646 
647  // Emit copy.
648  CopyGen(DestElementCurrent, SrcElementCurrent);
649 
650  // Shift the address forward by one element.
651  llvm::Value *DestElementNext = Builder.CreateConstGEP1_32(
652  DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
653  llvm::Value *SrcElementNext = Builder.CreateConstGEP1_32(
654  SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
655  // Check whether we've reached the end.
656  llvm::Value *Done =
657  Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
658  Builder.CreateCondBr(Done, DoneBB, BodyBB);
659  DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
660  SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
661 
662  // Done.
663  EmitBlock(DoneBB, /*IsFinished=*/true);
664 }
665 
666 void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
667  Address SrcAddr, const VarDecl *DestVD,
668  const VarDecl *SrcVD, const Expr *Copy) {
669  if (OriginalType->isArrayType()) {
670  const auto *BO = dyn_cast<BinaryOperator>(Copy);
671  if (BO && BO->getOpcode() == BO_Assign) {
672  // Perform simple memcpy for simple copying.
673  LValue Dest = MakeAddrLValue(DestAddr, OriginalType);
674  LValue Src = MakeAddrLValue(SrcAddr, OriginalType);
675  EmitAggregateAssign(Dest, Src, OriginalType);
676  } else {
677  // For arrays with complex element types perform element by element
678  // copying.
679  EmitOMPAggregateAssign(
680  DestAddr, SrcAddr, OriginalType,
681  [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
682  // Working with the single array element, so have to remap
683  // destination and source variables to corresponding array
684  // elements.
686  Remap.addPrivate(DestVD, [DestElement]() { return DestElement; });
687  Remap.addPrivate(SrcVD, [SrcElement]() { return SrcElement; });
688  (void)Remap.Privatize();
689  EmitIgnoredExpr(Copy);
690  });
691  }
692  } else {
693  // Remap pseudo source variable to private copy.
695  Remap.addPrivate(SrcVD, [SrcAddr]() { return SrcAddr; });
696  Remap.addPrivate(DestVD, [DestAddr]() { return DestAddr; });
697  (void)Remap.Privatize();
698  // Emit copying of the whole variable.
699  EmitIgnoredExpr(Copy);
700  }
701 }
702 
704  OMPPrivateScope &PrivateScope) {
705  if (!HaveInsertPoint())
706  return false;
707  bool DeviceConstTarget =
708  getLangOpts().OpenMPIsDevice &&
710  bool FirstprivateIsLastprivate = false;
711  llvm::DenseSet<const VarDecl *> Lastprivates;
712  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
713  for (const auto *D : C->varlists())
714  Lastprivates.insert(
715  cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl());
716  }
717  llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
719  getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind());
720  // Force emission of the firstprivate copy if the directive does not emit
721  // outlined function, like omp for, omp simd, omp distribute etc.
722  bool MustEmitFirstprivateCopy =
723  CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
724  for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
725  auto IRef = C->varlist_begin();
726  auto InitsRef = C->inits().begin();
727  for (const Expr *IInit : C->private_copies()) {
728  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
729  bool ThisFirstprivateIsLastprivate =
730  Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
731  const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
732  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
733  if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
734  !FD->getType()->isReferenceType() &&
735  (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
736  EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
737  ++IRef;
738  ++InitsRef;
739  continue;
740  }
741  // Do not emit copy for firstprivate constant variables in target regions,
742  // captured by reference.
743  if (DeviceConstTarget && OrigVD->getType().isConstant(getContext()) &&
744  FD && FD->getType()->isReferenceType() &&
745  (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
746  (void)CGM.getOpenMPRuntime().registerTargetFirstprivateCopy(*this,
747  OrigVD);
748  ++IRef;
749  ++InitsRef;
750  continue;
751  }
752  FirstprivateIsLastprivate =
753  FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
754  if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
755  const auto *VDInit =
756  cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
757  bool IsRegistered;
758  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
759  /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
760  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
761  LValue OriginalLVal;
762  if (!FD) {
763  // Check if the firstprivate variable is just a constant value.
764  ConstantEmission CE = tryEmitAsConstant(&DRE);
765  if (CE && !CE.isReference()) {
766  // Constant value, no need to create a copy.
767  ++IRef;
768  ++InitsRef;
769  continue;
770  }
771  if (CE && CE.isReference()) {
772  OriginalLVal = CE.getReferenceLValue(*this, &DRE);
773  } else {
774  assert(!CE && "Expected non-constant firstprivate.");
775  OriginalLVal = EmitLValue(&DRE);
776  }
777  } else {
778  OriginalLVal = EmitLValue(&DRE);
779  }
780  QualType Type = VD->getType();
781  if (Type->isArrayType()) {
782  // Emit VarDecl with copy init for arrays.
783  // Get the address of the original variable captured in current
784  // captured region.
785  IsRegistered = PrivateScope.addPrivate(
786  OrigVD, [this, VD, Type, OriginalLVal, VDInit]() {
787  AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
788  const Expr *Init = VD->getInit();
789  if (!isa<CXXConstructExpr>(Init) ||
790  isTrivialInitializer(Init)) {
791  // Perform simple memcpy.
792  LValue Dest =
793  MakeAddrLValue(Emission.getAllocatedAddress(), Type);
794  EmitAggregateAssign(Dest, OriginalLVal, Type);
795  } else {
796  EmitOMPAggregateAssign(
797  Emission.getAllocatedAddress(), OriginalLVal.getAddress(),
798  Type,
799  [this, VDInit, Init](Address DestElement,
800  Address SrcElement) {
801  // Clean up any temporaries needed by the
802  // initialization.
803  RunCleanupsScope InitScope(*this);
804  // Emit initialization for single element.
805  setAddrOfLocalVar(VDInit, SrcElement);
806  EmitAnyExprToMem(Init, DestElement,
807  Init->getType().getQualifiers(),
808  /*IsInitializer*/ false);
809  LocalDeclMap.erase(VDInit);
810  });
811  }
812  EmitAutoVarCleanups(Emission);
813  return Emission.getAllocatedAddress();
814  });
815  } else {
816  Address OriginalAddr = OriginalLVal.getAddress();
817  IsRegistered = PrivateScope.addPrivate(
818  OrigVD, [this, VDInit, OriginalAddr, VD]() {
819  // Emit private VarDecl with copy init.
820  // Remap temp VDInit variable to the address of the original
821  // variable (for proper handling of captured global variables).
822  setAddrOfLocalVar(VDInit, OriginalAddr);
823  EmitDecl(*VD);
824  LocalDeclMap.erase(VDInit);
825  return GetAddrOfLocalVar(VD);
826  });
827  }
828  assert(IsRegistered &&
829  "firstprivate var already registered as private");
830  // Silence the warning about unused variable.
831  (void)IsRegistered;
832  }
833  ++IRef;
834  ++InitsRef;
835  }
836  }
837  return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
838 }
839 
841  const OMPExecutableDirective &D,
842  CodeGenFunction::OMPPrivateScope &PrivateScope) {
843  if (!HaveInsertPoint())
844  return;
845  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
846  for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
847  auto IRef = C->varlist_begin();
848  for (const Expr *IInit : C->private_copies()) {
849  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
850  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
851  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
852  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
853  // Emit private VarDecl with copy init.
854  EmitDecl(*VD);
855  return GetAddrOfLocalVar(VD);
856  });
857  assert(IsRegistered && "private var already registered as private");
858  // Silence the warning about unused variable.
859  (void)IsRegistered;
860  }
861  ++IRef;
862  }
863  }
864 }
865 
867  if (!HaveInsertPoint())
868  return false;
869  // threadprivate_var1 = master_threadprivate_var1;
870  // operator=(threadprivate_var2, master_threadprivate_var2);
871  // ...
872  // __kmpc_barrier(&loc, global_tid);
874  llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
875  for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
876  auto IRef = C->varlist_begin();
877  auto ISrcRef = C->source_exprs().begin();
878  auto IDestRef = C->destination_exprs().begin();
879  for (const Expr *AssignOp : C->assignment_ops()) {
880  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
881  QualType Type = VD->getType();
882  if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
883  // Get the address of the master variable. If we are emitting code with
884  // TLS support, the address is passed from the master as field in the
885  // captured declaration.
886  Address MasterAddr = Address::invalid();
887  if (getLangOpts().OpenMPUseTLS &&
888  getContext().getTargetInfo().isTLSSupported()) {
889  assert(CapturedStmtInfo->lookup(VD) &&
890  "Copyin threadprivates should have been captured!");
891  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), true,
892  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
893  MasterAddr = EmitLValue(&DRE).getAddress();
894  LocalDeclMap.erase(VD);
895  } else {
896  MasterAddr =
897  Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
898  : CGM.GetAddrOfGlobal(VD),
899  getContext().getDeclAlign(VD));
900  }
901  // Get the address of the threadprivate variable.
902  Address PrivateAddr = EmitLValue(*IRef).getAddress();
903  if (CopiedVars.size() == 1) {
904  // At first check if current thread is a master thread. If it is, no
905  // need to copy data.
906  CopyBegin = createBasicBlock("copyin.not.master");
907  CopyEnd = createBasicBlock("copyin.not.master.end");
908  Builder.CreateCondBr(
909  Builder.CreateICmpNE(
910  Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy),
911  Builder.CreatePtrToInt(PrivateAddr.getPointer(),
912  CGM.IntPtrTy)),
913  CopyBegin, CopyEnd);
914  EmitBlock(CopyBegin);
915  }
916  const auto *SrcVD =
917  cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
918  const auto *DestVD =
919  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
920  EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
921  }
922  ++IRef;
923  ++ISrcRef;
924  ++IDestRef;
925  }
926  }
927  if (CopyEnd) {
928  // Exit out of copying procedure for non-master thread.
929  EmitBlock(CopyEnd, /*IsFinished=*/true);
930  return true;
931  }
932  return false;
933 }
934 
936  const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
937  if (!HaveInsertPoint())
938  return false;
939  bool HasAtLeastOneLastprivate = false;
942  const auto *LoopDirective = cast<OMPLoopDirective>(&D);
943  for (const Expr *C : LoopDirective->counters()) {
944  SIMDLCVs.insert(
945  cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
946  }
947  }
948  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
949  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
950  HasAtLeastOneLastprivate = true;
952  !getLangOpts().OpenMPSimd)
953  break;
954  auto IRef = C->varlist_begin();
955  auto IDestRef = C->destination_exprs().begin();
956  for (const Expr *IInit : C->private_copies()) {
957  // Keep the address of the original variable for future update at the end
958  // of the loop.
959  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
960  // Taskloops do not require additional initialization, it is done in
961  // runtime support library.
962  if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
963  const auto *DestVD =
964  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
965  PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() {
966  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
967  /*RefersToEnclosingVariableOrCapture=*/
968  CapturedStmtInfo->lookup(OrigVD) != nullptr,
969  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
970  return EmitLValue(&DRE).getAddress();
971  });
972  // Check if the variable is also a firstprivate: in this case IInit is
973  // not generated. Initialization of this variable will happen in codegen
974  // for 'firstprivate' clause.
975  if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
976  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
977  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
978  // Emit private VarDecl with copy init.
979  EmitDecl(*VD);
980  return GetAddrOfLocalVar(VD);
981  });
982  assert(IsRegistered &&
983  "lastprivate var already registered as private");
984  (void)IsRegistered;
985  }
986  }
987  ++IRef;
988  ++IDestRef;
989  }
990  }
991  return HasAtLeastOneLastprivate;
992 }
993 
995  const OMPExecutableDirective &D, bool NoFinals,
996  llvm::Value *IsLastIterCond) {
997  if (!HaveInsertPoint())
998  return;
999  // Emit following code:
1000  // if (<IsLastIterCond>) {
1001  // orig_var1 = private_orig_var1;
1002  // ...
1003  // orig_varn = private_orig_varn;
1004  // }
1005  llvm::BasicBlock *ThenBB = nullptr;
1006  llvm::BasicBlock *DoneBB = nullptr;
1007  if (IsLastIterCond) {
1008  ThenBB = createBasicBlock(".omp.lastprivate.then");
1009  DoneBB = createBasicBlock(".omp.lastprivate.done");
1010  Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1011  EmitBlock(ThenBB);
1012  }
1013  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1014  llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1015  if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1016  auto IC = LoopDirective->counters().begin();
1017  for (const Expr *F : LoopDirective->finals()) {
1018  const auto *D =
1019  cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1020  if (NoFinals)
1021  AlreadyEmittedVars.insert(D);
1022  else
1023  LoopCountersAndUpdates[D] = F;
1024  ++IC;
1025  }
1026  }
1027  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1028  auto IRef = C->varlist_begin();
1029  auto ISrcRef = C->source_exprs().begin();
1030  auto IDestRef = C->destination_exprs().begin();
1031  for (const Expr *AssignOp : C->assignment_ops()) {
1032  const auto *PrivateVD =
1033  cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1034  QualType Type = PrivateVD->getType();
1035  const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1036  if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1037  // If lastprivate variable is a loop control variable for loop-based
1038  // directive, update its value before copyin back to original
1039  // variable.
1040  if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1041  EmitIgnoredExpr(FinalExpr);
1042  const auto *SrcVD =
1043  cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1044  const auto *DestVD =
1045  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1046  // Get the address of the original variable.
1047  Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1048  // Get the address of the private variable.
1049  Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
1050  if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1051  PrivateAddr =
1052  Address(Builder.CreateLoad(PrivateAddr),
1053  getNaturalTypeAlignment(RefTy->getPointeeType()));
1054  EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1055  }
1056  ++IRef;
1057  ++ISrcRef;
1058  ++IDestRef;
1059  }
1060  if (const Expr *PostUpdate = C->getPostUpdateExpr())
1061  EmitIgnoredExpr(PostUpdate);
1062  }
1063  if (IsLastIterCond)
1064  EmitBlock(DoneBB, /*IsFinished=*/true);
1065 }
1066 
1068  const OMPExecutableDirective &D,
1069  CodeGenFunction::OMPPrivateScope &PrivateScope) {
1070  if (!HaveInsertPoint())
1071  return;
1074  SmallVector<const Expr *, 4> ReductionOps;
1077  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1078  auto IPriv = C->privates().begin();
1079  auto IRed = C->reduction_ops().begin();
1080  auto ILHS = C->lhs_exprs().begin();
1081  auto IRHS = C->rhs_exprs().begin();
1082  for (const Expr *Ref : C->varlists()) {
1083  Shareds.emplace_back(Ref);
1084  Privates.emplace_back(*IPriv);
1085  ReductionOps.emplace_back(*IRed);
1086  LHSs.emplace_back(*ILHS);
1087  RHSs.emplace_back(*IRHS);
1088  std::advance(IPriv, 1);
1089  std::advance(IRed, 1);
1090  std::advance(ILHS, 1);
1091  std::advance(IRHS, 1);
1092  }
1093  }
1094  ReductionCodeGen RedCG(Shareds, Privates, ReductionOps);
1095  unsigned Count = 0;
1096  auto ILHS = LHSs.begin();
1097  auto IRHS = RHSs.begin();
1098  auto IPriv = Privates.begin();
1099  for (const Expr *IRef : Shareds) {
1100  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1101  // Emit private VarDecl with reduction init.
1102  RedCG.emitSharedLValue(*this, Count);
1103  RedCG.emitAggregateType(*this, Count);
1104  AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);
1105  RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
1106  RedCG.getSharedLValue(Count),
1107  [&Emission](CodeGenFunction &CGF) {
1108  CGF.EmitAutoVarInit(Emission);
1109  return true;
1110  });
1111  EmitAutoVarCleanups(Emission);
1112  Address BaseAddr = RedCG.adjustPrivateAddress(
1113  *this, Count, Emission.getAllocatedAddress());
1114  bool IsRegistered = PrivateScope.addPrivate(
1115  RedCG.getBaseDecl(Count), [BaseAddr]() { return BaseAddr; });
1116  assert(IsRegistered && "private var already registered as private");
1117  // Silence the warning about unused variable.
1118  (void)IsRegistered;
1119 
1120  const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1121  const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1122  QualType Type = PrivateVD->getType();
1123  bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1124  if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1125  // Store the address of the original variable associated with the LHS
1126  // implicit variable.
1127  PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
1128  return RedCG.getSharedLValue(Count).getAddress();
1129  });
1130  PrivateScope.addPrivate(
1131  RHSVD, [this, PrivateVD]() { return GetAddrOfLocalVar(PrivateVD); });
1132  } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1133  isa<ArraySubscriptExpr>(IRef)) {
1134  // Store the address of the original variable associated with the LHS
1135  // implicit variable.
1136  PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
1137  return RedCG.getSharedLValue(Count).getAddress();
1138  });
1139  PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() {
1140  return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD),
1141  ConvertTypeForMem(RHSVD->getType()),
1142  "rhs.begin");
1143  });
1144  } else {
1145  QualType Type = PrivateVD->getType();
1146  bool IsArray = getContext().getAsArrayType(Type) != nullptr;
1147  Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress();
1148  // Store the address of the original variable associated with the LHS
1149  // implicit variable.
1150  if (IsArray) {
1151  OriginalAddr = Builder.CreateElementBitCast(
1152  OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin");
1153  }
1154  PrivateScope.addPrivate(LHSVD, [OriginalAddr]() { return OriginalAddr; });
1155  PrivateScope.addPrivate(
1156  RHSVD, [this, PrivateVD, RHSVD, IsArray]() {
1157  return IsArray
1158  ? Builder.CreateElementBitCast(
1159  GetAddrOfLocalVar(PrivateVD),
1160  ConvertTypeForMem(RHSVD->getType()), "rhs.begin")
1161  : GetAddrOfLocalVar(PrivateVD);
1162  });
1163  }
1164  ++ILHS;
1165  ++IRHS;
1166  ++IPriv;
1167  ++Count;
1168  }
1169 }
1170 
1172  const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1173  if (!HaveInsertPoint())
1174  return;
1179  bool HasAtLeastOneReduction = false;
1180  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1181  HasAtLeastOneReduction = true;
1182  Privates.append(C->privates().begin(), C->privates().end());
1183  LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1184  RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1185  ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1186  }
1187  if (HasAtLeastOneReduction) {
1188  bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1190  ReductionKind == OMPD_simd;
1191  bool SimpleReduction = ReductionKind == OMPD_simd;
1192  // Emit nowait reduction if nowait clause is present or directive is a
1193  // parallel directive (it always has implicit barrier).
1194  CGM.getOpenMPRuntime().emitReduction(
1195  *this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1196  {WithNowait, SimpleReduction, ReductionKind});
1197  }
1198 }
1199 
1201  CodeGenFunction &CGF, const OMPExecutableDirective &D,
1202  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1203  if (!CGF.HaveInsertPoint())
1204  return;
1205  llvm::BasicBlock *DoneBB = nullptr;
1206  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1207  if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
1208  if (!DoneBB) {
1209  if (llvm::Value *Cond = CondGen(CGF)) {
1210  // If the first post-update expression is found, emit conditional
1211  // block if it was requested.
1212  llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1213  DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1214  CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1215  CGF.EmitBlock(ThenBB);
1216  }
1217  }
1218  CGF.EmitIgnoredExpr(PostUpdate);
1219  }
1220  }
1221  if (DoneBB)
1222  CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1223 }
1224 
1225 namespace {
1226 /// Codegen lambda for appending distribute lower and upper bounds to outlined
1227 /// parallel function. This is necessary for combined constructs such as
1228 /// 'distribute parallel for'
1229 typedef llvm::function_ref<void(CodeGenFunction &,
1230  const OMPExecutableDirective &,
1232  CodeGenBoundParametersTy;
1233 } // anonymous namespace
1234 
1236  CodeGenFunction &CGF, const OMPExecutableDirective &S,
1237  OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1238  const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1239  const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1240  llvm::Function *OutlinedFn =
1241  CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
1242  S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
1243  if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
1244  CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1245  llvm::Value *NumThreads =
1246  CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1247  /*IgnoreResultAssign=*/true);
1248  CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
1249  CGF, NumThreads, NumThreadsClause->getBeginLoc());
1250  }
1251  if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
1252  CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1253  CGF.CGM.getOpenMPRuntime().emitProcBindClause(
1254  CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1255  }
1256  const Expr *IfCond = nullptr;
1257  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1258  if (C->getNameModifier() == OMPD_unknown ||
1259  C->getNameModifier() == OMPD_parallel) {
1260  IfCond = C->getCondition();
1261  break;
1262  }
1263  }
1264 
1265  OMPParallelScope Scope(CGF, S);
1267  // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
1268  // lower and upper bounds with the pragma 'for' chunking mechanism.
1269  // The following lambda takes care of appending the lower and upper bound
1270  // parameters when necessary
1271  CodeGenBoundParameters(CGF, S, CapturedVars);
1272  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
1273  CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn,
1274  CapturedVars, IfCond);
1275 }
1276 
1278  const OMPExecutableDirective &,
1280 
1282  // Emit parallel region as a standalone region.
1283  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1284  Action.Enter(CGF);
1285  OMPPrivateScope PrivateScope(CGF);
1286  bool Copyins = CGF.EmitOMPCopyinClause(S);
1287  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1288  if (Copyins) {
1289  // Emit implicit barrier to synchronize threads and avoid data races on
1290  // propagation master's thread values of threadprivate variables to local
1291  // instances of that variables of all other implicit threads.
1292  CGF.CGM.getOpenMPRuntime().emitBarrierCall(
1293  CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
1294  /*ForceSimpleCall=*/true);
1295  }
1296  CGF.EmitOMPPrivateClause(S, PrivateScope);
1297  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1298  (void)PrivateScope.Privatize();
1299  CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1300  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
1301  };
1302  emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
1305  [](CodeGenFunction &) { return nullptr; });
1306 }
1307 
1309  JumpDest LoopExit) {
1310  RunCleanupsScope BodyScope(*this);
1311  // Update counters values on current iteration.
1312  for (const Expr *UE : D.updates())
1313  EmitIgnoredExpr(UE);
1314  // Update the linear variables.
1315  // In distribute directives only loop counters may be marked as linear, no
1316  // need to generate the code for them.
1318  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1319  for (const Expr *UE : C->updates())
1320  EmitIgnoredExpr(UE);
1321  }
1322  }
1323 
1324  // On a continue in the body, jump to the end.
1325  JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue");
1326  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1327  // Emit loop body.
1328  EmitStmt(D.getBody());
1329  // The end (updates/cleanups).
1330  EmitBlock(Continue.getBlock());
1331  BreakContinueStack.pop_back();
1332 }
1333 
1335  const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
1336  const Expr *IncExpr,
1337  const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
1338  const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
1339  auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
1340 
1341  // Start the loop with a block that tests the condition.
1342  auto CondBlock = createBasicBlock("omp.inner.for.cond");
1343  EmitBlock(CondBlock);
1344  const SourceRange R = S.getSourceRange();
1345  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1346  SourceLocToDebugLoc(R.getEnd()));
1347 
1348  // If there are any cleanups between here and the loop-exit scope,
1349  // create a block to stage a loop exit along.
1350  llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1351  if (RequiresCleanup)
1352  ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
1353 
1354  llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body");
1355 
1356  // Emit condition.
1357  EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
1358  if (ExitBlock != LoopExit.getBlock()) {
1359  EmitBlock(ExitBlock);
1360  EmitBranchThroughCleanup(LoopExit);
1361  }
1362 
1363  EmitBlock(LoopBody);
1364  incrementProfileCounter(&S);
1365 
1366  // Create a block for the increment.
1367  JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
1368  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1369 
1370  BodyGen(*this);
1371 
1372  // Emit "IV = IV + 1" and a back-edge to the condition block.
1373  EmitBlock(Continue.getBlock());
1374  EmitIgnoredExpr(IncExpr);
1375  PostIncGen(*this);
1376  BreakContinueStack.pop_back();
1377  EmitBranch(CondBlock);
1378  LoopStack.pop();
1379  // Emit the fall-through block.
1380  EmitBlock(LoopExit.getBlock());
1381 }
1382 
1384  if (!HaveInsertPoint())
1385  return false;
1386  // Emit inits for the linear variables.
1387  bool HasLinears = false;
1388  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1389  for (const Expr *Init : C->inits()) {
1390  HasLinears = true;
1391  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
1392  if (const auto *Ref =
1393  dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
1394  AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
1395  const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
1396  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1397  CapturedStmtInfo->lookup(OrigVD) != nullptr,
1398  VD->getInit()->getType(), VK_LValue,
1399  VD->getInit()->getExprLoc());
1400  EmitExprAsInit(&DRE, VD, MakeAddrLValue(Emission.getAllocatedAddress(),
1401  VD->getType()),
1402  /*capturedByInit=*/false);
1403  EmitAutoVarCleanups(Emission);
1404  } else {
1405  EmitVarDecl(*VD);
1406  }
1407  }
1408  // Emit the linear steps for the linear clauses.
1409  // If a step is not constant, it is pre-calculated before the loop.
1410  if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
1411  if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
1412  EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
1413  // Emit calculation of the linear step.
1414  EmitIgnoredExpr(CS);
1415  }
1416  }
1417  return HasLinears;
1418 }
1419 
1421  const OMPLoopDirective &D,
1422  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1423  if (!HaveInsertPoint())
1424  return;
1425  llvm::BasicBlock *DoneBB = nullptr;
1426  // Emit the final values of the linear variables.
1427  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1428  auto IC = C->varlist_begin();
1429  for (const Expr *F : C->finals()) {
1430  if (!DoneBB) {
1431  if (llvm::Value *Cond = CondGen(*this)) {
1432  // If the first post-update expression is found, emit conditional
1433  // block if it was requested.
1434  llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu");
1435  DoneBB = createBasicBlock(".omp.linear.pu.done");
1436  Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1437  EmitBlock(ThenBB);
1438  }
1439  }
1440  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
1441  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1442  CapturedStmtInfo->lookup(OrigVD) != nullptr,
1443  (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
1444  Address OrigAddr = EmitLValue(&DRE).getAddress();
1445  CodeGenFunction::OMPPrivateScope VarScope(*this);
1446  VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1447  (void)VarScope.Privatize();
1448  EmitIgnoredExpr(F);
1449  ++IC;
1450  }
1451  if (const Expr *PostUpdate = C->getPostUpdateExpr())
1452  EmitIgnoredExpr(PostUpdate);
1453  }
1454  if (DoneBB)
1455  EmitBlock(DoneBB, /*IsFinished=*/true);
1456 }
1457 
1459  const OMPExecutableDirective &D) {
1460  if (!CGF.HaveInsertPoint())
1461  return;
1462  for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
1463  unsigned ClauseAlignment = 0;
1464  if (const Expr *AlignmentExpr = Clause->getAlignment()) {
1465  auto *AlignmentCI =
1466  cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
1467  ClauseAlignment = static_cast<unsigned>(AlignmentCI->getZExtValue());
1468  }
1469  for (const Expr *E : Clause->varlists()) {
1470  unsigned Alignment = ClauseAlignment;
1471  if (Alignment == 0) {
1472  // OpenMP [2.8.1, Description]
1473  // If no optional parameter is specified, implementation-defined default
1474  // alignments for SIMD instructions on the target platforms are assumed.
1475  Alignment =
1476  CGF.getContext()
1478  E->getType()->getPointeeType()))
1479  .getQuantity();
1480  }
1481  assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) &&
1482  "alignment is not power of 2");
1483  if (Alignment != 0) {
1484  llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
1486  PtrValue, E, /*No second loc needed*/ SourceLocation(), Alignment);
1487  }
1488  }
1489  }
1490 }
1491 
1493  const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
1494  if (!HaveInsertPoint())
1495  return;
1496  auto I = S.private_counters().begin();
1497  for (const Expr *E : S.counters()) {
1498  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1499  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
1500  // Emit var without initialization.
1501  AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD);
1502  EmitAutoVarCleanups(VarEmission);
1503  LocalDeclMap.erase(PrivateVD);
1504  (void)LoopScope.addPrivate(VD, [&VarEmission]() {
1505  return VarEmission.getAllocatedAddress();
1506  });
1507  if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
1508  VD->hasGlobalStorage()) {
1509  (void)LoopScope.addPrivate(PrivateVD, [this, VD, E]() {
1510  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD),
1511  LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
1512  E->getType(), VK_LValue, E->getExprLoc());
1513  return EmitLValue(&DRE).getAddress();
1514  });
1515  } else {
1516  (void)LoopScope.addPrivate(PrivateVD, [&VarEmission]() {
1517  return VarEmission.getAllocatedAddress();
1518  });
1519  }
1520  ++I;
1521  }
1522  // Privatize extra loop counters used in loops for ordered(n) clauses.
1523  for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) {
1524  if (!C->getNumForLoops())
1525  continue;
1526  for (unsigned I = S.getCollapsedNumber(),
1527  E = C->getLoopNumIterations().size();
1528  I < E; ++I) {
1529  const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I));
1530  const auto *VD = cast<VarDecl>(DRE->getDecl());
1531  // Override only those variables that can be captured to avoid re-emission
1532  // of the variables declared within the loops.
1533  if (DRE->refersToEnclosingVariableOrCapture()) {
1534  (void)LoopScope.addPrivate(VD, [this, DRE, VD]() {
1535  return CreateMemTemp(DRE->getType(), VD->getName());
1536  });
1537  }
1538  }
1539  }
1540 }
1541 
1542 static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
1543  const Expr *Cond, llvm::BasicBlock *TrueBlock,
1544  llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
1545  if (!CGF.HaveInsertPoint())
1546  return;
1547  {
1548  CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
1549  CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
1550  (void)PreCondScope.Privatize();
1551  // Get initial values of real counters.
1552  for (const Expr *I : S.inits()) {
1553  CGF.EmitIgnoredExpr(I);
1554  }
1555  }
1556  // Check that loop is executed at least one time.
1557  CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
1558 }
1559 
1561  const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
1562  if (!HaveInsertPoint())
1563  return;
1566  const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1567  for (const Expr *C : LoopDirective->counters()) {
1568  SIMDLCVs.insert(
1569  cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
1570  }
1571  }
1572  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1573  auto CurPrivate = C->privates().begin();
1574  for (const Expr *E : C->varlists()) {
1575  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1576  const auto *PrivateVD =
1577  cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
1578  if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
1579  bool IsRegistered = PrivateScope.addPrivate(VD, [this, PrivateVD]() {
1580  // Emit private VarDecl with copy init.
1581  EmitVarDecl(*PrivateVD);
1582  return GetAddrOfLocalVar(PrivateVD);
1583  });
1584  assert(IsRegistered && "linear var already registered as private");
1585  // Silence the warning about unused variable.
1586  (void)IsRegistered;
1587  } else {
1588  EmitVarDecl(*PrivateVD);
1589  }
1590  ++CurPrivate;
1591  }
1592  }
1593 }
1594 
1596  const OMPExecutableDirective &D,
1597  bool IsMonotonic) {
1598  if (!CGF.HaveInsertPoint())
1599  return;
1600  if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
1601  RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
1602  /*ignoreResult=*/true);
1603  auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1604  CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1605  // In presence of finite 'safelen', it may be unsafe to mark all
1606  // the memory instructions parallel, because loop-carried
1607  // dependences of 'safelen' iterations are possible.
1608  if (!IsMonotonic)
1610  } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
1611  RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
1612  /*ignoreResult=*/true);
1613  auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1614  CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1615  // In presence of finite 'safelen', it may be unsafe to mark all
1616  // the memory instructions parallel, because loop-carried
1617  // dependences of 'safelen' iterations are possible.
1618  CGF.LoopStack.setParallel(/*Enable=*/false);
1619  }
1620 }
1621 
1623  bool IsMonotonic) {
1624  // Walk clauses and process safelen/lastprivate.
1625  LoopStack.setParallel(!IsMonotonic);
1626  LoopStack.setVectorizeEnable();
1627  emitSimdlenSafelenClause(*this, D, IsMonotonic);
1628 }
1629 
1631  const OMPLoopDirective &D,
1632  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1633  if (!HaveInsertPoint())
1634  return;
1635  llvm::BasicBlock *DoneBB = nullptr;
1636  auto IC = D.counters().begin();
1637  auto IPC = D.private_counters().begin();
1638  for (const Expr *F : D.finals()) {
1639  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
1640  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
1641  const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
1642  if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
1643  OrigVD->hasGlobalStorage() || CED) {
1644  if (!DoneBB) {
1645  if (llvm::Value *Cond = CondGen(*this)) {
1646  // If the first post-update expression is found, emit conditional
1647  // block if it was requested.
1648  llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then");
1649  DoneBB = createBasicBlock(".omp.final.done");
1650  Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1651  EmitBlock(ThenBB);
1652  }
1653  }
1654  Address OrigAddr = Address::invalid();
1655  if (CED) {
1656  OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();
1657  } else {
1658  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(PrivateVD),
1659  /*RefersToEnclosingVariableOrCapture=*/false,
1660  (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
1661  OrigAddr = EmitLValue(&DRE).getAddress();
1662  }
1663  OMPPrivateScope VarScope(*this);
1664  VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1665  (void)VarScope.Privatize();
1666  EmitIgnoredExpr(F);
1667  }
1668  ++IC;
1669  ++IPC;
1670  }
1671  if (DoneBB)
1672  EmitBlock(DoneBB, /*IsFinished=*/true);
1673 }
1674 
1676  const OMPLoopDirective &S,
1678  CGF.EmitOMPLoopBody(S, LoopExit);
1679  CGF.EmitStopPoint(&S);
1680 }
1681 
1682 /// Emit a helper variable and return corresponding lvalue.
1684  const DeclRefExpr *Helper) {
1685  auto VDecl = cast<VarDecl>(Helper->getDecl());
1686  CGF.EmitVarDecl(*VDecl);
1687  return CGF.EmitLValue(Helper);
1688 }
1689 
1691  PrePostActionTy &Action) {
1692  Action.Enter(CGF);
1694  "Expected simd directive");
1695  OMPLoopScope PreInitScope(CGF, S);
1696  // if (PreCond) {
1697  // for (IV in 0..LastIteration) BODY;
1698  // <Final counter/linear vars updates>;
1699  // }
1700  //
1704  (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));
1705  (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));
1706  }
1707 
1708  // Emit: if (PreCond) - begin.
1709  // If the condition constant folds and can be elided, avoid emitting the
1710  // whole loop.
1711  bool CondConstant;
1712  llvm::BasicBlock *ContBlock = nullptr;
1713  if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
1714  if (!CondConstant)
1715  return;
1716  } else {
1717  llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");
1718  ContBlock = CGF.createBasicBlock("simd.if.end");
1719  emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
1720  CGF.getProfileCount(&S));
1721  CGF.EmitBlock(ThenBlock);
1722  CGF.incrementProfileCounter(&S);
1723  }
1724 
1725  // Emit the loop iteration variable.
1726  const Expr *IVExpr = S.getIterationVariable();
1727  const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
1728  CGF.EmitVarDecl(*IVDecl);
1729  CGF.EmitIgnoredExpr(S.getInit());
1730 
1731  // Emit the iterations count variable.
1732  // If it is not a variable, Sema decided to calculate iterations count on
1733  // each iteration (e.g., it is foldable into a constant).
1734  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
1735  CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
1736  // Emit calculation of the iterations count.
1738  }
1739 
1740  CGF.EmitOMPSimdInit(S);
1741 
1742  emitAlignedClause(CGF, S);
1743  (void)CGF.EmitOMPLinearClauseInit(S);
1744  {
1745  CodeGenFunction::OMPPrivateScope LoopScope(CGF);
1746  CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
1747  CGF.EmitOMPLinearClause(S, LoopScope);
1748  CGF.EmitOMPPrivateClause(S, LoopScope);
1749  CGF.EmitOMPReductionClauseInit(S, LoopScope);
1750  bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
1751  (void)LoopScope.Privatize();
1753  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
1754  CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
1755  S.getInc(),
1756  [&S](CodeGenFunction &CGF) {
1758  CGF.EmitStopPoint(&S);
1759  },
1760  [](CodeGenFunction &) {});
1761  CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; });
1762  // Emit final copy of the lastprivate variables at the end of loops.
1763  if (HasLastprivateClause)
1764  CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
1765  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
1767  [](CodeGenFunction &) { return nullptr; });
1768  }
1769  CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; });
1770  // Emit: if (PreCond) - end.
1771  if (ContBlock) {
1772  CGF.EmitBranch(ContBlock);
1773  CGF.EmitBlock(ContBlock, true);
1774  }
1775 }
1776 
1778  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1779  emitOMPSimdRegion(CGF, S, Action);
1780  };
1781  OMPLexicalScope Scope(*this, S, OMPD_unknown);
1782  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
1783 }
1784 
1785 void CodeGenFunction::EmitOMPOuterLoop(
1786  bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
1788  const CodeGenFunction::OMPLoopArguments &LoopArgs,
1789  const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
1790  const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
1791  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1792 
1793  const Expr *IVExpr = S.getIterationVariable();
1794  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1795  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1796 
1797  JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
1798 
1799  // Start the loop with a block that tests the condition.
1800  llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond");
1801  EmitBlock(CondBlock);
1802  const SourceRange R = S.getSourceRange();
1803  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1804  SourceLocToDebugLoc(R.getEnd()));
1805 
1806  llvm::Value *BoolCondVal = nullptr;
1807  if (!DynamicOrOrdered) {
1808  // UB = min(UB, GlobalUB) or
1809  // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
1810  // 'distribute parallel for')
1811  EmitIgnoredExpr(LoopArgs.EUB);
1812  // IV = LB
1813  EmitIgnoredExpr(LoopArgs.Init);
1814  // IV < UB
1815  BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
1816  } else {
1817  BoolCondVal =
1818  RT.emitForNext(*this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
1819  LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
1820  }
1821 
1822  // If there are any cleanups between here and the loop-exit scope,
1823  // create a block to stage a loop exit along.
1824  llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1825  if (LoopScope.requiresCleanups())
1826  ExitBlock = createBasicBlock("omp.dispatch.cleanup");
1827 
1828  llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body");
1829  Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
1830  if (ExitBlock != LoopExit.getBlock()) {
1831  EmitBlock(ExitBlock);
1832  EmitBranchThroughCleanup(LoopExit);
1833  }
1834  EmitBlock(LoopBody);
1835 
1836  // Emit "IV = LB" (in case of static schedule, we have already calculated new
1837  // LB for loop condition and emitted it above).
1838  if (DynamicOrOrdered)
1839  EmitIgnoredExpr(LoopArgs.Init);
1840 
1841  // Create a block for the increment.
1842  JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
1843  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1844 
1845  // Generate !llvm.loop.parallel metadata for loads and stores for loops
1846  // with dynamic/guided scheduling and without ordered clause.
1848  LoopStack.setParallel(!IsMonotonic);
1849  else
1850  EmitOMPSimdInit(S, IsMonotonic);
1851 
1852  SourceLocation Loc = S.getBeginLoc();
1853 
1854  // when 'distribute' is not combined with a 'for':
1855  // while (idx <= UB) { BODY; ++idx; }
1856  // when 'distribute' is combined with a 'for'
1857  // (e.g. 'distribute parallel for')
1858  // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
1859  EmitOMPInnerLoop(
1860  S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
1861  [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
1862  CodeGenLoop(CGF, S, LoopExit);
1863  },
1864  [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
1865  CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
1866  });
1867 
1868  EmitBlock(Continue.getBlock());
1869  BreakContinueStack.pop_back();
1870  if (!DynamicOrOrdered) {
1871  // Emit "LB = LB + Stride", "UB = UB + Stride".
1872  EmitIgnoredExpr(LoopArgs.NextLB);
1873  EmitIgnoredExpr(LoopArgs.NextUB);
1874  }
1875 
1876  EmitBranch(CondBlock);
1877  LoopStack.pop();
1878  // Emit the fall-through block.
1879  EmitBlock(LoopExit.getBlock());
1880 
1881  // Tell the runtime we are done.
1882  auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) {
1883  if (!DynamicOrOrdered)
1884  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
1885  S.getDirectiveKind());
1886  };
1887  OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
1888 }
1889 
1890 void CodeGenFunction::EmitOMPForOuterLoop(
1891  const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
1892  const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
1893  const OMPLoopArguments &LoopArgs,
1894  const CodeGenDispatchBoundsTy &CGDispatchBounds) {
1895  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1896 
1897  // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
1898  const bool DynamicOrOrdered =
1899  Ordered || RT.isDynamic(ScheduleKind.Schedule);
1900 
1901  assert((Ordered ||
1902  !RT.isStaticNonchunked(ScheduleKind.Schedule,
1903  LoopArgs.Chunk != nullptr)) &&
1904  "static non-chunked schedule does not need outer loop");
1905 
1906  // Emit outer loop.
1907  //
1908  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1909  // When schedule(dynamic,chunk_size) is specified, the iterations are
1910  // distributed to threads in the team in chunks as the threads request them.
1911  // Each thread executes a chunk of iterations, then requests another chunk,
1912  // until no chunks remain to be distributed. Each chunk contains chunk_size
1913  // iterations, except for the last chunk to be distributed, which may have
1914  // fewer iterations. When no chunk_size is specified, it defaults to 1.
1915  //
1916  // When schedule(guided,chunk_size) is specified, the iterations are assigned
1917  // to threads in the team in chunks as the executing threads request them.
1918  // Each thread executes a chunk of iterations, then requests another chunk,
1919  // until no chunks remain to be assigned. For a chunk_size of 1, the size of
1920  // each chunk is proportional to the number of unassigned iterations divided
1921  // by the number of threads in the team, decreasing to 1. For a chunk_size
1922  // with value k (greater than 1), the size of each chunk is determined in the
1923  // same way, with the restriction that the chunks do not contain fewer than k
1924  // iterations (except for the last chunk to be assigned, which may have fewer
1925  // than k iterations).
1926  //
1927  // When schedule(auto) is specified, the decision regarding scheduling is
1928  // delegated to the compiler and/or runtime system. The programmer gives the
1929  // implementation the freedom to choose any possible mapping of iterations to
1930  // threads in the team.
1931  //
1932  // When schedule(runtime) is specified, the decision regarding scheduling is
1933  // deferred until run time, and the schedule and chunk size are taken from the
1934  // run-sched-var ICV. If the ICV is set to auto, the schedule is
1935  // implementation defined
1936  //
1937  // while(__kmpc_dispatch_next(&LB, &UB)) {
1938  // idx = LB;
1939  // while (idx <= UB) { BODY; ++idx;
1940  // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
1941  // } // inner loop
1942  // }
1943  //
1944  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1945  // When schedule(static, chunk_size) is specified, iterations are divided into
1946  // chunks of size chunk_size, and the chunks are assigned to the threads in
1947  // the team in a round-robin fashion in the order of the thread number.
1948  //
1949  // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
1950  // while (idx <= UB) { BODY; ++idx; } // inner loop
1951  // LB = LB + ST;
1952  // UB = UB + ST;
1953  // }
1954  //
1955 
1956  const Expr *IVExpr = S.getIterationVariable();
1957  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1958  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1959 
1960  if (DynamicOrOrdered) {
1961  const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
1962  CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
1963  llvm::Value *LBVal = DispatchBounds.first;
1964  llvm::Value *UBVal = DispatchBounds.second;
1965  CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
1966  LoopArgs.Chunk};
1967  RT.emitForDispatchInit(*this, S.getBeginLoc(), ScheduleKind, IVSize,
1968  IVSigned, Ordered, DipatchRTInputValues);
1969  } else {
1970  CGOpenMPRuntime::StaticRTInput StaticInit(
1971  IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
1972  LoopArgs.ST, LoopArgs.Chunk);
1973  RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
1974  ScheduleKind, StaticInit);
1975  }
1976 
1977  auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
1978  const unsigned IVSize,
1979  const bool IVSigned) {
1980  if (Ordered) {
1981  CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
1982  IVSigned);
1983  }
1984  };
1985 
1986  OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
1987  LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
1988  OuterLoopArgs.IncExpr = S.getInc();
1989  OuterLoopArgs.Init = S.getInit();
1990  OuterLoopArgs.Cond = S.getCond();
1991  OuterLoopArgs.NextLB = S.getNextLowerBound();
1992  OuterLoopArgs.NextUB = S.getNextUpperBound();
1993  EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
1994  emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
1995 }
1996 
1998  const unsigned IVSize, const bool IVSigned) {}
1999 
2000 void CodeGenFunction::EmitOMPDistributeOuterLoop(
2001  OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
2002  OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
2003  const CodeGenLoopTy &CodeGenLoopContent) {
2004 
2005  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
2006 
2007  // Emit outer loop.
2008  // Same behavior as a OMPForOuterLoop, except that schedule cannot be
2009  // dynamic
2010  //
2011 
2012  const Expr *IVExpr = S.getIterationVariable();
2013  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2014  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2015 
2016  CGOpenMPRuntime::StaticRTInput StaticInit(
2017  IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
2018  LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
2019  RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, StaticInit);
2020 
2021  // for combined 'distribute' and 'for' the increment expression of distribute
2022  // is stored in DistInc. For 'distribute' alone, it is in Inc.
2023  Expr *IncExpr;
2025  IncExpr = S.getDistInc();
2026  else
2027  IncExpr = S.getInc();
2028 
2029  // this routine is shared by 'omp distribute parallel for' and
2030  // 'omp distribute': select the right EUB expression depending on the
2031  // directive
2032  OMPLoopArguments OuterLoopArgs;
2033  OuterLoopArgs.LB = LoopArgs.LB;
2034  OuterLoopArgs.UB = LoopArgs.UB;
2035  OuterLoopArgs.ST = LoopArgs.ST;
2036  OuterLoopArgs.IL = LoopArgs.IL;
2037  OuterLoopArgs.Chunk = LoopArgs.Chunk;
2038  OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2040  : S.getEnsureUpperBound();
2041  OuterLoopArgs.IncExpr = IncExpr;
2042  OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2043  ? S.getCombinedInit()
2044  : S.getInit();
2045  OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2046  ? S.getCombinedCond()
2047  : S.getCond();
2048  OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2050  : S.getNextLowerBound();
2051  OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2053  : S.getNextUpperBound();
2054 
2055  EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
2056  LoopScope, OuterLoopArgs, CodeGenLoopContent,
2058 }
2059 
2060 static std::pair<LValue, LValue>
2062  const OMPExecutableDirective &S) {
2063  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2064  LValue LB =
2065  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2066  LValue UB =
2067  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2068 
2069  // When composing 'distribute' with 'for' (e.g. as in 'distribute
2070  // parallel for') we need to use the 'distribute'
2071  // chunk lower and upper bounds rather than the whole loop iteration
2072  // space. These are parameters to the outlined function for 'parallel'
2073  // and we copy the bounds of the previous schedule into the
2074  // the current ones.
2075  LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
2076  LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
2077  llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
2078  PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());
2079  PrevLBVal = CGF.EmitScalarConversion(
2080  PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
2081  LS.getIterationVariable()->getType(),
2083  llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
2084  PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());
2085  PrevUBVal = CGF.EmitScalarConversion(
2086  PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
2087  LS.getIterationVariable()->getType(),
2089 
2090  CGF.EmitStoreOfScalar(PrevLBVal, LB);
2091  CGF.EmitStoreOfScalar(PrevUBVal, UB);
2092 
2093  return {LB, UB};
2094 }
2095 
2096 /// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
2097 /// we need to use the LB and UB expressions generated by the worksharing
2098 /// code generation support, whereas in non combined situations we would
2099 /// just emit 0 and the LastIteration expression
2100 /// This function is necessary due to the difference of the LB and UB
2101 /// types for the RT emission routines for 'for_static_init' and
2102 /// 'for_dispatch_init'
2103 static std::pair<llvm::Value *, llvm::Value *>
2105  const OMPExecutableDirective &S,
2106  Address LB, Address UB) {
2107  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2108  const Expr *IVExpr = LS.getIterationVariable();
2109  // when implementing a dynamic schedule for a 'for' combined with a
2110  // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
2111  // is not normalized as each team only executes its own assigned
2112  // distribute chunk
2113  QualType IteratorTy = IVExpr->getType();
2114  llvm::Value *LBVal =
2115  CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
2116  llvm::Value *UBVal =
2117  CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
2118  return {LBVal, UBVal};
2119 }
2120 
2122  CodeGenFunction &CGF, const OMPExecutableDirective &S,
2123  llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) {
2124  const auto &Dir = cast<OMPLoopDirective>(S);
2125  LValue LB =
2126  CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
2127  llvm::Value *LBCast = CGF.Builder.CreateIntCast(
2128  CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2129  CapturedVars.push_back(LBCast);
2130  LValue UB =
2131  CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
2132 
2133  llvm::Value *UBCast = CGF.Builder.CreateIntCast(
2134  CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2135  CapturedVars.push_back(UBCast);
2136 }
2137 
2138 static void
2140  const OMPLoopDirective &S,
2142  auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
2143  PrePostActionTy &Action) {
2144  Action.Enter(CGF);
2145  bool HasCancel = false;
2147  if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
2148  HasCancel = D->hasCancel();
2149  else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
2150  HasCancel = D->hasCancel();
2151  else if (const auto *D =
2152  dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
2153  HasCancel = D->hasCancel();
2154  }
2156  HasCancel);
2160  };
2161 
2163  CGF, S,
2164  isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for,
2165  CGInlinedWorksharingLoop,
2167 }
2168 
2171  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2173  S.getDistInc());
2174  };
2175  OMPLexicalScope Scope(*this, S, OMPD_parallel);
2176  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2177 }
2178 
2181  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2183  S.getDistInc());
2184  };
2185  OMPLexicalScope Scope(*this, S, OMPD_parallel);
2186  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2187 }
2188 
2190  const OMPDistributeSimdDirective &S) {
2191  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2193  };
2194  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2195  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2196 }
2197 
2199  CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
2200  // Emit SPMD target parallel for region as a standalone region.
2201  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2202  emitOMPSimdRegion(CGF, S, Action);
2203  };
2204  llvm::Function *Fn;
2205  llvm::Constant *Addr;
2206  // Emit target region as a standalone region.
2207  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
2208  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
2209  assert(Fn && Addr && "Target device function emission failed.");
2210 }
2211 
2213  const OMPTargetSimdDirective &S) {
2214  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2215  emitOMPSimdRegion(CGF, S, Action);
2216  };
2217  emitCommonOMPTargetDirective(*this, S, CodeGen);
2218 }
2219 
2220 namespace {
2221  struct ScheduleKindModifiersTy {
2225  ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
2228  : Kind(Kind), M1(M1), M2(M2) {}
2229  };
2230 } // namespace
2231 
2233  const OMPLoopDirective &S, Expr *EUB,
2234  const CodeGenLoopBoundsTy &CodeGenLoopBounds,
2235  const CodeGenDispatchBoundsTy &CGDispatchBounds) {
2236  // Emit the loop iteration variable.
2237  const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
2238  const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
2239  EmitVarDecl(*IVDecl);
2240 
2241  // Emit the iterations count variable.
2242  // If it is not a variable, Sema decided to calculate iterations count on each
2243  // iteration (e.g., it is foldable into a constant).
2244  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2245  EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2246  // Emit calculation of the iterations count.
2247  EmitIgnoredExpr(S.getCalcLastIteration());
2248  }
2249 
2250  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
2251 
2252  bool HasLastprivateClause;
2253  // Check pre-condition.
2254  {
2255  OMPLoopScope PreInitScope(*this, S);
2256  // Skip the entire loop if we don't meet the precondition.
2257  // If the condition constant folds and can be elided, avoid emitting the
2258  // whole loop.
2259  bool CondConstant;
2260  llvm::BasicBlock *ContBlock = nullptr;
2261  if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2262  if (!CondConstant)
2263  return false;
2264  } else {
2265  llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
2266  ContBlock = createBasicBlock("omp.precond.end");
2267  emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
2268  getProfileCount(&S));
2269  EmitBlock(ThenBlock);
2270  incrementProfileCounter(&S);
2271  }
2272 
2273  RunCleanupsScope DoacrossCleanupScope(*this);
2274  bool Ordered = false;
2275  if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
2276  if (OrderedClause->getNumForLoops())
2277  RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations());
2278  else
2279  Ordered = true;
2280  }
2281 
2282  llvm::DenseSet<const Expr *> EmittedFinals;
2283  emitAlignedClause(*this, S);
2284  bool HasLinears = EmitOMPLinearClauseInit(S);
2285  // Emit helper vars inits.
2286 
2287  std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
2288  LValue LB = Bounds.first;
2289  LValue UB = Bounds.second;
2290  LValue ST =
2291  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
2292  LValue IL =
2293  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
2294 
2295  // Emit 'then' code.
2296  {
2297  OMPPrivateScope LoopScope(*this);
2298  if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
2299  // Emit implicit barrier to synchronize threads and avoid data races on
2300  // initialization of firstprivate variables and post-update of
2301  // lastprivate variables.
2302  CGM.getOpenMPRuntime().emitBarrierCall(
2303  *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
2304  /*ForceSimpleCall=*/true);
2305  }
2306  EmitOMPPrivateClause(S, LoopScope);
2307  HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
2308  EmitOMPReductionClauseInit(S, LoopScope);
2309  EmitOMPPrivateLoopCounters(S, LoopScope);
2310  EmitOMPLinearClause(S, LoopScope);
2311  (void)LoopScope.Privatize();
2313  CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
2314 
2315  // Detect the loop schedule kind and chunk.
2316  const Expr *ChunkExpr = nullptr;
2317  OpenMPScheduleTy ScheduleKind;
2318  if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
2319  ScheduleKind.Schedule = C->getScheduleKind();
2320  ScheduleKind.M1 = C->getFirstScheduleModifier();
2321  ScheduleKind.M2 = C->getSecondScheduleModifier();
2322  ChunkExpr = C->getChunkSize();
2323  } else {
2324  // Default behaviour for schedule clause.
2325  CGM.getOpenMPRuntime().getDefaultScheduleAndChunk(
2326  *this, S, ScheduleKind.Schedule, ChunkExpr);
2327  }
2328  bool HasChunkSizeOne = false;
2329  llvm::Value *Chunk = nullptr;
2330  if (ChunkExpr) {
2331  Chunk = EmitScalarExpr(ChunkExpr);
2332  Chunk = EmitScalarConversion(Chunk, ChunkExpr->getType(),
2334  S.getBeginLoc());
2335  Expr::EvalResult Result;
2336  if (ChunkExpr->EvaluateAsInt(Result, getContext())) {
2337  llvm::APSInt EvaluatedChunk = Result.Val.getInt();
2338  HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
2339  }
2340  }
2341  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2342  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2343  // OpenMP 4.5, 2.7.1 Loop Construct, Description.
2344  // If the static schedule kind is specified or if the ordered clause is
2345  // specified, and if no monotonic modifier is specified, the effect will
2346  // be as if the monotonic modifier was specified.
2347  bool StaticChunkedOne = RT.isStaticChunked(ScheduleKind.Schedule,
2348  /* Chunked */ Chunk != nullptr) && HasChunkSizeOne &&
2350  if ((RT.isStaticNonchunked(ScheduleKind.Schedule,
2351  /* Chunked */ Chunk != nullptr) ||
2352  StaticChunkedOne) &&
2353  !Ordered) {
2355  EmitOMPSimdInit(S, /*IsMonotonic=*/true);
2356  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2357  // When no chunk_size is specified, the iteration space is divided into
2358  // chunks that are approximately equal in size, and at most one chunk is
2359  // distributed to each thread. Note that the size of the chunks is
2360  // unspecified in this case.
2361  CGOpenMPRuntime::StaticRTInput StaticInit(
2362  IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
2363  UB.getAddress(), ST.getAddress(),
2364  StaticChunkedOne ? Chunk : nullptr);
2365  RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
2366  ScheduleKind, StaticInit);
2367  JumpDest LoopExit =
2368  getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
2369  // UB = min(UB, GlobalUB);
2370  if (!StaticChunkedOne)
2371  EmitIgnoredExpr(S.getEnsureUpperBound());
2372  // IV = LB;
2373  EmitIgnoredExpr(S.getInit());
2374  // For unchunked static schedule generate:
2375  //
2376  // while (idx <= UB) {
2377  // BODY;
2378  // ++idx;
2379  // }
2380  //
2381  // For static schedule with chunk one:
2382  //
2383  // while (IV <= PrevUB) {
2384  // BODY;
2385  // IV += ST;
2386  // }
2387  EmitOMPInnerLoop(S, LoopScope.requiresCleanups(),
2388  StaticChunkedOne ? S.getCombinedParForInDistCond() : S.getCond(),
2389  StaticChunkedOne ? S.getDistInc() : S.getInc(),
2390  [&S, LoopExit](CodeGenFunction &CGF) {
2391  CGF.EmitOMPLoopBody(S, LoopExit);
2392  CGF.EmitStopPoint(&S);
2393  },
2394  [](CodeGenFunction &) {});
2395  EmitBlock(LoopExit.getBlock());
2396  // Tell the runtime we are done.
2397  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2398  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2399  S.getDirectiveKind());
2400  };
2401  OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
2402  } else {
2403  const bool IsMonotonic =
2404  Ordered || ScheduleKind.Schedule == OMPC_SCHEDULE_static ||
2405  ScheduleKind.Schedule == OMPC_SCHEDULE_unknown ||
2406  ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
2407  ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
2408  // Emit the outer loop, which requests its work chunk [LB..UB] from
2409  // runtime and runs the inner loop to process it.
2410  const OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),
2411  ST.getAddress(), IL.getAddress(),
2412  Chunk, EUB);
2413  EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
2414  LoopArguments, CGDispatchBounds);
2415  }
2417  EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
2418  return CGF.Builder.CreateIsNotNull(
2419  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2420  });
2421  }
2422  EmitOMPReductionClauseFinal(
2423  S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind())
2424  ? /*Parallel and Simd*/ OMPD_parallel_for_simd
2425  : /*Parallel only*/ OMPD_parallel);
2426  // Emit post-update of the reduction variables if IsLastIter != 0.
2428  *this, S, [IL, &S](CodeGenFunction &CGF) {
2429  return CGF.Builder.CreateIsNotNull(
2430  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2431  });
2432  // Emit final copy of the lastprivate variables if IsLastIter != 0.
2433  if (HasLastprivateClause)
2434  EmitOMPLastprivateClauseFinal(
2436  Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
2437  }
2438  EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) {
2439  return CGF.Builder.CreateIsNotNull(
2440  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2441  });
2442  DoacrossCleanupScope.ForceCleanup();
2443  // We're now done with the loop, so jump to the continuation block.
2444  if (ContBlock) {
2445  EmitBranch(ContBlock);
2446  EmitBlock(ContBlock, /*IsFinished=*/true);
2447  }
2448  }
2449  return HasLastprivateClause;
2450 }
2451 
2452 /// The following two functions generate expressions for the loop lower
2453 /// and upper bounds in case of static and dynamic (dispatch) schedule
2454 /// of the associated 'for' or 'distribute' loop.
2455 static std::pair<LValue, LValue>
2457  const auto &LS = cast<OMPLoopDirective>(S);
2458  LValue LB =
2459  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2460  LValue UB =
2461  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2462  return {LB, UB};
2463 }
2464 
2465 /// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
2466 /// consider the lower and upper bound expressions generated by the
2467 /// worksharing loop support, but we use 0 and the iteration space size as
2468 /// constants
2469 static std::pair<llvm::Value *, llvm::Value *>
2471  Address LB, Address UB) {
2472  const auto &LS = cast<OMPLoopDirective>(S);
2473  const Expr *IVExpr = LS.getIterationVariable();
2474  const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
2475  llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
2476  llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
2477  return {LBVal, UBVal};
2478 }
2479 
2481  bool HasLastprivates = false;
2482  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2483  PrePostActionTy &) {
2484  OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel());
2485  HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2488  };
2489  {
2490  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2491  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
2492  S.hasCancel());
2493  }
2494 
2495  // Emit an implicit barrier at the end.
2496  if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
2497  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
2498 }
2499 
2501  bool HasLastprivates = false;
2502  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2503  PrePostActionTy &) {
2504  HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2507  };
2508  {
2509  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2510  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2511  }
2512 
2513  // Emit an implicit barrier at the end.
2514  if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
2515  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
2516 }
2517 
2519  const Twine &Name,
2520  llvm::Value *Init = nullptr) {
2521  LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
2522  if (Init)
2523  CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
2524  return LVal;
2525 }
2526 
2527 void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
2529  const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
2530  bool HasLastprivates = false;
2531  auto &&CodeGen = [&S, CapturedStmt, CS,
2532  &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
2533  ASTContext &C = CGF.getContext();
2534  QualType KmpInt32Ty =
2535  C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2536  // Emit helper vars inits.
2537  LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
2538  CGF.Builder.getInt32(0));
2539  llvm::ConstantInt *GlobalUBVal = CS != nullptr
2540  ? CGF.Builder.getInt32(CS->size() - 1)
2541  : CGF.Builder.getInt32(0);
2542  LValue UB =
2543  createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
2544  LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
2545  CGF.Builder.getInt32(1));
2546  LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
2547  CGF.Builder.getInt32(0));
2548  // Loop counter.
2549  LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
2550  OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
2551  CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
2552  OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
2553  CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
2554  // Generate condition for loop.
2555  BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue,
2557  // Increment for loop counter.
2558  UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary,
2559  S.getBeginLoc(), true);
2560  auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
2561  // Iterate through all sections and emit a switch construct:
2562  // switch (IV) {
2563  // case 0:
2564  // <SectionStmt[0]>;
2565  // break;
2566  // ...
2567  // case <NumSection> - 1:
2568  // <SectionStmt[<NumSection> - 1]>;
2569  // break;
2570  // }
2571  // .omp.sections.exit:
2572  llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
2573  llvm::SwitchInst *SwitchStmt =
2574  CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
2575  ExitBB, CS == nullptr ? 1 : CS->size());
2576  if (CS) {
2577  unsigned CaseNumber = 0;
2578  for (const Stmt *SubStmt : CS->children()) {
2579  auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2580  CGF.EmitBlock(CaseBB);
2581  SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
2582  CGF.EmitStmt(SubStmt);
2583  CGF.EmitBranch(ExitBB);
2584  ++CaseNumber;
2585  }
2586  } else {
2587  llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");
2588  CGF.EmitBlock(CaseBB);
2589  SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
2590  CGF.EmitStmt(CapturedStmt);
2591  CGF.EmitBranch(ExitBB);
2592  }
2593  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2594  };
2595 
2596  CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2597  if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
2598  // Emit implicit barrier to synchronize threads and avoid data races on
2599  // initialization of firstprivate variables and post-update of lastprivate
2600  // variables.
2601  CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2602  CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
2603  /*ForceSimpleCall=*/true);
2604  }
2605  CGF.EmitOMPPrivateClause(S, LoopScope);
2606  HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2607  CGF.EmitOMPReductionClauseInit(S, LoopScope);
2608  (void)LoopScope.Privatize();
2609  if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
2610  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
2611 
2612  // Emit static non-chunked loop.
2613  OpenMPScheduleTy ScheduleKind;
2614  ScheduleKind.Schedule = OMPC_SCHEDULE_static;
2615  CGOpenMPRuntime::StaticRTInput StaticInit(
2616  /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(),
2617  LB.getAddress(), UB.getAddress(), ST.getAddress());
2618  CGF.CGM.getOpenMPRuntime().emitForStaticInit(
2619  CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit);
2620  // UB = min(UB, GlobalUB);
2621  llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
2622  llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
2623  CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
2624  CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
2625  // IV = LB;
2626  CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
2627  // while (idx <= UB) { BODY; ++idx; }
2628  CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen,
2629  [](CodeGenFunction &) {});
2630  // Tell the runtime we are done.
2631  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2632  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2633  S.getDirectiveKind());
2634  };
2635  CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
2636  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
2637  // Emit post-update of the reduction variables if IsLastIter != 0.
2638  emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) {
2639  return CGF.Builder.CreateIsNotNull(
2640  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2641  });
2642 
2643  // Emit final copy of the lastprivate variables if IsLastIter != 0.
2644  if (HasLastprivates)
2646  S, /*NoFinals=*/false,
2647  CGF.Builder.CreateIsNotNull(
2648  CGF.EmitLoadOfScalar(IL, S.getBeginLoc())));
2649  };
2650 
2651  bool HasCancel = false;
2652  if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
2653  HasCancel = OSD->hasCancel();
2654  else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
2655  HasCancel = OPSD->hasCancel();
2656  OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
2657  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
2658  HasCancel);
2659  // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
2660  // clause. Otherwise the barrier will be generated by the codegen for the
2661  // directive.
2662  if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
2663  // Emit implicit barrier to synchronize threads and avoid data races on
2664  // initialization of firstprivate variables.
2665  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
2666  OMPD_unknown);
2667  }
2668 }
2669 
2671  {
2672  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2673  EmitSections(S);
2674  }
2675  // Emit an implicit barrier at the end.
2676  if (!S.getSingleClause<OMPNowaitClause>()) {
2677  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
2678  OMPD_sections);
2679  }
2680 }
2681 
2683  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2685  };
2686  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2687  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen,
2688  S.hasCancel());
2689 }
2690 
2692  llvm::SmallVector<const Expr *, 8> CopyprivateVars;
2695  llvm::SmallVector<const Expr *, 8> AssignmentOps;
2696  // Check if there are any 'copyprivate' clauses associated with this
2697  // 'single' construct.
2698  // Build a list of copyprivate variables along with helper expressions
2699  // (<source>, <destination>, <destination>=<source> expressions)
2700  for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
2701  CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
2702  DestExprs.append(C->destination_exprs().begin(),
2703  C->destination_exprs().end());
2704  SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
2705  AssignmentOps.append(C->assignment_ops().begin(),
2706  C->assignment_ops().end());
2707  }
2708  // Emit code for 'single' region along with 'copyprivate' clauses
2709  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2710  Action.Enter(CGF);
2711  OMPPrivateScope SingleScope(CGF);
2712  (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
2713  CGF.EmitOMPPrivateClause(S, SingleScope);
2714  (void)SingleScope.Privatize();
2716  };
2717  {
2718  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2719  CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getBeginLoc(),
2720  CopyprivateVars, DestExprs,
2721  SrcExprs, AssignmentOps);
2722  }
2723  // Emit an implicit barrier at the end (to avoid data race on firstprivate
2724  // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
2725  if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
2726  CGM.getOpenMPRuntime().emitBarrierCall(
2727  *this, S.getBeginLoc(),
2728  S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
2729  }
2730 }
2731 
2733  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2734  Action.Enter(CGF);
2736  };
2737  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2738  CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
2739 }
2740 
2742  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2743  Action.Enter(CGF);
2745  };
2746  const Expr *Hint = nullptr;
2747  if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
2748  Hint = HintClause->getHint();
2749  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2750  CGM.getOpenMPRuntime().emitCriticalRegion(*this,
2752  CodeGen, S.getBeginLoc(), Hint);
2753 }
2754 
2756  const OMPParallelForDirective &S) {
2757  // Emit directive as a combined directive that consists of two implicit
2758  // directives: 'parallel' with 'for' directive.
2759  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2760  Action.Enter(CGF);
2761  OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel());
2764  };
2765  emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
2767 }
2768 
2770  const OMPParallelForSimdDirective &S) {
2771  // Emit directive as a combined directive that consists of two implicit
2772  // directives: 'parallel' with 'for' directive.
2773  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2774  Action.Enter(CGF);
2777  };
2778  emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen,
2780 }
2781 
2783  const OMPParallelSectionsDirective &S) {
2784  // Emit directive as a combined directive that consists of two implicit
2785  // directives: 'parallel' with 'sections' directive.
2786  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2787  Action.Enter(CGF);
2788  CGF.EmitSections(S);
2789  };
2790  emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
2792 }
2793 
2795  const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
2796  const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
2797  OMPTaskDataTy &Data) {
2798  // Emit outlined function for task construct.
2799  const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
2800  auto I = CS->getCapturedDecl()->param_begin();
2801  auto PartId = std::next(I);
2802  auto TaskT = std::next(I, 4);
2803  // Check if the task is final
2804  if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
2805  // If the condition constant folds and can be elided, try to avoid emitting
2806  // the condition and the dead arm of the if/else.
2807  const Expr *Cond = Clause->getCondition();
2808  bool CondConstant;
2809  if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
2810  Data.Final.setInt(CondConstant);
2811  else
2812  Data.Final.setPointer(EvaluateExprAsBool(Cond));
2813  } else {
2814  // By default the task is not final.
2815  Data.Final.setInt(/*IntVal=*/false);
2816  }
2817  // Check if the task has 'priority' clause.
2818  if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
2819  const Expr *Prio = Clause->getPriority();
2820  Data.Priority.setInt(/*IntVal=*/true);
2821  Data.Priority.setPointer(EmitScalarConversion(
2822  EmitScalarExpr(Prio), Prio->getType(),
2823  getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
2824  Prio->getExprLoc()));
2825  }
2826  // The first function argument for tasks is a thread id, the second one is a
2827  // part id (0 for tied tasks, >=0 for untied task).
2828  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
2829  // Get list of private variables.
2830  for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
2831  auto IRef = C->varlist_begin();
2832  for (const Expr *IInit : C->private_copies()) {
2833  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2834  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2835  Data.PrivateVars.push_back(*IRef);
2836  Data.PrivateCopies.push_back(IInit);
2837  }
2838  ++IRef;
2839  }
2840  }
2841  EmittedAsPrivate.clear();
2842  // Get list of firstprivate variables.
2843  for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
2844  auto IRef = C->varlist_begin();
2845  auto IElemInitRef = C->inits().begin();
2846  for (const Expr *IInit : C->private_copies()) {
2847  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2848  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2849  Data.FirstprivateVars.push_back(*IRef);
2850  Data.FirstprivateCopies.push_back(IInit);
2851  Data.FirstprivateInits.push_back(*IElemInitRef);
2852  }
2853  ++IRef;
2854  ++IElemInitRef;
2855  }
2856  }
2857  // Get list of lastprivate variables (for taskloops).
2858  llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
2859  for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
2860  auto IRef = C->varlist_begin();
2861  auto ID = C->destination_exprs().begin();
2862  for (const Expr *IInit : C->private_copies()) {
2863  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2864  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2865  Data.LastprivateVars.push_back(*IRef);
2866  Data.LastprivateCopies.push_back(IInit);
2867  }
2868  LastprivateDstsOrigs.insert(
2869  {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
2870  cast<DeclRefExpr>(*IRef)});
2871  ++IRef;
2872  ++ID;
2873  }
2874  }
2877  for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
2878  auto IPriv = C->privates().begin();
2879  auto IRed = C->reduction_ops().begin();
2880  auto ILHS = C->lhs_exprs().begin();
2881  auto IRHS = C->rhs_exprs().begin();
2882  for (const Expr *Ref : C->varlists()) {
2883  Data.ReductionVars.emplace_back(Ref);
2884  Data.ReductionCopies.emplace_back(*IPriv);
2885  Data.ReductionOps.emplace_back(*IRed);
2886  LHSs.emplace_back(*ILHS);
2887  RHSs.emplace_back(*IRHS);
2888  std::advance(IPriv, 1);
2889  std::advance(IRed, 1);
2890  std::advance(ILHS, 1);
2891  std::advance(IRHS, 1);
2892  }
2893  }
2894  Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
2895  *this, S.getBeginLoc(), LHSs, RHSs, Data);
2896  // Build list of dependences.
2897  for (const auto *C : S.getClausesOfKind<OMPDependClause>())
2898  for (const Expr *IRef : C->varlists())
2899  Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
2900  auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
2901  CapturedRegion](CodeGenFunction &CGF,
2902  PrePostActionTy &Action) {
2903  // Set proper addresses for generated private copies.
2904  OMPPrivateScope Scope(CGF);
2905  if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
2906  !Data.LastprivateVars.empty()) {
2907  llvm::FunctionType *CopyFnTy = llvm::FunctionType::get(
2908  CGF.Builder.getVoidTy(), {CGF.Builder.getInt8PtrTy()}, true);
2909  enum { PrivatesParam = 2, CopyFnParam = 3 };
2910  llvm::Value *CopyFn = CGF.Builder.CreateLoad(
2911  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
2912  llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
2913  CS->getCapturedDecl()->getParam(PrivatesParam)));
2914  // Map privates.
2917  CallArgs.push_back(PrivatesPtr);
2918  for (const Expr *E : Data.PrivateVars) {
2919  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2920  Address PrivatePtr = CGF.CreateMemTemp(
2921  CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
2922  PrivatePtrs.emplace_back(VD, PrivatePtr);
2923  CallArgs.push_back(PrivatePtr.getPointer());
2924  }
2925  for (const Expr *E : Data.FirstprivateVars) {
2926  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2927  Address PrivatePtr =
2928  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2929  ".firstpriv.ptr.addr");
2930  PrivatePtrs.emplace_back(VD, PrivatePtr);
2931  CallArgs.push_back(PrivatePtr.getPointer());
2932  }
2933  for (const Expr *E : Data.LastprivateVars) {
2934  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2935  Address PrivatePtr =
2936  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2937  ".lastpriv.ptr.addr");
2938  PrivatePtrs.emplace_back(VD, PrivatePtr);
2939  CallArgs.push_back(PrivatePtr.getPointer());
2940  }
2941  CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
2942  CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
2943  for (const auto &Pair : LastprivateDstsOrigs) {
2944  const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
2945  DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD),
2946  /*RefersToEnclosingVariableOrCapture=*/
2947  CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr,
2948  Pair.second->getType(), VK_LValue,
2949  Pair.second->getExprLoc());
2950  Scope.addPrivate(Pair.first, [&CGF, &DRE]() {
2951  return CGF.EmitLValue(&DRE).getAddress();
2952  });
2953  }
2954  for (const auto &Pair : PrivatePtrs) {
2955  Address Replacement(CGF.Builder.CreateLoad(Pair.second),
2956  CGF.getContext().getDeclAlign(Pair.first));
2957  Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
2958  }
2959  }
2960  if (Data.Reductions) {
2961  OMPLexicalScope LexScope(CGF, S, CapturedRegion);
2963  Data.ReductionOps);
2964  llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
2965  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
2966  for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
2967  RedCG.emitSharedLValue(CGF, Cnt);
2968  RedCG.emitAggregateType(CGF, Cnt);
2969  // FIXME: This must removed once the runtime library is fixed.
2970  // Emit required threadprivate variables for
2971  // initializer/combiner/finalizer.
2972  CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
2973  RedCG, Cnt);
2974  Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
2975  CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
2976  Replacement =
2977  Address(CGF.EmitScalarConversion(
2978  Replacement.getPointer(), CGF.getContext().VoidPtrTy,
2979  CGF.getContext().getPointerType(
2980  Data.ReductionCopies[Cnt]->getType()),
2981  Data.ReductionCopies[Cnt]->getExprLoc()),
2982  Replacement.getAlignment());
2983  Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
2984  Scope.addPrivate(RedCG.getBaseDecl(Cnt),
2985  [Replacement]() { return Replacement; });
2986  }
2987  }
2988  // Privatize all private variables except for in_reduction items.
2989  (void)Scope.Privatize();
2990  SmallVector<const Expr *, 4> InRedVars;
2991  SmallVector<const Expr *, 4> InRedPrivs;
2993  SmallVector<const Expr *, 4> TaskgroupDescriptors;
2994  for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
2995  auto IPriv = C->privates().begin();
2996  auto IRed = C->reduction_ops().begin();
2997  auto ITD = C->taskgroup_descriptors().begin();
2998  for (const Expr *Ref : C->varlists()) {
2999  InRedVars.emplace_back(Ref);
3000  InRedPrivs.emplace_back(*IPriv);
3001  InRedOps.emplace_back(*IRed);
3002  TaskgroupDescriptors.emplace_back(*ITD);
3003  std::advance(IPriv, 1);
3004  std::advance(IRed, 1);
3005  std::advance(ITD, 1);
3006  }
3007  }
3008  // Privatize in_reduction items here, because taskgroup descriptors must be
3009  // privatized earlier.
3010  OMPPrivateScope InRedScope(CGF);
3011  if (!InRedVars.empty()) {
3012  ReductionCodeGen RedCG(InRedVars, InRedPrivs, InRedOps);
3013  for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
3014  RedCG.emitSharedLValue(CGF, Cnt);
3015  RedCG.emitAggregateType(CGF, Cnt);
3016  // The taskgroup descriptor variable is always implicit firstprivate and
3017  // privatized already during processing of the firstprivates.
3018  // FIXME: This must removed once the runtime library is fixed.
3019  // Emit required threadprivate variables for
3020  // initializer/combiner/finalizer.
3021  CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
3022  RedCG, Cnt);
3023  llvm::Value *ReductionsPtr =
3024  CGF.EmitLoadOfScalar(CGF.EmitLValue(TaskgroupDescriptors[Cnt]),
3025  TaskgroupDescriptors[Cnt]->getExprLoc());
3026  Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
3027  CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
3028  Replacement = Address(
3029  CGF.EmitScalarConversion(
3030  Replacement.getPointer(), CGF.getContext().VoidPtrTy,
3031  CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
3032  InRedPrivs[Cnt]->getExprLoc()),
3033  Replacement.getAlignment());
3034  Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
3035  InRedScope.addPrivate(RedCG.getBaseDecl(Cnt),
3036  [Replacement]() { return Replacement; });
3037  }
3038  }
3039  (void)InRedScope.Privatize();
3040 
3041  Action.Enter(CGF);
3042  BodyGen(CGF);
3043  };
3044  llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3045  S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied,
3046  Data.NumberOfParts);
3047  OMPLexicalScope Scope(*this, S);
3048  TaskGen(*this, OutlinedFn, Data);
3049 }
3050 
3051 static ImplicitParamDecl *
3053  QualType Ty, CapturedDecl *CD,
3054  SourceLocation Loc) {
3055  auto *OrigVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3057  auto *OrigRef = DeclRefExpr::Create(
3058  C, NestedNameSpecifierLoc(), SourceLocation(), OrigVD,
3059  /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
3060  auto *PrivateVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3062  auto *PrivateRef = DeclRefExpr::Create(
3063  C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD,
3064  /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
3065  QualType ElemType = C.getBaseElementType(Ty);
3066  auto *InitVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, ElemType,
3068  auto *InitRef = DeclRefExpr::Create(
3069  C, NestedNameSpecifierLoc(), SourceLocation(), InitVD,
3070  /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue);
3071  PrivateVD->setInitStyle(VarDecl::CInit);
3072  PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue,
3073  InitRef, /*BasePath=*/nullptr,
3074  VK_RValue));
3075  Data.FirstprivateVars.emplace_back(OrigRef);
3076  Data.FirstprivateCopies.emplace_back(PrivateRef);
3077  Data.FirstprivateInits.emplace_back(InitRef);
3078  return OrigVD;
3079 }
3080 
3082  const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,
3083  OMPTargetDataInfo &InputInfo) {
3084  // Emit outlined function for task construct.
3085  const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3086  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3087  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3088  auto I = CS->getCapturedDecl()->param_begin();
3089  auto PartId = std::next(I);
3090  auto TaskT = std::next(I, 4);
3091  OMPTaskDataTy Data;
3092  // The task is not final.
3093  Data.Final.setInt(/*IntVal=*/false);
3094  // Get list of firstprivate variables.
3095  for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
3096  auto IRef = C->varlist_begin();
3097  auto IElemInitRef = C->inits().begin();
3098  for (auto *IInit : C->private_copies()) {
3099  Data.FirstprivateVars.push_back(*IRef);
3100  Data.FirstprivateCopies.push_back(IInit);
3101  Data.FirstprivateInits.push_back(*IElemInitRef);
3102  ++IRef;
3103  ++IElemInitRef;
3104  }
3105  }
3106  OMPPrivateScope TargetScope(*this);
3107  VarDecl *BPVD = nullptr;
3108  VarDecl *PVD = nullptr;
3109  VarDecl *SVD = nullptr;
3110  if (InputInfo.NumberOfTargetItems > 0) {
3111  auto *CD = CapturedDecl::Create(
3112  getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0);
3113  llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);
3114  QualType BaseAndPointersType = getContext().getConstantArrayType(
3115  getContext().VoidPtrTy, ArrSize, ArrayType::Normal,
3116  /*IndexTypeQuals=*/0);
3118  getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
3120  getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
3121  QualType SizesType = getContext().getConstantArrayType(
3122  getContext().getSizeType(), ArrSize, ArrayType::Normal,
3123  /*IndexTypeQuals=*/0);
3124  SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD,
3125  S.getBeginLoc());
3126  TargetScope.addPrivate(
3127  BPVD, [&InputInfo]() { return InputInfo.BasePointersArray; });
3128  TargetScope.addPrivate(PVD,
3129  [&InputInfo]() { return InputInfo.PointersArray; });
3130  TargetScope.addPrivate(SVD,
3131  [&InputInfo]() { return InputInfo.SizesArray; });
3132  }
3133  (void)TargetScope.Privatize();
3134  // Build list of dependences.
3135  for (const auto *C : S.getClausesOfKind<OMPDependClause>())
3136  for (const Expr *IRef : C->varlists())
3137  Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
3138  auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD,
3139  &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
3140  // Set proper addresses for generated private copies.
3141  OMPPrivateScope Scope(CGF);
3142  if (!Data.FirstprivateVars.empty()) {
3143  llvm::FunctionType *CopyFnTy = llvm::FunctionType::get(
3144  CGF.Builder.getVoidTy(), {CGF.Builder.getInt8PtrTy()}, true);
3145  enum { PrivatesParam = 2, CopyFnParam = 3 };
3146  llvm::Value *CopyFn = CGF.Builder.CreateLoad(
3147  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
3148  llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
3149  CS->getCapturedDecl()->getParam(PrivatesParam)));
3150  // Map privates.
3153  CallArgs.push_back(PrivatesPtr);
3154  for (const Expr *E : Data.FirstprivateVars) {
3155  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3156  Address PrivatePtr =
3157  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
3158  ".firstpriv.ptr.addr");
3159  PrivatePtrs.emplace_back(VD, PrivatePtr);
3160  CallArgs.push_back(PrivatePtr.getPointer());
3161  }
3162  CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
3163  CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
3164  for (const auto &Pair : PrivatePtrs) {
3165  Address Replacement(CGF.Builder.CreateLoad(Pair.second),
3166  CGF.getContext().getDeclAlign(Pair.first));
3167  Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
3168  }
3169  }
3170  // Privatize all private variables except for in_reduction items.
3171  (void)Scope.Privatize();
3172  if (InputInfo.NumberOfTargetItems > 0) {
3173  InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
3174  CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0);
3175  InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
3176  CGF.GetAddrOfLocalVar(PVD), /*Index=*/0);
3177  InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
3178  CGF.GetAddrOfLocalVar(SVD), /*Index=*/0);
3179  }
3180 
3181  Action.Enter(CGF);
3182  OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);
3183  BodyGen(CGF);
3184  };
3185  llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3186  S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true,
3187  Data.NumberOfParts);
3188  llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
3189  IntegerLiteral IfCond(getContext(), TrueOrFalse,
3190  getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3191  SourceLocation());
3192 
3193  CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn,
3194  SharedsTy, CapturedStruct, &IfCond, Data);
3195 }
3196 
3198  // Emit outlined function for task construct.
3199  const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3200  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3201  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3202  const Expr *IfCond = nullptr;
3203  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3204  if (C->getNameModifier() == OMPD_unknown ||
3205  C->getNameModifier() == OMPD_task) {
3206  IfCond = C->getCondition();
3207  break;
3208  }
3209  }
3210 
3211  OMPTaskDataTy Data;
3212  // Check if we should emit tied or untied task.
3213  Data.Tied = !S.getSingleClause<OMPUntiedClause>();
3214  auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
3215  CGF.EmitStmt(CS->getCapturedStmt());
3216  };
3217  auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
3218  IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
3219  const OMPTaskDataTy &Data) {
3220  CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn,
3221  SharedsTy, CapturedStruct, IfCond,
3222  Data);
3223  };
3224  EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data);
3225 }
3226 
3228  const OMPTaskyieldDirective &S) {
3229  CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getBeginLoc());
3230 }
3231 
3233  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_barrier);
3234 }
3235 
3237  CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc());
3238 }
3239 
3241  const OMPTaskgroupDirective &S) {
3242  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3243  Action.Enter(CGF);
3244  if (const Expr *E = S.getReductionRef()) {
3247  OMPTaskDataTy Data;
3248  for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
3249  auto IPriv = C->privates().begin();
3250  auto IRed = C->reduction_ops().begin();
3251  auto ILHS = C->lhs_exprs().begin();
3252  auto IRHS = C->rhs_exprs().begin();
3253  for (const Expr *Ref : C->varlists()) {
3254  Data.ReductionVars.emplace_back(Ref);
3255  Data.ReductionCopies.emplace_back(*IPriv);
3256  Data.ReductionOps.emplace_back(*IRed);
3257  LHSs.emplace_back(*ILHS);
3258  RHSs.emplace_back(*IRHS);
3259  std::advance(IPriv, 1);
3260  std::advance(IRed, 1);
3261  std::advance(ILHS, 1);
3262  std::advance(IRHS, 1);
3263  }
3264  }
3265  llvm::Value *ReductionDesc =
3266  CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(),
3267  LHSs, RHSs, Data);
3268  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3269  CGF.EmitVarDecl(*VD);
3270  CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),
3271  /*Volatile=*/false, E->getType());
3272  }
3274  };
3275  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3276  CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getBeginLoc());
3277 }
3278 
3280  CGM.getOpenMPRuntime().emitFlush(
3281  *this,
3282  [&S]() -> ArrayRef<const Expr *> {
3283  if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>())
3284  return llvm::makeArrayRef(FlushClause->varlist_begin(),
3285  FlushClause->varlist_end());
3286  return llvm::None;
3287  }(),
3288  S.getBeginLoc());
3289 }
3290 
3292  const CodeGenLoopTy &CodeGenLoop,
3293  Expr *IncExpr) {
3294  // Emit the loop iteration variable.
3295  const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3296  const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3297  EmitVarDecl(*IVDecl);
3298 
3299  // Emit the iterations count variable.
3300  // If it is not a variable, Sema decided to calculate iterations count on each
3301  // iteration (e.g., it is foldable into a constant).
3302  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3303  EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3304  // Emit calculation of the iterations count.
3305  EmitIgnoredExpr(S.getCalcLastIteration());
3306  }
3307 
3308  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3309 
3310  bool HasLastprivateClause = false;
3311  // Check pre-condition.
3312  {
3313  OMPLoopScope PreInitScope(*this, S);
3314  // Skip the entire loop if we don't meet the precondition.
3315  // If the condition constant folds and can be elided, avoid emitting the
3316  // whole loop.
3317  bool CondConstant;
3318  llvm::BasicBlock *ContBlock = nullptr;
3319  if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
3320  if (!CondConstant)
3321  return;
3322  } else {
3323  llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
3324  ContBlock = createBasicBlock("omp.precond.end");
3325  emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
3326  getProfileCount(&S));
3327  EmitBlock(ThenBlock);
3328  incrementProfileCounter(&S);
3329  }
3330 
3331  emitAlignedClause(*this, S);
3332  // Emit 'then' code.
3333  {
3334  // Emit helper vars inits.
3335 
3336  LValue LB = EmitOMPHelperVar(
3337  *this, cast<DeclRefExpr>(
3340  : S.getLowerBoundVariable())));
3341  LValue UB = EmitOMPHelperVar(
3342  *this, cast<DeclRefExpr>(
3345  : S.getUpperBoundVariable())));
3346  LValue ST =
3347  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
3348  LValue IL =
3349  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
3350 
3351  OMPPrivateScope LoopScope(*this);
3352  if (EmitOMPFirstprivateClause(S, LoopScope)) {
3353  // Emit implicit barrier to synchronize threads and avoid data races
3354  // on initialization of firstprivate variables and post-update of
3355  // lastprivate variables.
3356  CGM.getOpenMPRuntime().emitBarrierCall(
3357  *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
3358  /*ForceSimpleCall=*/true);
3359  }
3360  EmitOMPPrivateClause(S, LoopScope);
3364  EmitOMPReductionClauseInit(S, LoopScope);
3365  HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
3366  EmitOMPPrivateLoopCounters(S, LoopScope);
3367  (void)LoopScope.Privatize();
3369  CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
3370 
3371  // Detect the distribute schedule kind and chunk.
3372  llvm::Value *Chunk = nullptr;
3374  if (const auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
3375  ScheduleKind = C->getDistScheduleKind();
3376  if (const Expr *Ch = C->getChunkSize()) {
3377  Chunk = EmitScalarExpr(Ch);
3378  Chunk = EmitScalarConversion(Chunk, Ch->getType(),
3380  S.getBeginLoc());
3381  }
3382  } else {
3383  // Default behaviour for dist_schedule clause.
3384  CGM.getOpenMPRuntime().getDefaultDistScheduleAndChunk(
3385  *this, S, ScheduleKind, Chunk);
3386  }
3387  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3388  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3389 
3390  // OpenMP [2.10.8, distribute Construct, Description]
3391  // If dist_schedule is specified, kind must be static. If specified,
3392  // iterations are divided into chunks of size chunk_size, chunks are
3393  // assigned to the teams of the league in a round-robin fashion in the
3394  // order of the team number. When no chunk_size is specified, the
3395  // iteration space is divided into chunks that are approximately equal
3396  // in size, and at most one chunk is distributed to each team of the
3397  // league. The size of the chunks is unspecified in this case.
3398  bool StaticChunked = RT.isStaticChunked(
3399  ScheduleKind, /* Chunked */ Chunk != nullptr) &&
3401  if (RT.isStaticNonchunked(ScheduleKind,
3402  /* Chunked */ Chunk != nullptr) ||
3403  StaticChunked) {
3405  EmitOMPSimdInit(S, /*IsMonotonic=*/true);
3406  CGOpenMPRuntime::StaticRTInput StaticInit(
3407  IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(),
3408  LB.getAddress(), UB.getAddress(), ST.getAddress(),
3409  StaticChunked ? Chunk : nullptr);
3410  RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind,
3411  StaticInit);
3412  JumpDest LoopExit =
3413  getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
3414  // UB = min(UB, GlobalUB);
3417  : S.getEnsureUpperBound());
3418  // IV = LB;
3420  ? S.getCombinedInit()
3421  : S.getInit());
3422 
3423  const Expr *Cond =
3425  ? S.getCombinedCond()
3426  : S.getCond();
3427 
3428  if (StaticChunked)
3429  Cond = S.getCombinedDistCond();
3430 
3431  // For static unchunked schedules generate:
3432  //
3433  // 1. For distribute alone, codegen
3434  // while (idx <= UB) {
3435  // BODY;
3436  // ++idx;
3437  // }
3438  //
3439  // 2. When combined with 'for' (e.g. as in 'distribute parallel for')
3440  // while (idx <= UB) {
3441  // <CodeGen rest of pragma>(LB, UB);
3442  // idx += ST;
3443  // }
3444  //
3445  // For static chunk one schedule generate:
3446  //
3447  // while (IV <= GlobalUB) {
3448  // <CodeGen rest of pragma>(LB, UB);
3449  // LB += ST;
3450  // UB += ST;
3451  // UB = min(UB, GlobalUB);
3452  // IV = LB;
3453  // }
3454  //
3455  EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), Cond, IncExpr,
3456  [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3457  CodeGenLoop(CGF, S, LoopExit);
3458  },
3459  [&S, StaticChunked](CodeGenFunction &CGF) {
3460  if (StaticChunked) {
3461  CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());
3462  CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());
3463  CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());
3464  CGF.EmitIgnoredExpr(S.getCombinedInit());
3465  }
3466  });
3467  EmitBlock(LoopExit.getBlock());
3468  // Tell the runtime we are done.
3469  RT.emitForStaticFinish(*this, S.getBeginLoc(), S.getDirectiveKind());
3470  } else {
3471  // Emit the outer loop, which requests its work chunk [LB..UB] from
3472  // runtime and runs the inner loop to process it.
3473  const OMPLoopArguments LoopArguments = {
3474  LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(),
3475  Chunk};
3476  EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
3477  CodeGenLoop);
3478  }
3480  EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
3481  return CGF.Builder.CreateIsNotNull(
3482  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3483  });
3484  }
3488  EmitOMPReductionClauseFinal(S, OMPD_simd);
3489  // Emit post-update of the reduction variables if IsLastIter != 0.
3491  *this, S, [IL, &S](CodeGenFunction &CGF) {
3492  return CGF.Builder.CreateIsNotNull(
3493  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3494  });
3495  }
3496  // Emit final copy of the lastprivate variables if IsLastIter != 0.
3497  if (HasLastprivateClause) {
3498  EmitOMPLastprivateClauseFinal(
3499  S, /*NoFinals=*/false,
3500  Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
3501  }
3502  }
3503 
3504  // We're now done with the loop, so jump to the continuation block.
3505  if (ContBlock) {
3506  EmitBranch(ContBlock);
3507  EmitBlock(ContBlock, true);
3508  }
3509  }
3510 }
3511 
3513  const OMPDistributeDirective &S) {
3514  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3516  };
3517  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3518  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3519 }
3520 
3521 static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM,
3522  const CapturedStmt *S) {
3523  CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
3525  CGF.CapturedStmtInfo = &CapStmtInfo;
3526  llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S);
3527  Fn->setDoesNotRecurse();
3528  return Fn;
3529 }
3530 
3532  if (S.hasClausesOfKind<OMPDependClause>()) {
3533  assert(!S.getAssociatedStmt() &&
3534  "No associated statement must be in ordered depend construct.");
3535  for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
3536  CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
3537  return;
3538  }
3539  const auto *C = S.getSingleClause<OMPSIMDClause>();
3540  auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
3541  PrePostActionTy &Action) {
3542  const CapturedStmt *CS = S.getInnermostCapturedStmt();
3543  if (C) {
3545  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
3546  llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS);
3547  CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
3548  OutlinedFn, CapturedVars);
3549  } else {
3550  Action.Enter(CGF);
3551  CGF.EmitStmt(CS->getCapturedStmt());
3552  }
3553  };
3554  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3555  CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getBeginLoc(), !C);
3556 }
3557 
3559  QualType SrcType, QualType DestType,
3560  SourceLocation Loc) {
3561  assert(CGF.hasScalarEvaluationKind(DestType) &&
3562  "DestType must have scalar evaluation kind.");
3563  assert(!Val.isAggregate() && "Must be a scalar or complex.");
3564  return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
3565  DestType, Loc)
3567  Val.getComplexVal(), SrcType, DestType, Loc);
3568 }
3569 
3572  QualType DestType, SourceLocation Loc) {
3573  assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
3574  "DestType must have complex evaluation kind.");
3575  CodeGenFunction::ComplexPairTy ComplexVal;
3576  if (Val.isScalar()) {
3577  // Convert the input element to the element type of the complex.
3578  QualType DestElementType =
3579  DestType->castAs<ComplexType>()->getElementType();
3580  llvm::Value *ScalarVal = CGF.EmitScalarConversion(
3581  Val.getScalarVal(), SrcType, DestElementType, Loc);
3582  ComplexVal = CodeGenFunction::ComplexPairTy(
3583  ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
3584  } else {
3585  assert(Val.isComplex() && "Must be a scalar or complex.");
3586  QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
3587  QualType DestElementType =
3588  DestType->castAs<ComplexType>()->getElementType();
3589  ComplexVal.first = CGF.EmitScalarConversion(
3590  Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
3591  ComplexVal.second = CGF.EmitScalarConversion(
3592  Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
3593  }
3594  return ComplexVal;
3595 }
3596 
3597 static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst,
3598  LValue LVal, RValue RVal) {
3599  if (LVal.isGlobalReg()) {
3600  CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
3601  } else {
3602  CGF.EmitAtomicStore(RVal, LVal,
3603  IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3604  : llvm::AtomicOrdering::Monotonic,
3605  LVal.isVolatile(), /*IsInit=*/false);
3606  }
3607 }
3608 
3610  QualType RValTy, SourceLocation Loc) {
3611  switch (getEvaluationKind(LVal.getType())) {
3612  case TEK_Scalar:
3613  EmitStoreThroughLValue(RValue::get(convertToScalarValue(
3614  *this, RVal, RValTy, LVal.getType(), Loc)),
3615  LVal);
3616  break;
3617  case TEK_Complex:
3618  EmitStoreOfComplex(
3619  convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,
3620  /*isInit=*/false);
3621  break;
3622  case TEK_Aggregate:
3623  llvm_unreachable("Must be a scalar or complex.");
3624  }
3625 }
3626 
3627 static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst,
3628  const Expr *X, const Expr *V,
3629  SourceLocation Loc) {
3630  // v = x;
3631  assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
3632  assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
3633  LValue XLValue = CGF.EmitLValue(X);
3634  LValue VLValue = CGF.EmitLValue(V);
3635  RValue Res = XLValue.isGlobalReg()
3636  ? CGF.EmitLoadOfLValue(XLValue, Loc)
3637  : CGF.EmitAtomicLoad(
3638  XLValue, Loc,
3639  IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3640  : llvm::AtomicOrdering::Monotonic,
3641  XLValue.isVolatile());
3642  // OpenMP, 2.12.6, atomic Construct
3643  // Any atomic construct with a seq_cst clause forces the atomically
3644  // performed operation to include an implicit flush operation without a
3645  // list.
3646  if (IsSeqCst)
3647  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3648  CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
3649 }
3650 
3651 static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst,
3652  const Expr *X, const Expr *E,
3653  SourceLocation Loc) {
3654  // x = expr;
3655  assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
3656  emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
3657  // OpenMP, 2.12.6, atomic Construct
3658  // Any atomic construct with a seq_cst clause forces the atomically
3659  // performed operation to include an implicit flush operation without a
3660  // list.
3661  if (IsSeqCst)
3662  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3663 }
3664 
3665 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
3666  RValue Update,
3667  BinaryOperatorKind BO,
3668  llvm::AtomicOrdering AO,
3669  bool IsXLHSInRHSPart) {
3670  ASTContext &Context = CGF.getContext();
3671  // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
3672  // expression is simple and atomic is allowed for the given type for the
3673  // target platform.
3674  if (BO == BO_Comma || !Update.isScalar() ||
3675  !Update.getScalarVal()->getType()->isIntegerTy() ||
3676  !X.isSimple() || (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
3677  (Update.getScalarVal()->getType() !=
3678  X.getAddress().getElementType())) ||
3679  !X.getAddress().getElementType()->isIntegerTy() ||
3680  !Context.getTargetInfo().hasBuiltinAtomic(
3681  Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
3682  return std::make_pair(false, RValue::get(nullptr));
3683 
3684  llvm::AtomicRMWInst::BinOp RMWOp;
3685  switch (BO) {
3686  case BO_Add:
3687  RMWOp = llvm::AtomicRMWInst::Add;
3688  break;
3689  case BO_Sub:
3690  if (!IsXLHSInRHSPart)
3691  return std::make_pair(false, RValue::get(nullptr));
3692  RMWOp = llvm::AtomicRMWInst::Sub;
3693  break;
3694  case BO_And:
3695  RMWOp = llvm::AtomicRMWInst::And;
3696  break;
3697  case BO_Or:
3698  RMWOp = llvm::AtomicRMWInst::Or;
3699  break;
3700  case BO_Xor:
3701  RMWOp = llvm::AtomicRMWInst::Xor;
3702  break;
3703  case BO_LT:
3705  ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
3706  : llvm::AtomicRMWInst::Max)
3707  : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
3708  : llvm::AtomicRMWInst::UMax);
3709  break;
3710  case BO_GT:
3712  ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
3713  : llvm::AtomicRMWInst::Min)
3714  : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
3715  : llvm::AtomicRMWInst::UMin);
3716  break;
3717  case BO_Assign:
3718  RMWOp = llvm::AtomicRMWInst::Xchg;
3719  break;
3720  case BO_Mul:
3721  case BO_Div:
3722  case BO_Rem:
3723  case BO_Shl:
3724  case BO_Shr:
3725  case BO_LAnd:
3726  case BO_LOr:
3727  return std::make_pair(false, RValue::get(nullptr));
3728  case BO_PtrMemD:
3729  case BO_PtrMemI:
3730  case BO_LE:
3731  case BO_GE:
3732  case BO_EQ:
3733  case BO_NE:
3734  case BO_Cmp:
3735  case BO_AddAssign:
3736  case BO_SubAssign:
3737  case BO_AndAssign:
3738  case BO_OrAssign:
3739  case BO_XorAssign:
3740  case BO_MulAssign:
3741  case BO_DivAssign:
3742  case BO_RemAssign:
3743  case BO_ShlAssign:
3744  case BO_ShrAssign:
3745  case BO_Comma:
3746  llvm_unreachable("Unsupported atomic update operation");
3747  }
3748  llvm::Value *UpdateVal = Update.getScalarVal();
3749  if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
3750  UpdateVal = CGF.Builder.CreateIntCast(
3751  IC, X.getAddress().getElementType(),
3753  }
3754  llvm::Value *Res =
3755  CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO);
3756  return std::make_pair(true, RValue::get(Res));
3757 }
3758 
3760  LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3761  llvm::AtomicOrdering AO, SourceLocation Loc,
3762  const llvm::function_ref<RValue(RValue)> CommonGen) {
3763  // Update expressions are allowed to have the following forms:
3764  // x binop= expr; -> xrval + expr;
3765  // x++, ++x -> xrval + 1;
3766  // x--, --x -> xrval - 1;
3767  // x = x binop expr; -> xrval binop expr
3768  // x = expr Op x; - > expr binop xrval;
3769  auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);
3770  if (!Res.first) {
3771  if (X.isGlobalReg()) {
3772  // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
3773  // 'xrval'.
3774  EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
3775  } else {
3776  // Perform compare-and-swap procedure.
3777  EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
3778  }
3779  }
3780  return Res;
3781 }
3782 
3783 static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst,
3784  const Expr *X, const Expr *E,
3785  const Expr *UE, bool IsXLHSInRHSPart,
3786  SourceLocation Loc) {
3787  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3788  "Update expr in 'atomic update' must be a binary operator.");
3789  const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3790  // Update expressions are allowed to have the following forms:
3791  // x binop= expr; -> xrval + expr;
3792  // x++, ++x -> xrval + 1;
3793  // x--, --x -> xrval - 1;
3794  // x = x binop expr; -> xrval binop expr
3795  // x = expr Op x; - > expr binop xrval;
3796  assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
3797  LValue XLValue = CGF.EmitLValue(X);
3798  RValue ExprRValue = CGF.EmitAnyExpr(E);
3799  llvm::AtomicOrdering AO = IsSeqCst
3800  ? llvm::AtomicOrdering::SequentiallyConsistent
3801  : llvm::AtomicOrdering::Monotonic;
3802  const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3803  const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3804  const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3805  const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3806  auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {
3807  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3808  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3809  return CGF.EmitAnyExpr(UE);
3810  };
3812  XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3813  // OpenMP, 2.12.6, atomic Construct
3814  // Any atomic construct with a seq_cst clause forces the atomically
3815  // performed operation to include an implicit flush operation without a
3816  // list.
3817  if (IsSeqCst)
3818  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3819 }
3820 
3822  QualType SourceType, QualType ResType,
3823  SourceLocation Loc) {
3824  switch (CGF.getEvaluationKind(ResType)) {
3825  case TEK_Scalar:
3826  return RValue::get(
3827  convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
3828  case TEK_Complex: {
3829  auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
3830  return RValue::getComplex(Res.first, Res.second);
3831  }
3832  case TEK_Aggregate:
3833  break;
3834  }
3835  llvm_unreachable("Must be a scalar or complex.");
3836 }
3837 
3838 static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst,
3839  bool IsPostfixUpdate, const Expr *V,
3840  const Expr *X, const Expr *E,
3841  const Expr *UE, bool IsXLHSInRHSPart,
3842  SourceLocation Loc) {
3843  assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
3844  assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
3845  RValue NewVVal;
3846  LValue VLValue = CGF.EmitLValue(V);
3847  LValue XLValue = CGF.EmitLValue(X);
3848  RValue ExprRValue = CGF.EmitAnyExpr(E);
3849  llvm::AtomicOrdering AO = IsSeqCst
3850  ? llvm::AtomicOrdering::SequentiallyConsistent
3851  : llvm::AtomicOrdering::Monotonic;
3852  QualType NewVValType;
3853  if (UE) {
3854  // 'x' is updated with some additional value.
3855  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3856  "Update expr in 'atomic capture' must be a binary operator.");
3857  const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3858  // Update expressions are allowed to have the following forms:
3859  // x binop= expr; -> xrval + expr;
3860  // x++, ++x -> xrval + 1;
3861  // x--, --x -> xrval - 1;
3862  // x = x binop expr; -> xrval binop expr
3863  // x = expr Op x; - > expr binop xrval;
3864  const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3865  const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3866  const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3867  NewVValType = XRValExpr->getType();
3868  const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3869  auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
3870  IsPostfixUpdate](RValue XRValue) {
3871  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3872  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3873  RValue Res = CGF.EmitAnyExpr(UE);
3874  NewVVal = IsPostfixUpdate ? XRValue : Res;
3875  return Res;
3876  };
3877  auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3878  XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3879  if (Res.first) {
3880  // 'atomicrmw' instruction was generated.
3881  if (IsPostfixUpdate) {
3882  // Use old value from 'atomicrmw'.
3883  NewVVal = Res.second;
3884  } else {
3885  // 'atomicrmw' does not provide new value, so evaluate it using old
3886  // value of 'x'.
3887  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3888  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
3889  NewVVal = CGF.EmitAnyExpr(UE);
3890  }
3891  }
3892  } else {
3893  // 'x' is simply rewritten with some 'expr'.
3894  NewVValType = X->getType().getNonReferenceType();
3895  ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
3896  X->getType().getNonReferenceType(), Loc);
3897  auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) {
3898  NewVVal = XRValue;
3899  return ExprRValue;
3900  };
3901  // Try to perform atomicrmw xchg, otherwise simple exchange.
3902  auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3903  XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
3904  Loc, Gen);
3905  if (Res.first) {
3906  // 'atomicrmw' instruction was generated.
3907  NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
3908  }
3909  }
3910  // Emit post-update store to 'v' of old/new 'x' value.
3911  CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
3912  // OpenMP, 2.12.6, atomic Construct
3913  // Any atomic construct with a seq_cst clause forces the atomically
3914  // performed operation to include an implicit flush operation without a
3915  // list.
3916  if (IsSeqCst)
3917  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3918 }
3919 
3921  bool IsSeqCst, bool IsPostfixUpdate,
3922  const Expr *X, const Expr *V, const Expr *E,
3923  const Expr *UE, bool IsXLHSInRHSPart,
3924  SourceLocation Loc) {
3925  switch (Kind) {
3926  case OMPC_read:
3927  emitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc);
3928  break;
3929  case OMPC_write:
3930  emitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc);
3931  break;
3932  case OMPC_unknown:
3933  case OMPC_update:
3934  emitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc);
3935  break;
3936  case OMPC_capture:
3937  emitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE,
3938  IsXLHSInRHSPart, Loc);
3939  break;
3940  case OMPC_if:
3941  case OMPC_final:
3942  case OMPC_num_threads:
3943  case OMPC_private:
3944  case OMPC_firstprivate:
3945  case OMPC_lastprivate:
3946  case OMPC_reduction:
3947  case OMPC_task_reduction:
3948  case OMPC_in_reduction:
3949  case OMPC_safelen:
3950  case OMPC_simdlen:
3951  case OMPC_allocator:
3952  case OMPC_allocate:
3953  case OMPC_collapse:
3954  case OMPC_default:
3955  case OMPC_seq_cst:
3956  case OMPC_shared:
3957  case OMPC_linear:
3958  case OMPC_aligned:
3959  case OMPC_copyin:
3960  case OMPC_copyprivate:
3961  case OMPC_flush:
3962  case OMPC_proc_bind:
3963  case OMPC_schedule:
3964  case OMPC_ordered:
3965  case OMPC_nowait:
3966  case OMPC_untied:
3967  case OMPC_threadprivate:
3968  case OMPC_depend:
3969  case OMPC_mergeable:
3970  case OMPC_device:
3971  case OMPC_threads:
3972  case OMPC_simd:
3973  case OMPC_map:
3974  case OMPC_num_teams:
3975  case OMPC_thread_limit:
3976  case OMPC_priority:
3977  case OMPC_grainsize:
3978  case OMPC_nogroup:
3979  case OMPC_num_tasks:
3980  case OMPC_hint:
3981  case OMPC_dist_schedule:
3982  case OMPC_defaultmap:
3983  case OMPC_uniform:
3984  case OMPC_to:
3985  case OMPC_from:
3986  case OMPC_use_device_ptr:
3987  case OMPC_is_device_ptr:
3988  case OMPC_unified_address:
3989  case OMPC_unified_shared_memory:
3990  case OMPC_reverse_offload:
3991  case OMPC_dynamic_allocators:
3992  case OMPC_atomic_default_mem_order:
3993  llvm_unreachable("Clause is not allowed in 'omp atomic'.");
3994  }
3995 }
3996 
3998  bool IsSeqCst = S.getSingleClause<OMPSeqCstClause>();
4000  for (const OMPClause *C : S.clauses()) {
4001  // Find first clause (skip seq_cst clause, if it is first).
4002  if (C->getClauseKind() != OMPC_seq_cst) {
4003  Kind = C->getClauseKind();
4004  break;
4005  }
4006  }
4007 
4008  const Stmt *CS = S.getInnermostCapturedStmt()->IgnoreContainers();
4009  if (const auto *FE = dyn_cast<FullExpr>(CS))
4010  enterFullExpression(FE);
4011  // Processing for statements under 'atomic capture'.
4012  if (const auto *Compound = dyn_cast<CompoundStmt>(CS)) {
4013  for (const Stmt *C : Compound->body()) {
4014  if (const auto *FE = dyn_cast<FullExpr>(C))
4015  enterFullExpression(FE);
4016  }
4017  }
4018 
4019  auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF,
4020  PrePostActionTy &) {
4021  CGF.EmitStopPoint(CS);
4022  emitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(),
4023  S.getV(), S.getExpr(), S.getUpdateExpr(),
4024  S.isXLHSInRHSPart(), S.getBeginLoc());
4025  };
4026  OMPLexicalScope Scope(*this, S, OMPD_unknown);
4027  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen);
4028 }
4029 
4031  const OMPExecutableDirective &S,
4032  const RegionCodeGenTy &CodeGen) {
4034  CodeGenModule &CGM = CGF.CGM;
4035 
4036  // On device emit this construct as inlined code.
4037  if (CGM.getLangOpts().OpenMPIsDevice) {
4038  OMPLexicalScope Scope(CGF, S, OMPD_target);
4039  CGM.getOpenMPRuntime().emitInlinedDirective(
4040  CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4042  });
4043  return;
4044  }
4045 
4046  llvm::Function *Fn = nullptr;
4047  llvm::Constant *FnID = nullptr;
4048 
4049  const Expr *IfCond = nullptr;
4050  // Check for the at most one if clause associated with the target region.
4051  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4052  if (C->getNameModifier() == OMPD_unknown ||
4053  C->getNameModifier() == OMPD_target) {
4054  IfCond = C->getCondition();
4055  break;
4056  }
4057  }
4058 
4059  // Check if we have any device clause associated with the directive.
4060  const Expr *Device = nullptr;
4061  if (auto *C = S.getSingleClause<OMPDeviceClause>())
4062  Device = C->getDevice();
4063 
4064  // Check if we have an if clause whose conditional always evaluates to false
4065  // or if we do not have any targets specified. If so the target region is not
4066  // an offload entry point.
4067  bool IsOffloadEntry = true;
4068  if (IfCond) {
4069  bool Val;
4070  if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
4071  IsOffloadEntry = false;
4072  }
4073  if (CGM.getLangOpts().OMPTargetTriples.empty())
4074  IsOffloadEntry = false;
4075 
4076  assert(CGF.CurFuncDecl && "No parent declaration for target region!");
4077  StringRef ParentName;
4078  // In case we have Ctors/Dtors we use the complete type variant to produce
4079  // the mangling of the device outlined kernel.
4080  if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
4081  ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));
4082  else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
4083  ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));
4084  else
4085  ParentName =
4086  CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));
4087 
4088  // Emit target region as a standalone region.
4089  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
4090  IsOffloadEntry, CodeGen);
4091  OMPLexicalScope Scope(CGF, S, OMPD_task);
4092  auto &&SizeEmitter = [](CodeGenFunction &CGF, const OMPLoopDirective &D) {
4093  OMPLoopScope(CGF, D);
4094  // Emit calculation of the iterations count.
4095  llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations());
4096  NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty,
4097  /*IsSigned=*/false);
4098  return NumIterations;
4099  };
4100  if (IsOffloadEntry)
4101  CGM.getOpenMPRuntime().emitTargetNumIterationsCall(CGF, S, Device,
4102  SizeEmitter);
4103  CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device);
4104 }
4105 
4107  PrePostActionTy &Action) {
4108  Action.Enter(CGF);
4109  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4110  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4111  CGF.EmitOMPPrivateClause(S, PrivateScope);
4112  (void)PrivateScope.Privatize();
4114  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4115 
4116  CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
4117 }
4118 
4120  StringRef ParentName,
4121  const OMPTargetDirective &S) {
4122  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4123  emitTargetRegion(CGF, S, Action);
4124  };
4125  llvm::Function *Fn;
4126  llvm::Constant *Addr;
4127  // Emit target region as a standalone region.
4128  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4129  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4130  assert(Fn && Addr && "Target device function emission failed.");
4131 }
4132 
4134  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4135  emitTargetRegion(CGF, S, Action);
4136  };
4137  emitCommonOMPTargetDirective(*this, S, CodeGen);
4138 }
4139 
4141  const OMPExecutableDirective &S,
4142  OpenMPDirectiveKind InnermostKind,
4143  const RegionCodeGenTy &CodeGen) {
4144  const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
4145  llvm::Function *OutlinedFn =
4146  CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
4147  S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
4148 
4149  const auto *NT = S.getSingleClause<OMPNumTeamsClause>();
4150  const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
4151  if (NT || TL) {
4152  const Expr *NumTeams = NT ? NT->getNumTeams() : nullptr;
4153  const Expr *ThreadLimit = TL ? TL->getThreadLimit() : nullptr;
4154 
4155  CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
4156  S.getBeginLoc());
4157  }
4158 
4159  OMPTeamsScope Scope(CGF, S);
4161  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
4162  CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn,
4163  CapturedVars);
4164 }
4165 
4167  // Emit teams region as a standalone region.
4168  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4169  Action.Enter(CGF);
4170  OMPPrivateScope PrivateScope(CGF);
4171  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4172  CGF.EmitOMPPrivateClause(S, PrivateScope);
4173  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4174  (void)PrivateScope.Privatize();
4175  CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
4176  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4177  };
4178  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4180  [](CodeGenFunction &) { return nullptr; });
4181 }
4182 
4184  const OMPTargetTeamsDirective &S) {
4185  auto *CS = S.getCapturedStmt(OMPD_teams);
4186  Action.Enter(CGF);
4187  // Emit teams region as a standalone region.
4188  auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4189  Action.Enter(CGF);
4190  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4191  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4192  CGF.EmitOMPPrivateClause(S, PrivateScope);
4193  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4194  (void)PrivateScope.Privatize();
4196  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4197  CGF.EmitStmt(CS->getCapturedStmt());
4198  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4199  };
4200  emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
4202  [](CodeGenFunction &) { return nullptr; });
4203 }
4204 
4206  CodeGenModule &CGM, StringRef ParentName,
4207  const OMPTargetTeamsDirective &S) {
4208  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4209  emitTargetTeamsRegion(CGF, Action, S);
4210  };
4211  llvm::Function *Fn;
4212  llvm::Constant *Addr;
4213  // Emit target region as a standalone region.
4214  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4215  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4216  assert(Fn && Addr && "Target device function emission failed.");
4217 }
4218 
4220  const OMPTargetTeamsDirective &S) {
4221  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4222  emitTargetTeamsRegion(CGF, Action, S);
4223  };
4224  emitCommonOMPTargetDirective(*this, S, CodeGen);
4225 }
4226 
4227 static void
4230  Action.Enter(CGF);
4231  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4233  };
4234 
4235  // Emit teams region as a standalone region.
4236  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4237  PrePostActionTy &Action) {
4238  Action.Enter(CGF);
4239  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4240  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4241  (void)PrivateScope.Privatize();
4242  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4243  CodeGenDistribute);
4244  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4245  };
4246  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
4248  [](CodeGenFunction &) { return nullptr; });
4249 }
4250 
4252  CodeGenModule &CGM, StringRef ParentName,
4254  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4255  emitTargetTeamsDistributeRegion(CGF, Action, S);
4256  };
4257  llvm::Function *Fn;
4258  llvm::Constant *Addr;
4259  // Emit target region as a standalone region.
4260  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4261  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4262  assert(Fn && Addr && "Target device function emission failed.");
4263 }
4264 
4267  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4268  emitTargetTeamsDistributeRegion(CGF, Action, S);
4269  };
4270  emitCommonOMPTargetDirective(*this, S, CodeGen);
4271 }
4272 
4274  CodeGenFunction &CGF, PrePostActionTy &Action,
4276  Action.Enter(CGF);
4277  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4279  };
4280 
4281  // Emit teams region as a standalone region.
4282  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4283  PrePostActionTy &Action) {
4284  Action.Enter(CGF);
4285  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4286  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4287  (void)PrivateScope.Privatize();
4288  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4289  CodeGenDistribute);
4290  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4291  };
4292  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);
4294  [](CodeGenFunction &) { return nullptr; });
4295 }
4296 
4298  CodeGenModule &CGM, StringRef ParentName,
4300  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4301  emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4302  };
4303  llvm::Function *Fn;
4304  llvm::Constant *Addr;
4305  // Emit target region as a standalone region.
4306  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4307  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4308  assert(Fn && Addr && "Target device function emission failed.");
4309 }
4310 
4313  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4314  emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4315  };
4316  emitCommonOMPTargetDirective(*this, S, CodeGen);
4317 }
4318 
4320  const OMPTeamsDistributeDirective &S) {
4321 
4322  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4324  };
4325 
4326  // Emit teams region as a standalone region.
4327  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4328  PrePostActionTy &Action) {
4329  Action.Enter(CGF);
4330  OMPPrivateScope PrivateScope(CGF);
4331  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4332  (void)PrivateScope.Privatize();
4333  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4334  CodeGenDistribute);
4335  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4336  };
4337  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4339  [](CodeGenFunction &) { return nullptr; });
4340 }
4341 
4344  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4346  };
4347 
4348  // Emit teams region as a standalone region.
4349  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4350  PrePostActionTy &Action) {
4351  Action.Enter(CGF);
4352  OMPPrivateScope PrivateScope(CGF);
4353  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4354  (void)PrivateScope.Privatize();
4355  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
4356  CodeGenDistribute);
4357  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4358  };
4359  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen);
4361  [](CodeGenFunction &) { return nullptr; });
4362 }
4363 
4366  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4368  S.getDistInc());
4369  };
4370 
4371  // Emit teams region as a standalone region.
4372  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4373  PrePostActionTy &Action) {
4374  Action.Enter(CGF);
4375  OMPPrivateScope PrivateScope(CGF);
4376  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4377  (void)PrivateScope.Privatize();
4378  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4379  CodeGenDistribute);
4380  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4381  };
4382  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4384  [](CodeGenFunction &) { return nullptr; });
4385 }
4386 
4389  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4391  S.getDistInc());
4392  };
4393 
4394  // Emit teams region as a standalone region.
4395  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4396  PrePostActionTy &Action) {
4397  Action.Enter(CGF);
4398  OMPPrivateScope PrivateScope(CGF);
4399  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4400  (void)PrivateScope.Privatize();
4401  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4402  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4403  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4404  };
4405  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4407  [](CodeGenFunction &) { return nullptr; });
4408 }
4409 
4412  PrePostActionTy &Action) {
4413  Action.Enter(CGF);
4414  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4416  S.getDistInc());
4417  };
4418 
4419  // Emit teams region as a standalone region.
4420  auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4421  PrePostActionTy &Action) {
4422  Action.Enter(CGF);
4423  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4424  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4425  (void)PrivateScope.Privatize();
4426  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4427  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4428  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4429  };
4430 
4431  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
4432  CodeGenTeams);
4434  [](CodeGenFunction &) { return nullptr; });
4435 }
4436 
4438  CodeGenModule &CGM, StringRef ParentName,
4440  // Emit SPMD target teams distribute parallel for region as a standalone
4441  // region.
4442  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4444  };
4445  llvm::Function *Fn;
4446  llvm::Constant *Addr;
4447  // Emit target region as a standalone region.
4448  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4449  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4450  assert(Fn && Addr && "Target device function emission failed.");
4451 }
4452 
4455  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4457  };
4458  emitCommonOMPTargetDirective(*this, S, CodeGen);
4459 }
4460 
4462  CodeGenFunction &CGF,
4464  PrePostActionTy &Action) {
4465  Action.Enter(CGF);
4466  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4468  S.getDistInc());
4469  };
4470 
4471  // Emit teams region as a standalone region.
4472  auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4473  PrePostActionTy &Action) {
4474  Action.Enter(CGF);
4475  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4476  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4477  (void)PrivateScope.Privatize();
4478  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4479  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4480  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4481  };
4482 
4483  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd,
4484  CodeGenTeams);
4486  [](CodeGenFunction &) { return nullptr; });
4487 }
4488 
4490  CodeGenModule &CGM, StringRef ParentName,
4492  // Emit SPMD target teams distribute parallel for simd region as a standalone
4493  // region.
4494  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4496  };
4497  llvm::Function *Fn;
4498  llvm::Constant *Addr;
4499  // Emit target region as a standalone region.
4500  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4501  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4502  assert(Fn && Addr && "Target device function emission failed.");
4503 }
4504 
4507  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4509  };
4510  emitCommonOMPTargetDirective(*this, S, CodeGen);
4511 }
4512 
4514  const OMPCancellationPointDirective &S) {
4515  CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getBeginLoc(),
4516  S.getCancelRegion());
4517 }
4518 
4520  const Expr *IfCond = nullptr;
4521  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4522  if (C->getNameModifier() == OMPD_unknown ||
4523  C->getNameModifier() == OMPD_cancel) {
4524  IfCond = C->getCondition();
4525  break;
4526  }
4527  }
4528  CGM.getOpenMPRuntime().emitCancelCall(*this, S.getBeginLoc(), IfCond,
4529  S.getCancelRegion());
4530 }
4531 
4534  if (Kind == OMPD_parallel || Kind == OMPD_task ||
4535  Kind == OMPD_target_parallel)
4536  return ReturnBlock;
4537  assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
4538  Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
4539  Kind == OMPD_distribute_parallel_for ||
4540  Kind == OMPD_target_parallel_for ||
4541  Kind == OMPD_teams_distribute_parallel_for ||
4542  Kind == OMPD_target_teams_distribute_parallel_for);
4543  return OMPCancelStack.getExitBlock();
4544 }
4545 
4547  const OMPClause &NC, OMPPrivateScope &PrivateScope,
4548  const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) {
4549  const auto &C = cast<OMPUseDevicePtrClause>(NC);
4550  auto OrigVarIt = C.varlist_begin();
4551  auto InitIt = C.inits().begin();
4552  for (const Expr *PvtVarIt : C.private_copies()) {
4553  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl());
4554  const auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl());
4555  const auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl());
4556 
4557  // In order to identify the right initializer we need to match the
4558  // declaration used by the mapping logic. In some cases we may get
4559  // OMPCapturedExprDecl that refers to the original declaration.
4560  const ValueDecl *MatchingVD = OrigVD;
4561  if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
4562  // OMPCapturedExprDecl are used to privative fields of the current
4563  // structure.
4564  const auto *ME = cast<MemberExpr>(OED->getInit());
4565  assert(isa<CXXThisExpr>(ME->getBase()) &&
4566  "Base should be the current struct!");
4567  MatchingVD = ME->getMemberDecl();
4568  }
4569 
4570  // If we don't have information about the current list item, move on to
4571  // the next one.
4572  auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
4573  if (InitAddrIt == CaptureDeviceAddrMap.end())
4574  continue;
4575 
4576  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, OrigVD,
4577  InitAddrIt, InitVD,
4578  PvtVD]() {
4579  // Initialize the temporary initialization variable with the address we
4580  // get from the runtime library. We have to cast the source address
4581  // because it is always a void *. References are materialized in the
4582  // privatization scope, so the initialization here disregards the fact
4583  // the original variable is a reference.
4584  QualType AddrQTy =
4585  getContext().getPointerType(OrigVD->getType().getNonReferenceType());
4586  llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy);
4587  Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy);
4588  setAddrOfLocalVar(InitVD, InitAddr);
4589 
4590  // Emit private declaration, it will be initialized by the value we
4591  // declaration we just added to the local declarations map.
4592  EmitDecl(*PvtVD);
4593 
4594  // The initialization variables reached its purpose in the emission
4595  // of the previous declaration, so we don't need it anymore.
4596  LocalDeclMap.erase(InitVD);
4597 
4598  // Return the address of the private variable.
4599  return GetAddrOfLocalVar(PvtVD);
4600  });
4601  assert(IsRegistered && "firstprivate var already registered as private");
4602  // Silence the warning about unused variable.
4603  (void)IsRegistered;
4604 
4605  ++OrigVarIt;
4606  ++InitIt;
4607  }
4608 }
4609 
4610 // Generate the instructions for '#pragma omp target data' directive.
4612  const OMPTargetDataDirective &S) {
4613  CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true);
4614 
4615  // Create a pre/post action to signal the privatization of the device pointer.
4616  // This action can be replaced by the OpenMP runtime code generation to
4617  // deactivate privatization.
4618  bool PrivatizeDevicePointers = false;
4619  class DevicePointerPrivActionTy : public PrePostActionTy {
4620  bool &PrivatizeDevicePointers;
4621 
4622  public:
4623  explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
4624  : PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {}
4625  void Enter(CodeGenFunction &CGF) override {
4626  PrivatizeDevicePointers = true;
4627  }
4628  };
4629  DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
4630 
4631  auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers](
4632  CodeGenFunction &CGF, PrePostActionTy &Action) {
4633  auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4635  };
4636 
4637  // Codegen that selects whether to generate the privatization code or not.
4638  auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers,
4639  &InnermostCodeGen](CodeGenFunction &CGF,
4640  PrePostActionTy &Action) {
4641  RegionCodeGenTy RCG(InnermostCodeGen);
4642  PrivatizeDevicePointers = false;
4643 
4644  // Call the pre-action to change the status of PrivatizeDevicePointers if
4645  // needed.
4646  Action.Enter(CGF);
4647 
4648  if (PrivatizeDevicePointers) {
4649  OMPPrivateScope PrivateScope(CGF);
4650  // Emit all instances of the use_device_ptr clause.
4651  for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
4652  CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
4653  Info.CaptureDeviceAddrMap);
4654  (void)PrivateScope.Privatize();
4655  RCG(CGF);
4656  } else {
4657  RCG(CGF);
4658  }
4659  };
4660 
4661  // Forward the provided action to the privatization codegen.
4662  RegionCodeGenTy PrivRCG(PrivCodeGen);
4663  PrivRCG.setAction(Action);
4664 
4665  // Notwithstanding the body of the region is emitted as inlined directive,
4666  // we don't use an inline scope as changes in the references inside the
4667  // region are expected to be visible outside, so we do not privative them.
4668  OMPLexicalScope Scope(CGF, S);
4669  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
4670  PrivRCG);
4671  };
4672 
4673  RegionCodeGenTy RCG(CodeGen);
4674 
4675  // If we don't have target devices, don't bother emitting the data mapping
4676  // code.
4677  if (CGM.getLangOpts().OMPTargetTriples.empty()) {
4678  RCG(*this);
4679  return;
4680  }
4681 
4682  // Check if we have any if clause associated with the directive.
4683  const Expr *IfCond = nullptr;
4684  if (const auto *C = S.getSingleClause<OMPIfClause>())
4685  IfCond = C->getCondition();
4686 
4687  // Check if we have any device clause associated with the directive.
4688  const Expr *Device = nullptr;
4689  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4690  Device = C->getDevice();
4691 
4692  // Set the action to signal privatization of device pointers.
4693  RCG.setAction(PrivAction);
4694 
4695  // Emit region code.
4696  CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
4697  Info);
4698 }
4699 
4701  const OMPTargetEnterDataDirective &S) {
4702  // If we don't have target devices, don't bother emitting the data mapping
4703  // code.
4704  if (CGM.getLangOpts().OMPTargetTriples.empty())
4705  return;
4706 
4707  // Check if we have any if clause associated with the directive.
4708  const Expr *IfCond = nullptr;
4709  if (const auto *C = S.getSingleClause<OMPIfClause>())
4710  IfCond = C->getCondition();
4711 
4712  // Check if we have any device clause associated with the directive.
4713  const Expr *Device = nullptr;
4714  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4715  Device = C->getDevice();
4716 
4717  OMPLexicalScope Scope(*this, S, OMPD_task);
4718  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4719 }
4720 
4722  const OMPTargetExitDataDirective &S) {
4723  // If we don't have target devices, don't bother emitting the data mapping
4724  // code.
4725  if (CGM.getLangOpts().OMPTargetTriples.empty())
4726  return;
4727 
4728  // Check if we have any if clause associated with the directive.
4729  const Expr *IfCond = nullptr;
4730  if (const auto *C = S.getSingleClause<OMPIfClause>())
4731  IfCond = C->getCondition();
4732 
4733  // Check if we have any device clause associated with the directive.
4734  const Expr *Device = nullptr;
4735  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4736  Device = C->getDevice();
4737 
4738  OMPLexicalScope Scope(*this, S, OMPD_task);
4739  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4740 }
4741 
4743  const OMPTargetParallelDirective &S,
4744  PrePostActionTy &Action) {
4745  // Get the captured statement associated with the 'parallel' region.
4746  const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
4747  Action.Enter(CGF);
4748  auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4749  Action.Enter(CGF);
4750  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4751  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4752  CGF.EmitOMPPrivateClause(S, PrivateScope);
4753  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4754  (void)PrivateScope.Privatize();
4756  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4757  // TODO: Add support for clauses.
4758  CGF.EmitStmt(CS->getCapturedStmt());
4759  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4760  };
4761  emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
4764  [](CodeGenFunction &) { return nullptr; });
4765 }
4766 
4768  CodeGenModule &CGM, StringRef ParentName,
4769  const OMPTargetParallelDirective &S) {
4770  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4771  emitTargetParallelRegion(CGF, S, Action);
4772  };
4773  llvm::Function *Fn;
4774  llvm::Constant *Addr;
4775  // Emit target region as a standalone region.
4776  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4777  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4778  assert(Fn && Addr && "Target device function emission failed.");
4779 }
4780 
4782  const OMPTargetParallelDirective &S) {
4783  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4784  emitTargetParallelRegion(CGF, S, Action);
4785  };
4786  emitCommonOMPTargetDirective(*this, S, CodeGen);
4787 }
4788 
4791  PrePostActionTy &Action) {
4792  Action.Enter(CGF);
4793  // Emit directive as a combined directive that consists of two implicit
4794  // directives: 'parallel' with 'for' directive.
4795  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4796  Action.Enter(CGF);
4798  CGF, OMPD_target_parallel_for, S.hasCancel());
4801  };
4802  emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
4804 }
4805 
4807  CodeGenModule &CGM, StringRef ParentName,
4808  const OMPTargetParallelForDirective &S) {
4809  // Emit SPMD target parallel for region as a standalone region.
4810  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4811  emitTargetParallelForRegion(CGF, S, Action);
4812  };
4813  llvm::Function *Fn;
4814  llvm::Constant *Addr;
4815  // Emit target region as a standalone region.
4816  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4817  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4818  assert(Fn && Addr && "Target device function emission failed.");
4819 }
4820 
4822  const OMPTargetParallelForDirective &S) {
4823  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4824  emitTargetParallelForRegion(CGF, S, Action);
4825  };
4826  emitCommonOMPTargetDirective(*this, S, CodeGen);
4827 }
4828 
4829 static void
4832  PrePostActionTy &Action) {
4833  Action.Enter(CGF);
4834  // Emit directive as a combined directive that consists of two implicit
4835  // directives: 'parallel' with 'for' directive.
4836  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4837  Action.Enter(CGF);
4840  };
4841  emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
4843 }
4844 
4846  CodeGenModule &CGM, StringRef ParentName,
4848  // Emit SPMD target parallel for region as a standalone region.
4849  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4850  emitTargetParallelForSimdRegion(CGF, S, Action);
4851  };
4852  llvm::Function *Fn;
4853  llvm::Constant *Addr;
4854  // Emit target region as a standalone region.
4855  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4856  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4857  assert(Fn && Addr && "Target device function emission failed.");
4858 }
4859 
4862  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4863  emitTargetParallelForSimdRegion(CGF, S, Action);
4864  };
4865  emitCommonOMPTargetDirective(*this, S, CodeGen);
4866 }
4867 
4868 /// Emit a helper variable and return corresponding lvalue.
4869 static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
4870  const ImplicitParamDecl *PVD,
4872  const auto *VDecl = cast<VarDecl>(Helper->getDecl());
4873  Privates.addPrivate(VDecl,
4874  [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); });
4875 }
4876 
4879  // Emit outlined function for task construct.
4880  const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);
4881  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
4882  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
4883  const Expr *IfCond = nullptr;
4884  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4885  if (C->getNameModifier() == OMPD_unknown ||
4886  C->getNameModifier() == OMPD_taskloop) {
4887  IfCond = C->getCondition();
4888  break;
4889  }
4890  }
4891 
4892  OMPTaskDataTy Data;
4893  // Check if taskloop must be emitted without taskgroup.
4895  // TODO: Check if we should emit tied or untied task.
4896  Data.Tied = true;
4897  // Set scheduling for taskloop
4898  if (const auto* Clause = S.getSingleClause<OMPGrainsizeClause>()) {
4899  // grainsize clause
4900  Data.Schedule.setInt(/*IntVal=*/false);
4901  Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
4902  } else if (const auto* Clause = S.getSingleClause<OMPNumTasksClause>()) {
4903  // num_tasks clause
4904  Data.Schedule.setInt(/*IntVal=*/true);
4905  Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
4906  }
4907 
4908  auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
4909  // if (PreCond) {
4910  // for (IV in 0..LastIteration) BODY;
4911  // <Final counter/linear vars updates>;
4912  // }
4913  //
4914 
4915  // Emit: if (PreCond) - begin.
4916  // If the condition constant folds and can be elided, avoid emitting the
4917  // whole loop.
4918  bool CondConstant;
4919  llvm::BasicBlock *ContBlock = nullptr;
4920  OMPLoopScope PreInitScope(CGF, S);
4921  if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
4922  if (!CondConstant)
4923  return;
4924  } else {
4925  llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
4926  ContBlock = CGF.createBasicBlock("taskloop.if.end");
4927  emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
4928  CGF.getProfileCount(&S));
4929  CGF.EmitBlock(ThenBlock);
4930  CGF.incrementProfileCounter(&S);
4931  }
4932 
4934  CGF.EmitOMPSimdInit(S);
4935 
4936  OMPPrivateScope LoopScope(CGF);
4937  // Emit helper vars inits.
4938  enum { LowerBound = 5, UpperBound, Stride, LastIter };
4939  auto *I = CS->getCapturedDecl()->param_begin();
4940  auto *LBP = std::next(I, LowerBound);
4941  auto *UBP = std::next(I, UpperBound);
4942  auto *STP = std::next(I, Stride);
4943  auto *LIP = std::next(I, LastIter);
4944  mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
4945  LoopScope);
4946  mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
4947  LoopScope);
4948  mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
4949  mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
4950  LoopScope);
4951  CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
4952  bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4953  (void)LoopScope.Privatize();
4954  // Emit the loop iteration variable.
4955  const Expr *IVExpr = S.getIterationVariable();
4956  const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
4957  CGF.EmitVarDecl(*IVDecl);
4958  CGF.EmitIgnoredExpr(S.getInit());
4959 
4960  // Emit the iterations count variable.
4961  // If it is not a variable, Sema decided to calculate iterations count on
4962  // each iteration (e.g., it is foldable into a constant).
4963  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
4964  CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
4965  // Emit calculation of the iterations count.
4966  CGF.EmitIgnoredExpr(S.getCalcLastIteration());
4967  }
4968 
4969  CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
4970  S.getInc(),
4971  [&S](CodeGenFunction &CGF) {
4972  CGF.EmitOMPLoopBody(S, JumpDest());
4973  CGF.EmitStopPoint(&S);
4974  },
4975  [](CodeGenFunction &) {});
4976  // Emit: if (PreCond) - end.
4977  if (ContBlock) {
4978  CGF.EmitBranch(ContBlock);
4979  CGF.EmitBlock(ContBlock, true);
4980  }
4981  // Emit final copy of the lastprivate variables if IsLastIter != 0.
4982  if (HasLastprivateClause) {
4983  CGF.EmitOMPLastprivateClauseFinal(
4985  CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
4986  CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
4987  (*LIP)->getType(), S.getBeginLoc())));
4988  }
4989  };
4990  auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
4991  IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
4992  const OMPTaskDataTy &Data) {
4993  auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
4994  &Data](CodeGenFunction &CGF, PrePostActionTy &) {
4995  OMPLoopScope PreInitScope(CGF, S);
4996  CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
4997  OutlinedFn, SharedsTy,
4998  CapturedStruct, IfCond, Data);
4999  };
5000  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
5001  CodeGen);
5002  };
5003  if (Data.Nogroup) {
5004  EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data);
5005  } else {
5006  CGM.getOpenMPRuntime().emitTaskgroupRegion(
5007  *this,
5008  [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
5009  PrePostActionTy &Action) {
5010  Action.Enter(CGF);
5011  CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen,
5012  Data);
5013  },
5014  S.getBeginLoc());
5015  }
5016 }
5017 
5019  EmitOMPTaskLoopBasedDirective(S);
5020 }
5021 
5023  const OMPTaskLoopSimdDirective &S) {
5024  EmitOMPTaskLoopBasedDirective(S);
5025 }
5026 
5027 // Generate the instructions for '#pragma omp target update' directive.
5029  const OMPTargetUpdateDirective &S) {
5030  // If we don't have target devices, don't bother emitting the data mapping
5031  // code.
5032  if (CGM.getLangOpts().OMPTargetTriples.empty())
5033  return;
5034 
5035  // Check if we have any if clause associated with the directive.
5036  const Expr *IfCond = nullptr;
5037  if (const auto *C = S.getSingleClause<OMPIfClause>())
5038  IfCond = C->getCondition();
5039 
5040  // Check if we have any device clause associated with the directive.
5041  const Expr *Device = nullptr;
5042  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
5043  Device = C->getDevice();
5044 
5045  OMPLexicalScope Scope(*this, S, OMPD_task);
5046  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
5047 }
5048 
5050  const OMPExecutableDirective &D) {
5051  if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
5052  return;
5053  auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {
5055  emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action);
5056  } else {
5057  OMPPrivateScope LoopGlobals(CGF);
5058  if (const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
5059  for (const Expr *E : LD->counters()) {
5060  const auto *VD = dyn_cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5061  if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) {
5062  LValue GlobLVal = CGF.EmitLValue(E);
5063  LoopGlobals.addPrivate(
5064  VD, [&GlobLVal]() { return GlobLVal.getAddress(); });
5065  }
5066  if (isa<OMPCapturedExprDecl>(VD)) {
5067  // Emit only those that were not explicitly referenced in clauses.
5068  if (!CGF.LocalDeclMap.count(VD))
5069  CGF.EmitVarDecl(*VD);
5070  }
5071  }
5072  for (const auto *C : D.getClausesOfKind<OMPOrderedClause>()) {
5073  if (!C->getNumForLoops())
5074  continue;
5075  for (unsigned I = LD->getCollapsedNumber(),
5076  E = C->getLoopNumIterations().size();
5077  I < E; ++I) {
5078  if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
5079  cast<DeclRefExpr>(C->getLoopCounter(I))->getDecl())) {
5080  // Emit only those that were not explicitly referenced in clauses.
5081  if (!CGF.LocalDeclMap.count(VD))
5082  CGF.EmitVarDecl(*VD);
5083  }
5084  }
5085  }
5086  }
5087  LoopGlobals.Privatize();
5089  }
5090  };
5091  OMPSimdLexicalScope Scope(*this, D);
5092  CGM.getOpenMPRuntime().emitInlinedDirective(
5093  *this,
5094  isOpenMPSimdDirective(D.getDirectiveKind()) ? OMPD_simd
5095  : D.getDirectiveKind(),
5096  CodeGen);
5097 }
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
Definition: CGCall.cpp:652
bool isAggregate() const
Definition: CGValue.h:53
This represents &#39;#pragma omp distribute simd&#39; composite directive.
Definition: StmtOpenMP.h:3271
Expr * getNextUpperBound() const
Definition: StmtOpenMP.h:888
This represents &#39;#pragma omp master&#39; directive.
Definition: StmtOpenMP.h:1454
This represents &#39;#pragma omp task&#39; directive.
Definition: StmtOpenMP.h:1794
static const Decl * getCanonicalDecl(const Decl *D)
Represents a function declaration or definition.
Definition: Decl.h:1748
This represents &#39;thread_limit&#39; clause in the &#39;#pragma omp ...&#39; directive.
static FunctionDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation NLoc, DeclarationName N, QualType T, TypeSourceInfo *TInfo, StorageClass SC, bool isInlineSpecified=false, bool hasWrittenPrototype=true, ConstexprSpecKind ConstexprKind=CSK_unspecified)
Definition: Decl.h:1895
Expr * getUpperBoundVariable() const
Definition: StmtOpenMP.h:856
Other implicit parameter.
Definition: Decl.h:1524
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
LValue getReferenceLValue(CodeGenFunction &CGF, Expr *refExpr) const
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, SourceLocation Loc)
This represents clause &#39;copyin&#39; in the &#39;#pragma omp ...&#39; directives.
Complete object ctor.
Definition: ABI.h:25
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition: Type.h:2554
Scheduling data for loop-based OpenMP directives.
Definition: OpenMPKinds.h:156
A (possibly-)qualified type.
Definition: Type.h:643
capture_init_iterator capture_init_begin()
Retrieve the first initialization argument.
Definition: Stmt.h:3509
bool isArrayType() const
Definition: Type.h:6407
ArrayRef< OMPClause * > clauses()
Definition: StmtOpenMP.h:267
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *V, SourceLocation Loc)
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
Definition: CGDecl.cpp:162
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
Definition: Decl.h:4162
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Definition: CGExpr.cpp:138
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
Definition: Dominators.h:29
static std::pair< LValue, LValue > emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static OMPClauseWithPreInit * get(OMPClause *C)
Stmt - This represents one statement.
Definition: Stmt.h:66
This represents clause &#39;in_reduction&#39; in the &#39;#pragma omp task&#39; directives.
Expr * getLowerBoundVariable() const
Definition: StmtOpenMP.h:848
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
Store of global named registers are always calls to intrinsics.
Definition: CGExpr.cpp:2160
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
Definition: Type.cpp:505
bool requiresCleanups() const
Determine whether this scope requires any cleanups.
void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S)
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
SmallVector< std::pair< OpenMPDependClauseKind, const Expr * >, 4 > Dependences
This represents &#39;#pragma omp for simd&#39; directive.
Definition: StmtOpenMP.h:1204
SmallVector< const Expr *, 4 > LastprivateCopies
QualType getLValueReferenceType(QualType T, bool SpelledAsLValue=true) const
Return the uniqued reference to the type for an lvalue reference to the specified type...
Decl - This represents one declaration (or definition), e.g.
Definition: DeclBase.h:88
This represents &#39;grainsize&#39; clause in the &#39;#pragma omp ...&#39; directive.
This represents &#39;#pragma omp teams distribute parallel for&#39; composite directive.
Definition: StmtOpenMP.h:3682
SourceLocation getBeginLoc() const
Returns starting location of directive kind.
Definition: StmtOpenMP.h:167
bool isNothrow() const
Definition: Decl.cpp:4621
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
This represents &#39;if&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:399
CapturedStmt * getInnermostCapturedStmt()
Get innermost captured statement for the construct.
Definition: StmtOpenMP.h:225
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Definition: Type.h:6300
void EmitOMPOrderedDirective(const OMPOrderedDirective &S)
bool isVolatile() const
Definition: CGValue.h:300
This represents &#39;priority&#39; clause in the &#39;#pragma omp ...&#39; directive.
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
The base class of the type hierarchy.
Definition: Type.h:1418
This represents &#39;#pragma omp target teams distribute&#39; combined directive.
Definition: StmtOpenMP.h:3819
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
Definition: CGExpr.cpp:1925
static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, bool IsSeqCst, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Definition: Stmt.h:3453
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Definition: Type.h:2829
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
const TargetInfo & getTargetInfo() const
Definition: ASTContext.h:693
Expr * getCombinedParForInDistCond() const
Definition: StmtOpenMP.h:976
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
const llvm::function_ref< void(CodeGenFunction &, llvm::Function *, const OMPTaskDataTy &)> TaskGenTy
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters)
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Definition: CGExpr.cpp:2340
Floating point control options.
Definition: LangOptions.h:307
This represents &#39;#pragma omp parallel for&#39; directive.
Definition: StmtOpenMP.h:1575
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value *> &CapturedVars)