clang  10.0.0svn
CGStmtOpenMP.cpp
Go to the documentation of this file.
1 //===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This contains code to emit OpenMP nodes as LLVM code.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CGCleanup.h"
14 #include "CGOpenMPRuntime.h"
15 #include "CodeGenFunction.h"
16 #include "CodeGenModule.h"
17 #include "TargetInfo.h"
18 #include "clang/AST/Stmt.h"
19 #include "clang/AST/StmtOpenMP.h"
20 #include "clang/AST/DeclOpenMP.h"
21 using namespace clang;
22 using namespace CodeGen;
23 
24 namespace {
25 /// Lexical scope for OpenMP executable constructs, that handles correct codegen
26 /// for captured expressions.
27 class OMPLexicalScope : public CodeGenFunction::LexicalScope {
28  void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
29  for (const auto *C : S.clauses()) {
30  if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
31  if (const auto *PreInit =
32  cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
33  for (const auto *I : PreInit->decls()) {
34  if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
35  CGF.EmitVarDecl(cast<VarDecl>(*I));
36  } else {
38  CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
39  CGF.EmitAutoVarCleanups(Emission);
40  }
41  }
42  }
43  }
44  }
45  }
46  CodeGenFunction::OMPPrivateScope InlinedShareds;
47 
48  static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
49  return CGF.LambdaCaptureFields.lookup(VD) ||
50  (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
51  (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl));
52  }
53 
54 public:
55  OMPLexicalScope(
57  const llvm::Optional<OpenMPDirectiveKind> CapturedRegion = llvm::None,
58  const bool EmitPreInitStmt = true)
59  : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
60  InlinedShareds(CGF) {
61  if (EmitPreInitStmt)
62  emitPreInitStmt(CGF, S);
63  if (!CapturedRegion.hasValue())
64  return;
65  assert(S.hasAssociatedStmt() &&
66  "Expected associated statement for inlined directive.");
67  const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
68  for (const auto &C : CS->captures()) {
69  if (C.capturesVariable() || C.capturesVariableByCopy()) {
70  auto *VD = C.getCapturedVar();
71  assert(VD == VD->getCanonicalDecl() &&
72  "Canonical decl must be captured.");
73  DeclRefExpr DRE(
74  CGF.getContext(), const_cast<VarDecl *>(VD),
75  isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
76  InlinedShareds.isGlobalVarCaptured(VD)),
77  VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
78  InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
79  return CGF.EmitLValue(&DRE).getAddress();
80  });
81  }
82  }
83  (void)InlinedShareds.Privatize();
84  }
85 };
86 
87 /// Lexical scope for OpenMP parallel construct, that handles correct codegen
88 /// for captured expressions.
89 class OMPParallelScope final : public OMPLexicalScope {
90  bool EmitPreInitStmt(const OMPExecutableDirective &S) {
92  return !(isOpenMPTargetExecutionDirective(Kind) ||
95  }
96 
97 public:
98  OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
99  : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
100  EmitPreInitStmt(S)) {}
101 };
102 
103 /// Lexical scope for OpenMP teams construct, that handles correct codegen
104 /// for captured expressions.
105 class OMPTeamsScope final : public OMPLexicalScope {
106  bool EmitPreInitStmt(const OMPExecutableDirective &S) {
108  return !isOpenMPTargetExecutionDirective(Kind) &&
110  }
111 
112 public:
113  OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
114  : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
115  EmitPreInitStmt(S)) {}
116 };
117 
118 /// Private scope for OpenMP loop-based directives, that supports capturing
119 /// of used expression from loop statement.
120 class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
121  void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) {
122  CodeGenFunction::OMPMapVars PreCondVars;
123  for (const auto *E : S.counters()) {
124  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
125  (void)PreCondVars.setVarAddr(
126  CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
127  }
128  (void)PreCondVars.apply(CGF);
129  if (const auto *PreInits = cast_or_null<DeclStmt>(S.getPreInits())) {
130  for (const auto *I : PreInits->decls())
131  CGF.EmitVarDecl(cast<VarDecl>(*I));
132  }
133  PreCondVars.restore(CGF);
134  }
135 
136 public:
137  OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S)
138  : CodeGenFunction::RunCleanupsScope(CGF) {
139  emitPreInitStmt(CGF, S);
140  }
141 };
142 
143 class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
144  CodeGenFunction::OMPPrivateScope InlinedShareds;
145 
146  static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
147  return CGF.LambdaCaptureFields.lookup(VD) ||
148  (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
149  (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) &&
150  cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
151  }
152 
153 public:
154  OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
155  : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
156  InlinedShareds(CGF) {
157  for (const auto *C : S.clauses()) {
158  if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
159  if (const auto *PreInit =
160  cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
161  for (const auto *I : PreInit->decls()) {
162  if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
163  CGF.EmitVarDecl(cast<VarDecl>(*I));
164  } else {
166  CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
167  CGF.EmitAutoVarCleanups(Emission);
168  }
169  }
170  }
171  } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
172  for (const Expr *E : UDP->varlists()) {
173  const Decl *D = cast<DeclRefExpr>(E)->getDecl();
174  if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
175  CGF.EmitVarDecl(*OED);
176  }
177  }
178  }
180  CGF.EmitOMPPrivateClause(S, InlinedShareds);
181  if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
182  if (const Expr *E = TG->getReductionRef())
183  CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
184  }
185  const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
186  while (CS) {
187  for (auto &C : CS->captures()) {
188  if (C.capturesVariable() || C.capturesVariableByCopy()) {
189  auto *VD = C.getCapturedVar();
190  assert(VD == VD->getCanonicalDecl() &&
191  "Canonical decl must be captured.");
192  DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
193  isCapturedVar(CGF, VD) ||
194  (CGF.CapturedStmtInfo &&
195  InlinedShareds.isGlobalVarCaptured(VD)),
197  C.getLocation());
198  InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
199  return CGF.EmitLValue(&DRE).getAddress();
200  });
201  }
202  }
203  CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
204  }
205  (void)InlinedShareds.Privatize();
206  }
207 };
208 
209 } // namespace
210 
212  const OMPExecutableDirective &S,
213  const RegionCodeGenTy &CodeGen);
214 
216  if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
217  if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
218  OrigVD = OrigVD->getCanonicalDecl();
219  bool IsCaptured =
220  LambdaCaptureFields.lookup(OrigVD) ||
221  (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
222  (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
223  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), IsCaptured,
224  OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
225  return EmitLValue(&DRE);
226  }
227  }
228  return EmitLValue(E);
229 }
230 
232  ASTContext &C = getContext();
233  llvm::Value *Size = nullptr;
234  auto SizeInChars = C.getTypeSizeInChars(Ty);
235  if (SizeInChars.isZero()) {
236  // getTypeSizeInChars() returns 0 for a VLA.
237  while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) {
238  VlaSizePair VlaSize = getVLASize(VAT);
239  Ty = VlaSize.Type;
240  Size = Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts)
241  : VlaSize.NumElts;
242  }
243  SizeInChars = C.getTypeSizeInChars(Ty);
244  if (SizeInChars.isZero())
245  return llvm::ConstantInt::get(SizeTy, /*V=*/0);
246  return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
247  }
248  return CGM.getSize(SizeInChars);
249 }
250 
252  const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
253  const RecordDecl *RD = S.getCapturedRecordDecl();
254  auto CurField = RD->field_begin();
255  auto CurCap = S.captures().begin();
257  E = S.capture_init_end();
258  I != E; ++I, ++CurField, ++CurCap) {
259  if (CurField->hasCapturedVLAType()) {
260  const VariableArrayType *VAT = CurField->getCapturedVLAType();
261  llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
262  CapturedVars.push_back(Val);
263  } else if (CurCap->capturesThis()) {
264  CapturedVars.push_back(CXXThisValue);
265  } else if (CurCap->capturesVariableByCopy()) {
266  llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
267 
268  // If the field is not a pointer, we need to save the actual value
269  // and load it as a void pointer.
270  if (!CurField->getType()->isAnyPointerType()) {
271  ASTContext &Ctx = getContext();
272  Address DstAddr = CreateMemTemp(
273  Ctx.getUIntPtrType(),
274  Twine(CurCap->getCapturedVar()->getName(), ".casted"));
275  LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
276 
277  llvm::Value *SrcAddrVal = EmitScalarConversion(
278  DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
279  Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
280  LValue SrcLV =
281  MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
282 
283  // Store the value using the source type pointer.
284  EmitStoreThroughLValue(RValue::get(CV), SrcLV);
285 
286  // Load the value using the destination type pointer.
287  CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
288  }
289  CapturedVars.push_back(CV);
290  } else {
291  assert(CurCap->capturesVariable() && "Expected capture by reference.");
292  CapturedVars.push_back(EmitLValue(*I).getAddress().getPointer());
293  }
294  }
295 }
296 
298  QualType DstType, StringRef Name,
299  LValue AddrLV) {
300  ASTContext &Ctx = CGF.getContext();
301 
302  llvm::Value *CastedPtr = CGF.EmitScalarConversion(
303  AddrLV.getAddress().getPointer(), Ctx.getUIntPtrType(),
304  Ctx.getPointerType(DstType), Loc);
305  Address TmpAddr =
306  CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType))
307  .getAddress();
308  return TmpAddr;
309 }
310 
312  if (T->isLValueReferenceType())
313  return C.getLValueReferenceType(
315  /*SpelledAsLValue=*/false);
316  if (T->isPointerType())
318  if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
319  if (const auto *VLA = dyn_cast<VariableArrayType>(A))
320  return getCanonicalParamType(C, VLA->getElementType());
321  if (!A->isVariablyModifiedType())
322  return C.getCanonicalType(T);
323  }
324  return C.getCanonicalParamType(T);
325 }
326 
327 namespace {
328  /// Contains required data for proper outlined function codegen.
329  struct FunctionOptions {
330  /// Captured statement for which the function is generated.
331  const CapturedStmt *S = nullptr;
332  /// true if cast to/from UIntPtr is required for variables captured by
333  /// value.
334  const bool UIntPtrCastRequired = true;
335  /// true if only casted arguments must be registered as local args or VLA
336  /// sizes.
337  const bool RegisterCastedArgsOnly = false;
338  /// Name of the generated function.
339  const StringRef FunctionName;
340  explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
341  bool RegisterCastedArgsOnly,
342  StringRef FunctionName)
343  : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
344  RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
345  FunctionName(FunctionName) {}
346  };
347 }
348 
349 static llvm::Function *emitOutlinedFunctionPrologue(
350  CodeGenFunction &CGF, FunctionArgList &Args,
351  llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
352  &LocalAddrs,
353  llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
354  &VLASizes,
355  llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
356  const CapturedDecl *CD = FO.S->getCapturedDecl();
357  const RecordDecl *RD = FO.S->getCapturedRecordDecl();
358  assert(CD->hasBody() && "missing CapturedDecl body");
359 
360  CXXThisValue = nullptr;
361  // Build the argument list.
362  CodeGenModule &CGM = CGF.CGM;
363  ASTContext &Ctx = CGM.getContext();
364  FunctionArgList TargetArgs;
365  Args.append(CD->param_begin(),
366  std::next(CD->param_begin(), CD->getContextParamPosition()));
367  TargetArgs.append(
368  CD->param_begin(),
369  std::next(CD->param_begin(), CD->getContextParamPosition()));
370  auto I = FO.S->captures().begin();
371  FunctionDecl *DebugFunctionDecl = nullptr;
372  if (!FO.UIntPtrCastRequired) {
374  QualType FunctionTy = Ctx.getFunctionType(Ctx.VoidTy, llvm::None, EPI);
375  DebugFunctionDecl = FunctionDecl::Create(
376  Ctx, Ctx.getTranslationUnitDecl(), FO.S->getBeginLoc(),
377  SourceLocation(), DeclarationName(), FunctionTy,
378  Ctx.getTrivialTypeSourceInfo(FunctionTy), SC_Static,
379  /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/false);
380  }
381  for (const FieldDecl *FD : RD->fields()) {
382  QualType ArgType = FD->getType();
383  IdentifierInfo *II = nullptr;
384  VarDecl *CapVar = nullptr;
385 
386  // If this is a capture by copy and the type is not a pointer, the outlined
387  // function argument type should be uintptr and the value properly casted to
388  // uintptr. This is necessary given that the runtime library is only able to
389  // deal with pointers. We can pass in the same way the VLA type sizes to the
390  // outlined function.
391  if (FO.UIntPtrCastRequired &&
392  ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
393  I->capturesVariableArrayType()))
394  ArgType = Ctx.getUIntPtrType();
395 
396  if (I->capturesVariable() || I->capturesVariableByCopy()) {
397  CapVar = I->getCapturedVar();
398  II = CapVar->getIdentifier();
399  } else if (I->capturesThis()) {
400  II = &Ctx.Idents.get("this");
401  } else {
402  assert(I->capturesVariableArrayType());
403  II = &Ctx.Idents.get("vla");
404  }
405  if (ArgType->isVariablyModifiedType())
406  ArgType = getCanonicalParamType(Ctx, ArgType);
407  VarDecl *Arg;
408  if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
409  Arg = ParmVarDecl::Create(
410  Ctx, DebugFunctionDecl,
411  CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),
412  CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
413  /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
414  } else {
415  Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
416  II, ArgType, ImplicitParamDecl::Other);
417  }
418  Args.emplace_back(Arg);
419  // Do not cast arguments if we emit function with non-original types.
420  TargetArgs.emplace_back(
421  FO.UIntPtrCastRequired
422  ? Arg
423  : CGM.getOpenMPRuntime().translateParameter(FD, Arg));
424  ++I;
425  }
426  Args.append(
427  std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
428  CD->param_end());
429  TargetArgs.append(
430  std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
431  CD->param_end());
432 
433  // Create the function declaration.
434  const CGFunctionInfo &FuncInfo =
435  CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, TargetArgs);
436  llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
437 
438  auto *F =
440  FO.FunctionName, &CGM.getModule());
441  CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
442  if (CD->isNothrow())
443  F->setDoesNotThrow();
444  F->setDoesNotRecurse();
445 
446  // Generate the function.
447  CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
448  FO.S->getBeginLoc(), CD->getBody()->getBeginLoc());
449  unsigned Cnt = CD->getContextParamPosition();
450  I = FO.S->captures().begin();
451  for (const FieldDecl *FD : RD->fields()) {
452  // Do not map arguments if we emit function with non-original types.
453  Address LocalAddr(Address::invalid());
454  if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
455  LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
456  TargetArgs[Cnt]);
457  } else {
458  LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
459  }
460  // If we are capturing a pointer by copy we don't need to do anything, just
461  // use the value that we get from the arguments.
462  if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
463  const VarDecl *CurVD = I->getCapturedVar();
464  if (!FO.RegisterCastedArgsOnly)
465  LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
466  ++Cnt;
467  ++I;
468  continue;
469  }
470 
471  LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
473  if (FD->hasCapturedVLAType()) {
474  if (FO.UIntPtrCastRequired) {
475  ArgLVal = CGF.MakeAddrLValue(
476  castValueFromUintptr(CGF, I->getLocation(), FD->getType(),
477  Args[Cnt]->getName(), ArgLVal),
479  }
480  llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
481  const VariableArrayType *VAT = FD->getCapturedVLAType();
482  VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg);
483  } else if (I->capturesVariable()) {
484  const VarDecl *Var = I->getCapturedVar();
485  QualType VarTy = Var->getType();
486  Address ArgAddr = ArgLVal.getAddress();
487  if (ArgLVal.getType()->isLValueReferenceType()) {
488  ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
489  } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) {
490  assert(ArgLVal.getType()->isPointerType());
491  ArgAddr = CGF.EmitLoadOfPointer(
492  ArgAddr, ArgLVal.getType()->castAs<PointerType>());
493  }
494  if (!FO.RegisterCastedArgsOnly) {
495  LocalAddrs.insert(
496  {Args[Cnt],
497  {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}});
498  }
499  } else if (I->capturesVariableByCopy()) {
500  assert(!FD->getType()->isAnyPointerType() &&
501  "Not expecting a captured pointer.");
502  const VarDecl *Var = I->getCapturedVar();
503  LocalAddrs.insert({Args[Cnt],
504  {Var, FO.UIntPtrCastRequired
506  CGF, I->getLocation(), FD->getType(),
507  Args[Cnt]->getName(), ArgLVal)
508  : ArgLVal.getAddress()}});
509  } else {
510  // If 'this' is captured, load it into CXXThisValue.
511  assert(I->capturesThis());
512  CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
513  LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress()}});
514  }
515  ++Cnt;
516  ++I;
517  }
518 
519  return F;
520 }
521 
522 llvm::Function *
524  assert(
525  CapturedStmtInfo &&
526  "CapturedStmtInfo should be set when generating the captured function");
527  const CapturedDecl *CD = S.getCapturedDecl();
528  // Build the argument list.
529  bool NeedWrapperFunction =
530  getDebugInfo() &&
531  CGM.getCodeGenOpts().getDebugInfo() >= codegenoptions::LimitedDebugInfo;
532  FunctionArgList Args;
533  llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
534  llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
535  SmallString<256> Buffer;
536  llvm::raw_svector_ostream Out(Buffer);
537  Out << CapturedStmtInfo->getHelperName();
538  if (NeedWrapperFunction)
539  Out << "_debug__";
540  FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
541  Out.str());
542  llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs,
543  VLASizes, CXXThisValue, FO);
544  CodeGenFunction::OMPPrivateScope LocalScope(*this);
545  for (const auto &LocalAddrPair : LocalAddrs) {
546  if (LocalAddrPair.second.first) {
547  LocalScope.addPrivate(LocalAddrPair.second.first, [&LocalAddrPair]() {
548  return LocalAddrPair.second.second;
549  });
550  }
551  }
552  (void)LocalScope.Privatize();
553  for (const auto &VLASizePair : VLASizes)
554  VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
555  PGO.assignRegionCounters(GlobalDecl(CD), F);
556  CapturedStmtInfo->EmitBody(*this, CD->getBody());
557  (void)LocalScope.ForceCleanup();
558  FinishFunction(CD->getBodyRBrace());
559  if (!NeedWrapperFunction)
560  return F;
561 
562  FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
563  /*RegisterCastedArgsOnly=*/true,
564  CapturedStmtInfo->getHelperName());
565  CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
566  WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
567  Args.clear();
568  LocalAddrs.clear();
569  VLASizes.clear();
570  llvm::Function *WrapperF =
571  emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
572  WrapperCGF.CXXThisValue, WrapperFO);
574  for (const auto *Arg : Args) {
576  auto I = LocalAddrs.find(Arg);
577  if (I != LocalAddrs.end()) {
578  LValue LV = WrapperCGF.MakeAddrLValue(
579  I->second.second,
580  I->second.first ? I->second.first->getType() : Arg->getType(),
582  CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
583  } else {
584  auto EI = VLASizes.find(Arg);
585  if (EI != VLASizes.end()) {
586  CallArg = EI->second.second;
587  } else {
588  LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
589  Arg->getType(),
591  CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
592  }
593  }
594  CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
595  }
596  CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.getBeginLoc(),
597  F, CallArgs);
598  WrapperCGF.FinishFunction();
599  return WrapperF;
600 }
601 
602 //===----------------------------------------------------------------------===//
603 // OpenMP Directive Emission
604 //===----------------------------------------------------------------------===//
606  Address DestAddr, Address SrcAddr, QualType OriginalType,
607  const llvm::function_ref<void(Address, Address)> CopyGen) {
608  // Perform element-by-element initialization.
609  QualType ElementTy;
610 
611  // Drill down to the base element type on both arrays.
612  const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
613  llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
614  SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
615 
616  llvm::Value *SrcBegin = SrcAddr.getPointer();
617  llvm::Value *DestBegin = DestAddr.getPointer();
618  // Cast from pointer to array type to pointer to single element.
619  llvm::Value *DestEnd = Builder.CreateGEP(DestBegin, NumElements);
620  // The basic structure here is a while-do loop.
621  llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body");
622  llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done");
623  llvm::Value *IsEmpty =
624  Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
625  Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
626 
627  // Enter the loop body, making that address the current address.
628  llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
629  EmitBlock(BodyBB);
630 
631  CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
632 
633  llvm::PHINode *SrcElementPHI =
634  Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
635  SrcElementPHI->addIncoming(SrcBegin, EntryBB);
636  Address SrcElementCurrent =
637  Address(SrcElementPHI,
638  SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
639 
640  llvm::PHINode *DestElementPHI =
641  Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
642  DestElementPHI->addIncoming(DestBegin, EntryBB);
643  Address DestElementCurrent =
644  Address(DestElementPHI,
645  DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
646 
647  // Emit copy.
648  CopyGen(DestElementCurrent, SrcElementCurrent);
649 
650  // Shift the address forward by one element.
651  llvm::Value *DestElementNext = Builder.CreateConstGEP1_32(
652  DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
653  llvm::Value *SrcElementNext = Builder.CreateConstGEP1_32(
654  SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
655  // Check whether we've reached the end.
656  llvm::Value *Done =
657  Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
658  Builder.CreateCondBr(Done, DoneBB, BodyBB);
659  DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
660  SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
661 
662  // Done.
663  EmitBlock(DoneBB, /*IsFinished=*/true);
664 }
665 
666 void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
667  Address SrcAddr, const VarDecl *DestVD,
668  const VarDecl *SrcVD, const Expr *Copy) {
669  if (OriginalType->isArrayType()) {
670  const auto *BO = dyn_cast<BinaryOperator>(Copy);
671  if (BO && BO->getOpcode() == BO_Assign) {
672  // Perform simple memcpy for simple copying.
673  LValue Dest = MakeAddrLValue(DestAddr, OriginalType);
674  LValue Src = MakeAddrLValue(SrcAddr, OriginalType);
675  EmitAggregateAssign(Dest, Src, OriginalType);
676  } else {
677  // For arrays with complex element types perform element by element
678  // copying.
679  EmitOMPAggregateAssign(
680  DestAddr, SrcAddr, OriginalType,
681  [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
682  // Working with the single array element, so have to remap
683  // destination and source variables to corresponding array
684  // elements.
686  Remap.addPrivate(DestVD, [DestElement]() { return DestElement; });
687  Remap.addPrivate(SrcVD, [SrcElement]() { return SrcElement; });
688  (void)Remap.Privatize();
689  EmitIgnoredExpr(Copy);
690  });
691  }
692  } else {
693  // Remap pseudo source variable to private copy.
695  Remap.addPrivate(SrcVD, [SrcAddr]() { return SrcAddr; });
696  Remap.addPrivate(DestVD, [DestAddr]() { return DestAddr; });
697  (void)Remap.Privatize();
698  // Emit copying of the whole variable.
699  EmitIgnoredExpr(Copy);
700  }
701 }
702 
704  OMPPrivateScope &PrivateScope) {
705  if (!HaveInsertPoint())
706  return false;
707  bool DeviceConstTarget =
708  getLangOpts().OpenMPIsDevice &&
710  bool FirstprivateIsLastprivate = false;
711  llvm::DenseSet<const VarDecl *> Lastprivates;
712  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
713  for (const auto *D : C->varlists())
714  Lastprivates.insert(
715  cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl());
716  }
717  llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
719  getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind());
720  // Force emission of the firstprivate copy if the directive does not emit
721  // outlined function, like omp for, omp simd, omp distribute etc.
722  bool MustEmitFirstprivateCopy =
723  CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
724  for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
725  auto IRef = C->varlist_begin();
726  auto InitsRef = C->inits().begin();
727  for (const Expr *IInit : C->private_copies()) {
728  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
729  bool ThisFirstprivateIsLastprivate =
730  Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
731  const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
732  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
733  if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
734  !FD->getType()->isReferenceType() &&
735  (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
736  EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
737  ++IRef;
738  ++InitsRef;
739  continue;
740  }
741  // Do not emit copy for firstprivate constant variables in target regions,
742  // captured by reference.
743  if (DeviceConstTarget && OrigVD->getType().isConstant(getContext()) &&
744  FD && FD->getType()->isReferenceType() &&
745  (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
746  (void)CGM.getOpenMPRuntime().registerTargetFirstprivateCopy(*this,
747  OrigVD);
748  ++IRef;
749  ++InitsRef;
750  continue;
751  }
752  FirstprivateIsLastprivate =
753  FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
754  if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
755  const auto *VDInit =
756  cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
757  bool IsRegistered;
758  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
759  /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
760  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
761  LValue OriginalLVal;
762  if (!FD) {
763  // Check if the firstprivate variable is just a constant value.
764  ConstantEmission CE = tryEmitAsConstant(&DRE);
765  if (CE && !CE.isReference()) {
766  // Constant value, no need to create a copy.
767  ++IRef;
768  ++InitsRef;
769  continue;
770  }
771  if (CE && CE.isReference()) {
772  OriginalLVal = CE.getReferenceLValue(*this, &DRE);
773  } else {
774  assert(!CE && "Expected non-constant firstprivate.");
775  OriginalLVal = EmitLValue(&DRE);
776  }
777  } else {
778  OriginalLVal = EmitLValue(&DRE);
779  }
780  QualType Type = VD->getType();
781  if (Type->isArrayType()) {
782  // Emit VarDecl with copy init for arrays.
783  // Get the address of the original variable captured in current
784  // captured region.
785  IsRegistered = PrivateScope.addPrivate(
786  OrigVD, [this, VD, Type, OriginalLVal, VDInit]() {
787  AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
788  const Expr *Init = VD->getInit();
789  if (!isa<CXXConstructExpr>(Init) ||
790  isTrivialInitializer(Init)) {
791  // Perform simple memcpy.
792  LValue Dest =
793  MakeAddrLValue(Emission.getAllocatedAddress(), Type);
794  EmitAggregateAssign(Dest, OriginalLVal, Type);
795  } else {
796  EmitOMPAggregateAssign(
797  Emission.getAllocatedAddress(), OriginalLVal.getAddress(),
798  Type,
799  [this, VDInit, Init](Address DestElement,
800  Address SrcElement) {
801  // Clean up any temporaries needed by the
802  // initialization.
803  RunCleanupsScope InitScope(*this);
804  // Emit initialization for single element.
805  setAddrOfLocalVar(VDInit, SrcElement);
806  EmitAnyExprToMem(Init, DestElement,
807  Init->getType().getQualifiers(),
808  /*IsInitializer*/ false);
809  LocalDeclMap.erase(VDInit);
810  });
811  }
812  EmitAutoVarCleanups(Emission);
813  return Emission.getAllocatedAddress();
814  });
815  } else {
816  Address OriginalAddr = OriginalLVal.getAddress();
817  IsRegistered = PrivateScope.addPrivate(
818  OrigVD, [this, VDInit, OriginalAddr, VD]() {
819  // Emit private VarDecl with copy init.
820  // Remap temp VDInit variable to the address of the original
821  // variable (for proper handling of captured global variables).
822  setAddrOfLocalVar(VDInit, OriginalAddr);
823  EmitDecl(*VD);
824  LocalDeclMap.erase(VDInit);
825  return GetAddrOfLocalVar(VD);
826  });
827  }
828  assert(IsRegistered &&
829  "firstprivate var already registered as private");
830  // Silence the warning about unused variable.
831  (void)IsRegistered;
832  }
833  ++IRef;
834  ++InitsRef;
835  }
836  }
837  return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
838 }
839 
841  const OMPExecutableDirective &D,
842  CodeGenFunction::OMPPrivateScope &PrivateScope) {
843  if (!HaveInsertPoint())
844  return;
845  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
846  for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
847  auto IRef = C->varlist_begin();
848  for (const Expr *IInit : C->private_copies()) {
849  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
850  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
851  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
852  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
853  // Emit private VarDecl with copy init.
854  EmitDecl(*VD);
855  return GetAddrOfLocalVar(VD);
856  });
857  assert(IsRegistered && "private var already registered as private");
858  // Silence the warning about unused variable.
859  (void)IsRegistered;
860  }
861  ++IRef;
862  }
863  }
864 }
865 
867  if (!HaveInsertPoint())
868  return false;
869  // threadprivate_var1 = master_threadprivate_var1;
870  // operator=(threadprivate_var2, master_threadprivate_var2);
871  // ...
872  // __kmpc_barrier(&loc, global_tid);
874  llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
875  for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
876  auto IRef = C->varlist_begin();
877  auto ISrcRef = C->source_exprs().begin();
878  auto IDestRef = C->destination_exprs().begin();
879  for (const Expr *AssignOp : C->assignment_ops()) {
880  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
881  QualType Type = VD->getType();
882  if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
883  // Get the address of the master variable. If we are emitting code with
884  // TLS support, the address is passed from the master as field in the
885  // captured declaration.
886  Address MasterAddr = Address::invalid();
887  if (getLangOpts().OpenMPUseTLS &&
888  getContext().getTargetInfo().isTLSSupported()) {
889  assert(CapturedStmtInfo->lookup(VD) &&
890  "Copyin threadprivates should have been captured!");
891  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), true,
892  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
893  MasterAddr = EmitLValue(&DRE).getAddress();
894  LocalDeclMap.erase(VD);
895  } else {
896  MasterAddr =
897  Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
898  : CGM.GetAddrOfGlobal(VD),
899  getContext().getDeclAlign(VD));
900  }
901  // Get the address of the threadprivate variable.
902  Address PrivateAddr = EmitLValue(*IRef).getAddress();
903  if (CopiedVars.size() == 1) {
904  // At first check if current thread is a master thread. If it is, no
905  // need to copy data.
906  CopyBegin = createBasicBlock("copyin.not.master");
907  CopyEnd = createBasicBlock("copyin.not.master.end");
908  Builder.CreateCondBr(
909  Builder.CreateICmpNE(
910  Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy),
911  Builder.CreatePtrToInt(PrivateAddr.getPointer(),
912  CGM.IntPtrTy)),
913  CopyBegin, CopyEnd);
914  EmitBlock(CopyBegin);
915  }
916  const auto *SrcVD =
917  cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
918  const auto *DestVD =
919  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
920  EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
921  }
922  ++IRef;
923  ++ISrcRef;
924  ++IDestRef;
925  }
926  }
927  if (CopyEnd) {
928  // Exit out of copying procedure for non-master thread.
929  EmitBlock(CopyEnd, /*IsFinished=*/true);
930  return true;
931  }
932  return false;
933 }
934 
936  const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
937  if (!HaveInsertPoint())
938  return false;
939  bool HasAtLeastOneLastprivate = false;
942  const auto *LoopDirective = cast<OMPLoopDirective>(&D);
943  for (const Expr *C : LoopDirective->counters()) {
944  SIMDLCVs.insert(
945  cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
946  }
947  }
948  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
949  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
950  HasAtLeastOneLastprivate = true;
952  !getLangOpts().OpenMPSimd)
953  break;
954  auto IRef = C->varlist_begin();
955  auto IDestRef = C->destination_exprs().begin();
956  for (const Expr *IInit : C->private_copies()) {
957  // Keep the address of the original variable for future update at the end
958  // of the loop.
959  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
960  // Taskloops do not require additional initialization, it is done in
961  // runtime support library.
962  if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
963  const auto *DestVD =
964  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
965  PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() {
966  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
967  /*RefersToEnclosingVariableOrCapture=*/
968  CapturedStmtInfo->lookup(OrigVD) != nullptr,
969  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
970  return EmitLValue(&DRE).getAddress();
971  });
972  // Check if the variable is also a firstprivate: in this case IInit is
973  // not generated. Initialization of this variable will happen in codegen
974  // for 'firstprivate' clause.
975  if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
976  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
977  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
978  // Emit private VarDecl with copy init.
979  EmitDecl(*VD);
980  return GetAddrOfLocalVar(VD);
981  });
982  assert(IsRegistered &&
983  "lastprivate var already registered as private");
984  (void)IsRegistered;
985  }
986  }
987  ++IRef;
988  ++IDestRef;
989  }
990  }
991  return HasAtLeastOneLastprivate;
992 }
993 
995  const OMPExecutableDirective &D, bool NoFinals,
996  llvm::Value *IsLastIterCond) {
997  if (!HaveInsertPoint())
998  return;
999  // Emit following code:
1000  // if (<IsLastIterCond>) {
1001  // orig_var1 = private_orig_var1;
1002  // ...
1003  // orig_varn = private_orig_varn;
1004  // }
1005  llvm::BasicBlock *ThenBB = nullptr;
1006  llvm::BasicBlock *DoneBB = nullptr;
1007  if (IsLastIterCond) {
1008  ThenBB = createBasicBlock(".omp.lastprivate.then");
1009  DoneBB = createBasicBlock(".omp.lastprivate.done");
1010  Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1011  EmitBlock(ThenBB);
1012  }
1013  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1014  llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1015  if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1016  auto IC = LoopDirective->counters().begin();
1017  for (const Expr *F : LoopDirective->finals()) {
1018  const auto *D =
1019  cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1020  if (NoFinals)
1021  AlreadyEmittedVars.insert(D);
1022  else
1023  LoopCountersAndUpdates[D] = F;
1024  ++IC;
1025  }
1026  }
1027  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1028  auto IRef = C->varlist_begin();
1029  auto ISrcRef = C->source_exprs().begin();
1030  auto IDestRef = C->destination_exprs().begin();
1031  for (const Expr *AssignOp : C->assignment_ops()) {
1032  const auto *PrivateVD =
1033  cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1034  QualType Type = PrivateVD->getType();
1035  const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1036  if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1037  // If lastprivate variable is a loop control variable for loop-based
1038  // directive, update its value before copyin back to original
1039  // variable.
1040  if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1041  EmitIgnoredExpr(FinalExpr);
1042  const auto *SrcVD =
1043  cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1044  const auto *DestVD =
1045  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1046  // Get the address of the original variable.
1047  Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1048  // Get the address of the private variable.
1049  Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
1050  if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1051  PrivateAddr =
1052  Address(Builder.CreateLoad(PrivateAddr),
1053  getNaturalTypeAlignment(RefTy->getPointeeType()));
1054  EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1055  }
1056  ++IRef;
1057  ++ISrcRef;
1058  ++IDestRef;
1059  }
1060  if (const Expr *PostUpdate = C->getPostUpdateExpr())
1061  EmitIgnoredExpr(PostUpdate);
1062  }
1063  if (IsLastIterCond)
1064  EmitBlock(DoneBB, /*IsFinished=*/true);
1065 }
1066 
1068  const OMPExecutableDirective &D,
1069  CodeGenFunction::OMPPrivateScope &PrivateScope) {
1070  if (!HaveInsertPoint())
1071  return;
1074  SmallVector<const Expr *, 4> ReductionOps;
1077  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1078  auto IPriv = C->privates().begin();
1079  auto IRed = C->reduction_ops().begin();
1080  auto ILHS = C->lhs_exprs().begin();
1081  auto IRHS = C->rhs_exprs().begin();
1082  for (const Expr *Ref : C->varlists()) {
1083  Shareds.emplace_back(Ref);
1084  Privates.emplace_back(*IPriv);
1085  ReductionOps.emplace_back(*IRed);
1086  LHSs.emplace_back(*ILHS);
1087  RHSs.emplace_back(*IRHS);
1088  std::advance(IPriv, 1);
1089  std::advance(IRed, 1);
1090  std::advance(ILHS, 1);
1091  std::advance(IRHS, 1);
1092  }
1093  }
1094  ReductionCodeGen RedCG(Shareds, Privates, ReductionOps);
1095  unsigned Count = 0;
1096  auto ILHS = LHSs.begin();
1097  auto IRHS = RHSs.begin();
1098  auto IPriv = Privates.begin();
1099  for (const Expr *IRef : Shareds) {
1100  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1101  // Emit private VarDecl with reduction init.
1102  RedCG.emitSharedLValue(*this, Count);
1103  RedCG.emitAggregateType(*this, Count);
1104  AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);
1105  RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
1106  RedCG.getSharedLValue(Count),
1107  [&Emission](CodeGenFunction &CGF) {
1108  CGF.EmitAutoVarInit(Emission);
1109  return true;
1110  });
1111  EmitAutoVarCleanups(Emission);
1112  Address BaseAddr = RedCG.adjustPrivateAddress(
1113  *this, Count, Emission.getAllocatedAddress());
1114  bool IsRegistered = PrivateScope.addPrivate(
1115  RedCG.getBaseDecl(Count), [BaseAddr]() { return BaseAddr; });
1116  assert(IsRegistered && "private var already registered as private");
1117  // Silence the warning about unused variable.
1118  (void)IsRegistered;
1119 
1120  const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1121  const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1122  QualType Type = PrivateVD->getType();
1123  bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1124  if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1125  // Store the address of the original variable associated with the LHS
1126  // implicit variable.
1127  PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
1128  return RedCG.getSharedLValue(Count).getAddress();
1129  });
1130  PrivateScope.addPrivate(
1131  RHSVD, [this, PrivateVD]() { return GetAddrOfLocalVar(PrivateVD); });
1132  } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1133  isa<ArraySubscriptExpr>(IRef)) {
1134  // Store the address of the original variable associated with the LHS
1135  // implicit variable.
1136  PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
1137  return RedCG.getSharedLValue(Count).getAddress();
1138  });
1139  PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() {
1140  return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD),
1141  ConvertTypeForMem(RHSVD->getType()),
1142  "rhs.begin");
1143  });
1144  } else {
1145  QualType Type = PrivateVD->getType();
1146  bool IsArray = getContext().getAsArrayType(Type) != nullptr;
1147  Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress();
1148  // Store the address of the original variable associated with the LHS
1149  // implicit variable.
1150  if (IsArray) {
1151  OriginalAddr = Builder.CreateElementBitCast(
1152  OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin");
1153  }
1154  PrivateScope.addPrivate(LHSVD, [OriginalAddr]() { return OriginalAddr; });
1155  PrivateScope.addPrivate(
1156  RHSVD, [this, PrivateVD, RHSVD, IsArray]() {
1157  return IsArray
1158  ? Builder.CreateElementBitCast(
1159  GetAddrOfLocalVar(PrivateVD),
1160  ConvertTypeForMem(RHSVD->getType()), "rhs.begin")
1161  : GetAddrOfLocalVar(PrivateVD);
1162  });
1163  }
1164  ++ILHS;
1165  ++IRHS;
1166  ++IPriv;
1167  ++Count;
1168  }
1169 }
1170 
1172  const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1173  if (!HaveInsertPoint())
1174  return;
1179  bool HasAtLeastOneReduction = false;
1180  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1181  HasAtLeastOneReduction = true;
1182  Privates.append(C->privates().begin(), C->privates().end());
1183  LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1184  RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1185  ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1186  }
1187  if (HasAtLeastOneReduction) {
1188  bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1190  ReductionKind == OMPD_simd;
1191  bool SimpleReduction = ReductionKind == OMPD_simd;
1192  // Emit nowait reduction if nowait clause is present or directive is a
1193  // parallel directive (it always has implicit barrier).
1194  CGM.getOpenMPRuntime().emitReduction(
1195  *this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1196  {WithNowait, SimpleReduction, ReductionKind});
1197  }
1198 }
1199 
1201  CodeGenFunction &CGF, const OMPExecutableDirective &D,
1202  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1203  if (!CGF.HaveInsertPoint())
1204  return;
1205  llvm::BasicBlock *DoneBB = nullptr;
1206  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1207  if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
1208  if (!DoneBB) {
1209  if (llvm::Value *Cond = CondGen(CGF)) {
1210  // If the first post-update expression is found, emit conditional
1211  // block if it was requested.
1212  llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1213  DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1214  CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1215  CGF.EmitBlock(ThenBB);
1216  }
1217  }
1218  CGF.EmitIgnoredExpr(PostUpdate);
1219  }
1220  }
1221  if (DoneBB)
1222  CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1223 }
1224 
1225 namespace {
1226 /// Codegen lambda for appending distribute lower and upper bounds to outlined
1227 /// parallel function. This is necessary for combined constructs such as
1228 /// 'distribute parallel for'
1229 typedef llvm::function_ref<void(CodeGenFunction &,
1230  const OMPExecutableDirective &,
1232  CodeGenBoundParametersTy;
1233 } // anonymous namespace
1234 
1236  CodeGenFunction &CGF, const OMPExecutableDirective &S,
1237  OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1238  const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1239  const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1240  llvm::Function *OutlinedFn =
1241  CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
1242  S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
1243  if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
1244  CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1245  llvm::Value *NumThreads =
1246  CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1247  /*IgnoreResultAssign=*/true);
1248  CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
1249  CGF, NumThreads, NumThreadsClause->getBeginLoc());
1250  }
1251  if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
1252  CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1253  CGF.CGM.getOpenMPRuntime().emitProcBindClause(
1254  CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1255  }
1256  const Expr *IfCond = nullptr;
1257  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1258  if (C->getNameModifier() == OMPD_unknown ||
1259  C->getNameModifier() == OMPD_parallel) {
1260  IfCond = C->getCondition();
1261  break;
1262  }
1263  }
1264 
1265  OMPParallelScope Scope(CGF, S);
1267  // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
1268  // lower and upper bounds with the pragma 'for' chunking mechanism.
1269  // The following lambda takes care of appending the lower and upper bound
1270  // parameters when necessary
1271  CodeGenBoundParameters(CGF, S, CapturedVars);
1272  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
1273  CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn,
1274  CapturedVars, IfCond);
1275 }
1276 
1278  const OMPExecutableDirective &,
1280 
1282  // Emit parallel region as a standalone region.
1283  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1284  Action.Enter(CGF);
1285  OMPPrivateScope PrivateScope(CGF);
1286  bool Copyins = CGF.EmitOMPCopyinClause(S);
1287  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1288  if (Copyins) {
1289  // Emit implicit barrier to synchronize threads and avoid data races on
1290  // propagation master's thread values of threadprivate variables to local
1291  // instances of that variables of all other implicit threads.
1292  CGF.CGM.getOpenMPRuntime().emitBarrierCall(
1293  CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
1294  /*ForceSimpleCall=*/true);
1295  }
1296  CGF.EmitOMPPrivateClause(S, PrivateScope);
1297  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1298  (void)PrivateScope.Privatize();
1299  CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1300  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
1301  };
1302  emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
1305  [](CodeGenFunction &) { return nullptr; });
1306 }
1307 
1309  JumpDest LoopExit) {
1310  RunCleanupsScope BodyScope(*this);
1311  // Update counters values on current iteration.
1312  for (const Expr *UE : D.updates())
1313  EmitIgnoredExpr(UE);
1314  // Update the linear variables.
1315  // In distribute directives only loop counters may be marked as linear, no
1316  // need to generate the code for them.
1318  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1319  for (const Expr *UE : C->updates())
1320  EmitIgnoredExpr(UE);
1321  }
1322  }
1323 
1324  // On a continue in the body, jump to the end.
1325  JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue");
1326  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1327  for (const Expr *E : D.finals_conditions()) {
1328  if (!E)
1329  continue;
1330  // Check that loop counter in non-rectangular nest fits into the iteration
1331  // space.
1332  llvm::BasicBlock *NextBB = createBasicBlock("omp.body.next");
1333  EmitBranchOnBoolExpr(E, NextBB, Continue.getBlock(),
1334  getProfileCount(D.getBody()));
1335  EmitBlock(NextBB);
1336  }
1337  // Emit loop body.
1338  EmitStmt(D.getBody());
1339  // The end (updates/cleanups).
1340  EmitBlock(Continue.getBlock());
1341  BreakContinueStack.pop_back();
1342 }
1343 
1345  const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
1346  const Expr *IncExpr,
1347  const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
1348  const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
1349  auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
1350 
1351  // Start the loop with a block that tests the condition.
1352  auto CondBlock = createBasicBlock("omp.inner.for.cond");
1353  EmitBlock(CondBlock);
1354  const SourceRange R = S.getSourceRange();
1355  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1356  SourceLocToDebugLoc(R.getEnd()));
1357 
1358  // If there are any cleanups between here and the loop-exit scope,
1359  // create a block to stage a loop exit along.
1360  llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1361  if (RequiresCleanup)
1362  ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
1363 
1364  llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body");
1365 
1366  // Emit condition.
1367  EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
1368  if (ExitBlock != LoopExit.getBlock()) {
1369  EmitBlock(ExitBlock);
1370  EmitBranchThroughCleanup(LoopExit);
1371  }
1372 
1373  EmitBlock(LoopBody);
1374  incrementProfileCounter(&S);
1375 
1376  // Create a block for the increment.
1377  JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
1378  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1379 
1380  BodyGen(*this);
1381 
1382  // Emit "IV = IV + 1" and a back-edge to the condition block.
1383  EmitBlock(Continue.getBlock());
1384  EmitIgnoredExpr(IncExpr);
1385  PostIncGen(*this);
1386  BreakContinueStack.pop_back();
1387  EmitBranch(CondBlock);
1388  LoopStack.pop();
1389  // Emit the fall-through block.
1390  EmitBlock(LoopExit.getBlock());
1391 }
1392 
1394  if (!HaveInsertPoint())
1395  return false;
1396  // Emit inits for the linear variables.
1397  bool HasLinears = false;
1398  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1399  for (const Expr *Init : C->inits()) {
1400  HasLinears = true;
1401  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
1402  if (const auto *Ref =
1403  dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
1404  AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
1405  const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
1406  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1407  CapturedStmtInfo->lookup(OrigVD) != nullptr,
1408  VD->getInit()->getType(), VK_LValue,
1409  VD->getInit()->getExprLoc());
1410  EmitExprAsInit(&DRE, VD, MakeAddrLValue(Emission.getAllocatedAddress(),
1411  VD->getType()),
1412  /*capturedByInit=*/false);
1413  EmitAutoVarCleanups(Emission);
1414  } else {
1415  EmitVarDecl(*VD);
1416  }
1417  }
1418  // Emit the linear steps for the linear clauses.
1419  // If a step is not constant, it is pre-calculated before the loop.
1420  if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
1421  if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
1422  EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
1423  // Emit calculation of the linear step.
1424  EmitIgnoredExpr(CS);
1425  }
1426  }
1427  return HasLinears;
1428 }
1429 
1431  const OMPLoopDirective &D,
1432  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1433  if (!HaveInsertPoint())
1434  return;
1435  llvm::BasicBlock *DoneBB = nullptr;
1436  // Emit the final values of the linear variables.
1437  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1438  auto IC = C->varlist_begin();
1439  for (const Expr *F : C->finals()) {
1440  if (!DoneBB) {
1441  if (llvm::Value *Cond = CondGen(*this)) {
1442  // If the first post-update expression is found, emit conditional
1443  // block if it was requested.
1444  llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu");
1445  DoneBB = createBasicBlock(".omp.linear.pu.done");
1446  Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1447  EmitBlock(ThenBB);
1448  }
1449  }
1450  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
1451  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1452  CapturedStmtInfo->lookup(OrigVD) != nullptr,
1453  (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
1454  Address OrigAddr = EmitLValue(&DRE).getAddress();
1455  CodeGenFunction::OMPPrivateScope VarScope(*this);
1456  VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1457  (void)VarScope.Privatize();
1458  EmitIgnoredExpr(F);
1459  ++IC;
1460  }
1461  if (const Expr *PostUpdate = C->getPostUpdateExpr())
1462  EmitIgnoredExpr(PostUpdate);
1463  }
1464  if (DoneBB)
1465  EmitBlock(DoneBB, /*IsFinished=*/true);
1466 }
1467 
1469  const OMPExecutableDirective &D) {
1470  if (!CGF.HaveInsertPoint())
1471  return;
1472  for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
1473  unsigned ClauseAlignment = 0;
1474  if (const Expr *AlignmentExpr = Clause->getAlignment()) {
1475  auto *AlignmentCI =
1476  cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
1477  ClauseAlignment = static_cast<unsigned>(AlignmentCI->getZExtValue());
1478  }
1479  for (const Expr *E : Clause->varlists()) {
1480  unsigned Alignment = ClauseAlignment;
1481  if (Alignment == 0) {
1482  // OpenMP [2.8.1, Description]
1483  // If no optional parameter is specified, implementation-defined default
1484  // alignments for SIMD instructions on the target platforms are assumed.
1485  Alignment =
1486  CGF.getContext()
1488  E->getType()->getPointeeType()))
1489  .getQuantity();
1490  }
1491  assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) &&
1492  "alignment is not power of 2");
1493  if (Alignment != 0) {
1494  llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
1496  PtrValue, E, /*No second loc needed*/ SourceLocation(), Alignment);
1497  }
1498  }
1499  }
1500 }
1501 
1503  const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
1504  if (!HaveInsertPoint())
1505  return;
1506  auto I = S.private_counters().begin();
1507  for (const Expr *E : S.counters()) {
1508  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1509  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
1510  // Emit var without initialization.
1511  AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD);
1512  EmitAutoVarCleanups(VarEmission);
1513  LocalDeclMap.erase(PrivateVD);
1514  (void)LoopScope.addPrivate(VD, [&VarEmission]() {
1515  return VarEmission.getAllocatedAddress();
1516  });
1517  if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
1518  VD->hasGlobalStorage()) {
1519  (void)LoopScope.addPrivate(PrivateVD, [this, VD, E]() {
1520  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD),
1521  LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
1522  E->getType(), VK_LValue, E->getExprLoc());
1523  return EmitLValue(&DRE).getAddress();
1524  });
1525  } else {
1526  (void)LoopScope.addPrivate(PrivateVD, [&VarEmission]() {
1527  return VarEmission.getAllocatedAddress();
1528  });
1529  }
1530  ++I;
1531  }
1532  // Privatize extra loop counters used in loops for ordered(n) clauses.
1533  for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) {
1534  if (!C->getNumForLoops())
1535  continue;
1536  for (unsigned I = S.getCollapsedNumber(),
1537  E = C->getLoopNumIterations().size();
1538  I < E; ++I) {
1539  const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I));
1540  const auto *VD = cast<VarDecl>(DRE->getDecl());
1541  // Override only those variables that can be captured to avoid re-emission
1542  // of the variables declared within the loops.
1543  if (DRE->refersToEnclosingVariableOrCapture()) {
1544  (void)LoopScope.addPrivate(VD, [this, DRE, VD]() {
1545  return CreateMemTemp(DRE->getType(), VD->getName());
1546  });
1547  }
1548  }
1549  }
1550 }
1551 
1552 static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
1553  const Expr *Cond, llvm::BasicBlock *TrueBlock,
1554  llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
1555  if (!CGF.HaveInsertPoint())
1556  return;
1557  {
1558  CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
1559  CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
1560  (void)PreCondScope.Privatize();
1561  // Get initial values of real counters.
1562  for (const Expr *I : S.inits()) {
1563  CGF.EmitIgnoredExpr(I);
1564  }
1565  }
1566  // Create temp loop control variables with their init values to support
1567  // non-rectangular loops.
1568  CodeGenFunction::OMPMapVars PreCondVars;
1569  for (const Expr * E: S.dependent_counters()) {
1570  if (!E)
1571  continue;
1572  assert(!E->getType().getNonReferenceType()->isRecordType() &&
1573  "dependent counter must not be an iterator.");
1574  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1575  Address CounterAddr =
1576  CGF.CreateMemTemp(VD->getType().getNonReferenceType());
1577  (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr);
1578  }
1579  (void)PreCondVars.apply(CGF);
1580  for (const Expr *E : S.dependent_inits()) {
1581  if (!E)
1582  continue;
1583  CGF.EmitIgnoredExpr(E);
1584  }
1585  // Check that loop is executed at least one time.
1586  CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
1587  PreCondVars.restore(CGF);
1588 }
1589 
1591  const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
1592  if (!HaveInsertPoint())
1593  return;
1596  const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1597  for (const Expr *C : LoopDirective->counters()) {
1598  SIMDLCVs.insert(
1599  cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
1600  }
1601  }
1602  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1603  auto CurPrivate = C->privates().begin();
1604  for (const Expr *E : C->varlists()) {
1605  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1606  const auto *PrivateVD =
1607  cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
1608  if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
1609  bool IsRegistered = PrivateScope.addPrivate(VD, [this, PrivateVD]() {
1610  // Emit private VarDecl with copy init.
1611  EmitVarDecl(*PrivateVD);
1612  return GetAddrOfLocalVar(PrivateVD);
1613  });
1614  assert(IsRegistered && "linear var already registered as private");
1615  // Silence the warning about unused variable.
1616  (void)IsRegistered;
1617  } else {
1618  EmitVarDecl(*PrivateVD);
1619  }
1620  ++CurPrivate;
1621  }
1622  }
1623 }
1624 
1626  const OMPExecutableDirective &D,
1627  bool IsMonotonic) {
1628  if (!CGF.HaveInsertPoint())
1629  return;
1630  if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
1631  RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
1632  /*ignoreResult=*/true);
1633  auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1634  CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1635  // In presence of finite 'safelen', it may be unsafe to mark all
1636  // the memory instructions parallel, because loop-carried
1637  // dependences of 'safelen' iterations are possible.
1638  if (!IsMonotonic)
1640  } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
1641  RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
1642  /*ignoreResult=*/true);
1643  auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1644  CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1645  // In presence of finite 'safelen', it may be unsafe to mark all
1646  // the memory instructions parallel, because loop-carried
1647  // dependences of 'safelen' iterations are possible.
1648  CGF.LoopStack.setParallel(/*Enable=*/false);
1649  }
1650 }
1651 
1653  bool IsMonotonic) {
1654  // Walk clauses and process safelen/lastprivate.
1655  LoopStack.setParallel(!IsMonotonic);
1656  LoopStack.setVectorizeEnable();
1657  emitSimdlenSafelenClause(*this, D, IsMonotonic);
1658 }
1659 
1661  const OMPLoopDirective &D,
1662  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1663  if (!HaveInsertPoint())
1664  return;
1665  llvm::BasicBlock *DoneBB = nullptr;
1666  auto IC = D.counters().begin();
1667  auto IPC = D.private_counters().begin();
1668  for (const Expr *F : D.finals()) {
1669  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
1670  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
1671  const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
1672  if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
1673  OrigVD->hasGlobalStorage() || CED) {
1674  if (!DoneBB) {
1675  if (llvm::Value *Cond = CondGen(*this)) {
1676  // If the first post-update expression is found, emit conditional
1677  // block if it was requested.
1678  llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then");
1679  DoneBB = createBasicBlock(".omp.final.done");
1680  Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1681  EmitBlock(ThenBB);
1682  }
1683  }
1684  Address OrigAddr = Address::invalid();
1685  if (CED) {
1686  OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();
1687  } else {
1688  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(PrivateVD),
1689  /*RefersToEnclosingVariableOrCapture=*/false,
1690  (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
1691  OrigAddr = EmitLValue(&DRE).getAddress();
1692  }
1693  OMPPrivateScope VarScope(*this);
1694  VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1695  (void)VarScope.Privatize();
1696  EmitIgnoredExpr(F);
1697  }
1698  ++IC;
1699  ++IPC;
1700  }
1701  if (DoneBB)
1702  EmitBlock(DoneBB, /*IsFinished=*/true);
1703 }
1704 
1706  const OMPLoopDirective &S,
1708  CGF.EmitOMPLoopBody(S, LoopExit);
1709  CGF.EmitStopPoint(&S);
1710 }
1711 
1712 /// Emit a helper variable and return corresponding lvalue.
1714  const DeclRefExpr *Helper) {
1715  auto VDecl = cast<VarDecl>(Helper->getDecl());
1716  CGF.EmitVarDecl(*VDecl);
1717  return CGF.EmitLValue(Helper);
1718 }
1719 
1721  PrePostActionTy &Action) {
1722  Action.Enter(CGF);
1724  "Expected simd directive");
1725  OMPLoopScope PreInitScope(CGF, S);
1726  // if (PreCond) {
1727  // for (IV in 0..LastIteration) BODY;
1728  // <Final counter/linear vars updates>;
1729  // }
1730  //
1734  (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));
1735  (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));
1736  }
1737 
1738  // Emit: if (PreCond) - begin.
1739  // If the condition constant folds and can be elided, avoid emitting the
1740  // whole loop.
1741  bool CondConstant;
1742  llvm::BasicBlock *ContBlock = nullptr;
1743  if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
1744  if (!CondConstant)
1745  return;
1746  } else {
1747  llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");
1748  ContBlock = CGF.createBasicBlock("simd.if.end");
1749  emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
1750  CGF.getProfileCount(&S));
1751  CGF.EmitBlock(ThenBlock);
1752  CGF.incrementProfileCounter(&S);
1753  }
1754 
1755  // Emit the loop iteration variable.
1756  const Expr *IVExpr = S.getIterationVariable();
1757  const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
1758  CGF.EmitVarDecl(*IVDecl);
1759  CGF.EmitIgnoredExpr(S.getInit());
1760 
1761  // Emit the iterations count variable.
1762  // If it is not a variable, Sema decided to calculate iterations count on
1763  // each iteration (e.g., it is foldable into a constant).
1764  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
1765  CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
1766  // Emit calculation of the iterations count.
1768  }
1769 
1770  CGF.EmitOMPSimdInit(S);
1771 
1772  emitAlignedClause(CGF, S);
1773  (void)CGF.EmitOMPLinearClauseInit(S);
1774  {
1775  CodeGenFunction::OMPPrivateScope LoopScope(CGF);
1776  CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
1777  CGF.EmitOMPLinearClause(S, LoopScope);
1778  CGF.EmitOMPPrivateClause(S, LoopScope);
1779  CGF.EmitOMPReductionClauseInit(S, LoopScope);
1780  bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
1781  (void)LoopScope.Privatize();
1783  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
1784  CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
1785  S.getInc(),
1786  [&S](CodeGenFunction &CGF) {
1788  CGF.EmitStopPoint(&S);
1789  },
1790  [](CodeGenFunction &) {});
1791  CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; });
1792  // Emit final copy of the lastprivate variables at the end of loops.
1793  if (HasLastprivateClause)
1794  CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
1795  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
1797  [](CodeGenFunction &) { return nullptr; });
1798  }
1799  CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; });
1800  // Emit: if (PreCond) - end.
1801  if (ContBlock) {
1802  CGF.EmitBranch(ContBlock);
1803  CGF.EmitBlock(ContBlock, true);
1804  }
1805 }
1806 
1808  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1809  emitOMPSimdRegion(CGF, S, Action);
1810  };
1811  OMPLexicalScope Scope(*this, S, OMPD_unknown);
1812  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
1813 }
1814 
1815 void CodeGenFunction::EmitOMPOuterLoop(
1816  bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
1818  const CodeGenFunction::OMPLoopArguments &LoopArgs,
1819  const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
1820  const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
1821  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1822 
1823  const Expr *IVExpr = S.getIterationVariable();
1824  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1825  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1826 
1827  JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
1828 
1829  // Start the loop with a block that tests the condition.
1830  llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond");
1831  EmitBlock(CondBlock);
1832  const SourceRange R = S.getSourceRange();
1833  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1834  SourceLocToDebugLoc(R.getEnd()));
1835 
1836  llvm::Value *BoolCondVal = nullptr;
1837  if (!DynamicOrOrdered) {
1838  // UB = min(UB, GlobalUB) or
1839  // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
1840  // 'distribute parallel for')
1841  EmitIgnoredExpr(LoopArgs.EUB);
1842  // IV = LB
1843  EmitIgnoredExpr(LoopArgs.Init);
1844  // IV < UB
1845  BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
1846  } else {
1847  BoolCondVal =
1848  RT.emitForNext(*this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
1849  LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
1850  }
1851 
1852  // If there are any cleanups between here and the loop-exit scope,
1853  // create a block to stage a loop exit along.
1854  llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1855  if (LoopScope.requiresCleanups())
1856  ExitBlock = createBasicBlock("omp.dispatch.cleanup");
1857 
1858  llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body");
1859  Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
1860  if (ExitBlock != LoopExit.getBlock()) {
1861  EmitBlock(ExitBlock);
1862  EmitBranchThroughCleanup(LoopExit);
1863  }
1864  EmitBlock(LoopBody);
1865 
1866  // Emit "IV = LB" (in case of static schedule, we have already calculated new
1867  // LB for loop condition and emitted it above).
1868  if (DynamicOrOrdered)
1869  EmitIgnoredExpr(LoopArgs.Init);
1870 
1871  // Create a block for the increment.
1872  JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
1873  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1874 
1875  // Generate !llvm.loop.parallel metadata for loads and stores for loops
1876  // with dynamic/guided scheduling and without ordered clause.
1878  LoopStack.setParallel(!IsMonotonic);
1879  else
1880  EmitOMPSimdInit(S, IsMonotonic);
1881 
1882  SourceLocation Loc = S.getBeginLoc();
1883 
1884  // when 'distribute' is not combined with a 'for':
1885  // while (idx <= UB) { BODY; ++idx; }
1886  // when 'distribute' is combined with a 'for'
1887  // (e.g. 'distribute parallel for')
1888  // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
1889  EmitOMPInnerLoop(
1890  S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
1891  [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
1892  CodeGenLoop(CGF, S, LoopExit);
1893  },
1894  [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
1895  CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
1896  });
1897 
1898  EmitBlock(Continue.getBlock());
1899  BreakContinueStack.pop_back();
1900  if (!DynamicOrOrdered) {
1901  // Emit "LB = LB + Stride", "UB = UB + Stride".
1902  EmitIgnoredExpr(LoopArgs.NextLB);
1903  EmitIgnoredExpr(LoopArgs.NextUB);
1904  }
1905 
1906  EmitBranch(CondBlock);
1907  LoopStack.pop();
1908  // Emit the fall-through block.
1909  EmitBlock(LoopExit.getBlock());
1910 
1911  // Tell the runtime we are done.
1912  auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) {
1913  if (!DynamicOrOrdered)
1914  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
1915  S.getDirectiveKind());
1916  };
1917  OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
1918 }
1919 
1920 void CodeGenFunction::EmitOMPForOuterLoop(
1921  const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
1922  const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
1923  const OMPLoopArguments &LoopArgs,
1924  const CodeGenDispatchBoundsTy &CGDispatchBounds) {
1925  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1926 
1927  // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
1928  const bool DynamicOrOrdered =
1929  Ordered || RT.isDynamic(ScheduleKind.Schedule);
1930 
1931  assert((Ordered ||
1932  !RT.isStaticNonchunked(ScheduleKind.Schedule,
1933  LoopArgs.Chunk != nullptr)) &&
1934  "static non-chunked schedule does not need outer loop");
1935 
1936  // Emit outer loop.
1937  //
1938  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1939  // When schedule(dynamic,chunk_size) is specified, the iterations are
1940  // distributed to threads in the team in chunks as the threads request them.
1941  // Each thread executes a chunk of iterations, then requests another chunk,
1942  // until no chunks remain to be distributed. Each chunk contains chunk_size
1943  // iterations, except for the last chunk to be distributed, which may have
1944  // fewer iterations. When no chunk_size is specified, it defaults to 1.
1945  //
1946  // When schedule(guided,chunk_size) is specified, the iterations are assigned
1947  // to threads in the team in chunks as the executing threads request them.
1948  // Each thread executes a chunk of iterations, then requests another chunk,
1949  // until no chunks remain to be assigned. For a chunk_size of 1, the size of
1950  // each chunk is proportional to the number of unassigned iterations divided
1951  // by the number of threads in the team, decreasing to 1. For a chunk_size
1952  // with value k (greater than 1), the size of each chunk is determined in the
1953  // same way, with the restriction that the chunks do not contain fewer than k
1954  // iterations (except for the last chunk to be assigned, which may have fewer
1955  // than k iterations).
1956  //
1957  // When schedule(auto) is specified, the decision regarding scheduling is
1958  // delegated to the compiler and/or runtime system. The programmer gives the
1959  // implementation the freedom to choose any possible mapping of iterations to
1960  // threads in the team.
1961  //
1962  // When schedule(runtime) is specified, the decision regarding scheduling is
1963  // deferred until run time, and the schedule and chunk size are taken from the
1964  // run-sched-var ICV. If the ICV is set to auto, the schedule is
1965  // implementation defined
1966  //
1967  // while(__kmpc_dispatch_next(&LB, &UB)) {
1968  // idx = LB;
1969  // while (idx <= UB) { BODY; ++idx;
1970  // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
1971  // } // inner loop
1972  // }
1973  //
1974  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1975  // When schedule(static, chunk_size) is specified, iterations are divided into
1976  // chunks of size chunk_size, and the chunks are assigned to the threads in
1977  // the team in a round-robin fashion in the order of the thread number.
1978  //
1979  // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
1980  // while (idx <= UB) { BODY; ++idx; } // inner loop
1981  // LB = LB + ST;
1982  // UB = UB + ST;
1983  // }
1984  //
1985 
1986  const Expr *IVExpr = S.getIterationVariable();
1987  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1988  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1989 
1990  if (DynamicOrOrdered) {
1991  const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
1992  CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
1993  llvm::Value *LBVal = DispatchBounds.first;
1994  llvm::Value *UBVal = DispatchBounds.second;
1995  CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
1996  LoopArgs.Chunk};
1997  RT.emitForDispatchInit(*this, S.getBeginLoc(), ScheduleKind, IVSize,
1998  IVSigned, Ordered, DipatchRTInputValues);
1999  } else {
2000  CGOpenMPRuntime::StaticRTInput StaticInit(
2001  IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
2002  LoopArgs.ST, LoopArgs.Chunk);
2003  RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
2004  ScheduleKind, StaticInit);
2005  }
2006 
2007  auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
2008  const unsigned IVSize,
2009  const bool IVSigned) {
2010  if (Ordered) {
2011  CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
2012  IVSigned);
2013  }
2014  };
2015 
2016  OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
2017  LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
2018  OuterLoopArgs.IncExpr = S.getInc();
2019  OuterLoopArgs.Init = S.getInit();
2020  OuterLoopArgs.Cond = S.getCond();
2021  OuterLoopArgs.NextLB = S.getNextLowerBound();
2022  OuterLoopArgs.NextUB = S.getNextUpperBound();
2023  EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
2024  emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
2025 }
2026 
2028  const unsigned IVSize, const bool IVSigned) {}
2029 
2030 void CodeGenFunction::EmitOMPDistributeOuterLoop(
2031  OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
2032  OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
2033  const CodeGenLoopTy &CodeGenLoopContent) {
2034 
2035  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
2036 
2037  // Emit outer loop.
2038  // Same behavior as a OMPForOuterLoop, except that schedule cannot be
2039  // dynamic
2040  //
2041 
2042  const Expr *IVExpr = S.getIterationVariable();
2043  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2044  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2045 
2046  CGOpenMPRuntime::StaticRTInput StaticInit(
2047  IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
2048  LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
2049  RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, StaticInit);
2050 
2051  // for combined 'distribute' and 'for' the increment expression of distribute
2052  // is stored in DistInc. For 'distribute' alone, it is in Inc.
2053  Expr *IncExpr;
2055  IncExpr = S.getDistInc();
2056  else
2057  IncExpr = S.getInc();
2058 
2059  // this routine is shared by 'omp distribute parallel for' and
2060  // 'omp distribute': select the right EUB expression depending on the
2061  // directive
2062  OMPLoopArguments OuterLoopArgs;
2063  OuterLoopArgs.LB = LoopArgs.LB;
2064  OuterLoopArgs.UB = LoopArgs.UB;
2065  OuterLoopArgs.ST = LoopArgs.ST;
2066  OuterLoopArgs.IL = LoopArgs.IL;
2067  OuterLoopArgs.Chunk = LoopArgs.Chunk;
2068  OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2070  : S.getEnsureUpperBound();
2071  OuterLoopArgs.IncExpr = IncExpr;
2072  OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2073  ? S.getCombinedInit()
2074  : S.getInit();
2075  OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2076  ? S.getCombinedCond()
2077  : S.getCond();
2078  OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2080  : S.getNextLowerBound();
2081  OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2083  : S.getNextUpperBound();
2084 
2085  EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
2086  LoopScope, OuterLoopArgs, CodeGenLoopContent,
2088 }
2089 
2090 static std::pair<LValue, LValue>
2092  const OMPExecutableDirective &S) {
2093  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2094  LValue LB =
2095  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2096  LValue UB =
2097  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2098 
2099  // When composing 'distribute' with 'for' (e.g. as in 'distribute
2100  // parallel for') we need to use the 'distribute'
2101  // chunk lower and upper bounds rather than the whole loop iteration
2102  // space. These are parameters to the outlined function for 'parallel'
2103  // and we copy the bounds of the previous schedule into the
2104  // the current ones.
2105  LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
2106  LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
2107  llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
2108  PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());
2109  PrevLBVal = CGF.EmitScalarConversion(
2110  PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
2111  LS.getIterationVariable()->getType(),
2113  llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
2114  PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());
2115  PrevUBVal = CGF.EmitScalarConversion(
2116  PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
2117  LS.getIterationVariable()->getType(),
2119 
2120  CGF.EmitStoreOfScalar(PrevLBVal, LB);
2121  CGF.EmitStoreOfScalar(PrevUBVal, UB);
2122 
2123  return {LB, UB};
2124 }
2125 
2126 /// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
2127 /// we need to use the LB and UB expressions generated by the worksharing
2128 /// code generation support, whereas in non combined situations we would
2129 /// just emit 0 and the LastIteration expression
2130 /// This function is necessary due to the difference of the LB and UB
2131 /// types for the RT emission routines for 'for_static_init' and
2132 /// 'for_dispatch_init'
2133 static std::pair<llvm::Value *, llvm::Value *>
2135  const OMPExecutableDirective &S,
2136  Address LB, Address UB) {
2137  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2138  const Expr *IVExpr = LS.getIterationVariable();
2139  // when implementing a dynamic schedule for a 'for' combined with a
2140  // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
2141  // is not normalized as each team only executes its own assigned
2142  // distribute chunk
2143  QualType IteratorTy = IVExpr->getType();
2144  llvm::Value *LBVal =
2145  CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
2146  llvm::Value *UBVal =
2147  CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
2148  return {LBVal, UBVal};
2149 }
2150 
2152  CodeGenFunction &CGF, const OMPExecutableDirective &S,
2153  llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) {
2154  const auto &Dir = cast<OMPLoopDirective>(S);
2155  LValue LB =
2156  CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
2157  llvm::Value *LBCast = CGF.Builder.CreateIntCast(
2158  CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2159  CapturedVars.push_back(LBCast);
2160  LValue UB =
2161  CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
2162 
2163  llvm::Value *UBCast = CGF.Builder.CreateIntCast(
2164  CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2165  CapturedVars.push_back(UBCast);
2166 }
2167 
2168 static void
2170  const OMPLoopDirective &S,
2172  auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
2173  PrePostActionTy &Action) {
2174  Action.Enter(CGF);
2175  bool HasCancel = false;
2177  if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
2178  HasCancel = D->hasCancel();
2179  else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
2180  HasCancel = D->hasCancel();
2181  else if (const auto *D =
2182  dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
2183  HasCancel = D->hasCancel();
2184  }
2186  HasCancel);
2190  };
2191 
2193  CGF, S,
2194  isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for,
2195  CGInlinedWorksharingLoop,
2197 }
2198 
2201  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2203  S.getDistInc());
2204  };
2205  OMPLexicalScope Scope(*this, S, OMPD_parallel);
2206  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2207 }
2208 
2211  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2213  S.getDistInc());
2214  };
2215  OMPLexicalScope Scope(*this, S, OMPD_parallel);
2216  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2217 }
2218 
2220  const OMPDistributeSimdDirective &S) {
2221  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2223  };
2224  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2225  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2226 }
2227 
2229  CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
2230  // Emit SPMD target parallel for region as a standalone region.
2231  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2232  emitOMPSimdRegion(CGF, S, Action);
2233  };
2234  llvm::Function *Fn;
2235  llvm::Constant *Addr;
2236  // Emit target region as a standalone region.
2237  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
2238  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
2239  assert(Fn && Addr && "Target device function emission failed.");
2240 }
2241 
2243  const OMPTargetSimdDirective &S) {
2244  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2245  emitOMPSimdRegion(CGF, S, Action);
2246  };
2247  emitCommonOMPTargetDirective(*this, S, CodeGen);
2248 }
2249 
2250 namespace {
2251  struct ScheduleKindModifiersTy {
2255  ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
2258  : Kind(Kind), M1(M1), M2(M2) {}
2259  };
2260 } // namespace
2261 
2263  const OMPLoopDirective &S, Expr *EUB,
2264  const CodeGenLoopBoundsTy &CodeGenLoopBounds,
2265  const CodeGenDispatchBoundsTy &CGDispatchBounds) {
2266  // Emit the loop iteration variable.
2267  const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
2268  const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
2269  EmitVarDecl(*IVDecl);
2270 
2271  // Emit the iterations count variable.
2272  // If it is not a variable, Sema decided to calculate iterations count on each
2273  // iteration (e.g., it is foldable into a constant).
2274  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2275  EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2276  // Emit calculation of the iterations count.
2277  EmitIgnoredExpr(S.getCalcLastIteration());
2278  }
2279 
2280  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
2281 
2282  bool HasLastprivateClause;
2283  // Check pre-condition.
2284  {
2285  OMPLoopScope PreInitScope(*this, S);
2286  // Skip the entire loop if we don't meet the precondition.
2287  // If the condition constant folds and can be elided, avoid emitting the
2288  // whole loop.
2289  bool CondConstant;
2290  llvm::BasicBlock *ContBlock = nullptr;
2291  if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2292  if (!CondConstant)
2293  return false;
2294  } else {
2295  llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
2296  ContBlock = createBasicBlock("omp.precond.end");
2297  emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
2298  getProfileCount(&S));
2299  EmitBlock(ThenBlock);
2300  incrementProfileCounter(&S);
2301  }
2302 
2303  RunCleanupsScope DoacrossCleanupScope(*this);
2304  bool Ordered = false;
2305  if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
2306  if (OrderedClause->getNumForLoops())
2307  RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations());
2308  else
2309  Ordered = true;
2310  }
2311 
2312  llvm::DenseSet<const Expr *> EmittedFinals;
2313  emitAlignedClause(*this, S);
2314  bool HasLinears = EmitOMPLinearClauseInit(S);
2315  // Emit helper vars inits.
2316 
2317  std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
2318  LValue LB = Bounds.first;
2319  LValue UB = Bounds.second;
2320  LValue ST =
2321  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
2322  LValue IL =
2323  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
2324 
2325  // Emit 'then' code.
2326  {
2327  OMPPrivateScope LoopScope(*this);
2328  if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
2329  // Emit implicit barrier to synchronize threads and avoid data races on
2330  // initialization of firstprivate variables and post-update of
2331  // lastprivate variables.
2332  CGM.getOpenMPRuntime().emitBarrierCall(
2333  *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
2334  /*ForceSimpleCall=*/true);
2335  }
2336  EmitOMPPrivateClause(S, LoopScope);
2337  HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
2338  EmitOMPReductionClauseInit(S, LoopScope);
2339  EmitOMPPrivateLoopCounters(S, LoopScope);
2340  EmitOMPLinearClause(S, LoopScope);
2341  (void)LoopScope.Privatize();
2343  CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
2344 
2345  // Detect the loop schedule kind and chunk.
2346  const Expr *ChunkExpr = nullptr;
2347  OpenMPScheduleTy ScheduleKind;
2348  if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
2349  ScheduleKind.Schedule = C->getScheduleKind();
2350  ScheduleKind.M1 = C->getFirstScheduleModifier();
2351  ScheduleKind.M2 = C->getSecondScheduleModifier();
2352  ChunkExpr = C->getChunkSize();
2353  } else {
2354  // Default behaviour for schedule clause.
2355  CGM.getOpenMPRuntime().getDefaultScheduleAndChunk(
2356  *this, S, ScheduleKind.Schedule, ChunkExpr);
2357  }
2358  bool HasChunkSizeOne = false;
2359  llvm::Value *Chunk = nullptr;
2360  if (ChunkExpr) {
2361  Chunk = EmitScalarExpr(ChunkExpr);
2362  Chunk = EmitScalarConversion(Chunk, ChunkExpr->getType(),
2364  S.getBeginLoc());
2365  Expr::EvalResult Result;
2366  if (ChunkExpr->EvaluateAsInt(Result, getContext())) {
2367  llvm::APSInt EvaluatedChunk = Result.Val.getInt();
2368  HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
2369  }
2370  }
2371  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2372  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2373  // OpenMP 4.5, 2.7.1 Loop Construct, Description.
2374  // If the static schedule kind is specified or if the ordered clause is
2375  // specified, and if no monotonic modifier is specified, the effect will
2376  // be as if the monotonic modifier was specified.
2377  bool StaticChunkedOne = RT.isStaticChunked(ScheduleKind.Schedule,
2378  /* Chunked */ Chunk != nullptr) && HasChunkSizeOne &&
2380  if ((RT.isStaticNonchunked(ScheduleKind.Schedule,
2381  /* Chunked */ Chunk != nullptr) ||
2382  StaticChunkedOne) &&
2383  !Ordered) {
2385  EmitOMPSimdInit(S, /*IsMonotonic=*/true);
2386  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2387  // When no chunk_size is specified, the iteration space is divided into
2388  // chunks that are approximately equal in size, and at most one chunk is
2389  // distributed to each thread. Note that the size of the chunks is
2390  // unspecified in this case.
2391  CGOpenMPRuntime::StaticRTInput StaticInit(
2392  IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
2393  UB.getAddress(), ST.getAddress(),
2394  StaticChunkedOne ? Chunk : nullptr);
2395  RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
2396  ScheduleKind, StaticInit);
2397  JumpDest LoopExit =
2398  getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
2399  // UB = min(UB, GlobalUB);
2400  if (!StaticChunkedOne)
2401  EmitIgnoredExpr(S.getEnsureUpperBound());
2402  // IV = LB;
2403  EmitIgnoredExpr(S.getInit());
2404  // For unchunked static schedule generate:
2405  //
2406  // while (idx <= UB) {
2407  // BODY;
2408  // ++idx;
2409  // }
2410  //
2411  // For static schedule with chunk one:
2412  //
2413  // while (IV <= PrevUB) {
2414  // BODY;
2415  // IV += ST;
2416  // }
2417  EmitOMPInnerLoop(S, LoopScope.requiresCleanups(),
2418  StaticChunkedOne ? S.getCombinedParForInDistCond() : S.getCond(),
2419  StaticChunkedOne ? S.getDistInc() : S.getInc(),
2420  [&S, LoopExit](CodeGenFunction &CGF) {
2421  CGF.EmitOMPLoopBody(S, LoopExit);
2422  CGF.EmitStopPoint(&S);
2423  },
2424  [](CodeGenFunction &) {});
2425  EmitBlock(LoopExit.getBlock());
2426  // Tell the runtime we are done.
2427  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2428  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2429  S.getDirectiveKind());
2430  };
2431  OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
2432  } else {
2433  const bool IsMonotonic =
2434  Ordered || ScheduleKind.Schedule == OMPC_SCHEDULE_static ||
2435  ScheduleKind.Schedule == OMPC_SCHEDULE_unknown ||
2436  ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
2437  ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
2438  // Emit the outer loop, which requests its work chunk [LB..UB] from
2439  // runtime and runs the inner loop to process it.
2440  const OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),
2441  ST.getAddress(), IL.getAddress(),
2442  Chunk, EUB);
2443  EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
2444  LoopArguments, CGDispatchBounds);
2445  }
2447  EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
2448  return CGF.Builder.CreateIsNotNull(
2449  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2450  });
2451  }
2452  EmitOMPReductionClauseFinal(
2453  S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind())
2454  ? /*Parallel and Simd*/ OMPD_parallel_for_simd
2455  : /*Parallel only*/ OMPD_parallel);
2456  // Emit post-update of the reduction variables if IsLastIter != 0.
2458  *this, S, [IL, &S](CodeGenFunction &CGF) {
2459  return CGF.Builder.CreateIsNotNull(
2460  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2461  });
2462  // Emit final copy of the lastprivate variables if IsLastIter != 0.
2463  if (HasLastprivateClause)
2464  EmitOMPLastprivateClauseFinal(
2466  Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
2467  }
2468  EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) {
2469  return CGF.Builder.CreateIsNotNull(
2470  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2471  });
2472  DoacrossCleanupScope.ForceCleanup();
2473  // We're now done with the loop, so jump to the continuation block.
2474  if (ContBlock) {
2475  EmitBranch(ContBlock);
2476  EmitBlock(ContBlock, /*IsFinished=*/true);
2477  }
2478  }
2479  return HasLastprivateClause;
2480 }
2481 
2482 /// The following two functions generate expressions for the loop lower
2483 /// and upper bounds in case of static and dynamic (dispatch) schedule
2484 /// of the associated 'for' or 'distribute' loop.
2485 static std::pair<LValue, LValue>
2487  const auto &LS = cast<OMPLoopDirective>(S);
2488  LValue LB =
2489  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2490  LValue UB =
2491  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2492  return {LB, UB};
2493 }
2494 
2495 /// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
2496 /// consider the lower and upper bound expressions generated by the
2497 /// worksharing loop support, but we use 0 and the iteration space size as
2498 /// constants
2499 static std::pair<llvm::Value *, llvm::Value *>
2501  Address LB, Address UB) {
2502  const auto &LS = cast<OMPLoopDirective>(S);
2503  const Expr *IVExpr = LS.getIterationVariable();
2504  const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
2505  llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
2506  llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
2507  return {LBVal, UBVal};
2508 }
2509 
2511  bool HasLastprivates = false;
2512  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2513  PrePostActionTy &) {
2514  OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel());
2515  HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2518  };
2519  {
2520  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2521  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
2522  S.hasCancel());
2523  }
2524 
2525  // Emit an implicit barrier at the end.
2526  if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
2527  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
2528 }
2529 
2531  bool HasLastprivates = false;
2532  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2533  PrePostActionTy &) {
2534  HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2537  };
2538  {
2539  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2540  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2541  }
2542 
2543  // Emit an implicit barrier at the end.
2544  if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
2545  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
2546 }
2547 
2549  const Twine &Name,
2550  llvm::Value *Init = nullptr) {
2551  LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
2552  if (Init)
2553  CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
2554  return LVal;
2555 }
2556 
2557 void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
2559  const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
2560  bool HasLastprivates = false;
2561  auto &&CodeGen = [&S, CapturedStmt, CS,
2562  &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
2563  ASTContext &C = CGF.getContext();
2564  QualType KmpInt32Ty =
2565  C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2566  // Emit helper vars inits.
2567  LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
2568  CGF.Builder.getInt32(0));
2569  llvm::ConstantInt *GlobalUBVal = CS != nullptr
2570  ? CGF.Builder.getInt32(CS->size() - 1)
2571  : CGF.Builder.getInt32(0);
2572  LValue UB =
2573  createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
2574  LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
2575  CGF.Builder.getInt32(1));
2576  LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
2577  CGF.Builder.getInt32(0));
2578  // Loop counter.
2579  LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
2580  OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
2581  CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
2582  OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
2583  CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
2584  // Generate condition for loop.
2585  BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue,
2587  // Increment for loop counter.
2588  UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary,
2589  S.getBeginLoc(), true);
2590  auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
2591  // Iterate through all sections and emit a switch construct:
2592  // switch (IV) {
2593  // case 0:
2594  // <SectionStmt[0]>;
2595  // break;
2596  // ...
2597  // case <NumSection> - 1:
2598  // <SectionStmt[<NumSection> - 1]>;
2599  // break;
2600  // }
2601  // .omp.sections.exit:
2602  llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
2603  llvm::SwitchInst *SwitchStmt =
2604  CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
2605  ExitBB, CS == nullptr ? 1 : CS->size());
2606  if (CS) {
2607  unsigned CaseNumber = 0;
2608  for (const Stmt *SubStmt : CS->children()) {
2609  auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2610  CGF.EmitBlock(CaseBB);
2611  SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
2612  CGF.EmitStmt(SubStmt);
2613  CGF.EmitBranch(ExitBB);
2614  ++CaseNumber;
2615  }
2616  } else {
2617  llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");
2618  CGF.EmitBlock(CaseBB);
2619  SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
2620  CGF.EmitStmt(CapturedStmt);
2621  CGF.EmitBranch(ExitBB);
2622  }
2623  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2624  };
2625 
2626  CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2627  if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
2628  // Emit implicit barrier to synchronize threads and avoid data races on
2629  // initialization of firstprivate variables and post-update of lastprivate
2630  // variables.
2631  CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2632  CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
2633  /*ForceSimpleCall=*/true);
2634  }
2635  CGF.EmitOMPPrivateClause(S, LoopScope);
2636  HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2637  CGF.EmitOMPReductionClauseInit(S, LoopScope);
2638  (void)LoopScope.Privatize();
2639  if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
2640  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
2641 
2642  // Emit static non-chunked loop.
2643  OpenMPScheduleTy ScheduleKind;
2644  ScheduleKind.Schedule = OMPC_SCHEDULE_static;
2645  CGOpenMPRuntime::StaticRTInput StaticInit(
2646  /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(),
2647  LB.getAddress(), UB.getAddress(), ST.getAddress());
2648  CGF.CGM.getOpenMPRuntime().emitForStaticInit(
2649  CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit);
2650  // UB = min(UB, GlobalUB);
2651  llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
2652  llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
2653  CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
2654  CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
2655  // IV = LB;
2656  CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
2657  // while (idx <= UB) { BODY; ++idx; }
2658  CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen,
2659  [](CodeGenFunction &) {});
2660  // Tell the runtime we are done.
2661  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2662  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2663  S.getDirectiveKind());
2664  };
2665  CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
2666  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
2667  // Emit post-update of the reduction variables if IsLastIter != 0.
2668  emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) {
2669  return CGF.Builder.CreateIsNotNull(
2670  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2671  });
2672 
2673  // Emit final copy of the lastprivate variables if IsLastIter != 0.
2674  if (HasLastprivates)
2676  S, /*NoFinals=*/false,
2677  CGF.Builder.CreateIsNotNull(
2678  CGF.EmitLoadOfScalar(IL, S.getBeginLoc())));
2679  };
2680 
2681  bool HasCancel = false;
2682  if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
2683  HasCancel = OSD->hasCancel();
2684  else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
2685  HasCancel = OPSD->hasCancel();
2686  OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
2687  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
2688  HasCancel);
2689  // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
2690  // clause. Otherwise the barrier will be generated by the codegen for the
2691  // directive.
2692  if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
2693  // Emit implicit barrier to synchronize threads and avoid data races on
2694  // initialization of firstprivate variables.
2695  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
2696  OMPD_unknown);
2697  }
2698 }
2699 
2701  {
2702  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2703  EmitSections(S);
2704  }
2705  // Emit an implicit barrier at the end.
2706  if (!S.getSingleClause<OMPNowaitClause>()) {
2707  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
2708  OMPD_sections);
2709  }
2710 }
2711 
2713  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2715  };
2716  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2717  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen,
2718  S.hasCancel());
2719 }
2720 
2722  llvm::SmallVector<const Expr *, 8> CopyprivateVars;
2725  llvm::SmallVector<const Expr *, 8> AssignmentOps;
2726  // Check if there are any 'copyprivate' clauses associated with this
2727  // 'single' construct.
2728  // Build a list of copyprivate variables along with helper expressions
2729  // (<source>, <destination>, <destination>=<source> expressions)
2730  for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
2731  CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
2732  DestExprs.append(C->destination_exprs().begin(),
2733  C->destination_exprs().end());
2734  SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
2735  AssignmentOps.append(C->assignment_ops().begin(),
2736  C->assignment_ops().end());
2737  }
2738  // Emit code for 'single' region along with 'copyprivate' clauses
2739  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2740  Action.Enter(CGF);
2741  OMPPrivateScope SingleScope(CGF);
2742  (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
2743  CGF.EmitOMPPrivateClause(S, SingleScope);
2744  (void)SingleScope.Privatize();
2746  };
2747  {
2748  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2749  CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getBeginLoc(),
2750  CopyprivateVars, DestExprs,
2751  SrcExprs, AssignmentOps);
2752  }
2753  // Emit an implicit barrier at the end (to avoid data race on firstprivate
2754  // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
2755  if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
2756  CGM.getOpenMPRuntime().emitBarrierCall(
2757  *this, S.getBeginLoc(),
2758  S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
2759  }
2760 }
2761 
2763  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2764  Action.Enter(CGF);
2766  };
2767  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2768  CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
2769 }
2770 
2772  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2773  Action.Enter(CGF);
2775  };
2776  const Expr *Hint = nullptr;
2777  if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
2778  Hint = HintClause->getHint();
2779  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2780  CGM.getOpenMPRuntime().emitCriticalRegion(*this,
2782  CodeGen, S.getBeginLoc(), Hint);
2783 }
2784 
2786  const OMPParallelForDirective &S) {
2787  // Emit directive as a combined directive that consists of two implicit
2788  // directives: 'parallel' with 'for' directive.
2789  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2790  Action.Enter(CGF);
2791  OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel());
2794  };
2795  emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
2797 }
2798 
2800  const OMPParallelForSimdDirective &S) {
2801  // Emit directive as a combined directive that consists of two implicit
2802  // directives: 'parallel' with 'for' directive.
2803  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2804  Action.Enter(CGF);
2807  };
2808  emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen,
2810 }
2811 
2813  const OMPParallelSectionsDirective &S) {
2814  // Emit directive as a combined directive that consists of two implicit
2815  // directives: 'parallel' with 'sections' directive.
2816  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2817  Action.Enter(CGF);
2818  CGF.EmitSections(S);
2819  };
2820  emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
2822 }
2823 
2825  const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
2826  const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
2827  OMPTaskDataTy &Data) {
2828  // Emit outlined function for task construct.
2829  const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
2830  auto I = CS->getCapturedDecl()->param_begin();
2831  auto PartId = std::next(I);
2832  auto TaskT = std::next(I, 4);
2833  // Check if the task is final
2834  if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
2835  // If the condition constant folds and can be elided, try to avoid emitting
2836  // the condition and the dead arm of the if/else.
2837  const Expr *Cond = Clause->getCondition();
2838  bool CondConstant;
2839  if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
2840  Data.Final.setInt(CondConstant);
2841  else
2842  Data.Final.setPointer(EvaluateExprAsBool(Cond));
2843  } else {
2844  // By default the task is not final.
2845  Data.Final.setInt(/*IntVal=*/false);
2846  }
2847  // Check if the task has 'priority' clause.
2848  if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
2849  const Expr *Prio = Clause->getPriority();
2850  Data.Priority.setInt(/*IntVal=*/true);
2851  Data.Priority.setPointer(EmitScalarConversion(
2852  EmitScalarExpr(Prio), Prio->getType(),
2853  getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
2854  Prio->getExprLoc()));
2855  }
2856  // The first function argument for tasks is a thread id, the second one is a
2857  // part id (0 for tied tasks, >=0 for untied task).
2858  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
2859  // Get list of private variables.
2860  for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
2861  auto IRef = C->varlist_begin();
2862  for (const Expr *IInit : C->private_copies()) {
2863  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2864  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2865  Data.PrivateVars.push_back(*IRef);
2866  Data.PrivateCopies.push_back(IInit);
2867  }
2868  ++IRef;
2869  }
2870  }
2871  EmittedAsPrivate.clear();
2872  // Get list of firstprivate variables.
2873  for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
2874  auto IRef = C->varlist_begin();
2875  auto IElemInitRef = C->inits().begin();
2876  for (const Expr *IInit : C->private_copies()) {
2877  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2878  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2879  Data.FirstprivateVars.push_back(*IRef);
2880  Data.FirstprivateCopies.push_back(IInit);
2881  Data.FirstprivateInits.push_back(*IElemInitRef);
2882  }
2883  ++IRef;
2884  ++IElemInitRef;
2885  }
2886  }
2887  // Get list of lastprivate variables (for taskloops).
2888  llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
2889  for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
2890  auto IRef = C->varlist_begin();
2891  auto ID = C->destination_exprs().begin();
2892  for (const Expr *IInit : C->private_copies()) {
2893  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2894  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2895  Data.LastprivateVars.push_back(*IRef);
2896  Data.LastprivateCopies.push_back(IInit);
2897  }
2898  LastprivateDstsOrigs.insert(
2899  {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
2900  cast<DeclRefExpr>(*IRef)});
2901  ++IRef;
2902  ++ID;
2903  }
2904  }
2907  for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
2908  auto IPriv = C->privates().begin();
2909  auto IRed = C->reduction_ops().begin();
2910  auto ILHS = C->lhs_exprs().begin();
2911  auto IRHS = C->rhs_exprs().begin();
2912  for (const Expr *Ref : C->varlists()) {
2913  Data.ReductionVars.emplace_back(Ref);
2914  Data.ReductionCopies.emplace_back(*IPriv);
2915  Data.ReductionOps.emplace_back(*IRed);
2916  LHSs.emplace_back(*ILHS);
2917  RHSs.emplace_back(*IRHS);
2918  std::advance(IPriv, 1);
2919  std::advance(IRed, 1);
2920  std::advance(ILHS, 1);
2921  std::advance(IRHS, 1);
2922  }
2923  }
2924  Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
2925  *this, S.getBeginLoc(), LHSs, RHSs, Data);
2926  // Build list of dependences.
2927  for (const auto *C : S.getClausesOfKind<OMPDependClause>())
2928  for (const Expr *IRef : C->varlists())
2929  Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
2930  auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
2931  CapturedRegion](CodeGenFunction &CGF,
2932  PrePostActionTy &Action) {
2933  // Set proper addresses for generated private copies.
2934  OMPPrivateScope Scope(CGF);
2935  if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
2936  !Data.LastprivateVars.empty()) {
2937  llvm::FunctionType *CopyFnTy = llvm::FunctionType::get(
2938  CGF.Builder.getVoidTy(), {CGF.Builder.getInt8PtrTy()}, true);
2939  enum { PrivatesParam = 2, CopyFnParam = 3 };
2940  llvm::Value *CopyFn = CGF.Builder.CreateLoad(
2941  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
2942  llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
2943  CS->getCapturedDecl()->getParam(PrivatesParam)));
2944  // Map privates.
2947  CallArgs.push_back(PrivatesPtr);
2948  for (const Expr *E : Data.PrivateVars) {
2949  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2950  Address PrivatePtr = CGF.CreateMemTemp(
2951  CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
2952  PrivatePtrs.emplace_back(VD, PrivatePtr);
2953  CallArgs.push_back(PrivatePtr.getPointer());
2954  }
2955  for (const Expr *E : Data.FirstprivateVars) {
2956  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2957  Address PrivatePtr =
2958  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2959  ".firstpriv.ptr.addr");
2960  PrivatePtrs.emplace_back(VD, PrivatePtr);
2961  CallArgs.push_back(PrivatePtr.getPointer());
2962  }
2963  for (const Expr *E : Data.LastprivateVars) {
2964  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2965  Address PrivatePtr =
2966  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2967  ".lastpriv.ptr.addr");
2968  PrivatePtrs.emplace_back(VD, PrivatePtr);
2969  CallArgs.push_back(PrivatePtr.getPointer());
2970  }
2971  CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
2972  CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
2973  for (const auto &Pair : LastprivateDstsOrigs) {
2974  const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
2975  DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD),
2976  /*RefersToEnclosingVariableOrCapture=*/
2977  CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr,
2978  Pair.second->getType(), VK_LValue,
2979  Pair.second->getExprLoc());
2980  Scope.addPrivate(Pair.first, [&CGF, &DRE]() {
2981  return CGF.EmitLValue(&DRE).getAddress();
2982  });
2983  }
2984  for (const auto &Pair : PrivatePtrs) {
2985  Address Replacement(CGF.Builder.CreateLoad(Pair.second),
2986  CGF.getContext().getDeclAlign(Pair.first));
2987  Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
2988  }
2989  }
2990  if (Data.Reductions) {
2991  OMPLexicalScope LexScope(CGF, S, CapturedRegion);
2993  Data.ReductionOps);
2994  llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
2995  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
2996  for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
2997  RedCG.emitSharedLValue(CGF, Cnt);
2998  RedCG.emitAggregateType(CGF, Cnt);
2999  // FIXME: This must removed once the runtime library is fixed.
3000  // Emit required threadprivate variables for
3001  // initializer/combiner/finalizer.
3002  CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
3003  RedCG, Cnt);
3004  Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
3005  CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
3006  Replacement =
3007  Address(CGF.EmitScalarConversion(
3008  Replacement.getPointer(), CGF.getContext().VoidPtrTy,
3009  CGF.getContext().getPointerType(
3010  Data.ReductionCopies[Cnt]->getType()),
3011  Data.ReductionCopies[Cnt]->getExprLoc()),
3012  Replacement.getAlignment());
3013  Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
3014  Scope.addPrivate(RedCG.getBaseDecl(Cnt),
3015  [Replacement]() { return Replacement; });
3016  }
3017  }
3018  // Privatize all private variables except for in_reduction items.
3019  (void)Scope.Privatize();
3020  SmallVector<const Expr *, 4> InRedVars;
3021  SmallVector<const Expr *, 4> InRedPrivs;
3023  SmallVector<const Expr *, 4> TaskgroupDescriptors;
3024  for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
3025  auto IPriv = C->privates().begin();
3026  auto IRed = C->reduction_ops().begin();
3027  auto ITD = C->taskgroup_descriptors().begin();
3028  for (const Expr *Ref : C->varlists()) {
3029  InRedVars.emplace_back(Ref);
3030  InRedPrivs.emplace_back(*IPriv);
3031  InRedOps.emplace_back(*IRed);
3032  TaskgroupDescriptors.emplace_back(*ITD);
3033  std::advance(IPriv, 1);
3034  std::advance(IRed, 1);
3035  std::advance(ITD, 1);
3036  }
3037  }
3038  // Privatize in_reduction items here, because taskgroup descriptors must be
3039  // privatized earlier.
3040  OMPPrivateScope InRedScope(CGF);
3041  if (!InRedVars.empty()) {
3042  ReductionCodeGen RedCG(InRedVars, InRedPrivs, InRedOps);
3043  for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
3044  RedCG.emitSharedLValue(CGF, Cnt);
3045  RedCG.emitAggregateType(CGF, Cnt);
3046  // The taskgroup descriptor variable is always implicit firstprivate and
3047  // privatized already during processing of the firstprivates.
3048  // FIXME: This must removed once the runtime library is fixed.
3049  // Emit required threadprivate variables for
3050  // initializer/combiner/finalizer.
3051  CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
3052  RedCG, Cnt);
3053  llvm::Value *ReductionsPtr =
3054  CGF.EmitLoadOfScalar(CGF.EmitLValue(TaskgroupDescriptors[Cnt]),
3055  TaskgroupDescriptors[Cnt]->getExprLoc());
3056  Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
3057  CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
3058  Replacement = Address(
3059  CGF.EmitScalarConversion(
3060  Replacement.getPointer(), CGF.getContext().VoidPtrTy,
3061  CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
3062  InRedPrivs[Cnt]->getExprLoc()),
3063  Replacement.getAlignment());
3064  Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
3065  InRedScope.addPrivate(RedCG.getBaseDecl(Cnt),
3066  [Replacement]() { return Replacement; });
3067  }
3068  }
3069  (void)InRedScope.Privatize();
3070 
3071  Action.Enter(CGF);
3072  BodyGen(CGF);
3073  };
3074  llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3075  S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied,
3076  Data.NumberOfParts);
3077  OMPLexicalScope Scope(*this, S);
3078  TaskGen(*this, OutlinedFn, Data);
3079 }
3080 
3081 static ImplicitParamDecl *
3083  QualType Ty, CapturedDecl *CD,
3084  SourceLocation Loc) {
3085  auto *OrigVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3087  auto *OrigRef = DeclRefExpr::Create(
3088  C, NestedNameSpecifierLoc(), SourceLocation(), OrigVD,
3089  /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
3090  auto *PrivateVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3092  auto *PrivateRef = DeclRefExpr::Create(
3093  C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD,
3094  /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
3095  QualType ElemType = C.getBaseElementType(Ty);
3096  auto *InitVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, ElemType,
3098  auto *InitRef = DeclRefExpr::Create(
3099  C, NestedNameSpecifierLoc(), SourceLocation(), InitVD,
3100  /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue);
3101  PrivateVD->setInitStyle(VarDecl::CInit);
3102  PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue,
3103  InitRef, /*BasePath=*/nullptr,
3104  VK_RValue));
3105  Data.FirstprivateVars.emplace_back(OrigRef);
3106  Data.FirstprivateCopies.emplace_back(PrivateRef);
3107  Data.FirstprivateInits.emplace_back(InitRef);
3108  return OrigVD;
3109 }
3110 
3112  const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,
3113  OMPTargetDataInfo &InputInfo) {
3114  // Emit outlined function for task construct.
3115  const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3116  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3117  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3118  auto I = CS->getCapturedDecl()->param_begin();
3119  auto PartId = std::next(I);
3120  auto TaskT = std::next(I, 4);
3121  OMPTaskDataTy Data;
3122  // The task is not final.
3123  Data.Final.setInt(/*IntVal=*/false);
3124  // Get list of firstprivate variables.
3125  for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
3126  auto IRef = C->varlist_begin();
3127  auto IElemInitRef = C->inits().begin();
3128  for (auto *IInit : C->private_copies()) {
3129  Data.FirstprivateVars.push_back(*IRef);
3130  Data.FirstprivateCopies.push_back(IInit);
3131  Data.FirstprivateInits.push_back(*IElemInitRef);
3132  ++IRef;
3133  ++IElemInitRef;
3134  }
3135  }
3136  OMPPrivateScope TargetScope(*this);
3137  VarDecl *BPVD = nullptr;
3138  VarDecl *PVD = nullptr;
3139  VarDecl *SVD = nullptr;
3140  if (InputInfo.NumberOfTargetItems > 0) {
3141  auto *CD = CapturedDecl::Create(
3142  getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0);
3143  llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);
3144  QualType BaseAndPointersType = getContext().getConstantArrayType(
3145  getContext().VoidPtrTy, ArrSize, ArrayType::Normal,
3146  /*IndexTypeQuals=*/0);
3148  getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
3150  getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
3151  QualType SizesType = getContext().getConstantArrayType(
3152  getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1),
3153  ArrSize, ArrayType::Normal,
3154  /*IndexTypeQuals=*/0);
3155  SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD,
3156  S.getBeginLoc());
3157  TargetScope.addPrivate(
3158  BPVD, [&InputInfo]() { return InputInfo.BasePointersArray; });
3159  TargetScope.addPrivate(PVD,
3160  [&InputInfo]() { return InputInfo.PointersArray; });
3161  TargetScope.addPrivate(SVD,
3162  [&InputInfo]() { return InputInfo.SizesArray; });
3163  }
3164  (void)TargetScope.Privatize();
3165  // Build list of dependences.
3166  for (const auto *C : S.getClausesOfKind<OMPDependClause>())
3167  for (const Expr *IRef : C->varlists())
3168  Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
3169  auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD,
3170  &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
3171  // Set proper addresses for generated private copies.
3172  OMPPrivateScope Scope(CGF);
3173  if (!Data.FirstprivateVars.empty()) {
3174  llvm::FunctionType *CopyFnTy = llvm::FunctionType::get(
3175  CGF.Builder.getVoidTy(), {CGF.Builder.getInt8PtrTy()}, true);
3176  enum { PrivatesParam = 2, CopyFnParam = 3 };
3177  llvm::Value *CopyFn = CGF.Builder.CreateLoad(
3178  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
3179  llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
3180  CS->getCapturedDecl()->getParam(PrivatesParam)));
3181  // Map privates.
3184  CallArgs.push_back(PrivatesPtr);
3185  for (const Expr *E : Data.FirstprivateVars) {
3186  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3187  Address PrivatePtr =
3188  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
3189  ".firstpriv.ptr.addr");
3190  PrivatePtrs.emplace_back(VD, PrivatePtr);
3191  CallArgs.push_back(PrivatePtr.getPointer());
3192  }
3193  CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
3194  CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
3195  for (const auto &Pair : PrivatePtrs) {
3196  Address Replacement(CGF.Builder.CreateLoad(Pair.second),
3197  CGF.getContext().getDeclAlign(Pair.first));
3198  Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
3199  }
3200  }
3201  // Privatize all private variables except for in_reduction items.
3202  (void)Scope.Privatize();
3203  if (InputInfo.NumberOfTargetItems > 0) {
3204  InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
3205  CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0);
3206  InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
3207  CGF.GetAddrOfLocalVar(PVD), /*Index=*/0);
3208  InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
3209  CGF.GetAddrOfLocalVar(SVD), /*Index=*/0);
3210  }
3211 
3212  Action.Enter(CGF);
3213  OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);
3214  BodyGen(CGF);
3215  };
3216  llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3217  S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true,
3218  Data.NumberOfParts);
3219  llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
3220  IntegerLiteral IfCond(getContext(), TrueOrFalse,
3221  getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3222  SourceLocation());
3223 
3224  CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn,
3225  SharedsTy, CapturedStruct, &IfCond, Data);
3226 }
3227 
3229  // Emit outlined function for task construct.
3230  const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3231  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3232  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3233  const Expr *IfCond = nullptr;
3234  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3235  if (C->getNameModifier() == OMPD_unknown ||
3236  C->getNameModifier() == OMPD_task) {
3237  IfCond = C->getCondition();
3238  break;
3239  }
3240  }
3241 
3242  OMPTaskDataTy Data;
3243  // Check if we should emit tied or untied task.
3244  Data.Tied = !S.getSingleClause<OMPUntiedClause>();
3245  auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
3246  CGF.EmitStmt(CS->getCapturedStmt());
3247  };
3248  auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
3249  IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
3250  const OMPTaskDataTy &Data) {
3251  CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn,
3252  SharedsTy, CapturedStruct, IfCond,
3253  Data);
3254  };
3255  EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data);
3256 }
3257 
3259  const OMPTaskyieldDirective &S) {
3260  CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getBeginLoc());
3261 }
3262 
3264  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_barrier);
3265 }
3266 
3268  CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc());
3269 }
3270 
3272  const OMPTaskgroupDirective &S) {
3273  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3274  Action.Enter(CGF);
3275  if (const Expr *E = S.getReductionRef()) {
3278  OMPTaskDataTy Data;
3279  for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
3280  auto IPriv = C->privates().begin();
3281  auto IRed = C->reduction_ops().begin();
3282  auto ILHS = C->lhs_exprs().begin();
3283  auto IRHS = C->rhs_exprs().begin();
3284  for (const Expr *Ref : C->varlists()) {
3285  Data.ReductionVars.emplace_back(Ref);
3286  Data.ReductionCopies.emplace_back(*IPriv);
3287  Data.ReductionOps.emplace_back(*IRed);
3288  LHSs.emplace_back(*ILHS);
3289  RHSs.emplace_back(*IRHS);
3290  std::advance(IPriv, 1);
3291  std::advance(IRed, 1);
3292  std::advance(ILHS, 1);
3293  std::advance(IRHS, 1);
3294  }
3295  }
3296  llvm::Value *ReductionDesc =
3297  CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(),
3298  LHSs, RHSs, Data);
3299  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3300  CGF.EmitVarDecl(*VD);
3301  CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),
3302  /*Volatile=*/false, E->getType());
3303  }
3305  };
3306  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3307  CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getBeginLoc());
3308 }
3309 
3311  CGM.getOpenMPRuntime().emitFlush(
3312  *this,
3313  [&S]() -> ArrayRef<const Expr *> {
3314  if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>())
3315  return llvm::makeArrayRef(FlushClause->varlist_begin(),
3316  FlushClause->varlist_end());
3317  return llvm::None;
3318  }(),
3319  S.getBeginLoc());
3320 }
3321 
3323  const CodeGenLoopTy &CodeGenLoop,
3324  Expr *IncExpr) {
3325  // Emit the loop iteration variable.
3326  const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3327  const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3328  EmitVarDecl(*IVDecl);
3329 
3330  // Emit the iterations count variable.
3331  // If it is not a variable, Sema decided to calculate iterations count on each
3332  // iteration (e.g., it is foldable into a constant).
3333  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3334  EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3335  // Emit calculation of the iterations count.
3336  EmitIgnoredExpr(S.getCalcLastIteration());
3337  }
3338 
3339  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3340 
3341  bool HasLastprivateClause = false;
3342  // Check pre-condition.
3343  {
3344  OMPLoopScope PreInitScope(*this, S);
3345  // Skip the entire loop if we don't meet the precondition.
3346  // If the condition constant folds and can be elided, avoid emitting the
3347  // whole loop.
3348  bool CondConstant;
3349  llvm::BasicBlock *ContBlock = nullptr;
3350  if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
3351  if (!CondConstant)
3352  return;
3353  } else {
3354  llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
3355  ContBlock = createBasicBlock("omp.precond.end");
3356  emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
3357  getProfileCount(&S));
3358  EmitBlock(ThenBlock);
3359  incrementProfileCounter(&S);
3360  }
3361 
3362  emitAlignedClause(*this, S);
3363  // Emit 'then' code.
3364  {
3365  // Emit helper vars inits.
3366 
3367  LValue LB = EmitOMPHelperVar(
3368  *this, cast<DeclRefExpr>(
3371  : S.getLowerBoundVariable())));
3372  LValue UB = EmitOMPHelperVar(
3373  *this, cast<DeclRefExpr>(
3376  : S.getUpperBoundVariable())));
3377  LValue ST =
3378  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
3379  LValue IL =
3380  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
3381 
3382  OMPPrivateScope LoopScope(*this);
3383  if (EmitOMPFirstprivateClause(S, LoopScope)) {
3384  // Emit implicit barrier to synchronize threads and avoid data races
3385  // on initialization of firstprivate variables and post-update of
3386  // lastprivate variables.
3387  CGM.getOpenMPRuntime().emitBarrierCall(
3388  *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
3389  /*ForceSimpleCall=*/true);
3390  }
3391  EmitOMPPrivateClause(S, LoopScope);
3395  EmitOMPReductionClauseInit(S, LoopScope);
3396  HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
3397  EmitOMPPrivateLoopCounters(S, LoopScope);
3398  (void)LoopScope.Privatize();
3400  CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
3401 
3402  // Detect the distribute schedule kind and chunk.
3403  llvm::Value *Chunk = nullptr;
3405  if (const auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
3406  ScheduleKind = C->getDistScheduleKind();
3407  if (const Expr *Ch = C->getChunkSize()) {
3408  Chunk = EmitScalarExpr(Ch);
3409  Chunk = EmitScalarConversion(Chunk, Ch->getType(),
3411  S.getBeginLoc());
3412  }
3413  } else {
3414  // Default behaviour for dist_schedule clause.
3415  CGM.getOpenMPRuntime().getDefaultDistScheduleAndChunk(
3416  *this, S, ScheduleKind, Chunk);
3417  }
3418  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3419  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3420 
3421  // OpenMP [2.10.8, distribute Construct, Description]
3422  // If dist_schedule is specified, kind must be static. If specified,
3423  // iterations are divided into chunks of size chunk_size, chunks are
3424  // assigned to the teams of the league in a round-robin fashion in the
3425  // order of the team number. When no chunk_size is specified, the
3426  // iteration space is divided into chunks that are approximately equal
3427  // in size, and at most one chunk is distributed to each team of the
3428  // league. The size of the chunks is unspecified in this case.
3429  bool StaticChunked = RT.isStaticChunked(
3430  ScheduleKind, /* Chunked */ Chunk != nullptr) &&
3432  if (RT.isStaticNonchunked(ScheduleKind,
3433  /* Chunked */ Chunk != nullptr) ||
3434  StaticChunked) {
3436  EmitOMPSimdInit(S, /*IsMonotonic=*/true);
3437  CGOpenMPRuntime::StaticRTInput StaticInit(
3438  IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(),
3439  LB.getAddress(), UB.getAddress(), ST.getAddress(),
3440  StaticChunked ? Chunk : nullptr);
3441  RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind,
3442  StaticInit);
3443  JumpDest LoopExit =
3444  getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
3445  // UB = min(UB, GlobalUB);
3448  : S.getEnsureUpperBound());
3449  // IV = LB;
3451  ? S.getCombinedInit()
3452  : S.getInit());
3453 
3454  const Expr *Cond =
3456  ? S.getCombinedCond()
3457  : S.getCond();
3458 
3459  if (StaticChunked)
3460  Cond = S.getCombinedDistCond();
3461 
3462  // For static unchunked schedules generate:
3463  //
3464  // 1. For distribute alone, codegen
3465  // while (idx <= UB) {
3466  // BODY;
3467  // ++idx;
3468  // }
3469  //
3470  // 2. When combined with 'for' (e.g. as in 'distribute parallel for')
3471  // while (idx <= UB) {
3472  // <CodeGen rest of pragma>(LB, UB);
3473  // idx += ST;
3474  // }
3475  //
3476  // For static chunk one schedule generate:
3477  //
3478  // while (IV <= GlobalUB) {
3479  // <CodeGen rest of pragma>(LB, UB);
3480  // LB += ST;
3481  // UB += ST;
3482  // UB = min(UB, GlobalUB);
3483  // IV = LB;
3484  // }
3485  //
3486  EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), Cond, IncExpr,
3487  [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3488  CodeGenLoop(CGF, S, LoopExit);
3489  },
3490  [&S, StaticChunked](CodeGenFunction &CGF) {
3491  if (StaticChunked) {
3492  CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());
3493  CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());
3494  CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());
3495  CGF.EmitIgnoredExpr(S.getCombinedInit());
3496  }
3497  });
3498  EmitBlock(LoopExit.getBlock());
3499  // Tell the runtime we are done.
3500  RT.emitForStaticFinish(*this, S.getBeginLoc(), S.getDirectiveKind());
3501  } else {
3502  // Emit the outer loop, which requests its work chunk [LB..UB] from
3503  // runtime and runs the inner loop to process it.
3504  const OMPLoopArguments LoopArguments = {
3505  LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(),
3506  Chunk};
3507  EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
3508  CodeGenLoop);
3509  }
3511  EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
3512  return CGF.Builder.CreateIsNotNull(
3513  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3514  });
3515  }
3519  EmitOMPReductionClauseFinal(S, OMPD_simd);
3520  // Emit post-update of the reduction variables if IsLastIter != 0.
3522  *this, S, [IL, &S](CodeGenFunction &CGF) {
3523  return CGF.Builder.CreateIsNotNull(
3524  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3525  });
3526  }
3527  // Emit final copy of the lastprivate variables if IsLastIter != 0.
3528  if (HasLastprivateClause) {
3529  EmitOMPLastprivateClauseFinal(
3530  S, /*NoFinals=*/false,
3531  Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
3532  }
3533  }
3534 
3535  // We're now done with the loop, so jump to the continuation block.
3536  if (ContBlock) {
3537  EmitBranch(ContBlock);
3538  EmitBlock(ContBlock, true);
3539  }
3540  }
3541 }
3542 
3544  const OMPDistributeDirective &S) {
3545  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3547  };
3548  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3549  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3550 }
3551 
3552 static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM,
3553  const CapturedStmt *S) {
3554  CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
3556  CGF.CapturedStmtInfo = &CapStmtInfo;
3557  llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S);
3558  Fn->setDoesNotRecurse();
3559  return Fn;
3560 }
3561 
3563  if (S.hasClausesOfKind<OMPDependClause>()) {
3564  assert(!S.getAssociatedStmt() &&
3565  "No associated statement must be in ordered depend construct.");
3566  for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
3567  CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
3568  return;
3569  }
3570  const auto *C = S.getSingleClause<OMPSIMDClause>();
3571  auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
3572  PrePostActionTy &Action) {
3573  const CapturedStmt *CS = S.getInnermostCapturedStmt();
3574  if (C) {
3576  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
3577  llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS);
3578  CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
3579  OutlinedFn, CapturedVars);
3580  } else {
3581  Action.Enter(CGF);
3582  CGF.EmitStmt(CS->getCapturedStmt());
3583  }
3584  };
3585  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3586  CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getBeginLoc(), !C);
3587 }
3588 
3590  QualType SrcType, QualType DestType,
3591  SourceLocation Loc) {
3592  assert(CGF.hasScalarEvaluationKind(DestType) &&
3593  "DestType must have scalar evaluation kind.");
3594  assert(!Val.isAggregate() && "Must be a scalar or complex.");
3595  return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
3596  DestType, Loc)
3598  Val.getComplexVal(), SrcType, DestType, Loc);
3599 }
3600 
3603  QualType DestType, SourceLocation Loc) {
3604  assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
3605  "DestType must have complex evaluation kind.");
3606  CodeGenFunction::ComplexPairTy ComplexVal;
3607  if (Val.isScalar()) {
3608  // Convert the input element to the element type of the complex.
3609  QualType DestElementType =
3610  DestType->castAs<ComplexType>()->getElementType();
3611  llvm::Value *ScalarVal = CGF.EmitScalarConversion(
3612  Val.getScalarVal(), SrcType, DestElementType, Loc);
3613  ComplexVal = CodeGenFunction::ComplexPairTy(
3614  ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
3615  } else {
3616  assert(Val.isComplex() && "Must be a scalar or complex.");
3617  QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
3618  QualType DestElementType =
3619  DestType->castAs<ComplexType>()->getElementType();
3620  ComplexVal.first = CGF.EmitScalarConversion(
3621  Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
3622  ComplexVal.second = CGF.EmitScalarConversion(
3623  Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
3624  }
3625  return ComplexVal;
3626 }
3627 
3628 static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst,
3629  LValue LVal, RValue RVal) {
3630  if (LVal.isGlobalReg()) {
3631  CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
3632  } else {
3633  CGF.EmitAtomicStore(RVal, LVal,
3634  IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3635  : llvm::AtomicOrdering::Monotonic,
3636  LVal.isVolatile(), /*isInit=*/false);
3637  }
3638 }
3639 
3641  QualType RValTy, SourceLocation Loc) {
3642  switch (getEvaluationKind(LVal.getType())) {
3643  case TEK_Scalar:
3644  EmitStoreThroughLValue(RValue::get(convertToScalarValue(
3645  *this, RVal, RValTy, LVal.getType(), Loc)),
3646  LVal);
3647  break;
3648  case TEK_Complex:
3649  EmitStoreOfComplex(
3650  convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,
3651  /*isInit=*/false);
3652  break;
3653  case TEK_Aggregate:
3654  llvm_unreachable("Must be a scalar or complex.");
3655  }
3656 }
3657 
3658 static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst,
3659  const Expr *X, const Expr *V,
3660  SourceLocation Loc) {
3661  // v = x;
3662  assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
3663  assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
3664  LValue XLValue = CGF.EmitLValue(X);
3665  LValue VLValue = CGF.EmitLValue(V);
3666  RValue Res = XLValue.isGlobalReg()
3667  ? CGF.EmitLoadOfLValue(XLValue, Loc)
3668  : CGF.EmitAtomicLoad(
3669  XLValue, Loc,
3670  IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3671  : llvm::AtomicOrdering::Monotonic,
3672  XLValue.isVolatile());
3673  // OpenMP, 2.12.6, atomic Construct
3674  // Any atomic construct with a seq_cst clause forces the atomically
3675  // performed operation to include an implicit flush operation without a
3676  // list.
3677  if (IsSeqCst)
3678  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3679  CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
3680 }
3681 
3682 static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst,
3683  const Expr *X, const Expr *E,
3684  SourceLocation Loc) {
3685  // x = expr;
3686  assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
3687  emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
3688  // OpenMP, 2.12.6, atomic Construct
3689  // Any atomic construct with a seq_cst clause forces the atomically
3690  // performed operation to include an implicit flush operation without a
3691  // list.
3692  if (IsSeqCst)
3693  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3694 }
3695 
3696 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
3697  RValue Update,
3698  BinaryOperatorKind BO,
3699  llvm::AtomicOrdering AO,
3700  bool IsXLHSInRHSPart) {
3701  ASTContext &Context = CGF.getContext();
3702  // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
3703  // expression is simple and atomic is allowed for the given type for the
3704  // target platform.
3705  if (BO == BO_Comma || !Update.isScalar() ||
3706  !Update.getScalarVal()->getType()->isIntegerTy() ||
3707  !X.isSimple() || (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
3708  (Update.getScalarVal()->getType() !=
3709  X.getAddress().getElementType())) ||
3710  !X.getAddress().getElementType()->isIntegerTy() ||
3711  !Context.getTargetInfo().hasBuiltinAtomic(
3712  Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
3713  return std::make_pair(false, RValue::get(nullptr));
3714 
3715  llvm::AtomicRMWInst::BinOp RMWOp;
3716  switch (BO) {
3717  case BO_Add:
3718  RMWOp = llvm::AtomicRMWInst::Add;
3719  break;
3720  case BO_Sub:
3721  if (!IsXLHSInRHSPart)
3722  return std::make_pair(false, RValue::get(nullptr));
3723  RMWOp = llvm::AtomicRMWInst::Sub;
3724  break;
3725  case BO_And:
3726  RMWOp = llvm::AtomicRMWInst::And;
3727  break;
3728  case BO_Or:
3729  RMWOp = llvm::AtomicRMWInst::Or;
3730  break;
3731  case BO_Xor:
3732  RMWOp = llvm::AtomicRMWInst::Xor;
3733  break;
3734  case BO_LT:
3736  ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
3737  : llvm::AtomicRMWInst::Max)
3738  : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
3739  : llvm::AtomicRMWInst::UMax);
3740  break;
3741  case BO_GT:
3743  ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
3744  : llvm::AtomicRMWInst::Min)
3745  : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
3746  : llvm::AtomicRMWInst::UMin);
3747  break;
3748  case BO_Assign:
3749  RMWOp = llvm::AtomicRMWInst::Xchg;
3750  break;
3751  case BO_Mul:
3752  case BO_Div:
3753  case BO_Rem:
3754  case BO_Shl:
3755  case BO_Shr:
3756  case BO_LAnd:
3757  case BO_LOr:
3758  return std::make_pair(false, RValue::get(nullptr));
3759  case BO_PtrMemD:
3760  case BO_PtrMemI:
3761  case BO_LE:
3762  case BO_GE:
3763  case BO_EQ:
3764  case BO_NE:
3765  case BO_Cmp:
3766  case BO_AddAssign:
3767  case BO_SubAssign:
3768  case BO_AndAssign:
3769  case BO_OrAssign:
3770  case BO_XorAssign:
3771  case BO_MulAssign:
3772  case BO_DivAssign:
3773  case BO_RemAssign:
3774  case BO_ShlAssign:
3775  case BO_ShrAssign:
3776  case BO_Comma:
3777  llvm_unreachable("Unsupported atomic update operation");
3778  }
3779  llvm::Value *UpdateVal = Update.getScalarVal();
3780  if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
3781  UpdateVal = CGF.Builder.CreateIntCast(
3782  IC, X.getAddress().getElementType(),
3784  }
3785  llvm::Value *Res =
3786  CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO);
3787  return std::make_pair(true, RValue::get(Res));
3788 }
3789 
3791  LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3792  llvm::AtomicOrdering AO, SourceLocation Loc,
3793  const llvm::function_ref<RValue(RValue)> CommonGen) {
3794  // Update expressions are allowed to have the following forms:
3795  // x binop= expr; -> xrval + expr;
3796  // x++, ++x -> xrval + 1;
3797  // x--, --x -> xrval - 1;
3798  // x = x binop expr; -> xrval binop expr
3799  // x = expr Op x; - > expr binop xrval;
3800  auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);
3801  if (!Res.first) {
3802  if (X.isGlobalReg()) {
3803  // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
3804  // 'xrval'.
3805  EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
3806  } else {
3807  // Perform compare-and-swap procedure.
3808  EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
3809  }
3810  }
3811  return Res;
3812 }
3813 
3814 static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst,
3815  const Expr *X, const Expr *E,
3816  const Expr *UE, bool IsXLHSInRHSPart,
3817  SourceLocation Loc) {
3818  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3819  "Update expr in 'atomic update' must be a binary operator.");
3820  const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3821  // Update expressions are allowed to have the following forms:
3822  // x binop= expr; -> xrval + expr;
3823  // x++, ++x -> xrval + 1;
3824  // x--, --x -> xrval - 1;
3825  // x = x binop expr; -> xrval binop expr
3826  // x = expr Op x; - > expr binop xrval;
3827  assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
3828  LValue XLValue = CGF.EmitLValue(X);
3829  RValue ExprRValue = CGF.EmitAnyExpr(E);
3830  llvm::AtomicOrdering AO = IsSeqCst
3831  ? llvm::AtomicOrdering::SequentiallyConsistent
3832  : llvm::AtomicOrdering::Monotonic;
3833  const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3834  const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3835  const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3836  const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3837  auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {
3838  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3839  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3840  return CGF.EmitAnyExpr(UE);
3841  };
3843  XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3844  // OpenMP, 2.12.6, atomic Construct
3845  // Any atomic construct with a seq_cst clause forces the atomically
3846  // performed operation to include an implicit flush operation without a
3847  // list.
3848  if (IsSeqCst)
3849  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3850 }
3851 
3853  QualType SourceType, QualType ResType,
3854  SourceLocation Loc) {
3855  switch (CGF.getEvaluationKind(ResType)) {
3856  case TEK_Scalar:
3857  return RValue::get(
3858  convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
3859  case TEK_Complex: {
3860  auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
3861  return RValue::getComplex(Res.first, Res.second);
3862  }
3863  case TEK_Aggregate:
3864  break;
3865  }
3866  llvm_unreachable("Must be a scalar or complex.");
3867 }
3868 
3869 static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst,
3870  bool IsPostfixUpdate, const Expr *V,
3871  const Expr *X, const Expr *E,
3872  const Expr *UE, bool IsXLHSInRHSPart,
3873  SourceLocation Loc) {
3874  assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
3875  assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
3876  RValue NewVVal;
3877  LValue VLValue = CGF.EmitLValue(V);
3878  LValue XLValue = CGF.EmitLValue(X);
3879  RValue ExprRValue = CGF.EmitAnyExpr(E);
3880  llvm::AtomicOrdering AO = IsSeqCst
3881  ? llvm::AtomicOrdering::SequentiallyConsistent
3882  : llvm::AtomicOrdering::Monotonic;
3883  QualType NewVValType;
3884  if (UE) {
3885  // 'x' is updated with some additional value.
3886  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3887  "Update expr in 'atomic capture' must be a binary operator.");
3888  const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3889  // Update expressions are allowed to have the following forms:
3890  // x binop= expr; -> xrval + expr;
3891  // x++, ++x -> xrval + 1;
3892  // x--, --x -> xrval - 1;
3893  // x = x binop expr; -> xrval binop expr
3894  // x = expr Op x; - > expr binop xrval;
3895  const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3896  const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3897  const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3898  NewVValType = XRValExpr->getType();
3899  const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3900  auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
3901  IsPostfixUpdate](RValue XRValue) {
3902  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3903  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3904  RValue Res = CGF.EmitAnyExpr(UE);
3905  NewVVal = IsPostfixUpdate ? XRValue : Res;
3906  return Res;
3907  };
3908  auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3909  XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3910  if (Res.first) {
3911  // 'atomicrmw' instruction was generated.
3912  if (IsPostfixUpdate) {
3913  // Use old value from 'atomicrmw'.
3914  NewVVal = Res.second;
3915  } else {
3916  // 'atomicrmw' does not provide new value, so evaluate it using old
3917  // value of 'x'.
3918  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3919  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
3920  NewVVal = CGF.EmitAnyExpr(UE);
3921  }
3922  }
3923  } else {
3924  // 'x' is simply rewritten with some 'expr'.
3925  NewVValType = X->getType().getNonReferenceType();
3926  ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
3927  X->getType().getNonReferenceType(), Loc);
3928  auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) {
3929  NewVVal = XRValue;
3930  return ExprRValue;
3931  };
3932  // Try to perform atomicrmw xchg, otherwise simple exchange.
3933  auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3934  XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
3935  Loc, Gen);
3936  if (Res.first) {
3937  // 'atomicrmw' instruction was generated.
3938  NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
3939  }
3940  }
3941  // Emit post-update store to 'v' of old/new 'x' value.
3942  CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
3943  // OpenMP, 2.12.6, atomic Construct
3944  // Any atomic construct with a seq_cst clause forces the atomically
3945  // performed operation to include an implicit flush operation without a
3946  // list.
3947  if (IsSeqCst)
3948  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3949 }
3950 
3952  bool IsSeqCst, bool IsPostfixUpdate,
3953  const Expr *X, const Expr *V, const Expr *E,
3954  const Expr *UE, bool IsXLHSInRHSPart,
3955  SourceLocation Loc) {
3956  switch (Kind) {
3957  case OMPC_read:
3958  emitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc);
3959  break;
3960  case OMPC_write:
3961  emitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc);
3962  break;
3963  case OMPC_unknown:
3964  case OMPC_update:
3965  emitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc);
3966  break;
3967  case OMPC_capture:
3968  emitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE,
3969  IsXLHSInRHSPart, Loc);
3970  break;
3971  case OMPC_if:
3972  case OMPC_final:
3973  case OMPC_num_threads:
3974  case OMPC_private:
3975  case OMPC_firstprivate:
3976  case OMPC_lastprivate:
3977  case OMPC_reduction:
3978  case OMPC_task_reduction:
3979  case OMPC_in_reduction:
3980  case OMPC_safelen:
3981  case OMPC_simdlen:
3982  case OMPC_allocator:
3983  case OMPC_allocate:
3984  case OMPC_collapse:
3985  case OMPC_default:
3986  case OMPC_seq_cst:
3987  case OMPC_shared:
3988  case OMPC_linear:
3989  case OMPC_aligned:
3990  case OMPC_copyin:
3991  case OMPC_copyprivate:
3992  case OMPC_flush:
3993  case OMPC_proc_bind:
3994  case OMPC_schedule:
3995  case OMPC_ordered:
3996  case OMPC_nowait:
3997  case OMPC_untied:
3998  case OMPC_threadprivate:
3999  case OMPC_depend:
4000  case OMPC_mergeable:
4001  case OMPC_device:
4002  case OMPC_threads:
4003  case OMPC_simd:
4004  case OMPC_map:
4005  case OMPC_num_teams:
4006  case OMPC_thread_limit:
4007  case OMPC_priority:
4008  case OMPC_grainsize:
4009  case OMPC_nogroup:
4010  case OMPC_num_tasks:
4011  case OMPC_hint:
4012  case OMPC_dist_schedule:
4013  case OMPC_defaultmap:
4014  case OMPC_uniform:
4015  case OMPC_to:
4016  case OMPC_from:
4017  case OMPC_use_device_ptr:
4018  case OMPC_is_device_ptr:
4019  case OMPC_unified_address:
4020  case OMPC_unified_shared_memory:
4021  case OMPC_reverse_offload:
4022  case OMPC_dynamic_allocators:
4023  case OMPC_atomic_default_mem_order:
4024  case OMPC_device_type:
4025  llvm_unreachable("Clause is not allowed in 'omp atomic'.");
4026  }
4027 }
4028 
4030  bool IsSeqCst = S.getSingleClause<OMPSeqCstClause>();
4032  for (const OMPClause *C : S.clauses()) {
4033  // Find first clause (skip seq_cst clause, if it is first).
4034  if (C->getClauseKind() != OMPC_seq_cst) {
4035  Kind = C->getClauseKind();
4036  break;
4037  }
4038  }
4039 
4040  const Stmt *CS = S.getInnermostCapturedStmt()->IgnoreContainers();
4041  if (const auto *FE = dyn_cast<FullExpr>(CS))
4042  enterFullExpression(FE);
4043  // Processing for statements under 'atomic capture'.
4044  if (const auto *Compound = dyn_cast<CompoundStmt>(CS)) {
4045  for (const Stmt *C : Compound->body()) {
4046  if (const auto *FE = dyn_cast<FullExpr>(C))
4047  enterFullExpression(FE);
4048  }
4049  }
4050 
4051  auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF,
4052  PrePostActionTy &) {
4053  CGF.EmitStopPoint(CS);
4054  emitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(),
4055  S.getV(), S.getExpr(), S.getUpdateExpr(),
4056  S.isXLHSInRHSPart(), S.getBeginLoc());
4057  };
4058  OMPLexicalScope Scope(*this, S, OMPD_unknown);
4059  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen);
4060 }
4061 
4063  const OMPExecutableDirective &S,
4064  const RegionCodeGenTy &CodeGen) {
4066  CodeGenModule &CGM = CGF.CGM;
4067 
4068  // On device emit this construct as inlined code.
4069  if (CGM.getLangOpts().OpenMPIsDevice) {
4070  OMPLexicalScope Scope(CGF, S, OMPD_target);
4071  CGM.getOpenMPRuntime().emitInlinedDirective(
4072  CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4074  });
4075  return;
4076  }
4077 
4078  llvm::Function *Fn = nullptr;
4079  llvm::Constant *FnID = nullptr;
4080 
4081  const Expr *IfCond = nullptr;
4082  // Check for the at most one if clause associated with the target region.
4083  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4084  if (C->getNameModifier() == OMPD_unknown ||
4085  C->getNameModifier() == OMPD_target) {
4086  IfCond = C->getCondition();
4087  break;
4088  }
4089  }
4090 
4091  // Check if we have any device clause associated with the directive.
4092  const Expr *Device = nullptr;
4093  if (auto *C = S.getSingleClause<OMPDeviceClause>())
4094  Device = C->getDevice();
4095 
4096  // Check if we have an if clause whose conditional always evaluates to false
4097  // or if we do not have any targets specified. If so the target region is not
4098  // an offload entry point.
4099  bool IsOffloadEntry = true;
4100  if (IfCond) {
4101  bool Val;
4102  if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
4103  IsOffloadEntry = false;
4104  }
4105  if (CGM.getLangOpts().OMPTargetTriples.empty())
4106  IsOffloadEntry = false;
4107 
4108  assert(CGF.CurFuncDecl && "No parent declaration for target region!");
4109  StringRef ParentName;
4110  // In case we have Ctors/Dtors we use the complete type variant to produce
4111  // the mangling of the device outlined kernel.
4112  if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
4113  ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));
4114  else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
4115  ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));
4116  else
4117  ParentName =
4118  CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));
4119 
4120  // Emit target region as a standalone region.
4121  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
4122  IsOffloadEntry, CodeGen);
4123  OMPLexicalScope Scope(CGF, S, OMPD_task);
4124  auto &&SizeEmitter = [](CodeGenFunction &CGF, const OMPLoopDirective &D) {
4125  OMPLoopScope(CGF, D);
4126  // Emit calculation of the iterations count.
4127  llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations());
4128  NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty,
4129  /*isSigned=*/false);
4130  return NumIterations;
4131  };
4132  if (IsOffloadEntry)
4133  CGM.getOpenMPRuntime().emitTargetNumIterationsCall(CGF, S, Device,
4134  SizeEmitter);
4135  CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device);
4136 }
4137 
4139  PrePostActionTy &Action) {
4140  Action.Enter(CGF);
4141  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4142  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4143  CGF.EmitOMPPrivateClause(S, PrivateScope);
4144  (void)PrivateScope.Privatize();
4146  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4147 
4148  CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
4149 }
4150 
4152  StringRef ParentName,
4153  const OMPTargetDirective &S) {
4154  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4155  emitTargetRegion(CGF, S, Action);
4156  };
4157  llvm::Function *Fn;
4158  llvm::Constant *Addr;
4159  // Emit target region as a standalone region.
4160  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4161  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4162  assert(Fn && Addr && "Target device function emission failed.");
4163 }
4164 
4166  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4167  emitTargetRegion(CGF, S, Action);
4168  };
4169  emitCommonOMPTargetDirective(*this, S, CodeGen);
4170 }
4171 
4173  const OMPExecutableDirective &S,
4174  OpenMPDirectiveKind InnermostKind,
4175  const RegionCodeGenTy &CodeGen) {
4176  const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
4177  llvm::Function *OutlinedFn =
4178  CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
4179  S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
4180 
4181  const auto *NT = S.getSingleClause<OMPNumTeamsClause>();
4182  const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
4183  if (NT || TL) {
4184  const Expr *NumTeams = NT ? NT->getNumTeams() : nullptr;
4185  const Expr *ThreadLimit = TL ? TL->getThreadLimit() : nullptr;
4186 
4187  CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
4188  S.getBeginLoc());
4189  }
4190 
4191  OMPTeamsScope Scope(CGF, S);
4193  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
4194  CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn,
4195  CapturedVars);
4196 }
4197 
4199  // Emit teams region as a standalone region.
4200  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4201  Action.Enter(CGF);
4202  OMPPrivateScope PrivateScope(CGF);
4203  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4204  CGF.EmitOMPPrivateClause(S, PrivateScope);
4205  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4206  (void)PrivateScope.Privatize();
4207  CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
4208  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4209  };
4210  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4212  [](CodeGenFunction &) { return nullptr; });
4213 }
4214 
4216  const OMPTargetTeamsDirective &S) {
4217  auto *CS = S.getCapturedStmt(OMPD_teams);
4218  Action.Enter(CGF);
4219  // Emit teams region as a standalone region.
4220  auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4221  Action.Enter(CGF);
4222  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4223  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4224  CGF.EmitOMPPrivateClause(S, PrivateScope);
4225  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4226  (void)PrivateScope.Privatize();
4228  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4229  CGF.EmitStmt(CS->getCapturedStmt());
4230  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4231  };
4232  emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
4234  [](CodeGenFunction &) { return nullptr; });
4235 }
4236 
4238  CodeGenModule &CGM, StringRef ParentName,
4239  const OMPTargetTeamsDirective &S) {
4240  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4241  emitTargetTeamsRegion(CGF, Action, S);
4242  };
4243  llvm::Function *Fn;
4244  llvm::Constant *Addr;
4245  // Emit target region as a standalone region.
4246  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4247  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4248  assert(Fn && Addr && "Target device function emission failed.");
4249 }
4250 
4252  const OMPTargetTeamsDirective &S) {
4253  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4254  emitTargetTeamsRegion(CGF, Action, S);
4255  };
4256  emitCommonOMPTargetDirective(*this, S, CodeGen);
4257 }
4258 
4259 static void
4262  Action.Enter(CGF);
4263  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4265  };
4266 
4267  // Emit teams region as a standalone region.
4268  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4269  PrePostActionTy &Action) {
4270  Action.Enter(CGF);
4271  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4272  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4273  (void)PrivateScope.Privatize();
4274  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4275  CodeGenDistribute);
4276  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4277  };
4278  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
4280  [](CodeGenFunction &) { return nullptr; });
4281 }
4282 
4284  CodeGenModule &CGM, StringRef ParentName,
4286  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4287  emitTargetTeamsDistributeRegion(CGF, Action, S);
4288  };
4289  llvm::Function *Fn;
4290  llvm::Constant *Addr;
4291  // Emit target region as a standalone region.
4292  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4293  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4294  assert(Fn && Addr && "Target device function emission failed.");
4295 }
4296 
4299  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4300  emitTargetTeamsDistributeRegion(CGF, Action, S);
4301  };
4302  emitCommonOMPTargetDirective(*this, S, CodeGen);
4303 }
4304 
4306  CodeGenFunction &CGF, PrePostActionTy &Action,
4308  Action.Enter(CGF);
4309  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4311  };
4312 
4313  // Emit teams region as a standalone region.
4314  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4315  PrePostActionTy &Action) {
4316  Action.Enter(CGF);
4317  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4318  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4319  (void)PrivateScope.Privatize();
4320  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4321  CodeGenDistribute);
4322  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4323  };
4324  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);
4326  [](CodeGenFunction &) { return nullptr; });
4327 }
4328 
4330  CodeGenModule &CGM, StringRef ParentName,
4332  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4333  emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4334  };
4335  llvm::Function *Fn;
4336  llvm::Constant *Addr;
4337  // Emit target region as a standalone region.
4338  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4339  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4340  assert(Fn && Addr && "Target device function emission failed.");
4341 }
4342 
4345  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4346  emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4347  };
4348  emitCommonOMPTargetDirective(*this, S, CodeGen);
4349 }
4350 
4352  const OMPTeamsDistributeDirective &S) {
4353 
4354  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4356  };
4357 
4358  // Emit teams region as a standalone region.
4359  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4360  PrePostActionTy &Action) {
4361  Action.Enter(CGF);
4362  OMPPrivateScope PrivateScope(CGF);
4363  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4364  (void)PrivateScope.Privatize();
4365  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4366  CodeGenDistribute);
4367  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4368  };
4369  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4371  [](CodeGenFunction &) { return nullptr; });
4372 }
4373 
4376  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4378  };
4379 
4380  // Emit teams region as a standalone region.
4381  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4382  PrePostActionTy &Action) {
4383  Action.Enter(CGF);
4384  OMPPrivateScope PrivateScope(CGF);
4385  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4386  (void)PrivateScope.Privatize();
4387  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
4388  CodeGenDistribute);
4389  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4390  };
4391  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen);
4393  [](CodeGenFunction &) { return nullptr; });
4394 }
4395 
4398  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4400  S.getDistInc());
4401  };
4402 
4403  // Emit teams region as a standalone region.
4404  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4405  PrePostActionTy &Action) {
4406  Action.Enter(CGF);
4407  OMPPrivateScope PrivateScope(CGF);
4408  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4409  (void)PrivateScope.Privatize();
4410  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4411  CodeGenDistribute);
4412  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4413  };
4414  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4416  [](CodeGenFunction &) { return nullptr; });
4417 }
4418 
4421  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4423  S.getDistInc());
4424  };
4425 
4426  // Emit teams region as a standalone region.
4427  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4428  PrePostActionTy &Action) {
4429  Action.Enter(CGF);
4430  OMPPrivateScope PrivateScope(CGF);
4431  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4432  (void)PrivateScope.Privatize();
4433  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4434  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4435  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4436  };
4437  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4439  [](CodeGenFunction &) { return nullptr; });
4440 }
4441 
4444  PrePostActionTy &Action) {
4445  Action.Enter(CGF);
4446  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4448  S.getDistInc());
4449  };
4450 
4451  // Emit teams region as a standalone region.
4452  auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4453  PrePostActionTy &Action) {
4454  Action.Enter(CGF);
4455  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4456  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4457  (void)PrivateScope.Privatize();
4458  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4459  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4460  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4461  };
4462 
4463  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
4464  CodeGenTeams);
4466  [](CodeGenFunction &) { return nullptr; });
4467 }
4468 
4470  CodeGenModule &CGM, StringRef ParentName,
4472  // Emit SPMD target teams distribute parallel for region as a standalone
4473  // region.
4474  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4476  };
4477  llvm::Function *Fn;
4478  llvm::Constant *Addr;
4479  // Emit target region as a standalone region.
4480  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4481  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4482  assert(Fn && Addr && "Target device function emission failed.");
4483 }
4484 
4487  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4489  };
4490  emitCommonOMPTargetDirective(*this, S, CodeGen);
4491 }
4492 
4494  CodeGenFunction &CGF,
4496  PrePostActionTy &Action) {
4497  Action.Enter(CGF);
4498  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4500  S.getDistInc());
4501  };
4502 
4503  // Emit teams region as a standalone region.
4504  auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4505  PrePostActionTy &Action) {
4506  Action.Enter(CGF);
4507  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4508  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4509  (void)PrivateScope.Privatize();
4510  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4511  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4512  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4513  };
4514 
4515  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd,
4516  CodeGenTeams);
4518  [](CodeGenFunction &) { return nullptr; });
4519 }
4520 
4522  CodeGenModule &CGM, StringRef ParentName,
4524  // Emit SPMD target teams distribute parallel for simd region as a standalone
4525  // region.
4526  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4528  };
4529  llvm::Function *Fn;
4530  llvm::Constant *Addr;
4531  // Emit target region as a standalone region.
4532  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4533  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4534  assert(Fn && Addr && "Target device function emission failed.");
4535 }
4536 
4539  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4541  };
4542  emitCommonOMPTargetDirective(*this, S, CodeGen);
4543 }
4544 
4546  const OMPCancellationPointDirective &S) {
4547  CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getBeginLoc(),
4548  S.getCancelRegion());
4549 }
4550 
4552  const Expr *IfCond = nullptr;
4553  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4554  if (C->getNameModifier() == OMPD_unknown ||
4555  C->getNameModifier() == OMPD_cancel) {
4556  IfCond = C->getCondition();
4557  break;
4558  }
4559  }
4560  CGM.getOpenMPRuntime().emitCancelCall(*this, S.getBeginLoc(), IfCond,
4561  S.getCancelRegion());
4562 }
4563 
4566  if (Kind == OMPD_parallel || Kind == OMPD_task ||
4567  Kind == OMPD_target_parallel)
4568  return ReturnBlock;
4569  assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
4570  Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
4571  Kind == OMPD_distribute_parallel_for ||
4572  Kind == OMPD_target_parallel_for ||
4573  Kind == OMPD_teams_distribute_parallel_for ||
4574  Kind == OMPD_target_teams_distribute_parallel_for);
4575  return OMPCancelStack.getExitBlock();
4576 }
4577 
4579  const OMPClause &NC, OMPPrivateScope &PrivateScope,
4580  const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) {
4581  const auto &C = cast<OMPUseDevicePtrClause>(NC);
4582  auto OrigVarIt = C.varlist_begin();
4583  auto InitIt = C.inits().begin();
4584  for (const Expr *PvtVarIt : C.private_copies()) {
4585  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl());
4586  const auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl());
4587  const auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl());
4588 
4589  // In order to identify the right initializer we need to match the
4590  // declaration used by the mapping logic. In some cases we may get
4591  // OMPCapturedExprDecl that refers to the original declaration.
4592  const ValueDecl *MatchingVD = OrigVD;
4593  if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
4594  // OMPCapturedExprDecl are used to privative fields of the current
4595  // structure.
4596  const auto *ME = cast<MemberExpr>(OED->getInit());
4597  assert(isa<CXXThisExpr>(ME->getBase()) &&
4598  "Base should be the current struct!");
4599  MatchingVD = ME->getMemberDecl();
4600  }
4601 
4602  // If we don't have information about the current list item, move on to
4603  // the next one.
4604  auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
4605  if (InitAddrIt == CaptureDeviceAddrMap.end())
4606  continue;
4607 
4608  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, OrigVD,
4609  InitAddrIt, InitVD,
4610  PvtVD]() {
4611  // Initialize the temporary initialization variable with the address we
4612  // get from the runtime library. We have to cast the source address
4613  // because it is always a void *. References are materialized in the
4614  // privatization scope, so the initialization here disregards the fact
4615  // the original variable is a reference.
4616  QualType AddrQTy =
4617  getContext().getPointerType(OrigVD->getType().getNonReferenceType());
4618  llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy);
4619  Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy);
4620  setAddrOfLocalVar(InitVD, InitAddr);
4621 
4622  // Emit private declaration, it will be initialized by the value we
4623  // declaration we just added to the local declarations map.
4624  EmitDecl(*PvtVD);
4625 
4626  // The initialization variables reached its purpose in the emission
4627  // of the previous declaration, so we don't need it anymore.
4628  LocalDeclMap.erase(InitVD);
4629 
4630  // Return the address of the private variable.
4631  return GetAddrOfLocalVar(PvtVD);
4632  });
4633  assert(IsRegistered && "firstprivate var already registered as private");
4634  // Silence the warning about unused variable.
4635  (void)IsRegistered;
4636 
4637  ++OrigVarIt;
4638  ++InitIt;
4639  }
4640 }
4641 
4642 // Generate the instructions for '#pragma omp target data' directive.
4644  const OMPTargetDataDirective &S) {
4645  CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true);
4646 
4647  // Create a pre/post action to signal the privatization of the device pointer.
4648  // This action can be replaced by the OpenMP runtime code generation to
4649  // deactivate privatization.
4650  bool PrivatizeDevicePointers = false;
4651  class DevicePointerPrivActionTy : public PrePostActionTy {
4652  bool &PrivatizeDevicePointers;
4653 
4654  public:
4655  explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
4656  : PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {}
4657  void Enter(CodeGenFunction &CGF) override {
4658  PrivatizeDevicePointers = true;
4659  }
4660  };
4661  DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
4662 
4663  auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers](
4664  CodeGenFunction &CGF, PrePostActionTy &Action) {
4665  auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4667  };
4668 
4669  // Codegen that selects whether to generate the privatization code or not.
4670  auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers,
4671  &InnermostCodeGen](CodeGenFunction &CGF,
4672  PrePostActionTy &Action) {
4673  RegionCodeGenTy RCG(InnermostCodeGen);
4674  PrivatizeDevicePointers = false;
4675 
4676  // Call the pre-action to change the status of PrivatizeDevicePointers if
4677  // needed.
4678  Action.Enter(CGF);
4679 
4680  if (PrivatizeDevicePointers) {
4681  OMPPrivateScope PrivateScope(CGF);
4682  // Emit all instances of the use_device_ptr clause.
4683  for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
4684  CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
4685  Info.CaptureDeviceAddrMap);
4686  (void)PrivateScope.Privatize();
4687  RCG(CGF);
4688  } else {
4689  RCG(CGF);
4690  }
4691  };
4692 
4693  // Forward the provided action to the privatization codegen.
4694  RegionCodeGenTy PrivRCG(PrivCodeGen);
4695  PrivRCG.setAction(Action);
4696 
4697  // Notwithstanding the body of the region is emitted as inlined directive,
4698  // we don't use an inline scope as changes in the references inside the
4699  // region are expected to be visible outside, so we do not privative them.
4700  OMPLexicalScope Scope(CGF, S);
4701  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
4702  PrivRCG);
4703  };
4704 
4705  RegionCodeGenTy RCG(CodeGen);
4706 
4707  // If we don't have target devices, don't bother emitting the data mapping
4708  // code.
4709  if (CGM.getLangOpts().OMPTargetTriples.empty()) {
4710  RCG(*this);
4711  return;
4712  }
4713 
4714  // Check if we have any if clause associated with the directive.
4715  const Expr *IfCond = nullptr;
4716  if (const auto *C = S.getSingleClause<OMPIfClause>())
4717  IfCond = C->getCondition();
4718 
4719  // Check if we have any device clause associated with the directive.
4720  const Expr *Device = nullptr;
4721  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4722  Device = C->getDevice();
4723 
4724  // Set the action to signal privatization of device pointers.
4725  RCG.setAction(PrivAction);
4726 
4727  // Emit region code.
4728  CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
4729  Info);
4730 }
4731 
4733  const OMPTargetEnterDataDirective &S) {
4734  // If we don't have target devices, don't bother emitting the data mapping
4735  // code.
4736  if (CGM.getLangOpts().OMPTargetTriples.empty())
4737  return;
4738 
4739  // Check if we have any if clause associated with the directive.
4740  const Expr *IfCond = nullptr;
4741  if (const auto *C = S.getSingleClause<OMPIfClause>())
4742  IfCond = C->getCondition();
4743 
4744  // Check if we have any device clause associated with the directive.
4745  const Expr *Device = nullptr;
4746  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4747  Device = C->getDevice();
4748 
4749  OMPLexicalScope Scope(*this, S, OMPD_task);
4750  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4751 }
4752 
4754  const OMPTargetExitDataDirective &S) {
4755  // If we don't have target devices, don't bother emitting the data mapping
4756  // code.
4757  if (CGM.getLangOpts().OMPTargetTriples.empty())
4758  return;
4759 
4760  // Check if we have any if clause associated with the directive.
4761  const Expr *IfCond = nullptr;
4762  if (const auto *C = S.getSingleClause<OMPIfClause>())
4763  IfCond = C->getCondition();
4764 
4765  // Check if we have any device clause associated with the directive.
4766  const Expr *Device = nullptr;
4767  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4768  Device = C->getDevice();
4769 
4770  OMPLexicalScope Scope(*this, S, OMPD_task);
4771  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4772 }
4773 
4775  const OMPTargetParallelDirective &S,
4776  PrePostActionTy &Action) {
4777  // Get the captured statement associated with the 'parallel' region.
4778  const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
4779  Action.Enter(CGF);
4780  auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4781  Action.Enter(CGF);
4782  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4783  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4784  CGF.EmitOMPPrivateClause(S, PrivateScope);
4785  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4786  (void)PrivateScope.Privatize();
4788  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4789  // TODO: Add support for clauses.
4790  CGF.EmitStmt(CS->getCapturedStmt());
4791  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4792  };
4793  emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
4796  [](CodeGenFunction &) { return nullptr; });
4797 }
4798 
4800  CodeGenModule &CGM, StringRef ParentName,
4801  const OMPTargetParallelDirective &S) {
4802  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4803  emitTargetParallelRegion(CGF, S, Action);
4804  };
4805  llvm::Function *Fn;
4806  llvm::Constant *Addr;
4807  // Emit target region as a standalone region.
4808  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4809  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4810  assert(Fn && Addr && "Target device function emission failed.");
4811 }
4812 
4814  const OMPTargetParallelDirective &S) {
4815  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4816  emitTargetParallelRegion(CGF, S, Action);
4817  };
4818  emitCommonOMPTargetDirective(*this, S, CodeGen);
4819 }
4820 
4823  PrePostActionTy &Action) {
4824  Action.Enter(CGF);
4825  // Emit directive as a combined directive that consists of two implicit
4826  // directives: 'parallel' with 'for' directive.
4827  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4828  Action.Enter(CGF);
4830  CGF, OMPD_target_parallel_for, S.hasCancel());
4833  };
4834  emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
4836 }
4837 
4839  CodeGenModule &CGM, StringRef ParentName,
4840  const OMPTargetParallelForDirective &S) {
4841  // Emit SPMD target parallel for region as a standalone region.
4842  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4843  emitTargetParallelForRegion(CGF, S, Action);
4844  };
4845  llvm::Function *Fn;
4846  llvm::Constant *Addr;
4847  // Emit target region as a standalone region.
4848  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4849  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4850  assert(Fn && Addr && "Target device function emission failed.");
4851 }
4852 
4854  const OMPTargetParallelForDirective &S) {
4855  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4856  emitTargetParallelForRegion(CGF, S, Action);
4857  };
4858  emitCommonOMPTargetDirective(*this, S, CodeGen);
4859 }
4860 
4861 static void
4864  PrePostActionTy &Action) {
4865  Action.Enter(CGF);
4866  // Emit directive as a combined directive that consists of two implicit
4867  // directives: 'parallel' with 'for' directive.
4868  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4869  Action.Enter(CGF);
4872  };
4873  emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
4875 }
4876 
4878  CodeGenModule &CGM, StringRef ParentName,
4880  // Emit SPMD target parallel for region as a standalone region.
4881  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4882  emitTargetParallelForSimdRegion(CGF, S, Action);
4883  };
4884  llvm::Function *Fn;
4885  llvm::Constant *Addr;
4886  // Emit target region as a standalone region.
4887  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4888  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4889  assert(Fn && Addr && "Target device function emission failed.");
4890 }
4891 
4894  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4895  emitTargetParallelForSimdRegion(CGF, S, Action);
4896  };
4897  emitCommonOMPTargetDirective(*this, S, CodeGen);
4898 }
4899 
4900 /// Emit a helper variable and return corresponding lvalue.
4901 static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
4902  const ImplicitParamDecl *PVD,
4904  const auto *VDecl = cast<VarDecl>(Helper->getDecl());
4905  Privates.addPrivate(VDecl,
4906  [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); });
4907 }
4908 
4911  // Emit outlined function for task construct.
4912  const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);
4913  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
4914  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
4915  const Expr *IfCond = nullptr;
4916  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4917  if (C->getNameModifier() == OMPD_unknown ||
4918  C->getNameModifier() == OMPD_taskloop) {
4919  IfCond = C->getCondition();
4920  break;
4921  }
4922  }
4923 
4924  OMPTaskDataTy Data;
4925  // Check if taskloop must be emitted without taskgroup.
4927  // TODO: Check if we should emit tied or untied task.
4928  Data.Tied = true;
4929  // Set scheduling for taskloop
4930  if (const auto* Clause = S.getSingleClause<OMPGrainsizeClause>()) {
4931  // grainsize clause
4932  Data.Schedule.setInt(/*IntVal=*/false);
4933  Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
4934  } else if (const auto* Clause = S.getSingleClause<OMPNumTasksClause>()) {
4935  // num_tasks clause
4936  Data.Schedule.setInt(/*IntVal=*/true);
4937  Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
4938  }
4939 
4940  auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
4941  // if (PreCond) {
4942  // for (IV in 0..LastIteration) BODY;
4943  // <Final counter/linear vars updates>;
4944  // }
4945  //
4946 
4947  // Emit: if (PreCond) - begin.
4948  // If the condition constant folds and can be elided, avoid emitting the
4949  // whole loop.
4950  bool CondConstant;
4951  llvm::BasicBlock *ContBlock = nullptr;
4952  OMPLoopScope PreInitScope(CGF, S);
4953  if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
4954  if (!CondConstant)
4955  return;
4956  } else {
4957  llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
4958  ContBlock = CGF.createBasicBlock("taskloop.if.end");
4959  emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
4960  CGF.getProfileCount(&S));
4961  CGF.EmitBlock(ThenBlock);
4962  CGF.incrementProfileCounter(&S);
4963  }
4964 
4966  CGF.EmitOMPSimdInit(S);
4967 
4968  OMPPrivateScope LoopScope(CGF);
4969  // Emit helper vars inits.
4970  enum { LowerBound = 5, UpperBound, Stride, LastIter };
4971  auto *I = CS->getCapturedDecl()->param_begin();
4972  auto *LBP = std::next(I, LowerBound);
4973  auto *UBP = std::next(I, UpperBound);
4974  auto *STP = std::next(I, Stride);
4975  auto *LIP = std::next(I, LastIter);
4976  mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
4977  LoopScope);
4978  mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
4979  LoopScope);
4980  mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
4981  mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
4982  LoopScope);
4983  CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
4984  bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4985  (void)LoopScope.Privatize();
4986  // Emit the loop iteration variable.
4987  const Expr *IVExpr = S.getIterationVariable();
4988  const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
4989  CGF.EmitVarDecl(*IVDecl);
4990  CGF.EmitIgnoredExpr(S.getInit());
4991 
4992  // Emit the iterations count variable.
4993  // If it is not a variable, Sema decided to calculate iterations count on
4994  // each iteration (e.g., it is foldable into a constant).
4995  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
4996  CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
4997  // Emit calculation of the iterations count.
4998  CGF.EmitIgnoredExpr(S.getCalcLastIteration());
4999  }
5000 
5001  CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
5002  S.getInc(),
5003  [&S](CodeGenFunction &CGF) {
5004  CGF.EmitOMPLoopBody(S, JumpDest());
5005  CGF.EmitStopPoint(&S);
5006  },
5007  [](CodeGenFunction &) {});
5008  // Emit: if (PreCond) - end.
5009  if (ContBlock) {
5010  CGF.EmitBranch(ContBlock);
5011  CGF.EmitBlock(ContBlock, true);
5012  }
5013  // Emit final copy of the lastprivate variables if IsLastIter != 0.
5014  if (HasLastprivateClause) {
5015  CGF.EmitOMPLastprivateClauseFinal(
5017  CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
5018  CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
5019  (*LIP)->getType(), S.getBeginLoc())));
5020  }
5021  };
5022  auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
5023  IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
5024  const OMPTaskDataTy &Data) {
5025  auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
5026  &Data](CodeGenFunction &CGF, PrePostActionTy &) {
5027  OMPLoopScope PreInitScope(CGF, S);
5028  CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
5029  OutlinedFn, SharedsTy,
5030  CapturedStruct, IfCond, Data);
5031  };
5032  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
5033  CodeGen);
5034  };
5035  if (Data.Nogroup) {
5036  EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data);
5037  } else {
5038  CGM.getOpenMPRuntime().emitTaskgroupRegion(
5039  *this,
5040  [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
5041  PrePostActionTy &Action) {
5042  Action.Enter(CGF);
5043  CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen,
5044  Data);
5045  },
5046  S.getBeginLoc());
5047  }
5048 }
5049 
5051  EmitOMPTaskLoopBasedDirective(S);
5052 }
5053 
5055  const OMPTaskLoopSimdDirective &S) {
5056  EmitOMPTaskLoopBasedDirective(S);
5057 }
5058 
5059 // Generate the instructions for '#pragma omp target update' directive.
5061  const OMPTargetUpdateDirective &S) {
5062  // If we don't have target devices, don't bother emitting the data mapping
5063  // code.
5064  if (CGM.getLangOpts().OMPTargetTriples.empty())
5065  return;
5066 
5067  // Check if we have any if clause associated with the directive.
5068  const Expr *IfCond = nullptr;
5069  if (const auto *C = S.getSingleClause<OMPIfClause>())
5070  IfCond = C->getCondition();
5071 
5072  // Check if we have any device clause associated with the directive.
5073  const Expr *Device = nullptr;
5074  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
5075  Device = C->getDevice();
5076 
5077  OMPLexicalScope Scope(*this, S, OMPD_task);
5078  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
5079 }
5080 
5082  const OMPExecutableDirective &D) {
5083  if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
5084  return;
5085  auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {
5087  emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action);
5088  } else {
5089  OMPPrivateScope LoopGlobals(CGF);
5090  if (const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
5091  for (const Expr *E : LD->counters()) {
5092  const auto *VD = dyn_cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5093  if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) {
5094  LValue GlobLVal = CGF.EmitLValue(E);
5095  LoopGlobals.addPrivate(
5096  VD, [&GlobLVal]() { return GlobLVal.getAddress(); });
5097  }
5098  if (isa<OMPCapturedExprDecl>(VD)) {
5099  // Emit only those that were not explicitly referenced in clauses.
5100  if (!CGF.LocalDeclMap.count(VD))
5101  CGF.EmitVarDecl(*VD);
5102  }
5103  }
5104  for (const auto *C : D.getClausesOfKind<OMPOrderedClause>()) {
5105  if (!C->getNumForLoops())
5106  continue;
5107  for (unsigned I = LD->getCollapsedNumber(),
5108  E = C->getLoopNumIterations().size();
5109  I < E; ++I) {
5110  if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
5111  cast<DeclRefExpr>(C->getLoopCounter(I))->getDecl())) {
5112  // Emit only those that were not explicitly referenced in clauses.
5113  if (!CGF.LocalDeclMap.count(VD))
5114  CGF.EmitVarDecl(*VD);
5115  }
5116  }
5117  }
5118  }
5119  LoopGlobals.Privatize();
5121  }
5122  };
5123  OMPSimdLexicalScope Scope(*this, D);
5124  CGM.getOpenMPRuntime().emitInlinedDirective(
5125  *this,
5126  isOpenMPSimdDirective(D.getDirectiveKind()) ? OMPD_simd
5127  : D.getDirectiveKind(),
5128  CodeGen);
5129 }
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
Definition: CGCall.cpp:652
bool isAggregate() const
Definition: CGValue.h:53
This represents &#39;#pragma omp distribute simd&#39; composite directive.
Definition: StmtOpenMP.h:3393
Expr * getNextUpperBound() const
Definition: StmtOpenMP.h:992
This represents &#39;#pragma omp master&#39; directive.
Definition: StmtOpenMP.h:1576
This represents &#39;#pragma omp task&#39; directive.
Definition: StmtOpenMP.h:1916
static const Decl * getCanonicalDecl(const Decl *D)
Represents a function declaration or definition.
Definition: Decl.h:1748
This represents &#39;thread_limit&#39; clause in the &#39;#pragma omp ...&#39; directive.
static FunctionDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation NLoc, DeclarationName N, QualType T, TypeSourceInfo *TInfo, StorageClass SC, bool isInlineSpecified=false, bool hasWrittenPrototype=true, ConstexprSpecKind ConstexprKind=CSK_unspecified)
Definition: Decl.h:1895
Expr * getUpperBoundVariable() const
Definition: StmtOpenMP.h:960
Other implicit parameter.
Definition: Decl.h:1524
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
LValue getReferenceLValue(CodeGenFunction &CGF, Expr *refExpr) const
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, SourceLocation Loc)
This represents clause &#39;copyin&#39; in the &#39;#pragma omp ...&#39; directives.
Complete object ctor.
Definition: ABI.h:25
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition: Type.h:2560
Scheduling data for loop-based OpenMP directives.
Definition: OpenMPKinds.h:165
A (possibly-)qualified type.
Definition: Type.h:643
capture_init_iterator capture_init_begin()
Retrieve the first initialization argument.
Definition: Stmt.h:3524
bool isArrayType() const
Definition: Type.h:6413
ArrayRef< OMPClause * > clauses()
Definition: StmtOpenMP.h:324
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *V, SourceLocation Loc)
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
Definition: CGDecl.cpp:165
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
Definition: Decl.h:4164
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Definition: CGExpr.cpp:139
ArrayRef< Expr * > dependent_counters()
Definition: StmtOpenMP.h:1128
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
Specialize PointerLikeTypeTraits to allow LazyGenerationalUpdatePtr to be placed into a PointerUnion...
Definition: Dominators.h:30
static std::pair< LValue, LValue > emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static OMPClauseWithPreInit * get(OMPClause *C)
Stmt - This represents one statement.
Definition: Stmt.h:66
This represents clause &#39;in_reduction&#39; in the &#39;#pragma omp task&#39; directives.
Expr * getLowerBoundVariable() const
Definition: StmtOpenMP.h:952
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
Store of global named registers are always calls to intrinsics.
Definition: CGExpr.cpp:2160
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
Definition: Type.cpp:505
bool requiresCleanups() const
Determine whether this scope requires any cleanups.
void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S)
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
SmallVector< std::pair< OpenMPDependClauseKind, const Expr * >, 4 > Dependences
This represents &#39;#pragma omp for simd&#39; directive.
Definition: StmtOpenMP.h:1326
SmallVector< const Expr *, 4 > LastprivateCopies
QualType getLValueReferenceType(QualType T, bool SpelledAsLValue=true) const
Return the uniqued reference to the type for an lvalue reference to the specified type...
Decl - This represents one declaration (or definition), e.g.
Definition: DeclBase.h:88
This represents &#39;grainsize&#39; clause in the &#39;#pragma omp ...&#39; directive.
This represents &#39;#pragma omp teams distribute parallel for&#39; composite directive.
Definition: StmtOpenMP.h:3804
SourceLocation getBeginLoc() const
Returns starting location of directive kind.
Definition: StmtOpenMP.h:224
bool isNothrow() const
Definition: Decl.cpp:4621
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
This represents &#39;if&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:422
CapturedStmt * getInnermostCapturedStmt()
Get innermost captured statement for the construct.
Definition: StmtOpenMP.h:282
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Definition: Type.h:6306
void EmitOMPOrderedDirective(const OMPOrderedDirective &S)
bool isVolatile() const
Definition: CGValue.h:300
This represents &#39;priority&#39; clause in the &#39;#pragma omp ...&#39; directive.
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
The base class of the type hierarchy.
Definition: Type.h:1421
This represents &#39;#pragma omp target teams distribute&#39; combined directive.
Definition: StmtOpenMP.h:3941