clang  9.0.0svn
CGStmtOpenMP.cpp
Go to the documentation of this file.
1 //===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This contains code to emit OpenMP nodes as LLVM code.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CGCleanup.h"
14 #include "CGOpenMPRuntime.h"
15 #include "CodeGenFunction.h"
16 #include "CodeGenModule.h"
17 #include "TargetInfo.h"
18 #include "clang/AST/Stmt.h"
19 #include "clang/AST/StmtOpenMP.h"
20 #include "clang/AST/DeclOpenMP.h"
21 using namespace clang;
22 using namespace CodeGen;
23 
24 namespace {
25 /// Lexical scope for OpenMP executable constructs, that handles correct codegen
26 /// for captured expressions.
27 class OMPLexicalScope : public CodeGenFunction::LexicalScope {
28  void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
29  for (const auto *C : S.clauses()) {
30  if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
31  if (const auto *PreInit =
32  cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
33  for (const auto *I : PreInit->decls()) {
34  if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
35  CGF.EmitVarDecl(cast<VarDecl>(*I));
36  } else {
38  CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
39  CGF.EmitAutoVarCleanups(Emission);
40  }
41  }
42  }
43  }
44  }
45  }
46  CodeGenFunction::OMPPrivateScope InlinedShareds;
47 
48  static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
49  return CGF.LambdaCaptureFields.lookup(VD) ||
50  (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
51  (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl));
52  }
53 
54 public:
55  OMPLexicalScope(
57  const llvm::Optional<OpenMPDirectiveKind> CapturedRegion = llvm::None,
58  const bool EmitPreInitStmt = true)
59  : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
60  InlinedShareds(CGF) {
61  if (EmitPreInitStmt)
62  emitPreInitStmt(CGF, S);
63  if (!CapturedRegion.hasValue())
64  return;
65  assert(S.hasAssociatedStmt() &&
66  "Expected associated statement for inlined directive.");
67  const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
68  for (const auto &C : CS->captures()) {
69  if (C.capturesVariable() || C.capturesVariableByCopy()) {
70  auto *VD = C.getCapturedVar();
71  assert(VD == VD->getCanonicalDecl() &&
72  "Canonical decl must be captured.");
73  DeclRefExpr DRE(
74  CGF.getContext(), const_cast<VarDecl *>(VD),
75  isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
76  InlinedShareds.isGlobalVarCaptured(VD)),
77  VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
78  InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
79  return CGF.EmitLValue(&DRE).getAddress();
80  });
81  }
82  }
83  (void)InlinedShareds.Privatize();
84  }
85 };
86 
87 /// Lexical scope for OpenMP parallel construct, that handles correct codegen
88 /// for captured expressions.
89 class OMPParallelScope final : public OMPLexicalScope {
90  bool EmitPreInitStmt(const OMPExecutableDirective &S) {
92  return !(isOpenMPTargetExecutionDirective(Kind) ||
95  }
96 
97 public:
98  OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
99  : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
100  EmitPreInitStmt(S)) {}
101 };
102 
103 /// Lexical scope for OpenMP teams construct, that handles correct codegen
104 /// for captured expressions.
105 class OMPTeamsScope final : public OMPLexicalScope {
106  bool EmitPreInitStmt(const OMPExecutableDirective &S) {
108  return !isOpenMPTargetExecutionDirective(Kind) &&
110  }
111 
112 public:
113  OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
114  : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
115  EmitPreInitStmt(S)) {}
116 };
117 
118 /// Private scope for OpenMP loop-based directives, that supports capturing
119 /// of used expression from loop statement.
120 class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
121  void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) {
122  CodeGenFunction::OMPMapVars PreCondVars;
123  for (const auto *E : S.counters()) {
124  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
125  (void)PreCondVars.setVarAddr(
126  CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
127  }
128  (void)PreCondVars.apply(CGF);
129  if (const auto *PreInits = cast_or_null<DeclStmt>(S.getPreInits())) {
130  for (const auto *I : PreInits->decls())
131  CGF.EmitVarDecl(cast<VarDecl>(*I));
132  }
133  PreCondVars.restore(CGF);
134  }
135 
136 public:
137  OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S)
138  : CodeGenFunction::RunCleanupsScope(CGF) {
139  emitPreInitStmt(CGF, S);
140  }
141 };
142 
143 class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
144  CodeGenFunction::OMPPrivateScope InlinedShareds;
145 
146  static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
147  return CGF.LambdaCaptureFields.lookup(VD) ||
148  (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
149  (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) &&
150  cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
151  }
152 
153 public:
154  OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
155  : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
156  InlinedShareds(CGF) {
157  for (const auto *C : S.clauses()) {
158  if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
159  if (const auto *PreInit =
160  cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
161  for (const auto *I : PreInit->decls()) {
162  if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
163  CGF.EmitVarDecl(cast<VarDecl>(*I));
164  } else {
166  CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
167  CGF.EmitAutoVarCleanups(Emission);
168  }
169  }
170  }
171  } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
172  for (const Expr *E : UDP->varlists()) {
173  const Decl *D = cast<DeclRefExpr>(E)->getDecl();
174  if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
175  CGF.EmitVarDecl(*OED);
176  }
177  }
178  }
180  CGF.EmitOMPPrivateClause(S, InlinedShareds);
181  if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
182  if (const Expr *E = TG->getReductionRef())
183  CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
184  }
185  const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
186  while (CS) {
187  for (auto &C : CS->captures()) {
188  if (C.capturesVariable() || C.capturesVariableByCopy()) {
189  auto *VD = C.getCapturedVar();
190  assert(VD == VD->getCanonicalDecl() &&
191  "Canonical decl must be captured.");
192  DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
193  isCapturedVar(CGF, VD) ||
194  (CGF.CapturedStmtInfo &&
195  InlinedShareds.isGlobalVarCaptured(VD)),
197  C.getLocation());
198  InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
199  return CGF.EmitLValue(&DRE).getAddress();
200  });
201  }
202  }
203  CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
204  }
205  (void)InlinedShareds.Privatize();
206  }
207 };
208 
209 } // namespace
210 
212  const OMPExecutableDirective &S,
213  const RegionCodeGenTy &CodeGen);
214 
216  if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
217  if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
218  OrigVD = OrigVD->getCanonicalDecl();
219  bool IsCaptured =
220  LambdaCaptureFields.lookup(OrigVD) ||
221  (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
222  (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
223  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), IsCaptured,
224  OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
225  return EmitLValue(&DRE);
226  }
227  }
228  return EmitLValue(E);
229 }
230 
232  ASTContext &C = getContext();
233  llvm::Value *Size = nullptr;
234  auto SizeInChars = C.getTypeSizeInChars(Ty);
235  if (SizeInChars.isZero()) {
236  // getTypeSizeInChars() returns 0 for a VLA.
237  while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) {
238  VlaSizePair VlaSize = getVLASize(VAT);
239  Ty = VlaSize.Type;
240  Size = Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts)
241  : VlaSize.NumElts;
242  }
243  SizeInChars = C.getTypeSizeInChars(Ty);
244  if (SizeInChars.isZero())
245  return llvm::ConstantInt::get(SizeTy, /*V=*/0);
246  return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
247  }
248  return CGM.getSize(SizeInChars);
249 }
250 
252  const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
253  const RecordDecl *RD = S.getCapturedRecordDecl();
254  auto CurField = RD->field_begin();
255  auto CurCap = S.captures().begin();
257  E = S.capture_init_end();
258  I != E; ++I, ++CurField, ++CurCap) {
259  if (CurField->hasCapturedVLAType()) {
260  const VariableArrayType *VAT = CurField->getCapturedVLAType();
261  llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
262  CapturedVars.push_back(Val);
263  } else if (CurCap->capturesThis()) {
264  CapturedVars.push_back(CXXThisValue);
265  } else if (CurCap->capturesVariableByCopy()) {
266  llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
267 
268  // If the field is not a pointer, we need to save the actual value
269  // and load it as a void pointer.
270  if (!CurField->getType()->isAnyPointerType()) {
271  ASTContext &Ctx = getContext();
272  Address DstAddr = CreateMemTemp(
273  Ctx.getUIntPtrType(),
274  Twine(CurCap->getCapturedVar()->getName(), ".casted"));
275  LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
276 
277  llvm::Value *SrcAddrVal = EmitScalarConversion(
278  DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
279  Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
280  LValue SrcLV =
281  MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
282 
283  // Store the value using the source type pointer.
284  EmitStoreThroughLValue(RValue::get(CV), SrcLV);
285 
286  // Load the value using the destination type pointer.
287  CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
288  }
289  CapturedVars.push_back(CV);
290  } else {
291  assert(CurCap->capturesVariable() && "Expected capture by reference.");
292  CapturedVars.push_back(EmitLValue(*I).getAddress().getPointer());
293  }
294  }
295 }
296 
298  QualType DstType, StringRef Name,
299  LValue AddrLV,
300  bool isReferenceType = false) {
301  ASTContext &Ctx = CGF.getContext();
302 
303  llvm::Value *CastedPtr = CGF.EmitScalarConversion(
304  AddrLV.getAddress().getPointer(), Ctx.getUIntPtrType(),
305  Ctx.getPointerType(DstType), Loc);
306  Address TmpAddr =
307  CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType))
308  .getAddress();
309 
310  // If we are dealing with references we need to return the address of the
311  // reference instead of the reference of the value.
312  if (isReferenceType) {
313  QualType RefType = Ctx.getLValueReferenceType(DstType);
314  llvm::Value *RefVal = TmpAddr.getPointer();
315  TmpAddr = CGF.CreateMemTemp(RefType, Twine(Name, ".ref"));
316  LValue TmpLVal = CGF.MakeAddrLValue(TmpAddr, RefType);
317  CGF.EmitStoreThroughLValue(RValue::get(RefVal), TmpLVal, /*isInit=*/true);
318  }
319 
320  return TmpAddr;
321 }
322 
324  if (T->isLValueReferenceType())
325  return C.getLValueReferenceType(
327  /*SpelledAsLValue=*/false);
328  if (T->isPointerType())
330  if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
331  if (const auto *VLA = dyn_cast<VariableArrayType>(A))
332  return getCanonicalParamType(C, VLA->getElementType());
333  if (!A->isVariablyModifiedType())
334  return C.getCanonicalType(T);
335  }
336  return C.getCanonicalParamType(T);
337 }
338 
339 namespace {
340  /// Contains required data for proper outlined function codegen.
341  struct FunctionOptions {
342  /// Captured statement for which the function is generated.
343  const CapturedStmt *S = nullptr;
344  /// true if cast to/from UIntPtr is required for variables captured by
345  /// value.
346  const bool UIntPtrCastRequired = true;
347  /// true if only casted arguments must be registered as local args or VLA
348  /// sizes.
349  const bool RegisterCastedArgsOnly = false;
350  /// Name of the generated function.
351  const StringRef FunctionName;
352  explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
353  bool RegisterCastedArgsOnly,
354  StringRef FunctionName)
355  : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
356  RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
357  FunctionName(FunctionName) {}
358  };
359 }
360 
361 static llvm::Function *emitOutlinedFunctionPrologue(
362  CodeGenFunction &CGF, FunctionArgList &Args,
363  llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
364  &LocalAddrs,
365  llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
366  &VLASizes,
367  llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
368  const CapturedDecl *CD = FO.S->getCapturedDecl();
369  const RecordDecl *RD = FO.S->getCapturedRecordDecl();
370  assert(CD->hasBody() && "missing CapturedDecl body");
371 
372  CXXThisValue = nullptr;
373  // Build the argument list.
374  CodeGenModule &CGM = CGF.CGM;
375  ASTContext &Ctx = CGM.getContext();
376  FunctionArgList TargetArgs;
377  Args.append(CD->param_begin(),
378  std::next(CD->param_begin(), CD->getContextParamPosition()));
379  TargetArgs.append(
380  CD->param_begin(),
381  std::next(CD->param_begin(), CD->getContextParamPosition()));
382  auto I = FO.S->captures().begin();
383  FunctionDecl *DebugFunctionDecl = nullptr;
384  if (!FO.UIntPtrCastRequired) {
386  QualType FunctionTy = Ctx.getFunctionType(Ctx.VoidTy, llvm::None, EPI);
387  DebugFunctionDecl = FunctionDecl::Create(
388  Ctx, Ctx.getTranslationUnitDecl(), FO.S->getBeginLoc(),
389  SourceLocation(), DeclarationName(), FunctionTy,
390  Ctx.getTrivialTypeSourceInfo(FunctionTy), SC_Static,
391  /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/false);
392  }
393  for (const FieldDecl *FD : RD->fields()) {
394  QualType ArgType = FD->getType();
395  IdentifierInfo *II = nullptr;
396  VarDecl *CapVar = nullptr;
397 
398  // If this is a capture by copy and the type is not a pointer, the outlined
399  // function argument type should be uintptr and the value properly casted to
400  // uintptr. This is necessary given that the runtime library is only able to
401  // deal with pointers. We can pass in the same way the VLA type sizes to the
402  // outlined function.
403  if (FO.UIntPtrCastRequired &&
404  ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
405  I->capturesVariableArrayType()))
406  ArgType = Ctx.getUIntPtrType();
407 
408  if (I->capturesVariable() || I->capturesVariableByCopy()) {
409  CapVar = I->getCapturedVar();
410  II = CapVar->getIdentifier();
411  } else if (I->capturesThis()) {
412  II = &Ctx.Idents.get("this");
413  } else {
414  assert(I->capturesVariableArrayType());
415  II = &Ctx.Idents.get("vla");
416  }
417  if (ArgType->isVariablyModifiedType())
418  ArgType = getCanonicalParamType(Ctx, ArgType);
419  VarDecl *Arg;
420  if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
421  Arg = ParmVarDecl::Create(
422  Ctx, DebugFunctionDecl,
423  CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),
424  CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
425  /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
426  } else {
427  Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
428  II, ArgType, ImplicitParamDecl::Other);
429  }
430  Args.emplace_back(Arg);
431  // Do not cast arguments if we emit function with non-original types.
432  TargetArgs.emplace_back(
433  FO.UIntPtrCastRequired
434  ? Arg
435  : CGM.getOpenMPRuntime().translateParameter(FD, Arg));
436  ++I;
437  }
438  Args.append(
439  std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
440  CD->param_end());
441  TargetArgs.append(
442  std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
443  CD->param_end());
444 
445  // Create the function declaration.
446  const CGFunctionInfo &FuncInfo =
447  CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, TargetArgs);
448  llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
449 
450  auto *F =
452  FO.FunctionName, &CGM.getModule());
453  CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
454  if (CD->isNothrow())
455  F->setDoesNotThrow();
456  F->setDoesNotRecurse();
457 
458  // Generate the function.
459  CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
460  FO.S->getBeginLoc(), CD->getBody()->getBeginLoc());
461  unsigned Cnt = CD->getContextParamPosition();
462  I = FO.S->captures().begin();
463  for (const FieldDecl *FD : RD->fields()) {
464  // Do not map arguments if we emit function with non-original types.
465  Address LocalAddr(Address::invalid());
466  if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
467  LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
468  TargetArgs[Cnt]);
469  } else {
470  LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
471  }
472  // If we are capturing a pointer by copy we don't need to do anything, just
473  // use the value that we get from the arguments.
474  if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
475  const VarDecl *CurVD = I->getCapturedVar();
476  // If the variable is a reference we need to materialize it here.
477  if (CurVD->getType()->isReferenceType()) {
478  Address RefAddr = CGF.CreateMemTemp(
479  CurVD->getType(), CGM.getPointerAlign(), ".materialized_ref");
480  CGF.EmitStoreOfScalar(LocalAddr.getPointer(), RefAddr,
481  /*Volatile=*/false, CurVD->getType());
482  LocalAddr = RefAddr;
483  }
484  if (!FO.RegisterCastedArgsOnly)
485  LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
486  ++Cnt;
487  ++I;
488  continue;
489  }
490 
491  LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
493  if (FD->hasCapturedVLAType()) {
494  if (FO.UIntPtrCastRequired) {
495  ArgLVal = CGF.MakeAddrLValue(
496  castValueFromUintptr(CGF, I->getLocation(), FD->getType(),
497  Args[Cnt]->getName(), ArgLVal),
499  }
500  llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
501  const VariableArrayType *VAT = FD->getCapturedVLAType();
502  VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg);
503  } else if (I->capturesVariable()) {
504  const VarDecl *Var = I->getCapturedVar();
505  QualType VarTy = Var->getType();
506  Address ArgAddr = ArgLVal.getAddress();
507  if (!VarTy->isReferenceType()) {
508  if (ArgLVal.getType()->isLValueReferenceType()) {
509  ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
510  } else if (!VarTy->isVariablyModifiedType() ||
511  !VarTy->isPointerType()) {
512  assert(ArgLVal.getType()->isPointerType());
513  ArgAddr = CGF.EmitLoadOfPointer(
514  ArgAddr, ArgLVal.getType()->castAs<PointerType>());
515  }
516  }
517  if (!FO.RegisterCastedArgsOnly) {
518  LocalAddrs.insert(
519  {Args[Cnt],
520  {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}});
521  }
522  } else if (I->capturesVariableByCopy()) {
523  assert(!FD->getType()->isAnyPointerType() &&
524  "Not expecting a captured pointer.");
525  const VarDecl *Var = I->getCapturedVar();
526  QualType VarTy = Var->getType();
527  LocalAddrs.insert(
528  {Args[Cnt],
529  {Var, FO.UIntPtrCastRequired
530  ? castValueFromUintptr(CGF, I->getLocation(),
531  FD->getType(), Args[Cnt]->getName(),
532  ArgLVal, VarTy->isReferenceType())
533  : ArgLVal.getAddress()}});
534  } else {
535  // If 'this' is captured, load it into CXXThisValue.
536  assert(I->capturesThis());
537  CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
538  LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress()}});
539  }
540  ++Cnt;
541  ++I;
542  }
543 
544  return F;
545 }
546 
547 llvm::Function *
549  assert(
550  CapturedStmtInfo &&
551  "CapturedStmtInfo should be set when generating the captured function");
552  const CapturedDecl *CD = S.getCapturedDecl();
553  // Build the argument list.
554  bool NeedWrapperFunction =
555  getDebugInfo() &&
556  CGM.getCodeGenOpts().getDebugInfo() >= codegenoptions::LimitedDebugInfo;
557  FunctionArgList Args;
558  llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
559  llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
560  SmallString<256> Buffer;
561  llvm::raw_svector_ostream Out(Buffer);
562  Out << CapturedStmtInfo->getHelperName();
563  if (NeedWrapperFunction)
564  Out << "_debug__";
565  FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
566  Out.str());
567  llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs,
568  VLASizes, CXXThisValue, FO);
569  for (const auto &LocalAddrPair : LocalAddrs) {
570  if (LocalAddrPair.second.first) {
571  setAddrOfLocalVar(LocalAddrPair.second.first,
572  LocalAddrPair.second.second);
573  }
574  }
575  for (const auto &VLASizePair : VLASizes)
576  VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
577  PGO.assignRegionCounters(GlobalDecl(CD), F);
578  CapturedStmtInfo->EmitBody(*this, CD->getBody());
579  FinishFunction(CD->getBodyRBrace());
580  if (!NeedWrapperFunction)
581  return F;
582 
583  FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
584  /*RegisterCastedArgsOnly=*/true,
585  CapturedStmtInfo->getHelperName());
586  CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
587  WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
588  Args.clear();
589  LocalAddrs.clear();
590  VLASizes.clear();
591  llvm::Function *WrapperF =
592  emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
593  WrapperCGF.CXXThisValue, WrapperFO);
595  for (const auto *Arg : Args) {
597  auto I = LocalAddrs.find(Arg);
598  if (I != LocalAddrs.end()) {
599  LValue LV = WrapperCGF.MakeAddrLValue(
600  I->second.second,
601  I->second.first ? I->second.first->getType() : Arg->getType(),
603  CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
604  } else {
605  auto EI = VLASizes.find(Arg);
606  if (EI != VLASizes.end()) {
607  CallArg = EI->second.second;
608  } else {
609  LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
610  Arg->getType(),
612  CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
613  }
614  }
615  CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
616  }
617  CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.getBeginLoc(),
618  F, CallArgs);
619  WrapperCGF.FinishFunction();
620  return WrapperF;
621 }
622 
623 //===----------------------------------------------------------------------===//
624 // OpenMP Directive Emission
625 //===----------------------------------------------------------------------===//
627  Address DestAddr, Address SrcAddr, QualType OriginalType,
628  const llvm::function_ref<void(Address, Address)> CopyGen) {
629  // Perform element-by-element initialization.
630  QualType ElementTy;
631 
632  // Drill down to the base element type on both arrays.
633  const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
634  llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
635  SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
636 
637  llvm::Value *SrcBegin = SrcAddr.getPointer();
638  llvm::Value *DestBegin = DestAddr.getPointer();
639  // Cast from pointer to array type to pointer to single element.
640  llvm::Value *DestEnd = Builder.CreateGEP(DestBegin, NumElements);
641  // The basic structure here is a while-do loop.
642  llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body");
643  llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done");
644  llvm::Value *IsEmpty =
645  Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
646  Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
647 
648  // Enter the loop body, making that address the current address.
649  llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
650  EmitBlock(BodyBB);
651 
652  CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
653 
654  llvm::PHINode *SrcElementPHI =
655  Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
656  SrcElementPHI->addIncoming(SrcBegin, EntryBB);
657  Address SrcElementCurrent =
658  Address(SrcElementPHI,
659  SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
660 
661  llvm::PHINode *DestElementPHI =
662  Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
663  DestElementPHI->addIncoming(DestBegin, EntryBB);
664  Address DestElementCurrent =
665  Address(DestElementPHI,
666  DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
667 
668  // Emit copy.
669  CopyGen(DestElementCurrent, SrcElementCurrent);
670 
671  // Shift the address forward by one element.
672  llvm::Value *DestElementNext = Builder.CreateConstGEP1_32(
673  DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
674  llvm::Value *SrcElementNext = Builder.CreateConstGEP1_32(
675  SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
676  // Check whether we've reached the end.
677  llvm::Value *Done =
678  Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
679  Builder.CreateCondBr(Done, DoneBB, BodyBB);
680  DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
681  SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
682 
683  // Done.
684  EmitBlock(DoneBB, /*IsFinished=*/true);
685 }
686 
687 void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
688  Address SrcAddr, const VarDecl *DestVD,
689  const VarDecl *SrcVD, const Expr *Copy) {
690  if (OriginalType->isArrayType()) {
691  const auto *BO = dyn_cast<BinaryOperator>(Copy);
692  if (BO && BO->getOpcode() == BO_Assign) {
693  // Perform simple memcpy for simple copying.
694  LValue Dest = MakeAddrLValue(DestAddr, OriginalType);
695  LValue Src = MakeAddrLValue(SrcAddr, OriginalType);
696  EmitAggregateAssign(Dest, Src, OriginalType);
697  } else {
698  // For arrays with complex element types perform element by element
699  // copying.
700  EmitOMPAggregateAssign(
701  DestAddr, SrcAddr, OriginalType,
702  [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
703  // Working with the single array element, so have to remap
704  // destination and source variables to corresponding array
705  // elements.
707  Remap.addPrivate(DestVD, [DestElement]() { return DestElement; });
708  Remap.addPrivate(SrcVD, [SrcElement]() { return SrcElement; });
709  (void)Remap.Privatize();
710  EmitIgnoredExpr(Copy);
711  });
712  }
713  } else {
714  // Remap pseudo source variable to private copy.
716  Remap.addPrivate(SrcVD, [SrcAddr]() { return SrcAddr; });
717  Remap.addPrivate(DestVD, [DestAddr]() { return DestAddr; });
718  (void)Remap.Privatize();
719  // Emit copying of the whole variable.
720  EmitIgnoredExpr(Copy);
721  }
722 }
723 
725  OMPPrivateScope &PrivateScope) {
726  if (!HaveInsertPoint())
727  return false;
728  bool FirstprivateIsLastprivate = false;
729  llvm::DenseSet<const VarDecl *> Lastprivates;
730  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
731  for (const auto *D : C->varlists())
732  Lastprivates.insert(
733  cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl());
734  }
735  llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
737  getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind());
738  // Force emission of the firstprivate copy if the directive does not emit
739  // outlined function, like omp for, omp simd, omp distribute etc.
740  bool MustEmitFirstprivateCopy =
741  CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
742  for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
743  auto IRef = C->varlist_begin();
744  auto InitsRef = C->inits().begin();
745  for (const Expr *IInit : C->private_copies()) {
746  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
747  bool ThisFirstprivateIsLastprivate =
748  Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
749  const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
750  if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
751  !FD->getType()->isReferenceType()) {
752  EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
753  ++IRef;
754  ++InitsRef;
755  continue;
756  }
757  FirstprivateIsLastprivate =
758  FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
759  if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
760  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
761  const auto *VDInit =
762  cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
763  bool IsRegistered;
764  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
765  /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
766  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
767  LValue OriginalLVal = EmitLValue(&DRE);
768  QualType Type = VD->getType();
769  if (Type->isArrayType()) {
770  // Emit VarDecl with copy init for arrays.
771  // Get the address of the original variable captured in current
772  // captured region.
773  IsRegistered = PrivateScope.addPrivate(
774  OrigVD, [this, VD, Type, OriginalLVal, VDInit]() {
775  AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
776  const Expr *Init = VD->getInit();
777  if (!isa<CXXConstructExpr>(Init) ||
778  isTrivialInitializer(Init)) {
779  // Perform simple memcpy.
780  LValue Dest =
781  MakeAddrLValue(Emission.getAllocatedAddress(), Type);
782  EmitAggregateAssign(Dest, OriginalLVal, Type);
783  } else {
784  EmitOMPAggregateAssign(
785  Emission.getAllocatedAddress(), OriginalLVal.getAddress(),
786  Type,
787  [this, VDInit, Init](Address DestElement,
788  Address SrcElement) {
789  // Clean up any temporaries needed by the
790  // initialization.
791  RunCleanupsScope InitScope(*this);
792  // Emit initialization for single element.
793  setAddrOfLocalVar(VDInit, SrcElement);
794  EmitAnyExprToMem(Init, DestElement,
795  Init->getType().getQualifiers(),
796  /*IsInitializer*/ false);
797  LocalDeclMap.erase(VDInit);
798  });
799  }
800  EmitAutoVarCleanups(Emission);
801  return Emission.getAllocatedAddress();
802  });
803  } else {
804  Address OriginalAddr = OriginalLVal.getAddress();
805  IsRegistered = PrivateScope.addPrivate(
806  OrigVD, [this, VDInit, OriginalAddr, VD]() {
807  // Emit private VarDecl with copy init.
808  // Remap temp VDInit variable to the address of the original
809  // variable (for proper handling of captured global variables).
810  setAddrOfLocalVar(VDInit, OriginalAddr);
811  EmitDecl(*VD);
812  LocalDeclMap.erase(VDInit);
813  return GetAddrOfLocalVar(VD);
814  });
815  }
816  assert(IsRegistered &&
817  "firstprivate var already registered as private");
818  // Silence the warning about unused variable.
819  (void)IsRegistered;
820  }
821  ++IRef;
822  ++InitsRef;
823  }
824  }
825  return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
826 }
827 
829  const OMPExecutableDirective &D,
830  CodeGenFunction::OMPPrivateScope &PrivateScope) {
831  if (!HaveInsertPoint())
832  return;
833  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
834  for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
835  auto IRef = C->varlist_begin();
836  for (const Expr *IInit : C->private_copies()) {
837  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
838  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
839  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
840  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
841  // Emit private VarDecl with copy init.
842  EmitDecl(*VD);
843  return GetAddrOfLocalVar(VD);
844  });
845  assert(IsRegistered && "private var already registered as private");
846  // Silence the warning about unused variable.
847  (void)IsRegistered;
848  }
849  ++IRef;
850  }
851  }
852 }
853 
855  if (!HaveInsertPoint())
856  return false;
857  // threadprivate_var1 = master_threadprivate_var1;
858  // operator=(threadprivate_var2, master_threadprivate_var2);
859  // ...
860  // __kmpc_barrier(&loc, global_tid);
862  llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
863  for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
864  auto IRef = C->varlist_begin();
865  auto ISrcRef = C->source_exprs().begin();
866  auto IDestRef = C->destination_exprs().begin();
867  for (const Expr *AssignOp : C->assignment_ops()) {
868  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
869  QualType Type = VD->getType();
870  if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
871  // Get the address of the master variable. If we are emitting code with
872  // TLS support, the address is passed from the master as field in the
873  // captured declaration.
874  Address MasterAddr = Address::invalid();
875  if (getLangOpts().OpenMPUseTLS &&
876  getContext().getTargetInfo().isTLSSupported()) {
877  assert(CapturedStmtInfo->lookup(VD) &&
878  "Copyin threadprivates should have been captured!");
879  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), true,
880  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
881  MasterAddr = EmitLValue(&DRE).getAddress();
882  LocalDeclMap.erase(VD);
883  } else {
884  MasterAddr =
885  Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
886  : CGM.GetAddrOfGlobal(VD),
887  getContext().getDeclAlign(VD));
888  }
889  // Get the address of the threadprivate variable.
890  Address PrivateAddr = EmitLValue(*IRef).getAddress();
891  if (CopiedVars.size() == 1) {
892  // At first check if current thread is a master thread. If it is, no
893  // need to copy data.
894  CopyBegin = createBasicBlock("copyin.not.master");
895  CopyEnd = createBasicBlock("copyin.not.master.end");
896  Builder.CreateCondBr(
897  Builder.CreateICmpNE(
898  Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy),
899  Builder.CreatePtrToInt(PrivateAddr.getPointer(),
900  CGM.IntPtrTy)),
901  CopyBegin, CopyEnd);
902  EmitBlock(CopyBegin);
903  }
904  const auto *SrcVD =
905  cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
906  const auto *DestVD =
907  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
908  EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
909  }
910  ++IRef;
911  ++ISrcRef;
912  ++IDestRef;
913  }
914  }
915  if (CopyEnd) {
916  // Exit out of copying procedure for non-master thread.
917  EmitBlock(CopyEnd, /*IsFinished=*/true);
918  return true;
919  }
920  return false;
921 }
922 
924  const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
925  if (!HaveInsertPoint())
926  return false;
927  bool HasAtLeastOneLastprivate = false;
930  const auto *LoopDirective = cast<OMPLoopDirective>(&D);
931  for (const Expr *C : LoopDirective->counters()) {
932  SIMDLCVs.insert(
933  cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
934  }
935  }
936  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
937  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
938  HasAtLeastOneLastprivate = true;
940  !getLangOpts().OpenMPSimd)
941  break;
942  auto IRef = C->varlist_begin();
943  auto IDestRef = C->destination_exprs().begin();
944  for (const Expr *IInit : C->private_copies()) {
945  // Keep the address of the original variable for future update at the end
946  // of the loop.
947  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
948  // Taskloops do not require additional initialization, it is done in
949  // runtime support library.
950  if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
951  const auto *DestVD =
952  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
953  PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() {
954  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
955  /*RefersToEnclosingVariableOrCapture=*/
956  CapturedStmtInfo->lookup(OrigVD) != nullptr,
957  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
958  return EmitLValue(&DRE).getAddress();
959  });
960  // Check if the variable is also a firstprivate: in this case IInit is
961  // not generated. Initialization of this variable will happen in codegen
962  // for 'firstprivate' clause.
963  if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
964  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
965  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
966  // Emit private VarDecl with copy init.
967  EmitDecl(*VD);
968  return GetAddrOfLocalVar(VD);
969  });
970  assert(IsRegistered &&
971  "lastprivate var already registered as private");
972  (void)IsRegistered;
973  }
974  }
975  ++IRef;
976  ++IDestRef;
977  }
978  }
979  return HasAtLeastOneLastprivate;
980 }
981 
983  const OMPExecutableDirective &D, bool NoFinals,
984  llvm::Value *IsLastIterCond) {
985  if (!HaveInsertPoint())
986  return;
987  // Emit following code:
988  // if (<IsLastIterCond>) {
989  // orig_var1 = private_orig_var1;
990  // ...
991  // orig_varn = private_orig_varn;
992  // }
993  llvm::BasicBlock *ThenBB = nullptr;
994  llvm::BasicBlock *DoneBB = nullptr;
995  if (IsLastIterCond) {
996  ThenBB = createBasicBlock(".omp.lastprivate.then");
997  DoneBB = createBasicBlock(".omp.lastprivate.done");
998  Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
999  EmitBlock(ThenBB);
1000  }
1001  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1002  llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1003  if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1004  auto IC = LoopDirective->counters().begin();
1005  for (const Expr *F : LoopDirective->finals()) {
1006  const auto *D =
1007  cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1008  if (NoFinals)
1009  AlreadyEmittedVars.insert(D);
1010  else
1011  LoopCountersAndUpdates[D] = F;
1012  ++IC;
1013  }
1014  }
1015  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1016  auto IRef = C->varlist_begin();
1017  auto ISrcRef = C->source_exprs().begin();
1018  auto IDestRef = C->destination_exprs().begin();
1019  for (const Expr *AssignOp : C->assignment_ops()) {
1020  const auto *PrivateVD =
1021  cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1022  QualType Type = PrivateVD->getType();
1023  const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1024  if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1025  // If lastprivate variable is a loop control variable for loop-based
1026  // directive, update its value before copyin back to original
1027  // variable.
1028  if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1029  EmitIgnoredExpr(FinalExpr);
1030  const auto *SrcVD =
1031  cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1032  const auto *DestVD =
1033  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1034  // Get the address of the original variable.
1035  Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1036  // Get the address of the private variable.
1037  Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
1038  if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1039  PrivateAddr =
1040  Address(Builder.CreateLoad(PrivateAddr),
1041  getNaturalTypeAlignment(RefTy->getPointeeType()));
1042  EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1043  }
1044  ++IRef;
1045  ++ISrcRef;
1046  ++IDestRef;
1047  }
1048  if (const Expr *PostUpdate = C->getPostUpdateExpr())
1049  EmitIgnoredExpr(PostUpdate);
1050  }
1051  if (IsLastIterCond)
1052  EmitBlock(DoneBB, /*IsFinished=*/true);
1053 }
1054 
1056  const OMPExecutableDirective &D,
1057  CodeGenFunction::OMPPrivateScope &PrivateScope) {
1058  if (!HaveInsertPoint())
1059  return;
1062  SmallVector<const Expr *, 4> ReductionOps;
1065  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1066  auto IPriv = C->privates().begin();
1067  auto IRed = C->reduction_ops().begin();
1068  auto ILHS = C->lhs_exprs().begin();
1069  auto IRHS = C->rhs_exprs().begin();
1070  for (const Expr *Ref : C->varlists()) {
1071  Shareds.emplace_back(Ref);
1072  Privates.emplace_back(*IPriv);
1073  ReductionOps.emplace_back(*IRed);
1074  LHSs.emplace_back(*ILHS);
1075  RHSs.emplace_back(*IRHS);
1076  std::advance(IPriv, 1);
1077  std::advance(IRed, 1);
1078  std::advance(ILHS, 1);
1079  std::advance(IRHS, 1);
1080  }
1081  }
1082  ReductionCodeGen RedCG(Shareds, Privates, ReductionOps);
1083  unsigned Count = 0;
1084  auto ILHS = LHSs.begin();
1085  auto IRHS = RHSs.begin();
1086  auto IPriv = Privates.begin();
1087  for (const Expr *IRef : Shareds) {
1088  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1089  // Emit private VarDecl with reduction init.
1090  RedCG.emitSharedLValue(*this, Count);
1091  RedCG.emitAggregateType(*this, Count);
1092  AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);
1093  RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
1094  RedCG.getSharedLValue(Count),
1095  [&Emission](CodeGenFunction &CGF) {
1096  CGF.EmitAutoVarInit(Emission);
1097  return true;
1098  });
1099  EmitAutoVarCleanups(Emission);
1100  Address BaseAddr = RedCG.adjustPrivateAddress(
1101  *this, Count, Emission.getAllocatedAddress());
1102  bool IsRegistered = PrivateScope.addPrivate(
1103  RedCG.getBaseDecl(Count), [BaseAddr]() { return BaseAddr; });
1104  assert(IsRegistered && "private var already registered as private");
1105  // Silence the warning about unused variable.
1106  (void)IsRegistered;
1107 
1108  const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1109  const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1110  QualType Type = PrivateVD->getType();
1111  bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1112  if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1113  // Store the address of the original variable associated with the LHS
1114  // implicit variable.
1115  PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
1116  return RedCG.getSharedLValue(Count).getAddress();
1117  });
1118  PrivateScope.addPrivate(
1119  RHSVD, [this, PrivateVD]() { return GetAddrOfLocalVar(PrivateVD); });
1120  } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1121  isa<ArraySubscriptExpr>(IRef)) {
1122  // Store the address of the original variable associated with the LHS
1123  // implicit variable.
1124  PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
1125  return RedCG.getSharedLValue(Count).getAddress();
1126  });
1127  PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() {
1128  return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD),
1129  ConvertTypeForMem(RHSVD->getType()),
1130  "rhs.begin");
1131  });
1132  } else {
1133  QualType Type = PrivateVD->getType();
1134  bool IsArray = getContext().getAsArrayType(Type) != nullptr;
1135  Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress();
1136  // Store the address of the original variable associated with the LHS
1137  // implicit variable.
1138  if (IsArray) {
1139  OriginalAddr = Builder.CreateElementBitCast(
1140  OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin");
1141  }
1142  PrivateScope.addPrivate(LHSVD, [OriginalAddr]() { return OriginalAddr; });
1143  PrivateScope.addPrivate(
1144  RHSVD, [this, PrivateVD, RHSVD, IsArray]() {
1145  return IsArray
1146  ? Builder.CreateElementBitCast(
1147  GetAddrOfLocalVar(PrivateVD),
1148  ConvertTypeForMem(RHSVD->getType()), "rhs.begin")
1149  : GetAddrOfLocalVar(PrivateVD);
1150  });
1151  }
1152  ++ILHS;
1153  ++IRHS;
1154  ++IPriv;
1155  ++Count;
1156  }
1157 }
1158 
1160  const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1161  if (!HaveInsertPoint())
1162  return;
1167  bool HasAtLeastOneReduction = false;
1168  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1169  HasAtLeastOneReduction = true;
1170  Privates.append(C->privates().begin(), C->privates().end());
1171  LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1172  RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1173  ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1174  }
1175  if (HasAtLeastOneReduction) {
1176  bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1178  ReductionKind == OMPD_simd;
1179  bool SimpleReduction = ReductionKind == OMPD_simd;
1180  // Emit nowait reduction if nowait clause is present or directive is a
1181  // parallel directive (it always has implicit barrier).
1182  CGM.getOpenMPRuntime().emitReduction(
1183  *this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1184  {WithNowait, SimpleReduction, ReductionKind});
1185  }
1186 }
1187 
1189  CodeGenFunction &CGF, const OMPExecutableDirective &D,
1190  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1191  if (!CGF.HaveInsertPoint())
1192  return;
1193  llvm::BasicBlock *DoneBB = nullptr;
1194  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1195  if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
1196  if (!DoneBB) {
1197  if (llvm::Value *Cond = CondGen(CGF)) {
1198  // If the first post-update expression is found, emit conditional
1199  // block if it was requested.
1200  llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1201  DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1202  CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1203  CGF.EmitBlock(ThenBB);
1204  }
1205  }
1206  CGF.EmitIgnoredExpr(PostUpdate);
1207  }
1208  }
1209  if (DoneBB)
1210  CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1211 }
1212 
1213 namespace {
1214 /// Codegen lambda for appending distribute lower and upper bounds to outlined
1215 /// parallel function. This is necessary for combined constructs such as
1216 /// 'distribute parallel for'
1217 typedef llvm::function_ref<void(CodeGenFunction &,
1218  const OMPExecutableDirective &,
1220  CodeGenBoundParametersTy;
1221 } // anonymous namespace
1222 
1224  CodeGenFunction &CGF, const OMPExecutableDirective &S,
1225  OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1226  const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1227  const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1228  llvm::Function *OutlinedFn =
1229  CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
1230  S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
1231  if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
1232  CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1233  llvm::Value *NumThreads =
1234  CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1235  /*IgnoreResultAssign=*/true);
1236  CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
1237  CGF, NumThreads, NumThreadsClause->getBeginLoc());
1238  }
1239  if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
1240  CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1241  CGF.CGM.getOpenMPRuntime().emitProcBindClause(
1242  CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1243  }
1244  const Expr *IfCond = nullptr;
1245  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1246  if (C->getNameModifier() == OMPD_unknown ||
1247  C->getNameModifier() == OMPD_parallel) {
1248  IfCond = C->getCondition();
1249  break;
1250  }
1251  }
1252 
1253  OMPParallelScope Scope(CGF, S);
1255  // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
1256  // lower and upper bounds with the pragma 'for' chunking mechanism.
1257  // The following lambda takes care of appending the lower and upper bound
1258  // parameters when necessary
1259  CodeGenBoundParameters(CGF, S, CapturedVars);
1260  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
1261  CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn,
1262  CapturedVars, IfCond);
1263 }
1264 
1266  const OMPExecutableDirective &,
1268 
1270  // Emit parallel region as a standalone region.
1271  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1272  Action.Enter(CGF);
1273  OMPPrivateScope PrivateScope(CGF);
1274  bool Copyins = CGF.EmitOMPCopyinClause(S);
1275  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1276  if (Copyins) {
1277  // Emit implicit barrier to synchronize threads and avoid data races on
1278  // propagation master's thread values of threadprivate variables to local
1279  // instances of that variables of all other implicit threads.
1280  CGF.CGM.getOpenMPRuntime().emitBarrierCall(
1281  CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
1282  /*ForceSimpleCall=*/true);
1283  }
1284  CGF.EmitOMPPrivateClause(S, PrivateScope);
1285  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1286  (void)PrivateScope.Privatize();
1287  CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1288  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
1289  };
1290  emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
1293  [](CodeGenFunction &) { return nullptr; });
1294 }
1295 
1297  JumpDest LoopExit) {
1298  RunCleanupsScope BodyScope(*this);
1299  // Update counters values on current iteration.
1300  for (const Expr *UE : D.updates())
1301  EmitIgnoredExpr(UE);
1302  // Update the linear variables.
1303  // In distribute directives only loop counters may be marked as linear, no
1304  // need to generate the code for them.
1306  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1307  for (const Expr *UE : C->updates())
1308  EmitIgnoredExpr(UE);
1309  }
1310  }
1311 
1312  // On a continue in the body, jump to the end.
1313  JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue");
1314  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1315  // Emit loop body.
1316  EmitStmt(D.getBody());
1317  // The end (updates/cleanups).
1318  EmitBlock(Continue.getBlock());
1319  BreakContinueStack.pop_back();
1320 }
1321 
1323  const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
1324  const Expr *IncExpr,
1325  const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
1326  const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
1327  auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
1328 
1329  // Start the loop with a block that tests the condition.
1330  auto CondBlock = createBasicBlock("omp.inner.for.cond");
1331  EmitBlock(CondBlock);
1332  const SourceRange R = S.getSourceRange();
1333  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1334  SourceLocToDebugLoc(R.getEnd()));
1335 
1336  // If there are any cleanups between here and the loop-exit scope,
1337  // create a block to stage a loop exit along.
1338  llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1339  if (RequiresCleanup)
1340  ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
1341 
1342  llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body");
1343 
1344  // Emit condition.
1345  EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
1346  if (ExitBlock != LoopExit.getBlock()) {
1347  EmitBlock(ExitBlock);
1348  EmitBranchThroughCleanup(LoopExit);
1349  }
1350 
1351  EmitBlock(LoopBody);
1352  incrementProfileCounter(&S);
1353 
1354  // Create a block for the increment.
1355  JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
1356  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1357 
1358  BodyGen(*this);
1359 
1360  // Emit "IV = IV + 1" and a back-edge to the condition block.
1361  EmitBlock(Continue.getBlock());
1362  EmitIgnoredExpr(IncExpr);
1363  PostIncGen(*this);
1364  BreakContinueStack.pop_back();
1365  EmitBranch(CondBlock);
1366  LoopStack.pop();
1367  // Emit the fall-through block.
1368  EmitBlock(LoopExit.getBlock());
1369 }
1370 
1372  if (!HaveInsertPoint())
1373  return false;
1374  // Emit inits for the linear variables.
1375  bool HasLinears = false;
1376  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1377  for (const Expr *Init : C->inits()) {
1378  HasLinears = true;
1379  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
1380  if (const auto *Ref =
1381  dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
1382  AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
1383  const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
1384  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1385  CapturedStmtInfo->lookup(OrigVD) != nullptr,
1386  VD->getInit()->getType(), VK_LValue,
1387  VD->getInit()->getExprLoc());
1388  EmitExprAsInit(&DRE, VD, MakeAddrLValue(Emission.getAllocatedAddress(),
1389  VD->getType()),
1390  /*capturedByInit=*/false);
1391  EmitAutoVarCleanups(Emission);
1392  } else {
1393  EmitVarDecl(*VD);
1394  }
1395  }
1396  // Emit the linear steps for the linear clauses.
1397  // If a step is not constant, it is pre-calculated before the loop.
1398  if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
1399  if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
1400  EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
1401  // Emit calculation of the linear step.
1402  EmitIgnoredExpr(CS);
1403  }
1404  }
1405  return HasLinears;
1406 }
1407 
1409  const OMPLoopDirective &D,
1410  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1411  if (!HaveInsertPoint())
1412  return;
1413  llvm::BasicBlock *DoneBB = nullptr;
1414  // Emit the final values of the linear variables.
1415  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1416  auto IC = C->varlist_begin();
1417  for (const Expr *F : C->finals()) {
1418  if (!DoneBB) {
1419  if (llvm::Value *Cond = CondGen(*this)) {
1420  // If the first post-update expression is found, emit conditional
1421  // block if it was requested.
1422  llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu");
1423  DoneBB = createBasicBlock(".omp.linear.pu.done");
1424  Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1425  EmitBlock(ThenBB);
1426  }
1427  }
1428  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
1429  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1430  CapturedStmtInfo->lookup(OrigVD) != nullptr,
1431  (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
1432  Address OrigAddr = EmitLValue(&DRE).getAddress();
1433  CodeGenFunction::OMPPrivateScope VarScope(*this);
1434  VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1435  (void)VarScope.Privatize();
1436  EmitIgnoredExpr(F);
1437  ++IC;
1438  }
1439  if (const Expr *PostUpdate = C->getPostUpdateExpr())
1440  EmitIgnoredExpr(PostUpdate);
1441  }
1442  if (DoneBB)
1443  EmitBlock(DoneBB, /*IsFinished=*/true);
1444 }
1445 
1447  const OMPExecutableDirective &D) {
1448  if (!CGF.HaveInsertPoint())
1449  return;
1450  for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
1451  unsigned ClauseAlignment = 0;
1452  if (const Expr *AlignmentExpr = Clause->getAlignment()) {
1453  auto *AlignmentCI =
1454  cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
1455  ClauseAlignment = static_cast<unsigned>(AlignmentCI->getZExtValue());
1456  }
1457  for (const Expr *E : Clause->varlists()) {
1458  unsigned Alignment = ClauseAlignment;
1459  if (Alignment == 0) {
1460  // OpenMP [2.8.1, Description]
1461  // If no optional parameter is specified, implementation-defined default
1462  // alignments for SIMD instructions on the target platforms are assumed.
1463  Alignment =
1464  CGF.getContext()
1466  E->getType()->getPointeeType()))
1467  .getQuantity();
1468  }
1469  assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) &&
1470  "alignment is not power of 2");
1471  if (Alignment != 0) {
1472  llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
1474  PtrValue, E, /*No second loc needed*/ SourceLocation(), Alignment);
1475  }
1476  }
1477  }
1478 }
1479 
1481  const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
1482  if (!HaveInsertPoint())
1483  return;
1484  auto I = S.private_counters().begin();
1485  for (const Expr *E : S.counters()) {
1486  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1487  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
1488  // Emit var without initialization.
1489  AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD);
1490  EmitAutoVarCleanups(VarEmission);
1491  LocalDeclMap.erase(PrivateVD);
1492  (void)LoopScope.addPrivate(VD, [&VarEmission]() {
1493  return VarEmission.getAllocatedAddress();
1494  });
1495  if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
1496  VD->hasGlobalStorage()) {
1497  (void)LoopScope.addPrivate(PrivateVD, [this, VD, E]() {
1498  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD),
1499  LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
1500  E->getType(), VK_LValue, E->getExprLoc());
1501  return EmitLValue(&DRE).getAddress();
1502  });
1503  } else {
1504  (void)LoopScope.addPrivate(PrivateVD, [&VarEmission]() {
1505  return VarEmission.getAllocatedAddress();
1506  });
1507  }
1508  ++I;
1509  }
1510  // Privatize extra loop counters used in loops for ordered(n) clauses.
1511  for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) {
1512  if (!C->getNumForLoops())
1513  continue;
1514  for (unsigned I = S.getCollapsedNumber(),
1515  E = C->getLoopNumIterations().size();
1516  I < E; ++I) {
1517  const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I));
1518  const auto *VD = cast<VarDecl>(DRE->getDecl());
1519  // Override only those variables that are really emitted already.
1520  if (LocalDeclMap.count(VD)) {
1521  (void)LoopScope.addPrivate(VD, [this, DRE, VD]() {
1522  return CreateMemTemp(DRE->getType(), VD->getName());
1523  });
1524  }
1525  }
1526  }
1527 }
1528 
1529 static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
1530  const Expr *Cond, llvm::BasicBlock *TrueBlock,
1531  llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
1532  if (!CGF.HaveInsertPoint())
1533  return;
1534  {
1535  CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
1536  CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
1537  (void)PreCondScope.Privatize();
1538  // Get initial values of real counters.
1539  for (const Expr *I : S.inits()) {
1540  CGF.EmitIgnoredExpr(I);
1541  }
1542  }
1543  // Check that loop is executed at least one time.
1544  CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
1545 }
1546 
1548  const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
1549  if (!HaveInsertPoint())
1550  return;
1553  const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1554  for (const Expr *C : LoopDirective->counters()) {
1555  SIMDLCVs.insert(
1556  cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
1557  }
1558  }
1559  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1560  auto CurPrivate = C->privates().begin();
1561  for (const Expr *E : C->varlists()) {
1562  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1563  const auto *PrivateVD =
1564  cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
1565  if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
1566  bool IsRegistered = PrivateScope.addPrivate(VD, [this, PrivateVD]() {
1567  // Emit private VarDecl with copy init.
1568  EmitVarDecl(*PrivateVD);
1569  return GetAddrOfLocalVar(PrivateVD);
1570  });
1571  assert(IsRegistered && "linear var already registered as private");
1572  // Silence the warning about unused variable.
1573  (void)IsRegistered;
1574  } else {
1575  EmitVarDecl(*PrivateVD);
1576  }
1577  ++CurPrivate;
1578  }
1579  }
1580 }
1581 
1583  const OMPExecutableDirective &D,
1584  bool IsMonotonic) {
1585  if (!CGF.HaveInsertPoint())
1586  return;
1587  if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
1588  RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
1589  /*ignoreResult=*/true);
1590  auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1591  CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1592  // In presence of finite 'safelen', it may be unsafe to mark all
1593  // the memory instructions parallel, because loop-carried
1594  // dependences of 'safelen' iterations are possible.
1595  if (!IsMonotonic)
1597  } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
1598  RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
1599  /*ignoreResult=*/true);
1600  auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1601  CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1602  // In presence of finite 'safelen', it may be unsafe to mark all
1603  // the memory instructions parallel, because loop-carried
1604  // dependences of 'safelen' iterations are possible.
1605  CGF.LoopStack.setParallel(/*Enable=*/false);
1606  }
1607 }
1608 
1610  bool IsMonotonic) {
1611  // Walk clauses and process safelen/lastprivate.
1612  LoopStack.setParallel(!IsMonotonic);
1613  LoopStack.setVectorizeEnable();
1614  emitSimdlenSafelenClause(*this, D, IsMonotonic);
1615 }
1616 
1618  const OMPLoopDirective &D,
1619  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1620  if (!HaveInsertPoint())
1621  return;
1622  llvm::BasicBlock *DoneBB = nullptr;
1623  auto IC = D.counters().begin();
1624  auto IPC = D.private_counters().begin();
1625  for (const Expr *F : D.finals()) {
1626  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
1627  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
1628  const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
1629  if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
1630  OrigVD->hasGlobalStorage() || CED) {
1631  if (!DoneBB) {
1632  if (llvm::Value *Cond = CondGen(*this)) {
1633  // If the first post-update expression is found, emit conditional
1634  // block if it was requested.
1635  llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then");
1636  DoneBB = createBasicBlock(".omp.final.done");
1637  Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1638  EmitBlock(ThenBB);
1639  }
1640  }
1641  Address OrigAddr = Address::invalid();
1642  if (CED) {
1643  OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();
1644  } else {
1645  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(PrivateVD),
1646  /*RefersToEnclosingVariableOrCapture=*/false,
1647  (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
1648  OrigAddr = EmitLValue(&DRE).getAddress();
1649  }
1650  OMPPrivateScope VarScope(*this);
1651  VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1652  (void)VarScope.Privatize();
1653  EmitIgnoredExpr(F);
1654  }
1655  ++IC;
1656  ++IPC;
1657  }
1658  if (DoneBB)
1659  EmitBlock(DoneBB, /*IsFinished=*/true);
1660 }
1661 
1663  const OMPLoopDirective &S,
1665  CGF.EmitOMPLoopBody(S, LoopExit);
1666  CGF.EmitStopPoint(&S);
1667 }
1668 
1669 /// Emit a helper variable and return corresponding lvalue.
1671  const DeclRefExpr *Helper) {
1672  auto VDecl = cast<VarDecl>(Helper->getDecl());
1673  CGF.EmitVarDecl(*VDecl);
1674  return CGF.EmitLValue(Helper);
1675 }
1676 
1678  PrePostActionTy &Action) {
1679  Action.Enter(CGF);
1681  "Expected simd directive");
1682  OMPLoopScope PreInitScope(CGF, S);
1683  // if (PreCond) {
1684  // for (IV in 0..LastIteration) BODY;
1685  // <Final counter/linear vars updates>;
1686  // }
1687  //
1691  (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));
1692  (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));
1693  }
1694 
1695  // Emit: if (PreCond) - begin.
1696  // If the condition constant folds and can be elided, avoid emitting the
1697  // whole loop.
1698  bool CondConstant;
1699  llvm::BasicBlock *ContBlock = nullptr;
1700  if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
1701  if (!CondConstant)
1702  return;
1703  } else {
1704  llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");
1705  ContBlock = CGF.createBasicBlock("simd.if.end");
1706  emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
1707  CGF.getProfileCount(&S));
1708  CGF.EmitBlock(ThenBlock);
1709  CGF.incrementProfileCounter(&S);
1710  }
1711 
1712  // Emit the loop iteration variable.
1713  const Expr *IVExpr = S.getIterationVariable();
1714  const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
1715  CGF.EmitVarDecl(*IVDecl);
1716  CGF.EmitIgnoredExpr(S.getInit());
1717 
1718  // Emit the iterations count variable.
1719  // If it is not a variable, Sema decided to calculate iterations count on
1720  // each iteration (e.g., it is foldable into a constant).
1721  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
1722  CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
1723  // Emit calculation of the iterations count.
1725  }
1726 
1727  CGF.EmitOMPSimdInit(S);
1728 
1729  emitAlignedClause(CGF, S);
1730  (void)CGF.EmitOMPLinearClauseInit(S);
1731  {
1732  CodeGenFunction::OMPPrivateScope LoopScope(CGF);
1733  CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
1734  CGF.EmitOMPLinearClause(S, LoopScope);
1735  CGF.EmitOMPPrivateClause(S, LoopScope);
1736  CGF.EmitOMPReductionClauseInit(S, LoopScope);
1737  bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
1738  (void)LoopScope.Privatize();
1740  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
1741  CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
1742  S.getInc(),
1743  [&S](CodeGenFunction &CGF) {
1745  CGF.EmitStopPoint(&S);
1746  },
1747  [](CodeGenFunction &) {});
1748  CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; });
1749  // Emit final copy of the lastprivate variables at the end of loops.
1750  if (HasLastprivateClause)
1751  CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
1752  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
1754  [](CodeGenFunction &) { return nullptr; });
1755  }
1756  CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; });
1757  // Emit: if (PreCond) - end.
1758  if (ContBlock) {
1759  CGF.EmitBranch(ContBlock);
1760  CGF.EmitBlock(ContBlock, true);
1761  }
1762 }
1763 
1765  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1766  emitOMPSimdRegion(CGF, S, Action);
1767  };
1768  OMPLexicalScope Scope(*this, S, OMPD_unknown);
1769  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
1770 }
1771 
1772 void CodeGenFunction::EmitOMPOuterLoop(
1773  bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
1775  const CodeGenFunction::OMPLoopArguments &LoopArgs,
1776  const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
1777  const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
1778  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1779 
1780  const Expr *IVExpr = S.getIterationVariable();
1781  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1782  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1783 
1784  JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
1785 
1786  // Start the loop with a block that tests the condition.
1787  llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond");
1788  EmitBlock(CondBlock);
1789  const SourceRange R = S.getSourceRange();
1790  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1791  SourceLocToDebugLoc(R.getEnd()));
1792 
1793  llvm::Value *BoolCondVal = nullptr;
1794  if (!DynamicOrOrdered) {
1795  // UB = min(UB, GlobalUB) or
1796  // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
1797  // 'distribute parallel for')
1798  EmitIgnoredExpr(LoopArgs.EUB);
1799  // IV = LB
1800  EmitIgnoredExpr(LoopArgs.Init);
1801  // IV < UB
1802  BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
1803  } else {
1804  BoolCondVal =
1805  RT.emitForNext(*this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
1806  LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
1807  }
1808 
1809  // If there are any cleanups between here and the loop-exit scope,
1810  // create a block to stage a loop exit along.
1811  llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1812  if (LoopScope.requiresCleanups())
1813  ExitBlock = createBasicBlock("omp.dispatch.cleanup");
1814 
1815  llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body");
1816  Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
1817  if (ExitBlock != LoopExit.getBlock()) {
1818  EmitBlock(ExitBlock);
1819  EmitBranchThroughCleanup(LoopExit);
1820  }
1821  EmitBlock(LoopBody);
1822 
1823  // Emit "IV = LB" (in case of static schedule, we have already calculated new
1824  // LB for loop condition and emitted it above).
1825  if (DynamicOrOrdered)
1826  EmitIgnoredExpr(LoopArgs.Init);
1827 
1828  // Create a block for the increment.
1829  JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
1830  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1831 
1832  // Generate !llvm.loop.parallel metadata for loads and stores for loops
1833  // with dynamic/guided scheduling and without ordered clause.
1835  LoopStack.setParallel(!IsMonotonic);
1836  else
1837  EmitOMPSimdInit(S, IsMonotonic);
1838 
1839  SourceLocation Loc = S.getBeginLoc();
1840 
1841  // when 'distribute' is not combined with a 'for':
1842  // while (idx <= UB) { BODY; ++idx; }
1843  // when 'distribute' is combined with a 'for'
1844  // (e.g. 'distribute parallel for')
1845  // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
1846  EmitOMPInnerLoop(
1847  S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
1848  [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
1849  CodeGenLoop(CGF, S, LoopExit);
1850  },
1851  [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
1852  CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
1853  });
1854 
1855  EmitBlock(Continue.getBlock());
1856  BreakContinueStack.pop_back();
1857  if (!DynamicOrOrdered) {
1858  // Emit "LB = LB + Stride", "UB = UB + Stride".
1859  EmitIgnoredExpr(LoopArgs.NextLB);
1860  EmitIgnoredExpr(LoopArgs.NextUB);
1861  }
1862 
1863  EmitBranch(CondBlock);
1864  LoopStack.pop();
1865  // Emit the fall-through block.
1866  EmitBlock(LoopExit.getBlock());
1867 
1868  // Tell the runtime we are done.
1869  auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) {
1870  if (!DynamicOrOrdered)
1871  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
1872  S.getDirectiveKind());
1873  };
1874  OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
1875 }
1876 
1877 void CodeGenFunction::EmitOMPForOuterLoop(
1878  const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
1879  const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
1880  const OMPLoopArguments &LoopArgs,
1881  const CodeGenDispatchBoundsTy &CGDispatchBounds) {
1882  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1883 
1884  // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
1885  const bool DynamicOrOrdered =
1886  Ordered || RT.isDynamic(ScheduleKind.Schedule);
1887 
1888  assert((Ordered ||
1889  !RT.isStaticNonchunked(ScheduleKind.Schedule,
1890  LoopArgs.Chunk != nullptr)) &&
1891  "static non-chunked schedule does not need outer loop");
1892 
1893  // Emit outer loop.
1894  //
1895  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1896  // When schedule(dynamic,chunk_size) is specified, the iterations are
1897  // distributed to threads in the team in chunks as the threads request them.
1898  // Each thread executes a chunk of iterations, then requests another chunk,
1899  // until no chunks remain to be distributed. Each chunk contains chunk_size
1900  // iterations, except for the last chunk to be distributed, which may have
1901  // fewer iterations. When no chunk_size is specified, it defaults to 1.
1902  //
1903  // When schedule(guided,chunk_size) is specified, the iterations are assigned
1904  // to threads in the team in chunks as the executing threads request them.
1905  // Each thread executes a chunk of iterations, then requests another chunk,
1906  // until no chunks remain to be assigned. For a chunk_size of 1, the size of
1907  // each chunk is proportional to the number of unassigned iterations divided
1908  // by the number of threads in the team, decreasing to 1. For a chunk_size
1909  // with value k (greater than 1), the size of each chunk is determined in the
1910  // same way, with the restriction that the chunks do not contain fewer than k
1911  // iterations (except for the last chunk to be assigned, which may have fewer
1912  // than k iterations).
1913  //
1914  // When schedule(auto) is specified, the decision regarding scheduling is
1915  // delegated to the compiler and/or runtime system. The programmer gives the
1916  // implementation the freedom to choose any possible mapping of iterations to
1917  // threads in the team.
1918  //
1919  // When schedule(runtime) is specified, the decision regarding scheduling is
1920  // deferred until run time, and the schedule and chunk size are taken from the
1921  // run-sched-var ICV. If the ICV is set to auto, the schedule is
1922  // implementation defined
1923  //
1924  // while(__kmpc_dispatch_next(&LB, &UB)) {
1925  // idx = LB;
1926  // while (idx <= UB) { BODY; ++idx;
1927  // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
1928  // } // inner loop
1929  // }
1930  //
1931  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1932  // When schedule(static, chunk_size) is specified, iterations are divided into
1933  // chunks of size chunk_size, and the chunks are assigned to the threads in
1934  // the team in a round-robin fashion in the order of the thread number.
1935  //
1936  // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
1937  // while (idx <= UB) { BODY; ++idx; } // inner loop
1938  // LB = LB + ST;
1939  // UB = UB + ST;
1940  // }
1941  //
1942 
1943  const Expr *IVExpr = S.getIterationVariable();
1944  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1945  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1946 
1947  if (DynamicOrOrdered) {
1948  const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
1949  CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
1950  llvm::Value *LBVal = DispatchBounds.first;
1951  llvm::Value *UBVal = DispatchBounds.second;
1952  CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
1953  LoopArgs.Chunk};
1954  RT.emitForDispatchInit(*this, S.getBeginLoc(), ScheduleKind, IVSize,
1955  IVSigned, Ordered, DipatchRTInputValues);
1956  } else {
1957  CGOpenMPRuntime::StaticRTInput StaticInit(
1958  IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
1959  LoopArgs.ST, LoopArgs.Chunk);
1960  RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
1961  ScheduleKind, StaticInit);
1962  }
1963 
1964  auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
1965  const unsigned IVSize,
1966  const bool IVSigned) {
1967  if (Ordered) {
1968  CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
1969  IVSigned);
1970  }
1971  };
1972 
1973  OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
1974  LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
1975  OuterLoopArgs.IncExpr = S.getInc();
1976  OuterLoopArgs.Init = S.getInit();
1977  OuterLoopArgs.Cond = S.getCond();
1978  OuterLoopArgs.NextLB = S.getNextLowerBound();
1979  OuterLoopArgs.NextUB = S.getNextUpperBound();
1980  EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
1981  emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
1982 }
1983 
1985  const unsigned IVSize, const bool IVSigned) {}
1986 
1987 void CodeGenFunction::EmitOMPDistributeOuterLoop(
1988  OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
1989  OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
1990  const CodeGenLoopTy &CodeGenLoopContent) {
1991 
1992  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1993 
1994  // Emit outer loop.
1995  // Same behavior as a OMPForOuterLoop, except that schedule cannot be
1996  // dynamic
1997  //
1998 
1999  const Expr *IVExpr = S.getIterationVariable();
2000  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2001  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2002 
2003  CGOpenMPRuntime::StaticRTInput StaticInit(
2004  IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
2005  LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
2006  RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, StaticInit);
2007 
2008  // for combined 'distribute' and 'for' the increment expression of distribute
2009  // is stored in DistInc. For 'distribute' alone, it is in Inc.
2010  Expr *IncExpr;
2012  IncExpr = S.getDistInc();
2013  else
2014  IncExpr = S.getInc();
2015 
2016  // this routine is shared by 'omp distribute parallel for' and
2017  // 'omp distribute': select the right EUB expression depending on the
2018  // directive
2019  OMPLoopArguments OuterLoopArgs;
2020  OuterLoopArgs.LB = LoopArgs.LB;
2021  OuterLoopArgs.UB = LoopArgs.UB;
2022  OuterLoopArgs.ST = LoopArgs.ST;
2023  OuterLoopArgs.IL = LoopArgs.IL;
2024  OuterLoopArgs.Chunk = LoopArgs.Chunk;
2025  OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2027  : S.getEnsureUpperBound();
2028  OuterLoopArgs.IncExpr = IncExpr;
2029  OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2030  ? S.getCombinedInit()
2031  : S.getInit();
2032  OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2033  ? S.getCombinedCond()
2034  : S.getCond();
2035  OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2037  : S.getNextLowerBound();
2038  OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2040  : S.getNextUpperBound();
2041 
2042  EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
2043  LoopScope, OuterLoopArgs, CodeGenLoopContent,
2045 }
2046 
2047 static std::pair<LValue, LValue>
2049  const OMPExecutableDirective &S) {
2050  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2051  LValue LB =
2052  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2053  LValue UB =
2054  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2055 
2056  // When composing 'distribute' with 'for' (e.g. as in 'distribute
2057  // parallel for') we need to use the 'distribute'
2058  // chunk lower and upper bounds rather than the whole loop iteration
2059  // space. These are parameters to the outlined function for 'parallel'
2060  // and we copy the bounds of the previous schedule into the
2061  // the current ones.
2062  LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
2063  LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
2064  llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
2065  PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());
2066  PrevLBVal = CGF.EmitScalarConversion(
2067  PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
2068  LS.getIterationVariable()->getType(),
2070  llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
2071  PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());
2072  PrevUBVal = CGF.EmitScalarConversion(
2073  PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
2074  LS.getIterationVariable()->getType(),
2076 
2077  CGF.EmitStoreOfScalar(PrevLBVal, LB);
2078  CGF.EmitStoreOfScalar(PrevUBVal, UB);
2079 
2080  return {LB, UB};
2081 }
2082 
2083 /// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
2084 /// we need to use the LB and UB expressions generated by the worksharing
2085 /// code generation support, whereas in non combined situations we would
2086 /// just emit 0 and the LastIteration expression
2087 /// This function is necessary due to the difference of the LB and UB
2088 /// types for the RT emission routines for 'for_static_init' and
2089 /// 'for_dispatch_init'
2090 static std::pair<llvm::Value *, llvm::Value *>
2092  const OMPExecutableDirective &S,
2093  Address LB, Address UB) {
2094  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2095  const Expr *IVExpr = LS.getIterationVariable();
2096  // when implementing a dynamic schedule for a 'for' combined with a
2097  // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
2098  // is not normalized as each team only executes its own assigned
2099  // distribute chunk
2100  QualType IteratorTy = IVExpr->getType();
2101  llvm::Value *LBVal =
2102  CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
2103  llvm::Value *UBVal =
2104  CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
2105  return {LBVal, UBVal};
2106 }
2107 
2109  CodeGenFunction &CGF, const OMPExecutableDirective &S,
2110  llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) {
2111  const auto &Dir = cast<OMPLoopDirective>(S);
2112  LValue LB =
2113  CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
2114  llvm::Value *LBCast = CGF.Builder.CreateIntCast(
2115  CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2116  CapturedVars.push_back(LBCast);
2117  LValue UB =
2118  CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
2119 
2120  llvm::Value *UBCast = CGF.Builder.CreateIntCast(
2121  CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2122  CapturedVars.push_back(UBCast);
2123 }
2124 
2125 static void
2127  const OMPLoopDirective &S,
2129  auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
2130  PrePostActionTy &Action) {
2131  Action.Enter(CGF);
2132  bool HasCancel = false;
2134  if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
2135  HasCancel = D->hasCancel();
2136  else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
2137  HasCancel = D->hasCancel();
2138  else if (const auto *D =
2139  dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
2140  HasCancel = D->hasCancel();
2141  }
2143  HasCancel);
2147  };
2148 
2150  CGF, S,
2151  isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for,
2152  CGInlinedWorksharingLoop,
2154 }
2155 
2158  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2160  S.getDistInc());
2161  };
2162  OMPLexicalScope Scope(*this, S, OMPD_parallel);
2163  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2164 }
2165 
2168  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2170  S.getDistInc());
2171  };
2172  OMPLexicalScope Scope(*this, S, OMPD_parallel);
2173  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2174 }
2175 
2177  const OMPDistributeSimdDirective &S) {
2178  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2180  };
2181  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2182  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2183 }
2184 
2186  CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
2187  // Emit SPMD target parallel for region as a standalone region.
2188  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2189  emitOMPSimdRegion(CGF, S, Action);
2190  };
2191  llvm::Function *Fn;
2192  llvm::Constant *Addr;
2193  // Emit target region as a standalone region.
2194  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
2195  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
2196  assert(Fn && Addr && "Target device function emission failed.");
2197 }
2198 
2200  const OMPTargetSimdDirective &S) {
2201  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2202  emitOMPSimdRegion(CGF, S, Action);
2203  };
2204  emitCommonOMPTargetDirective(*this, S, CodeGen);
2205 }
2206 
2207 namespace {
2208  struct ScheduleKindModifiersTy {
2212  ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
2215  : Kind(Kind), M1(M1), M2(M2) {}
2216  };
2217 } // namespace
2218 
2220  const OMPLoopDirective &S, Expr *EUB,
2221  const CodeGenLoopBoundsTy &CodeGenLoopBounds,
2222  const CodeGenDispatchBoundsTy &CGDispatchBounds) {
2223  // Emit the loop iteration variable.
2224  const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
2225  const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
2226  EmitVarDecl(*IVDecl);
2227 
2228  // Emit the iterations count variable.
2229  // If it is not a variable, Sema decided to calculate iterations count on each
2230  // iteration (e.g., it is foldable into a constant).
2231  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2232  EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2233  // Emit calculation of the iterations count.
2234  EmitIgnoredExpr(S.getCalcLastIteration());
2235  }
2236 
2237  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
2238 
2239  bool HasLastprivateClause;
2240  // Check pre-condition.
2241  {
2242  OMPLoopScope PreInitScope(*this, S);
2243  // Skip the entire loop if we don't meet the precondition.
2244  // If the condition constant folds and can be elided, avoid emitting the
2245  // whole loop.
2246  bool CondConstant;
2247  llvm::BasicBlock *ContBlock = nullptr;
2248  if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2249  if (!CondConstant)
2250  return false;
2251  } else {
2252  llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
2253  ContBlock = createBasicBlock("omp.precond.end");
2254  emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
2255  getProfileCount(&S));
2256  EmitBlock(ThenBlock);
2257  incrementProfileCounter(&S);
2258  }
2259 
2260  RunCleanupsScope DoacrossCleanupScope(*this);
2261  bool Ordered = false;
2262  if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
2263  if (OrderedClause->getNumForLoops())
2264  RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations());
2265  else
2266  Ordered = true;
2267  }
2268 
2269  llvm::DenseSet<const Expr *> EmittedFinals;
2270  emitAlignedClause(*this, S);
2271  bool HasLinears = EmitOMPLinearClauseInit(S);
2272  // Emit helper vars inits.
2273 
2274  std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
2275  LValue LB = Bounds.first;
2276  LValue UB = Bounds.second;
2277  LValue ST =
2278  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
2279  LValue IL =
2280  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
2281 
2282  // Emit 'then' code.
2283  {
2284  OMPPrivateScope LoopScope(*this);
2285  if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
2286  // Emit implicit barrier to synchronize threads and avoid data races on
2287  // initialization of firstprivate variables and post-update of
2288  // lastprivate variables.
2289  CGM.getOpenMPRuntime().emitBarrierCall(
2290  *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
2291  /*ForceSimpleCall=*/true);
2292  }
2293  EmitOMPPrivateClause(S, LoopScope);
2294  HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
2295  EmitOMPReductionClauseInit(S, LoopScope);
2296  EmitOMPPrivateLoopCounters(S, LoopScope);
2297  EmitOMPLinearClause(S, LoopScope);
2298  (void)LoopScope.Privatize();
2300  CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
2301 
2302  // Detect the loop schedule kind and chunk.
2303  const Expr *ChunkExpr = nullptr;
2304  OpenMPScheduleTy ScheduleKind;
2305  if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
2306  ScheduleKind.Schedule = C->getScheduleKind();
2307  ScheduleKind.M1 = C->getFirstScheduleModifier();
2308  ScheduleKind.M2 = C->getSecondScheduleModifier();
2309  ChunkExpr = C->getChunkSize();
2310  } else {
2311  // Default behaviour for schedule clause.
2312  CGM.getOpenMPRuntime().getDefaultScheduleAndChunk(
2313  *this, S, ScheduleKind.Schedule, ChunkExpr);
2314  }
2315  bool HasChunkSizeOne = false;
2316  llvm::Value *Chunk = nullptr;
2317  if (ChunkExpr) {
2318  Chunk = EmitScalarExpr(ChunkExpr);
2319  Chunk = EmitScalarConversion(Chunk, ChunkExpr->getType(),
2321  S.getBeginLoc());
2322  Expr::EvalResult Result;
2323  if (ChunkExpr->EvaluateAsInt(Result, getContext())) {
2324  llvm::APSInt EvaluatedChunk = Result.Val.getInt();
2325  HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
2326  }
2327  }
2328  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2329  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2330  // OpenMP 4.5, 2.7.1 Loop Construct, Description.
2331  // If the static schedule kind is specified or if the ordered clause is
2332  // specified, and if no monotonic modifier is specified, the effect will
2333  // be as if the monotonic modifier was specified.
2334  bool StaticChunkedOne = RT.isStaticChunked(ScheduleKind.Schedule,
2335  /* Chunked */ Chunk != nullptr) && HasChunkSizeOne &&
2337  if ((RT.isStaticNonchunked(ScheduleKind.Schedule,
2338  /* Chunked */ Chunk != nullptr) ||
2339  StaticChunkedOne) &&
2340  !Ordered) {
2342  EmitOMPSimdInit(S, /*IsMonotonic=*/true);
2343  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2344  // When no chunk_size is specified, the iteration space is divided into
2345  // chunks that are approximately equal in size, and at most one chunk is
2346  // distributed to each thread. Note that the size of the chunks is
2347  // unspecified in this case.
2348  CGOpenMPRuntime::StaticRTInput StaticInit(
2349  IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
2350  UB.getAddress(), ST.getAddress(),
2351  StaticChunkedOne ? Chunk : nullptr);
2352  RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
2353  ScheduleKind, StaticInit);
2354  JumpDest LoopExit =
2355  getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
2356  // UB = min(UB, GlobalUB);
2357  if (!StaticChunkedOne)
2358  EmitIgnoredExpr(S.getEnsureUpperBound());
2359  // IV = LB;
2360  EmitIgnoredExpr(S.getInit());
2361  // For unchunked static schedule generate:
2362  //
2363  // while (idx <= UB) {
2364  // BODY;
2365  // ++idx;
2366  // }
2367  //
2368  // For static schedule with chunk one:
2369  //
2370  // while (IV <= PrevUB) {
2371  // BODY;
2372  // IV += ST;
2373  // }
2374  EmitOMPInnerLoop(S, LoopScope.requiresCleanups(),
2375  StaticChunkedOne ? S.getCombinedParForInDistCond() : S.getCond(),
2376  StaticChunkedOne ? S.getDistInc() : S.getInc(),
2377  [&S, LoopExit](CodeGenFunction &CGF) {
2378  CGF.EmitOMPLoopBody(S, LoopExit);
2379  CGF.EmitStopPoint(&S);
2380  },
2381  [](CodeGenFunction &) {});
2382  EmitBlock(LoopExit.getBlock());
2383  // Tell the runtime we are done.
2384  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2385  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2386  S.getDirectiveKind());
2387  };
2388  OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
2389  } else {
2390  const bool IsMonotonic =
2391  Ordered || ScheduleKind.Schedule == OMPC_SCHEDULE_static ||
2392  ScheduleKind.Schedule == OMPC_SCHEDULE_unknown ||
2393  ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
2394  ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
2395  // Emit the outer loop, which requests its work chunk [LB..UB] from
2396  // runtime and runs the inner loop to process it.
2397  const OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),
2398  ST.getAddress(), IL.getAddress(),
2399  Chunk, EUB);
2400  EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
2401  LoopArguments, CGDispatchBounds);
2402  }
2404  EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
2405  return CGF.Builder.CreateIsNotNull(
2406  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2407  });
2408  }
2409  EmitOMPReductionClauseFinal(
2410  S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind())
2411  ? /*Parallel and Simd*/ OMPD_parallel_for_simd
2412  : /*Parallel only*/ OMPD_parallel);
2413  // Emit post-update of the reduction variables if IsLastIter != 0.
2415  *this, S, [IL, &S](CodeGenFunction &CGF) {
2416  return CGF.Builder.CreateIsNotNull(
2417  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2418  });
2419  // Emit final copy of the lastprivate variables if IsLastIter != 0.
2420  if (HasLastprivateClause)
2421  EmitOMPLastprivateClauseFinal(
2423  Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
2424  }
2425  EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) {
2426  return CGF.Builder.CreateIsNotNull(
2427  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2428  });
2429  DoacrossCleanupScope.ForceCleanup();
2430  // We're now done with the loop, so jump to the continuation block.
2431  if (ContBlock) {
2432  EmitBranch(ContBlock);
2433  EmitBlock(ContBlock, /*IsFinished=*/true);
2434  }
2435  }
2436  return HasLastprivateClause;
2437 }
2438 
2439 /// The following two functions generate expressions for the loop lower
2440 /// and upper bounds in case of static and dynamic (dispatch) schedule
2441 /// of the associated 'for' or 'distribute' loop.
2442 static std::pair<LValue, LValue>
2444  const auto &LS = cast<OMPLoopDirective>(S);
2445  LValue LB =
2446  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2447  LValue UB =
2448  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2449  return {LB, UB};
2450 }
2451 
2452 /// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
2453 /// consider the lower and upper bound expressions generated by the
2454 /// worksharing loop support, but we use 0 and the iteration space size as
2455 /// constants
2456 static std::pair<llvm::Value *, llvm::Value *>
2458  Address LB, Address UB) {
2459  const auto &LS = cast<OMPLoopDirective>(S);
2460  const Expr *IVExpr = LS.getIterationVariable();
2461  const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
2462  llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
2463  llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
2464  return {LBVal, UBVal};
2465 }
2466 
2468  bool HasLastprivates = false;
2469  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2470  PrePostActionTy &) {
2471  OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel());
2472  HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2475  };
2476  {
2477  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2478  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
2479  S.hasCancel());
2480  }
2481 
2482  // Emit an implicit barrier at the end.
2483  if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
2484  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
2485 }
2486 
2488  bool HasLastprivates = false;
2489  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2490  PrePostActionTy &) {
2491  HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2494  };
2495  {
2496  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2497  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2498  }
2499 
2500  // Emit an implicit barrier at the end.
2501  if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
2502  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
2503 }
2504 
2506  const Twine &Name,
2507  llvm::Value *Init = nullptr) {
2508  LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
2509  if (Init)
2510  CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
2511  return LVal;
2512 }
2513 
2514 void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
2516  const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
2517  bool HasLastprivates = false;
2518  auto &&CodeGen = [&S, CapturedStmt, CS,
2519  &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
2520  ASTContext &C = CGF.getContext();
2521  QualType KmpInt32Ty =
2522  C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2523  // Emit helper vars inits.
2524  LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
2525  CGF.Builder.getInt32(0));
2526  llvm::ConstantInt *GlobalUBVal = CS != nullptr
2527  ? CGF.Builder.getInt32(CS->size() - 1)
2528  : CGF.Builder.getInt32(0);
2529  LValue UB =
2530  createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
2531  LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
2532  CGF.Builder.getInt32(1));
2533  LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
2534  CGF.Builder.getInt32(0));
2535  // Loop counter.
2536  LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
2537  OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
2538  CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
2539  OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
2540  CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
2541  // Generate condition for loop.
2542  BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue,
2544  // Increment for loop counter.
2545  UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary,
2546  S.getBeginLoc(), true);
2547  auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
2548  // Iterate through all sections and emit a switch construct:
2549  // switch (IV) {
2550  // case 0:
2551  // <SectionStmt[0]>;
2552  // break;
2553  // ...
2554  // case <NumSection> - 1:
2555  // <SectionStmt[<NumSection> - 1]>;
2556  // break;
2557  // }
2558  // .omp.sections.exit:
2559  llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
2560  llvm::SwitchInst *SwitchStmt =
2561  CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
2562  ExitBB, CS == nullptr ? 1 : CS->size());
2563  if (CS) {
2564  unsigned CaseNumber = 0;
2565  for (const Stmt *SubStmt : CS->children()) {
2566  auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2567  CGF.EmitBlock(CaseBB);
2568  SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
2569  CGF.EmitStmt(SubStmt);
2570  CGF.EmitBranch(ExitBB);
2571  ++CaseNumber;
2572  }
2573  } else {
2574  llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");
2575  CGF.EmitBlock(CaseBB);
2576  SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
2577  CGF.EmitStmt(CapturedStmt);
2578  CGF.EmitBranch(ExitBB);
2579  }
2580  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2581  };
2582 
2583  CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2584  if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
2585  // Emit implicit barrier to synchronize threads and avoid data races on
2586  // initialization of firstprivate variables and post-update of lastprivate
2587  // variables.
2588  CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2589  CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
2590  /*ForceSimpleCall=*/true);
2591  }
2592  CGF.EmitOMPPrivateClause(S, LoopScope);
2593  HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2594  CGF.EmitOMPReductionClauseInit(S, LoopScope);
2595  (void)LoopScope.Privatize();
2596  if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
2597  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
2598 
2599  // Emit static non-chunked loop.
2600  OpenMPScheduleTy ScheduleKind;
2601  ScheduleKind.Schedule = OMPC_SCHEDULE_static;
2602  CGOpenMPRuntime::StaticRTInput StaticInit(
2603  /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(),
2604  LB.getAddress(), UB.getAddress(), ST.getAddress());
2605  CGF.CGM.getOpenMPRuntime().emitForStaticInit(
2606  CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit);
2607  // UB = min(UB, GlobalUB);
2608  llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
2609  llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
2610  CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
2611  CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
2612  // IV = LB;
2613  CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
2614  // while (idx <= UB) { BODY; ++idx; }
2615  CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen,
2616  [](CodeGenFunction &) {});
2617  // Tell the runtime we are done.
2618  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2619  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2620  S.getDirectiveKind());
2621  };
2622  CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
2623  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
2624  // Emit post-update of the reduction variables if IsLastIter != 0.
2625  emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) {
2626  return CGF.Builder.CreateIsNotNull(
2627  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2628  });
2629 
2630  // Emit final copy of the lastprivate variables if IsLastIter != 0.
2631  if (HasLastprivates)
2633  S, /*NoFinals=*/false,
2634  CGF.Builder.CreateIsNotNull(
2635  CGF.EmitLoadOfScalar(IL, S.getBeginLoc())));
2636  };
2637 
2638  bool HasCancel = false;
2639  if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
2640  HasCancel = OSD->hasCancel();
2641  else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
2642  HasCancel = OPSD->hasCancel();
2643  OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
2644  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
2645  HasCancel);
2646  // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
2647  // clause. Otherwise the barrier will be generated by the codegen for the
2648  // directive.
2649  if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
2650  // Emit implicit barrier to synchronize threads and avoid data races on
2651  // initialization of firstprivate variables.
2652  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
2653  OMPD_unknown);
2654  }
2655 }
2656 
2658  {
2659  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2660  EmitSections(S);
2661  }
2662  // Emit an implicit barrier at the end.
2663  if (!S.getSingleClause<OMPNowaitClause>()) {
2664  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
2665  OMPD_sections);
2666  }
2667 }
2668 
2670  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2672  };
2673  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2674  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen,
2675  S.hasCancel());
2676 }
2677 
2679  llvm::SmallVector<const Expr *, 8> CopyprivateVars;
2682  llvm::SmallVector<const Expr *, 8> AssignmentOps;
2683  // Check if there are any 'copyprivate' clauses associated with this
2684  // 'single' construct.
2685  // Build a list of copyprivate variables along with helper expressions
2686  // (<source>, <destination>, <destination>=<source> expressions)
2687  for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
2688  CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
2689  DestExprs.append(C->destination_exprs().begin(),
2690  C->destination_exprs().end());
2691  SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
2692  AssignmentOps.append(C->assignment_ops().begin(),
2693  C->assignment_ops().end());
2694  }
2695  // Emit code for 'single' region along with 'copyprivate' clauses
2696  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2697  Action.Enter(CGF);
2698  OMPPrivateScope SingleScope(CGF);
2699  (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
2700  CGF.EmitOMPPrivateClause(S, SingleScope);
2701  (void)SingleScope.Privatize();
2703  };
2704  {
2705  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2706  CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getBeginLoc(),
2707  CopyprivateVars, DestExprs,
2708  SrcExprs, AssignmentOps);
2709  }
2710  // Emit an implicit barrier at the end (to avoid data race on firstprivate
2711  // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
2712  if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
2713  CGM.getOpenMPRuntime().emitBarrierCall(
2714  *this, S.getBeginLoc(),
2715  S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
2716  }
2717 }
2718 
2720  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2721  Action.Enter(CGF);
2723  };
2724  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2725  CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
2726 }
2727 
2729  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2730  Action.Enter(CGF);
2732  };
2733  const Expr *Hint = nullptr;
2734  if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
2735  Hint = HintClause->getHint();
2736  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2737  CGM.getOpenMPRuntime().emitCriticalRegion(*this,
2739  CodeGen, S.getBeginLoc(), Hint);
2740 }
2741 
2743  const OMPParallelForDirective &S) {
2744  // Emit directive as a combined directive that consists of two implicit
2745  // directives: 'parallel' with 'for' directive.
2746  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2747  Action.Enter(CGF);
2748  OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel());
2751  };
2752  emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
2754 }
2755 
2757  const OMPParallelForSimdDirective &S) {
2758  // Emit directive as a combined directive that consists of two implicit
2759  // directives: 'parallel' with 'for' directive.
2760  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2761  Action.Enter(CGF);
2764  };
2765  emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen,
2767 }
2768 
2770  const OMPParallelSectionsDirective &S) {
2771  // Emit directive as a combined directive that consists of two implicit
2772  // directives: 'parallel' with 'sections' directive.
2773  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2774  Action.Enter(CGF);
2775  CGF.EmitSections(S);
2776  };
2777  emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
2779 }
2780 
2782  const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
2783  const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
2784  OMPTaskDataTy &Data) {
2785  // Emit outlined function for task construct.
2786  const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
2787  auto I = CS->getCapturedDecl()->param_begin();
2788  auto PartId = std::next(I);
2789  auto TaskT = std::next(I, 4);
2790  // Check if the task is final
2791  if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
2792  // If the condition constant folds and can be elided, try to avoid emitting
2793  // the condition and the dead arm of the if/else.
2794  const Expr *Cond = Clause->getCondition();
2795  bool CondConstant;
2796  if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
2797  Data.Final.setInt(CondConstant);
2798  else
2799  Data.Final.setPointer(EvaluateExprAsBool(Cond));
2800  } else {
2801  // By default the task is not final.
2802  Data.Final.setInt(/*IntVal=*/false);
2803  }
2804  // Check if the task has 'priority' clause.
2805  if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
2806  const Expr *Prio = Clause->getPriority();
2807  Data.Priority.setInt(/*IntVal=*/true);
2808  Data.Priority.setPointer(EmitScalarConversion(
2809  EmitScalarExpr(Prio), Prio->getType(),
2810  getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
2811  Prio->getExprLoc()));
2812  }
2813  // The first function argument for tasks is a thread id, the second one is a
2814  // part id (0 for tied tasks, >=0 for untied task).
2815  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
2816  // Get list of private variables.
2817  for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
2818  auto IRef = C->varlist_begin();
2819  for (const Expr *IInit : C->private_copies()) {
2820  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2821  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2822  Data.PrivateVars.push_back(*IRef);
2823  Data.PrivateCopies.push_back(IInit);
2824  }
2825  ++IRef;
2826  }
2827  }
2828  EmittedAsPrivate.clear();
2829  // Get list of firstprivate variables.
2830  for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
2831  auto IRef = C->varlist_begin();
2832  auto IElemInitRef = C->inits().begin();
2833  for (const Expr *IInit : C->private_copies()) {
2834  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2835  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2836  Data.FirstprivateVars.push_back(*IRef);
2837  Data.FirstprivateCopies.push_back(IInit);
2838  Data.FirstprivateInits.push_back(*IElemInitRef);
2839  }
2840  ++IRef;
2841  ++IElemInitRef;
2842  }
2843  }
2844  // Get list of lastprivate variables (for taskloops).
2845  llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
2846  for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
2847  auto IRef = C->varlist_begin();
2848  auto ID = C->destination_exprs().begin();
2849  for (const Expr *IInit : C->private_copies()) {
2850  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2851  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2852  Data.LastprivateVars.push_back(*IRef);
2853  Data.LastprivateCopies.push_back(IInit);
2854  }
2855  LastprivateDstsOrigs.insert(
2856  {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
2857  cast<DeclRefExpr>(*IRef)});
2858  ++IRef;
2859  ++ID;
2860  }
2861  }
2864  for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
2865  auto IPriv = C->privates().begin();
2866  auto IRed = C->reduction_ops().begin();
2867  auto ILHS = C->lhs_exprs().begin();
2868  auto IRHS = C->rhs_exprs().begin();
2869  for (const Expr *Ref : C->varlists()) {
2870  Data.ReductionVars.emplace_back(Ref);
2871  Data.ReductionCopies.emplace_back(*IPriv);
2872  Data.ReductionOps.emplace_back(*IRed);
2873  LHSs.emplace_back(*ILHS);
2874  RHSs.emplace_back(*IRHS);
2875  std::advance(IPriv, 1);
2876  std::advance(IRed, 1);
2877  std::advance(ILHS, 1);
2878  std::advance(IRHS, 1);
2879  }
2880  }
2881  Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
2882  *this, S.getBeginLoc(), LHSs, RHSs, Data);
2883  // Build list of dependences.
2884  for (const auto *C : S.getClausesOfKind<OMPDependClause>())
2885  for (const Expr *IRef : C->varlists())
2886  Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
2887  auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
2888  CapturedRegion](CodeGenFunction &CGF,
2889  PrePostActionTy &Action) {
2890  // Set proper addresses for generated private copies.
2891  OMPPrivateScope Scope(CGF);
2892  if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
2893  !Data.LastprivateVars.empty()) {
2894  llvm::FunctionType *CopyFnTy = llvm::FunctionType::get(
2895  CGF.Builder.getVoidTy(), {CGF.Builder.getInt8PtrTy()}, true);
2896  enum { PrivatesParam = 2, CopyFnParam = 3 };
2897  llvm::Value *CopyFn = CGF.Builder.CreateLoad(
2898  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
2899  llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
2900  CS->getCapturedDecl()->getParam(PrivatesParam)));
2901  // Map privates.
2904  CallArgs.push_back(PrivatesPtr);
2905  for (const Expr *E : Data.PrivateVars) {
2906  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2907  Address PrivatePtr = CGF.CreateMemTemp(
2908  CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
2909  PrivatePtrs.emplace_back(VD, PrivatePtr);
2910  CallArgs.push_back(PrivatePtr.getPointer());
2911  }
2912  for (const Expr *E : Data.FirstprivateVars) {
2913  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2914  Address PrivatePtr =
2915  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2916  ".firstpriv.ptr.addr");
2917  PrivatePtrs.emplace_back(VD, PrivatePtr);
2918  CallArgs.push_back(PrivatePtr.getPointer());
2919  }
2920  for (const Expr *E : Data.LastprivateVars) {
2921  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2922  Address PrivatePtr =
2923  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2924  ".lastpriv.ptr.addr");
2925  PrivatePtrs.emplace_back(VD, PrivatePtr);
2926  CallArgs.push_back(PrivatePtr.getPointer());
2927  }
2928  CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
2929  CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
2930  for (const auto &Pair : LastprivateDstsOrigs) {
2931  const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
2932  DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD),
2933  /*RefersToEnclosingVariableOrCapture=*/
2934  CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr,
2935  Pair.second->getType(), VK_LValue,
2936  Pair.second->getExprLoc());
2937  Scope.addPrivate(Pair.first, [&CGF, &DRE]() {
2938  return CGF.EmitLValue(&DRE).getAddress();
2939  });
2940  }
2941  for (const auto &Pair : PrivatePtrs) {
2942  Address Replacement(CGF.Builder.CreateLoad(Pair.second),
2943  CGF.getContext().getDeclAlign(Pair.first));
2944  Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
2945  }
2946  }
2947  if (Data.Reductions) {
2948  OMPLexicalScope LexScope(CGF, S, CapturedRegion);
2950  Data.ReductionOps);
2951  llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
2952  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
2953  for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
2954  RedCG.emitSharedLValue(CGF, Cnt);
2955  RedCG.emitAggregateType(CGF, Cnt);
2956  // FIXME: This must removed once the runtime library is fixed.
2957  // Emit required threadprivate variables for
2958  // initializer/combiner/finalizer.
2959  CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
2960  RedCG, Cnt);
2961  Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
2962  CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
2963  Replacement =
2964  Address(CGF.EmitScalarConversion(
2965  Replacement.getPointer(), CGF.getContext().VoidPtrTy,
2966  CGF.getContext().getPointerType(
2967  Data.ReductionCopies[Cnt]->getType()),
2968  Data.ReductionCopies[Cnt]->getExprLoc()),
2969  Replacement.getAlignment());
2970  Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
2971  Scope.addPrivate(RedCG.getBaseDecl(Cnt),
2972  [Replacement]() { return Replacement; });
2973  }
2974  }
2975  // Privatize all private variables except for in_reduction items.
2976  (void)Scope.Privatize();
2977  SmallVector<const Expr *, 4> InRedVars;
2978  SmallVector<const Expr *, 4> InRedPrivs;
2980  SmallVector<const Expr *, 4> TaskgroupDescriptors;
2981  for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
2982  auto IPriv = C->privates().begin();
2983  auto IRed = C->reduction_ops().begin();
2984  auto ITD = C->taskgroup_descriptors().begin();
2985  for (const Expr *Ref : C->varlists()) {
2986  InRedVars.emplace_back(Ref);
2987  InRedPrivs.emplace_back(*IPriv);
2988  InRedOps.emplace_back(*IRed);
2989  TaskgroupDescriptors.emplace_back(*ITD);
2990  std::advance(IPriv, 1);
2991  std::advance(IRed, 1);
2992  std::advance(ITD, 1);
2993  }
2994  }
2995  // Privatize in_reduction items here, because taskgroup descriptors must be
2996  // privatized earlier.
2997  OMPPrivateScope InRedScope(CGF);
2998  if (!InRedVars.empty()) {
2999  ReductionCodeGen RedCG(InRedVars, InRedPrivs, InRedOps);
3000  for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
3001  RedCG.emitSharedLValue(CGF, Cnt);
3002  RedCG.emitAggregateType(CGF, Cnt);
3003  // The taskgroup descriptor variable is always implicit firstprivate and
3004  // privatized already during processing of the firstprivates.
3005  // FIXME: This must removed once the runtime library is fixed.
3006  // Emit required threadprivate variables for
3007  // initializer/combiner/finalizer.
3008  CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
3009  RedCG, Cnt);
3010  llvm::Value *ReductionsPtr =
3011  CGF.EmitLoadOfScalar(CGF.EmitLValue(TaskgroupDescriptors[Cnt]),
3012  TaskgroupDescriptors[Cnt]->getExprLoc());
3013  Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
3014  CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
3015  Replacement = Address(
3016  CGF.EmitScalarConversion(
3017  Replacement.getPointer(), CGF.getContext().VoidPtrTy,
3018  CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
3019  InRedPrivs[Cnt]->getExprLoc()),
3020  Replacement.getAlignment());
3021  Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
3022  InRedScope.addPrivate(RedCG.getBaseDecl(Cnt),
3023  [Replacement]() { return Replacement; });
3024  }
3025  }
3026  (void)InRedScope.Privatize();
3027 
3028  Action.Enter(CGF);
3029  BodyGen(CGF);
3030  };
3031  llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3032  S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied,
3033  Data.NumberOfParts);
3034  OMPLexicalScope Scope(*this, S);
3035  TaskGen(*this, OutlinedFn, Data);
3036 }
3037 
3038 static ImplicitParamDecl *
3040  QualType Ty, CapturedDecl *CD,
3041  SourceLocation Loc) {
3042  auto *OrigVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3044  auto *OrigRef = DeclRefExpr::Create(
3045  C, NestedNameSpecifierLoc(), SourceLocation(), OrigVD,
3046  /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
3047  auto *PrivateVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3049  auto *PrivateRef = DeclRefExpr::Create(
3050  C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD,
3051  /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
3052  QualType ElemType = C.getBaseElementType(Ty);
3053  auto *InitVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, ElemType,
3055  auto *InitRef = DeclRefExpr::Create(
3056  C, NestedNameSpecifierLoc(), SourceLocation(), InitVD,
3057  /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue);
3058  PrivateVD->setInitStyle(VarDecl::CInit);
3059  PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue,
3060  InitRef, /*BasePath=*/nullptr,
3061  VK_RValue));
3062  Data.FirstprivateVars.emplace_back(OrigRef);
3063  Data.FirstprivateCopies.emplace_back(PrivateRef);
3064  Data.FirstprivateInits.emplace_back(InitRef);
3065  return OrigVD;
3066 }
3067 
3069  const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,
3070  OMPTargetDataInfo &InputInfo) {
3071  // Emit outlined function for task construct.
3072  const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3073  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3074  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3075  auto I = CS->getCapturedDecl()->param_begin();
3076  auto PartId = std::next(I);
3077  auto TaskT = std::next(I, 4);
3078  OMPTaskDataTy Data;
3079  // The task is not final.
3080  Data.Final.setInt(/*IntVal=*/false);
3081  // Get list of firstprivate variables.
3082  for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
3083  auto IRef = C->varlist_begin();
3084  auto IElemInitRef = C->inits().begin();
3085  for (auto *IInit : C->private_copies()) {
3086  Data.FirstprivateVars.push_back(*IRef);
3087  Data.FirstprivateCopies.push_back(IInit);
3088  Data.FirstprivateInits.push_back(*IElemInitRef);
3089  ++IRef;
3090  ++IElemInitRef;
3091  }
3092  }
3093  OMPPrivateScope TargetScope(*this);
3094  VarDecl *BPVD = nullptr;
3095  VarDecl *PVD = nullptr;
3096  VarDecl *SVD = nullptr;
3097  if (InputInfo.NumberOfTargetItems > 0) {
3098  auto *CD = CapturedDecl::Create(
3099  getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0);
3100  llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);
3101  QualType BaseAndPointersType = getContext().getConstantArrayType(
3102  getContext().VoidPtrTy, ArrSize, ArrayType::Normal,
3103  /*IndexTypeQuals=*/0);
3105  getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
3107  getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
3108  QualType SizesType = getContext().getConstantArrayType(
3109  getContext().getSizeType(), ArrSize, ArrayType::Normal,
3110  /*IndexTypeQuals=*/0);
3111  SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD,
3112  S.getBeginLoc());
3113  TargetScope.addPrivate(
3114  BPVD, [&InputInfo]() { return InputInfo.BasePointersArray; });
3115  TargetScope.addPrivate(PVD,
3116  [&InputInfo]() { return InputInfo.PointersArray; });
3117  TargetScope.addPrivate(SVD,
3118  [&InputInfo]() { return InputInfo.SizesArray; });
3119  }
3120  (void)TargetScope.Privatize();
3121  // Build list of dependences.
3122  for (const auto *C : S.getClausesOfKind<OMPDependClause>())
3123  for (const Expr *IRef : C->varlists())
3124  Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
3125  auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD,
3126  &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
3127  // Set proper addresses for generated private copies.
3128  OMPPrivateScope Scope(CGF);
3129  if (!Data.FirstprivateVars.empty()) {
3130  llvm::FunctionType *CopyFnTy = llvm::FunctionType::get(
3131  CGF.Builder.getVoidTy(), {CGF.Builder.getInt8PtrTy()}, true);
3132  enum { PrivatesParam = 2, CopyFnParam = 3 };
3133  llvm::Value *CopyFn = CGF.Builder.CreateLoad(
3134  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
3135  llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
3136  CS->getCapturedDecl()->getParam(PrivatesParam)));
3137  // Map privates.
3140  CallArgs.push_back(PrivatesPtr);
3141  for (const Expr *E : Data.FirstprivateVars) {
3142  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3143  Address PrivatePtr =
3144  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
3145  ".firstpriv.ptr.addr");
3146  PrivatePtrs.emplace_back(VD, PrivatePtr);
3147  CallArgs.push_back(PrivatePtr.getPointer());
3148  }
3149  CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
3150  CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
3151  for (const auto &Pair : PrivatePtrs) {
3152  Address Replacement(CGF.Builder.CreateLoad(Pair.second),
3153  CGF.getContext().getDeclAlign(Pair.first));
3154  Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
3155  }
3156  }
3157  // Privatize all private variables except for in_reduction items.
3158  (void)Scope.Privatize();
3159  if (InputInfo.NumberOfTargetItems > 0) {
3160  InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
3161  CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0);
3162  InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
3163  CGF.GetAddrOfLocalVar(PVD), /*Index=*/0);
3164  InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
3165  CGF.GetAddrOfLocalVar(SVD), /*Index=*/0);
3166  }
3167 
3168  Action.Enter(CGF);
3169  OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);
3170  BodyGen(CGF);
3171  };
3172  llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3173  S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true,
3174  Data.NumberOfParts);
3175  llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
3176  IntegerLiteral IfCond(getContext(), TrueOrFalse,
3177  getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3178  SourceLocation());
3179 
3180  CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn,
3181  SharedsTy, CapturedStruct, &IfCond, Data);
3182 }
3183 
3185  // Emit outlined function for task construct.
3186  const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3187  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3188  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3189  const Expr *IfCond = nullptr;
3190  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3191  if (C->getNameModifier() == OMPD_unknown ||
3192  C->getNameModifier() == OMPD_task) {
3193  IfCond = C->getCondition();
3194  break;
3195  }
3196  }
3197 
3198  OMPTaskDataTy Data;
3199  // Check if we should emit tied or untied task.
3200  Data.Tied = !S.getSingleClause<OMPUntiedClause>();
3201  auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
3202  CGF.EmitStmt(CS->getCapturedStmt());
3203  };
3204  auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
3205  IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
3206  const OMPTaskDataTy &Data) {
3207  CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn,
3208  SharedsTy, CapturedStruct, IfCond,
3209  Data);
3210  };
3211  EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data);
3212 }
3213 
3215  const OMPTaskyieldDirective &S) {
3216  CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getBeginLoc());
3217 }
3218 
3220  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_barrier);
3221 }
3222 
3224  CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc());
3225 }
3226 
3228  const OMPTaskgroupDirective &S) {
3229  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3230  Action.Enter(CGF);
3231  if (const Expr *E = S.getReductionRef()) {
3234  OMPTaskDataTy Data;
3235  for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
3236  auto IPriv = C->privates().begin();
3237  auto IRed = C->reduction_ops().begin();
3238  auto ILHS = C->lhs_exprs().begin();
3239  auto IRHS = C->rhs_exprs().begin();
3240  for (const Expr *Ref : C->varlists()) {
3241  Data.ReductionVars.emplace_back(Ref);
3242  Data.ReductionCopies.emplace_back(*IPriv);
3243  Data.ReductionOps.emplace_back(*IRed);
3244  LHSs.emplace_back(*ILHS);
3245  RHSs.emplace_back(*IRHS);
3246  std::advance(IPriv, 1);
3247  std::advance(IRed, 1);
3248  std::advance(ILHS, 1);
3249  std::advance(IRHS, 1);
3250  }
3251  }
3252  llvm::Value *ReductionDesc =
3253  CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(),
3254  LHSs, RHSs, Data);
3255  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3256  CGF.EmitVarDecl(*VD);
3257  CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),
3258  /*Volatile=*/false, E->getType());
3259  }
3261  };
3262  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3263  CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getBeginLoc());
3264 }
3265 
3267  CGM.getOpenMPRuntime().emitFlush(
3268  *this,
3269  [&S]() -> ArrayRef<const Expr *> {
3270  if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>())
3271  return llvm::makeArrayRef(FlushClause->varlist_begin(),
3272  FlushClause->varlist_end());
3273  return llvm::None;
3274  }(),
3275  S.getBeginLoc());
3276 }
3277 
3279  const CodeGenLoopTy &CodeGenLoop,
3280  Expr *IncExpr) {
3281  // Emit the loop iteration variable.
3282  const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3283  const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3284  EmitVarDecl(*IVDecl);
3285 
3286  // Emit the iterations count variable.
3287  // If it is not a variable, Sema decided to calculate iterations count on each
3288  // iteration (e.g., it is foldable into a constant).
3289  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3290  EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3291  // Emit calculation of the iterations count.
3292  EmitIgnoredExpr(S.getCalcLastIteration());
3293  }
3294 
3295  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3296 
3297  bool HasLastprivateClause = false;
3298  // Check pre-condition.
3299  {
3300  OMPLoopScope PreInitScope(*this, S);
3301  // Skip the entire loop if we don't meet the precondition.
3302  // If the condition constant folds and can be elided, avoid emitting the
3303  // whole loop.
3304  bool CondConstant;
3305  llvm::BasicBlock *ContBlock = nullptr;
3306  if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
3307  if (!CondConstant)
3308  return;
3309  } else {
3310  llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
3311  ContBlock = createBasicBlock("omp.precond.end");
3312  emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
3313  getProfileCount(&S));
3314  EmitBlock(ThenBlock);
3315  incrementProfileCounter(&S);
3316  }
3317 
3318  emitAlignedClause(*this, S);
3319  // Emit 'then' code.
3320  {
3321  // Emit helper vars inits.
3322 
3323  LValue LB = EmitOMPHelperVar(
3324  *this, cast<DeclRefExpr>(
3327  : S.getLowerBoundVariable())));
3328  LValue UB = EmitOMPHelperVar(
3329  *this, cast<DeclRefExpr>(
3332  : S.getUpperBoundVariable())));
3333  LValue ST =
3334  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
3335  LValue IL =
3336  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
3337 
3338  OMPPrivateScope LoopScope(*this);
3339  if (EmitOMPFirstprivateClause(S, LoopScope)) {
3340  // Emit implicit barrier to synchronize threads and avoid data races
3341  // on initialization of firstprivate variables and post-update of
3342  // lastprivate variables.
3343  CGM.getOpenMPRuntime().emitBarrierCall(
3344  *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
3345  /*ForceSimpleCall=*/true);
3346  }
3347  EmitOMPPrivateClause(S, LoopScope);
3351  EmitOMPReductionClauseInit(S, LoopScope);
3352  HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
3353  EmitOMPPrivateLoopCounters(S, LoopScope);
3354  (void)LoopScope.Privatize();
3356  CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
3357 
3358  // Detect the distribute schedule kind and chunk.
3359  llvm::Value *Chunk = nullptr;
3361  if (const auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
3362  ScheduleKind = C->getDistScheduleKind();
3363  if (const Expr *Ch = C->getChunkSize()) {
3364  Chunk = EmitScalarExpr(Ch);
3365  Chunk = EmitScalarConversion(Chunk, Ch->getType(),
3367  S.getBeginLoc());
3368  }
3369  } else {
3370  // Default behaviour for dist_schedule clause.
3371  CGM.getOpenMPRuntime().getDefaultDistScheduleAndChunk(
3372  *this, S, ScheduleKind, Chunk);
3373  }
3374  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3375  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3376 
3377  // OpenMP [2.10.8, distribute Construct, Description]
3378  // If dist_schedule is specified, kind must be static. If specified,
3379  // iterations are divided into chunks of size chunk_size, chunks are
3380  // assigned to the teams of the league in a round-robin fashion in the
3381  // order of the team number. When no chunk_size is specified, the
3382  // iteration space is divided into chunks that are approximately equal
3383  // in size, and at most one chunk is distributed to each team of the
3384  // league. The size of the chunks is unspecified in this case.
3385  bool StaticChunked = RT.isStaticChunked(
3386  ScheduleKind, /* Chunked */ Chunk != nullptr) &&
3388  if (RT.isStaticNonchunked(ScheduleKind,
3389  /* Chunked */ Chunk != nullptr) ||
3390  StaticChunked) {
3392  EmitOMPSimdInit(S, /*IsMonotonic=*/true);
3393  CGOpenMPRuntime::StaticRTInput StaticInit(
3394  IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(),
3395  LB.getAddress(), UB.getAddress(), ST.getAddress(),
3396  StaticChunked ? Chunk : nullptr);
3397  RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind,
3398  StaticInit);
3399  JumpDest LoopExit =
3400  getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
3401  // UB = min(UB, GlobalUB);
3404  : S.getEnsureUpperBound());
3405  // IV = LB;
3407  ? S.getCombinedInit()
3408  : S.getInit());
3409 
3410  const Expr *Cond =
3412  ? S.getCombinedCond()
3413  : S.getCond();
3414 
3415  if (StaticChunked)
3416  Cond = S.getCombinedDistCond();
3417 
3418  // For static unchunked schedules generate:
3419  //
3420  // 1. For distribute alone, codegen
3421  // while (idx <= UB) {
3422  // BODY;
3423  // ++idx;
3424  // }
3425  //
3426  // 2. When combined with 'for' (e.g. as in 'distribute parallel for')
3427  // while (idx <= UB) {
3428  // <CodeGen rest of pragma>(LB, UB);
3429  // idx += ST;
3430  // }
3431  //
3432  // For static chunk one schedule generate:
3433  //
3434  // while (IV <= GlobalUB) {
3435  // <CodeGen rest of pragma>(LB, UB);
3436  // LB += ST;
3437  // UB += ST;
3438  // UB = min(UB, GlobalUB);
3439  // IV = LB;
3440  // }
3441  //
3442  EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), Cond, IncExpr,
3443  [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3444  CodeGenLoop(CGF, S, LoopExit);
3445  },
3446  [&S, StaticChunked](CodeGenFunction &CGF) {
3447  if (StaticChunked) {
3448  CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());
3449  CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());
3450  CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());
3451  CGF.EmitIgnoredExpr(S.getCombinedInit());
3452  }
3453  });
3454  EmitBlock(LoopExit.getBlock());
3455  // Tell the runtime we are done.
3456  RT.emitForStaticFinish(*this, S.getBeginLoc(), S.getDirectiveKind());
3457  } else {
3458  // Emit the outer loop, which requests its work chunk [LB..UB] from
3459  // runtime and runs the inner loop to process it.
3460  const OMPLoopArguments LoopArguments = {
3461  LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(),
3462  Chunk};
3463  EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
3464  CodeGenLoop);
3465  }
3467  EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
3468  return CGF.Builder.CreateIsNotNull(
3469  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3470  });
3471  }
3475  EmitOMPReductionClauseFinal(S, OMPD_simd);
3476  // Emit post-update of the reduction variables if IsLastIter != 0.
3478  *this, S, [IL, &S](CodeGenFunction &CGF) {
3479  return CGF.Builder.CreateIsNotNull(
3480  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3481  });
3482  }
3483  // Emit final copy of the lastprivate variables if IsLastIter != 0.
3484  if (HasLastprivateClause) {
3485  EmitOMPLastprivateClauseFinal(
3486  S, /*NoFinals=*/false,
3487  Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
3488  }
3489  }
3490 
3491  // We're now done with the loop, so jump to the continuation block.
3492  if (ContBlock) {
3493  EmitBranch(ContBlock);
3494  EmitBlock(ContBlock, true);
3495  }
3496  }
3497 }
3498 
3500  const OMPDistributeDirective &S) {
3501  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3503  };
3504  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3505  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3506 }
3507 
3508 static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM,
3509  const CapturedStmt *S) {
3510  CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
3512  CGF.CapturedStmtInfo = &CapStmtInfo;
3513  llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S);
3514  Fn->setDoesNotRecurse();
3515  return Fn;
3516 }
3517 
3519  if (S.hasClausesOfKind<OMPDependClause>()) {
3520  assert(!S.getAssociatedStmt() &&
3521  "No associated statement must be in ordered depend construct.");
3522  for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
3523  CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
3524  return;
3525  }
3526  const auto *C = S.getSingleClause<OMPSIMDClause>();
3527  auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
3528  PrePostActionTy &Action) {
3529  const CapturedStmt *CS = S.getInnermostCapturedStmt();
3530  if (C) {
3532  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
3533  llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS);
3534  CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
3535  OutlinedFn, CapturedVars);
3536  } else {
3537  Action.Enter(CGF);
3538  CGF.EmitStmt(CS->getCapturedStmt());
3539  }
3540  };
3541  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3542  CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getBeginLoc(), !C);
3543 }
3544 
3546  QualType SrcType, QualType DestType,
3547  SourceLocation Loc) {
3548  assert(CGF.hasScalarEvaluationKind(DestType) &&
3549  "DestType must have scalar evaluation kind.");
3550  assert(!Val.isAggregate() && "Must be a scalar or complex.");
3551  return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
3552  DestType, Loc)
3554  Val.getComplexVal(), SrcType, DestType, Loc);
3555 }
3556 
3559  QualType DestType, SourceLocation Loc) {
3560  assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
3561  "DestType must have complex evaluation kind.");
3562  CodeGenFunction::ComplexPairTy ComplexVal;
3563  if (Val.isScalar()) {
3564  // Convert the input element to the element type of the complex.
3565  QualType DestElementType =
3566  DestType->castAs<ComplexType>()->getElementType();
3567  llvm::Value *ScalarVal = CGF.EmitScalarConversion(
3568  Val.getScalarVal(), SrcType, DestElementType, Loc);
3569  ComplexVal = CodeGenFunction::ComplexPairTy(
3570  ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
3571  } else {
3572  assert(Val.isComplex() && "Must be a scalar or complex.");
3573  QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
3574  QualType DestElementType =
3575  DestType->castAs<ComplexType>()->getElementType();
3576  ComplexVal.first = CGF.EmitScalarConversion(
3577  Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
3578  ComplexVal.second = CGF.EmitScalarConversion(
3579  Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
3580  }
3581  return ComplexVal;
3582 }
3583 
3584 static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst,
3585  LValue LVal, RValue RVal) {
3586  if (LVal.isGlobalReg()) {
3587  CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
3588  } else {
3589  CGF.EmitAtomicStore(RVal, LVal,
3590  IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3591  : llvm::AtomicOrdering::Monotonic,
3592  LVal.isVolatile(), /*IsInit=*/false);
3593  }
3594 }
3595 
3597  QualType RValTy, SourceLocation Loc) {
3598  switch (getEvaluationKind(LVal.getType())) {
3599  case TEK_Scalar:
3600  EmitStoreThroughLValue(RValue::get(convertToScalarValue(
3601  *this, RVal, RValTy, LVal.getType(), Loc)),
3602  LVal);
3603  break;
3604  case TEK_Complex:
3605  EmitStoreOfComplex(
3606  convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,
3607  /*isInit=*/false);
3608  break;
3609  case TEK_Aggregate:
3610  llvm_unreachable("Must be a scalar or complex.");
3611  }
3612 }
3613 
3614 static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst,
3615  const Expr *X, const Expr *V,
3616  SourceLocation Loc) {
3617  // v = x;
3618  assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
3619  assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
3620  LValue XLValue = CGF.EmitLValue(X);
3621  LValue VLValue = CGF.EmitLValue(V);
3622  RValue Res = XLValue.isGlobalReg()
3623  ? CGF.EmitLoadOfLValue(XLValue, Loc)
3624  : CGF.EmitAtomicLoad(
3625  XLValue, Loc,
3626  IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3627  : llvm::AtomicOrdering::Monotonic,
3628  XLValue.isVolatile());
3629  // OpenMP, 2.12.6, atomic Construct
3630  // Any atomic construct with a seq_cst clause forces the atomically
3631  // performed operation to include an implicit flush operation without a
3632  // list.
3633  if (IsSeqCst)
3634  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3635  CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
3636 }
3637 
3638 static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst,
3639  const Expr *X, const Expr *E,
3640  SourceLocation Loc) {
3641  // x = expr;
3642  assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
3643  emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
3644  // OpenMP, 2.12.6, atomic Construct
3645  // Any atomic construct with a seq_cst clause forces the atomically
3646  // performed operation to include an implicit flush operation without a
3647  // list.
3648  if (IsSeqCst)
3649  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3650 }
3651 
3652 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
3653  RValue Update,
3654  BinaryOperatorKind BO,
3655  llvm::AtomicOrdering AO,
3656  bool IsXLHSInRHSPart) {
3657  ASTContext &Context = CGF.getContext();
3658  // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
3659  // expression is simple and atomic is allowed for the given type for the
3660  // target platform.
3661  if (BO == BO_Comma || !Update.isScalar() ||
3662  !Update.getScalarVal()->getType()->isIntegerTy() ||
3663  !X.isSimple() || (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
3664  (Update.getScalarVal()->getType() !=
3665  X.getAddress().getElementType())) ||
3666  !X.getAddress().getElementType()->isIntegerTy() ||
3667  !Context.getTargetInfo().hasBuiltinAtomic(
3668  Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
3669  return std::make_pair(false, RValue::get(nullptr));
3670 
3671  llvm::AtomicRMWInst::BinOp RMWOp;
3672  switch (BO) {
3673  case BO_Add:
3674  RMWOp = llvm::AtomicRMWInst::Add;
3675  break;
3676  case BO_Sub:
3677  if (!IsXLHSInRHSPart)
3678  return std::make_pair(false, RValue::get(nullptr));
3679  RMWOp = llvm::AtomicRMWInst::Sub;
3680  break;
3681  case BO_And:
3682  RMWOp = llvm::AtomicRMWInst::And;
3683  break;
3684  case BO_Or:
3685  RMWOp = llvm::AtomicRMWInst::Or;
3686  break;
3687  case BO_Xor:
3688  RMWOp = llvm::AtomicRMWInst::Xor;
3689  break;
3690  case BO_LT:
3692  ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
3693  : llvm::AtomicRMWInst::Max)
3694  : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
3695  : llvm::AtomicRMWInst::UMax);
3696  break;
3697  case BO_GT:
3699  ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
3700  : llvm::AtomicRMWInst::Min)
3701  : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
3702  : llvm::AtomicRMWInst::UMin);
3703  break;
3704  case BO_Assign:
3705  RMWOp = llvm::AtomicRMWInst::Xchg;
3706  break;
3707  case BO_Mul:
3708  case BO_Div:
3709  case BO_Rem:
3710  case BO_Shl:
3711  case BO_Shr:
3712  case BO_LAnd:
3713  case BO_LOr:
3714  return std::make_pair(false, RValue::get(nullptr));
3715  case BO_PtrMemD:
3716  case BO_PtrMemI:
3717  case BO_LE:
3718  case BO_GE:
3719  case BO_EQ:
3720  case BO_NE:
3721  case BO_Cmp:
3722  case BO_AddAssign:
3723  case BO_SubAssign:
3724  case BO_AndAssign:
3725  case BO_OrAssign:
3726  case BO_XorAssign:
3727  case BO_MulAssign:
3728  case BO_DivAssign:
3729  case BO_RemAssign:
3730  case BO_ShlAssign:
3731  case BO_ShrAssign:
3732  case BO_Comma:
3733  llvm_unreachable("Unsupported atomic update operation");
3734  }
3735  llvm::Value *UpdateVal = Update.getScalarVal();
3736  if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
3737  UpdateVal = CGF.Builder.CreateIntCast(
3738  IC, X.getAddress().getElementType(),
3740  }
3741  llvm::Value *Res =
3742  CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO);
3743  return std::make_pair(true, RValue::get(Res));
3744 }
3745 
3747  LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3748  llvm::AtomicOrdering AO, SourceLocation Loc,
3749  const llvm::function_ref<RValue(RValue)> CommonGen) {
3750  // Update expressions are allowed to have the following forms:
3751  // x binop= expr; -> xrval + expr;
3752  // x++, ++x -> xrval + 1;
3753  // x--, --x -> xrval - 1;
3754  // x = x binop expr; -> xrval binop expr
3755  // x = expr Op x; - > expr binop xrval;
3756  auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);
3757  if (!Res.first) {
3758  if (X.isGlobalReg()) {
3759  // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
3760  // 'xrval'.
3761  EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
3762  } else {
3763  // Perform compare-and-swap procedure.
3764  EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
3765  }
3766  }
3767  return Res;
3768 }
3769 
3770 static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst,
3771  const Expr *X, const Expr *E,
3772  const Expr *UE, bool IsXLHSInRHSPart,
3773  SourceLocation Loc) {
3774  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3775  "Update expr in 'atomic update' must be a binary operator.");
3776  const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3777  // Update expressions are allowed to have the following forms:
3778  // x binop= expr; -> xrval + expr;
3779  // x++, ++x -> xrval + 1;
3780  // x--, --x -> xrval - 1;
3781  // x = x binop expr; -> xrval binop expr
3782  // x = expr Op x; - > expr binop xrval;
3783  assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
3784  LValue XLValue = CGF.EmitLValue(X);
3785  RValue ExprRValue = CGF.EmitAnyExpr(E);
3786  llvm::AtomicOrdering AO = IsSeqCst
3787  ? llvm::AtomicOrdering::SequentiallyConsistent
3788  : llvm::AtomicOrdering::Monotonic;
3789  const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3790  const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3791  const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3792  const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3793  auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {
3794  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3795  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3796  return CGF.EmitAnyExpr(UE);
3797  };
3799  XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3800  // OpenMP, 2.12.6, atomic Construct
3801  // Any atomic construct with a seq_cst clause forces the atomically
3802  // performed operation to include an implicit flush operation without a
3803  // list.
3804  if (IsSeqCst)
3805  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3806 }
3807 
3809  QualType SourceType, QualType ResType,
3810  SourceLocation Loc) {
3811  switch (CGF.getEvaluationKind(ResType)) {
3812  case TEK_Scalar:
3813  return RValue::get(
3814  convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
3815  case TEK_Complex: {
3816  auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
3817  return RValue::getComplex(Res.first, Res.second);
3818  }
3819  case TEK_Aggregate:
3820  break;
3821  }
3822  llvm_unreachable("Must be a scalar or complex.");
3823 }
3824 
3825 static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst,
3826  bool IsPostfixUpdate, const Expr *V,
3827  const Expr *X, const Expr *E,
3828  const Expr *UE, bool IsXLHSInRHSPart,
3829  SourceLocation Loc) {
3830  assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
3831  assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
3832  RValue NewVVal;
3833  LValue VLValue = CGF.EmitLValue(V);
3834  LValue XLValue = CGF.EmitLValue(X);
3835  RValue ExprRValue = CGF.EmitAnyExpr(E);
3836  llvm::AtomicOrdering AO = IsSeqCst
3837  ? llvm::AtomicOrdering::SequentiallyConsistent
3838  : llvm::AtomicOrdering::Monotonic;
3839  QualType NewVValType;
3840  if (UE) {
3841  // 'x' is updated with some additional value.
3842  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3843  "Update expr in 'atomic capture' must be a binary operator.");
3844  const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3845  // Update expressions are allowed to have the following forms:
3846  // x binop= expr; -> xrval + expr;
3847  // x++, ++x -> xrval + 1;
3848  // x--, --x -> xrval - 1;
3849  // x = x binop expr; -> xrval binop expr
3850  // x = expr Op x; - > expr binop xrval;
3851  const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3852  const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3853  const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3854  NewVValType = XRValExpr->getType();
3855  const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3856  auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
3857  IsPostfixUpdate](RValue XRValue) {
3858  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3859  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3860  RValue Res = CGF.EmitAnyExpr(UE);
3861  NewVVal = IsPostfixUpdate ? XRValue : Res;
3862  return Res;
3863  };
3864  auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3865  XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3866  if (Res.first) {
3867  // 'atomicrmw' instruction was generated.
3868  if (IsPostfixUpdate) {
3869  // Use old value from 'atomicrmw'.
3870  NewVVal = Res.second;
3871  } else {
3872  // 'atomicrmw' does not provide new value, so evaluate it using old
3873  // value of 'x'.
3874  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3875  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
3876  NewVVal = CGF.EmitAnyExpr(UE);
3877  }
3878  }
3879  } else {
3880  // 'x' is simply rewritten with some 'expr'.
3881  NewVValType = X->getType().getNonReferenceType();
3882  ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
3883  X->getType().getNonReferenceType(), Loc);
3884  auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) {
3885  NewVVal = XRValue;
3886  return ExprRValue;
3887  };
3888  // Try to perform atomicrmw xchg, otherwise simple exchange.
3889  auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3890  XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
3891  Loc, Gen);
3892  if (Res.first) {
3893  // 'atomicrmw' instruction was generated.
3894  NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
3895  }
3896  }
3897  // Emit post-update store to 'v' of old/new 'x' value.
3898  CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
3899  // OpenMP, 2.12.6, atomic Construct
3900  // Any atomic construct with a seq_cst clause forces the atomically
3901  // performed operation to include an implicit flush operation without a
3902  // list.
3903  if (IsSeqCst)
3904  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3905 }
3906 
3908  bool IsSeqCst, bool IsPostfixUpdate,
3909  const Expr *X, const Expr *V, const Expr *E,
3910  const Expr *UE, bool IsXLHSInRHSPart,
3911  SourceLocation Loc) {
3912  switch (Kind) {
3913  case OMPC_read:
3914  emitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc);
3915  break;
3916  case OMPC_write:
3917  emitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc);
3918  break;
3919  case OMPC_unknown:
3920  case OMPC_update:
3921  emitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc);
3922  break;
3923  case OMPC_capture:
3924  emitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE,
3925  IsXLHSInRHSPart, Loc);
3926  break;
3927  case OMPC_if:
3928  case OMPC_final:
3929  case OMPC_num_threads:
3930  case OMPC_private:
3931  case OMPC_firstprivate:
3932  case OMPC_lastprivate:
3933  case OMPC_reduction:
3934  case OMPC_task_reduction:
3935  case OMPC_in_reduction:
3936  case OMPC_safelen:
3937  case OMPC_simdlen:
3938  case OMPC_collapse:
3939  case OMPC_default:
3940  case OMPC_seq_cst:
3941  case OMPC_shared:
3942  case OMPC_linear:
3943  case OMPC_aligned:
3944  case OMPC_copyin:
3945  case OMPC_copyprivate:
3946  case OMPC_flush:
3947  case OMPC_proc_bind:
3948  case OMPC_schedule:
3949  case OMPC_ordered:
3950  case OMPC_nowait:
3951  case OMPC_untied:
3952  case OMPC_threadprivate:
3953  case OMPC_depend:
3954  case OMPC_mergeable:
3955  case OMPC_device:
3956  case OMPC_threads:
3957  case OMPC_simd:
3958  case OMPC_map:
3959  case OMPC_num_teams:
3960  case OMPC_thread_limit:
3961  case OMPC_priority:
3962  case OMPC_grainsize:
3963  case OMPC_nogroup:
3964  case OMPC_num_tasks:
3965  case OMPC_hint:
3966  case OMPC_dist_schedule:
3967  case OMPC_defaultmap:
3968  case OMPC_uniform:
3969  case OMPC_to:
3970  case OMPC_from:
3971  case OMPC_use_device_ptr:
3972  case OMPC_is_device_ptr:
3973  case OMPC_unified_address:
3974  case OMPC_unified_shared_memory:
3975  case OMPC_reverse_offload:
3976  case OMPC_dynamic_allocators:
3977  case OMPC_atomic_default_mem_order:
3978  llvm_unreachable("Clause is not allowed in 'omp atomic'.");
3979  }
3980 }
3981 
3983  bool IsSeqCst = S.getSingleClause<OMPSeqCstClause>();
3985  for (const OMPClause *C : S.clauses()) {
3986  // Find first clause (skip seq_cst clause, if it is first).
3987  if (C->getClauseKind() != OMPC_seq_cst) {
3988  Kind = C->getClauseKind();
3989  break;
3990  }
3991  }
3992 
3993  const Stmt *CS = S.getInnermostCapturedStmt()->IgnoreContainers();
3994  if (const auto *FE = dyn_cast<FullExpr>(CS))
3995  enterFullExpression(FE);
3996  // Processing for statements under 'atomic capture'.
3997  if (const auto *Compound = dyn_cast<CompoundStmt>(CS)) {
3998  for (const Stmt *C : Compound->body()) {
3999  if (const auto *FE = dyn_cast<FullExpr>(C))
4000  enterFullExpression(FE);
4001  }
4002  }
4003 
4004  auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF,
4005  PrePostActionTy &) {
4006  CGF.EmitStopPoint(CS);
4007  emitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(),
4008  S.getV(), S.getExpr(), S.getUpdateExpr(),
4009  S.isXLHSInRHSPart(), S.getBeginLoc());
4010  };
4011  OMPLexicalScope Scope(*this, S, OMPD_unknown);
4012  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen);
4013 }
4014 
4016  const OMPExecutableDirective &S,
4017  const RegionCodeGenTy &CodeGen) {
4019  CodeGenModule &CGM = CGF.CGM;
4020 
4021  // On device emit this construct as inlined code.
4022  if (CGM.getLangOpts().OpenMPIsDevice) {
4023  OMPLexicalScope Scope(CGF, S, OMPD_target);
4024  CGM.getOpenMPRuntime().emitInlinedDirective(
4025  CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4027  });
4028  return;
4029  }
4030 
4031  llvm::Function *Fn = nullptr;
4032  llvm::Constant *FnID = nullptr;
4033 
4034  const Expr *IfCond = nullptr;
4035  // Check for the at most one if clause associated with the target region.
4036  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4037  if (C->getNameModifier() == OMPD_unknown ||
4038  C->getNameModifier() == OMPD_target) {
4039  IfCond = C->getCondition();
4040  break;
4041  }
4042  }
4043 
4044  // Check if we have any device clause associated with the directive.
4045  const Expr *Device = nullptr;
4046  if (auto *C = S.getSingleClause<OMPDeviceClause>())
4047  Device = C->getDevice();
4048 
4049  // Check if we have an if clause whose conditional always evaluates to false
4050  // or if we do not have any targets specified. If so the target region is not
4051  // an offload entry point.
4052  bool IsOffloadEntry = true;
4053  if (IfCond) {
4054  bool Val;
4055  if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
4056  IsOffloadEntry = false;
4057  }
4058  if (CGM.getLangOpts().OMPTargetTriples.empty())
4059  IsOffloadEntry = false;
4060 
4061  assert(CGF.CurFuncDecl && "No parent declaration for target region!");
4062  StringRef ParentName;
4063  // In case we have Ctors/Dtors we use the complete type variant to produce
4064  // the mangling of the device outlined kernel.
4065  if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
4066  ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));
4067  else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
4068  ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));
4069  else
4070  ParentName =
4071  CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));
4072 
4073  // Emit target region as a standalone region.
4074  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
4075  IsOffloadEntry, CodeGen);
4076  OMPLexicalScope Scope(CGF, S, OMPD_task);
4077  auto &&SizeEmitter = [](CodeGenFunction &CGF, const OMPLoopDirective &D) {
4078  OMPLoopScope(CGF, D);
4079  // Emit calculation of the iterations count.
4080  llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations());
4081  NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty,
4082  /*IsSigned=*/false);
4083  return NumIterations;
4084  };
4085  if (IsOffloadEntry)
4086  CGM.getOpenMPRuntime().emitTargetNumIterationsCall(CGF, S, Device,
4087  SizeEmitter);
4088  CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device);
4089 }
4090 
4092  PrePostActionTy &Action) {
4093  Action.Enter(CGF);
4094  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4095  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4096  CGF.EmitOMPPrivateClause(S, PrivateScope);
4097  (void)PrivateScope.Privatize();
4099  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4100 
4101  CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
4102 }
4103 
4105  StringRef ParentName,
4106  const OMPTargetDirective &S) {
4107  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4108  emitTargetRegion(CGF, S, Action);
4109  };
4110  llvm::Function *Fn;
4111  llvm::Constant *Addr;
4112  // Emit target region as a standalone region.
4113  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4114  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4115  assert(Fn && Addr && "Target device function emission failed.");
4116 }
4117 
4119  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4120  emitTargetRegion(CGF, S, Action);
4121  };
4122  emitCommonOMPTargetDirective(*this, S, CodeGen);
4123 }
4124 
4126  const OMPExecutableDirective &S,
4127  OpenMPDirectiveKind InnermostKind,
4128  const RegionCodeGenTy &CodeGen) {
4129  const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
4130  llvm::Function *OutlinedFn =
4131  CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
4132  S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
4133 
4134  const auto *NT = S.getSingleClause<OMPNumTeamsClause>();
4135  const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
4136  if (NT || TL) {
4137  const Expr *NumTeams = NT ? NT->getNumTeams() : nullptr;
4138  const Expr *ThreadLimit = TL ? TL->getThreadLimit() : nullptr;
4139 
4140  CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
4141  S.getBeginLoc());
4142  }
4143 
4144  OMPTeamsScope Scope(CGF, S);
4146  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
4147  CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn,
4148  CapturedVars);
4149 }
4150 
4152  // Emit teams region as a standalone region.
4153  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4154  Action.Enter(CGF);
4155  OMPPrivateScope PrivateScope(CGF);
4156  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4157  CGF.EmitOMPPrivateClause(S, PrivateScope);
4158  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4159  (void)PrivateScope.Privatize();
4160  CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
4161  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4162  };
4163  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4165  [](CodeGenFunction &) { return nullptr; });
4166 }
4167 
4169  const OMPTargetTeamsDirective &S) {
4170  auto *CS = S.getCapturedStmt(OMPD_teams);
4171  Action.Enter(CGF);
4172  // Emit teams region as a standalone region.
4173  auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4174  Action.Enter(CGF);
4175  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4176  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4177  CGF.EmitOMPPrivateClause(S, PrivateScope);
4178  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4179  (void)PrivateScope.Privatize();
4181  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4182  CGF.EmitStmt(CS->getCapturedStmt());
4183  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4184  };
4185  emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
4187  [](CodeGenFunction &) { return nullptr; });
4188 }
4189 
4191  CodeGenModule &CGM, StringRef ParentName,
4192  const OMPTargetTeamsDirective &S) {
4193  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4194  emitTargetTeamsRegion(CGF, Action, S);
4195  };
4196  llvm::Function *Fn;
4197  llvm::Constant *Addr;
4198  // Emit target region as a standalone region.
4199  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4200  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4201  assert(Fn && Addr && "Target device function emission failed.");
4202 }
4203 
4205  const OMPTargetTeamsDirective &S) {
4206  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4207  emitTargetTeamsRegion(CGF, Action, S);
4208  };
4209  emitCommonOMPTargetDirective(*this, S, CodeGen);
4210 }
4211 
4212 static void
4215  Action.Enter(CGF);
4216  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4218  };
4219 
4220  // Emit teams region as a standalone region.
4221  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4222  PrePostActionTy &Action) {
4223  Action.Enter(CGF);
4224  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4225  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4226  (void)PrivateScope.Privatize();
4227  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4228  CodeGenDistribute);
4229  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4230  };
4231  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
4233  [](CodeGenFunction &) { return nullptr; });
4234 }
4235 
4237  CodeGenModule &CGM, StringRef ParentName,
4239  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4240  emitTargetTeamsDistributeRegion(CGF, Action, S);
4241  };
4242  llvm::Function *Fn;
4243  llvm::Constant *Addr;
4244  // Emit target region as a standalone region.
4245  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4246  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4247  assert(Fn && Addr && "Target device function emission failed.");
4248 }
4249 
4252  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4253  emitTargetTeamsDistributeRegion(CGF, Action, S);
4254  };
4255  emitCommonOMPTargetDirective(*this, S, CodeGen);
4256 }
4257 
4259  CodeGenFunction &CGF, PrePostActionTy &Action,
4261  Action.Enter(CGF);
4262  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4264  };
4265 
4266  // Emit teams region as a standalone region.
4267  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4268  PrePostActionTy &Action) {
4269  Action.Enter(CGF);
4270  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4271  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4272  (void)PrivateScope.Privatize();
4273  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4274  CodeGenDistribute);
4275  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4276  };
4277  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);
4279  [](CodeGenFunction &) { return nullptr; });
4280 }
4281 
4283  CodeGenModule &CGM, StringRef ParentName,
4285  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4286  emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4287  };
4288  llvm::Function *Fn;
4289  llvm::Constant *Addr;
4290  // Emit target region as a standalone region.
4291  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4292  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4293  assert(Fn && Addr && "Target device function emission failed.");
4294 }
4295 
4298  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4299  emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4300  };
4301  emitCommonOMPTargetDirective(*this, S, CodeGen);
4302 }
4303 
4305  const OMPTeamsDistributeDirective &S) {
4306 
4307  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4309  };
4310 
4311  // Emit teams region as a standalone region.
4312  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4313  PrePostActionTy &Action) {
4314  Action.Enter(CGF);
4315  OMPPrivateScope PrivateScope(CGF);
4316  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4317  (void)PrivateScope.Privatize();
4318  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4319  CodeGenDistribute);
4320  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4321  };
4322  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4324  [](CodeGenFunction &) { return nullptr; });
4325 }
4326 
4329  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4331  };
4332 
4333  // Emit teams region as a standalone region.
4334  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4335  PrePostActionTy &Action) {
4336  Action.Enter(CGF);
4337  OMPPrivateScope PrivateScope(CGF);
4338  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4339  (void)PrivateScope.Privatize();
4340  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
4341  CodeGenDistribute);
4342  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4343  };
4344  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen);
4346  [](CodeGenFunction &) { return nullptr; });
4347 }
4348 
4351  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4353  S.getDistInc());
4354  };
4355 
4356  // Emit teams region as a standalone region.
4357  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4358  PrePostActionTy &Action) {
4359  Action.Enter(CGF);
4360  OMPPrivateScope PrivateScope(CGF);
4361  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4362  (void)PrivateScope.Privatize();
4363  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4364  CodeGenDistribute);
4365  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4366  };
4367  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4369  [](CodeGenFunction &) { return nullptr; });
4370 }
4371 
4374  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4376  S.getDistInc());
4377  };
4378 
4379  // Emit teams region as a standalone region.
4380  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4381  PrePostActionTy &Action) {
4382  Action.Enter(CGF);
4383  OMPPrivateScope PrivateScope(CGF);
4384  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4385  (void)PrivateScope.Privatize();
4386  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4387  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4388  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4389  };
4390  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4392  [](CodeGenFunction &) { return nullptr; });
4393 }
4394 
4397  PrePostActionTy &Action) {
4398  Action.Enter(CGF);
4399  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4401  S.getDistInc());
4402  };
4403 
4404  // Emit teams region as a standalone region.
4405  auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4406  PrePostActionTy &Action) {
4407  Action.Enter(CGF);
4408  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4409  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4410  (void)PrivateScope.Privatize();
4411  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4412  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4413  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4414  };
4415 
4416  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
4417  CodeGenTeams);
4419  [](CodeGenFunction &) { return nullptr; });
4420 }
4421 
4423  CodeGenModule &CGM, StringRef ParentName,
4425  // Emit SPMD target teams distribute parallel for region as a standalone
4426  // region.
4427  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4429  };
4430  llvm::Function *Fn;
4431  llvm::Constant *Addr;
4432  // Emit target region as a standalone region.
4433  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4434  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4435  assert(Fn && Addr && "Target device function emission failed.");
4436 }
4437 
4440  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4442  };
4443  emitCommonOMPTargetDirective(*this, S, CodeGen);
4444 }
4445 
4447  CodeGenFunction &CGF,
4449  PrePostActionTy &Action) {
4450  Action.Enter(CGF);
4451  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4453  S.getDistInc());
4454  };
4455 
4456  // Emit teams region as a standalone region.
4457  auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4458  PrePostActionTy &Action) {
4459  Action.Enter(CGF);
4460  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4461  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4462  (void)PrivateScope.Privatize();
4463  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4464  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4465  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4466  };
4467 
4468  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd,
4469  CodeGenTeams);
4471  [](CodeGenFunction &) { return nullptr; });
4472 }
4473 
4475  CodeGenModule &CGM, StringRef ParentName,
4477  // Emit SPMD target teams distribute parallel for simd region as a standalone
4478  // region.
4479  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4481  };
4482  llvm::Function *Fn;
4483  llvm::Constant *Addr;
4484  // Emit target region as a standalone region.
4485  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4486  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4487  assert(Fn && Addr && "Target device function emission failed.");
4488 }
4489 
4492  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4494  };
4495  emitCommonOMPTargetDirective(*this, S, CodeGen);
4496 }
4497 
4499  const OMPCancellationPointDirective &S) {
4500  CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getBeginLoc(),
4501  S.getCancelRegion());
4502 }
4503 
4505  const Expr *IfCond = nullptr;
4506  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4507  if (C->getNameModifier() == OMPD_unknown ||
4508  C->getNameModifier() == OMPD_cancel) {
4509  IfCond = C->getCondition();
4510  break;
4511  }
4512  }
4513  CGM.getOpenMPRuntime().emitCancelCall(*this, S.getBeginLoc(), IfCond,
4514  S.getCancelRegion());
4515 }
4516 
4519  if (Kind == OMPD_parallel || Kind == OMPD_task ||
4520  Kind == OMPD_target_parallel)
4521  return ReturnBlock;
4522  assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
4523  Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
4524  Kind == OMPD_distribute_parallel_for ||
4525  Kind == OMPD_target_parallel_for ||
4526  Kind == OMPD_teams_distribute_parallel_for ||
4527  Kind == OMPD_target_teams_distribute_parallel_for);
4528  return OMPCancelStack.getExitBlock();
4529 }
4530 
4532  const OMPClause &NC, OMPPrivateScope &PrivateScope,
4533  const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) {
4534  const auto &C = cast<OMPUseDevicePtrClause>(NC);
4535  auto OrigVarIt = C.varlist_begin();
4536  auto InitIt = C.inits().begin();
4537  for (const Expr *PvtVarIt : C.private_copies()) {
4538  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl());
4539  const auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl());
4540  const auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl());
4541 
4542  // In order to identify the right initializer we need to match the
4543  // declaration used by the mapping logic. In some cases we may get
4544  // OMPCapturedExprDecl that refers to the original declaration.
4545  const ValueDecl *MatchingVD = OrigVD;
4546  if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
4547  // OMPCapturedExprDecl are used to privative fields of the current
4548  // structure.
4549  const auto *ME = cast<MemberExpr>(OED->getInit());
4550  assert(isa<CXXThisExpr>(ME->getBase()) &&
4551  "Base should be the current struct!");
4552  MatchingVD = ME->getMemberDecl();
4553  }
4554 
4555  // If we don't have information about the current list item, move on to
4556  // the next one.
4557  auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
4558  if (InitAddrIt == CaptureDeviceAddrMap.end())
4559  continue;
4560 
4561  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, OrigVD,
4562  InitAddrIt, InitVD,
4563  PvtVD]() {
4564  // Initialize the temporary initialization variable with the address we
4565  // get from the runtime library. We have to cast the source address
4566  // because it is always a void *. References are materialized in the
4567  // privatization scope, so the initialization here disregards the fact
4568  // the original variable is a reference.
4569  QualType AddrQTy =
4570  getContext().getPointerType(OrigVD->getType().getNonReferenceType());
4571  llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy);
4572  Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy);
4573  setAddrOfLocalVar(InitVD, InitAddr);
4574 
4575  // Emit private declaration, it will be initialized by the value we
4576  // declaration we just added to the local declarations map.
4577  EmitDecl(*PvtVD);
4578 
4579  // The initialization variables reached its purpose in the emission
4580  // of the previous declaration, so we don't need it anymore.
4581  LocalDeclMap.erase(InitVD);
4582 
4583  // Return the address of the private variable.
4584  return GetAddrOfLocalVar(PvtVD);
4585  });
4586  assert(IsRegistered && "firstprivate var already registered as private");
4587  // Silence the warning about unused variable.
4588  (void)IsRegistered;
4589 
4590  ++OrigVarIt;
4591  ++InitIt;
4592  }
4593 }
4594 
4595 // Generate the instructions for '#pragma omp target data' directive.
4597  const OMPTargetDataDirective &S) {
4598  CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true);
4599 
4600  // Create a pre/post action to signal the privatization of the device pointer.
4601  // This action can be replaced by the OpenMP runtime code generation to
4602  // deactivate privatization.
4603  bool PrivatizeDevicePointers = false;
4604  class DevicePointerPrivActionTy : public PrePostActionTy {
4605  bool &PrivatizeDevicePointers;
4606 
4607  public:
4608  explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
4609  : PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {}
4610  void Enter(CodeGenFunction &CGF) override {
4611  PrivatizeDevicePointers = true;
4612  }
4613  };
4614  DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
4615 
4616  auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers](
4617  CodeGenFunction &CGF, PrePostActionTy &Action) {
4618  auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4620  };
4621 
4622  // Codegen that selects whether to generate the privatization code or not.
4623  auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers,
4624  &InnermostCodeGen](CodeGenFunction &CGF,
4625  PrePostActionTy &Action) {
4626  RegionCodeGenTy RCG(InnermostCodeGen);
4627  PrivatizeDevicePointers = false;
4628 
4629  // Call the pre-action to change the status of PrivatizeDevicePointers if
4630  // needed.
4631  Action.Enter(CGF);
4632 
4633  if (PrivatizeDevicePointers) {
4634  OMPPrivateScope PrivateScope(CGF);
4635  // Emit all instances of the use_device_ptr clause.
4636  for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
4637  CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
4638  Info.CaptureDeviceAddrMap);
4639  (void)PrivateScope.Privatize();
4640  RCG(CGF);
4641  } else {
4642  RCG(CGF);
4643  }
4644  };
4645 
4646  // Forward the provided action to the privatization codegen.
4647  RegionCodeGenTy PrivRCG(PrivCodeGen);
4648  PrivRCG.setAction(Action);
4649 
4650  // Notwithstanding the body of the region is emitted as inlined directive,
4651  // we don't use an inline scope as changes in the references inside the
4652  // region are expected to be visible outside, so we do not privative them.
4653  OMPLexicalScope Scope(CGF, S);
4654  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
4655  PrivRCG);
4656  };
4657 
4658  RegionCodeGenTy RCG(CodeGen);
4659 
4660  // If we don't have target devices, don't bother emitting the data mapping
4661  // code.
4662  if (CGM.getLangOpts().OMPTargetTriples.empty()) {
4663  RCG(*this);
4664  return;
4665  }
4666 
4667  // Check if we have any if clause associated with the directive.
4668  const Expr *IfCond = nullptr;
4669  if (const auto *C = S.getSingleClause<OMPIfClause>())
4670  IfCond = C->getCondition();
4671 
4672  // Check if we have any device clause associated with the directive.
4673  const Expr *Device = nullptr;
4674  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4675  Device = C->getDevice();
4676 
4677  // Set the action to signal privatization of device pointers.
4678  RCG.setAction(PrivAction);
4679 
4680  // Emit region code.
4681  CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
4682  Info);
4683 }
4684 
4686  const OMPTargetEnterDataDirective &S) {
4687  // If we don't have target devices, don't bother emitting the data mapping
4688  // code.
4689  if (CGM.getLangOpts().OMPTargetTriples.empty())
4690  return;
4691 
4692  // Check if we have any if clause associated with the directive.
4693  const Expr *IfCond = nullptr;
4694  if (const auto *C = S.getSingleClause<OMPIfClause>())
4695  IfCond = C->getCondition();
4696 
4697  // Check if we have any device clause associated with the directive.
4698  const Expr *Device = nullptr;
4699  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4700  Device = C->getDevice();
4701 
4702  OMPLexicalScope Scope(*this, S, OMPD_task);
4703  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4704 }
4705 
4707  const OMPTargetExitDataDirective &S) {
4708  // If we don't have target devices, don't bother emitting the data mapping
4709  // code.
4710  if (CGM.getLangOpts().OMPTargetTriples.empty())
4711  return;
4712 
4713  // Check if we have any if clause associated with the directive.
4714  const Expr *IfCond = nullptr;
4715  if (const auto *C = S.getSingleClause<OMPIfClause>())
4716  IfCond = C->getCondition();
4717 
4718  // Check if we have any device clause associated with the directive.
4719  const Expr *Device = nullptr;
4720  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4721  Device = C->getDevice();
4722 
4723  OMPLexicalScope Scope(*this, S, OMPD_task);
4724  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4725 }
4726 
4728  const OMPTargetParallelDirective &S,
4729  PrePostActionTy &Action) {
4730  // Get the captured statement associated with the 'parallel' region.
4731  const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
4732  Action.Enter(CGF);
4733  auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4734  Action.Enter(CGF);
4735  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4736  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4737  CGF.EmitOMPPrivateClause(S, PrivateScope);
4738  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4739  (void)PrivateScope.Privatize();
4741  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4742  // TODO: Add support for clauses.
4743  CGF.EmitStmt(CS->getCapturedStmt());
4744  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4745  };
4746  emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
4749  [](CodeGenFunction &) { return nullptr; });
4750 }
4751 
4753  CodeGenModule &CGM, StringRef ParentName,
4754  const OMPTargetParallelDirective &S) {
4755  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4756  emitTargetParallelRegion(CGF, S, Action);
4757  };
4758  llvm::Function *Fn;
4759  llvm::Constant *Addr;
4760  // Emit target region as a standalone region.
4761  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4762  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4763  assert(Fn && Addr && "Target device function emission failed.");
4764 }
4765 
4767  const OMPTargetParallelDirective &S) {
4768  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4769  emitTargetParallelRegion(CGF, S, Action);
4770  };
4771  emitCommonOMPTargetDirective(*this, S, CodeGen);
4772 }
4773 
4776  PrePostActionTy &Action) {
4777  Action.Enter(CGF);
4778  // Emit directive as a combined directive that consists of two implicit
4779  // directives: 'parallel' with 'for' directive.
4780  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4781  Action.Enter(CGF);
4783  CGF, OMPD_target_parallel_for, S.hasCancel());
4786  };
4787  emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
4789 }
4790 
4792  CodeGenModule &CGM, StringRef ParentName,
4793  const OMPTargetParallelForDirective &S) {
4794  // Emit SPMD target parallel for region as a standalone region.
4795  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4796  emitTargetParallelForRegion(CGF, S, Action);
4797  };
4798  llvm::Function *Fn;
4799  llvm::Constant *Addr;
4800  // Emit target region as a standalone region.
4801  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4802  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4803  assert(Fn && Addr && "Target device function emission failed.");
4804 }
4805 
4807  const OMPTargetParallelForDirective &S) {
4808  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4809  emitTargetParallelForRegion(CGF, S, Action);
4810  };
4811  emitCommonOMPTargetDirective(*this, S, CodeGen);
4812 }
4813 
4814 static void
4817  PrePostActionTy &Action) {
4818  Action.Enter(CGF);
4819  // Emit directive as a combined directive that consists of two implicit
4820  // directives: 'parallel' with 'for' directive.
4821  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4822  Action.Enter(CGF);
4825  };
4826  emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
4828 }
4829 
4831  CodeGenModule &CGM, StringRef ParentName,
4833  // Emit SPMD target parallel for region as a standalone region.
4834  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4835  emitTargetParallelForSimdRegion(CGF, S, Action);
4836  };
4837  llvm::Function *Fn;
4838  llvm::Constant *Addr;
4839  // Emit target region as a standalone region.
4840  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4841  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4842  assert(Fn && Addr && "Target device function emission failed.");
4843 }
4844 
4847  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4848  emitTargetParallelForSimdRegion(CGF, S, Action);
4849  };
4850  emitCommonOMPTargetDirective(*this, S, CodeGen);
4851 }
4852 
4853 /// Emit a helper variable and return corresponding lvalue.
4854 static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
4855  const ImplicitParamDecl *PVD,
4857  const auto *VDecl = cast<VarDecl>(Helper->getDecl());
4858  Privates.addPrivate(VDecl,
4859  [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); });
4860 }
4861 
4864  // Emit outlined function for task construct.
4865  const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);
4866  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
4867  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
4868  const Expr *IfCond = nullptr;
4869  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4870  if (C->getNameModifier() == OMPD_unknown ||
4871  C->getNameModifier() == OMPD_taskloop) {
4872  IfCond = C->getCondition();
4873  break;
4874  }
4875  }
4876 
4877  OMPTaskDataTy Data;
4878  // Check if taskloop must be emitted without taskgroup.
4880  // TODO: Check if we should emit tied or untied task.
4881  Data.Tied = true;
4882  // Set scheduling for taskloop
4883  if (const auto* Clause = S.getSingleClause<OMPGrainsizeClause>()) {
4884  // grainsize clause
4885  Data.Schedule.setInt(/*IntVal=*/false);
4886  Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
4887  } else if (const auto* Clause = S.getSingleClause<OMPNumTasksClause>()) {
4888  // num_tasks clause
4889  Data.Schedule.setInt(/*IntVal=*/true);
4890  Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
4891  }
4892 
4893  auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
4894  // if (PreCond) {
4895  // for (IV in 0..LastIteration) BODY;
4896  // <Final counter/linear vars updates>;
4897  // }
4898  //
4899 
4900  // Emit: if (PreCond) - begin.
4901  // If the condition constant folds and can be elided, avoid emitting the
4902  // whole loop.
4903  bool CondConstant;
4904  llvm::BasicBlock *ContBlock = nullptr;
4905  OMPLoopScope PreInitScope(CGF, S);
4906  if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
4907  if (!CondConstant)
4908  return;
4909  } else {
4910  llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
4911  ContBlock = CGF.createBasicBlock("taskloop.if.end");
4912  emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
4913  CGF.getProfileCount(&S));
4914  CGF.EmitBlock(ThenBlock);
4915  CGF.incrementProfileCounter(&S);
4916  }
4917 
4919  CGF.EmitOMPSimdInit(S);
4920 
4921  OMPPrivateScope LoopScope(CGF);
4922  // Emit helper vars inits.
4923  enum { LowerBound = 5, UpperBound, Stride, LastIter };
4924  auto *I = CS->getCapturedDecl()->param_begin();
4925  auto *LBP = std::next(I, LowerBound);
4926  auto *UBP = std::next(I, UpperBound);
4927  auto *STP = std::next(I, Stride);
4928  auto *LIP = std::next(I, LastIter);
4929  mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
4930  LoopScope);
4931  mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
4932  LoopScope);
4933  mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
4934  mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
4935  LoopScope);
4936  CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
4937  bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4938  (void)LoopScope.Privatize();
4939  // Emit the loop iteration variable.
4940  const Expr *IVExpr = S.getIterationVariable();
4941  const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
4942  CGF.EmitVarDecl(*IVDecl);
4943  CGF.EmitIgnoredExpr(S.getInit());
4944 
4945  // Emit the iterations count variable.
4946  // If it is not a variable, Sema decided to calculate iterations count on
4947  // each iteration (e.g., it is foldable into a constant).
4948  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
4949  CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
4950  // Emit calculation of the iterations count.
4951  CGF.EmitIgnoredExpr(S.getCalcLastIteration());
4952  }
4953 
4954  CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
4955  S.getInc(),
4956  [&S](CodeGenFunction &CGF) {
4957  CGF.EmitOMPLoopBody(S, JumpDest());
4958  CGF.EmitStopPoint(&S);
4959  },
4960  [](CodeGenFunction &) {});
4961  // Emit: if (PreCond) - end.
4962  if (ContBlock) {
4963  CGF.EmitBranch(ContBlock);
4964  CGF.EmitBlock(ContBlock, true);
4965  }
4966  // Emit final copy of the lastprivate variables if IsLastIter != 0.
4967  if (HasLastprivateClause) {
4968  CGF.EmitOMPLastprivateClauseFinal(
4970  CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
4971  CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
4972  (*LIP)->getType(), S.getBeginLoc())));
4973  }
4974  };
4975  auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
4976  IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
4977  const OMPTaskDataTy &Data) {
4978  auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
4979  &Data](CodeGenFunction &CGF, PrePostActionTy &) {
4980  OMPLoopScope PreInitScope(CGF, S);
4981  CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
4982  OutlinedFn, SharedsTy,
4983  CapturedStruct, IfCond, Data);
4984  };
4985  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
4986  CodeGen);
4987  };
4988  if (Data.Nogroup) {
4989  EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data);
4990  } else {
4991  CGM.getOpenMPRuntime().emitTaskgroupRegion(
4992  *this,
4993  [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
4994  PrePostActionTy &Action) {
4995  Action.Enter(CGF);
4996  CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen,
4997  Data);
4998  },
4999  S.getBeginLoc());
5000  }
5001 }
5002 
5004  EmitOMPTaskLoopBasedDirective(S);
5005 }
5006 
5008  const OMPTaskLoopSimdDirective &S) {
5009  EmitOMPTaskLoopBasedDirective(S);
5010 }
5011 
5012 // Generate the instructions for '#pragma omp target update' directive.
5014  const OMPTargetUpdateDirective &S) {
5015  // If we don't have target devices, don't bother emitting the data mapping
5016  // code.
5017  if (CGM.getLangOpts().OMPTargetTriples.empty())
5018  return;
5019 
5020  // Check if we have any if clause associated with the directive.
5021  const Expr *IfCond = nullptr;
5022  if (const auto *C = S.getSingleClause<OMPIfClause>())
5023  IfCond = C->getCondition();
5024 
5025  // Check if we have any device clause associated with the directive.
5026  const Expr *Device = nullptr;
5027  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
5028  Device = C->getDevice();
5029 
5030  OMPLexicalScope Scope(*this, S, OMPD_task);
5031  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
5032 }
5033 
5035  const OMPExecutableDirective &D) {
5036  if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
5037  return;
5038  auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {
5040  emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action);
5041  } else {
5042  OMPPrivateScope LoopGlobals(CGF);
5043  if (const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
5044  for (const Expr *E : LD->counters()) {
5045  const auto *VD = dyn_cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5046  if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) {
5047  LValue GlobLVal = CGF.EmitLValue(E);
5048  LoopGlobals.addPrivate(
5049  VD, [&GlobLVal]() { return GlobLVal.getAddress(); });
5050  }
5051  if (isa<OMPCapturedExprDecl>(VD)) {
5052  // Emit only those that were not explicitly referenced in clauses.
5053  if (!CGF.LocalDeclMap.count(VD))
5054  CGF.EmitVarDecl(*VD);
5055  }
5056  }
5057  for (const auto *C : D.getClausesOfKind<OMPOrderedClause>()) {
5058  if (!C->getNumForLoops())
5059  continue;
5060  for (unsigned I = LD->getCollapsedNumber(),
5061  E = C->getLoopNumIterations().size();
5062  I < E; ++I) {
5063  if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
5064  cast<DeclRefExpr>(C->getLoopCounter(I))->getDecl())) {
5065  // Emit only those that were not explicitly referenced in clauses.
5066  if (!CGF.LocalDeclMap.count(VD))
5067  CGF.EmitVarDecl(*VD);
5068  }
5069  }
5070  }
5071  }
5072  LoopGlobals.Privatize();
5074  }
5075  };
5076  OMPSimdLexicalScope Scope(*this, D);
5077  CGM.getOpenMPRuntime().emitInlinedDirective(
5078  *this,
5079  isOpenMPSimdDirective(D.getDirectiveKind()) ? OMPD_simd
5080  : D.getDirectiveKind(),
5081  CodeGen);
5082 }
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
Definition: CGCall.cpp:660
bool isAggregate() const
Definition: CGValue.h:53
This represents &#39;#pragma omp distribute simd&#39; composite directive.
Definition: StmtOpenMP.h:3247
Expr * getNextUpperBound() const
Definition: StmtOpenMP.h:864
This represents &#39;#pragma omp master&#39; directive.
Definition: StmtOpenMP.h:1430
This represents &#39;#pragma omp task&#39; directive.
Definition: StmtOpenMP.h:1770
static const Decl * getCanonicalDecl(const Decl *D)
Represents a function declaration or definition.
Definition: Decl.h:1737
This represents &#39;thread_limit&#39; clause in the &#39;#pragma omp ...&#39; directive.
Expr * getUpperBoundVariable() const
Definition: StmtOpenMP.h:832
Other implicit parameter.
Definition: Decl.h:1509
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, SourceLocation Loc)
This represents clause &#39;copyin&#39; in the &#39;#pragma omp ...&#39; directives.
Complete object ctor.
Definition: ABI.h:25
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition: Type.h:2542
Scheduling data for loop-based OpenMP directives.
Definition: OpenMPKinds.h:141
A (possibly-)qualified type.
Definition: Type.h:634
capture_init_iterator capture_init_begin()
Retrieve the first initialization argument.
Definition: Stmt.h:3298
bool isArrayType() const
Definition: Type.h:6350
ArrayRef< OMPClause * > clauses()
Definition: StmtOpenMP.h:259
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *V, SourceLocation Loc)
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
Definition: CGDecl.cpp:160
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
Definition: Decl.h:4123
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Definition: CGExpr.cpp:138
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
Definition: Dominators.h:29
static std::pair< LValue, LValue > emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer...
static OMPClauseWithPreInit * get(OMPClause *C)
Stmt - This represents one statement.
Definition: Stmt.h:65
This represents clause &#39;in_reduction&#39; in the &#39;#pragma omp task&#39; directives.
Expr * getLowerBoundVariable() const
Definition: StmtOpenMP.h:824
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
Store of global named registers are always calls to intrinsics.
Definition: CGExpr.cpp:2153
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
Definition: Type.cpp:505
bool requiresCleanups() const
Determine whether this scope requires any cleanups.
void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S)
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
SmallVector< std::pair< OpenMPDependClauseKind, const Expr * >, 4 > Dependences
This represents &#39;#pragma omp for simd&#39; directive.
Definition: StmtOpenMP.h:1180
SmallVector< const Expr *, 4 > LastprivateCopies
QualType getLValueReferenceType(QualType T, bool SpelledAsLValue=true) const
Return the uniqued reference to the type for an lvalue reference to the specified type...
Decl - This represents one declaration (or definition), e.g.
Definition: DeclBase.h:86
This represents &#39;grainsize&#39; clause in the &#39;#pragma omp ...&#39; directive.
This represents &#39;#pragma omp teams distribute parallel for&#39; composite directive.
Definition: StmtOpenMP.h:3658
SourceLocation getBeginLoc() const
Returns starting location of directive kind.
Definition: StmtOpenMP.h:167
bool isNothrow() const
Definition: Decl.cpp:4474
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
This represents &#39;if&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:239
CapturedStmt * getInnermostCapturedStmt()
Get innermost captured statement for the construct.
Definition: StmtOpenMP.h:225
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Definition: Type.h:6250
void EmitOMPOrderedDirective(const OMPOrderedDirective &S)
bool isVolatile() const
Definition: CGValue.h:300
This represents &#39;priority&#39; clause in the &#39;#pragma omp ...&#39; directive.
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
The base class of the type hierarchy.
Definition: Type.h:1409
This represents &#39;#pragma omp target teams distribute&#39; combined directive.
Definition: StmtOpenMP.h:3795
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
Definition: CGExpr.cpp:1918
static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, bool IsSeqCst, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Definition: Stmt.h:3242
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Definition: Type.h:2817
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
const TargetInfo & getTargetInfo() const
Definition: ASTContext.h:689
Expr * getCombinedParForInDistCond() const
Definition: StmtOpenMP.h:952
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
const llvm::function_ref< void(CodeGenFunction &, llvm::Function *, const OMPTaskDataTy &)> TaskGenTy
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters)
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Definition: CGExpr.cpp:2326
Floating point control options.
Definition: LangOptions.h:306
This represents &#39;#pragma omp parallel for&#39; directive.
Definition: StmtOpenMP.h:1551
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value *> &CapturedVars)
This represents &#39;#pragma omp target teams distribute parallel for&#39; combined directive.
Definition: StmtOpenMP.h:3863
Expr * getCombinedEnsureUpperBound() const
Definition: StmtOpenMP.h:916
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
Definition: CGAtomic.cpp:1894
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant, or if it does but contains a label, return false.
Represents a point when we exit a loop.
Definition: ProgramPoint.h:714