clang  9.0.0svn
CGStmtOpenMP.cpp
Go to the documentation of this file.
1 //===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This contains code to emit OpenMP nodes as LLVM code.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CGCleanup.h"
14 #include "CGOpenMPRuntime.h"
15 #include "CodeGenFunction.h"
16 #include "CodeGenModule.h"
17 #include "TargetInfo.h"
18 #include "clang/AST/Stmt.h"
19 #include "clang/AST/StmtOpenMP.h"
20 #include "clang/AST/DeclOpenMP.h"
21 using namespace clang;
22 using namespace CodeGen;
23 
24 namespace {
25 /// Lexical scope for OpenMP executable constructs, that handles correct codegen
26 /// for captured expressions.
27 class OMPLexicalScope : public CodeGenFunction::LexicalScope {
28  void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
29  for (const auto *C : S.clauses()) {
30  if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
31  if (const auto *PreInit =
32  cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
33  for (const auto *I : PreInit->decls()) {
34  if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
35  CGF.EmitVarDecl(cast<VarDecl>(*I));
36  } else {
38  CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
39  CGF.EmitAutoVarCleanups(Emission);
40  }
41  }
42  }
43  }
44  }
45  }
46  CodeGenFunction::OMPPrivateScope InlinedShareds;
47 
48  static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
49  return CGF.LambdaCaptureFields.lookup(VD) ||
50  (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
51  (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl));
52  }
53 
54 public:
55  OMPLexicalScope(
57  const llvm::Optional<OpenMPDirectiveKind> CapturedRegion = llvm::None,
58  const bool EmitPreInitStmt = true)
59  : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
60  InlinedShareds(CGF) {
61  if (EmitPreInitStmt)
62  emitPreInitStmt(CGF, S);
63  if (!CapturedRegion.hasValue())
64  return;
65  assert(S.hasAssociatedStmt() &&
66  "Expected associated statement for inlined directive.");
67  const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
68  for (const auto &C : CS->captures()) {
69  if (C.capturesVariable() || C.capturesVariableByCopy()) {
70  auto *VD = C.getCapturedVar();
71  assert(VD == VD->getCanonicalDecl() &&
72  "Canonical decl must be captured.");
73  DeclRefExpr DRE(
74  CGF.getContext(), const_cast<VarDecl *>(VD),
75  isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
76  InlinedShareds.isGlobalVarCaptured(VD)),
77  VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
78  InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
79  return CGF.EmitLValue(&DRE).getAddress();
80  });
81  }
82  }
83  (void)InlinedShareds.Privatize();
84  }
85 };
86 
87 /// Lexical scope for OpenMP parallel construct, that handles correct codegen
88 /// for captured expressions.
89 class OMPParallelScope final : public OMPLexicalScope {
90  bool EmitPreInitStmt(const OMPExecutableDirective &S) {
92  return !(isOpenMPTargetExecutionDirective(Kind) ||
95  }
96 
97 public:
98  OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
99  : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
100  EmitPreInitStmt(S)) {}
101 };
102 
103 /// Lexical scope for OpenMP teams construct, that handles correct codegen
104 /// for captured expressions.
105 class OMPTeamsScope final : public OMPLexicalScope {
106  bool EmitPreInitStmt(const OMPExecutableDirective &S) {
108  return !isOpenMPTargetExecutionDirective(Kind) &&
110  }
111 
112 public:
113  OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
114  : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
115  EmitPreInitStmt(S)) {}
116 };
117 
118 /// Private scope for OpenMP loop-based directives, that supports capturing
119 /// of used expression from loop statement.
120 class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
121  void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) {
122  CodeGenFunction::OMPMapVars PreCondVars;
123  for (const auto *E : S.counters()) {
124  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
125  (void)PreCondVars.setVarAddr(
126  CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
127  }
128  (void)PreCondVars.apply(CGF);
129  if (const auto *PreInits = cast_or_null<DeclStmt>(S.getPreInits())) {
130  for (const auto *I : PreInits->decls())
131  CGF.EmitVarDecl(cast<VarDecl>(*I));
132  }
133  PreCondVars.restore(CGF);
134  }
135 
136 public:
137  OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S)
138  : CodeGenFunction::RunCleanupsScope(CGF) {
139  emitPreInitStmt(CGF, S);
140  }
141 };
142 
143 class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
144  CodeGenFunction::OMPPrivateScope InlinedShareds;
145 
146  static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
147  return CGF.LambdaCaptureFields.lookup(VD) ||
148  (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
149  (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) &&
150  cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
151  }
152 
153 public:
154  OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
155  : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
156  InlinedShareds(CGF) {
157  for (const auto *C : S.clauses()) {
158  if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
159  if (const auto *PreInit =
160  cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
161  for (const auto *I : PreInit->decls()) {
162  if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
163  CGF.EmitVarDecl(cast<VarDecl>(*I));
164  } else {
166  CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
167  CGF.EmitAutoVarCleanups(Emission);
168  }
169  }
170  }
171  } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
172  for (const Expr *E : UDP->varlists()) {
173  const Decl *D = cast<DeclRefExpr>(E)->getDecl();
174  if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
175  CGF.EmitVarDecl(*OED);
176  }
177  }
178  }
180  CGF.EmitOMPPrivateClause(S, InlinedShareds);
181  if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
182  if (const Expr *E = TG->getReductionRef())
183  CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
184  }
185  const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
186  while (CS) {
187  for (auto &C : CS->captures()) {
188  if (C.capturesVariable() || C.capturesVariableByCopy()) {
189  auto *VD = C.getCapturedVar();
190  assert(VD == VD->getCanonicalDecl() &&
191  "Canonical decl must be captured.");
192  DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
193  isCapturedVar(CGF, VD) ||
194  (CGF.CapturedStmtInfo &&
195  InlinedShareds.isGlobalVarCaptured(VD)),
197  C.getLocation());
198  InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
199  return CGF.EmitLValue(&DRE).getAddress();
200  });
201  }
202  }
203  CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
204  }
205  (void)InlinedShareds.Privatize();
206  }
207 };
208 
209 } // namespace
210 
212  const OMPExecutableDirective &S,
213  const RegionCodeGenTy &CodeGen);
214 
216  if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
217  if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
218  OrigVD = OrigVD->getCanonicalDecl();
219  bool IsCaptured =
220  LambdaCaptureFields.lookup(OrigVD) ||
221  (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
222  (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
223  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), IsCaptured,
224  OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
225  return EmitLValue(&DRE);
226  }
227  }
228  return EmitLValue(E);
229 }
230 
232  ASTContext &C = getContext();
233  llvm::Value *Size = nullptr;
234  auto SizeInChars = C.getTypeSizeInChars(Ty);
235  if (SizeInChars.isZero()) {
236  // getTypeSizeInChars() returns 0 for a VLA.
237  while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) {
238  VlaSizePair VlaSize = getVLASize(VAT);
239  Ty = VlaSize.Type;
240  Size = Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts)
241  : VlaSize.NumElts;
242  }
243  SizeInChars = C.getTypeSizeInChars(Ty);
244  if (SizeInChars.isZero())
245  return llvm::ConstantInt::get(SizeTy, /*V=*/0);
246  return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
247  }
248  return CGM.getSize(SizeInChars);
249 }
250 
252  const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
253  const RecordDecl *RD = S.getCapturedRecordDecl();
254  auto CurField = RD->field_begin();
255  auto CurCap = S.captures().begin();
257  E = S.capture_init_end();
258  I != E; ++I, ++CurField, ++CurCap) {
259  if (CurField->hasCapturedVLAType()) {
260  const VariableArrayType *VAT = CurField->getCapturedVLAType();
261  llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
262  CapturedVars.push_back(Val);
263  } else if (CurCap->capturesThis()) {
264  CapturedVars.push_back(CXXThisValue);
265  } else if (CurCap->capturesVariableByCopy()) {
266  llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
267 
268  // If the field is not a pointer, we need to save the actual value
269  // and load it as a void pointer.
270  if (!CurField->getType()->isAnyPointerType()) {
271  ASTContext &Ctx = getContext();
272  Address DstAddr = CreateMemTemp(
273  Ctx.getUIntPtrType(),
274  Twine(CurCap->getCapturedVar()->getName(), ".casted"));
275  LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
276 
277  llvm::Value *SrcAddrVal = EmitScalarConversion(
278  DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
279  Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
280  LValue SrcLV =
281  MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
282 
283  // Store the value using the source type pointer.
284  EmitStoreThroughLValue(RValue::get(CV), SrcLV);
285 
286  // Load the value using the destination type pointer.
287  CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
288  }
289  CapturedVars.push_back(CV);
290  } else {
291  assert(CurCap->capturesVariable() && "Expected capture by reference.");
292  CapturedVars.push_back(EmitLValue(*I).getAddress().getPointer());
293  }
294  }
295 }
296 
298  QualType DstType, StringRef Name,
299  LValue AddrLV,
300  bool isReferenceType = false) {
301  ASTContext &Ctx = CGF.getContext();
302 
303  llvm::Value *CastedPtr = CGF.EmitScalarConversion(
304  AddrLV.getAddress().getPointer(), Ctx.getUIntPtrType(),
305  Ctx.getPointerType(DstType), Loc);
306  Address TmpAddr =
307  CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType))
308  .getAddress();
309 
310  // If we are dealing with references we need to return the address of the
311  // reference instead of the reference of the value.
312  if (isReferenceType) {
313  QualType RefType = Ctx.getLValueReferenceType(DstType);
314  llvm::Value *RefVal = TmpAddr.getPointer();
315  TmpAddr = CGF.CreateMemTemp(RefType, Twine(Name, ".ref"));
316  LValue TmpLVal = CGF.MakeAddrLValue(TmpAddr, RefType);
317  CGF.EmitStoreThroughLValue(RValue::get(RefVal), TmpLVal, /*isInit=*/true);
318  }
319 
320  return TmpAddr;
321 }
322 
324  if (T->isLValueReferenceType())
325  return C.getLValueReferenceType(
327  /*SpelledAsLValue=*/false);
328  if (T->isPointerType())
330  if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
331  if (const auto *VLA = dyn_cast<VariableArrayType>(A))
332  return getCanonicalParamType(C, VLA->getElementType());
333  if (!A->isVariablyModifiedType())
334  return C.getCanonicalType(T);
335  }
336  return C.getCanonicalParamType(T);
337 }
338 
339 namespace {
340  /// Contains required data for proper outlined function codegen.
341  struct FunctionOptions {
342  /// Captured statement for which the function is generated.
343  const CapturedStmt *S = nullptr;
344  /// true if cast to/from UIntPtr is required for variables captured by
345  /// value.
346  const bool UIntPtrCastRequired = true;
347  /// true if only casted arguments must be registered as local args or VLA
348  /// sizes.
349  const bool RegisterCastedArgsOnly = false;
350  /// Name of the generated function.
351  const StringRef FunctionName;
352  explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
353  bool RegisterCastedArgsOnly,
354  StringRef FunctionName)
355  : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
356  RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
357  FunctionName(FunctionName) {}
358  };
359 }
360 
361 static llvm::Function *emitOutlinedFunctionPrologue(
362  CodeGenFunction &CGF, FunctionArgList &Args,
363  llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
364  &LocalAddrs,
365  llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
366  &VLASizes,
367  llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
368  const CapturedDecl *CD = FO.S->getCapturedDecl();
369  const RecordDecl *RD = FO.S->getCapturedRecordDecl();
370  assert(CD->hasBody() && "missing CapturedDecl body");
371 
372  CXXThisValue = nullptr;
373  // Build the argument list.
374  CodeGenModule &CGM = CGF.CGM;
375  ASTContext &Ctx = CGM.getContext();
376  FunctionArgList TargetArgs;
377  Args.append(CD->param_begin(),
378  std::next(CD->param_begin(), CD->getContextParamPosition()));
379  TargetArgs.append(
380  CD->param_begin(),
381  std::next(CD->param_begin(), CD->getContextParamPosition()));
382  auto I = FO.S->captures().begin();
383  FunctionDecl *DebugFunctionDecl = nullptr;
384  if (!FO.UIntPtrCastRequired) {
386  QualType FunctionTy = Ctx.getFunctionType(Ctx.VoidTy, llvm::None, EPI);
387  DebugFunctionDecl = FunctionDecl::Create(
388  Ctx, Ctx.getTranslationUnitDecl(), FO.S->getBeginLoc(),
389  SourceLocation(), DeclarationName(), FunctionTy,
390  Ctx.getTrivialTypeSourceInfo(FunctionTy), SC_Static,
391  /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/false);
392  }
393  for (const FieldDecl *FD : RD->fields()) {
394  QualType ArgType = FD->getType();
395  IdentifierInfo *II = nullptr;
396  VarDecl *CapVar = nullptr;
397 
398  // If this is a capture by copy and the type is not a pointer, the outlined
399  // function argument type should be uintptr and the value properly casted to
400  // uintptr. This is necessary given that the runtime library is only able to
401  // deal with pointers. We can pass in the same way the VLA type sizes to the
402  // outlined function.
403  if (FO.UIntPtrCastRequired &&
404  ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
405  I->capturesVariableArrayType()))
406  ArgType = Ctx.getUIntPtrType();
407 
408  if (I->capturesVariable() || I->capturesVariableByCopy()) {
409  CapVar = I->getCapturedVar();
410  II = CapVar->getIdentifier();
411  } else if (I->capturesThis()) {
412  II = &Ctx.Idents.get("this");
413  } else {
414  assert(I->capturesVariableArrayType());
415  II = &Ctx.Idents.get("vla");
416  }
417  if (ArgType->isVariablyModifiedType())
418  ArgType = getCanonicalParamType(Ctx, ArgType);
419  VarDecl *Arg;
420  if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
421  Arg = ParmVarDecl::Create(
422  Ctx, DebugFunctionDecl,
423  CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),
424  CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
425  /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
426  } else {
427  Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
428  II, ArgType, ImplicitParamDecl::Other);
429  }
430  Args.emplace_back(Arg);
431  // Do not cast arguments if we emit function with non-original types.
432  TargetArgs.emplace_back(
433  FO.UIntPtrCastRequired
434  ? Arg
435  : CGM.getOpenMPRuntime().translateParameter(FD, Arg));
436  ++I;
437  }
438  Args.append(
439  std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
440  CD->param_end());
441  TargetArgs.append(
442  std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
443  CD->param_end());
444 
445  // Create the function declaration.
446  const CGFunctionInfo &FuncInfo =
447  CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, TargetArgs);
448  llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
449 
450  auto *F =
452  FO.FunctionName, &CGM.getModule());
453  CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
454  if (CD->isNothrow())
455  F->setDoesNotThrow();
456  F->setDoesNotRecurse();
457 
458  // Generate the function.
459  CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
460  FO.S->getBeginLoc(), CD->getBody()->getBeginLoc());
461  unsigned Cnt = CD->getContextParamPosition();
462  I = FO.S->captures().begin();
463  for (const FieldDecl *FD : RD->fields()) {
464  // Do not map arguments if we emit function with non-original types.
465  Address LocalAddr(Address::invalid());
466  if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
467  LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
468  TargetArgs[Cnt]);
469  } else {
470  LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
471  }
472  // If we are capturing a pointer by copy we don't need to do anything, just
473  // use the value that we get from the arguments.
474  if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
475  const VarDecl *CurVD = I->getCapturedVar();
476  // If the variable is a reference we need to materialize it here.
477  if (CurVD->getType()->isReferenceType()) {
478  Address RefAddr = CGF.CreateMemTemp(
479  CurVD->getType(), CGM.getPointerAlign(), ".materialized_ref");
480  CGF.EmitStoreOfScalar(LocalAddr.getPointer(), RefAddr,
481  /*Volatile=*/false, CurVD->getType());
482  LocalAddr = RefAddr;
483  }
484  if (!FO.RegisterCastedArgsOnly)
485  LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
486  ++Cnt;
487  ++I;
488  continue;
489  }
490 
491  LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
493  if (FD->hasCapturedVLAType()) {
494  if (FO.UIntPtrCastRequired) {
495  ArgLVal = CGF.MakeAddrLValue(
496  castValueFromUintptr(CGF, I->getLocation(), FD->getType(),
497  Args[Cnt]->getName(), ArgLVal),
499  }
500  llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
501  const VariableArrayType *VAT = FD->getCapturedVLAType();
502  VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg);
503  } else if (I->capturesVariable()) {
504  const VarDecl *Var = I->getCapturedVar();
505  QualType VarTy = Var->getType();
506  Address ArgAddr = ArgLVal.getAddress();
507  if (!VarTy->isReferenceType()) {
508  if (ArgLVal.getType()->isLValueReferenceType()) {
509  ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
510  } else if (!VarTy->isVariablyModifiedType() ||
511  !VarTy->isPointerType()) {
512  assert(ArgLVal.getType()->isPointerType());
513  ArgAddr = CGF.EmitLoadOfPointer(
514  ArgAddr, ArgLVal.getType()->castAs<PointerType>());
515  }
516  }
517  if (!FO.RegisterCastedArgsOnly) {
518  LocalAddrs.insert(
519  {Args[Cnt],
520  {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}});
521  }
522  } else if (I->capturesVariableByCopy()) {
523  assert(!FD->getType()->isAnyPointerType() &&
524  "Not expecting a captured pointer.");
525  const VarDecl *Var = I->getCapturedVar();
526  QualType VarTy = Var->getType();
527  LocalAddrs.insert(
528  {Args[Cnt],
529  {Var, FO.UIntPtrCastRequired
530  ? castValueFromUintptr(CGF, I->getLocation(),
531  FD->getType(), Args[Cnt]->getName(),
532  ArgLVal, VarTy->isReferenceType())
533  : ArgLVal.getAddress()}});
534  } else {
535  // If 'this' is captured, load it into CXXThisValue.
536  assert(I->capturesThis());
537  CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
538  LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress()}});
539  }
540  ++Cnt;
541  ++I;
542  }
543 
544  return F;
545 }
546 
547 llvm::Function *
549  assert(
550  CapturedStmtInfo &&
551  "CapturedStmtInfo should be set when generating the captured function");
552  const CapturedDecl *CD = S.getCapturedDecl();
553  // Build the argument list.
554  bool NeedWrapperFunction =
555  getDebugInfo() &&
556  CGM.getCodeGenOpts().getDebugInfo() >= codegenoptions::LimitedDebugInfo;
557  FunctionArgList Args;
558  llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
559  llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
560  SmallString<256> Buffer;
561  llvm::raw_svector_ostream Out(Buffer);
562  Out << CapturedStmtInfo->getHelperName();
563  if (NeedWrapperFunction)
564  Out << "_debug__";
565  FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
566  Out.str());
567  llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs,
568  VLASizes, CXXThisValue, FO);
569  for (const auto &LocalAddrPair : LocalAddrs) {
570  if (LocalAddrPair.second.first) {
571  setAddrOfLocalVar(LocalAddrPair.second.first,
572  LocalAddrPair.second.second);
573  }
574  }
575  for (const auto &VLASizePair : VLASizes)
576  VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
577  PGO.assignRegionCounters(GlobalDecl(CD), F);
578  CapturedStmtInfo->EmitBody(*this, CD->getBody());
579  FinishFunction(CD->getBodyRBrace());
580  if (!NeedWrapperFunction)
581  return F;
582 
583  FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
584  /*RegisterCastedArgsOnly=*/true,
585  CapturedStmtInfo->getHelperName());
586  CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
587  WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
588  Args.clear();
589  LocalAddrs.clear();
590  VLASizes.clear();
591  llvm::Function *WrapperF =
592  emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
593  WrapperCGF.CXXThisValue, WrapperFO);
595  for (const auto *Arg : Args) {
597  auto I = LocalAddrs.find(Arg);
598  if (I != LocalAddrs.end()) {
599  LValue LV = WrapperCGF.MakeAddrLValue(
600  I->second.second,
601  I->second.first ? I->second.first->getType() : Arg->getType(),
603  CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
604  } else {
605  auto EI = VLASizes.find(Arg);
606  if (EI != VLASizes.end()) {
607  CallArg = EI->second.second;
608  } else {
609  LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
610  Arg->getType(),
612  CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
613  }
614  }
615  CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
616  }
617  CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.getBeginLoc(),
618  F, CallArgs);
619  WrapperCGF.FinishFunction();
620  return WrapperF;
621 }
622 
623 //===----------------------------------------------------------------------===//
624 // OpenMP Directive Emission
625 //===----------------------------------------------------------------------===//
627  Address DestAddr, Address SrcAddr, QualType OriginalType,
628  const llvm::function_ref<void(Address, Address)> CopyGen) {
629  // Perform element-by-element initialization.
630  QualType ElementTy;
631 
632  // Drill down to the base element type on both arrays.
633  const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
634  llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
635  SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
636 
637  llvm::Value *SrcBegin = SrcAddr.getPointer();
638  llvm::Value *DestBegin = DestAddr.getPointer();
639  // Cast from pointer to array type to pointer to single element.
640  llvm::Value *DestEnd = Builder.CreateGEP(DestBegin, NumElements);
641  // The basic structure here is a while-do loop.
642  llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body");
643  llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done");
644  llvm::Value *IsEmpty =
645  Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
646  Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
647 
648  // Enter the loop body, making that address the current address.
649  llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
650  EmitBlock(BodyBB);
651 
652  CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
653 
654  llvm::PHINode *SrcElementPHI =
655  Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
656  SrcElementPHI->addIncoming(SrcBegin, EntryBB);
657  Address SrcElementCurrent =
658  Address(SrcElementPHI,
659  SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
660 
661  llvm::PHINode *DestElementPHI =
662  Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
663  DestElementPHI->addIncoming(DestBegin, EntryBB);
664  Address DestElementCurrent =
665  Address(DestElementPHI,
666  DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
667 
668  // Emit copy.
669  CopyGen(DestElementCurrent, SrcElementCurrent);
670 
671  // Shift the address forward by one element.
672  llvm::Value *DestElementNext = Builder.CreateConstGEP1_32(
673  DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
674  llvm::Value *SrcElementNext = Builder.CreateConstGEP1_32(
675  SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
676  // Check whether we've reached the end.
677  llvm::Value *Done =
678  Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
679  Builder.CreateCondBr(Done, DoneBB, BodyBB);
680  DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
681  SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
682 
683  // Done.
684  EmitBlock(DoneBB, /*IsFinished=*/true);
685 }
686 
687 void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
688  Address SrcAddr, const VarDecl *DestVD,
689  const VarDecl *SrcVD, const Expr *Copy) {
690  if (OriginalType->isArrayType()) {
691  const auto *BO = dyn_cast<BinaryOperator>(Copy);
692  if (BO && BO->getOpcode() == BO_Assign) {
693  // Perform simple memcpy for simple copying.
694  LValue Dest = MakeAddrLValue(DestAddr, OriginalType);
695  LValue Src = MakeAddrLValue(SrcAddr, OriginalType);
696  EmitAggregateAssign(Dest, Src, OriginalType);
697  } else {
698  // For arrays with complex element types perform element by element
699  // copying.
700  EmitOMPAggregateAssign(
701  DestAddr, SrcAddr, OriginalType,
702  [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
703  // Working with the single array element, so have to remap
704  // destination and source variables to corresponding array
705  // elements.
707  Remap.addPrivate(DestVD, [DestElement]() { return DestElement; });
708  Remap.addPrivate(SrcVD, [SrcElement]() { return SrcElement; });
709  (void)Remap.Privatize();
710  EmitIgnoredExpr(Copy);
711  });
712  }
713  } else {
714  // Remap pseudo source variable to private copy.
716  Remap.addPrivate(SrcVD, [SrcAddr]() { return SrcAddr; });
717  Remap.addPrivate(DestVD, [DestAddr]() { return DestAddr; });
718  (void)Remap.Privatize();
719  // Emit copying of the whole variable.
720  EmitIgnoredExpr(Copy);
721  }
722 }
723 
725  OMPPrivateScope &PrivateScope) {
726  if (!HaveInsertPoint())
727  return false;
728  bool DeviceConstTarget =
729  getLangOpts().OpenMPIsDevice &&
731  bool FirstprivateIsLastprivate = false;
732  llvm::DenseSet<const VarDecl *> Lastprivates;
733  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
734  for (const auto *D : C->varlists())
735  Lastprivates.insert(
736  cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl());
737  }
738  llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
740  getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind());
741  // Force emission of the firstprivate copy if the directive does not emit
742  // outlined function, like omp for, omp simd, omp distribute etc.
743  bool MustEmitFirstprivateCopy =
744  CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
745  for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
746  auto IRef = C->varlist_begin();
747  auto InitsRef = C->inits().begin();
748  for (const Expr *IInit : C->private_copies()) {
749  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
750  bool ThisFirstprivateIsLastprivate =
751  Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
752  const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
753  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
754  if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
755  !FD->getType()->isReferenceType() &&
756  (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
757  EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
758  ++IRef;
759  ++InitsRef;
760  continue;
761  }
762  // Do not emit copy for firstprivate constant variables in target regions,
763  // captured by reference.
764  if (DeviceConstTarget && OrigVD->getType().isConstant(getContext()) &&
765  FD && FD->getType()->isReferenceType() &&
766  (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
767  (void)CGM.getOpenMPRuntime().registerTargetFirstprivateCopy(*this,
768  OrigVD);
769  ++IRef;
770  ++InitsRef;
771  continue;
772  }
773  FirstprivateIsLastprivate =
774  FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
775  if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
776  const auto *VDInit =
777  cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
778  bool IsRegistered;
779  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
780  /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
781  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
782  LValue OriginalLVal = EmitLValue(&DRE);
783  QualType Type = VD->getType();
784  if (Type->isArrayType()) {
785  // Emit VarDecl with copy init for arrays.
786  // Get the address of the original variable captured in current
787  // captured region.
788  IsRegistered = PrivateScope.addPrivate(
789  OrigVD, [this, VD, Type, OriginalLVal, VDInit]() {
790  AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
791  const Expr *Init = VD->getInit();
792  if (!isa<CXXConstructExpr>(Init) ||
793  isTrivialInitializer(Init)) {
794  // Perform simple memcpy.
795  LValue Dest =
796  MakeAddrLValue(Emission.getAllocatedAddress(), Type);
797  EmitAggregateAssign(Dest, OriginalLVal, Type);
798  } else {
799  EmitOMPAggregateAssign(
800  Emission.getAllocatedAddress(), OriginalLVal.getAddress(),
801  Type,
802  [this, VDInit, Init](Address DestElement,
803  Address SrcElement) {
804  // Clean up any temporaries needed by the
805  // initialization.
806  RunCleanupsScope InitScope(*this);
807  // Emit initialization for single element.
808  setAddrOfLocalVar(VDInit, SrcElement);
809  EmitAnyExprToMem(Init, DestElement,
810  Init->getType().getQualifiers(),
811  /*IsInitializer*/ false);
812  LocalDeclMap.erase(VDInit);
813  });
814  }
815  EmitAutoVarCleanups(Emission);
816  return Emission.getAllocatedAddress();
817  });
818  } else {
819  Address OriginalAddr = OriginalLVal.getAddress();
820  IsRegistered = PrivateScope.addPrivate(
821  OrigVD, [this, VDInit, OriginalAddr, VD]() {
822  // Emit private VarDecl with copy init.
823  // Remap temp VDInit variable to the address of the original
824  // variable (for proper handling of captured global variables).
825  setAddrOfLocalVar(VDInit, OriginalAddr);
826  EmitDecl(*VD);
827  LocalDeclMap.erase(VDInit);
828  return GetAddrOfLocalVar(VD);
829  });
830  }
831  assert(IsRegistered &&
832  "firstprivate var already registered as private");
833  // Silence the warning about unused variable.
834  (void)IsRegistered;
835  }
836  ++IRef;
837  ++InitsRef;
838  }
839  }
840  return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
841 }
842 
844  const OMPExecutableDirective &D,
845  CodeGenFunction::OMPPrivateScope &PrivateScope) {
846  if (!HaveInsertPoint())
847  return;
848  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
849  for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
850  auto IRef = C->varlist_begin();
851  for (const Expr *IInit : C->private_copies()) {
852  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
853  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
854  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
855  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
856  // Emit private VarDecl with copy init.
857  EmitDecl(*VD);
858  return GetAddrOfLocalVar(VD);
859  });
860  assert(IsRegistered && "private var already registered as private");
861  // Silence the warning about unused variable.
862  (void)IsRegistered;
863  }
864  ++IRef;
865  }
866  }
867 }
868 
870  if (!HaveInsertPoint())
871  return false;
872  // threadprivate_var1 = master_threadprivate_var1;
873  // operator=(threadprivate_var2, master_threadprivate_var2);
874  // ...
875  // __kmpc_barrier(&loc, global_tid);
877  llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
878  for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
879  auto IRef = C->varlist_begin();
880  auto ISrcRef = C->source_exprs().begin();
881  auto IDestRef = C->destination_exprs().begin();
882  for (const Expr *AssignOp : C->assignment_ops()) {
883  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
884  QualType Type = VD->getType();
885  if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
886  // Get the address of the master variable. If we are emitting code with
887  // TLS support, the address is passed from the master as field in the
888  // captured declaration.
889  Address MasterAddr = Address::invalid();
890  if (getLangOpts().OpenMPUseTLS &&
891  getContext().getTargetInfo().isTLSSupported()) {
892  assert(CapturedStmtInfo->lookup(VD) &&
893  "Copyin threadprivates should have been captured!");
894  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), true,
895  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
896  MasterAddr = EmitLValue(&DRE).getAddress();
897  LocalDeclMap.erase(VD);
898  } else {
899  MasterAddr =
900  Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
901  : CGM.GetAddrOfGlobal(VD),
902  getContext().getDeclAlign(VD));
903  }
904  // Get the address of the threadprivate variable.
905  Address PrivateAddr = EmitLValue(*IRef).getAddress();
906  if (CopiedVars.size() == 1) {
907  // At first check if current thread is a master thread. If it is, no
908  // need to copy data.
909  CopyBegin = createBasicBlock("copyin.not.master");
910  CopyEnd = createBasicBlock("copyin.not.master.end");
911  Builder.CreateCondBr(
912  Builder.CreateICmpNE(
913  Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy),
914  Builder.CreatePtrToInt(PrivateAddr.getPointer(),
915  CGM.IntPtrTy)),
916  CopyBegin, CopyEnd);
917  EmitBlock(CopyBegin);
918  }
919  const auto *SrcVD =
920  cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
921  const auto *DestVD =
922  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
923  EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
924  }
925  ++IRef;
926  ++ISrcRef;
927  ++IDestRef;
928  }
929  }
930  if (CopyEnd) {
931  // Exit out of copying procedure for non-master thread.
932  EmitBlock(CopyEnd, /*IsFinished=*/true);
933  return true;
934  }
935  return false;
936 }
937 
939  const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
940  if (!HaveInsertPoint())
941  return false;
942  bool HasAtLeastOneLastprivate = false;
945  const auto *LoopDirective = cast<OMPLoopDirective>(&D);
946  for (const Expr *C : LoopDirective->counters()) {
947  SIMDLCVs.insert(
948  cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
949  }
950  }
951  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
952  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
953  HasAtLeastOneLastprivate = true;
955  !getLangOpts().OpenMPSimd)
956  break;
957  auto IRef = C->varlist_begin();
958  auto IDestRef = C->destination_exprs().begin();
959  for (const Expr *IInit : C->private_copies()) {
960  // Keep the address of the original variable for future update at the end
961  // of the loop.
962  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
963  // Taskloops do not require additional initialization, it is done in
964  // runtime support library.
965  if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
966  const auto *DestVD =
967  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
968  PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() {
969  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
970  /*RefersToEnclosingVariableOrCapture=*/
971  CapturedStmtInfo->lookup(OrigVD) != nullptr,
972  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
973  return EmitLValue(&DRE).getAddress();
974  });
975  // Check if the variable is also a firstprivate: in this case IInit is
976  // not generated. Initialization of this variable will happen in codegen
977  // for 'firstprivate' clause.
978  if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
979  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
980  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
981  // Emit private VarDecl with copy init.
982  EmitDecl(*VD);
983  return GetAddrOfLocalVar(VD);
984  });
985  assert(IsRegistered &&
986  "lastprivate var already registered as private");
987  (void)IsRegistered;
988  }
989  }
990  ++IRef;
991  ++IDestRef;
992  }
993  }
994  return HasAtLeastOneLastprivate;
995 }
996 
998  const OMPExecutableDirective &D, bool NoFinals,
999  llvm::Value *IsLastIterCond) {
1000  if (!HaveInsertPoint())
1001  return;
1002  // Emit following code:
1003  // if (<IsLastIterCond>) {
1004  // orig_var1 = private_orig_var1;
1005  // ...
1006  // orig_varn = private_orig_varn;
1007  // }
1008  llvm::BasicBlock *ThenBB = nullptr;
1009  llvm::BasicBlock *DoneBB = nullptr;
1010  if (IsLastIterCond) {
1011  ThenBB = createBasicBlock(".omp.lastprivate.then");
1012  DoneBB = createBasicBlock(".omp.lastprivate.done");
1013  Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1014  EmitBlock(ThenBB);
1015  }
1016  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1017  llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1018  if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1019  auto IC = LoopDirective->counters().begin();
1020  for (const Expr *F : LoopDirective->finals()) {
1021  const auto *D =
1022  cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1023  if (NoFinals)
1024  AlreadyEmittedVars.insert(D);
1025  else
1026  LoopCountersAndUpdates[D] = F;
1027  ++IC;
1028  }
1029  }
1030  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1031  auto IRef = C->varlist_begin();
1032  auto ISrcRef = C->source_exprs().begin();
1033  auto IDestRef = C->destination_exprs().begin();
1034  for (const Expr *AssignOp : C->assignment_ops()) {
1035  const auto *PrivateVD =
1036  cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1037  QualType Type = PrivateVD->getType();
1038  const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1039  if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1040  // If lastprivate variable is a loop control variable for loop-based
1041  // directive, update its value before copyin back to original
1042  // variable.
1043  if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1044  EmitIgnoredExpr(FinalExpr);
1045  const auto *SrcVD =
1046  cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1047  const auto *DestVD =
1048  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1049  // Get the address of the original variable.
1050  Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1051  // Get the address of the private variable.
1052  Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
1053  if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1054  PrivateAddr =
1055  Address(Builder.CreateLoad(PrivateAddr),
1056  getNaturalTypeAlignment(RefTy->getPointeeType()));
1057  EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1058  }
1059  ++IRef;
1060  ++ISrcRef;
1061  ++IDestRef;
1062  }
1063  if (const Expr *PostUpdate = C->getPostUpdateExpr())
1064  EmitIgnoredExpr(PostUpdate);
1065  }
1066  if (IsLastIterCond)
1067  EmitBlock(DoneBB, /*IsFinished=*/true);
1068 }
1069 
1071  const OMPExecutableDirective &D,
1072  CodeGenFunction::OMPPrivateScope &PrivateScope) {
1073  if (!HaveInsertPoint())
1074  return;
1077  SmallVector<const Expr *, 4> ReductionOps;
1080  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1081  auto IPriv = C->privates().begin();
1082  auto IRed = C->reduction_ops().begin();
1083  auto ILHS = C->lhs_exprs().begin();
1084  auto IRHS = C->rhs_exprs().begin();
1085  for (const Expr *Ref : C->varlists()) {
1086  Shareds.emplace_back(Ref);
1087  Privates.emplace_back(*IPriv);
1088  ReductionOps.emplace_back(*IRed);
1089  LHSs.emplace_back(*ILHS);
1090  RHSs.emplace_back(*IRHS);
1091  std::advance(IPriv, 1);
1092  std::advance(IRed, 1);
1093  std::advance(ILHS, 1);
1094  std::advance(IRHS, 1);
1095  }
1096  }
1097  ReductionCodeGen RedCG(Shareds, Privates, ReductionOps);
1098  unsigned Count = 0;
1099  auto ILHS = LHSs.begin();
1100  auto IRHS = RHSs.begin();
1101  auto IPriv = Privates.begin();
1102  for (const Expr *IRef : Shareds) {
1103  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1104  // Emit private VarDecl with reduction init.
1105  RedCG.emitSharedLValue(*this, Count);
1106  RedCG.emitAggregateType(*this, Count);
1107  AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);
1108  RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
1109  RedCG.getSharedLValue(Count),
1110  [&Emission](CodeGenFunction &CGF) {
1111  CGF.EmitAutoVarInit(Emission);
1112  return true;
1113  });
1114  EmitAutoVarCleanups(Emission);
1115  Address BaseAddr = RedCG.adjustPrivateAddress(
1116  *this, Count, Emission.getAllocatedAddress());
1117  bool IsRegistered = PrivateScope.addPrivate(
1118  RedCG.getBaseDecl(Count), [BaseAddr]() { return BaseAddr; });
1119  assert(IsRegistered && "private var already registered as private");
1120  // Silence the warning about unused variable.
1121  (void)IsRegistered;
1122 
1123  const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1124  const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1125  QualType Type = PrivateVD->getType();
1126  bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1127  if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1128  // Store the address of the original variable associated with the LHS
1129  // implicit variable.
1130  PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
1131  return RedCG.getSharedLValue(Count).getAddress();
1132  });
1133  PrivateScope.addPrivate(
1134  RHSVD, [this, PrivateVD]() { return GetAddrOfLocalVar(PrivateVD); });
1135  } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1136  isa<ArraySubscriptExpr>(IRef)) {
1137  // Store the address of the original variable associated with the LHS
1138  // implicit variable.
1139  PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
1140  return RedCG.getSharedLValue(Count).getAddress();
1141  });
1142  PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() {
1143  return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD),
1144  ConvertTypeForMem(RHSVD->getType()),
1145  "rhs.begin");
1146  });
1147  } else {
1148  QualType Type = PrivateVD->getType();
1149  bool IsArray = getContext().getAsArrayType(Type) != nullptr;
1150  Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress();
1151  // Store the address of the original variable associated with the LHS
1152  // implicit variable.
1153  if (IsArray) {
1154  OriginalAddr = Builder.CreateElementBitCast(
1155  OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin");
1156  }
1157  PrivateScope.addPrivate(LHSVD, [OriginalAddr]() { return OriginalAddr; });
1158  PrivateScope.addPrivate(
1159  RHSVD, [this, PrivateVD, RHSVD, IsArray]() {
1160  return IsArray
1161  ? Builder.CreateElementBitCast(
1162  GetAddrOfLocalVar(PrivateVD),
1163  ConvertTypeForMem(RHSVD->getType()), "rhs.begin")
1164  : GetAddrOfLocalVar(PrivateVD);
1165  });
1166  }
1167  ++ILHS;
1168  ++IRHS;
1169  ++IPriv;
1170  ++Count;
1171  }
1172 }
1173 
1175  const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1176  if (!HaveInsertPoint())
1177  return;
1182  bool HasAtLeastOneReduction = false;
1183  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1184  HasAtLeastOneReduction = true;
1185  Privates.append(C->privates().begin(), C->privates().end());
1186  LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1187  RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1188  ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1189  }
1190  if (HasAtLeastOneReduction) {
1191  bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1193  ReductionKind == OMPD_simd;
1194  bool SimpleReduction = ReductionKind == OMPD_simd;
1195  // Emit nowait reduction if nowait clause is present or directive is a
1196  // parallel directive (it always has implicit barrier).
1197  CGM.getOpenMPRuntime().emitReduction(
1198  *this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1199  {WithNowait, SimpleReduction, ReductionKind});
1200  }
1201 }
1202 
1204  CodeGenFunction &CGF, const OMPExecutableDirective &D,
1205  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1206  if (!CGF.HaveInsertPoint())
1207  return;
1208  llvm::BasicBlock *DoneBB = nullptr;
1209  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1210  if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
1211  if (!DoneBB) {
1212  if (llvm::Value *Cond = CondGen(CGF)) {
1213  // If the first post-update expression is found, emit conditional
1214  // block if it was requested.
1215  llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1216  DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1217  CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1218  CGF.EmitBlock(ThenBB);
1219  }
1220  }
1221  CGF.EmitIgnoredExpr(PostUpdate);
1222  }
1223  }
1224  if (DoneBB)
1225  CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1226 }
1227 
1228 namespace {
1229 /// Codegen lambda for appending distribute lower and upper bounds to outlined
1230 /// parallel function. This is necessary for combined constructs such as
1231 /// 'distribute parallel for'
1232 typedef llvm::function_ref<void(CodeGenFunction &,
1233  const OMPExecutableDirective &,
1235  CodeGenBoundParametersTy;
1236 } // anonymous namespace
1237 
1239  CodeGenFunction &CGF, const OMPExecutableDirective &S,
1240  OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1241  const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1242  const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1243  llvm::Function *OutlinedFn =
1244  CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
1245  S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
1246  if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
1247  CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1248  llvm::Value *NumThreads =
1249  CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1250  /*IgnoreResultAssign=*/true);
1251  CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
1252  CGF, NumThreads, NumThreadsClause->getBeginLoc());
1253  }
1254  if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
1255  CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1256  CGF.CGM.getOpenMPRuntime().emitProcBindClause(
1257  CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1258  }
1259  const Expr *IfCond = nullptr;
1260  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1261  if (C->getNameModifier() == OMPD_unknown ||
1262  C->getNameModifier() == OMPD_parallel) {
1263  IfCond = C->getCondition();
1264  break;
1265  }
1266  }
1267 
1268  OMPParallelScope Scope(CGF, S);
1270  // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
1271  // lower and upper bounds with the pragma 'for' chunking mechanism.
1272  // The following lambda takes care of appending the lower and upper bound
1273  // parameters when necessary
1274  CodeGenBoundParameters(CGF, S, CapturedVars);
1275  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
1276  CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn,
1277  CapturedVars, IfCond);
1278 }
1279 
1281  const OMPExecutableDirective &,
1283 
1285  // Emit parallel region as a standalone region.
1286  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1287  Action.Enter(CGF);
1288  OMPPrivateScope PrivateScope(CGF);
1289  bool Copyins = CGF.EmitOMPCopyinClause(S);
1290  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1291  if (Copyins) {
1292  // Emit implicit barrier to synchronize threads and avoid data races on
1293  // propagation master's thread values of threadprivate variables to local
1294  // instances of that variables of all other implicit threads.
1295  CGF.CGM.getOpenMPRuntime().emitBarrierCall(
1296  CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
1297  /*ForceSimpleCall=*/true);
1298  }
1299  CGF.EmitOMPPrivateClause(S, PrivateScope);
1300  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1301  (void)PrivateScope.Privatize();
1302  CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1303  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
1304  };
1305  emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
1308  [](CodeGenFunction &) { return nullptr; });
1309 }
1310 
1312  JumpDest LoopExit) {
1313  RunCleanupsScope BodyScope(*this);
1314  // Update counters values on current iteration.
1315  for (const Expr *UE : D.updates())
1316  EmitIgnoredExpr(UE);
1317  // Update the linear variables.
1318  // In distribute directives only loop counters may be marked as linear, no
1319  // need to generate the code for them.
1321  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1322  for (const Expr *UE : C->updates())
1323  EmitIgnoredExpr(UE);
1324  }
1325  }
1326 
1327  // On a continue in the body, jump to the end.
1328  JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue");
1329  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1330  // Emit loop body.
1331  EmitStmt(D.getBody());
1332  // The end (updates/cleanups).
1333  EmitBlock(Continue.getBlock());
1334  BreakContinueStack.pop_back();
1335 }
1336 
1338  const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
1339  const Expr *IncExpr,
1340  const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
1341  const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
1342  auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
1343 
1344  // Start the loop with a block that tests the condition.
1345  auto CondBlock = createBasicBlock("omp.inner.for.cond");
1346  EmitBlock(CondBlock);
1347  const SourceRange R = S.getSourceRange();
1348  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1349  SourceLocToDebugLoc(R.getEnd()));
1350 
1351  // If there are any cleanups between here and the loop-exit scope,
1352  // create a block to stage a loop exit along.
1353  llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1354  if (RequiresCleanup)
1355  ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
1356 
1357  llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body");
1358 
1359  // Emit condition.
1360  EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
1361  if (ExitBlock != LoopExit.getBlock()) {
1362  EmitBlock(ExitBlock);
1363  EmitBranchThroughCleanup(LoopExit);
1364  }
1365 
1366  EmitBlock(LoopBody);
1367  incrementProfileCounter(&S);
1368 
1369  // Create a block for the increment.
1370  JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
1371  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1372 
1373  BodyGen(*this);
1374 
1375  // Emit "IV = IV + 1" and a back-edge to the condition block.
1376  EmitBlock(Continue.getBlock());
1377  EmitIgnoredExpr(IncExpr);
1378  PostIncGen(*this);
1379  BreakContinueStack.pop_back();
1380  EmitBranch(CondBlock);
1381  LoopStack.pop();
1382  // Emit the fall-through block.
1383  EmitBlock(LoopExit.getBlock());
1384 }
1385 
1387  if (!HaveInsertPoint())
1388  return false;
1389  // Emit inits for the linear variables.
1390  bool HasLinears = false;
1391  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1392  for (const Expr *Init : C->inits()) {
1393  HasLinears = true;
1394  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
1395  if (const auto *Ref =
1396  dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
1397  AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
1398  const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
1399  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1400  CapturedStmtInfo->lookup(OrigVD) != nullptr,
1401  VD->getInit()->getType(), VK_LValue,
1402  VD->getInit()->getExprLoc());
1403  EmitExprAsInit(&DRE, VD, MakeAddrLValue(Emission.getAllocatedAddress(),
1404  VD->getType()),
1405  /*capturedByInit=*/false);
1406  EmitAutoVarCleanups(Emission);
1407  } else {
1408  EmitVarDecl(*VD);
1409  }
1410  }
1411  // Emit the linear steps for the linear clauses.
1412  // If a step is not constant, it is pre-calculated before the loop.
1413  if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
1414  if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
1415  EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
1416  // Emit calculation of the linear step.
1417  EmitIgnoredExpr(CS);
1418  }
1419  }
1420  return HasLinears;
1421 }
1422 
1424  const OMPLoopDirective &D,
1425  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1426  if (!HaveInsertPoint())
1427  return;
1428  llvm::BasicBlock *DoneBB = nullptr;
1429  // Emit the final values of the linear variables.
1430  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1431  auto IC = C->varlist_begin();
1432  for (const Expr *F : C->finals()) {
1433  if (!DoneBB) {
1434  if (llvm::Value *Cond = CondGen(*this)) {
1435  // If the first post-update expression is found, emit conditional
1436  // block if it was requested.
1437  llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu");
1438  DoneBB = createBasicBlock(".omp.linear.pu.done");
1439  Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1440  EmitBlock(ThenBB);
1441  }
1442  }
1443  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
1444  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1445  CapturedStmtInfo->lookup(OrigVD) != nullptr,
1446  (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
1447  Address OrigAddr = EmitLValue(&DRE).getAddress();
1448  CodeGenFunction::OMPPrivateScope VarScope(*this);
1449  VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1450  (void)VarScope.Privatize();
1451  EmitIgnoredExpr(F);
1452  ++IC;
1453  }
1454  if (const Expr *PostUpdate = C->getPostUpdateExpr())
1455  EmitIgnoredExpr(PostUpdate);
1456  }
1457  if (DoneBB)
1458  EmitBlock(DoneBB, /*IsFinished=*/true);
1459 }
1460 
1462  const OMPExecutableDirective &D) {
1463  if (!CGF.HaveInsertPoint())
1464  return;
1465  for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
1466  unsigned ClauseAlignment = 0;
1467  if (const Expr *AlignmentExpr = Clause->getAlignment()) {
1468  auto *AlignmentCI =
1469  cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
1470  ClauseAlignment = static_cast<unsigned>(AlignmentCI->getZExtValue());
1471  }
1472  for (const Expr *E : Clause->varlists()) {
1473  unsigned Alignment = ClauseAlignment;
1474  if (Alignment == 0) {
1475  // OpenMP [2.8.1, Description]
1476  // If no optional parameter is specified, implementation-defined default
1477  // alignments for SIMD instructions on the target platforms are assumed.
1478  Alignment =
1479  CGF.getContext()
1481  E->getType()->getPointeeType()))
1482  .getQuantity();
1483  }
1484  assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) &&
1485  "alignment is not power of 2");
1486  if (Alignment != 0) {
1487  llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
1489  PtrValue, E, /*No second loc needed*/ SourceLocation(), Alignment);
1490  }
1491  }
1492  }
1493 }
1494 
1496  const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
1497  if (!HaveInsertPoint())
1498  return;
1499  auto I = S.private_counters().begin();
1500  for (const Expr *E : S.counters()) {
1501  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1502  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
1503  // Emit var without initialization.
1504  AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD);
1505  EmitAutoVarCleanups(VarEmission);
1506  LocalDeclMap.erase(PrivateVD);
1507  (void)LoopScope.addPrivate(VD, [&VarEmission]() {
1508  return VarEmission.getAllocatedAddress();
1509  });
1510  if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
1511  VD->hasGlobalStorage()) {
1512  (void)LoopScope.addPrivate(PrivateVD, [this, VD, E]() {
1513  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD),
1514  LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
1515  E->getType(), VK_LValue, E->getExprLoc());
1516  return EmitLValue(&DRE).getAddress();
1517  });
1518  } else {
1519  (void)LoopScope.addPrivate(PrivateVD, [&VarEmission]() {
1520  return VarEmission.getAllocatedAddress();
1521  });
1522  }
1523  ++I;
1524  }
1525  // Privatize extra loop counters used in loops for ordered(n) clauses.
1526  for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) {
1527  if (!C->getNumForLoops())
1528  continue;
1529  for (unsigned I = S.getCollapsedNumber(),
1530  E = C->getLoopNumIterations().size();
1531  I < E; ++I) {
1532  const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I));
1533  const auto *VD = cast<VarDecl>(DRE->getDecl());
1534  // Override only those variables that can be captured to avoid re-emission
1535  // of the variables declared within the loops.
1536  if (DRE->refersToEnclosingVariableOrCapture()) {
1537  (void)LoopScope.addPrivate(VD, [this, DRE, VD]() {
1538  return CreateMemTemp(DRE->getType(), VD->getName());
1539  });
1540  }
1541  }
1542  }
1543 }
1544 
1545 static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
1546  const Expr *Cond, llvm::BasicBlock *TrueBlock,
1547  llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
1548  if (!CGF.HaveInsertPoint())
1549  return;
1550  {
1551  CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
1552  CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
1553  (void)PreCondScope.Privatize();
1554  // Get initial values of real counters.
1555  for (const Expr *I : S.inits()) {
1556  CGF.EmitIgnoredExpr(I);
1557  }
1558  }
1559  // Check that loop is executed at least one time.
1560  CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
1561 }
1562 
1564  const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
1565  if (!HaveInsertPoint())
1566  return;
1569  const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1570  for (const Expr *C : LoopDirective->counters()) {
1571  SIMDLCVs.insert(
1572  cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
1573  }
1574  }
1575  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1576  auto CurPrivate = C->privates().begin();
1577  for (const Expr *E : C->varlists()) {
1578  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1579  const auto *PrivateVD =
1580  cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
1581  if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
1582  bool IsRegistered = PrivateScope.addPrivate(VD, [this, PrivateVD]() {
1583  // Emit private VarDecl with copy init.
1584  EmitVarDecl(*PrivateVD);
1585  return GetAddrOfLocalVar(PrivateVD);
1586  });
1587  assert(IsRegistered && "linear var already registered as private");
1588  // Silence the warning about unused variable.
1589  (void)IsRegistered;
1590  } else {
1591  EmitVarDecl(*PrivateVD);
1592  }
1593  ++CurPrivate;
1594  }
1595  }
1596 }
1597 
1599  const OMPExecutableDirective &D,
1600  bool IsMonotonic) {
1601  if (!CGF.HaveInsertPoint())
1602  return;
1603  if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
1604  RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
1605  /*ignoreResult=*/true);
1606  auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1607  CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1608  // In presence of finite 'safelen', it may be unsafe to mark all
1609  // the memory instructions parallel, because loop-carried
1610  // dependences of 'safelen' iterations are possible.
1611  if (!IsMonotonic)
1613  } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
1614  RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
1615  /*ignoreResult=*/true);
1616  auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1617  CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1618  // In presence of finite 'safelen', it may be unsafe to mark all
1619  // the memory instructions parallel, because loop-carried
1620  // dependences of 'safelen' iterations are possible.
1621  CGF.LoopStack.setParallel(/*Enable=*/false);
1622  }
1623 }
1624 
1626  bool IsMonotonic) {
1627  // Walk clauses and process safelen/lastprivate.
1628  LoopStack.setParallel(!IsMonotonic);
1629  LoopStack.setVectorizeEnable();
1630  emitSimdlenSafelenClause(*this, D, IsMonotonic);
1631 }
1632 
1634  const OMPLoopDirective &D,
1635  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1636  if (!HaveInsertPoint())
1637  return;
1638  llvm::BasicBlock *DoneBB = nullptr;
1639  auto IC = D.counters().begin();
1640  auto IPC = D.private_counters().begin();
1641  for (const Expr *F : D.finals()) {
1642  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
1643  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
1644  const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
1645  if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
1646  OrigVD->hasGlobalStorage() || CED) {
1647  if (!DoneBB) {
1648  if (llvm::Value *Cond = CondGen(*this)) {
1649  // If the first post-update expression is found, emit conditional
1650  // block if it was requested.
1651  llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then");
1652  DoneBB = createBasicBlock(".omp.final.done");
1653  Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1654  EmitBlock(ThenBB);
1655  }
1656  }
1657  Address OrigAddr = Address::invalid();
1658  if (CED) {
1659  OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();
1660  } else {
1661  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(PrivateVD),
1662  /*RefersToEnclosingVariableOrCapture=*/false,
1663  (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
1664  OrigAddr = EmitLValue(&DRE).getAddress();
1665  }
1666  OMPPrivateScope VarScope(*this);
1667  VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1668  (void)VarScope.Privatize();
1669  EmitIgnoredExpr(F);
1670  }
1671  ++IC;
1672  ++IPC;
1673  }
1674  if (DoneBB)
1675  EmitBlock(DoneBB, /*IsFinished=*/true);
1676 }
1677 
1679  const OMPLoopDirective &S,
1681  CGF.EmitOMPLoopBody(S, LoopExit);
1682  CGF.EmitStopPoint(&S);
1683 }
1684 
1685 /// Emit a helper variable and return corresponding lvalue.
1687  const DeclRefExpr *Helper) {
1688  auto VDecl = cast<VarDecl>(Helper->getDecl());
1689  CGF.EmitVarDecl(*VDecl);
1690  return CGF.EmitLValue(Helper);
1691 }
1692 
1694  PrePostActionTy &Action) {
1695  Action.Enter(CGF);
1697  "Expected simd directive");
1698  OMPLoopScope PreInitScope(CGF, S);
1699  // if (PreCond) {
1700  // for (IV in 0..LastIteration) BODY;
1701  // <Final counter/linear vars updates>;
1702  // }
1703  //
1707  (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));
1708  (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));
1709  }
1710 
1711  // Emit: if (PreCond) - begin.
1712  // If the condition constant folds and can be elided, avoid emitting the
1713  // whole loop.
1714  bool CondConstant;
1715  llvm::BasicBlock *ContBlock = nullptr;
1716  if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
1717  if (!CondConstant)
1718  return;
1719  } else {
1720  llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");
1721  ContBlock = CGF.createBasicBlock("simd.if.end");
1722  emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
1723  CGF.getProfileCount(&S));
1724  CGF.EmitBlock(ThenBlock);
1725  CGF.incrementProfileCounter(&S);
1726  }
1727 
1728  // Emit the loop iteration variable.
1729  const Expr *IVExpr = S.getIterationVariable();
1730  const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
1731  CGF.EmitVarDecl(*IVDecl);
1732  CGF.EmitIgnoredExpr(S.getInit());
1733 
1734  // Emit the iterations count variable.
1735  // If it is not a variable, Sema decided to calculate iterations count on
1736  // each iteration (e.g., it is foldable into a constant).
1737  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
1738  CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
1739  // Emit calculation of the iterations count.
1741  }
1742 
1743  CGF.EmitOMPSimdInit(S);
1744 
1745  emitAlignedClause(CGF, S);
1746  (void)CGF.EmitOMPLinearClauseInit(S);
1747  {
1748  CodeGenFunction::OMPPrivateScope LoopScope(CGF);
1749  CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
1750  CGF.EmitOMPLinearClause(S, LoopScope);
1751  CGF.EmitOMPPrivateClause(S, LoopScope);
1752  CGF.EmitOMPReductionClauseInit(S, LoopScope);
1753  bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
1754  (void)LoopScope.Privatize();
1756  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
1757  CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
1758  S.getInc(),
1759  [&S](CodeGenFunction &CGF) {
1761  CGF.EmitStopPoint(&S);
1762  },
1763  [](CodeGenFunction &) {});
1764  CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; });
1765  // Emit final copy of the lastprivate variables at the end of loops.
1766  if (HasLastprivateClause)
1767  CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
1768  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
1770  [](CodeGenFunction &) { return nullptr; });
1771  }
1772  CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; });
1773  // Emit: if (PreCond) - end.
1774  if (ContBlock) {
1775  CGF.EmitBranch(ContBlock);
1776  CGF.EmitBlock(ContBlock, true);
1777  }
1778 }
1779 
1781  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1782  emitOMPSimdRegion(CGF, S, Action);
1783  };
1784  OMPLexicalScope Scope(*this, S, OMPD_unknown);
1785  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
1786 }
1787 
1788 void CodeGenFunction::EmitOMPOuterLoop(
1789  bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
1791  const CodeGenFunction::OMPLoopArguments &LoopArgs,
1792  const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
1793  const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
1794  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1795 
1796  const Expr *IVExpr = S.getIterationVariable();
1797  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1798  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1799 
1800  JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
1801 
1802  // Start the loop with a block that tests the condition.
1803  llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond");
1804  EmitBlock(CondBlock);
1805  const SourceRange R = S.getSourceRange();
1806  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1807  SourceLocToDebugLoc(R.getEnd()));
1808 
1809  llvm::Value *BoolCondVal = nullptr;
1810  if (!DynamicOrOrdered) {
1811  // UB = min(UB, GlobalUB) or
1812  // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
1813  // 'distribute parallel for')
1814  EmitIgnoredExpr(LoopArgs.EUB);
1815  // IV = LB
1816  EmitIgnoredExpr(LoopArgs.Init);
1817  // IV < UB
1818  BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
1819  } else {
1820  BoolCondVal =
1821  RT.emitForNext(*this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
1822  LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
1823  }
1824 
1825  // If there are any cleanups between here and the loop-exit scope,
1826  // create a block to stage a loop exit along.
1827  llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1828  if (LoopScope.requiresCleanups())
1829  ExitBlock = createBasicBlock("omp.dispatch.cleanup");
1830 
1831  llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body");
1832  Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
1833  if (ExitBlock != LoopExit.getBlock()) {
1834  EmitBlock(ExitBlock);
1835  EmitBranchThroughCleanup(LoopExit);
1836  }
1837  EmitBlock(LoopBody);
1838 
1839  // Emit "IV = LB" (in case of static schedule, we have already calculated new
1840  // LB for loop condition and emitted it above).
1841  if (DynamicOrOrdered)
1842  EmitIgnoredExpr(LoopArgs.Init);
1843 
1844  // Create a block for the increment.
1845  JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
1846  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1847 
1848  // Generate !llvm.loop.parallel metadata for loads and stores for loops
1849  // with dynamic/guided scheduling and without ordered clause.
1851  LoopStack.setParallel(!IsMonotonic);
1852  else
1853  EmitOMPSimdInit(S, IsMonotonic);
1854 
1855  SourceLocation Loc = S.getBeginLoc();
1856 
1857  // when 'distribute' is not combined with a 'for':
1858  // while (idx <= UB) { BODY; ++idx; }
1859  // when 'distribute' is combined with a 'for'
1860  // (e.g. 'distribute parallel for')
1861  // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
1862  EmitOMPInnerLoop(
1863  S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
1864  [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
1865  CodeGenLoop(CGF, S, LoopExit);
1866  },
1867  [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
1868  CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
1869  });
1870 
1871  EmitBlock(Continue.getBlock());
1872  BreakContinueStack.pop_back();
1873  if (!DynamicOrOrdered) {
1874  // Emit "LB = LB + Stride", "UB = UB + Stride".
1875  EmitIgnoredExpr(LoopArgs.NextLB);
1876  EmitIgnoredExpr(LoopArgs.NextUB);
1877  }
1878 
1879  EmitBranch(CondBlock);
1880  LoopStack.pop();
1881  // Emit the fall-through block.
1882  EmitBlock(LoopExit.getBlock());
1883 
1884  // Tell the runtime we are done.
1885  auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) {
1886  if (!DynamicOrOrdered)
1887  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
1888  S.getDirectiveKind());
1889  };
1890  OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
1891 }
1892 
1893 void CodeGenFunction::EmitOMPForOuterLoop(
1894  const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
1895  const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
1896  const OMPLoopArguments &LoopArgs,
1897  const CodeGenDispatchBoundsTy &CGDispatchBounds) {
1898  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1899 
1900  // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
1901  const bool DynamicOrOrdered =
1902  Ordered || RT.isDynamic(ScheduleKind.Schedule);
1903 
1904  assert((Ordered ||
1905  !RT.isStaticNonchunked(ScheduleKind.Schedule,
1906  LoopArgs.Chunk != nullptr)) &&
1907  "static non-chunked schedule does not need outer loop");
1908 
1909  // Emit outer loop.
1910  //
1911  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1912  // When schedule(dynamic,chunk_size) is specified, the iterations are
1913  // distributed to threads in the team in chunks as the threads request them.
1914  // Each thread executes a chunk of iterations, then requests another chunk,
1915  // until no chunks remain to be distributed. Each chunk contains chunk_size
1916  // iterations, except for the last chunk to be distributed, which may have
1917  // fewer iterations. When no chunk_size is specified, it defaults to 1.
1918  //
1919  // When schedule(guided,chunk_size) is specified, the iterations are assigned
1920  // to threads in the team in chunks as the executing threads request them.
1921  // Each thread executes a chunk of iterations, then requests another chunk,
1922  // until no chunks remain to be assigned. For a chunk_size of 1, the size of
1923  // each chunk is proportional to the number of unassigned iterations divided
1924  // by the number of threads in the team, decreasing to 1. For a chunk_size
1925  // with value k (greater than 1), the size of each chunk is determined in the
1926  // same way, with the restriction that the chunks do not contain fewer than k
1927  // iterations (except for the last chunk to be assigned, which may have fewer
1928  // than k iterations).
1929  //
1930  // When schedule(auto) is specified, the decision regarding scheduling is
1931  // delegated to the compiler and/or runtime system. The programmer gives the
1932  // implementation the freedom to choose any possible mapping of iterations to
1933  // threads in the team.
1934  //
1935  // When schedule(runtime) is specified, the decision regarding scheduling is
1936  // deferred until run time, and the schedule and chunk size are taken from the
1937  // run-sched-var ICV. If the ICV is set to auto, the schedule is
1938  // implementation defined
1939  //
1940  // while(__kmpc_dispatch_next(&LB, &UB)) {
1941  // idx = LB;
1942  // while (idx <= UB) { BODY; ++idx;
1943  // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
1944  // } // inner loop
1945  // }
1946  //
1947  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1948  // When schedule(static, chunk_size) is specified, iterations are divided into
1949  // chunks of size chunk_size, and the chunks are assigned to the threads in
1950  // the team in a round-robin fashion in the order of the thread number.
1951  //
1952  // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
1953  // while (idx <= UB) { BODY; ++idx; } // inner loop
1954  // LB = LB + ST;
1955  // UB = UB + ST;
1956  // }
1957  //
1958 
1959  const Expr *IVExpr = S.getIterationVariable();
1960  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1961  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1962 
1963  if (DynamicOrOrdered) {
1964  const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
1965  CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
1966  llvm::Value *LBVal = DispatchBounds.first;
1967  llvm::Value *UBVal = DispatchBounds.second;
1968  CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
1969  LoopArgs.Chunk};
1970  RT.emitForDispatchInit(*this, S.getBeginLoc(), ScheduleKind, IVSize,
1971  IVSigned, Ordered, DipatchRTInputValues);
1972  } else {
1973  CGOpenMPRuntime::StaticRTInput StaticInit(
1974  IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
1975  LoopArgs.ST, LoopArgs.Chunk);
1976  RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
1977  ScheduleKind, StaticInit);
1978  }
1979 
1980  auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
1981  const unsigned IVSize,
1982  const bool IVSigned) {
1983  if (Ordered) {
1984  CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
1985  IVSigned);
1986  }
1987  };
1988 
1989  OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
1990  LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
1991  OuterLoopArgs.IncExpr = S.getInc();
1992  OuterLoopArgs.Init = S.getInit();
1993  OuterLoopArgs.Cond = S.getCond();
1994  OuterLoopArgs.NextLB = S.getNextLowerBound();
1995  OuterLoopArgs.NextUB = S.getNextUpperBound();
1996  EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
1997  emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
1998 }
1999 
2001  const unsigned IVSize, const bool IVSigned) {}
2002 
2003 void CodeGenFunction::EmitOMPDistributeOuterLoop(
2004  OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
2005  OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
2006  const CodeGenLoopTy &CodeGenLoopContent) {
2007 
2008  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
2009 
2010  // Emit outer loop.
2011  // Same behavior as a OMPForOuterLoop, except that schedule cannot be
2012  // dynamic
2013  //
2014 
2015  const Expr *IVExpr = S.getIterationVariable();
2016  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2017  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2018 
2019  CGOpenMPRuntime::StaticRTInput StaticInit(
2020  IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
2021  LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
2022  RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, StaticInit);
2023 
2024  // for combined 'distribute' and 'for' the increment expression of distribute
2025  // is stored in DistInc. For 'distribute' alone, it is in Inc.
2026  Expr *IncExpr;
2028  IncExpr = S.getDistInc();
2029  else
2030  IncExpr = S.getInc();
2031 
2032  // this routine is shared by 'omp distribute parallel for' and
2033  // 'omp distribute': select the right EUB expression depending on the
2034  // directive
2035  OMPLoopArguments OuterLoopArgs;
2036  OuterLoopArgs.LB = LoopArgs.LB;
2037  OuterLoopArgs.UB = LoopArgs.UB;
2038  OuterLoopArgs.ST = LoopArgs.ST;
2039  OuterLoopArgs.IL = LoopArgs.IL;
2040  OuterLoopArgs.Chunk = LoopArgs.Chunk;
2041  OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2043  : S.getEnsureUpperBound();
2044  OuterLoopArgs.IncExpr = IncExpr;
2045  OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2046  ? S.getCombinedInit()
2047  : S.getInit();
2048  OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2049  ? S.getCombinedCond()
2050  : S.getCond();
2051  OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2053  : S.getNextLowerBound();
2054  OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2056  : S.getNextUpperBound();
2057 
2058  EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
2059  LoopScope, OuterLoopArgs, CodeGenLoopContent,
2061 }
2062 
2063 static std::pair<LValue, LValue>
2065  const OMPExecutableDirective &S) {
2066  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2067  LValue LB =
2068  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2069  LValue UB =
2070  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2071 
2072  // When composing 'distribute' with 'for' (e.g. as in 'distribute
2073  // parallel for') we need to use the 'distribute'
2074  // chunk lower and upper bounds rather than the whole loop iteration
2075  // space. These are parameters to the outlined function for 'parallel'
2076  // and we copy the bounds of the previous schedule into the
2077  // the current ones.
2078  LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
2079  LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
2080  llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
2081  PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());
2082  PrevLBVal = CGF.EmitScalarConversion(
2083  PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
2084  LS.getIterationVariable()->getType(),
2086  llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
2087  PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());
2088  PrevUBVal = CGF.EmitScalarConversion(
2089  PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
2090  LS.getIterationVariable()->getType(),
2092 
2093  CGF.EmitStoreOfScalar(PrevLBVal, LB);
2094  CGF.EmitStoreOfScalar(PrevUBVal, UB);
2095 
2096  return {LB, UB};
2097 }
2098 
2099 /// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
2100 /// we need to use the LB and UB expressions generated by the worksharing
2101 /// code generation support, whereas in non combined situations we would
2102 /// just emit 0 and the LastIteration expression
2103 /// This function is necessary due to the difference of the LB and UB
2104 /// types for the RT emission routines for 'for_static_init' and
2105 /// 'for_dispatch_init'
2106 static std::pair<llvm::Value *, llvm::Value *>
2108  const OMPExecutableDirective &S,
2109  Address LB, Address UB) {
2110  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2111  const Expr *IVExpr = LS.getIterationVariable();
2112  // when implementing a dynamic schedule for a 'for' combined with a
2113  // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
2114  // is not normalized as each team only executes its own assigned
2115  // distribute chunk
2116  QualType IteratorTy = IVExpr->getType();
2117  llvm::Value *LBVal =
2118  CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
2119  llvm::Value *UBVal =
2120  CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
2121  return {LBVal, UBVal};
2122 }
2123 
2125  CodeGenFunction &CGF, const OMPExecutableDirective &S,
2126  llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) {
2127  const auto &Dir = cast<OMPLoopDirective>(S);
2128  LValue LB =
2129  CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
2130  llvm::Value *LBCast = CGF.Builder.CreateIntCast(
2131  CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2132  CapturedVars.push_back(LBCast);
2133  LValue UB =
2134  CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
2135 
2136  llvm::Value *UBCast = CGF.Builder.CreateIntCast(
2137  CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2138  CapturedVars.push_back(UBCast);
2139 }
2140 
2141 static void
2143  const OMPLoopDirective &S,
2145  auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
2146  PrePostActionTy &Action) {
2147  Action.Enter(CGF);
2148  bool HasCancel = false;
2150  if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
2151  HasCancel = D->hasCancel();
2152  else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
2153  HasCancel = D->hasCancel();
2154  else if (const auto *D =
2155  dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
2156  HasCancel = D->hasCancel();
2157  }
2159  HasCancel);
2163  };
2164 
2166  CGF, S,
2167  isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for,
2168  CGInlinedWorksharingLoop,
2170 }
2171 
2174  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2176  S.getDistInc());
2177  };
2178  OMPLexicalScope Scope(*this, S, OMPD_parallel);
2179  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2180 }
2181 
2184  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2186  S.getDistInc());
2187  };
2188  OMPLexicalScope Scope(*this, S, OMPD_parallel);
2189  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2190 }
2191 
2193  const OMPDistributeSimdDirective &S) {
2194  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2196  };
2197  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2198  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2199 }
2200 
2202  CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
2203  // Emit SPMD target parallel for region as a standalone region.
2204  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2205  emitOMPSimdRegion(CGF, S, Action);
2206  };
2207  llvm::Function *Fn;
2208  llvm::Constant *Addr;
2209  // Emit target region as a standalone region.
2210  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
2211  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
2212  assert(Fn && Addr && "Target device function emission failed.");
2213 }
2214 
2216  const OMPTargetSimdDirective &S) {
2217  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2218  emitOMPSimdRegion(CGF, S, Action);
2219  };
2220  emitCommonOMPTargetDirective(*this, S, CodeGen);
2221 }
2222 
2223 namespace {
2224  struct ScheduleKindModifiersTy {
2228  ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
2231  : Kind(Kind), M1(M1), M2(M2) {}
2232  };
2233 } // namespace
2234 
2236  const OMPLoopDirective &S, Expr *EUB,
2237  const CodeGenLoopBoundsTy &CodeGenLoopBounds,
2238  const CodeGenDispatchBoundsTy &CGDispatchBounds) {
2239  // Emit the loop iteration variable.
2240  const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
2241  const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
2242  EmitVarDecl(*IVDecl);
2243 
2244  // Emit the iterations count variable.
2245  // If it is not a variable, Sema decided to calculate iterations count on each
2246  // iteration (e.g., it is foldable into a constant).
2247  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2248  EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2249  // Emit calculation of the iterations count.
2250  EmitIgnoredExpr(S.getCalcLastIteration());
2251  }
2252 
2253  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
2254 
2255  bool HasLastprivateClause;
2256  // Check pre-condition.
2257  {
2258  OMPLoopScope PreInitScope(*this, S);
2259  // Skip the entire loop if we don't meet the precondition.
2260  // If the condition constant folds and can be elided, avoid emitting the
2261  // whole loop.
2262  bool CondConstant;
2263  llvm::BasicBlock *ContBlock = nullptr;
2264  if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2265  if (!CondConstant)
2266  return false;
2267  } else {
2268  llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
2269  ContBlock = createBasicBlock("omp.precond.end");
2270  emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
2271  getProfileCount(&S));
2272  EmitBlock(ThenBlock);
2273  incrementProfileCounter(&S);
2274  }
2275 
2276  RunCleanupsScope DoacrossCleanupScope(*this);
2277  bool Ordered = false;
2278  if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
2279  if (OrderedClause->getNumForLoops())
2280  RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations());
2281  else
2282  Ordered = true;
2283  }
2284 
2285  llvm::DenseSet<const Expr *> EmittedFinals;
2286  emitAlignedClause(*this, S);
2287  bool HasLinears = EmitOMPLinearClauseInit(S);
2288  // Emit helper vars inits.
2289 
2290  std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
2291  LValue LB = Bounds.first;
2292  LValue UB = Bounds.second;
2293  LValue ST =
2294  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
2295  LValue IL =
2296  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
2297 
2298  // Emit 'then' code.
2299  {
2300  OMPPrivateScope LoopScope(*this);
2301  if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
2302  // Emit implicit barrier to synchronize threads and avoid data races on
2303  // initialization of firstprivate variables and post-update of
2304  // lastprivate variables.
2305  CGM.getOpenMPRuntime().emitBarrierCall(
2306  *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
2307  /*ForceSimpleCall=*/true);
2308  }
2309  EmitOMPPrivateClause(S, LoopScope);
2310  HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
2311  EmitOMPReductionClauseInit(S, LoopScope);
2312  EmitOMPPrivateLoopCounters(S, LoopScope);
2313  EmitOMPLinearClause(S, LoopScope);
2314  (void)LoopScope.Privatize();
2316  CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
2317 
2318  // Detect the loop schedule kind and chunk.
2319  const Expr *ChunkExpr = nullptr;
2320  OpenMPScheduleTy ScheduleKind;
2321  if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
2322  ScheduleKind.Schedule = C->getScheduleKind();
2323  ScheduleKind.M1 = C->getFirstScheduleModifier();
2324  ScheduleKind.M2 = C->getSecondScheduleModifier();
2325  ChunkExpr = C->getChunkSize();
2326  } else {
2327  // Default behaviour for schedule clause.
2328  CGM.getOpenMPRuntime().getDefaultScheduleAndChunk(
2329  *this, S, ScheduleKind.Schedule, ChunkExpr);
2330  }
2331  bool HasChunkSizeOne = false;
2332  llvm::Value *Chunk = nullptr;
2333  if (ChunkExpr) {
2334  Chunk = EmitScalarExpr(ChunkExpr);
2335  Chunk = EmitScalarConversion(Chunk, ChunkExpr->getType(),
2337  S.getBeginLoc());
2338  Expr::EvalResult Result;
2339  if (ChunkExpr->EvaluateAsInt(Result, getContext())) {
2340  llvm::APSInt EvaluatedChunk = Result.Val.getInt();
2341  HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
2342  }
2343  }
2344  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2345  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2346  // OpenMP 4.5, 2.7.1 Loop Construct, Description.
2347  // If the static schedule kind is specified or if the ordered clause is
2348  // specified, and if no monotonic modifier is specified, the effect will
2349  // be as if the monotonic modifier was specified.
2350  bool StaticChunkedOne = RT.isStaticChunked(ScheduleKind.Schedule,
2351  /* Chunked */ Chunk != nullptr) && HasChunkSizeOne &&
2353  if ((RT.isStaticNonchunked(ScheduleKind.Schedule,
2354  /* Chunked */ Chunk != nullptr) ||
2355  StaticChunkedOne) &&
2356  !Ordered) {
2358  EmitOMPSimdInit(S, /*IsMonotonic=*/true);
2359  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2360  // When no chunk_size is specified, the iteration space is divided into
2361  // chunks that are approximately equal in size, and at most one chunk is
2362  // distributed to each thread. Note that the size of the chunks is
2363  // unspecified in this case.
2364  CGOpenMPRuntime::StaticRTInput StaticInit(
2365  IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
2366  UB.getAddress(), ST.getAddress(),
2367  StaticChunkedOne ? Chunk : nullptr);
2368  RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
2369  ScheduleKind, StaticInit);
2370  JumpDest LoopExit =
2371  getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
2372  // UB = min(UB, GlobalUB);
2373  if (!StaticChunkedOne)
2374  EmitIgnoredExpr(S.getEnsureUpperBound());
2375  // IV = LB;
2376  EmitIgnoredExpr(S.getInit());
2377  // For unchunked static schedule generate:
2378  //
2379  // while (idx <= UB) {
2380  // BODY;
2381  // ++idx;
2382  // }
2383  //
2384  // For static schedule with chunk one:
2385  //
2386  // while (IV <= PrevUB) {
2387  // BODY;
2388  // IV += ST;
2389  // }
2390  EmitOMPInnerLoop(S, LoopScope.requiresCleanups(),
2391  StaticChunkedOne ? S.getCombinedParForInDistCond() : S.getCond(),
2392  StaticChunkedOne ? S.getDistInc() : S.getInc(),
2393  [&S, LoopExit](CodeGenFunction &CGF) {
2394  CGF.EmitOMPLoopBody(S, LoopExit);
2395  CGF.EmitStopPoint(&S);
2396  },
2397  [](CodeGenFunction &) {});
2398  EmitBlock(LoopExit.getBlock());
2399  // Tell the runtime we are done.
2400  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2401  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2402  S.getDirectiveKind());
2403  };
2404  OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
2405  } else {
2406  const bool IsMonotonic =
2407  Ordered || ScheduleKind.Schedule == OMPC_SCHEDULE_static ||
2408  ScheduleKind.Schedule == OMPC_SCHEDULE_unknown ||
2409  ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
2410  ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
2411  // Emit the outer loop, which requests its work chunk [LB..UB] from
2412  // runtime and runs the inner loop to process it.
2413  const OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),
2414  ST.getAddress(), IL.getAddress(),
2415  Chunk, EUB);
2416  EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
2417  LoopArguments, CGDispatchBounds);
2418  }
2420  EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
2421  return CGF.Builder.CreateIsNotNull(
2422  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2423  });
2424  }
2425  EmitOMPReductionClauseFinal(
2426  S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind())
2427  ? /*Parallel and Simd*/ OMPD_parallel_for_simd
2428  : /*Parallel only*/ OMPD_parallel);
2429  // Emit post-update of the reduction variables if IsLastIter != 0.
2431  *this, S, [IL, &S](CodeGenFunction &CGF) {
2432  return CGF.Builder.CreateIsNotNull(
2433  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2434  });
2435  // Emit final copy of the lastprivate variables if IsLastIter != 0.
2436  if (HasLastprivateClause)
2437  EmitOMPLastprivateClauseFinal(
2439  Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
2440  }
2441  EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) {
2442  return CGF.Builder.CreateIsNotNull(
2443  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2444  });
2445  DoacrossCleanupScope.ForceCleanup();
2446  // We're now done with the loop, so jump to the continuation block.
2447  if (ContBlock) {
2448  EmitBranch(ContBlock);
2449  EmitBlock(ContBlock, /*IsFinished=*/true);
2450  }
2451  }
2452  return HasLastprivateClause;
2453 }
2454 
2455 /// The following two functions generate expressions for the loop lower
2456 /// and upper bounds in case of static and dynamic (dispatch) schedule
2457 /// of the associated 'for' or 'distribute' loop.
2458 static std::pair<LValue, LValue>
2460  const auto &LS = cast<OMPLoopDirective>(S);
2461  LValue LB =
2462  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2463  LValue UB =
2464  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2465  return {LB, UB};
2466 }
2467 
2468 /// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
2469 /// consider the lower and upper bound expressions generated by the
2470 /// worksharing loop support, but we use 0 and the iteration space size as
2471 /// constants
2472 static std::pair<llvm::Value *, llvm::Value *>
2474  Address LB, Address UB) {
2475  const auto &LS = cast<OMPLoopDirective>(S);
2476  const Expr *IVExpr = LS.getIterationVariable();
2477  const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
2478  llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
2479  llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
2480  return {LBVal, UBVal};
2481 }
2482 
2484  bool HasLastprivates = false;
2485  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2486  PrePostActionTy &) {
2487  OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel());
2488  HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2491  };
2492  {
2493  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2494  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
2495  S.hasCancel());
2496  }
2497 
2498  // Emit an implicit barrier at the end.
2499  if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
2500  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
2501 }
2502 
2504  bool HasLastprivates = false;
2505  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2506  PrePostActionTy &) {
2507  HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2510  };
2511  {
2512  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2513  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2514  }
2515 
2516  // Emit an implicit barrier at the end.
2517  if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
2518  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
2519 }
2520 
2522  const Twine &Name,
2523  llvm::Value *Init = nullptr) {
2524  LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
2525  if (Init)
2526  CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
2527  return LVal;
2528 }
2529 
2530 void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
2532  const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
2533  bool HasLastprivates = false;
2534  auto &&CodeGen = [&S, CapturedStmt, CS,
2535  &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
2536  ASTContext &C = CGF.getContext();
2537  QualType KmpInt32Ty =
2538  C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2539  // Emit helper vars inits.
2540  LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
2541  CGF.Builder.getInt32(0));
2542  llvm::ConstantInt *GlobalUBVal = CS != nullptr
2543  ? CGF.Builder.getInt32(CS->size() - 1)
2544  : CGF.Builder.getInt32(0);
2545  LValue UB =
2546  createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
2547  LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
2548  CGF.Builder.getInt32(1));
2549  LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
2550  CGF.Builder.getInt32(0));
2551  // Loop counter.
2552  LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
2553  OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
2554  CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
2555  OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
2556  CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
2557  // Generate condition for loop.
2558  BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue,
2560  // Increment for loop counter.
2561  UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary,
2562  S.getBeginLoc(), true);
2563  auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
2564  // Iterate through all sections and emit a switch construct:
2565  // switch (IV) {
2566  // case 0:
2567  // <SectionStmt[0]>;
2568  // break;
2569  // ...
2570  // case <NumSection> - 1:
2571  // <SectionStmt[<NumSection> - 1]>;
2572  // break;
2573  // }
2574  // .omp.sections.exit:
2575  llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
2576  llvm::SwitchInst *SwitchStmt =
2577  CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
2578  ExitBB, CS == nullptr ? 1 : CS->size());
2579  if (CS) {
2580  unsigned CaseNumber = 0;
2581  for (const Stmt *SubStmt : CS->children()) {
2582  auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2583  CGF.EmitBlock(CaseBB);
2584  SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
2585  CGF.EmitStmt(SubStmt);
2586  CGF.EmitBranch(ExitBB);
2587  ++CaseNumber;
2588  }
2589  } else {
2590  llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");
2591  CGF.EmitBlock(CaseBB);
2592  SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
2593  CGF.EmitStmt(CapturedStmt);
2594  CGF.EmitBranch(ExitBB);
2595  }
2596  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2597  };
2598 
2599  CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2600  if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
2601  // Emit implicit barrier to synchronize threads and avoid data races on
2602  // initialization of firstprivate variables and post-update of lastprivate
2603  // variables.
2604  CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2605  CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
2606  /*ForceSimpleCall=*/true);
2607  }
2608  CGF.EmitOMPPrivateClause(S, LoopScope);
2609  HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2610  CGF.EmitOMPReductionClauseInit(S, LoopScope);
2611  (void)LoopScope.Privatize();
2612  if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
2613  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
2614 
2615  // Emit static non-chunked loop.
2616  OpenMPScheduleTy ScheduleKind;
2617  ScheduleKind.Schedule = OMPC_SCHEDULE_static;
2618  CGOpenMPRuntime::StaticRTInput StaticInit(
2619  /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(),
2620  LB.getAddress(), UB.getAddress(), ST.getAddress());
2621  CGF.CGM.getOpenMPRuntime().emitForStaticInit(
2622  CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit);
2623  // UB = min(UB, GlobalUB);
2624  llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
2625  llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
2626  CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
2627  CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
2628  // IV = LB;
2629  CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
2630  // while (idx <= UB) { BODY; ++idx; }
2631  CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen,
2632  [](CodeGenFunction &) {});
2633  // Tell the runtime we are done.
2634  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2635  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2636  S.getDirectiveKind());
2637  };
2638  CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
2639  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
2640  // Emit post-update of the reduction variables if IsLastIter != 0.
2641  emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) {
2642  return CGF.Builder.CreateIsNotNull(
2643  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2644  });
2645 
2646  // Emit final copy of the lastprivate variables if IsLastIter != 0.
2647  if (HasLastprivates)
2649  S, /*NoFinals=*/false,
2650  CGF.Builder.CreateIsNotNull(
2651  CGF.EmitLoadOfScalar(IL, S.getBeginLoc())));
2652  };
2653 
2654  bool HasCancel = false;
2655  if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
2656  HasCancel = OSD->hasCancel();
2657  else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
2658  HasCancel = OPSD->hasCancel();
2659  OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
2660  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
2661  HasCancel);
2662  // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
2663  // clause. Otherwise the barrier will be generated by the codegen for the
2664  // directive.
2665  if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
2666  // Emit implicit barrier to synchronize threads and avoid data races on
2667  // initialization of firstprivate variables.
2668  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
2669  OMPD_unknown);
2670  }
2671 }
2672 
2674  {
2675  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2676  EmitSections(S);
2677  }
2678  // Emit an implicit barrier at the end.
2679  if (!S.getSingleClause<OMPNowaitClause>()) {
2680  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
2681  OMPD_sections);
2682  }
2683 }
2684 
2686  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2688  };
2689  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2690  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen,
2691  S.hasCancel());
2692 }
2693 
2695  llvm::SmallVector<const Expr *, 8> CopyprivateVars;
2698  llvm::SmallVector<const Expr *, 8> AssignmentOps;
2699  // Check if there are any 'copyprivate' clauses associated with this
2700  // 'single' construct.
2701  // Build a list of copyprivate variables along with helper expressions
2702  // (<source>, <destination>, <destination>=<source> expressions)
2703  for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
2704  CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
2705  DestExprs.append(C->destination_exprs().begin(),
2706  C->destination_exprs().end());
2707  SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
2708  AssignmentOps.append(C->assignment_ops().begin(),
2709  C->assignment_ops().end());
2710  }
2711  // Emit code for 'single' region along with 'copyprivate' clauses
2712  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2713  Action.Enter(CGF);
2714  OMPPrivateScope SingleScope(CGF);
2715  (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
2716  CGF.EmitOMPPrivateClause(S, SingleScope);
2717  (void)SingleScope.Privatize();
2719  };
2720  {
2721  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2722  CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getBeginLoc(),
2723  CopyprivateVars, DestExprs,
2724  SrcExprs, AssignmentOps);
2725  }
2726  // Emit an implicit barrier at the end (to avoid data race on firstprivate
2727  // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
2728  if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
2729  CGM.getOpenMPRuntime().emitBarrierCall(
2730  *this, S.getBeginLoc(),
2731  S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
2732  }
2733 }
2734 
2736  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2737  Action.Enter(CGF);
2739  };
2740  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2741  CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
2742 }
2743 
2745  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2746  Action.Enter(CGF);
2748  };
2749  const Expr *Hint = nullptr;
2750  if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
2751  Hint = HintClause->getHint();
2752  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2753  CGM.getOpenMPRuntime().emitCriticalRegion(*this,
2755  CodeGen, S.getBeginLoc(), Hint);
2756 }
2757 
2759  const OMPParallelForDirective &S) {
2760  // Emit directive as a combined directive that consists of two implicit
2761  // directives: 'parallel' with 'for' directive.
2762  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2763  Action.Enter(CGF);
2764  OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel());
2767  };
2768  emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
2770 }
2771 
2773  const OMPParallelForSimdDirective &S) {
2774  // Emit directive as a combined directive that consists of two implicit
2775  // directives: 'parallel' with 'for' directive.
2776  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2777  Action.Enter(CGF);
2780  };
2781  emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen,
2783 }
2784 
2786  const OMPParallelSectionsDirective &S) {
2787  // Emit directive as a combined directive that consists of two implicit
2788  // directives: 'parallel' with 'sections' directive.
2789  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2790  Action.Enter(CGF);
2791  CGF.EmitSections(S);
2792  };
2793  emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
2795 }
2796 
2798  const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
2799  const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
2800  OMPTaskDataTy &Data) {
2801  // Emit outlined function for task construct.
2802  const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
2803  auto I = CS->getCapturedDecl()->param_begin();
2804  auto PartId = std::next(I);
2805  auto TaskT = std::next(I, 4);
2806  // Check if the task is final
2807  if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
2808  // If the condition constant folds and can be elided, try to avoid emitting
2809  // the condition and the dead arm of the if/else.
2810  const Expr *Cond = Clause->getCondition();
2811  bool CondConstant;
2812  if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
2813  Data.Final.setInt(CondConstant);
2814  else
2815  Data.Final.setPointer(EvaluateExprAsBool(Cond));
2816  } else {
2817  // By default the task is not final.
2818  Data.Final.setInt(/*IntVal=*/false);
2819  }
2820  // Check if the task has 'priority' clause.
2821  if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
2822  const Expr *Prio = Clause->getPriority();
2823  Data.Priority.setInt(/*IntVal=*/true);
2824  Data.Priority.setPointer(EmitScalarConversion(
2825  EmitScalarExpr(Prio), Prio->getType(),
2826  getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
2827  Prio->getExprLoc()));
2828  }
2829  // The first function argument for tasks is a thread id, the second one is a
2830  // part id (0 for tied tasks, >=0 for untied task).
2831  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
2832  // Get list of private variables.
2833  for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
2834  auto IRef = C->varlist_begin();
2835  for (const Expr *IInit : C->private_copies()) {
2836  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2837  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2838  Data.PrivateVars.push_back(*IRef);
2839  Data.PrivateCopies.push_back(IInit);
2840  }
2841  ++IRef;
2842  }
2843  }
2844  EmittedAsPrivate.clear();
2845  // Get list of firstprivate variables.
2846  for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
2847  auto IRef = C->varlist_begin();
2848  auto IElemInitRef = C->inits().begin();
2849  for (const Expr *IInit : C->private_copies()) {
2850  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2851  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2852  Data.FirstprivateVars.push_back(*IRef);
2853  Data.FirstprivateCopies.push_back(IInit);
2854  Data.FirstprivateInits.push_back(*IElemInitRef);
2855  }
2856  ++IRef;
2857  ++IElemInitRef;
2858  }
2859  }
2860  // Get list of lastprivate variables (for taskloops).
2861  llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
2862  for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
2863  auto IRef = C->varlist_begin();
2864  auto ID = C->destination_exprs().begin();
2865  for (const Expr *IInit : C->private_copies()) {
2866  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2867  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2868  Data.LastprivateVars.push_back(*IRef);
2869  Data.LastprivateCopies.push_back(IInit);
2870  }
2871  LastprivateDstsOrigs.insert(
2872  {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
2873  cast<DeclRefExpr>(*IRef)});
2874  ++IRef;
2875  ++ID;
2876  }
2877  }
2880  for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
2881  auto IPriv = C->privates().begin();
2882  auto IRed = C->reduction_ops().begin();
2883  auto ILHS = C->lhs_exprs().begin();
2884  auto IRHS = C->rhs_exprs().begin();
2885  for (const Expr *Ref : C->varlists()) {
2886  Data.ReductionVars.emplace_back(Ref);
2887  Data.ReductionCopies.emplace_back(*IPriv);
2888  Data.ReductionOps.emplace_back(*IRed);
2889  LHSs.emplace_back(*ILHS);
2890  RHSs.emplace_back(*IRHS);
2891  std::advance(IPriv, 1);
2892  std::advance(IRed, 1);
2893  std::advance(ILHS, 1);
2894  std::advance(IRHS, 1);
2895  }
2896  }
2897  Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
2898  *this, S.getBeginLoc(), LHSs, RHSs, Data);
2899  // Build list of dependences.
2900  for (const auto *C : S.getClausesOfKind<OMPDependClause>())
2901  for (const Expr *IRef : C->varlists())
2902  Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
2903  auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
2904  CapturedRegion](CodeGenFunction &CGF,
2905  PrePostActionTy &Action) {
2906  // Set proper addresses for generated private copies.
2907  OMPPrivateScope Scope(CGF);
2908  if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
2909  !Data.LastprivateVars.empty()) {
2910  llvm::FunctionType *CopyFnTy = llvm::FunctionType::get(
2911  CGF.Builder.getVoidTy(), {CGF.Builder.getInt8PtrTy()}, true);
2912  enum { PrivatesParam = 2, CopyFnParam = 3 };
2913  llvm::Value *CopyFn = CGF.Builder.CreateLoad(
2914  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
2915  llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
2916  CS->getCapturedDecl()->getParam(PrivatesParam)));
2917  // Map privates.
2920  CallArgs.push_back(PrivatesPtr);
2921  for (const Expr *E : Data.PrivateVars) {
2922  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2923  Address PrivatePtr = CGF.CreateMemTemp(
2924  CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
2925  PrivatePtrs.emplace_back(VD, PrivatePtr);
2926  CallArgs.push_back(PrivatePtr.getPointer());
2927  }
2928  for (const Expr *E : Data.FirstprivateVars) {
2929  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2930  Address PrivatePtr =
2931  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2932  ".firstpriv.ptr.addr");
2933  PrivatePtrs.emplace_back(VD, PrivatePtr);
2934  CallArgs.push_back(PrivatePtr.getPointer());
2935  }
2936  for (const Expr *E : Data.LastprivateVars) {
2937  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2938  Address PrivatePtr =
2939  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2940  ".lastpriv.ptr.addr");
2941  PrivatePtrs.emplace_back(VD, PrivatePtr);
2942  CallArgs.push_back(PrivatePtr.getPointer());
2943  }
2944  CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
2945  CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
2946  for (const auto &Pair : LastprivateDstsOrigs) {
2947  const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
2948  DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD),
2949  /*RefersToEnclosingVariableOrCapture=*/
2950  CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr,
2951  Pair.second->getType(), VK_LValue,
2952  Pair.second->getExprLoc());
2953  Scope.addPrivate(Pair.first, [&CGF, &DRE]() {
2954  return CGF.EmitLValue(&DRE).getAddress();
2955  });
2956  }
2957  for (const auto &Pair : PrivatePtrs) {
2958  Address Replacement(CGF.Builder.CreateLoad(Pair.second),
2959  CGF.getContext().getDeclAlign(Pair.first));
2960  Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
2961  }
2962  }
2963  if (Data.Reductions) {
2964  OMPLexicalScope LexScope(CGF, S, CapturedRegion);
2966  Data.ReductionOps);
2967  llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
2968  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
2969  for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
2970  RedCG.emitSharedLValue(CGF, Cnt);
2971  RedCG.emitAggregateType(CGF, Cnt);
2972  // FIXME: This must removed once the runtime library is fixed.
2973  // Emit required threadprivate variables for
2974  // initializer/combiner/finalizer.
2975  CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
2976  RedCG, Cnt);
2977  Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
2978  CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
2979  Replacement =
2980  Address(CGF.EmitScalarConversion(
2981  Replacement.getPointer(), CGF.getContext().VoidPtrTy,
2982  CGF.getContext().getPointerType(
2983  Data.ReductionCopies[Cnt]->getType()),
2984  Data.ReductionCopies[Cnt]->getExprLoc()),
2985  Replacement.getAlignment());
2986  Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
2987  Scope.addPrivate(RedCG.getBaseDecl(Cnt),
2988  [Replacement]() { return Replacement; });
2989  }
2990  }
2991  // Privatize all private variables except for in_reduction items.
2992  (void)Scope.Privatize();
2993  SmallVector<const Expr *, 4> InRedVars;
2994  SmallVector<const Expr *, 4> InRedPrivs;
2996  SmallVector<const Expr *, 4> TaskgroupDescriptors;
2997  for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
2998  auto IPriv = C->privates().begin();
2999  auto IRed = C->reduction_ops().begin();
3000  auto ITD = C->taskgroup_descriptors().begin();
3001  for (const Expr *Ref : C->varlists()) {
3002  InRedVars.emplace_back(Ref);
3003  InRedPrivs.emplace_back(*IPriv);
3004  InRedOps.emplace_back(*IRed);
3005  TaskgroupDescriptors.emplace_back(*ITD);
3006  std::advance(IPriv, 1);
3007  std::advance(IRed, 1);
3008  std::advance(ITD, 1);
3009  }
3010  }
3011  // Privatize in_reduction items here, because taskgroup descriptors must be
3012  // privatized earlier.
3013  OMPPrivateScope InRedScope(CGF);
3014  if (!InRedVars.empty()) {
3015  ReductionCodeGen RedCG(InRedVars, InRedPrivs, InRedOps);
3016  for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
3017  RedCG.emitSharedLValue(CGF, Cnt);
3018  RedCG.emitAggregateType(CGF, Cnt);
3019  // The taskgroup descriptor variable is always implicit firstprivate and
3020  // privatized already during processing of the firstprivates.
3021  // FIXME: This must removed once the runtime library is fixed.
3022  // Emit required threadprivate variables for
3023  // initializer/combiner/finalizer.
3024  CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
3025  RedCG, Cnt);
3026  llvm::Value *ReductionsPtr =
3027  CGF.EmitLoadOfScalar(CGF.EmitLValue(TaskgroupDescriptors[Cnt]),
3028  TaskgroupDescriptors[Cnt]->getExprLoc());
3029  Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
3030  CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
3031  Replacement = Address(
3032  CGF.EmitScalarConversion(
3033  Replacement.getPointer(), CGF.getContext().VoidPtrTy,
3034  CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
3035  InRedPrivs[Cnt]->getExprLoc()),
3036  Replacement.getAlignment());
3037  Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
3038  InRedScope.addPrivate(RedCG.getBaseDecl(Cnt),
3039  [Replacement]() { return Replacement; });
3040  }
3041  }
3042  (void)InRedScope.Privatize();
3043 
3044  Action.Enter(CGF);
3045  BodyGen(CGF);
3046  };
3047  llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3048  S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied,
3049  Data.NumberOfParts);
3050  OMPLexicalScope Scope(*this, S);
3051  TaskGen(*this, OutlinedFn, Data);
3052 }
3053 
3054 static ImplicitParamDecl *
3056  QualType Ty, CapturedDecl *CD,
3057  SourceLocation Loc) {
3058  auto *OrigVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3060  auto *OrigRef = DeclRefExpr::Create(
3061  C, NestedNameSpecifierLoc(), SourceLocation(), OrigVD,
3062  /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
3063  auto *PrivateVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3065  auto *PrivateRef = DeclRefExpr::Create(
3066  C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD,
3067  /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
3068  QualType ElemType = C.getBaseElementType(Ty);
3069  auto *InitVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, ElemType,
3071  auto *InitRef = DeclRefExpr::Create(
3072  C, NestedNameSpecifierLoc(), SourceLocation(), InitVD,
3073  /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue);
3074  PrivateVD->setInitStyle(VarDecl::CInit);
3075  PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue,
3076  InitRef, /*BasePath=*/nullptr,
3077  VK_RValue));
3078  Data.FirstprivateVars.emplace_back(OrigRef);
3079  Data.FirstprivateCopies.emplace_back(PrivateRef);
3080  Data.FirstprivateInits.emplace_back(InitRef);
3081  return OrigVD;
3082 }
3083 
3085  const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,
3086  OMPTargetDataInfo &InputInfo) {
3087  // Emit outlined function for task construct.
3088  const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3089  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3090  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3091  auto I = CS->getCapturedDecl()->param_begin();
3092  auto PartId = std::next(I);
3093  auto TaskT = std::next(I, 4);
3094  OMPTaskDataTy Data;
3095  // The task is not final.
3096  Data.Final.setInt(/*IntVal=*/false);
3097  // Get list of firstprivate variables.
3098  for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
3099  auto IRef = C->varlist_begin();
3100  auto IElemInitRef = C->inits().begin();
3101  for (auto *IInit : C->private_copies()) {
3102  Data.FirstprivateVars.push_back(*IRef);
3103  Data.FirstprivateCopies.push_back(IInit);
3104  Data.FirstprivateInits.push_back(*IElemInitRef);
3105  ++IRef;
3106  ++IElemInitRef;
3107  }
3108  }
3109  OMPPrivateScope TargetScope(*this);
3110  VarDecl *BPVD = nullptr;
3111  VarDecl *PVD = nullptr;
3112  VarDecl *SVD = nullptr;
3113  if (InputInfo.NumberOfTargetItems > 0) {
3114  auto *CD = CapturedDecl::Create(
3115  getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0);
3116  llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);
3117  QualType BaseAndPointersType = getContext().getConstantArrayType(
3118  getContext().VoidPtrTy, ArrSize, ArrayType::Normal,
3119  /*IndexTypeQuals=*/0);
3121  getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
3123  getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
3124  QualType SizesType = getContext().getConstantArrayType(
3125  getContext().getSizeType(), ArrSize, ArrayType::Normal,
3126  /*IndexTypeQuals=*/0);
3127  SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD,
3128  S.getBeginLoc());
3129  TargetScope.addPrivate(
3130  BPVD, [&InputInfo]() { return InputInfo.BasePointersArray; });
3131  TargetScope.addPrivate(PVD,
3132  [&InputInfo]() { return InputInfo.PointersArray; });
3133  TargetScope.addPrivate(SVD,
3134  [&InputInfo]() { return InputInfo.SizesArray; });
3135  }
3136  (void)TargetScope.Privatize();
3137  // Build list of dependences.
3138  for (const auto *C : S.getClausesOfKind<OMPDependClause>())
3139  for (const Expr *IRef : C->varlists())
3140  Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
3141  auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD,
3142  &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
3143  // Set proper addresses for generated private copies.
3144  OMPPrivateScope Scope(CGF);
3145  if (!Data.FirstprivateVars.empty()) {
3146  llvm::FunctionType *CopyFnTy = llvm::FunctionType::get(
3147  CGF.Builder.getVoidTy(), {CGF.Builder.getInt8PtrTy()}, true);
3148  enum { PrivatesParam = 2, CopyFnParam = 3 };
3149  llvm::Value *CopyFn = CGF.Builder.CreateLoad(
3150  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
3151  llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
3152  CS->getCapturedDecl()->getParam(PrivatesParam)));
3153  // Map privates.
3156  CallArgs.push_back(PrivatesPtr);
3157  for (const Expr *E : Data.FirstprivateVars) {
3158  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3159  Address PrivatePtr =
3160  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
3161  ".firstpriv.ptr.addr");
3162  PrivatePtrs.emplace_back(VD, PrivatePtr);
3163  CallArgs.push_back(PrivatePtr.getPointer());
3164  }
3165  CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
3166  CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
3167  for (const auto &Pair : PrivatePtrs) {
3168  Address Replacement(CGF.Builder.CreateLoad(Pair.second),
3169  CGF.getContext().getDeclAlign(Pair.first));
3170  Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
3171  }
3172  }
3173  // Privatize all private variables except for in_reduction items.
3174  (void)Scope.Privatize();
3175  if (InputInfo.NumberOfTargetItems > 0) {
3176  InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
3177  CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0);
3178  InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
3179  CGF.GetAddrOfLocalVar(PVD), /*Index=*/0);
3180  InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
3181  CGF.GetAddrOfLocalVar(SVD), /*Index=*/0);
3182  }
3183 
3184  Action.Enter(CGF);
3185  OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);
3186  BodyGen(CGF);
3187  };
3188  llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3189  S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true,
3190  Data.NumberOfParts);
3191  llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
3192  IntegerLiteral IfCond(getContext(), TrueOrFalse,
3193  getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3194  SourceLocation());
3195 
3196  CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn,
3197  SharedsTy, CapturedStruct, &IfCond, Data);
3198 }
3199 
3201  // Emit outlined function for task construct.
3202  const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3203  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3204  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3205  const Expr *IfCond = nullptr;
3206  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3207  if (C->getNameModifier() == OMPD_unknown ||
3208  C->getNameModifier() == OMPD_task) {
3209  IfCond = C->getCondition();
3210  break;
3211  }
3212  }
3213 
3214  OMPTaskDataTy Data;
3215  // Check if we should emit tied or untied task.
3216  Data.Tied = !S.getSingleClause<OMPUntiedClause>();
3217  auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
3218  CGF.EmitStmt(CS->getCapturedStmt());
3219  };
3220  auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
3221  IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
3222  const OMPTaskDataTy &Data) {
3223  CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn,
3224  SharedsTy, CapturedStruct, IfCond,
3225  Data);
3226  };
3227  EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data);
3228 }
3229 
3231  const OMPTaskyieldDirective &S) {
3232  CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getBeginLoc());
3233 }
3234 
3236  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_barrier);
3237 }
3238 
3240  CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc());
3241 }
3242 
3244  const OMPTaskgroupDirective &S) {
3245  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3246  Action.Enter(CGF);
3247  if (const Expr *E = S.getReductionRef()) {
3250  OMPTaskDataTy Data;
3251  for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
3252  auto IPriv = C->privates().begin();
3253  auto IRed = C->reduction_ops().begin();
3254  auto ILHS = C->lhs_exprs().begin();
3255  auto IRHS = C->rhs_exprs().begin();
3256  for (const Expr *Ref : C->varlists()) {
3257  Data.ReductionVars.emplace_back(Ref);
3258  Data.ReductionCopies.emplace_back(*IPriv);
3259  Data.ReductionOps.emplace_back(*IRed);
3260  LHSs.emplace_back(*ILHS);
3261  RHSs.emplace_back(*IRHS);
3262  std::advance(IPriv, 1);
3263  std::advance(IRed, 1);
3264  std::advance(ILHS, 1);
3265  std::advance(IRHS, 1);
3266  }
3267  }
3268  llvm::Value *ReductionDesc =
3269  CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(),
3270  LHSs, RHSs, Data);
3271  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3272  CGF.EmitVarDecl(*VD);
3273  CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),
3274  /*Volatile=*/false, E->getType());
3275  }
3277  };
3278  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3279  CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getBeginLoc());
3280 }
3281 
3283  CGM.getOpenMPRuntime().emitFlush(
3284  *this,
3285  [&S]() -> ArrayRef<const Expr *> {
3286  if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>())
3287  return llvm::makeArrayRef(FlushClause->varlist_begin(),
3288  FlushClause->varlist_end());
3289  return llvm::None;
3290  }(),
3291  S.getBeginLoc());
3292 }
3293 
3295  const CodeGenLoopTy &CodeGenLoop,
3296  Expr *IncExpr) {
3297  // Emit the loop iteration variable.
3298  const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3299  const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3300  EmitVarDecl(*IVDecl);
3301 
3302  // Emit the iterations count variable.
3303  // If it is not a variable, Sema decided to calculate iterations count on each
3304  // iteration (e.g., it is foldable into a constant).
3305  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3306  EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3307  // Emit calculation of the iterations count.
3308  EmitIgnoredExpr(S.getCalcLastIteration());
3309  }
3310 
3311  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3312 
3313  bool HasLastprivateClause = false;
3314  // Check pre-condition.
3315  {
3316  OMPLoopScope PreInitScope(*this, S);
3317  // Skip the entire loop if we don't meet the precondition.
3318  // If the condition constant folds and can be elided, avoid emitting the
3319  // whole loop.
3320  bool CondConstant;
3321  llvm::BasicBlock *ContBlock = nullptr;
3322  if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
3323  if (!CondConstant)
3324  return;
3325  } else {
3326  llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
3327  ContBlock = createBasicBlock("omp.precond.end");
3328  emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
3329  getProfileCount(&S));
3330  EmitBlock(ThenBlock);
3331  incrementProfileCounter(&S);
3332  }
3333 
3334  emitAlignedClause(*this, S);
3335  // Emit 'then' code.
3336  {
3337  // Emit helper vars inits.
3338 
3339  LValue LB = EmitOMPHelperVar(
3340  *this, cast<DeclRefExpr>(
3343  : S.getLowerBoundVariable())));
3344  LValue UB = EmitOMPHelperVar(
3345  *this, cast<DeclRefExpr>(
3348  : S.getUpperBoundVariable())));
3349  LValue ST =
3350  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
3351  LValue IL =
3352  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
3353 
3354  OMPPrivateScope LoopScope(*this);
3355  if (EmitOMPFirstprivateClause(S, LoopScope)) {
3356  // Emit implicit barrier to synchronize threads and avoid data races
3357  // on initialization of firstprivate variables and post-update of
3358  // lastprivate variables.
3359  CGM.getOpenMPRuntime().emitBarrierCall(
3360  *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
3361  /*ForceSimpleCall=*/true);
3362  }
3363  EmitOMPPrivateClause(S, LoopScope);
3367  EmitOMPReductionClauseInit(S, LoopScope);
3368  HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
3369  EmitOMPPrivateLoopCounters(S, LoopScope);
3370  (void)LoopScope.Privatize();
3372  CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
3373 
3374  // Detect the distribute schedule kind and chunk.
3375  llvm::Value *Chunk = nullptr;
3377  if (const auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
3378  ScheduleKind = C->getDistScheduleKind();
3379  if (const Expr *Ch = C->getChunkSize()) {
3380  Chunk = EmitScalarExpr(Ch);
3381  Chunk = EmitScalarConversion(Chunk, Ch->getType(),
3383  S.getBeginLoc());
3384  }
3385  } else {
3386  // Default behaviour for dist_schedule clause.
3387  CGM.getOpenMPRuntime().getDefaultDistScheduleAndChunk(
3388  *this, S, ScheduleKind, Chunk);
3389  }
3390  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3391  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3392 
3393  // OpenMP [2.10.8, distribute Construct, Description]
3394  // If dist_schedule is specified, kind must be static. If specified,
3395  // iterations are divided into chunks of size chunk_size, chunks are
3396  // assigned to the teams of the league in a round-robin fashion in the
3397  // order of the team number. When no chunk_size is specified, the
3398  // iteration space is divided into chunks that are approximately equal
3399  // in size, and at most one chunk is distributed to each team of the
3400  // league. The size of the chunks is unspecified in this case.
3401  bool StaticChunked = RT.isStaticChunked(
3402  ScheduleKind, /* Chunked */ Chunk != nullptr) &&
3404  if (RT.isStaticNonchunked(ScheduleKind,
3405  /* Chunked */ Chunk != nullptr) ||
3406  StaticChunked) {
3408  EmitOMPSimdInit(S, /*IsMonotonic=*/true);
3409  CGOpenMPRuntime::StaticRTInput StaticInit(
3410  IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(),
3411  LB.getAddress(), UB.getAddress(), ST.getAddress(),
3412  StaticChunked ? Chunk : nullptr);
3413  RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind,
3414  StaticInit);
3415  JumpDest LoopExit =
3416  getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
3417  // UB = min(UB, GlobalUB);
3420  : S.getEnsureUpperBound());
3421  // IV = LB;
3423  ? S.getCombinedInit()
3424  : S.getInit());
3425 
3426  const Expr *Cond =
3428  ? S.getCombinedCond()
3429  : S.getCond();
3430 
3431  if (StaticChunked)
3432  Cond = S.getCombinedDistCond();
3433 
3434  // For static unchunked schedules generate:
3435  //
3436  // 1. For distribute alone, codegen
3437  // while (idx <= UB) {
3438  // BODY;
3439  // ++idx;
3440  // }
3441  //
3442  // 2. When combined with 'for' (e.g. as in 'distribute parallel for')
3443  // while (idx <= UB) {
3444  // <CodeGen rest of pragma>(LB, UB);
3445  // idx += ST;
3446  // }
3447  //
3448  // For static chunk one schedule generate:
3449  //
3450  // while (IV <= GlobalUB) {
3451  // <CodeGen rest of pragma>(LB, UB);
3452  // LB += ST;
3453  // UB += ST;
3454  // UB = min(UB, GlobalUB);
3455  // IV = LB;
3456  // }
3457  //
3458  EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), Cond, IncExpr,
3459  [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3460  CodeGenLoop(CGF, S, LoopExit);
3461  },
3462  [&S, StaticChunked](CodeGenFunction &CGF) {
3463  if (StaticChunked) {
3464  CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());
3465  CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());
3466  CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());
3467  CGF.EmitIgnoredExpr(S.getCombinedInit());
3468  }
3469  });
3470  EmitBlock(LoopExit.getBlock());
3471  // Tell the runtime we are done.
3472  RT.emitForStaticFinish(*this, S.getBeginLoc(), S.getDirectiveKind());
3473  } else {
3474  // Emit the outer loop, which requests its work chunk [LB..UB] from
3475  // runtime and runs the inner loop to process it.
3476  const OMPLoopArguments LoopArguments = {
3477  LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(),
3478  Chunk};
3479  EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
3480  CodeGenLoop);
3481  }
3483  EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
3484  return CGF.Builder.CreateIsNotNull(
3485  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3486  });
3487  }
3491  EmitOMPReductionClauseFinal(S, OMPD_simd);
3492  // Emit post-update of the reduction variables if IsLastIter != 0.
3494  *this, S, [IL, &S](CodeGenFunction &CGF) {
3495  return CGF.Builder.CreateIsNotNull(
3496  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3497  });
3498  }
3499  // Emit final copy of the lastprivate variables if IsLastIter != 0.
3500  if (HasLastprivateClause) {
3501  EmitOMPLastprivateClauseFinal(
3502  S, /*NoFinals=*/false,
3503  Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
3504  }
3505  }
3506 
3507  // We're now done with the loop, so jump to the continuation block.
3508  if (ContBlock) {
3509  EmitBranch(ContBlock);
3510  EmitBlock(ContBlock, true);
3511  }
3512  }
3513 }
3514 
3516  const OMPDistributeDirective &S) {
3517  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3519  };
3520  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3521  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3522 }
3523 
3524 static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM,
3525  const CapturedStmt *S) {
3526  CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
3528  CGF.CapturedStmtInfo = &CapStmtInfo;
3529  llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S);
3530  Fn->setDoesNotRecurse();
3531  return Fn;
3532 }
3533 
3535  if (S.hasClausesOfKind<OMPDependClause>()) {
3536  assert(!S.getAssociatedStmt() &&
3537  "No associated statement must be in ordered depend construct.");
3538  for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
3539  CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
3540  return;
3541  }
3542  const auto *C = S.getSingleClause<OMPSIMDClause>();
3543  auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
3544  PrePostActionTy &Action) {
3545  const CapturedStmt *CS = S.getInnermostCapturedStmt();
3546  if (C) {
3548  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
3549  llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS);
3550  CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
3551  OutlinedFn, CapturedVars);
3552  } else {
3553  Action.Enter(CGF);
3554  CGF.EmitStmt(CS->getCapturedStmt());
3555  }
3556  };
3557  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3558  CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getBeginLoc(), !C);
3559 }
3560 
3562  QualType SrcType, QualType DestType,
3563  SourceLocation Loc) {
3564  assert(CGF.hasScalarEvaluationKind(DestType) &&
3565  "DestType must have scalar evaluation kind.");
3566  assert(!Val.isAggregate() && "Must be a scalar or complex.");
3567  return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
3568  DestType, Loc)
3570  Val.getComplexVal(), SrcType, DestType, Loc);
3571 }
3572 
3575  QualType DestType, SourceLocation Loc) {
3576  assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
3577  "DestType must have complex evaluation kind.");
3578  CodeGenFunction::ComplexPairTy ComplexVal;
3579  if (Val.isScalar()) {
3580  // Convert the input element to the element type of the complex.
3581  QualType DestElementType =
3582  DestType->castAs<ComplexType>()->getElementType();
3583  llvm::Value *ScalarVal = CGF.EmitScalarConversion(
3584  Val.getScalarVal(), SrcType, DestElementType, Loc);
3585  ComplexVal = CodeGenFunction::ComplexPairTy(
3586  ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
3587  } else {
3588  assert(Val.isComplex() && "Must be a scalar or complex.");
3589  QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
3590  QualType DestElementType =
3591  DestType->castAs<ComplexType>()->getElementType();
3592  ComplexVal.first = CGF.EmitScalarConversion(
3593  Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
3594  ComplexVal.second = CGF.EmitScalarConversion(
3595  Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
3596  }
3597  return ComplexVal;
3598 }
3599 
3600 static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst,
3601  LValue LVal, RValue RVal) {
3602  if (LVal.isGlobalReg()) {
3603  CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
3604  } else {
3605  CGF.EmitAtomicStore(RVal, LVal,
3606  IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3607  : llvm::AtomicOrdering::Monotonic,
3608  LVal.isVolatile(), /*IsInit=*/false);
3609  }
3610 }
3611 
3613  QualType RValTy, SourceLocation Loc) {
3614  switch (getEvaluationKind(LVal.getType())) {
3615  case TEK_Scalar:
3616  EmitStoreThroughLValue(RValue::get(convertToScalarValue(
3617  *this, RVal, RValTy, LVal.getType(), Loc)),
3618  LVal);
3619  break;
3620  case TEK_Complex:
3621  EmitStoreOfComplex(
3622  convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,
3623  /*isInit=*/false);
3624  break;
3625  case TEK_Aggregate:
3626  llvm_unreachable("Must be a scalar or complex.");
3627  }
3628 }
3629 
3630 static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst,
3631  const Expr *X, const Expr *V,
3632  SourceLocation Loc) {
3633  // v = x;
3634  assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
3635  assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
3636  LValue XLValue = CGF.EmitLValue(X);
3637  LValue VLValue = CGF.EmitLValue(V);
3638  RValue Res = XLValue.isGlobalReg()
3639  ? CGF.EmitLoadOfLValue(XLValue, Loc)
3640  : CGF.EmitAtomicLoad(
3641  XLValue, Loc,
3642  IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3643  : llvm::AtomicOrdering::Monotonic,
3644  XLValue.isVolatile());
3645  // OpenMP, 2.12.6, atomic Construct
3646  // Any atomic construct with a seq_cst clause forces the atomically
3647  // performed operation to include an implicit flush operation without a
3648  // list.
3649  if (IsSeqCst)
3650  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3651  CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
3652 }
3653 
3654 static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst,
3655  const Expr *X, const Expr *E,
3656  SourceLocation Loc) {
3657  // x = expr;
3658  assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
3659  emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
3660  // OpenMP, 2.12.6, atomic Construct
3661  // Any atomic construct with a seq_cst clause forces the atomically
3662  // performed operation to include an implicit flush operation without a
3663  // list.
3664  if (IsSeqCst)
3665  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3666 }
3667 
3668 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
3669  RValue Update,
3670  BinaryOperatorKind BO,
3671  llvm::AtomicOrdering AO,
3672  bool IsXLHSInRHSPart) {
3673  ASTContext &Context = CGF.getContext();
3674  // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
3675  // expression is simple and atomic is allowed for the given type for the
3676  // target platform.
3677  if (BO == BO_Comma || !Update.isScalar() ||
3678  !Update.getScalarVal()->getType()->isIntegerTy() ||
3679  !X.isSimple() || (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
3680  (Update.getScalarVal()->getType() !=
3681  X.getAddress().getElementType())) ||
3682  !X.getAddress().getElementType()->isIntegerTy() ||
3683  !Context.getTargetInfo().hasBuiltinAtomic(
3684  Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
3685  return std::make_pair(false, RValue::get(nullptr));
3686 
3687  llvm::AtomicRMWInst::BinOp RMWOp;
3688  switch (BO) {
3689  case BO_Add:
3690  RMWOp = llvm::AtomicRMWInst::Add;
3691  break;
3692  case BO_Sub:
3693  if (!IsXLHSInRHSPart)
3694  return std::make_pair(false, RValue::get(nullptr));
3695  RMWOp = llvm::AtomicRMWInst::Sub;
3696  break;
3697  case BO_And:
3698  RMWOp = llvm::AtomicRMWInst::And;
3699  break;
3700  case BO_Or:
3701  RMWOp = llvm::AtomicRMWInst::Or;
3702  break;
3703  case BO_Xor:
3704  RMWOp = llvm::AtomicRMWInst::Xor;
3705  break;
3706  case BO_LT:
3708  ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
3709  : llvm::AtomicRMWInst::Max)
3710  : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
3711  : llvm::AtomicRMWInst::UMax);
3712  break;
3713  case BO_GT:
3715  ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
3716  : llvm::AtomicRMWInst::Min)
3717  : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
3718  : llvm::AtomicRMWInst::UMin);
3719  break;
3720  case BO_Assign:
3721  RMWOp = llvm::AtomicRMWInst::Xchg;
3722  break;
3723  case BO_Mul:
3724  case BO_Div:
3725  case BO_Rem:
3726  case BO_Shl:
3727  case BO_Shr:
3728  case BO_LAnd:
3729  case BO_LOr:
3730  return std::make_pair(false, RValue::get(nullptr));
3731  case BO_PtrMemD:
3732  case BO_PtrMemI:
3733  case BO_LE:
3734  case BO_GE:
3735  case BO_EQ:
3736  case BO_NE:
3737  case BO_Cmp:
3738  case BO_AddAssign:
3739  case BO_SubAssign:
3740  case BO_AndAssign:
3741  case BO_OrAssign:
3742  case BO_XorAssign:
3743  case BO_MulAssign:
3744  case BO_DivAssign:
3745  case BO_RemAssign:
3746  case BO_ShlAssign:
3747  case BO_ShrAssign:
3748  case BO_Comma:
3749  llvm_unreachable("Unsupported atomic update operation");
3750  }
3751  llvm::Value *UpdateVal = Update.getScalarVal();
3752  if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
3753  UpdateVal = CGF.Builder.CreateIntCast(
3754  IC, X.getAddress().getElementType(),
3756  }
3757  llvm::Value *Res =
3758  CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO);
3759  return std::make_pair(true, RValue::get(Res));
3760 }
3761 
3763  LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3764  llvm::AtomicOrdering AO, SourceLocation Loc,
3765  const llvm::function_ref<RValue(RValue)> CommonGen) {
3766  // Update expressions are allowed to have the following forms:
3767  // x binop= expr; -> xrval + expr;
3768  // x++, ++x -> xrval + 1;
3769  // x--, --x -> xrval - 1;
3770  // x = x binop expr; -> xrval binop expr
3771  // x = expr Op x; - > expr binop xrval;
3772  auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);
3773  if (!Res.first) {
3774  if (X.isGlobalReg()) {
3775  // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
3776  // 'xrval'.
3777  EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
3778  } else {
3779  // Perform compare-and-swap procedure.
3780  EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
3781  }
3782  }
3783  return Res;
3784 }
3785 
3786 static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst,
3787  const Expr *X, const Expr *E,
3788  const Expr *UE, bool IsXLHSInRHSPart,
3789  SourceLocation Loc) {
3790  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3791  "Update expr in 'atomic update' must be a binary operator.");
3792  const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3793  // Update expressions are allowed to have the following forms:
3794  // x binop= expr; -> xrval + expr;
3795  // x++, ++x -> xrval + 1;
3796  // x--, --x -> xrval - 1;
3797  // x = x binop expr; -> xrval binop expr
3798  // x = expr Op x; - > expr binop xrval;
3799  assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
3800  LValue XLValue = CGF.EmitLValue(X);
3801  RValue ExprRValue = CGF.EmitAnyExpr(E);
3802  llvm::AtomicOrdering AO = IsSeqCst
3803  ? llvm::AtomicOrdering::SequentiallyConsistent
3804  : llvm::AtomicOrdering::Monotonic;
3805  const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3806  const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3807  const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3808  const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3809  auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {
3810  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3811  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3812  return CGF.EmitAnyExpr(UE);
3813  };
3815  XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3816  // OpenMP, 2.12.6, atomic Construct
3817  // Any atomic construct with a seq_cst clause forces the atomically
3818  // performed operation to include an implicit flush operation without a
3819  // list.
3820  if (IsSeqCst)
3821  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3822 }
3823 
3825  QualType SourceType, QualType ResType,
3826  SourceLocation Loc) {
3827  switch (CGF.getEvaluationKind(ResType)) {
3828  case TEK_Scalar:
3829  return RValue::get(
3830  convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
3831  case TEK_Complex: {
3832  auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
3833  return RValue::getComplex(Res.first, Res.second);
3834  }
3835  case TEK_Aggregate:
3836  break;
3837  }
3838  llvm_unreachable("Must be a scalar or complex.");
3839 }
3840 
3841 static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst,
3842  bool IsPostfixUpdate, const Expr *V,
3843  const Expr *X, const Expr *E,
3844  const Expr *UE, bool IsXLHSInRHSPart,
3845  SourceLocation Loc) {
3846  assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
3847  assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
3848  RValue NewVVal;
3849  LValue VLValue = CGF.EmitLValue(V);
3850  LValue XLValue = CGF.EmitLValue(X);
3851  RValue ExprRValue = CGF.EmitAnyExpr(E);
3852  llvm::AtomicOrdering AO = IsSeqCst
3853  ? llvm::AtomicOrdering::SequentiallyConsistent
3854  : llvm::AtomicOrdering::Monotonic;
3855  QualType NewVValType;
3856  if (UE) {
3857  // 'x' is updated with some additional value.
3858  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3859  "Update expr in 'atomic capture' must be a binary operator.");
3860  const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3861  // Update expressions are allowed to have the following forms:
3862  // x binop= expr; -> xrval + expr;
3863  // x++, ++x -> xrval + 1;
3864  // x--, --x -> xrval - 1;
3865  // x = x binop expr; -> xrval binop expr
3866  // x = expr Op x; - > expr binop xrval;
3867  const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3868  const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3869  const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3870  NewVValType = XRValExpr->getType();
3871  const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3872  auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
3873  IsPostfixUpdate](RValue XRValue) {
3874  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3875  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3876  RValue Res = CGF.EmitAnyExpr(UE);
3877  NewVVal = IsPostfixUpdate ? XRValue : Res;
3878  return Res;
3879  };
3880  auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3881  XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3882  if (Res.first) {
3883  // 'atomicrmw' instruction was generated.
3884  if (IsPostfixUpdate) {
3885  // Use old value from 'atomicrmw'.
3886  NewVVal = Res.second;
3887  } else {
3888  // 'atomicrmw' does not provide new value, so evaluate it using old
3889  // value of 'x'.
3890  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3891  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
3892  NewVVal = CGF.EmitAnyExpr(UE);
3893  }
3894  }
3895  } else {
3896  // 'x' is simply rewritten with some 'expr'.
3897  NewVValType = X->getType().getNonReferenceType();
3898  ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
3899  X->getType().getNonReferenceType(), Loc);
3900  auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) {
3901  NewVVal = XRValue;
3902  return ExprRValue;
3903  };
3904  // Try to perform atomicrmw xchg, otherwise simple exchange.
3905  auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3906  XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
3907  Loc, Gen);
3908  if (Res.first) {
3909  // 'atomicrmw' instruction was generated.
3910  NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
3911  }
3912  }
3913  // Emit post-update store to 'v' of old/new 'x' value.
3914  CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
3915  // OpenMP, 2.12.6, atomic Construct
3916  // Any atomic construct with a seq_cst clause forces the atomically
3917  // performed operation to include an implicit flush operation without a
3918  // list.
3919  if (IsSeqCst)
3920  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3921 }
3922 
3924  bool IsSeqCst, bool IsPostfixUpdate,
3925  const Expr *X, const Expr *V, const Expr *E,
3926  const Expr *UE, bool IsXLHSInRHSPart,
3927  SourceLocation Loc) {
3928  switch (Kind) {
3929  case OMPC_read:
3930  emitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc);
3931  break;
3932  case OMPC_write:
3933  emitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc);
3934  break;
3935  case OMPC_unknown:
3936  case OMPC_update:
3937  emitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc);
3938  break;
3939  case OMPC_capture:
3940  emitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE,
3941  IsXLHSInRHSPart, Loc);
3942  break;
3943  case OMPC_if:
3944  case OMPC_final:
3945  case OMPC_num_threads:
3946  case OMPC_private:
3947  case OMPC_firstprivate:
3948  case OMPC_lastprivate:
3949  case OMPC_reduction:
3950  case OMPC_task_reduction:
3951  case OMPC_in_reduction:
3952  case OMPC_safelen:
3953  case OMPC_simdlen:
3954  case OMPC_allocator:
3955  case OMPC_allocate:
3956  case OMPC_collapse:
3957  case OMPC_default:
3958  case OMPC_seq_cst:
3959  case OMPC_shared:
3960  case OMPC_linear:
3961  case OMPC_aligned:
3962  case OMPC_copyin:
3963  case OMPC_copyprivate:
3964  case OMPC_flush:
3965  case OMPC_proc_bind:
3966  case OMPC_schedule:
3967  case OMPC_ordered:
3968  case OMPC_nowait:
3969  case OMPC_untied:
3970  case OMPC_threadprivate:
3971  case OMPC_depend:
3972  case OMPC_mergeable:
3973  case OMPC_device:
3974  case OMPC_threads:
3975  case OMPC_simd:
3976  case OMPC_map:
3977  case OMPC_num_teams:
3978  case OMPC_thread_limit:
3979  case OMPC_priority:
3980  case OMPC_grainsize:
3981  case OMPC_nogroup:
3982  case OMPC_num_tasks:
3983  case OMPC_hint:
3984  case OMPC_dist_schedule:
3985  case OMPC_defaultmap:
3986  case OMPC_uniform:
3987  case OMPC_to:
3988  case OMPC_from:
3989  case OMPC_use_device_ptr:
3990  case OMPC_is_device_ptr:
3991  case OMPC_unified_address:
3992  case OMPC_unified_shared_memory:
3993  case OMPC_reverse_offload:
3994  case OMPC_dynamic_allocators:
3995  case OMPC_atomic_default_mem_order:
3996  llvm_unreachable("Clause is not allowed in 'omp atomic'.");
3997  }
3998 }
3999 
4001  bool IsSeqCst = S.getSingleClause<OMPSeqCstClause>();
4003  for (const OMPClause *C : S.clauses()) {
4004  // Find first clause (skip seq_cst clause, if it is first).
4005  if (C->getClauseKind() != OMPC_seq_cst) {
4006  Kind = C->getClauseKind();
4007  break;
4008  }
4009  }
4010 
4011  const Stmt *CS = S.getInnermostCapturedStmt()->IgnoreContainers();
4012  if (const auto *FE = dyn_cast<FullExpr>(CS))
4013  enterFullExpression(FE);
4014  // Processing for statements under 'atomic capture'.
4015  if (const auto *Compound = dyn_cast<CompoundStmt>(CS)) {
4016  for (const Stmt *C : Compound->body()) {
4017  if (const auto *FE = dyn_cast<FullExpr>(C))
4018  enterFullExpression(FE);
4019  }
4020  }
4021 
4022  auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF,
4023  PrePostActionTy &) {
4024  CGF.EmitStopPoint(CS);
4025  emitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(),
4026  S.getV(), S.getExpr(), S.getUpdateExpr(),
4027  S.isXLHSInRHSPart(), S.getBeginLoc());
4028  };
4029  OMPLexicalScope Scope(*this, S, OMPD_unknown);
4030  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen);
4031 }
4032 
4034  const OMPExecutableDirective &S,
4035  const RegionCodeGenTy &CodeGen) {
4037  CodeGenModule &CGM = CGF.CGM;
4038 
4039  // On device emit this construct as inlined code.
4040  if (CGM.getLangOpts().OpenMPIsDevice) {
4041  OMPLexicalScope Scope(CGF, S, OMPD_target);
4042  CGM.getOpenMPRuntime().emitInlinedDirective(
4043  CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4045  });
4046  return;
4047  }
4048 
4049  llvm::Function *Fn = nullptr;
4050  llvm::Constant *FnID = nullptr;
4051 
4052  const Expr *IfCond = nullptr;
4053  // Check for the at most one if clause associated with the target region.
4054  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4055  if (C->getNameModifier() == OMPD_unknown ||
4056  C->getNameModifier() == OMPD_target) {
4057  IfCond = C->getCondition();
4058  break;
4059  }
4060  }
4061 
4062  // Check if we have any device clause associated with the directive.
4063  const Expr *Device = nullptr;
4064  if (auto *C = S.getSingleClause<OMPDeviceClause>())
4065  Device = C->getDevice();
4066 
4067  // Check if we have an if clause whose conditional always evaluates to false
4068  // or if we do not have any targets specified. If so the target region is not
4069  // an offload entry point.
4070  bool IsOffloadEntry = true;
4071  if (IfCond) {
4072  bool Val;
4073  if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
4074  IsOffloadEntry = false;
4075  }
4076  if (CGM.getLangOpts().OMPTargetTriples.empty())
4077  IsOffloadEntry = false;
4078 
4079  assert(CGF.CurFuncDecl && "No parent declaration for target region!");
4080  StringRef ParentName;
4081  // In case we have Ctors/Dtors we use the complete type variant to produce
4082  // the mangling of the device outlined kernel.
4083  if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
4084  ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));
4085  else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
4086  ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));
4087  else
4088  ParentName =
4089  CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));
4090 
4091  // Emit target region as a standalone region.
4092  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
4093  IsOffloadEntry, CodeGen);
4094  OMPLexicalScope Scope(CGF, S, OMPD_task);
4095  auto &&SizeEmitter = [](CodeGenFunction &CGF, const OMPLoopDirective &D) {
4096  OMPLoopScope(CGF, D);
4097  // Emit calculation of the iterations count.
4098  llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations());
4099  NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty,
4100  /*IsSigned=*/false);
4101  return NumIterations;
4102  };
4103  if (IsOffloadEntry)
4104  CGM.getOpenMPRuntime().emitTargetNumIterationsCall(CGF, S, Device,
4105  SizeEmitter);
4106  CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device);
4107 }
4108 
4110  PrePostActionTy &Action) {
4111  Action.Enter(CGF);
4112  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4113  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4114  CGF.EmitOMPPrivateClause(S, PrivateScope);
4115  (void)PrivateScope.Privatize();
4117  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4118 
4119  CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
4120 }
4121 
4123  StringRef ParentName,
4124  const OMPTargetDirective &S) {
4125  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4126  emitTargetRegion(CGF, S, Action);
4127  };
4128  llvm::Function *Fn;
4129  llvm::Constant *Addr;
4130  // Emit target region as a standalone region.
4131  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4132  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4133  assert(Fn && Addr && "Target device function emission failed.");
4134 }
4135 
4137  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4138  emitTargetRegion(CGF, S, Action);
4139  };
4140  emitCommonOMPTargetDirective(*this, S, CodeGen);
4141 }
4142 
4144  const OMPExecutableDirective &S,
4145  OpenMPDirectiveKind InnermostKind,
4146  const RegionCodeGenTy &CodeGen) {
4147  const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
4148  llvm::Function *OutlinedFn =
4149  CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
4150  S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
4151 
4152  const auto *NT = S.getSingleClause<OMPNumTeamsClause>();
4153  const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
4154  if (NT || TL) {
4155  const Expr *NumTeams = NT ? NT->getNumTeams() : nullptr;
4156  const Expr *ThreadLimit = TL ? TL->getThreadLimit() : nullptr;
4157 
4158  CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
4159  S.getBeginLoc());
4160  }
4161 
4162  OMPTeamsScope Scope(CGF, S);
4164  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
4165  CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn,
4166  CapturedVars);
4167 }
4168 
4170  // Emit teams region as a standalone region.
4171  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4172  Action.Enter(CGF);
4173  OMPPrivateScope PrivateScope(CGF);
4174  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4175  CGF.EmitOMPPrivateClause(S, PrivateScope);
4176  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4177  (void)PrivateScope.Privatize();
4178  CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
4179  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4180  };
4181  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4183  [](CodeGenFunction &) { return nullptr; });
4184 }
4185 
4187  const OMPTargetTeamsDirective &S) {
4188  auto *CS = S.getCapturedStmt(OMPD_teams);
4189  Action.Enter(CGF);
4190  // Emit teams region as a standalone region.
4191  auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4192  Action.Enter(CGF);
4193  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4194  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4195  CGF.EmitOMPPrivateClause(S, PrivateScope);
4196  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4197  (void)PrivateScope.Privatize();
4199  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4200  CGF.EmitStmt(CS->getCapturedStmt());
4201  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4202  };
4203  emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
4205  [](CodeGenFunction &) { return nullptr; });
4206 }
4207 
4209  CodeGenModule &CGM, StringRef ParentName,
4210  const OMPTargetTeamsDirective &S) {
4211  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4212  emitTargetTeamsRegion(CGF, Action, S);
4213  };
4214  llvm::Function *Fn;
4215  llvm::Constant *Addr;
4216  // Emit target region as a standalone region.
4217  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4218  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4219  assert(Fn && Addr && "Target device function emission failed.");
4220 }
4221 
4223  const OMPTargetTeamsDirective &S) {
4224  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4225  emitTargetTeamsRegion(CGF, Action, S);
4226  };
4227  emitCommonOMPTargetDirective(*this, S, CodeGen);
4228 }
4229 
4230 static void
4233  Action.Enter(CGF);
4234  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4236  };
4237 
4238  // Emit teams region as a standalone region.
4239  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4240  PrePostActionTy &Action) {
4241  Action.Enter(CGF);
4242  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4243  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4244  (void)PrivateScope.Privatize();
4245  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4246  CodeGenDistribute);
4247  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4248  };
4249  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
4251  [](CodeGenFunction &) { return nullptr; });
4252 }
4253 
4255  CodeGenModule &CGM, StringRef ParentName,
4257  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4258  emitTargetTeamsDistributeRegion(CGF, Action, S);
4259  };
4260  llvm::Function *Fn;
4261  llvm::Constant *Addr;
4262  // Emit target region as a standalone region.
4263  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4264  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4265  assert(Fn && Addr && "Target device function emission failed.");
4266 }
4267 
4270  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4271  emitTargetTeamsDistributeRegion(CGF, Action, S);
4272  };
4273  emitCommonOMPTargetDirective(*this, S, CodeGen);
4274 }
4275 
4277  CodeGenFunction &CGF, PrePostActionTy &Action,
4279  Action.Enter(CGF);
4280  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4282  };
4283 
4284  // Emit teams region as a standalone region.
4285  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4286  PrePostActionTy &Action) {
4287  Action.Enter(CGF);
4288  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4289  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4290  (void)PrivateScope.Privatize();
4291  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4292  CodeGenDistribute);
4293  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4294  };
4295  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);
4297  [](CodeGenFunction &) { return nullptr; });
4298 }
4299 
4301  CodeGenModule &CGM, StringRef ParentName,
4303  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4304  emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4305  };
4306  llvm::Function *Fn;
4307  llvm::Constant *Addr;
4308  // Emit target region as a standalone region.
4309  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4310  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4311  assert(Fn && Addr && "Target device function emission failed.");
4312 }
4313 
4316  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4317  emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4318  };
4319  emitCommonOMPTargetDirective(*this, S, CodeGen);
4320 }
4321 
4323  const OMPTeamsDistributeDirective &S) {
4324 
4325  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4327  };
4328 
4329  // Emit teams region as a standalone region.
4330  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4331  PrePostActionTy &Action) {
4332  Action.Enter(CGF);
4333  OMPPrivateScope PrivateScope(CGF);
4334  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4335  (void)PrivateScope.Privatize();
4336  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4337  CodeGenDistribute);
4338  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4339  };
4340  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4342  [](CodeGenFunction &) { return nullptr; });
4343 }
4344 
4347  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4349  };
4350 
4351  // Emit teams region as a standalone region.
4352  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4353  PrePostActionTy &Action) {
4354  Action.Enter(CGF);
4355  OMPPrivateScope PrivateScope(CGF);
4356  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4357  (void)PrivateScope.Privatize();
4358  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
4359  CodeGenDistribute);
4360  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4361  };
4362  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen);
4364  [](CodeGenFunction &) { return nullptr; });
4365 }
4366 
4369  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4371  S.getDistInc());
4372  };
4373 
4374  // Emit teams region as a standalone region.
4375  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4376  PrePostActionTy &Action) {
4377  Action.Enter(CGF);
4378  OMPPrivateScope PrivateScope(CGF);
4379  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4380  (void)PrivateScope.Privatize();
4381  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4382  CodeGenDistribute);
4383  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4384  };
4385  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4387  [](CodeGenFunction &) { return nullptr; });
4388 }
4389 
4392  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4394  S.getDistInc());
4395  };
4396 
4397  // Emit teams region as a standalone region.
4398  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4399  PrePostActionTy &Action) {
4400  Action.Enter(CGF);
4401  OMPPrivateScope PrivateScope(CGF);
4402  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4403  (void)PrivateScope.Privatize();
4404  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4405  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4406  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4407  };
4408  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4410  [](CodeGenFunction &) { return nullptr; });
4411 }
4412 
4415  PrePostActionTy &Action) {
4416  Action.Enter(CGF);
4417  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4419  S.getDistInc());
4420  };
4421 
4422  // Emit teams region as a standalone region.
4423  auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4424  PrePostActionTy &Action) {
4425  Action.Enter(CGF);
4426  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4427  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4428  (void)PrivateScope.Privatize();
4429  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4430  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4431  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4432  };
4433 
4434  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
4435  CodeGenTeams);
4437  [](CodeGenFunction &) { return nullptr; });
4438 }
4439 
4441  CodeGenModule &CGM, StringRef ParentName,
4443  // Emit SPMD target teams distribute parallel for region as a standalone
4444  // region.
4445  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4447  };
4448  llvm::Function *Fn;
4449  llvm::Constant *Addr;
4450  // Emit target region as a standalone region.
4451  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4452  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4453  assert(Fn && Addr && "Target device function emission failed.");
4454 }
4455 
4458  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4460  };
4461  emitCommonOMPTargetDirective(*this, S, CodeGen);
4462 }
4463 
4465  CodeGenFunction &CGF,
4467  PrePostActionTy &Action) {
4468  Action.Enter(CGF);
4469  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4471  S.getDistInc());
4472  };
4473 
4474  // Emit teams region as a standalone region.
4475  auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4476  PrePostActionTy &Action) {
4477  Action.Enter(CGF);
4478  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4479  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4480  (void)PrivateScope.Privatize();
4481  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4482  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4483  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4484  };
4485 
4486  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd,
4487  CodeGenTeams);
4489  [](CodeGenFunction &) { return nullptr; });
4490 }
4491 
4493  CodeGenModule &CGM, StringRef ParentName,
4495  // Emit SPMD target teams distribute parallel for simd region as a standalone
4496  // region.
4497  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4499  };
4500  llvm::Function *Fn;
4501  llvm::Constant *Addr;
4502  // Emit target region as a standalone region.
4503  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4504  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4505  assert(Fn && Addr && "Target device function emission failed.");
4506 }
4507 
4510  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4512  };
4513  emitCommonOMPTargetDirective(*this, S, CodeGen);
4514 }
4515 
4517  const OMPCancellationPointDirective &S) {
4518  CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getBeginLoc(),
4519  S.getCancelRegion());
4520 }
4521 
4523  const Expr *IfCond = nullptr;
4524  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4525  if (C->getNameModifier() == OMPD_unknown ||
4526  C->getNameModifier() == OMPD_cancel) {
4527  IfCond = C->getCondition();
4528  break;
4529  }
4530  }
4531  CGM.getOpenMPRuntime().emitCancelCall(*this, S.getBeginLoc(), IfCond,
4532  S.getCancelRegion());
4533 }
4534 
4537  if (Kind == OMPD_parallel || Kind == OMPD_task ||
4538  Kind == OMPD_target_parallel)
4539  return ReturnBlock;
4540  assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
4541  Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
4542  Kind == OMPD_distribute_parallel_for ||
4543  Kind == OMPD_target_parallel_for ||
4544  Kind == OMPD_teams_distribute_parallel_for ||
4545  Kind == OMPD_target_teams_distribute_parallel_for);
4546  return OMPCancelStack.getExitBlock();
4547 }
4548 
4550  const OMPClause &NC, OMPPrivateScope &PrivateScope,
4551  const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) {
4552  const auto &C = cast<OMPUseDevicePtrClause>(NC);
4553  auto OrigVarIt = C.varlist_begin();
4554  auto InitIt = C.inits().begin();
4555  for (const Expr *PvtVarIt : C.private_copies()) {
4556  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl());
4557  const auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl());
4558  const auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl());
4559 
4560  // In order to identify the right initializer we need to match the
4561  // declaration used by the mapping logic. In some cases we may get
4562  // OMPCapturedExprDecl that refers to the original declaration.
4563  const ValueDecl *MatchingVD = OrigVD;
4564  if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
4565  // OMPCapturedExprDecl are used to privative fields of the current
4566  // structure.
4567  const auto *ME = cast<MemberExpr>(OED->getInit());
4568  assert(isa<CXXThisExpr>(ME->getBase()) &&
4569  "Base should be the current struct!");
4570  MatchingVD = ME->getMemberDecl();
4571  }
4572 
4573  // If we don't have information about the current list item, move on to
4574  // the next one.
4575  auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
4576  if (InitAddrIt == CaptureDeviceAddrMap.end())
4577  continue;
4578 
4579  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, OrigVD,
4580  InitAddrIt, InitVD,
4581  PvtVD]() {
4582  // Initialize the temporary initialization variable with the address we
4583  // get from the runtime library. We have to cast the source address
4584  // because it is always a void *. References are materialized in the
4585  // privatization scope, so the initialization here disregards the fact
4586  // the original variable is a reference.
4587  QualType AddrQTy =
4588  getContext().getPointerType(OrigVD->getType().getNonReferenceType());
4589  llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy);
4590  Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy);
4591  setAddrOfLocalVar(InitVD, InitAddr);
4592 
4593  // Emit private declaration, it will be initialized by the value we
4594  // declaration we just added to the local declarations map.
4595  EmitDecl(*PvtVD);
4596 
4597  // The initialization variables reached its purpose in the emission
4598  // of the previous declaration, so we don't need it anymore.
4599  LocalDeclMap.erase(InitVD);
4600 
4601  // Return the address of the private variable.
4602  return GetAddrOfLocalVar(PvtVD);
4603  });
4604  assert(IsRegistered && "firstprivate var already registered as private");
4605  // Silence the warning about unused variable.
4606  (void)IsRegistered;
4607 
4608  ++OrigVarIt;
4609  ++InitIt;
4610  }
4611 }
4612 
4613 // Generate the instructions for '#pragma omp target data' directive.
4615  const OMPTargetDataDirective &S) {
4616  CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true);
4617 
4618  // Create a pre/post action to signal the privatization of the device pointer.
4619  // This action can be replaced by the OpenMP runtime code generation to
4620  // deactivate privatization.
4621  bool PrivatizeDevicePointers = false;
4622  class DevicePointerPrivActionTy : public PrePostActionTy {
4623  bool &PrivatizeDevicePointers;
4624 
4625  public:
4626  explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
4627  : PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {}
4628  void Enter(CodeGenFunction &CGF) override {
4629  PrivatizeDevicePointers = true;
4630  }
4631  };
4632  DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
4633 
4634  auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers](
4635  CodeGenFunction &CGF, PrePostActionTy &Action) {
4636  auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4638  };
4639 
4640  // Codegen that selects whether to generate the privatization code or not.
4641  auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers,
4642  &InnermostCodeGen](CodeGenFunction &CGF,
4643  PrePostActionTy &Action) {
4644  RegionCodeGenTy RCG(InnermostCodeGen);
4645  PrivatizeDevicePointers = false;
4646 
4647  // Call the pre-action to change the status of PrivatizeDevicePointers if
4648  // needed.
4649  Action.Enter(CGF);
4650 
4651  if (PrivatizeDevicePointers) {
4652  OMPPrivateScope PrivateScope(CGF);
4653  // Emit all instances of the use_device_ptr clause.
4654  for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
4655  CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
4656  Info.CaptureDeviceAddrMap);
4657  (void)PrivateScope.Privatize();
4658  RCG(CGF);
4659  } else {
4660  RCG(CGF);
4661  }
4662  };
4663 
4664  // Forward the provided action to the privatization codegen.
4665  RegionCodeGenTy PrivRCG(PrivCodeGen);
4666  PrivRCG.setAction(Action);
4667 
4668  // Notwithstanding the body of the region is emitted as inlined directive,
4669  // we don't use an inline scope as changes in the references inside the
4670  // region are expected to be visible outside, so we do not privative them.
4671  OMPLexicalScope Scope(CGF, S);
4672  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
4673  PrivRCG);
4674  };
4675 
4676  RegionCodeGenTy RCG(CodeGen);
4677 
4678  // If we don't have target devices, don't bother emitting the data mapping
4679  // code.
4680  if (CGM.getLangOpts().OMPTargetTriples.empty()) {
4681  RCG(*this);
4682  return;
4683  }
4684 
4685  // Check if we have any if clause associated with the directive.
4686  const Expr *IfCond = nullptr;
4687  if (const auto *C = S.getSingleClause<OMPIfClause>())
4688  IfCond = C->getCondition();
4689 
4690  // Check if we have any device clause associated with the directive.
4691  const Expr *Device = nullptr;
4692  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4693  Device = C->getDevice();
4694 
4695  // Set the action to signal privatization of device pointers.
4696  RCG.setAction(PrivAction);
4697 
4698  // Emit region code.
4699  CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
4700  Info);
4701 }
4702 
4704  const OMPTargetEnterDataDirective &S) {
4705  // If we don't have target devices, don't bother emitting the data mapping
4706  // code.
4707  if (CGM.getLangOpts().OMPTargetTriples.empty())
4708  return;
4709 
4710  // Check if we have any if clause associated with the directive.
4711  const Expr *IfCond = nullptr;
4712  if (const auto *C = S.getSingleClause<OMPIfClause>())
4713  IfCond = C->getCondition();
4714 
4715  // Check if we have any device clause associated with the directive.
4716  const Expr *Device = nullptr;
4717  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4718  Device = C->getDevice();
4719 
4720  OMPLexicalScope Scope(*this, S, OMPD_task);
4721  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4722 }
4723 
4725  const OMPTargetExitDataDirective &S) {
4726  // If we don't have target devices, don't bother emitting the data mapping
4727  // code.
4728  if (CGM.getLangOpts().OMPTargetTriples.empty())
4729  return;
4730 
4731  // Check if we have any if clause associated with the directive.
4732  const Expr *IfCond = nullptr;
4733  if (const auto *C = S.getSingleClause<OMPIfClause>())
4734  IfCond = C->getCondition();
4735 
4736  // Check if we have any device clause associated with the directive.
4737  const Expr *Device = nullptr;
4738  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4739  Device = C->getDevice();
4740 
4741  OMPLexicalScope Scope(*this, S, OMPD_task);
4742  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4743 }
4744 
4746  const OMPTargetParallelDirective &S,
4747  PrePostActionTy &Action) {
4748  // Get the captured statement associated with the 'parallel' region.
4749  const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
4750  Action.Enter(CGF);
4751  auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4752  Action.Enter(CGF);
4753  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4754  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4755  CGF.EmitOMPPrivateClause(S, PrivateScope);
4756  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4757  (void)PrivateScope.Privatize();
4759  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4760  // TODO: Add support for clauses.
4761  CGF.EmitStmt(CS->getCapturedStmt());
4762  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4763  };
4764  emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
4767  [](CodeGenFunction &) { return nullptr; });
4768 }
4769 
4771  CodeGenModule &CGM, StringRef ParentName,
4772  const OMPTargetParallelDirective &S) {
4773  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4774  emitTargetParallelRegion(CGF, S, Action);
4775  };
4776  llvm::Function *Fn;
4777  llvm::Constant *Addr;
4778  // Emit target region as a standalone region.
4779  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4780  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4781  assert(Fn && Addr && "Target device function emission failed.");
4782 }
4783 
4785  const OMPTargetParallelDirective &S) {
4786  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4787  emitTargetParallelRegion(CGF, S, Action);
4788  };
4789  emitCommonOMPTargetDirective(*this, S, CodeGen);
4790 }
4791 
4794  PrePostActionTy &Action) {
4795  Action.Enter(CGF);
4796  // Emit directive as a combined directive that consists of two implicit
4797  // directives: 'parallel' with 'for' directive.
4798  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4799  Action.Enter(CGF);
4801  CGF, OMPD_target_parallel_for, S.hasCancel());
4804  };
4805  emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
4807 }
4808 
4810  CodeGenModule &CGM, StringRef ParentName,
4811  const OMPTargetParallelForDirective &S) {
4812  // Emit SPMD target parallel for region as a standalone region.
4813  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4814  emitTargetParallelForRegion(CGF, S, Action);
4815  };
4816  llvm::Function *Fn;
4817  llvm::Constant *Addr;
4818  // Emit target region as a standalone region.
4819  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4820  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4821  assert(Fn && Addr && "Target device function emission failed.");
4822 }
4823 
4825  const OMPTargetParallelForDirective &S) {
4826  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4827  emitTargetParallelForRegion(CGF, S, Action);
4828  };
4829  emitCommonOMPTargetDirective(*this, S, CodeGen);
4830 }
4831 
4832 static void
4835  PrePostActionTy &Action) {
4836  Action.Enter(CGF);
4837  // Emit directive as a combined directive that consists of two implicit
4838  // directives: 'parallel' with 'for' directive.
4839  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4840  Action.Enter(CGF);
4843  };
4844  emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
4846 }
4847 
4849  CodeGenModule &CGM, StringRef ParentName,
4851  // Emit SPMD target parallel for region as a standalone region.
4852  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4853  emitTargetParallelForSimdRegion(CGF, S, Action);
4854  };
4855  llvm::Function *Fn;
4856  llvm::Constant *Addr;
4857  // Emit target region as a standalone region.
4858  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4859  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4860  assert(Fn && Addr && "Target device function emission failed.");
4861 }
4862 
4865  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4866  emitTargetParallelForSimdRegion(CGF, S, Action);
4867  };
4868  emitCommonOMPTargetDirective(*this, S, CodeGen);
4869 }
4870 
4871 /// Emit a helper variable and return corresponding lvalue.
4872 static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
4873  const ImplicitParamDecl *PVD,
4875  const auto *VDecl = cast<VarDecl>(Helper->getDecl());
4876  Privates.addPrivate(VDecl,
4877  [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); });
4878 }
4879 
4882  // Emit outlined function for task construct.
4883  const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);
4884  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
4885  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
4886  const Expr *IfCond = nullptr;
4887  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4888  if (C->getNameModifier() == OMPD_unknown ||
4889  C->getNameModifier() == OMPD_taskloop) {
4890  IfCond = C->getCondition();
4891  break;
4892  }
4893  }
4894 
4895  OMPTaskDataTy Data;
4896  // Check if taskloop must be emitted without taskgroup.
4898  // TODO: Check if we should emit tied or untied task.
4899  Data.Tied = true;
4900  // Set scheduling for taskloop
4901  if (const auto* Clause = S.getSingleClause<OMPGrainsizeClause>()) {
4902  // grainsize clause
4903  Data.Schedule.setInt(/*IntVal=*/false);
4904  Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
4905  } else if (const auto* Clause = S.getSingleClause<OMPNumTasksClause>()) {
4906  // num_tasks clause
4907  Data.Schedule.setInt(/*IntVal=*/true);
4908  Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
4909  }
4910 
4911  auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
4912  // if (PreCond) {
4913  // for (IV in 0..LastIteration) BODY;
4914  // <Final counter/linear vars updates>;
4915  // }
4916  //
4917 
4918  // Emit: if (PreCond) - begin.
4919  // If the condition constant folds and can be elided, avoid emitting the
4920  // whole loop.
4921  bool CondConstant;
4922  llvm::BasicBlock *ContBlock = nullptr;
4923  OMPLoopScope PreInitScope(CGF, S);
4924  if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
4925  if (!CondConstant)
4926  return;
4927  } else {
4928  llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
4929  ContBlock = CGF.createBasicBlock("taskloop.if.end");
4930  emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
4931  CGF.getProfileCount(&S));
4932  CGF.EmitBlock(ThenBlock);
4933  CGF.incrementProfileCounter(&S);
4934  }
4935 
4937  CGF.EmitOMPSimdInit(S);
4938 
4939  OMPPrivateScope LoopScope(CGF);
4940  // Emit helper vars inits.
4941  enum { LowerBound = 5, UpperBound, Stride, LastIter };
4942  auto *I = CS->getCapturedDecl()->param_begin();
4943  auto *LBP = std::next(I, LowerBound);
4944  auto *UBP = std::next(I, UpperBound);
4945  auto *STP = std::next(I, Stride);
4946  auto *LIP = std::next(I, LastIter);
4947  mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
4948  LoopScope);
4949  mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
4950  LoopScope);
4951  mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
4952  mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
4953  LoopScope);
4954  CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
4955  bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4956  (void)LoopScope.Privatize();
4957  // Emit the loop iteration variable.
4958  const Expr *IVExpr = S.getIterationVariable();
4959  const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
4960  CGF.EmitVarDecl(*IVDecl);
4961  CGF.EmitIgnoredExpr(S.getInit());
4962 
4963  // Emit the iterations count variable.
4964  // If it is not a variable, Sema decided to calculate iterations count on
4965  // each iteration (e.g., it is foldable into a constant).
4966  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
4967  CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
4968  // Emit calculation of the iterations count.
4969  CGF.EmitIgnoredExpr(S.getCalcLastIteration());
4970  }
4971 
4972  CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
4973  S.getInc(),
4974  [&S](CodeGenFunction &CGF) {
4975  CGF.EmitOMPLoopBody(S, JumpDest());
4976  CGF.EmitStopPoint(&S);
4977  },
4978  [](CodeGenFunction &) {});
4979  // Emit: if (PreCond) - end.
4980  if (ContBlock) {
4981  CGF.EmitBranch(ContBlock);
4982  CGF.EmitBlock(ContBlock, true);
4983  }
4984  // Emit final copy of the lastprivate variables if IsLastIter != 0.
4985  if (HasLastprivateClause) {
4986  CGF.EmitOMPLastprivateClauseFinal(
4988  CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
4989  CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
4990  (*LIP)->getType(), S.getBeginLoc())));
4991  }
4992  };
4993  auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
4994  IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
4995  const OMPTaskDataTy &Data) {
4996  auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
4997  &Data](CodeGenFunction &CGF, PrePostActionTy &) {
4998  OMPLoopScope PreInitScope(CGF, S);
4999  CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
5000  OutlinedFn, SharedsTy,
5001  CapturedStruct, IfCond, Data);
5002  };
5003  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
5004  CodeGen);
5005  };
5006  if (Data.Nogroup) {
5007  EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data);
5008  } else {
5009  CGM.getOpenMPRuntime().emitTaskgroupRegion(
5010  *this,
5011  [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
5012  PrePostActionTy &Action) {
5013  Action.Enter(CGF);
5014  CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen,
5015  Data);
5016  },
5017  S.getBeginLoc());
5018  }
5019 }
5020 
5022  EmitOMPTaskLoopBasedDirective(S);
5023 }
5024 
5026  const OMPTaskLoopSimdDirective &S) {
5027  EmitOMPTaskLoopBasedDirective(S);
5028 }
5029 
5030 // Generate the instructions for '#pragma omp target update' directive.
5032  const OMPTargetUpdateDirective &S) {
5033  // If we don't have target devices, don't bother emitting the data mapping
5034  // code.
5035  if (CGM.getLangOpts().OMPTargetTriples.empty())
5036  return;
5037 
5038  // Check if we have any if clause associated with the directive.
5039  const Expr *IfCond = nullptr;
5040  if (const auto *C = S.getSingleClause<OMPIfClause>())
5041  IfCond = C->getCondition();
5042 
5043  // Check if we have any device clause associated with the directive.
5044  const Expr *Device = nullptr;
5045  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
5046  Device = C->getDevice();
5047 
5048  OMPLexicalScope Scope(*this, S, OMPD_task);
5049  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
5050 }
5051 
5053  const OMPExecutableDirective &D) {
5054  if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
5055  return;
5056  auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {
5058  emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action);
5059  } else {
5060  OMPPrivateScope LoopGlobals(CGF);
5061  if (const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
5062  for (const Expr *E : LD->counters()) {
5063  const auto *VD = dyn_cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5064  if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) {
5065  LValue GlobLVal = CGF.EmitLValue(E);
5066  LoopGlobals.addPrivate(
5067  VD, [&GlobLVal]() { return GlobLVal.getAddress(); });
5068  }
5069  if (isa<OMPCapturedExprDecl>(VD)) {
5070  // Emit only those that were not explicitly referenced in clauses.
5071  if (!CGF.LocalDeclMap.count(VD))
5072  CGF.EmitVarDecl(*VD);
5073  }
5074  }
5075  for (const auto *C : D.getClausesOfKind<OMPOrderedClause>()) {
5076  if (!C->getNumForLoops())
5077  continue;
5078  for (unsigned I = LD->getCollapsedNumber(),
5079  E = C->getLoopNumIterations().size();
5080  I < E; ++I) {
5081  if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
5082  cast<DeclRefExpr>(C->getLoopCounter(I))->getDecl())) {
5083  // Emit only those that were not explicitly referenced in clauses.
5084  if (!CGF.LocalDeclMap.count(VD))
5085  CGF.EmitVarDecl(*VD);
5086  }
5087  }
5088  }
5089  }
5090  LoopGlobals.Privatize();
5092  }
5093  };
5094  OMPSimdLexicalScope Scope(*this, D);
5095  CGM.getOpenMPRuntime().emitInlinedDirective(
5096  *this,
5097  isOpenMPSimdDirective(D.getDirectiveKind()) ? OMPD_simd
5098  : D.getDirectiveKind(),
5099  CodeGen);
5100 }
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
Definition: CGCall.cpp:652
bool isAggregate() const
Definition: CGValue.h:53
This represents &#39;#pragma omp distribute simd&#39; composite directive.
Definition: StmtOpenMP.h:3271
Expr * getNextUpperBound() const
Definition: StmtOpenMP.h:888
This represents &#39;#pragma omp master&#39; directive.
Definition: StmtOpenMP.h:1454
This represents &#39;#pragma omp task&#39; directive.
Definition: StmtOpenMP.h:1794
static const Decl * getCanonicalDecl(const Decl *D)
Represents a function declaration or definition.
Definition: Decl.h:1737
This represents &#39;thread_limit&#39; clause in the &#39;#pragma omp ...&#39; directive.
Expr * getUpperBoundVariable() const
Definition: StmtOpenMP.h:856
Other implicit parameter.
Definition: Decl.h:1509
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, SourceLocation Loc)
This represents clause &#39;copyin&#39; in the &#39;#pragma omp ...&#39; directives.
Complete object ctor.
Definition: ABI.h:25
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition: Type.h:2549
Scheduling data for loop-based OpenMP directives.
Definition: OpenMPKinds.h:156
A (possibly-)qualified type.
Definition: Type.h:639
capture_init_iterator capture_init_begin()
Retrieve the first initialization argument.
Definition: Stmt.h:3416
bool isArrayType() const
Definition: Type.h:6357
ArrayRef< OMPClause * > clauses()
Definition: StmtOpenMP.h:267
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *V, SourceLocation Loc)
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
Definition: CGDecl.cpp:162
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
Definition: Decl.h:4130
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Definition: CGExpr.cpp:138
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
Definition: Dominators.h:29
static std::pair< LValue, LValue > emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer...
static OMPClauseWithPreInit * get(OMPClause *C)
Stmt - This represents one statement.
Definition: Stmt.h:65
This represents clause &#39;in_reduction&#39; in the &#39;#pragma omp task&#39; directives.
Expr * getLowerBoundVariable() const
Definition: StmtOpenMP.h:848
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
Store of global named registers are always calls to intrinsics.
Definition: CGExpr.cpp:2155
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
Definition: Type.cpp:505
bool requiresCleanups() const
Determine whether this scope requires any cleanups.
void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S)
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
SmallVector< std::pair< OpenMPDependClauseKind, const Expr * >, 4 > Dependences
This represents &#39;#pragma omp for simd&#39; directive.
Definition: StmtOpenMP.h:1204
SmallVector< const Expr *, 4 > LastprivateCopies
QualType getLValueReferenceType(QualType T, bool SpelledAsLValue=true) const
Return the uniqued reference to the type for an lvalue reference to the specified type...
Decl - This represents one declaration (or definition), e.g.
Definition: DeclBase.h:87
This represents &#39;grainsize&#39; clause in the &#39;#pragma omp ...&#39; directive.
This represents &#39;#pragma omp teams distribute parallel for&#39; composite directive.
Definition: StmtOpenMP.h:3682
SourceLocation getBeginLoc() const
Returns starting location of directive kind.
Definition: StmtOpenMP.h:167
bool isNothrow() const
Definition: Decl.cpp:4493
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
This represents &#39;if&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:399
CapturedStmt * getInnermostCapturedStmt()
Get innermost captured statement for the construct.
Definition: StmtOpenMP.h:225
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Definition: Type.h:6257
void EmitOMPOrderedDirective(const OMPOrderedDirective &S)
bool isVolatile() const
Definition: CGValue.h:300
This represents &#39;priority&#39; clause in the &#39;#pragma omp ...&#39; directive.
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
The base class of the type hierarchy.
Definition: Type.h:1414
This represents &#39;#pragma omp target teams distribute&#39; combined directive.
Definition: StmtOpenMP.h:3819
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
Definition: CGExpr.cpp:1920
static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, bool IsSeqCst, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Definition: Stmt.h:3360
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Definition: Type.h:2824
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
const TargetInfo & getTargetInfo() const
Definition: ASTContext.h:689
Expr * getCombinedParForInDistCond() const
Definition: StmtOpenMP.h:976
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
const llvm::function_ref< void(CodeGenFunction &, llvm::Function *, const OMPTaskDataTy &)> TaskGenTy
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters)
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Definition: CGExpr.cpp:2328
Floating point control options.
Definition: LangOptions.h:306
This represents &#39;#pragma omp parallel for&#39; directive.
Defi