clang  6.0.0svn
CGStmtOpenMP.cpp
Go to the documentation of this file.
1 //===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This contains code to emit OpenMP nodes as LLVM code.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "CGCleanup.h"
15 #include "CGOpenMPRuntime.h"
16 #include "CodeGenFunction.h"
17 #include "CodeGenModule.h"
18 #include "TargetInfo.h"
19 #include "clang/AST/Stmt.h"
20 #include "clang/AST/StmtOpenMP.h"
21 #include "clang/AST/DeclOpenMP.h"
22 #include "llvm/IR/CallSite.h"
23 using namespace clang;
24 using namespace CodeGen;
25 
26 namespace {
27 /// Lexical scope for OpenMP executable constructs, that handles correct codegen
28 /// for captured expressions.
29 class OMPLexicalScope : public CodeGenFunction::LexicalScope {
30  void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
31  for (const auto *C : S.clauses()) {
32  if (auto *CPI = OMPClauseWithPreInit::get(C)) {
33  if (auto *PreInit = cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
34  for (const auto *I : PreInit->decls()) {
35  if (!I->hasAttr<OMPCaptureNoInitAttr>())
36  CGF.EmitVarDecl(cast<VarDecl>(*I));
37  else {
39  CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
40  CGF.EmitAutoVarCleanups(Emission);
41  }
42  }
43  }
44  }
45  }
46  }
47  CodeGenFunction::OMPPrivateScope InlinedShareds;
48 
49  static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
50  return CGF.LambdaCaptureFields.lookup(VD) ||
51  (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
52  (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl));
53  }
54 
55 public:
56  OMPLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S,
57  bool AsInlined = false, bool EmitPreInitStmt = true)
58  : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
59  InlinedShareds(CGF) {
60  if (EmitPreInitStmt)
61  emitPreInitStmt(CGF, S);
62  if (AsInlined) {
63  if (S.hasAssociatedStmt()) {
64  auto *CS = cast<CapturedStmt>(S.getAssociatedStmt());
65  for (auto &C : CS->captures()) {
66  if (C.capturesVariable() || C.capturesVariableByCopy()) {
67  auto *VD = C.getCapturedVar();
68  assert(VD == VD->getCanonicalDecl() &&
69  "Canonical decl must be captured.");
70  DeclRefExpr DRE(const_cast<VarDecl *>(VD),
71  isCapturedVar(CGF, VD) ||
72  (CGF.CapturedStmtInfo &&
73  InlinedShareds.isGlobalVarCaptured(VD)),
75  SourceLocation());
76  InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
77  return CGF.EmitLValue(&DRE).getAddress();
78  });
79  }
80  }
81  (void)InlinedShareds.Privatize();
82  }
83  }
84  }
85 };
86 
87 /// Lexical scope for OpenMP parallel construct, that handles correct codegen
88 /// for captured expressions.
89 class OMPParallelScope final : public OMPLexicalScope {
90  bool EmitPreInitStmt(const OMPExecutableDirective &S) {
92  return !(isOpenMPTargetExecutionDirective(Kind) ||
95  }
96 
97 public:
98  OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
99  : OMPLexicalScope(CGF, S,
100  /*AsInlined=*/false,
101  /*EmitPreInitStmt=*/EmitPreInitStmt(S)) {}
102 };
103 
104 /// Lexical scope for OpenMP teams construct, that handles correct codegen
105 /// for captured expressions.
106 class OMPTeamsScope final : public OMPLexicalScope {
107  bool EmitPreInitStmt(const OMPExecutableDirective &S) {
109  return !isOpenMPTargetExecutionDirective(Kind) &&
111  }
112 
113 public:
114  OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
115  : OMPLexicalScope(CGF, S,
116  /*AsInlined=*/false,
117  /*EmitPreInitStmt=*/EmitPreInitStmt(S)) {}
118 };
119 
120 /// Private scope for OpenMP loop-based directives, that supports capturing
121 /// of used expression from loop statement.
122 class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
123  void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) {
124  CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
125  for (auto *E : S.counters()) {
126  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
127  (void)PreCondScope.addPrivate(VD, [&CGF, VD]() {
128  return CGF.CreateMemTemp(VD->getType().getNonReferenceType());
129  });
130  }
131  (void)PreCondScope.Privatize();
132  if (auto *LD = dyn_cast<OMPLoopDirective>(&S)) {
133  if (auto *PreInits = cast_or_null<DeclStmt>(LD->getPreInits())) {
134  for (const auto *I : PreInits->decls())
135  CGF.EmitVarDecl(cast<VarDecl>(*I));
136  }
137  }
138  }
139 
140 public:
141  OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S)
142  : CodeGenFunction::RunCleanupsScope(CGF) {
143  emitPreInitStmt(CGF, S);
144  }
145 };
146 
147 } // namespace
148 
150  const OMPExecutableDirective &S,
151  const RegionCodeGenTy &CodeGen);
152 
154  if (auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
155  if (auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
156  OrigVD = OrigVD->getCanonicalDecl();
157  bool IsCaptured =
158  LambdaCaptureFields.lookup(OrigVD) ||
159  (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
160  (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
161  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), IsCaptured,
162  OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
163  return EmitLValue(&DRE);
164  }
165  }
166  return EmitLValue(E);
167 }
168 
170  auto &C = getContext();
171  llvm::Value *Size = nullptr;
172  auto SizeInChars = C.getTypeSizeInChars(Ty);
173  if (SizeInChars.isZero()) {
174  // getTypeSizeInChars() returns 0 for a VLA.
175  while (auto *VAT = C.getAsVariableArrayType(Ty)) {
176  llvm::Value *ArraySize;
177  std::tie(ArraySize, Ty) = getVLASize(VAT);
178  Size = Size ? Builder.CreateNUWMul(Size, ArraySize) : ArraySize;
179  }
180  SizeInChars = C.getTypeSizeInChars(Ty);
181  if (SizeInChars.isZero())
182  return llvm::ConstantInt::get(SizeTy, /*V=*/0);
183  Size = Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
184  } else
185  Size = CGM.getSize(SizeInChars);
186  return Size;
187 }
188 
190  const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
191  const RecordDecl *RD = S.getCapturedRecordDecl();
192  auto CurField = RD->field_begin();
193  auto CurCap = S.captures().begin();
195  E = S.capture_init_end();
196  I != E; ++I, ++CurField, ++CurCap) {
197  if (CurField->hasCapturedVLAType()) {
198  auto VAT = CurField->getCapturedVLAType();
199  auto *Val = VLASizeMap[VAT->getSizeExpr()];
200  CapturedVars.push_back(Val);
201  } else if (CurCap->capturesThis())
202  CapturedVars.push_back(CXXThisValue);
203  else if (CurCap->capturesVariableByCopy()) {
204  llvm::Value *CV =
205  EmitLoadOfLValue(EmitLValue(*I), SourceLocation()).getScalarVal();
206 
207  // If the field is not a pointer, we need to save the actual value
208  // and load it as a void pointer.
209  if (!CurField->getType()->isAnyPointerType()) {
210  auto &Ctx = getContext();
211  auto DstAddr = CreateMemTemp(
212  Ctx.getUIntPtrType(),
213  Twine(CurCap->getCapturedVar()->getName()) + ".casted");
214  LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
215 
216  auto *SrcAddrVal = EmitScalarConversion(
217  DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
218  Ctx.getPointerType(CurField->getType()), SourceLocation());
219  LValue SrcLV =
220  MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
221 
222  // Store the value using the source type pointer.
223  EmitStoreThroughLValue(RValue::get(CV), SrcLV);
224 
225  // Load the value using the destination type pointer.
226  CV = EmitLoadOfLValue(DstLV, SourceLocation()).getScalarVal();
227  }
228  CapturedVars.push_back(CV);
229  } else {
230  assert(CurCap->capturesVariable() && "Expected capture by reference.");
231  CapturedVars.push_back(EmitLValue(*I).getAddress().getPointer());
232  }
233  }
234 }
235 
237  StringRef Name, LValue AddrLV,
238  bool isReferenceType = false) {
239  ASTContext &Ctx = CGF.getContext();
240 
241  auto *CastedPtr = CGF.EmitScalarConversion(
242  AddrLV.getAddress().getPointer(), Ctx.getUIntPtrType(),
243  Ctx.getPointerType(DstType), SourceLocation());
244  auto TmpAddr =
245  CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType))
246  .getAddress();
247 
248  // If we are dealing with references we need to return the address of the
249  // reference instead of the reference of the value.
250  if (isReferenceType) {
251  QualType RefType = Ctx.getLValueReferenceType(DstType);
252  auto *RefVal = TmpAddr.getPointer();
253  TmpAddr = CGF.CreateMemTemp(RefType, Twine(Name) + ".ref");
254  auto TmpLVal = CGF.MakeAddrLValue(TmpAddr, RefType);
255  CGF.EmitStoreThroughLValue(RValue::get(RefVal), TmpLVal, /*isInit*/ true);
256  }
257 
258  return TmpAddr;
259 }
260 
262  if (T->isLValueReferenceType()) {
263  return C.getLValueReferenceType(
265  /*SpelledAsLValue=*/false);
266  }
267  if (T->isPointerType())
269  if (auto *A = T->getAsArrayTypeUnsafe()) {
270  if (auto *VLA = dyn_cast<VariableArrayType>(A))
271  return getCanonicalParamType(C, VLA->getElementType());
272  else if (!A->isVariablyModifiedType())
273  return C.getCanonicalType(T);
274  }
275  return C.getCanonicalParamType(T);
276 }
277 
278 namespace {
279  /// Contains required data for proper outlined function codegen.
280  struct FunctionOptions {
281  /// Captured statement for which the function is generated.
282  const CapturedStmt *S = nullptr;
283  /// true if cast to/from UIntPtr is required for variables captured by
284  /// value.
285  const bool UIntPtrCastRequired = true;
286  /// true if only casted arguments must be registered as local args or VLA
287  /// sizes.
288  const bool RegisterCastedArgsOnly = false;
289  /// Name of the generated function.
290  const StringRef FunctionName;
291  explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
292  bool RegisterCastedArgsOnly,
293  StringRef FunctionName)
294  : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
295  RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
296  FunctionName(FunctionName) {}
297  };
298 }
299 
300 static llvm::Function *emitOutlinedFunctionPrologue(
301  CodeGenFunction &CGF, FunctionArgList &Args,
302  llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
303  &LocalAddrs,
304  llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
305  &VLASizes,
306  llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
307  const CapturedDecl *CD = FO.S->getCapturedDecl();
308  const RecordDecl *RD = FO.S->getCapturedRecordDecl();
309  assert(CD->hasBody() && "missing CapturedDecl body");
310 
311  CXXThisValue = nullptr;
312  // Build the argument list.
313  CodeGenModule &CGM = CGF.CGM;
314  ASTContext &Ctx = CGM.getContext();
315  FunctionArgList TargetArgs;
316  Args.append(CD->param_begin(),
317  std::next(CD->param_begin(), CD->getContextParamPosition()));
318  TargetArgs.append(
319  CD->param_begin(),
320  std::next(CD->param_begin(), CD->getContextParamPosition()));
321  auto I = FO.S->captures().begin();
322  FunctionDecl *DebugFunctionDecl = nullptr;
323  if (!FO.UIntPtrCastRequired) {
325  DebugFunctionDecl = FunctionDecl::Create(
326  Ctx, Ctx.getTranslationUnitDecl(), FO.S->getLocStart(),
329  Ctx.getFunctionType(Ctx.VoidTy, llvm::None, EPI)),
330  SC_Static, /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/false);
331  }
332  for (auto *FD : RD->fields()) {
333  QualType ArgType = FD->getType();
334  IdentifierInfo *II = nullptr;
335  VarDecl *CapVar = nullptr;
336 
337  // If this is a capture by copy and the type is not a pointer, the outlined
338  // function argument type should be uintptr and the value properly casted to
339  // uintptr. This is necessary given that the runtime library is only able to
340  // deal with pointers. We can pass in the same way the VLA type sizes to the
341  // outlined function.
342  if ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
343  I->capturesVariableArrayType()) {
344  if (FO.UIntPtrCastRequired)
345  ArgType = Ctx.getUIntPtrType();
346  }
347 
348  if (I->capturesVariable() || I->capturesVariableByCopy()) {
349  CapVar = I->getCapturedVar();
350  II = CapVar->getIdentifier();
351  } else if (I->capturesThis())
352  II = &Ctx.Idents.get("this");
353  else {
354  assert(I->capturesVariableArrayType());
355  II = &Ctx.Idents.get("vla");
356  }
357  if (ArgType->isVariablyModifiedType())
358  ArgType = getCanonicalParamType(Ctx, ArgType);
359  VarDecl *Arg;
360  if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
361  Arg = ParmVarDecl::Create(
362  Ctx, DebugFunctionDecl,
363  CapVar ? CapVar->getLocStart() : FD->getLocStart(),
364  CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
365  /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
366  } else {
367  Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
368  II, ArgType, ImplicitParamDecl::Other);
369  }
370  Args.emplace_back(Arg);
371  // Do not cast arguments if we emit function with non-original types.
372  TargetArgs.emplace_back(
373  FO.UIntPtrCastRequired
374  ? Arg
375  : CGM.getOpenMPRuntime().translateParameter(FD, Arg));
376  ++I;
377  }
378  Args.append(
379  std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
380  CD->param_end());
381  TargetArgs.append(
382  std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
383  CD->param_end());
384 
385  // Create the function declaration.
386  const CGFunctionInfo &FuncInfo =
387  CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, TargetArgs);
388  llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
389 
390  llvm::Function *F =
392  FO.FunctionName, &CGM.getModule());
393  CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
394  if (CD->isNothrow())
395  F->setDoesNotThrow();
396 
397  // Generate the function.
398  CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
399  FO.S->getLocStart(), CD->getBody()->getLocStart());
400  unsigned Cnt = CD->getContextParamPosition();
401  I = FO.S->captures().begin();
402  for (auto *FD : RD->fields()) {
403  // Do not map arguments if we emit function with non-original types.
404  Address LocalAddr(Address::invalid());
405  if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
406  LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
407  TargetArgs[Cnt]);
408  } else {
409  LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
410  }
411  // If we are capturing a pointer by copy we don't need to do anything, just
412  // use the value that we get from the arguments.
413  if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
414  const VarDecl *CurVD = I->getCapturedVar();
415  // If the variable is a reference we need to materialize it here.
416  if (CurVD->getType()->isReferenceType()) {
417  Address RefAddr = CGF.CreateMemTemp(
418  CurVD->getType(), CGM.getPointerAlign(), ".materialized_ref");
419  CGF.EmitStoreOfScalar(LocalAddr.getPointer(), RefAddr,
420  /*Volatile=*/false, CurVD->getType());
421  LocalAddr = RefAddr;
422  }
423  if (!FO.RegisterCastedArgsOnly)
424  LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
425  ++Cnt;
426  ++I;
427  continue;
428  }
429 
430  LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
432  if (FD->hasCapturedVLAType()) {
433  if (FO.UIntPtrCastRequired) {
434  ArgLVal = CGF.MakeAddrLValue(castValueFromUintptr(CGF, FD->getType(),
435  Args[Cnt]->getName(),
436  ArgLVal),
438  }
439  auto *ExprArg =
440  CGF.EmitLoadOfLValue(ArgLVal, SourceLocation()).getScalarVal();
441  auto VAT = FD->getCapturedVLAType();
442  VLASizes.insert({Args[Cnt], {VAT->getSizeExpr(), ExprArg}});
443  } else if (I->capturesVariable()) {
444  auto *Var = I->getCapturedVar();
445  QualType VarTy = Var->getType();
446  Address ArgAddr = ArgLVal.getAddress();
447  if (!VarTy->isReferenceType()) {
448  if (ArgLVal.getType()->isLValueReferenceType()) {
449  ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
450  } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) {
451  assert(ArgLVal.getType()->isPointerType());
452  ArgAddr = CGF.EmitLoadOfPointer(
453  ArgAddr, ArgLVal.getType()->castAs<PointerType>());
454  }
455  }
456  if (!FO.RegisterCastedArgsOnly) {
457  LocalAddrs.insert(
458  {Args[Cnt],
459  {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}});
460  }
461  } else if (I->capturesVariableByCopy()) {
462  assert(!FD->getType()->isAnyPointerType() &&
463  "Not expecting a captured pointer.");
464  auto *Var = I->getCapturedVar();
465  QualType VarTy = Var->getType();
466  LocalAddrs.insert(
467  {Args[Cnt],
468  {Var,
469  FO.UIntPtrCastRequired
470  ? castValueFromUintptr(CGF, FD->getType(), Args[Cnt]->getName(),
471  ArgLVal, VarTy->isReferenceType())
472  : ArgLVal.getAddress()}});
473  } else {
474  // If 'this' is captured, load it into CXXThisValue.
475  assert(I->capturesThis());
476  CXXThisValue = CGF.EmitLoadOfLValue(ArgLVal, Args[Cnt]->getLocation())
477  .getScalarVal();
478  LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress()}});
479  }
480  ++Cnt;
481  ++I;
482  }
483 
484  return F;
485 }
486 
487 llvm::Function *
489  assert(
490  CapturedStmtInfo &&
491  "CapturedStmtInfo should be set when generating the captured function");
492  const CapturedDecl *CD = S.getCapturedDecl();
493  // Build the argument list.
494  bool NeedWrapperFunction =
495  getDebugInfo() &&
496  CGM.getCodeGenOpts().getDebugInfo() >= codegenoptions::LimitedDebugInfo;
497  FunctionArgList Args;
498  llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
499  llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
500  SmallString<256> Buffer;
501  llvm::raw_svector_ostream Out(Buffer);
502  Out << CapturedStmtInfo->getHelperName();
503  if (NeedWrapperFunction)
504  Out << "_debug__";
505  FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
506  Out.str());
507  llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs,
508  VLASizes, CXXThisValue, FO);
509  for (const auto &LocalAddrPair : LocalAddrs) {
510  if (LocalAddrPair.second.first) {
511  setAddrOfLocalVar(LocalAddrPair.second.first,
512  LocalAddrPair.second.second);
513  }
514  }
515  for (const auto &VLASizePair : VLASizes)
516  VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
517  PGO.assignRegionCounters(GlobalDecl(CD), F);
518  CapturedStmtInfo->EmitBody(*this, CD->getBody());
519  FinishFunction(CD->getBodyRBrace());
520  if (!NeedWrapperFunction)
521  return F;
522 
523  FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
524  /*RegisterCastedArgsOnly=*/true,
525  CapturedStmtInfo->getHelperName());
526  CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
527  Args.clear();
528  LocalAddrs.clear();
529  VLASizes.clear();
530  llvm::Function *WrapperF =
531  emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
532  WrapperCGF.CXXThisValue, WrapperFO);
534  for (const auto *Arg : Args) {
536  auto I = LocalAddrs.find(Arg);
537  if (I != LocalAddrs.end()) {
538  LValue LV = WrapperCGF.MakeAddrLValue(
539  I->second.second,
540  I->second.first ? I->second.first->getType() : Arg->getType(),
542  CallArg = WrapperCGF.EmitLoadOfScalar(LV, SourceLocation());
543  } else {
544  auto EI = VLASizes.find(Arg);
545  if (EI != VLASizes.end())
546  CallArg = EI->second.second;
547  else {
548  LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
549  Arg->getType(),
551  CallArg = WrapperCGF.EmitLoadOfScalar(LV, SourceLocation());
552  }
553  }
554  CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
555  }
556  CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.getLocStart(),
557  F, CallArgs);
558  WrapperCGF.FinishFunction();
559  return WrapperF;
560 }
561 
562 //===----------------------------------------------------------------------===//
563 // OpenMP Directive Emission
564 //===----------------------------------------------------------------------===//
566  Address DestAddr, Address SrcAddr, QualType OriginalType,
567  const llvm::function_ref<void(Address, Address)> &CopyGen) {
568  // Perform element-by-element initialization.
569  QualType ElementTy;
570 
571  // Drill down to the base element type on both arrays.
572  auto ArrayTy = OriginalType->getAsArrayTypeUnsafe();
573  auto NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
574  SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
575 
576  auto SrcBegin = SrcAddr.getPointer();
577  auto DestBegin = DestAddr.getPointer();
578  // Cast from pointer to array type to pointer to single element.
579  auto DestEnd = Builder.CreateGEP(DestBegin, NumElements);
580  // The basic structure here is a while-do loop.
581  auto BodyBB = createBasicBlock("omp.arraycpy.body");
582  auto DoneBB = createBasicBlock("omp.arraycpy.done");
583  auto IsEmpty =
584  Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
585  Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
586 
587  // Enter the loop body, making that address the current address.
588  auto EntryBB = Builder.GetInsertBlock();
589  EmitBlock(BodyBB);
590 
591  CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
592 
593  llvm::PHINode *SrcElementPHI =
594  Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
595  SrcElementPHI->addIncoming(SrcBegin, EntryBB);
596  Address SrcElementCurrent =
597  Address(SrcElementPHI,
598  SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
599 
600  llvm::PHINode *DestElementPHI =
601  Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
602  DestElementPHI->addIncoming(DestBegin, EntryBB);
603  Address DestElementCurrent =
604  Address(DestElementPHI,
605  DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
606 
607  // Emit copy.
608  CopyGen(DestElementCurrent, SrcElementCurrent);
609 
610  // Shift the address forward by one element.
611  auto DestElementNext = Builder.CreateConstGEP1_32(
612  DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
613  auto SrcElementNext = Builder.CreateConstGEP1_32(
614  SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
615  // Check whether we've reached the end.
616  auto Done =
617  Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
618  Builder.CreateCondBr(Done, DoneBB, BodyBB);
619  DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
620  SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
621 
622  // Done.
623  EmitBlock(DoneBB, /*IsFinished=*/true);
624 }
625 
626 void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
627  Address SrcAddr, const VarDecl *DestVD,
628  const VarDecl *SrcVD, const Expr *Copy) {
629  if (OriginalType->isArrayType()) {
630  auto *BO = dyn_cast<BinaryOperator>(Copy);
631  if (BO && BO->getOpcode() == BO_Assign) {
632  // Perform simple memcpy for simple copying.
633  EmitAggregateAssign(DestAddr, SrcAddr, OriginalType);
634  } else {
635  // For arrays with complex element types perform element by element
636  // copying.
637  EmitOMPAggregateAssign(
638  DestAddr, SrcAddr, OriginalType,
639  [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
640  // Working with the single array element, so have to remap
641  // destination and source variables to corresponding array
642  // elements.
644  Remap.addPrivate(DestVD, [DestElement]() -> Address {
645  return DestElement;
646  });
647  Remap.addPrivate(
648  SrcVD, [SrcElement]() -> Address { return SrcElement; });
649  (void)Remap.Privatize();
650  EmitIgnoredExpr(Copy);
651  });
652  }
653  } else {
654  // Remap pseudo source variable to private copy.
656  Remap.addPrivate(SrcVD, [SrcAddr]() -> Address { return SrcAddr; });
657  Remap.addPrivate(DestVD, [DestAddr]() -> Address { return DestAddr; });
658  (void)Remap.Privatize();
659  // Emit copying of the whole variable.
660  EmitIgnoredExpr(Copy);
661  }
662 }
663 
665  OMPPrivateScope &PrivateScope) {
666  if (!HaveInsertPoint())
667  return false;
668  bool FirstprivateIsLastprivate = false;
669  llvm::DenseSet<const VarDecl *> Lastprivates;
670  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
671  for (const auto *D : C->varlists())
672  Lastprivates.insert(
673  cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl());
674  }
675  llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
676  CGCapturedStmtInfo CapturesInfo(cast<CapturedStmt>(*D.getAssociatedStmt()));
677  for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
678  auto IRef = C->varlist_begin();
679  auto InitsRef = C->inits().begin();
680  for (auto IInit : C->private_copies()) {
681  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
682  bool ThisFirstprivateIsLastprivate =
683  Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
684  auto *CapFD = CapturesInfo.lookup(OrigVD);
685  auto *FD = CapturedStmtInfo->lookup(OrigVD);
686  if (!ThisFirstprivateIsLastprivate && FD && (FD == CapFD) &&
687  !FD->getType()->isReferenceType()) {
688  EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
689  ++IRef;
690  ++InitsRef;
691  continue;
692  }
693  FirstprivateIsLastprivate =
694  FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
695  if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
696  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
697  auto *VDInit = cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
698  bool IsRegistered;
699  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
700  /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
701  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
702  Address OriginalAddr = EmitLValue(&DRE).getAddress();
703  QualType Type = VD->getType();
704  if (Type->isArrayType()) {
705  // Emit VarDecl with copy init for arrays.
706  // Get the address of the original variable captured in current
707  // captured region.
708  IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address {
709  auto Emission = EmitAutoVarAlloca(*VD);
710  auto *Init = VD->getInit();
711  if (!isa<CXXConstructExpr>(Init) || isTrivialInitializer(Init)) {
712  // Perform simple memcpy.
713  EmitAggregateAssign(Emission.getAllocatedAddress(), OriginalAddr,
714  Type);
715  } else {
716  EmitOMPAggregateAssign(
717  Emission.getAllocatedAddress(), OriginalAddr, Type,
718  [this, VDInit, Init](Address DestElement,
719  Address SrcElement) {
720  // Clean up any temporaries needed by the initialization.
721  RunCleanupsScope InitScope(*this);
722  // Emit initialization for single element.
723  setAddrOfLocalVar(VDInit, SrcElement);
724  EmitAnyExprToMem(Init, DestElement,
725  Init->getType().getQualifiers(),
726  /*IsInitializer*/ false);
727  LocalDeclMap.erase(VDInit);
728  });
729  }
730  EmitAutoVarCleanups(Emission);
731  return Emission.getAllocatedAddress();
732  });
733  } else {
734  IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address {
735  // Emit private VarDecl with copy init.
736  // Remap temp VDInit variable to the address of the original
737  // variable
738  // (for proper handling of captured global variables).
739  setAddrOfLocalVar(VDInit, OriginalAddr);
740  EmitDecl(*VD);
741  LocalDeclMap.erase(VDInit);
742  return GetAddrOfLocalVar(VD);
743  });
744  }
745  assert(IsRegistered &&
746  "firstprivate var already registered as private");
747  // Silence the warning about unused variable.
748  (void)IsRegistered;
749  }
750  ++IRef;
751  ++InitsRef;
752  }
753  }
754  return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
755 }
756 
758  const OMPExecutableDirective &D,
759  CodeGenFunction::OMPPrivateScope &PrivateScope) {
760  if (!HaveInsertPoint())
761  return;
762  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
763  for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
764  auto IRef = C->varlist_begin();
765  for (auto IInit : C->private_copies()) {
766  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
767  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
768  auto VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
769  bool IsRegistered =
770  PrivateScope.addPrivate(OrigVD, [&]() -> Address {
771  // Emit private VarDecl with copy init.
772  EmitDecl(*VD);
773  return GetAddrOfLocalVar(VD);
774  });
775  assert(IsRegistered && "private var already registered as private");
776  // Silence the warning about unused variable.
777  (void)IsRegistered;
778  }
779  ++IRef;
780  }
781  }
782 }
783 
785  if (!HaveInsertPoint())
786  return false;
787  // threadprivate_var1 = master_threadprivate_var1;
788  // operator=(threadprivate_var2, master_threadprivate_var2);
789  // ...
790  // __kmpc_barrier(&loc, global_tid);
792  llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
793  for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
794  auto IRef = C->varlist_begin();
795  auto ISrcRef = C->source_exprs().begin();
796  auto IDestRef = C->destination_exprs().begin();
797  for (auto *AssignOp : C->assignment_ops()) {
798  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
799  QualType Type = VD->getType();
800  if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
801  // Get the address of the master variable. If we are emitting code with
802  // TLS support, the address is passed from the master as field in the
803  // captured declaration.
804  Address MasterAddr = Address::invalid();
805  if (getLangOpts().OpenMPUseTLS &&
806  getContext().getTargetInfo().isTLSSupported()) {
807  assert(CapturedStmtInfo->lookup(VD) &&
808  "Copyin threadprivates should have been captured!");
809  DeclRefExpr DRE(const_cast<VarDecl *>(VD), true, (*IRef)->getType(),
810  VK_LValue, (*IRef)->getExprLoc());
811  MasterAddr = EmitLValue(&DRE).getAddress();
812  LocalDeclMap.erase(VD);
813  } else {
814  MasterAddr =
815  Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
816  : CGM.GetAddrOfGlobal(VD),
817  getContext().getDeclAlign(VD));
818  }
819  // Get the address of the threadprivate variable.
820  Address PrivateAddr = EmitLValue(*IRef).getAddress();
821  if (CopiedVars.size() == 1) {
822  // At first check if current thread is a master thread. If it is, no
823  // need to copy data.
824  CopyBegin = createBasicBlock("copyin.not.master");
825  CopyEnd = createBasicBlock("copyin.not.master.end");
826  Builder.CreateCondBr(
827  Builder.CreateICmpNE(
828  Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy),
829  Builder.CreatePtrToInt(PrivateAddr.getPointer(), CGM.IntPtrTy)),
830  CopyBegin, CopyEnd);
831  EmitBlock(CopyBegin);
832  }
833  auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
834  auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
835  EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
836  }
837  ++IRef;
838  ++ISrcRef;
839  ++IDestRef;
840  }
841  }
842  if (CopyEnd) {
843  // Exit out of copying procedure for non-master thread.
844  EmitBlock(CopyEnd, /*IsFinished=*/true);
845  return true;
846  }
847  return false;
848 }
849 
851  const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
852  if (!HaveInsertPoint())
853  return false;
854  bool HasAtLeastOneLastprivate = false;
857  auto *LoopDirective = cast<OMPLoopDirective>(&D);
858  for (auto *C : LoopDirective->counters()) {
859  SIMDLCVs.insert(
860  cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
861  }
862  }
863  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
864  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
865  HasAtLeastOneLastprivate = true;
867  break;
868  auto IRef = C->varlist_begin();
869  auto IDestRef = C->destination_exprs().begin();
870  for (auto *IInit : C->private_copies()) {
871  // Keep the address of the original variable for future update at the end
872  // of the loop.
873  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
874  // Taskloops do not require additional initialization, it is done in
875  // runtime support library.
876  if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
877  auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
878  PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() -> Address {
879  DeclRefExpr DRE(
880  const_cast<VarDecl *>(OrigVD),
881  /*RefersToEnclosingVariableOrCapture=*/CapturedStmtInfo->lookup(
882  OrigVD) != nullptr,
883  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
884  return EmitLValue(&DRE).getAddress();
885  });
886  // Check if the variable is also a firstprivate: in this case IInit is
887  // not generated. Initialization of this variable will happen in codegen
888  // for 'firstprivate' clause.
889  if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
890  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
891  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address {
892  // Emit private VarDecl with copy init.
893  EmitDecl(*VD);
894  return GetAddrOfLocalVar(VD);
895  });
896  assert(IsRegistered &&
897  "lastprivate var already registered as private");
898  (void)IsRegistered;
899  }
900  }
901  ++IRef;
902  ++IDestRef;
903  }
904  }
905  return HasAtLeastOneLastprivate;
906 }
907 
909  const OMPExecutableDirective &D, bool NoFinals,
910  llvm::Value *IsLastIterCond) {
911  if (!HaveInsertPoint())
912  return;
913  // Emit following code:
914  // if (<IsLastIterCond>) {
915  // orig_var1 = private_orig_var1;
916  // ...
917  // orig_varn = private_orig_varn;
918  // }
919  llvm::BasicBlock *ThenBB = nullptr;
920  llvm::BasicBlock *DoneBB = nullptr;
921  if (IsLastIterCond) {
922  ThenBB = createBasicBlock(".omp.lastprivate.then");
923  DoneBB = createBasicBlock(".omp.lastprivate.done");
924  Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
925  EmitBlock(ThenBB);
926  }
927  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
928  llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
929  if (auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
930  auto IC = LoopDirective->counters().begin();
931  for (auto F : LoopDirective->finals()) {
932  auto *D =
933  cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
934  if (NoFinals)
935  AlreadyEmittedVars.insert(D);
936  else
937  LoopCountersAndUpdates[D] = F;
938  ++IC;
939  }
940  }
941  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
942  auto IRef = C->varlist_begin();
943  auto ISrcRef = C->source_exprs().begin();
944  auto IDestRef = C->destination_exprs().begin();
945  for (auto *AssignOp : C->assignment_ops()) {
946  auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
947  QualType Type = PrivateVD->getType();
948  auto *CanonicalVD = PrivateVD->getCanonicalDecl();
949  if (AlreadyEmittedVars.insert(CanonicalVD).second) {
950  // If lastprivate variable is a loop control variable for loop-based
951  // directive, update its value before copyin back to original
952  // variable.
953  if (auto *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
954  EmitIgnoredExpr(FinalExpr);
955  auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
956  auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
957  // Get the address of the original variable.
958  Address OriginalAddr = GetAddrOfLocalVar(DestVD);
959  // Get the address of the private variable.
960  Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
961  if (auto RefTy = PrivateVD->getType()->getAs<ReferenceType>())
962  PrivateAddr =
963  Address(Builder.CreateLoad(PrivateAddr),
964  getNaturalTypeAlignment(RefTy->getPointeeType()));
965  EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
966  }
967  ++IRef;
968  ++ISrcRef;
969  ++IDestRef;
970  }
971  if (auto *PostUpdate = C->getPostUpdateExpr())
972  EmitIgnoredExpr(PostUpdate);
973  }
974  if (IsLastIterCond)
975  EmitBlock(DoneBB, /*IsFinished=*/true);
976 }
977 
979  const OMPExecutableDirective &D,
980  CodeGenFunction::OMPPrivateScope &PrivateScope) {
981  if (!HaveInsertPoint())
982  return;
985  SmallVector<const Expr *, 4> ReductionOps;
988  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
989  auto IPriv = C->privates().begin();
990  auto IRed = C->reduction_ops().begin();
991  auto ILHS = C->lhs_exprs().begin();
992  auto IRHS = C->rhs_exprs().begin();
993  for (const auto *Ref : C->varlists()) {
994  Shareds.emplace_back(Ref);
995  Privates.emplace_back(*IPriv);
996  ReductionOps.emplace_back(*IRed);
997  LHSs.emplace_back(*ILHS);
998  RHSs.emplace_back(*IRHS);
999  std::advance(IPriv, 1);
1000  std::advance(IRed, 1);
1001  std::advance(ILHS, 1);
1002  std::advance(IRHS, 1);
1003  }
1004  }
1005  ReductionCodeGen RedCG(Shareds, Privates, ReductionOps);
1006  unsigned Count = 0;
1007  auto ILHS = LHSs.begin();
1008  auto IRHS = RHSs.begin();
1009  auto IPriv = Privates.begin();
1010  for (const auto *IRef : Shareds) {
1011  auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1012  // Emit private VarDecl with reduction init.
1013  RedCG.emitSharedLValue(*this, Count);
1014  RedCG.emitAggregateType(*this, Count);
1015  auto Emission = EmitAutoVarAlloca(*PrivateVD);
1016  RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
1017  RedCG.getSharedLValue(Count),
1018  [&Emission](CodeGenFunction &CGF) {
1019  CGF.EmitAutoVarInit(Emission);
1020  return true;
1021  });
1022  EmitAutoVarCleanups(Emission);
1023  Address BaseAddr = RedCG.adjustPrivateAddress(
1024  *this, Count, Emission.getAllocatedAddress());
1025  bool IsRegistered = PrivateScope.addPrivate(
1026  RedCG.getBaseDecl(Count), [BaseAddr]() -> Address { return BaseAddr; });
1027  assert(IsRegistered && "private var already registered as private");
1028  // Silence the warning about unused variable.
1029  (void)IsRegistered;
1030 
1031  auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1032  auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1033  QualType Type = PrivateVD->getType();
1034  bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1035  if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1036  // Store the address of the original variable associated with the LHS
1037  // implicit variable.
1038  PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() -> Address {
1039  return RedCG.getSharedLValue(Count).getAddress();
1040  });
1041  PrivateScope.addPrivate(RHSVD, [this, PrivateVD]() -> Address {
1042  return GetAddrOfLocalVar(PrivateVD);
1043  });
1044  } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1045  isa<ArraySubscriptExpr>(IRef)) {
1046  // Store the address of the original variable associated with the LHS
1047  // implicit variable.
1048  PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() -> Address {
1049  return RedCG.getSharedLValue(Count).getAddress();
1050  });
1051  PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() -> Address {
1052  return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD),
1053  ConvertTypeForMem(RHSVD->getType()),
1054  "rhs.begin");
1055  });
1056  } else {
1057  QualType Type = PrivateVD->getType();
1058  bool IsArray = getContext().getAsArrayType(Type) != nullptr;
1059  Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress();
1060  // Store the address of the original variable associated with the LHS
1061  // implicit variable.
1062  if (IsArray) {
1063  OriginalAddr = Builder.CreateElementBitCast(
1064  OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin");
1065  }
1066  PrivateScope.addPrivate(
1067  LHSVD, [OriginalAddr]() -> Address { return OriginalAddr; });
1068  PrivateScope.addPrivate(
1069  RHSVD, [this, PrivateVD, RHSVD, IsArray]() -> Address {
1070  return IsArray
1071  ? Builder.CreateElementBitCast(
1072  GetAddrOfLocalVar(PrivateVD),
1073  ConvertTypeForMem(RHSVD->getType()), "rhs.begin")
1074  : GetAddrOfLocalVar(PrivateVD);
1075  });
1076  }
1077  ++ILHS;
1078  ++IRHS;
1079  ++IPriv;
1080  ++Count;
1081  }
1082 }
1083 
1085  const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1086  if (!HaveInsertPoint())
1087  return;
1092  bool HasAtLeastOneReduction = false;
1093  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1094  HasAtLeastOneReduction = true;
1095  Privates.append(C->privates().begin(), C->privates().end());
1096  LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1097  RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1098  ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1099  }
1100  if (HasAtLeastOneReduction) {
1101  bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1103  D.getDirectiveKind() == OMPD_simd;
1104  bool SimpleReduction = D.getDirectiveKind() == OMPD_simd ||
1105  D.getDirectiveKind() == OMPD_distribute_simd;
1106  // Emit nowait reduction if nowait clause is present or directive is a
1107  // parallel directive (it always has implicit barrier).
1108  CGM.getOpenMPRuntime().emitReduction(
1109  *this, D.getLocEnd(), Privates, LHSExprs, RHSExprs, ReductionOps,
1110  {WithNowait, SimpleReduction, ReductionKind});
1111  }
1112 }
1113 
1115  CodeGenFunction &CGF, const OMPExecutableDirective &D,
1116  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> &CondGen) {
1117  if (!CGF.HaveInsertPoint())
1118  return;
1119  llvm::BasicBlock *DoneBB = nullptr;
1120  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1121  if (auto *PostUpdate = C->getPostUpdateExpr()) {
1122  if (!DoneBB) {
1123  if (auto *Cond = CondGen(CGF)) {
1124  // If the first post-update expression is found, emit conditional
1125  // block if it was requested.
1126  auto *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1127  DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1128  CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1129  CGF.EmitBlock(ThenBB);
1130  }
1131  }
1132  CGF.EmitIgnoredExpr(PostUpdate);
1133  }
1134  }
1135  if (DoneBB)
1136  CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1137 }
1138 
1139 namespace {
1140 /// Codegen lambda for appending distribute lower and upper bounds to outlined
1141 /// parallel function. This is necessary for combined constructs such as
1142 /// 'distribute parallel for'
1143 typedef llvm::function_ref<void(CodeGenFunction &,
1144  const OMPExecutableDirective &,
1146  CodeGenBoundParametersTy;
1147 } // anonymous namespace
1148 
1150  CodeGenFunction &CGF, const OMPExecutableDirective &S,
1151  OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1152  const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1153  const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1154  auto OutlinedFn = CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
1155  S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
1156  if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
1157  CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1158  auto NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1159  /*IgnoreResultAssign*/ true);
1160  CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
1161  CGF, NumThreads, NumThreadsClause->getLocStart());
1162  }
1163  if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
1164  CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1165  CGF.CGM.getOpenMPRuntime().emitProcBindClause(
1166  CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getLocStart());
1167  }
1168  const Expr *IfCond = nullptr;
1169  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1170  if (C->getNameModifier() == OMPD_unknown ||
1171  C->getNameModifier() == OMPD_parallel) {
1172  IfCond = C->getCondition();
1173  break;
1174  }
1175  }
1176 
1177  OMPParallelScope Scope(CGF, S);
1179  // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
1180  // lower and upper bounds with the pragma 'for' chunking mechanism.
1181  // The following lambda takes care of appending the lower and upper bound
1182  // parameters when necessary
1183  CodeGenBoundParameters(CGF, S, CapturedVars);
1184  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
1185  CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getLocStart(), OutlinedFn,
1186  CapturedVars, IfCond);
1187 }
1188 
1190  const OMPExecutableDirective &,
1192 
1194  // Emit parallel region as a standalone region.
1195  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
1196  OMPPrivateScope PrivateScope(CGF);
1197  bool Copyins = CGF.EmitOMPCopyinClause(S);
1198  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1199  if (Copyins) {
1200  // Emit implicit barrier to synchronize threads and avoid data races on
1201  // propagation master's thread values of threadprivate variables to local
1202  // instances of that variables of all other implicit threads.
1203  CGF.CGM.getOpenMPRuntime().emitBarrierCall(
1204  CGF, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
1205  /*ForceSimpleCall=*/true);
1206  }
1207  CGF.EmitOMPPrivateClause(S, PrivateScope);
1208  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1209  (void)PrivateScope.Privatize();
1210  CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
1211  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
1212  };
1213  emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
1216  *this, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
1217 }
1218 
1220  JumpDest LoopExit) {
1221  RunCleanupsScope BodyScope(*this);
1222  // Update counters values on current iteration.
1223  for (auto I : D.updates()) {
1224  EmitIgnoredExpr(I);
1225  }
1226  // Update the linear variables.
1227  // In distribute directives only loop counters may be marked as linear, no
1228  // need to generate the code for them.
1230  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1231  for (auto *U : C->updates())
1232  EmitIgnoredExpr(U);
1233  }
1234  }
1235 
1236  // On a continue in the body, jump to the end.
1237  auto Continue = getJumpDestInCurrentScope("omp.body.continue");
1238  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1239  // Emit loop body.
1240  EmitStmt(D.getBody());
1241  // The end (updates/cleanups).
1242  EmitBlock(Continue.getBlock());
1243  BreakContinueStack.pop_back();
1244 }
1245 
1247  const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
1248  const Expr *IncExpr,
1249  const llvm::function_ref<void(CodeGenFunction &)> &BodyGen,
1250  const llvm::function_ref<void(CodeGenFunction &)> &PostIncGen) {
1251  auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
1252 
1253  // Start the loop with a block that tests the condition.
1254  auto CondBlock = createBasicBlock("omp.inner.for.cond");
1255  EmitBlock(CondBlock);
1256  const SourceRange &R = S.getSourceRange();
1257  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1258  SourceLocToDebugLoc(R.getEnd()));
1259 
1260  // If there are any cleanups between here and the loop-exit scope,
1261  // create a block to stage a loop exit along.
1262  auto ExitBlock = LoopExit.getBlock();
1263  if (RequiresCleanup)
1264  ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
1265 
1266  auto LoopBody = createBasicBlock("omp.inner.for.body");
1267 
1268  // Emit condition.
1269  EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
1270  if (ExitBlock != LoopExit.getBlock()) {
1271  EmitBlock(ExitBlock);
1272  EmitBranchThroughCleanup(LoopExit);
1273  }
1274 
1275  EmitBlock(LoopBody);
1276  incrementProfileCounter(&S);
1277 
1278  // Create a block for the increment.
1279  auto Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
1280  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1281 
1282  BodyGen(*this);
1283 
1284  // Emit "IV = IV + 1" and a back-edge to the condition block.
1285  EmitBlock(Continue.getBlock());
1286  EmitIgnoredExpr(IncExpr);
1287  PostIncGen(*this);
1288  BreakContinueStack.pop_back();
1289  EmitBranch(CondBlock);
1290  LoopStack.pop();
1291  // Emit the fall-through block.
1292  EmitBlock(LoopExit.getBlock());
1293 }
1294 
1296  if (!HaveInsertPoint())
1297  return false;
1298  // Emit inits for the linear variables.
1299  bool HasLinears = false;
1300  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1301  for (auto *Init : C->inits()) {
1302  HasLinears = true;
1303  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
1304  if (auto *Ref = dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
1305  AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
1306  auto *OrigVD = cast<VarDecl>(Ref->getDecl());
1307  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
1308  CapturedStmtInfo->lookup(OrigVD) != nullptr,
1309  VD->getInit()->getType(), VK_LValue,
1310  VD->getInit()->getExprLoc());
1311  EmitExprAsInit(&DRE, VD, MakeAddrLValue(Emission.getAllocatedAddress(),
1312  VD->getType()),
1313  /*capturedByInit=*/false);
1314  EmitAutoVarCleanups(Emission);
1315  } else
1316  EmitVarDecl(*VD);
1317  }
1318  // Emit the linear steps for the linear clauses.
1319  // If a step is not constant, it is pre-calculated before the loop.
1320  if (auto CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
1321  if (auto SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
1322  EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
1323  // Emit calculation of the linear step.
1324  EmitIgnoredExpr(CS);
1325  }
1326  }
1327  return HasLinears;
1328 }
1329 
1331  const OMPLoopDirective &D,
1332  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> &CondGen) {
1333  if (!HaveInsertPoint())
1334  return;
1335  llvm::BasicBlock *DoneBB = nullptr;
1336  // Emit the final values of the linear variables.
1337  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1338  auto IC = C->varlist_begin();
1339  for (auto *F : C->finals()) {
1340  if (!DoneBB) {
1341  if (auto *Cond = CondGen(*this)) {
1342  // If the first post-update expression is found, emit conditional
1343  // block if it was requested.
1344  auto *ThenBB = createBasicBlock(".omp.linear.pu");
1345  DoneBB = createBasicBlock(".omp.linear.pu.done");
1346  Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1347  EmitBlock(ThenBB);
1348  }
1349  }
1350  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
1351  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
1352  CapturedStmtInfo->lookup(OrigVD) != nullptr,
1353  (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
1354  Address OrigAddr = EmitLValue(&DRE).getAddress();
1355  CodeGenFunction::OMPPrivateScope VarScope(*this);
1356  VarScope.addPrivate(OrigVD, [OrigAddr]() -> Address { return OrigAddr; });
1357  (void)VarScope.Privatize();
1358  EmitIgnoredExpr(F);
1359  ++IC;
1360  }
1361  if (auto *PostUpdate = C->getPostUpdateExpr())
1362  EmitIgnoredExpr(PostUpdate);
1363  }
1364  if (DoneBB)
1365  EmitBlock(DoneBB, /*IsFinished=*/true);
1366 }
1367 
1369  const OMPExecutableDirective &D) {
1370  if (!CGF.HaveInsertPoint())
1371  return;
1372  for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
1373  unsigned ClauseAlignment = 0;
1374  if (auto AlignmentExpr = Clause->getAlignment()) {
1375  auto AlignmentCI =
1376  cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
1377  ClauseAlignment = static_cast<unsigned>(AlignmentCI->getZExtValue());
1378  }
1379  for (auto E : Clause->varlists()) {
1380  unsigned Alignment = ClauseAlignment;
1381  if (Alignment == 0) {
1382  // OpenMP [2.8.1, Description]
1383  // If no optional parameter is specified, implementation-defined default
1384  // alignments for SIMD instructions on the target platforms are assumed.
1385  Alignment =
1386  CGF.getContext()
1388  E->getType()->getPointeeType()))
1389  .getQuantity();
1390  }
1391  assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) &&
1392  "alignment is not power of 2");
1393  if (Alignment != 0) {
1394  llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
1395  CGF.EmitAlignmentAssumption(PtrValue, Alignment);
1396  }
1397  }
1398  }
1399 }
1400 
1402  const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
1403  if (!HaveInsertPoint())
1404  return;
1405  auto I = S.private_counters().begin();
1406  for (auto *E : S.counters()) {
1407  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1408  auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
1409  (void)LoopScope.addPrivate(VD, [&]() -> Address {
1410  // Emit var without initialization.
1411  if (!LocalDeclMap.count(PrivateVD)) {
1412  auto VarEmission = EmitAutoVarAlloca(*PrivateVD);
1413  EmitAutoVarCleanups(VarEmission);
1414  }
1415  DeclRefExpr DRE(const_cast<VarDecl *>(PrivateVD),
1416  /*RefersToEnclosingVariableOrCapture=*/false,
1417  (*I)->getType(), VK_LValue, (*I)->getExprLoc());
1418  return EmitLValue(&DRE).getAddress();
1419  });
1420  if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
1421  VD->hasGlobalStorage()) {
1422  (void)LoopScope.addPrivate(PrivateVD, [&]() -> Address {
1423  DeclRefExpr DRE(const_cast<VarDecl *>(VD),
1424  LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
1425  E->getType(), VK_LValue, E->getExprLoc());
1426  return EmitLValue(&DRE).getAddress();
1427  });
1428  }
1429  ++I;
1430  }
1431 }
1432 
1433 static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
1434  const Expr *Cond, llvm::BasicBlock *TrueBlock,
1435  llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
1436  if (!CGF.HaveInsertPoint())
1437  return;
1438  {
1439  CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
1440  CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
1441  (void)PreCondScope.Privatize();
1442  // Get initial values of real counters.
1443  for (auto I : S.inits()) {
1444  CGF.EmitIgnoredExpr(I);
1445  }
1446  }
1447  // Check that loop is executed at least one time.
1448  CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
1449 }
1450 
1452  const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
1453  if (!HaveInsertPoint())
1454  return;
1457  auto *LoopDirective = cast<OMPLoopDirective>(&D);
1458  for (auto *C : LoopDirective->counters()) {
1459  SIMDLCVs.insert(
1460  cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
1461  }
1462  }
1463  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1464  auto CurPrivate = C->privates().begin();
1465  for (auto *E : C->varlists()) {
1466  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1467  auto *PrivateVD =
1468  cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
1469  if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
1470  bool IsRegistered = PrivateScope.addPrivate(VD, [&]() -> Address {
1471  // Emit private VarDecl with copy init.
1472  EmitVarDecl(*PrivateVD);
1473  return GetAddrOfLocalVar(PrivateVD);
1474  });
1475  assert(IsRegistered && "linear var already registered as private");
1476  // Silence the warning about unused variable.
1477  (void)IsRegistered;
1478  } else
1479  EmitVarDecl(*PrivateVD);
1480  ++CurPrivate;
1481  }
1482  }
1483 }
1484 
1486  const OMPExecutableDirective &D,
1487  bool IsMonotonic) {
1488  if (!CGF.HaveInsertPoint())
1489  return;
1490  if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
1491  RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
1492  /*ignoreResult=*/true);
1493  llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1494  CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1495  // In presence of finite 'safelen', it may be unsafe to mark all
1496  // the memory instructions parallel, because loop-carried
1497  // dependences of 'safelen' iterations are possible.
1498  if (!IsMonotonic)
1500  } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
1501  RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
1502  /*ignoreResult=*/true);
1503  llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1504  CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1505  // In presence of finite 'safelen', it may be unsafe to mark all
1506  // the memory instructions parallel, because loop-carried
1507  // dependences of 'safelen' iterations are possible.
1508  CGF.LoopStack.setParallel(false);
1509  }
1510 }
1511 
1513  bool IsMonotonic) {
1514  // Walk clauses and process safelen/lastprivate.
1515  LoopStack.setParallel(!IsMonotonic);
1516  LoopStack.setVectorizeEnable(true);
1517  emitSimdlenSafelenClause(*this, D, IsMonotonic);
1518 }
1519 
1521  const OMPLoopDirective &D,
1522  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> &CondGen) {
1523  if (!HaveInsertPoint())
1524  return;
1525  llvm::BasicBlock *DoneBB = nullptr;
1526  auto IC = D.counters().begin();
1527  auto IPC = D.private_counters().begin();
1528  for (auto F : D.finals()) {
1529  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
1530  auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
1531  auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
1532  if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
1533  OrigVD->hasGlobalStorage() || CED) {
1534  if (!DoneBB) {
1535  if (auto *Cond = CondGen(*this)) {
1536  // If the first post-update expression is found, emit conditional
1537  // block if it was requested.
1538  auto *ThenBB = createBasicBlock(".omp.final.then");
1539  DoneBB = createBasicBlock(".omp.final.done");
1540  Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1541  EmitBlock(ThenBB);
1542  }
1543  }
1544  Address OrigAddr = Address::invalid();
1545  if (CED)
1546  OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();
1547  else {
1548  DeclRefExpr DRE(const_cast<VarDecl *>(PrivateVD),
1549  /*RefersToEnclosingVariableOrCapture=*/false,
1550  (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
1551  OrigAddr = EmitLValue(&DRE).getAddress();
1552  }
1553  OMPPrivateScope VarScope(*this);
1554  VarScope.addPrivate(OrigVD,
1555  [OrigAddr]() -> Address { return OrigAddr; });
1556  (void)VarScope.Privatize();
1557  EmitIgnoredExpr(F);
1558  }
1559  ++IC;
1560  ++IPC;
1561  }
1562  if (DoneBB)
1563  EmitBlock(DoneBB, /*IsFinished=*/true);
1564 }
1565 
1567  const OMPLoopDirective &S,
1569  CGF.EmitOMPLoopBody(S, LoopExit);
1570  CGF.EmitStopPoint(&S);
1571 }
1572 
1574  PrePostActionTy &Action) {
1575  Action.Enter(CGF);
1577  "Expected simd directive");
1578  OMPLoopScope PreInitScope(CGF, S);
1579  // if (PreCond) {
1580  // for (IV in 0..LastIteration) BODY;
1581  // <Final counter/linear vars updates>;
1582  // }
1583  //
1584 
1585  // Emit: if (PreCond) - begin.
1586  // If the condition constant folds and can be elided, avoid emitting the
1587  // whole loop.
1588  bool CondConstant;
1589  llvm::BasicBlock *ContBlock = nullptr;
1590  if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
1591  if (!CondConstant)
1592  return;
1593  } else {
1594  auto *ThenBlock = CGF.createBasicBlock("simd.if.then");
1595  ContBlock = CGF.createBasicBlock("simd.if.end");
1596  emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
1597  CGF.getProfileCount(&S));
1598  CGF.EmitBlock(ThenBlock);
1599  CGF.incrementProfileCounter(&S);
1600  }
1601 
1602  // Emit the loop iteration variable.
1603  const Expr *IVExpr = S.getIterationVariable();
1604  const VarDecl *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
1605  CGF.EmitVarDecl(*IVDecl);
1606  CGF.EmitIgnoredExpr(S.getInit());
1607 
1608  // Emit the iterations count variable.
1609  // If it is not a variable, Sema decided to calculate iterations count on
1610  // each iteration (e.g., it is foldable into a constant).
1611  if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
1612  CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
1613  // Emit calculation of the iterations count.
1615  }
1616 
1617  CGF.EmitOMPSimdInit(S);
1618 
1619  emitAlignedClause(CGF, S);
1620  (void)CGF.EmitOMPLinearClauseInit(S);
1621  {
1622  CodeGenFunction::OMPPrivateScope LoopScope(CGF);
1623  CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
1624  CGF.EmitOMPLinearClause(S, LoopScope);
1625  CGF.EmitOMPPrivateClause(S, LoopScope);
1626  CGF.EmitOMPReductionClauseInit(S, LoopScope);
1627  bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
1628  (void)LoopScope.Privatize();
1629  CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
1630  S.getInc(),
1631  [&S](CodeGenFunction &CGF) {
1633  CGF.EmitStopPoint(&S);
1634  },
1635  [](CodeGenFunction &) {});
1636  CGF.EmitOMPSimdFinal(
1637  S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
1638  // Emit final copy of the lastprivate variables at the end of loops.
1639  if (HasLastprivateClause)
1640  CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
1641  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
1643  CGF, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
1644  }
1646  S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
1647  // Emit: if (PreCond) - end.
1648  if (ContBlock) {
1649  CGF.EmitBranch(ContBlock);
1650  CGF.EmitBlock(ContBlock, true);
1651  }
1652 }
1653 
1655  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1656  emitOMPSimdRegion(CGF, S, Action);
1657  };
1658  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
1659  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
1660 }
1661 
1662 void CodeGenFunction::EmitOMPOuterLoop(
1663  bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
1665  const CodeGenFunction::OMPLoopArguments &LoopArgs,
1666  const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
1667  const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
1668  auto &RT = CGM.getOpenMPRuntime();
1669 
1670  const Expr *IVExpr = S.getIterationVariable();
1671  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1672  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1673 
1674  auto LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
1675 
1676  // Start the loop with a block that tests the condition.
1677  auto CondBlock = createBasicBlock("omp.dispatch.cond");
1678  EmitBlock(CondBlock);
1679  const SourceRange &R = S.getSourceRange();
1680  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1681  SourceLocToDebugLoc(R.getEnd()));
1682 
1683  llvm::Value *BoolCondVal = nullptr;
1684  if (!DynamicOrOrdered) {
1685  // UB = min(UB, GlobalUB) or
1686  // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
1687  // 'distribute parallel for')
1688  EmitIgnoredExpr(LoopArgs.EUB);
1689  // IV = LB
1690  EmitIgnoredExpr(LoopArgs.Init);
1691  // IV < UB
1692  BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
1693  } else {
1694  BoolCondVal =
1695  RT.emitForNext(*this, S.getLocStart(), IVSize, IVSigned, LoopArgs.IL,
1696  LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
1697  }
1698 
1699  // If there are any cleanups between here and the loop-exit scope,
1700  // create a block to stage a loop exit along.
1701  auto ExitBlock = LoopExit.getBlock();
1702  if (LoopScope.requiresCleanups())
1703  ExitBlock = createBasicBlock("omp.dispatch.cleanup");
1704 
1705  auto LoopBody = createBasicBlock("omp.dispatch.body");
1706  Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
1707  if (ExitBlock != LoopExit.getBlock()) {
1708  EmitBlock(ExitBlock);
1709  EmitBranchThroughCleanup(LoopExit);
1710  }
1711  EmitBlock(LoopBody);
1712 
1713  // Emit "IV = LB" (in case of static schedule, we have already calculated new
1714  // LB for loop condition and emitted it above).
1715  if (DynamicOrOrdered)
1716  EmitIgnoredExpr(LoopArgs.Init);
1717 
1718  // Create a block for the increment.
1719  auto Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
1720  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1721 
1722  // Generate !llvm.loop.parallel metadata for loads and stores for loops
1723  // with dynamic/guided scheduling and without ordered clause.
1725  LoopStack.setParallel(!IsMonotonic);
1726  else
1727  EmitOMPSimdInit(S, IsMonotonic);
1728 
1729  SourceLocation Loc = S.getLocStart();
1730 
1731  // when 'distribute' is not combined with a 'for':
1732  // while (idx <= UB) { BODY; ++idx; }
1733  // when 'distribute' is combined with a 'for'
1734  // (e.g. 'distribute parallel for')
1735  // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
1736  EmitOMPInnerLoop(
1737  S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
1738  [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
1739  CodeGenLoop(CGF, S, LoopExit);
1740  },
1741  [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
1742  CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
1743  });
1744 
1745  EmitBlock(Continue.getBlock());
1746  BreakContinueStack.pop_back();
1747  if (!DynamicOrOrdered) {
1748  // Emit "LB = LB + Stride", "UB = UB + Stride".
1749  EmitIgnoredExpr(LoopArgs.NextLB);
1750  EmitIgnoredExpr(LoopArgs.NextUB);
1751  }
1752 
1753  EmitBranch(CondBlock);
1754  LoopStack.pop();
1755  // Emit the fall-through block.
1756  EmitBlock(LoopExit.getBlock());
1757 
1758  // Tell the runtime we are done.
1759  auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) {
1760  if (!DynamicOrOrdered)
1761  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(),
1762  S.getDirectiveKind());
1763  };
1764  OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
1765 }
1766 
1767 void CodeGenFunction::EmitOMPForOuterLoop(
1768  const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
1769  const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
1770  const OMPLoopArguments &LoopArgs,
1771  const CodeGenDispatchBoundsTy &CGDispatchBounds) {
1772  auto &RT = CGM.getOpenMPRuntime();
1773 
1774  // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
1775  const bool DynamicOrOrdered =
1776  Ordered || RT.isDynamic(ScheduleKind.Schedule);
1777 
1778  assert((Ordered ||
1779  !RT.isStaticNonchunked(ScheduleKind.Schedule,
1780  LoopArgs.Chunk != nullptr)) &&
1781  "static non-chunked schedule does not need outer loop");
1782 
1783  // Emit outer loop.
1784  //
1785  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1786  // When schedule(dynamic,chunk_size) is specified, the iterations are
1787  // distributed to threads in the team in chunks as the threads request them.
1788  // Each thread executes a chunk of iterations, then requests another chunk,
1789  // until no chunks remain to be distributed. Each chunk contains chunk_size
1790  // iterations, except for the last chunk to be distributed, which may have
1791  // fewer iterations. When no chunk_size is specified, it defaults to 1.
1792  //
1793  // When schedule(guided,chunk_size) is specified, the iterations are assigned
1794  // to threads in the team in chunks as the executing threads request them.
1795  // Each thread executes a chunk of iterations, then requests another chunk,
1796  // until no chunks remain to be assigned. For a chunk_size of 1, the size of
1797  // each chunk is proportional to the number of unassigned iterations divided
1798  // by the number of threads in the team, decreasing to 1. For a chunk_size
1799  // with value k (greater than 1), the size of each chunk is determined in the
1800  // same way, with the restriction that the chunks do not contain fewer than k
1801  // iterations (except for the last chunk to be assigned, which may have fewer
1802  // than k iterations).
1803  //
1804  // When schedule(auto) is specified, the decision regarding scheduling is
1805  // delegated to the compiler and/or runtime system. The programmer gives the
1806  // implementation the freedom to choose any possible mapping of iterations to
1807  // threads in the team.
1808  //
1809  // When schedule(runtime) is specified, the decision regarding scheduling is
1810  // deferred until run time, and the schedule and chunk size are taken from the
1811  // run-sched-var ICV. If the ICV is set to auto, the schedule is
1812  // implementation defined
1813  //
1814  // while(__kmpc_dispatch_next(&LB, &UB)) {
1815  // idx = LB;
1816  // while (idx <= UB) { BODY; ++idx;
1817  // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
1818  // } // inner loop
1819  // }
1820  //
1821  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1822  // When schedule(static, chunk_size) is specified, iterations are divided into
1823  // chunks of size chunk_size, and the chunks are assigned to the threads in
1824  // the team in a round-robin fashion in the order of the thread number.
1825  //
1826  // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
1827  // while (idx <= UB) { BODY; ++idx; } // inner loop
1828  // LB = LB + ST;
1829  // UB = UB + ST;
1830  // }
1831  //
1832 
1833  const Expr *IVExpr = S.getIterationVariable();
1834  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1835  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1836 
1837  if (DynamicOrOrdered) {
1838  auto DispatchBounds = CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
1839  llvm::Value *LBVal = DispatchBounds.first;
1840  llvm::Value *UBVal = DispatchBounds.second;
1841  CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
1842  LoopArgs.Chunk};
1843  RT.emitForDispatchInit(*this, S.getLocStart(), ScheduleKind, IVSize,
1844  IVSigned, Ordered, DipatchRTInputValues);
1845  } else {
1846  CGOpenMPRuntime::StaticRTInput StaticInit(
1847  IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
1848  LoopArgs.ST, LoopArgs.Chunk);
1849  RT.emitForStaticInit(*this, S.getLocStart(), S.getDirectiveKind(),
1850  ScheduleKind, StaticInit);
1851  }
1852 
1853  auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
1854  const unsigned IVSize,
1855  const bool IVSigned) {
1856  if (Ordered) {
1857  CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
1858  IVSigned);
1859  }
1860  };
1861 
1862  OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
1863  LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
1864  OuterLoopArgs.IncExpr = S.getInc();
1865  OuterLoopArgs.Init = S.getInit();
1866  OuterLoopArgs.Cond = S.getCond();
1867  OuterLoopArgs.NextLB = S.getNextLowerBound();
1868  OuterLoopArgs.NextUB = S.getNextUpperBound();
1869  EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
1870  emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
1871 }
1872 
1874  const unsigned IVSize, const bool IVSigned) {}
1875 
1876 void CodeGenFunction::EmitOMPDistributeOuterLoop(
1877  OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
1878  OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
1879  const CodeGenLoopTy &CodeGenLoopContent) {
1880 
1881  auto &RT = CGM.getOpenMPRuntime();
1882 
1883  // Emit outer loop.
1884  // Same behavior as a OMPForOuterLoop, except that schedule cannot be
1885  // dynamic
1886  //
1887 
1888  const Expr *IVExpr = S.getIterationVariable();
1889  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1890  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1891 
1892  CGOpenMPRuntime::StaticRTInput StaticInit(
1893  IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
1894  LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
1895  RT.emitDistributeStaticInit(*this, S.getLocStart(), ScheduleKind, StaticInit);
1896 
1897  // for combined 'distribute' and 'for' the increment expression of distribute
1898  // is store in DistInc. For 'distribute' alone, it is in Inc.
1899  Expr *IncExpr;
1901  IncExpr = S.getDistInc();
1902  else
1903  IncExpr = S.getInc();
1904 
1905  // this routine is shared by 'omp distribute parallel for' and
1906  // 'omp distribute': select the right EUB expression depending on the
1907  // directive
1908  OMPLoopArguments OuterLoopArgs;
1909  OuterLoopArgs.LB = LoopArgs.LB;
1910  OuterLoopArgs.UB = LoopArgs.UB;
1911  OuterLoopArgs.ST = LoopArgs.ST;
1912  OuterLoopArgs.IL = LoopArgs.IL;
1913  OuterLoopArgs.Chunk = LoopArgs.Chunk;
1914  OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
1916  : S.getEnsureUpperBound();
1917  OuterLoopArgs.IncExpr = IncExpr;
1918  OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
1919  ? S.getCombinedInit()
1920  : S.getInit();
1921  OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
1922  ? S.getCombinedCond()
1923  : S.getCond();
1924  OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
1926  : S.getNextLowerBound();
1927  OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
1929  : S.getNextUpperBound();
1930 
1931  EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
1932  LoopScope, OuterLoopArgs, CodeGenLoopContent,
1934 }
1935 
1936 /// Emit a helper variable and return corresponding lvalue.
1938  const DeclRefExpr *Helper) {
1939  auto VDecl = cast<VarDecl>(Helper->getDecl());
1940  CGF.EmitVarDecl(*VDecl);
1941  return CGF.EmitLValue(Helper);
1942 }
1943 
1944 static std::pair<LValue, LValue>
1946  const OMPExecutableDirective &S) {
1947  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
1948  LValue LB =
1949  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
1950  LValue UB =
1951  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
1952 
1953  // When composing 'distribute' with 'for' (e.g. as in 'distribute
1954  // parallel for') we need to use the 'distribute'
1955  // chunk lower and upper bounds rather than the whole loop iteration
1956  // space. These are parameters to the outlined function for 'parallel'
1957  // and we copy the bounds of the previous schedule into the
1958  // the current ones.
1959  LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
1960  LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
1961  llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(PrevLB, SourceLocation());
1962  PrevLBVal = CGF.EmitScalarConversion(
1963  PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
1965  llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(PrevUB, SourceLocation());
1966  PrevUBVal = CGF.EmitScalarConversion(
1967  PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
1969 
1970  CGF.EmitStoreOfScalar(PrevLBVal, LB);
1971  CGF.EmitStoreOfScalar(PrevUBVal, UB);
1972 
1973  return {LB, UB};
1974 }
1975 
1976 /// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
1977 /// we need to use the LB and UB expressions generated by the worksharing
1978 /// code generation support, whereas in non combined situations we would
1979 /// just emit 0 and the LastIteration expression
1980 /// This function is necessary due to the difference of the LB and UB
1981 /// types for the RT emission routines for 'for_static_init' and
1982 /// 'for_dispatch_init'
1983 static std::pair<llvm::Value *, llvm::Value *>
1985  const OMPExecutableDirective &S,
1986  Address LB, Address UB) {
1987  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
1988  const Expr *IVExpr = LS.getIterationVariable();
1989  // when implementing a dynamic schedule for a 'for' combined with a
1990  // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
1991  // is not normalized as each team only executes its own assigned
1992  // distribute chunk
1993  QualType IteratorTy = IVExpr->getType();
1994  llvm::Value *LBVal = CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy,
1995  SourceLocation());
1996  llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy,
1997  SourceLocation());
1998  return {LBVal, UBVal};
1999 }
2000 
2002  CodeGenFunction &CGF, const OMPExecutableDirective &S,
2003  llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) {
2004  const auto &Dir = cast<OMPLoopDirective>(S);
2005  LValue LB =
2006  CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
2007  auto LBCast = CGF.Builder.CreateIntCast(
2008  CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2009  CapturedVars.push_back(LBCast);
2010  LValue UB =
2011  CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
2012 
2013  auto UBCast = CGF.Builder.CreateIntCast(
2014  CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2015  CapturedVars.push_back(UBCast);
2016 }
2017 
2018 static void
2020  const OMPLoopDirective &S,
2022  auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
2023  PrePostActionTy &) {
2024  bool HasCancel = false;
2026  if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
2027  HasCancel = D->hasCancel();
2028  else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
2029  HasCancel = D->hasCancel();
2030  else if (const auto *D =
2031  dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
2032  HasCancel = D->hasCancel();
2033  }
2035  HasCancel);
2039  };
2040 
2042  CGF, S,
2043  isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for,
2044  CGInlinedWorksharingLoop,
2046 }
2047 
2050  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2052  S.getDistInc());
2053  };
2054  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2055  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2056 }
2057 
2060  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2062  S.getDistInc());
2063  };
2064  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2065  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2066 }
2067 
2069  const OMPDistributeSimdDirective &S) {
2070  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2072  };
2073  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2074  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2075 }
2076 
2078  CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
2079  // Emit SPMD target parallel for region as a standalone region.
2080  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2081  emitOMPSimdRegion(CGF, S, Action);
2082  };
2083  llvm::Function *Fn;
2084  llvm::Constant *Addr;
2085  // Emit target region as a standalone region.
2086  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
2087  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
2088  assert(Fn && Addr && "Target device function emission failed.");
2089 }
2090 
2092  const OMPTargetSimdDirective &S) {
2093  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2094  emitOMPSimdRegion(CGF, S, Action);
2095  };
2096  emitCommonOMPTargetDirective(*this, S, CodeGen);
2097 }
2098 
2101  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2102  CGM.getOpenMPRuntime().emitInlinedDirective(
2103  *this, OMPD_target_teams_distribute_parallel_for,
2104  [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2105  CGF.EmitStmt(
2106  cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2107  });
2108 }
2109 
2112  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2113  CGM.getOpenMPRuntime().emitInlinedDirective(
2114  *this, OMPD_target_teams_distribute_parallel_for_simd,
2115  [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2116  CGF.EmitStmt(
2117  cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2118  });
2119 }
2120 
2121 namespace {
2122  struct ScheduleKindModifiersTy {
2126  ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
2129  : Kind(Kind), M1(M1), M2(M2) {}
2130  };
2131 } // namespace
2132 
2134  const OMPLoopDirective &S, Expr *EUB,
2135  const CodeGenLoopBoundsTy &CodeGenLoopBounds,
2136  const CodeGenDispatchBoundsTy &CGDispatchBounds) {
2137  // Emit the loop iteration variable.
2138  auto IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
2139  auto IVDecl = cast<VarDecl>(IVExpr->getDecl());
2140  EmitVarDecl(*IVDecl);
2141 
2142  // Emit the iterations count variable.
2143  // If it is not a variable, Sema decided to calculate iterations count on each
2144  // iteration (e.g., it is foldable into a constant).
2145  if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2146  EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2147  // Emit calculation of the iterations count.
2148  EmitIgnoredExpr(S.getCalcLastIteration());
2149  }
2150 
2151  auto &RT = CGM.getOpenMPRuntime();
2152 
2153  bool HasLastprivateClause;
2154  // Check pre-condition.
2155  {
2156  OMPLoopScope PreInitScope(*this, S);
2157  // Skip the entire loop if we don't meet the precondition.
2158  // If the condition constant folds and can be elided, avoid emitting the
2159  // whole loop.
2160  bool CondConstant;
2161  llvm::BasicBlock *ContBlock = nullptr;
2162  if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2163  if (!CondConstant)
2164  return false;
2165  } else {
2166  auto *ThenBlock = createBasicBlock("omp.precond.then");
2167  ContBlock = createBasicBlock("omp.precond.end");
2168  emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
2169  getProfileCount(&S));
2170  EmitBlock(ThenBlock);
2171  incrementProfileCounter(&S);
2172  }
2173 
2174  bool Ordered = false;
2175  if (auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
2176  if (OrderedClause->getNumForLoops())
2177  RT.emitDoacrossInit(*this, S);
2178  else
2179  Ordered = true;
2180  }
2181 
2182  llvm::DenseSet<const Expr *> EmittedFinals;
2183  emitAlignedClause(*this, S);
2184  bool HasLinears = EmitOMPLinearClauseInit(S);
2185  // Emit helper vars inits.
2186 
2187  std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
2188  LValue LB = Bounds.first;
2189  LValue UB = Bounds.second;
2190  LValue ST =
2191  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
2192  LValue IL =
2193  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
2194 
2195  // Emit 'then' code.
2196  {
2197  OMPPrivateScope LoopScope(*this);
2198  if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
2199  // Emit implicit barrier to synchronize threads and avoid data races on
2200  // initialization of firstprivate variables and post-update of
2201  // lastprivate variables.
2202  CGM.getOpenMPRuntime().emitBarrierCall(
2203  *this, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
2204  /*ForceSimpleCall=*/true);
2205  }
2206  EmitOMPPrivateClause(S, LoopScope);
2207  HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
2208  EmitOMPReductionClauseInit(S, LoopScope);
2209  EmitOMPPrivateLoopCounters(S, LoopScope);
2210  EmitOMPLinearClause(S, LoopScope);
2211  (void)LoopScope.Privatize();
2212 
2213  // Detect the loop schedule kind and chunk.
2214  llvm::Value *Chunk = nullptr;
2215  OpenMPScheduleTy ScheduleKind;
2216  if (auto *C = S.getSingleClause<OMPScheduleClause>()) {
2217  ScheduleKind.Schedule = C->getScheduleKind();
2218  ScheduleKind.M1 = C->getFirstScheduleModifier();
2219  ScheduleKind.M2 = C->getSecondScheduleModifier();
2220  if (const auto *Ch = C->getChunkSize()) {
2221  Chunk = EmitScalarExpr(Ch);
2222  Chunk = EmitScalarConversion(Chunk, Ch->getType(),
2224  S.getLocStart());
2225  }
2226  }
2227  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2228  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2229  // OpenMP 4.5, 2.7.1 Loop Construct, Description.
2230  // If the static schedule kind is specified or if the ordered clause is
2231  // specified, and if no monotonic modifier is specified, the effect will
2232  // be as if the monotonic modifier was specified.
2233  if (RT.isStaticNonchunked(ScheduleKind.Schedule,
2234  /* Chunked */ Chunk != nullptr) &&
2235  !Ordered) {
2237  EmitOMPSimdInit(S, /*IsMonotonic=*/true);
2238  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2239  // When no chunk_size is specified, the iteration space is divided into
2240  // chunks that are approximately equal in size, and at most one chunk is
2241  // distributed to each thread. Note that the size of the chunks is
2242  // unspecified in this case.
2243  CGOpenMPRuntime::StaticRTInput StaticInit(
2244  IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
2245  UB.getAddress(), ST.getAddress());
2246  RT.emitForStaticInit(*this, S.getLocStart(), S.getDirectiveKind(),
2247  ScheduleKind, StaticInit);
2248  auto LoopExit =
2249  getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
2250  // UB = min(UB, GlobalUB);
2251  EmitIgnoredExpr(S.getEnsureUpperBound());
2252  // IV = LB;
2253  EmitIgnoredExpr(S.getInit());
2254  // while (idx <= UB) { BODY; ++idx; }
2255  EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
2256  S.getInc(),
2257  [&S, LoopExit](CodeGenFunction &CGF) {
2258  CGF.EmitOMPLoopBody(S, LoopExit);
2259  CGF.EmitStopPoint(&S);
2260  },
2261  [](CodeGenFunction &) {});
2262  EmitBlock(LoopExit.getBlock());
2263  // Tell the runtime we are done.
2264  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2265  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(),
2266  S.getDirectiveKind());
2267  };
2268  OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
2269  } else {
2270  const bool IsMonotonic =
2271  Ordered || ScheduleKind.Schedule == OMPC_SCHEDULE_static ||
2272  ScheduleKind.Schedule == OMPC_SCHEDULE_unknown ||
2273  ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
2274  ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
2275  // Emit the outer loop, which requests its work chunk [LB..UB] from
2276  // runtime and runs the inner loop to process it.
2277  const OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),
2278  ST.getAddress(), IL.getAddress(),
2279  Chunk, EUB);
2280  EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
2281  LoopArguments, CGDispatchBounds);
2282  }
2284  EmitOMPSimdFinal(S,
2285  [&](CodeGenFunction &CGF) -> llvm::Value * {
2286  return CGF.Builder.CreateIsNotNull(
2287  CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2288  });
2289  }
2290  EmitOMPReductionClauseFinal(
2291  S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind())
2292  ? /*Parallel and Simd*/ OMPD_parallel_for_simd
2293  : /*Parallel only*/ OMPD_parallel);
2294  // Emit post-update of the reduction variables if IsLastIter != 0.
2296  *this, S, [&](CodeGenFunction &CGF) -> llvm::Value * {
2297  return CGF.Builder.CreateIsNotNull(
2298  CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2299  });
2300  // Emit final copy of the lastprivate variables if IsLastIter != 0.
2301  if (HasLastprivateClause)
2302  EmitOMPLastprivateClauseFinal(
2304  Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getLocStart())));
2305  }
2306  EmitOMPLinearClauseFinal(S, [&](CodeGenFunction &CGF) -> llvm::Value * {
2307  return CGF.Builder.CreateIsNotNull(
2308  CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2309  });
2310  // We're now done with the loop, so jump to the continuation block.
2311  if (ContBlock) {
2312  EmitBranch(ContBlock);
2313  EmitBlock(ContBlock, true);
2314  }
2315  }
2316  return HasLastprivateClause;
2317 }
2318 
2319 /// The following two functions generate expressions for the loop lower
2320 /// and upper bounds in case of static and dynamic (dispatch) schedule
2321 /// of the associated 'for' or 'distribute' loop.
2322 static std::pair<LValue, LValue>
2324  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2325  LValue LB =
2326  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2327  LValue UB =
2328  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2329  return {LB, UB};
2330 }
2331 
2332 /// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
2333 /// consider the lower and upper bound expressions generated by the
2334 /// worksharing loop support, but we use 0 and the iteration space size as
2335 /// constants
2336 static std::pair<llvm::Value *, llvm::Value *>
2338  Address LB, Address UB) {
2339  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2340  const Expr *IVExpr = LS.getIterationVariable();
2341  const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
2342  llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
2343  llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
2344  return {LBVal, UBVal};
2345 }
2346 
2348  bool HasLastprivates = false;
2349  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2350  PrePostActionTy &) {
2351  OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel());
2352  HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2355  };
2356  {
2357  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2358  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
2359  S.hasCancel());
2360  }
2361 
2362  // Emit an implicit barrier at the end.
2363  if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates) {
2364  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for);
2365  }
2366 }
2367 
2369  bool HasLastprivates = false;
2370  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2371  PrePostActionTy &) {
2372  HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2375  };
2376  {
2377  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2378  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2379  }
2380 
2381  // Emit an implicit barrier at the end.
2382  if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates) {
2383  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for);
2384  }
2385 }
2386 
2388  const Twine &Name,
2389  llvm::Value *Init = nullptr) {
2390  auto LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
2391  if (Init)
2392  CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
2393  return LVal;
2394 }
2395 
2396 void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
2397  auto *Stmt = cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt();
2398  auto *CS = dyn_cast<CompoundStmt>(Stmt);
2399  bool HasLastprivates = false;
2400  auto &&CodeGen = [&S, Stmt, CS, &HasLastprivates](CodeGenFunction &CGF,
2401  PrePostActionTy &) {
2402  auto &C = CGF.CGM.getContext();
2403  auto KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2404  // Emit helper vars inits.
2405  LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
2406  CGF.Builder.getInt32(0));
2407  auto *GlobalUBVal = CS != nullptr ? CGF.Builder.getInt32(CS->size() - 1)
2408  : CGF.Builder.getInt32(0);
2409  LValue UB =
2410  createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
2411  LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
2412  CGF.Builder.getInt32(1));
2413  LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
2414  CGF.Builder.getInt32(0));
2415  // Loop counter.
2416  LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
2417  OpaqueValueExpr IVRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue);
2418  CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
2419  OpaqueValueExpr UBRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue);
2420  CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
2421  // Generate condition for loop.
2422  BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue,
2424  // Increment for loop counter.
2425  UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary,
2426  S.getLocStart());
2427  auto BodyGen = [Stmt, CS, &S, &IV](CodeGenFunction &CGF) {
2428  // Iterate through all sections and emit a switch construct:
2429  // switch (IV) {
2430  // case 0:
2431  // <SectionStmt[0]>;
2432  // break;
2433  // ...
2434  // case <NumSection> - 1:
2435  // <SectionStmt[<NumSection> - 1]>;
2436  // break;
2437  // }
2438  // .omp.sections.exit:
2439  auto *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
2440  auto *SwitchStmt = CGF.Builder.CreateSwitch(
2441  CGF.EmitLoadOfLValue(IV, S.getLocStart()).getScalarVal(), ExitBB,
2442  CS == nullptr ? 1 : CS->size());
2443  if (CS) {
2444  unsigned CaseNumber = 0;
2445  for (auto *SubStmt : CS->children()) {
2446  auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2447  CGF.EmitBlock(CaseBB);
2448  SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
2449  CGF.EmitStmt(SubStmt);
2450  CGF.EmitBranch(ExitBB);
2451  ++CaseNumber;
2452  }
2453  } else {
2454  auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2455  CGF.EmitBlock(CaseBB);
2456  SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
2457  CGF.EmitStmt(Stmt);
2458  CGF.EmitBranch(ExitBB);
2459  }
2460  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2461  };
2462 
2463  CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2464  if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
2465  // Emit implicit barrier to synchronize threads and avoid data races on
2466  // initialization of firstprivate variables and post-update of lastprivate
2467  // variables.
2468  CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2469  CGF, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
2470  /*ForceSimpleCall=*/true);
2471  }
2472  CGF.EmitOMPPrivateClause(S, LoopScope);
2473  HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2474  CGF.EmitOMPReductionClauseInit(S, LoopScope);
2475  (void)LoopScope.Privatize();
2476 
2477  // Emit static non-chunked loop.
2478  OpenMPScheduleTy ScheduleKind;
2479  ScheduleKind.Schedule = OMPC_SCHEDULE_static;
2480  CGOpenMPRuntime::StaticRTInput StaticInit(
2481  /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(),
2482  LB.getAddress(), UB.getAddress(), ST.getAddress());
2483  CGF.CGM.getOpenMPRuntime().emitForStaticInit(
2484  CGF, S.getLocStart(), S.getDirectiveKind(), ScheduleKind, StaticInit);
2485  // UB = min(UB, GlobalUB);
2486  auto *UBVal = CGF.EmitLoadOfScalar(UB, S.getLocStart());
2487  auto *MinUBGlobalUB = CGF.Builder.CreateSelect(
2488  CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
2489  CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
2490  // IV = LB;
2491  CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getLocStart()), IV);
2492  // while (idx <= UB) { BODY; ++idx; }
2493  CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen,
2494  [](CodeGenFunction &) {});
2495  // Tell the runtime we are done.
2496  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2497  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(),
2498  S.getDirectiveKind());
2499  };
2500  CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
2501  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
2502  // Emit post-update of the reduction variables if IsLastIter != 0.
2504  CGF, S, [&](CodeGenFunction &CGF) -> llvm::Value * {
2505  return CGF.Builder.CreateIsNotNull(
2506  CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2507  });
2508 
2509  // Emit final copy of the lastprivate variables if IsLastIter != 0.
2510  if (HasLastprivates)
2512  S, /*NoFinals=*/false,
2513  CGF.Builder.CreateIsNotNull(
2514  CGF.EmitLoadOfScalar(IL, S.getLocStart())));
2515  };
2516 
2517  bool HasCancel = false;
2518  if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
2519  HasCancel = OSD->hasCancel();
2520  else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
2521  HasCancel = OPSD->hasCancel();
2522  OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
2523  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
2524  HasCancel);
2525  // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
2526  // clause. Otherwise the barrier will be generated by the codegen for the
2527  // directive.
2528  if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
2529  // Emit implicit barrier to synchronize threads and avoid data races on
2530  // initialization of firstprivate variables.
2531  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(),
2532  OMPD_unknown);
2533  }
2534 }
2535 
2537  {
2538  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2539  EmitSections(S);
2540  }
2541  // Emit an implicit barrier at the end.
2542  if (!S.getSingleClause<OMPNowaitClause>()) {
2543  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(),
2544  OMPD_sections);
2545  }
2546 }
2547 
2549  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2550  CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2551  };
2552  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2553  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen,
2554  S.hasCancel());
2555 }
2556 
2558  llvm::SmallVector<const Expr *, 8> CopyprivateVars;
2561  llvm::SmallVector<const Expr *, 8> AssignmentOps;
2562  // Check if there are any 'copyprivate' clauses associated with this
2563  // 'single' construct.
2564  // Build a list of copyprivate variables along with helper expressions
2565  // (<source>, <destination>, <destination>=<source> expressions)
2566  for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
2567  CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
2568  DestExprs.append(C->destination_exprs().begin(),
2569  C->destination_exprs().end());
2570  SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
2571  AssignmentOps.append(C->assignment_ops().begin(),
2572  C->assignment_ops().end());
2573  }
2574  // Emit code for 'single' region along with 'copyprivate' clauses
2575  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2576  Action.Enter(CGF);
2577  OMPPrivateScope SingleScope(CGF);
2578  (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
2579  CGF.EmitOMPPrivateClause(S, SingleScope);
2580  (void)SingleScope.Privatize();
2581  CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2582  };
2583  {
2584  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2585  CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getLocStart(),
2586  CopyprivateVars, DestExprs,
2587  SrcExprs, AssignmentOps);
2588  }
2589  // Emit an implicit barrier at the end (to avoid data race on firstprivate
2590  // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
2591  if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
2592  CGM.getOpenMPRuntime().emitBarrierCall(
2593  *this, S.getLocStart(),
2594  S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
2595  }
2596 }
2597 
2599  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2600  Action.Enter(CGF);
2601  CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2602  };
2603  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2604  CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getLocStart());
2605 }
2606 
2608  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2609  Action.Enter(CGF);
2610  CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2611  };
2612  Expr *Hint = nullptr;
2613  if (auto *HintClause = S.getSingleClause<OMPHintClause>())
2614  Hint = HintClause->getHint();
2615  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2616  CGM.getOpenMPRuntime().emitCriticalRegion(*this,
2618  CodeGen, S.getLocStart(), Hint);
2619 }
2620 
2622  const OMPParallelForDirective &S) {
2623  // Emit directive as a combined directive that consists of two implicit
2624  // directives: 'parallel' with 'for' directive.
2625  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2626  OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel());
2629  };
2630  emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
2632 }
2633 
2635  const OMPParallelForSimdDirective &S) {
2636  // Emit directive as a combined directive that consists of two implicit
2637  // directives: 'parallel' with 'for' directive.
2638  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2641  };
2642  emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen,
2644 }
2645 
2647  const OMPParallelSectionsDirective &S) {
2648  // Emit directive as a combined directive that consists of two implicit
2649  // directives: 'parallel' with 'sections' directive.
2650  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2651  CGF.EmitSections(S);
2652  };
2653  emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
2655 }
2656 
2658  const RegionCodeGenTy &BodyGen,
2659  const TaskGenTy &TaskGen,
2660  OMPTaskDataTy &Data) {
2661  // Emit outlined function for task construct.
2662  auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
2663  auto *I = CS->getCapturedDecl()->param_begin();
2664  auto *PartId = std::next(I);
2665  auto *TaskT = std::next(I, 4);
2666  // Check if the task is final
2667  if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
2668  // If the condition constant folds and can be elided, try to avoid emitting
2669  // the condition and the dead arm of the if/else.
2670  auto *Cond = Clause->getCondition();
2671  bool CondConstant;
2672  if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
2673  Data.Final.setInt(CondConstant);
2674  else
2675  Data.Final.setPointer(EvaluateExprAsBool(Cond));
2676  } else {
2677  // By default the task is not final.
2678  Data.Final.setInt(/*IntVal=*/false);
2679  }
2680  // Check if the task has 'priority' clause.
2681  if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
2682  auto *Prio = Clause->getPriority();
2683  Data.Priority.setInt(/*IntVal=*/true);
2684  Data.Priority.setPointer(EmitScalarConversion(
2685  EmitScalarExpr(Prio), Prio->getType(),
2686  getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
2687  Prio->getExprLoc()));
2688  }
2689  // The first function argument for tasks is a thread id, the second one is a
2690  // part id (0 for tied tasks, >=0 for untied task).
2691  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
2692  // Get list of private variables.
2693  for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
2694  auto IRef = C->varlist_begin();
2695  for (auto *IInit : C->private_copies()) {
2696  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2697  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2698  Data.PrivateVars.push_back(*IRef);
2699  Data.PrivateCopies.push_back(IInit);
2700  }
2701  ++IRef;
2702  }
2703  }
2704  EmittedAsPrivate.clear();
2705  // Get list of firstprivate variables.
2706  for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
2707  auto IRef = C->varlist_begin();
2708  auto IElemInitRef = C->inits().begin();
2709  for (auto *IInit : C->private_copies()) {
2710  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2711  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2712  Data.FirstprivateVars.push_back(*IRef);
2713  Data.FirstprivateCopies.push_back(IInit);
2714  Data.FirstprivateInits.push_back(*IElemInitRef);
2715  }
2716  ++IRef;
2717  ++IElemInitRef;
2718  }
2719  }
2720  // Get list of lastprivate variables (for taskloops).
2721  llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
2722  for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
2723  auto IRef = C->varlist_begin();
2724  auto ID = C->destination_exprs().begin();
2725  for (auto *IInit : C->private_copies()) {
2726  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2727  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2728  Data.LastprivateVars.push_back(*IRef);
2729  Data.LastprivateCopies.push_back(IInit);
2730  }
2731  LastprivateDstsOrigs.insert(
2732  {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
2733  cast<DeclRefExpr>(*IRef)});
2734  ++IRef;
2735  ++ID;
2736  }
2737  }
2740  for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
2741  auto IPriv = C->privates().begin();
2742  auto IRed = C->reduction_ops().begin();
2743  auto ILHS = C->lhs_exprs().begin();
2744  auto IRHS = C->rhs_exprs().begin();
2745  for (const auto *Ref : C->varlists()) {
2746  Data.ReductionVars.emplace_back(Ref);
2747  Data.ReductionCopies.emplace_back(*IPriv);
2748  Data.ReductionOps.emplace_back(*IRed);
2749  LHSs.emplace_back(*ILHS);
2750  RHSs.emplace_back(*IRHS);
2751  std::advance(IPriv, 1);
2752  std::advance(IRed, 1);
2753  std::advance(ILHS, 1);
2754  std::advance(IRHS, 1);
2755  }
2756  }
2757  Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
2758  *this, S.getLocStart(), LHSs, RHSs, Data);
2759  // Build list of dependences.
2760  for (const auto *C : S.getClausesOfKind<OMPDependClause>())
2761  for (auto *IRef : C->varlists())
2762  Data.Dependences.push_back(std::make_pair(C->getDependencyKind(), IRef));
2763  auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs](
2764  CodeGenFunction &CGF, PrePostActionTy &Action) {
2765  // Set proper addresses for generated private copies.
2766  OMPPrivateScope Scope(CGF);
2767  if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
2768  !Data.LastprivateVars.empty()) {
2769  enum { PrivatesParam = 2, CopyFnParam = 3 };
2770  auto *CopyFn = CGF.Builder.CreateLoad(
2771  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(3)));
2772  auto *PrivatesPtr = CGF.Builder.CreateLoad(
2773  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(2)));
2774  // Map privates.
2777  CallArgs.push_back(PrivatesPtr);
2778  for (auto *E : Data.PrivateVars) {
2779  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2780  Address PrivatePtr = CGF.CreateMemTemp(
2781  CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
2782  PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
2783  CallArgs.push_back(PrivatePtr.getPointer());
2784  }
2785  for (auto *E : Data.FirstprivateVars) {
2786  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2787  Address PrivatePtr =
2788  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2789  ".firstpriv.ptr.addr");
2790  PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
2791  CallArgs.push_back(PrivatePtr.getPointer());
2792  }
2793  for (auto *E : Data.LastprivateVars) {
2794  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2795  Address PrivatePtr =
2796  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2797  ".lastpriv.ptr.addr");
2798  PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
2799  CallArgs.push_back(PrivatePtr.getPointer());
2800  }
2801  CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(),
2802  CopyFn, CallArgs);
2803  for (auto &&Pair : LastprivateDstsOrigs) {
2804  auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
2805  DeclRefExpr DRE(
2806  const_cast<VarDecl *>(OrigVD),
2807  /*RefersToEnclosingVariableOrCapture=*/CGF.CapturedStmtInfo->lookup(
2808  OrigVD) != nullptr,
2809  Pair.second->getType(), VK_LValue, Pair.second->getExprLoc());
2810  Scope.addPrivate(Pair.first, [&CGF, &DRE]() {
2811  return CGF.EmitLValue(&DRE).getAddress();
2812  });
2813  }
2814  for (auto &&Pair : PrivatePtrs) {
2815  Address Replacement(CGF.Builder.CreateLoad(Pair.second),
2816  CGF.getContext().getDeclAlign(Pair.first));
2817  Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
2818  }
2819  }
2820  if (Data.Reductions) {
2821  OMPLexicalScope LexScope(CGF, S, /*AsInlined=*/true);
2823  Data.ReductionOps);
2824  llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
2825  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
2826  for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
2827  RedCG.emitSharedLValue(CGF, Cnt);
2828  RedCG.emitAggregateType(CGF, Cnt);
2829  Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
2830  CGF, S.getLocStart(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
2831  Replacement =
2832  Address(CGF.EmitScalarConversion(
2833  Replacement.getPointer(), CGF.getContext().VoidPtrTy,
2834  CGF.getContext().getPointerType(
2835  Data.ReductionCopies[Cnt]->getType()),
2836  SourceLocation()),
2837  Replacement.getAlignment());
2838  Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
2839  Scope.addPrivate(RedCG.getBaseDecl(Cnt),
2840  [Replacement]() { return Replacement; });
2841  // FIXME: This must removed once the runtime library is fixed.
2842  // Emit required threadprivate variables for
2843  // initilizer/combiner/finalizer.
2844  CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getLocStart(),
2845  RedCG, Cnt);
2846  }
2847  }
2848  // Privatize all private variables except for in_reduction items.
2849  (void)Scope.Privatize();
2850  SmallVector<const Expr *, 4> InRedVars;
2851  SmallVector<const Expr *, 4> InRedPrivs;
2853  SmallVector<const Expr *, 4> TaskgroupDescriptors;
2854  for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
2855  auto IPriv = C->privates().begin();
2856  auto IRed = C->reduction_ops().begin();
2857  auto ITD = C->taskgroup_descriptors().begin();
2858  for (const auto *Ref : C->varlists()) {
2859  InRedVars.emplace_back(Ref);
2860  InRedPrivs.emplace_back(*IPriv);
2861  InRedOps.emplace_back(*IRed);
2862  TaskgroupDescriptors.emplace_back(*ITD);
2863  std::advance(IPriv, 1);
2864  std::advance(IRed, 1);
2865  std::advance(ITD, 1);
2866  }
2867  }
2868  // Privatize in_reduction items here, because taskgroup descriptors must be
2869  // privatized earlier.
2870  OMPPrivateScope InRedScope(CGF);
2871  if (!InRedVars.empty()) {
2872  ReductionCodeGen RedCG(InRedVars, InRedPrivs, InRedOps);
2873  for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
2874  RedCG.emitSharedLValue(CGF, Cnt);
2875  RedCG.emitAggregateType(CGF, Cnt);
2876  // The taskgroup descriptor variable is always implicit firstprivate and
2877  // privatized already during procoessing of the firstprivates.
2878  llvm::Value *ReductionsPtr = CGF.EmitLoadOfScalar(
2879  CGF.EmitLValue(TaskgroupDescriptors[Cnt]), SourceLocation());
2880  Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
2881  CGF, S.getLocStart(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
2882  Replacement = Address(
2883  CGF.EmitScalarConversion(
2884  Replacement.getPointer(), CGF.getContext().VoidPtrTy,
2885  CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
2886  SourceLocation()),
2887  Replacement.getAlignment());
2888  Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
2889  InRedScope.addPrivate(RedCG.getBaseDecl(Cnt),
2890  [Replacement]() { return Replacement; });
2891  // FIXME: This must removed once the runtime library is fixed.
2892  // Emit required threadprivate variables for
2893  // initilizer/combiner/finalizer.
2894  CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getLocStart(),
2895  RedCG, Cnt);
2896  }
2897  }
2898  (void)InRedScope.Privatize();
2899 
2900  Action.Enter(CGF);
2901  BodyGen(CGF);
2902  };
2903  auto *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
2904  S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied,
2905  Data.NumberOfParts);
2906  OMPLexicalScope Scope(*this, S);
2907  TaskGen(*this, OutlinedFn, Data);
2908 }
2909 
2911  // Emit outlined function for task construct.
2912  auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
2913  auto CapturedStruct = GenerateCapturedStmtArgument(*CS);
2914  auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
2915  const Expr *IfCond = nullptr;
2916  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
2917  if (C->getNameModifier() == OMPD_unknown ||
2918  C->getNameModifier() == OMPD_task) {
2919  IfCond = C->getCondition();
2920  break;
2921  }
2922  }
2923 
2924  OMPTaskDataTy Data;
2925  // Check if we should emit tied or untied task.
2926  Data.Tied = !S.getSingleClause<OMPUntiedClause>();
2927  auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
2928  CGF.EmitStmt(CS->getCapturedStmt());
2929  };
2930  auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
2931  IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
2932  const OMPTaskDataTy &Data) {
2933  CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getLocStart(), S, OutlinedFn,
2934  SharedsTy, CapturedStruct, IfCond,
2935  Data);
2936  };
2937  EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data);
2938 }
2939 
2941  const OMPTaskyieldDirective &S) {
2942  CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getLocStart());
2943 }
2944 
2946  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_barrier);
2947 }
2948 
2950  CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getLocStart());
2951 }
2952 
2954  const OMPTaskgroupDirective &S) {
2955  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2956  Action.Enter(CGF);
2957  if (const Expr *E = S.getReductionRef()) {
2960  OMPTaskDataTy Data;
2961  for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
2962  auto IPriv = C->privates().begin();
2963  auto IRed = C->reduction_ops().begin();
2964  auto ILHS = C->lhs_exprs().begin();
2965  auto IRHS = C->rhs_exprs().begin();
2966  for (const auto *Ref : C->varlists()) {
2967  Data.ReductionVars.emplace_back(Ref);
2968  Data.ReductionCopies.emplace_back(*IPriv);
2969  Data.ReductionOps.emplace_back(*IRed);
2970  LHSs.emplace_back(*ILHS);
2971  RHSs.emplace_back(*IRHS);
2972  std::advance(IPriv, 1);
2973  std::advance(IRed, 1);
2974  std::advance(ILHS, 1);
2975  std::advance(IRHS, 1);
2976  }
2977  }
2978  llvm::Value *ReductionDesc =
2979  CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getLocStart(),
2980  LHSs, RHSs, Data);
2981  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2982  CGF.EmitVarDecl(*VD);
2983  CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),
2984  /*Volatile=*/false, E->getType());
2985  }
2986  CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2987  };
2988  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2989  CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getLocStart());
2990 }
2991 
2993  CGM.getOpenMPRuntime().emitFlush(*this, [&]() -> ArrayRef<const Expr *> {
2994  if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>()) {
2995  return llvm::makeArrayRef(FlushClause->varlist_begin(),
2996  FlushClause->varlist_end());
2997  }
2998  return llvm::None;
2999  }(), S.getLocStart());
3000 }
3001 
3003  const CodeGenLoopTy &CodeGenLoop,
3004  Expr *IncExpr) {
3005  // Emit the loop iteration variable.
3006  auto IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3007  auto IVDecl = cast<VarDecl>(IVExpr->getDecl());
3008  EmitVarDecl(*IVDecl);
3009 
3010  // Emit the iterations count variable.
3011  // If it is not a variable, Sema decided to calculate iterations count on each
3012  // iteration (e.g., it is foldable into a constant).
3013  if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3014  EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3015  // Emit calculation of the iterations count.
3016  EmitIgnoredExpr(S.getCalcLastIteration());
3017  }
3018 
3019  auto &RT = CGM.getOpenMPRuntime();
3020 
3021  bool HasLastprivateClause = false;
3022  // Check pre-condition.
3023  {
3024  OMPLoopScope PreInitScope(*this, S);
3025  // Skip the entire loop if we don't meet the precondition.
3026  // If the condition constant folds and can be elided, avoid emitting the
3027  // whole loop.
3028  bool CondConstant;
3029  llvm::BasicBlock *ContBlock = nullptr;
3030  if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
3031  if (!CondConstant)
3032  return;
3033  } else {
3034  auto *ThenBlock = createBasicBlock("omp.precond.then");
3035  ContBlock = createBasicBlock("omp.precond.end");
3036  emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
3037  getProfileCount(&S));
3038  EmitBlock(ThenBlock);
3039  incrementProfileCounter(&S);
3040  }
3041 
3042  emitAlignedClause(*this, S);
3043  // Emit 'then' code.
3044  {
3045  // Emit helper vars inits.
3046 
3047  LValue LB = EmitOMPHelperVar(
3048  *this, cast<DeclRefExpr>(
3051  : S.getLowerBoundVariable())));
3052  LValue UB = EmitOMPHelperVar(
3053  *this, cast<DeclRefExpr>(
3056  : S.getUpperBoundVariable())));
3057  LValue ST =
3058  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
3059  LValue IL =
3060  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
3061 
3062  OMPPrivateScope LoopScope(*this);
3063  if (EmitOMPFirstprivateClause(S, LoopScope)) {
3064  // Emit implicit barrier to synchronize threads and avoid data races
3065  // on initialization of firstprivate variables and post-update of
3066  // lastprivate variables.
3067  CGM.getOpenMPRuntime().emitBarrierCall(
3068  *this, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
3069  /*ForceSimpleCall=*/true);
3070  }
3071  EmitOMPPrivateClause(S, LoopScope);
3075  EmitOMPReductionClauseInit(S, LoopScope);
3076  HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
3077  EmitOMPPrivateLoopCounters(S, LoopScope);
3078  (void)LoopScope.Privatize();
3079 
3080  // Detect the distribute schedule kind and chunk.
3081  llvm::Value *Chunk = nullptr;
3083  if (auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
3084  ScheduleKind = C->getDistScheduleKind();
3085  if (const auto *Ch = C->getChunkSize()) {
3086  Chunk = EmitScalarExpr(Ch);
3087  Chunk = EmitScalarConversion(Chunk, Ch->getType(),
3089  S.getLocStart());
3090  }
3091  }
3092  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3093  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3094 
3095  // OpenMP [2.10.8, distribute Construct, Description]
3096  // If dist_schedule is specified, kind must be static. If specified,
3097  // iterations are divided into chunks of size chunk_size, chunks are
3098  // assigned to the teams of the league in a round-robin fashion in the
3099  // order of the team number. When no chunk_size is specified, the
3100  // iteration space is divided into chunks that are approximately equal
3101  // in size, and at most one chunk is distributed to each team of the
3102  // league. The size of the chunks is unspecified in this case.
3103  if (RT.isStaticNonchunked(ScheduleKind,
3104  /* Chunked */ Chunk != nullptr)) {
3106  EmitOMPSimdInit(S, /*IsMonotonic=*/true);
3107  CGOpenMPRuntime::StaticRTInput StaticInit(
3108  IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(),
3109  LB.getAddress(), UB.getAddress(), ST.getAddress());
3110  RT.emitDistributeStaticInit(*this, S.getLocStart(), ScheduleKind,
3111  StaticInit);
3112  auto LoopExit =
3113  getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
3114  // UB = min(UB, GlobalUB);
3117  : S.getEnsureUpperBound());
3118  // IV = LB;
3120  ? S.getCombinedInit()
3121  : S.getInit());
3122 
3124  ? S.getCombinedCond()
3125  : S.getCond();
3126 
3127  // for distribute alone, codegen
3128  // while (idx <= UB) { BODY; ++idx; }
3129  // when combined with 'for' (e.g. as in 'distribute parallel for')
3130  // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
3131  EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), Cond, IncExpr,
3132  [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3133  CodeGenLoop(CGF, S, LoopExit);
3134  },
3135  [](CodeGenFunction &) {});
3136  EmitBlock(LoopExit.getBlock());
3137  // Tell the runtime we are done.
3138  RT.emitForStaticFinish(*this, S.getLocStart(), S.getDirectiveKind());
3139  } else {
3140  // Emit the outer loop, which requests its work chunk [LB..UB] from
3141  // runtime and runs the inner loop to process it.
3142  const OMPLoopArguments LoopArguments = {
3143  LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(),
3144  Chunk};
3145  EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
3146  CodeGenLoop);
3147  }
3149  EmitOMPSimdFinal(S, [&](CodeGenFunction &CGF) -> llvm::Value * {
3150  return CGF.Builder.CreateIsNotNull(
3151  CGF.EmitLoadOfScalar(IL, S.getLocStart()));
3152  });
3153  }
3154  OpenMPDirectiveKind ReductionKind = OMPD_unknown;
3157  ReductionKind = OMPD_parallel_for_simd;
3158  } else if (isOpenMPParallelDirective(S.getDirectiveKind())) {
3159  ReductionKind = OMPD_parallel_for;
3160  } else if (isOpenMPSimdDirective(S.getDirectiveKind())) {
3161  ReductionKind = OMPD_simd;
3162  } else if (!isOpenMPTeamsDirective(S.getDirectiveKind()) &&
3164  llvm_unreachable(
3165  "No reduction clauses is allowed in distribute directive.");
3166  }
3167  EmitOMPReductionClauseFinal(S, ReductionKind);
3168  // Emit post-update of the reduction variables if IsLastIter != 0.
3170  *this, S, [&](CodeGenFunction &CGF) -> llvm::Value * {
3171  return CGF.Builder.CreateIsNotNull(
3172  CGF.EmitLoadOfScalar(IL, S.getLocStart()));
3173  });
3174  // Emit final copy of the lastprivate variables if IsLastIter != 0.
3175  if (HasLastprivateClause) {
3176  EmitOMPLastprivateClauseFinal(
3177  S, /*NoFinals=*/false,
3178  Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getLocStart())));
3179  }
3180  }
3181 
3182  // We're now done with the loop, so jump to the continuation block.
3183  if (ContBlock) {
3184  EmitBranch(ContBlock);
3185  EmitBlock(ContBlock, true);
3186  }
3187  }
3188 }
3189 
3191  const OMPDistributeDirective &S) {
3192  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3193 
3195  };
3196  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
3197  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3198 }
3199 
3200 static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM,
3201  const CapturedStmt *S) {
3202  CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
3204  CGF.CapturedStmtInfo = &CapStmtInfo;
3205  auto *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S);
3206  Fn->addFnAttr(llvm::Attribute::NoInline);
3207  return Fn;
3208 }
3209 
3211  if (!S.getAssociatedStmt()) {
3212  for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
3213  CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
3214  return;
3215  }
3216  auto *C = S.getSingleClause<OMPSIMDClause>();
3217  auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
3218  PrePostActionTy &Action) {
3219  if (C) {
3220  auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
3222  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
3223  auto *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS);
3224  CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(),
3225  OutlinedFn, CapturedVars);
3226  } else {
3227  Action.Enter(CGF);
3228  CGF.EmitStmt(
3229  cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
3230  }
3231  };
3232  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
3233  CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getLocStart(), !C);
3234 }
3235 
3237  QualType SrcType, QualType DestType,
3238  SourceLocation Loc) {
3239  assert(CGF.hasScalarEvaluationKind(DestType) &&
3240  "DestType must have scalar evaluation kind.");
3241  assert(!Val.isAggregate() && "Must be a scalar or complex.");
3242  return Val.isScalar()
3243  ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestType,
3244  Loc)
3245  : CGF.EmitComplexToScalarConversion(Val.getComplexVal(), SrcType,
3246  DestType, Loc);
3247 }
3248 
3251  QualType DestType, SourceLocation Loc) {
3252  assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
3253  "DestType must have complex evaluation kind.");
3254  CodeGenFunction::ComplexPairTy ComplexVal;
3255  if (Val.isScalar()) {
3256  // Convert the input element to the element type of the complex.
3257  auto DestElementType = DestType->castAs<ComplexType>()->getElementType();
3258  auto ScalarVal = CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
3259  DestElementType, Loc);
3260  ComplexVal = CodeGenFunction::ComplexPairTy(
3261  ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
3262  } else {
3263  assert(Val.isComplex() && "Must be a scalar or complex.");
3264  auto SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
3265  auto DestElementType = DestType->castAs<ComplexType>()->getElementType();
3266  ComplexVal.first = CGF.EmitScalarConversion(
3267  Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
3268  ComplexVal.second = CGF.EmitScalarConversion(
3269  Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
3270  }
3271  return ComplexVal;
3272 }
3273 
3274 static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst,
3275  LValue LVal, RValue RVal) {
3276  if (LVal.isGlobalReg()) {
3277  CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
3278  } else {
3279  CGF.EmitAtomicStore(RVal, LVal,
3280  IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3281  : llvm::AtomicOrdering::Monotonic,
3282  LVal.isVolatile(), /*IsInit=*/false);
3283  }
3284 }
3285 
3287  QualType RValTy, SourceLocation Loc) {
3288  switch (getEvaluationKind(LVal.getType())) {
3289  case TEK_Scalar:
3290  EmitStoreThroughLValue(RValue::get(convertToScalarValue(
3291  *this, RVal, RValTy, LVal.getType(), Loc)),
3292  LVal);
3293  break;
3294  case TEK_Complex:
3295  EmitStoreOfComplex(
3296  convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,
3297  /*isInit=*/false);
3298  break;
3299  case TEK_Aggregate:
3300  llvm_unreachable("Must be a scalar or complex.");
3301  }
3302 }
3303 
3304 static void EmitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst,
3305  const Expr *X, const Expr *V,
3306  SourceLocation Loc) {
3307  // v = x;
3308  assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
3309  assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
3310  LValue XLValue = CGF.EmitLValue(X);
3311  LValue VLValue = CGF.EmitLValue(V);
3312  RValue Res = XLValue.isGlobalReg()
3313  ? CGF.EmitLoadOfLValue(XLValue, Loc)
3314  : CGF.EmitAtomicLoad(
3315  XLValue, Loc,
3316  IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3317  : llvm::AtomicOrdering::Monotonic,
3318  XLValue.isVolatile());
3319  // OpenMP, 2.12.6, atomic Construct
3320  // Any atomic construct with a seq_cst clause forces the atomically
3321  // performed operation to include an implicit flush operation without a
3322  // list.
3323  if (IsSeqCst)
3324  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3325  CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
3326 }
3327 
3328 static void EmitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst,
3329  const Expr *X, const Expr *E,
3330  SourceLocation Loc) {
3331  // x = expr;
3332  assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
3333  emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
3334  // OpenMP, 2.12.6, atomic Construct
3335  // Any atomic construct with a seq_cst clause forces the atomically
3336  // performed operation to include an implicit flush operation without a
3337  // list.
3338  if (IsSeqCst)
3339  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3340 }
3341 
3342 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
3343  RValue Update,
3344  BinaryOperatorKind BO,
3345  llvm::AtomicOrdering AO,
3346  bool IsXLHSInRHSPart) {
3347  auto &Context = CGF.CGM.getContext();
3348  // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
3349  // expression is simple and atomic is allowed for the given type for the
3350  // target platform.
3351  if (BO == BO_Comma || !Update.isScalar() ||
3352  !Update.getScalarVal()->getType()->isIntegerTy() ||
3353  !X.isSimple() || (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
3354  (Update.getScalarVal()->getType() !=
3355  X.getAddress().getElementType())) ||
3356  !X.getAddress().getElementType()->isIntegerTy() ||
3357  !Context.getTargetInfo().hasBuiltinAtomic(
3358  Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
3359  return std::make_pair(false, RValue::get(nullptr));
3360 
3361  llvm::AtomicRMWInst::BinOp RMWOp;
3362  switch (BO) {
3363  case BO_Add:
3364  RMWOp = llvm::AtomicRMWInst::Add;
3365  break;
3366  case BO_Sub:
3367  if (!IsXLHSInRHSPart)
3368  return std::make_pair(false, RValue::get(nullptr));
3369  RMWOp = llvm::AtomicRMWInst::Sub;
3370  break;
3371  case BO_And:
3372  RMWOp = llvm::AtomicRMWInst::And;
3373  break;
3374  case BO_Or:
3375  RMWOp = llvm::AtomicRMWInst::Or;
3376  break;
3377  case BO_Xor:
3378  RMWOp = llvm::AtomicRMWInst::Xor;
3379  break;
3380  case BO_LT:
3382  ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
3383  : llvm::AtomicRMWInst::Max)
3384  : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
3385  : llvm::AtomicRMWInst::UMax);
3386  break;
3387  case BO_GT:
3389  ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
3390  : llvm::AtomicRMWInst::Min)
3391  : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
3392  : llvm::AtomicRMWInst::UMin);
3393  break;
3394  case BO_Assign:
3395  RMWOp = llvm::AtomicRMWInst::Xchg;
3396  break;
3397  case BO_Mul:
3398  case BO_Div:
3399  case BO_Rem:
3400  case BO_Shl:
3401  case BO_Shr:
3402  case BO_LAnd:
3403  case BO_LOr:
3404  return std::make_pair(false, RValue::get(nullptr));
3405  case BO_PtrMemD:
3406  case BO_PtrMemI:
3407  case BO_LE:
3408  case BO_GE:
3409  case BO_EQ:
3410  case BO_NE:
3411  case BO_AddAssign:
3412  case BO_SubAssign:
3413  case BO_AndAssign:
3414  case BO_OrAssign:
3415  case BO_XorAssign:
3416  case BO_MulAssign:
3417  case BO_DivAssign:
3418  case BO_RemAssign:
3419  case BO_ShlAssign:
3420  case BO_ShrAssign:
3421  case BO_Comma:
3422  llvm_unreachable("Unsupported atomic update operation");
3423  }
3424  auto *UpdateVal = Update.getScalarVal();
3425  if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
3426  UpdateVal = CGF.Builder.CreateIntCast(
3427  IC, X.getAddress().getElementType(),
3429  }
3430  auto *Res = CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO);
3431  return std::make_pair(true, RValue::get(Res));
3432 }
3433 
3435  LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3436  llvm::AtomicOrdering AO, SourceLocation Loc,
3437  const llvm::function_ref<RValue(RValue)> &CommonGen) {
3438  // Update expressions are allowed to have the following forms:
3439  // x binop= expr; -> xrval + expr;
3440  // x++, ++x -> xrval + 1;
3441  // x--, --x -> xrval - 1;
3442  // x = x binop expr; -> xrval binop expr
3443  // x = expr Op x; - > expr binop xrval;
3444  auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);
3445  if (!Res.first) {
3446  if (X.isGlobalReg()) {
3447  // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
3448  // 'xrval'.
3449  EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
3450  } else {
3451  // Perform compare-and-swap procedure.
3452  EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
3453  }
3454  }
3455  return Res;
3456 }
3457 
3458 static void EmitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst,
3459  const Expr *X, const Expr *E,
3460  const Expr *UE, bool IsXLHSInRHSPart,
3461  SourceLocation Loc) {
3462  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3463  "Update expr in 'atomic update' must be a binary operator.");
3464  auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3465  // Update expressions are allowed to have the following forms:
3466  // x binop= expr; -> xrval + expr;
3467  // x++, ++x -> xrval + 1;
3468  // x--, --x -> xrval - 1;
3469  // x = x binop expr; -> xrval binop expr
3470  // x = expr Op x; - > expr binop xrval;
3471  assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
3472  LValue XLValue = CGF.EmitLValue(X);
3473  RValue ExprRValue = CGF.EmitAnyExpr(E);
3474  auto AO = IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3475  : llvm::AtomicOrdering::Monotonic;
3476  auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3477  auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3478  auto *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3479  auto *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3480  auto Gen =
3481  [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) -> RValue {
3482  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3483  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3484  return CGF.EmitAnyExpr(UE);
3485  };
3487  XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3488  // OpenMP, 2.12.6, atomic Construct
3489  // Any atomic construct with a seq_cst clause forces the atomically
3490  // performed operation to include an implicit flush operation without a
3491  // list.
3492  if (IsSeqCst)
3493  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3494 }
3495 
3497  QualType SourceType, QualType ResType,
3498  SourceLocation Loc) {
3499  switch (CGF.getEvaluationKind(ResType)) {
3500  case TEK_Scalar:
3501  return RValue::get(
3502  convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
3503  case TEK_Complex: {
3504  auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
3505  return RValue::getComplex(Res.first, Res.second);
3506  }
3507  case TEK_Aggregate:
3508  break;
3509  }
3510  llvm_unreachable("Must be a scalar or complex.");
3511 }
3512 
3513 static void EmitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst,
3514  bool IsPostfixUpdate, const Expr *V,
3515  const Expr *X, const Expr *E,
3516  const Expr *UE, bool IsXLHSInRHSPart,
3517  SourceLocation Loc) {
3518  assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
3519  assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
3520  RValue NewVVal;
3521  LValue VLValue = CGF.EmitLValue(V);
3522  LValue XLValue = CGF.EmitLValue(X);
3523  RValue ExprRValue = CGF.EmitAnyExpr(E);
3524  auto AO = IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3525  : llvm::AtomicOrdering::Monotonic;
3526  QualType NewVValType;
3527  if (UE) {
3528  // 'x' is updated with some additional value.
3529  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3530  "Update expr in 'atomic capture' must be a binary operator.");
3531  auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3532  // Update expressions are allowed to have the following forms:
3533  // x binop= expr; -> xrval + expr;
3534  // x++, ++x -> xrval + 1;
3535  // x--, --x -> xrval - 1;
3536  // x = x binop expr; -> xrval binop expr
3537  // x = expr Op x; - > expr binop xrval;
3538  auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3539  auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3540  auto *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3541  NewVValType = XRValExpr->getType();
3542  auto *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3543  auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
3544  IsPostfixUpdate](RValue XRValue) -> RValue {
3545  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3546  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3547  RValue Res = CGF.EmitAnyExpr(UE);
3548  NewVVal = IsPostfixUpdate ? XRValue : Res;
3549  return Res;
3550  };
3551  auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3552  XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3553  if (Res.first) {
3554  // 'atomicrmw' instruction was generated.
3555  if (IsPostfixUpdate) {
3556  // Use old value from 'atomicrmw'.
3557  NewVVal = Res.second;
3558  } else {
3559  // 'atomicrmw' does not provide new value, so evaluate it using old
3560  // value of 'x'.
3561  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3562  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
3563  NewVVal = CGF.EmitAnyExpr(UE);
3564  }
3565  }
3566  } else {
3567  // 'x' is simply rewritten with some 'expr'.
3568  NewVValType = X->getType().getNonReferenceType();
3569  ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
3570  X->getType().getNonReferenceType(), Loc);
3571  auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) -> RValue {
3572  NewVVal = XRValue;
3573  return ExprRValue;
3574  };
3575  // Try to perform atomicrmw xchg, otherwise simple exchange.
3576  auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3577  XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
3578  Loc, Gen);
3579  if (Res.first) {
3580  // 'atomicrmw' instruction was generated.
3581  NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
3582  }
3583  }
3584  // Emit post-update store to 'v' of old/new 'x' value.
3585  CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
3586  // OpenMP, 2.12.6, atomic Construct
3587  // Any atomic construct with a seq_cst clause forces the atomically
3588  // performed operation to include an implicit flush operation without a
3589  // list.
3590  if (IsSeqCst)
3591  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3592 }
3593 
3595  bool IsSeqCst, bool IsPostfixUpdate,
3596  const Expr *X, const Expr *V, const Expr *E,
3597  const Expr *UE, bool IsXLHSInRHSPart,
3598  SourceLocation Loc) {
3599  switch (Kind) {
3600  case OMPC_read:
3601  EmitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc);
3602  break;
3603  case OMPC_write:
3604  EmitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc);
3605  break;
3606  case OMPC_unknown:
3607  case OMPC_update:
3608  EmitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc);
3609  break;
3610  case OMPC_capture:
3611  EmitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE,
3612  IsXLHSInRHSPart, Loc);
3613  break;
3614  case OMPC_if:
3615  case OMPC_final:
3616  case OMPC_num_threads:
3617  case OMPC_private:
3618  case OMPC_firstprivate:
3619  case OMPC_lastprivate:
3620  case OMPC_reduction:
3621  case OMPC_task_reduction:
3622  case OMPC_in_reduction:
3623  case OMPC_safelen:
3624  case OMPC_simdlen:
3625  case OMPC_collapse:
3626  case OMPC_default:
3627  case OMPC_seq_cst:
3628  case OMPC_shared:
3629  case OMPC_linear:
3630  case OMPC_aligned:
3631  case OMPC_copyin:
3632  case OMPC_copyprivate:
3633  case OMPC_flush:
3634  case OMPC_proc_bind:
3635  case OMPC_schedule:
3636  case OMPC_ordered:
3637  case OMPC_nowait:
3638  case OMPC_untied:
3639  case OMPC_threadprivate:
3640  case OMPC_depend:
3641  case OMPC_mergeable:
3642  case OMPC_device:
3643  case OMPC_threads:
3644  case OMPC_simd:
3645  case OMPC_map:
3646  case OMPC_num_teams:
3647  case OMPC_thread_limit:
3648  case OMPC_priority:
3649  case OMPC_grainsize:
3650  case OMPC_nogroup:
3651  case OMPC_num_tasks:
3652  case OMPC_hint:
3653  case OMPC_dist_schedule:
3654  case OMPC_defaultmap:
3655  case OMPC_uniform:
3656  case OMPC_to:
3657  case OMPC_from:
3658  case OMPC_use_device_ptr:
3659  case OMPC_is_device_ptr:
3660  llvm_unreachable("Clause is not allowed in 'omp atomic'.");
3661  }
3662 }
3663 
3665  bool IsSeqCst = S.getSingleClause<OMPSeqCstClause>();
3667  for (auto *C : S.clauses()) {
3668  // Find first clause (skip seq_cst clause, if it is first).
3669  if (C->getClauseKind() != OMPC_seq_cst) {
3670  Kind = C->getClauseKind();
3671  break;
3672  }
3673  }
3674 
3675  const auto *CS =
3676  S.getAssociatedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
3677  if (const auto *EWC = dyn_cast<ExprWithCleanups>(CS)) {
3678  enterFullExpression(EWC);
3679  }
3680  // Processing for statements under 'atomic capture'.
3681  if (const auto *Compound = dyn_cast<CompoundStmt>(CS)) {
3682  for (const auto *C : Compound->body()) {
3683  if (const auto *EWC = dyn_cast<ExprWithCleanups>(C)) {
3684  enterFullExpression(EWC);
3685  }
3686  }
3687  }
3688 
3689  auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF,
3690  PrePostActionTy &) {
3691  CGF.EmitStopPoint(CS);
3692  EmitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(),
3693  S.getV(), S.getExpr(), S.getUpdateExpr(),
3694  S.isXLHSInRHSPart(), S.getLocStart());
3695  };
3696  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
3697  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen);
3698 }
3699 
3701  const OMPExecutableDirective &S,
3702  const RegionCodeGenTy &CodeGen) {
3704  CodeGenModule &CGM = CGF.CGM;
3705  const CapturedStmt &CS = *S.getCapturedStmt(OMPD_target);
3706 
3707  llvm::Function *Fn = nullptr;
3708  llvm::Constant *FnID = nullptr;
3709 
3710  const Expr *IfCond = nullptr;
3711  // Check for the at most one if clause associated with the target region.
3712  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3713  if (C->getNameModifier() == OMPD_unknown ||
3714  C->getNameModifier() == OMPD_target) {
3715  IfCond = C->getCondition();
3716  break;
3717  }
3718  }
3719 
3720  // Check if we have any device clause associated with the directive.
3721  const Expr *Device = nullptr;
3722  if (auto *C = S.getSingleClause<OMPDeviceClause>()) {
3723  Device = C->getDevice();
3724  }
3725 
3726  // Check if we have an if clause whose conditional always evaluates to false
3727  // or if we do not have any targets specified. If so the target region is not
3728  // an offload entry point.
3729  bool IsOffloadEntry = true;
3730  if (IfCond) {
3731  bool Val;
3732  if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
3733  IsOffloadEntry = false;
3734  }
3735  if (CGM.getLangOpts().OMPTargetTriples.empty())
3736  IsOffloadEntry = false;
3737 
3738  assert(CGF.CurFuncDecl && "No parent declaration for target region!");
3739  StringRef ParentName;
3740  // In case we have Ctors/Dtors we use the complete type variant to produce
3741  // the mangling of the device outlined kernel.
3742  if (auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
3743  ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));
3744  else if (auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
3745  ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));
3746  else
3747  ParentName =
3748  CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));
3749 
3750  // Emit target region as a standalone region.
3751  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
3752  IsOffloadEntry, CodeGen);
3753  OMPLexicalScope Scope(CGF, S);
3755  CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
3756  CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device,
3757  CapturedVars);
3758 }
3759 
3761  PrePostActionTy &Action) {
3762  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
3763  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
3764  CGF.EmitOMPPrivateClause(S, PrivateScope);
3765  (void)PrivateScope.Privatize();
3766 
3767  Action.Enter(CGF);
3768  CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
3769 }
3770 
3772  StringRef ParentName,
3773  const OMPTargetDirective &S) {
3774  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3775  emitTargetRegion(CGF, S, Action);
3776  };
3777  llvm::Function *Fn;
3778  llvm::Constant *Addr;
3779  // Emit target region as a standalone region.
3780  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
3781  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
3782  assert(Fn && Addr && "Target device function emission failed.");
3783 }
3784 
3786  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3787  emitTargetRegion(CGF, S, Action);
3788  };
3789  emitCommonOMPTargetDirective(*this, S, CodeGen);
3790 }
3791 
3793  const OMPExecutableDirective &S,
3794  OpenMPDirectiveKind InnermostKind,
3795  const RegionCodeGenTy &CodeGen) {
3796  const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
3797  auto OutlinedFn = CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
3798  S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
3799 
3802  if (NT || TL) {
3803  Expr *NumTeams = (NT) ? NT->getNumTeams() : nullptr;
3804  Expr *ThreadLimit = (TL) ? TL->getThreadLimit() : nullptr;
3805 
3806  CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
3807  S.getLocStart());
3808  }
3809 
3810  OMPTeamsScope Scope(CGF, S);
3812  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
3813  CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getLocStart(), OutlinedFn,
3814  CapturedVars);
3815 }
3816 
3818  // Emit teams region as a standalone region.
3819  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3820  OMPPrivateScope PrivateScope(CGF);
3821  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
3822  CGF.EmitOMPPrivateClause(S, PrivateScope);
3823  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
3824  (void)PrivateScope.Privatize();
3825  CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
3826  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
3827  };
3828  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
3830  *this, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
3831 }
3832 
3834  const OMPTargetTeamsDirective &S) {
3835  auto *CS = S.getCapturedStmt(OMPD_teams);
3836  Action.Enter(CGF);
3837  // Emit teams region as a standalone region.
3838  auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
3839  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
3840  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
3841  CGF.EmitOMPPrivateClause(S, PrivateScope);
3842  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
3843  (void)PrivateScope.Privatize();
3844  Action.Enter(CGF);
3845  CGF.EmitStmt(CS->getCapturedStmt());
3846  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
3847  };
3848  emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
3850  CGF, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
3851 }
3852 
3854  CodeGenModule &CGM, StringRef ParentName,
3855  const OMPTargetTeamsDirective &S) {
3856  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3857  emitTargetTeamsRegion(CGF, Action, S);
3858  };
3859  llvm::Function *Fn;
3860  llvm::Constant *Addr;
3861  // Emit target region as a standalone region.
3862  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
3863  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
3864  assert(Fn && Addr && "Target device function emission failed.");
3865 }
3866 
3868  const OMPTargetTeamsDirective &S) {
3869  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3870  emitTargetTeamsRegion(CGF, Action, S);
3871  };
3872  emitCommonOMPTargetDirective(*this, S, CodeGen);
3873 }
3874 
3875 static void
3878  Action.Enter(CGF);
3879  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3881  };
3882 
3883  // Emit teams region as a standalone region.
3884  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
3885  PrePostActionTy &) {
3886  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
3887  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
3888  (void)PrivateScope.Privatize();
3889  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
3890  CodeGenDistribute);
3891  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
3892  };
3893  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
3895  [](CodeGenFunction &) { return nullptr; });
3896 }
3897 
3899  CodeGenModule &CGM, StringRef ParentName,
3901  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3902  emitTargetTeamsDistributeRegion(CGF, Action, S);
3903  };
3904  llvm::Function *Fn;
3905  llvm::Constant *Addr;
3906  // Emit target region as a standalone region.
3907  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
3908  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
3909  assert(Fn && Addr && "Target device function emission failed.");
3910 }
3911 
3914  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3915  emitTargetTeamsDistributeRegion(CGF, Action, S);
3916  };
3917  emitCommonOMPTargetDirective(*this, S, CodeGen);
3918 }
3919 
3921  CodeGenFunction &CGF, PrePostActionTy &Action,
3923  Action.Enter(CGF);
3924  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3926  };
3927 
3928  // Emit teams region as a standalone region.
3929  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
3930  PrePostActionTy &) {
3931  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
3932  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
3933  (void)PrivateScope.Privatize();
3934  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
3935  CodeGenDistribute);
3936  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
3937  };
3938  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);
3940  [](CodeGenFunction &) { return nullptr; });
3941 }
3942 
3944  CodeGenModule &CGM, StringRef ParentName,
3946  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3947  emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
3948  };
3949  llvm::Function *Fn;
3950  llvm::Constant *Addr;
3951  // Emit target region as a standalone region.
3952  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
3953  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
3954  assert(Fn && Addr && "Target device function emission failed.");
3955 }
3956 
3959  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3960  emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
3961  };
3962  emitCommonOMPTargetDirective(*this, S, CodeGen);
3963 }
3964 
3966  const OMPTeamsDistributeDirective &S) {
3967 
3968  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3970  };
3971 
3972  // Emit teams region as a standalone region.
3973  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
3974  PrePostActionTy &) {
3975  OMPPrivateScope PrivateScope(CGF);
3976  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
3977  (void)PrivateScope.Privatize();
3978  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
3979  CodeGenDistribute);
3980  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
3981  };
3982  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
3984  [](CodeGenFunction &) { return nullptr; });
3985 }
3986 
3989  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3991  };
3992 
3993  // Emit teams region as a standalone region.
3994  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
3995  PrePostActionTy &) {
3996  OMPPrivateScope PrivateScope(CGF);
3997  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
3998  (void)PrivateScope.Privatize();
3999  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
4000  CodeGenDistribute);
4001  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4002  };
4003  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen);
4005  [](CodeGenFunction &) { return nullptr; });
4006 }
4007 
4010  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4012  S.getDistInc());
4013  };
4014 
4015  // Emit teams region as a standalone region.
4016  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4017  PrePostActionTy &) {
4018  OMPPrivateScope PrivateScope(CGF);
4019  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4020  (void)PrivateScope.Privatize();
4021  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4022  CodeGenDistribute);
4023  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4024  };
4025  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4027  [](CodeGenFunction &) { return nullptr; });
4028 }
4029 
4032  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4034  S.getDistInc());
4035  };
4036 
4037  // Emit teams region as a standalone region.
4038  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4039  PrePostActionTy &) {
4040  OMPPrivateScope PrivateScope(CGF);
4041  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4042  (void)PrivateScope.Privatize();
4043  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4044  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4045  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4046  };
4047  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4049  [](CodeGenFunction &) { return nullptr; });
4050 }
4051 
4053  const OMPCancellationPointDirective &S) {
4054  CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getLocStart(),
4055  S.getCancelRegion());
4056 }
4057 
4059  const Expr *IfCond = nullptr;
4060  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4061  if (C->getNameModifier() == OMPD_unknown ||
4062  C->getNameModifier() == OMPD_cancel) {
4063  IfCond = C->getCondition();
4064  break;
4065  }
4066  }
4067  CGM.getOpenMPRuntime().emitCancelCall(*this, S.getLocStart(), IfCond,
4068  S.getCancelRegion());
4069 }
4070 
4073  if (Kind == OMPD_parallel || Kind == OMPD_task ||
4074  Kind == OMPD_target_parallel)
4075  return ReturnBlock;
4076  assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
4077  Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
4078  Kind == OMPD_distribute_parallel_for ||
4079  Kind == OMPD_target_parallel_for ||
4080  Kind == OMPD_teams_distribute_parallel_for ||
4081  Kind == OMPD_target_teams_distribute_parallel_for);
4082  return OMPCancelStack.getExitBlock();
4083 }
4084 
4086  const OMPClause &NC, OMPPrivateScope &PrivateScope,
4087  const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) {
4088  const auto &C = cast<OMPUseDevicePtrClause>(NC);
4089  auto OrigVarIt = C.varlist_begin();
4090  auto InitIt = C.inits().begin();
4091  for (auto PvtVarIt : C.private_copies()) {
4092  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl());
4093  auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl());
4094  auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl());
4095 
4096  // In order to identify the right initializer we need to match the
4097  // declaration used by the mapping logic. In some cases we may get
4098  // OMPCapturedExprDecl that refers to the original declaration.
4099  const ValueDecl *MatchingVD = OrigVD;
4100  if (auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
4101  // OMPCapturedExprDecl are used to privative fields of the current
4102  // structure.
4103  auto *ME = cast<MemberExpr>(OED->getInit());
4104  assert(isa<CXXThisExpr>(ME->getBase()) &&
4105  "Base should be the current struct!");
4106  MatchingVD = ME->getMemberDecl();
4107  }
4108 
4109  // If we don't have information about the current list item, move on to
4110  // the next one.
4111  auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
4112  if (InitAddrIt == CaptureDeviceAddrMap.end())
4113  continue;
4114 
4115  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address {
4116  // Initialize the temporary initialization variable with the address we
4117  // get from the runtime library. We have to cast the source address
4118  // because it is always a void *. References are materialized in the
4119  // privatization scope, so the initialization here disregards the fact
4120  // the original variable is a reference.
4121  QualType AddrQTy =
4122  getContext().getPointerType(OrigVD->getType().getNonReferenceType());
4123  llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy);
4124  Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy);
4125  setAddrOfLocalVar(InitVD, InitAddr);
4126 
4127  // Emit private declaration, it will be initialized by the value we
4128  // declaration we just added to the local declarations map.
4129  EmitDecl(*PvtVD);
4130 
4131  // The initialization variables reached its purpose in the emission
4132  // ofthe previous declaration, so we don't need it anymore.
4133  LocalDeclMap.erase(InitVD);
4134 
4135  // Return the address of the private variable.
4136  return GetAddrOfLocalVar(PvtVD);
4137  });
4138  assert(IsRegistered && "firstprivate var already registered as private");
4139  // Silence the warning about unused variable.
4140  (void)IsRegistered;
4141 
4142  ++OrigVarIt;
4143  ++InitIt;
4144  }
4145 }
4146 
4147 // Generate the instructions for '#pragma omp target data' directive.
4149  const OMPTargetDataDirective &S) {
4150  CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true);
4151 
4152  // Create a pre/post action to signal the privatization of the device pointer.
4153  // This action can be replaced by the OpenMP runtime code generation to
4154  // deactivate privatization.
4155  bool PrivatizeDevicePointers = false;
4156  class DevicePointerPrivActionTy : public PrePostActionTy {
4157  bool &PrivatizeDevicePointers;
4158 
4159  public:
4160  explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
4161  : PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {}
4162  void Enter(CodeGenFunction &CGF) override {
4163  PrivatizeDevicePointers = true;
4164  }
4165  };
4166  DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
4167 
4168  auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers](
4169  CodeGenFunction &CGF, PrePostActionTy &Action) {
4170  auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4171  CGF.EmitStmt(
4172  cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
4173  };
4174 
4175  // Codegen that selects wheather to generate the privatization code or not.
4176  auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers,
4177  &InnermostCodeGen](CodeGenFunction &CGF,
4178  PrePostActionTy &Action) {
4179  RegionCodeGenTy RCG(InnermostCodeGen);
4180  PrivatizeDevicePointers = false;
4181 
4182  // Call the pre-action to change the status of PrivatizeDevicePointers if
4183  // needed.
4184  Action.Enter(CGF);
4185 
4186  if (PrivatizeDevicePointers) {
4187  OMPPrivateScope PrivateScope(CGF);
4188  // Emit all instances of the use_device_ptr clause.
4189  for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
4190  CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
4191  Info.CaptureDeviceAddrMap);
4192  (void)PrivateScope.Privatize();
4193  RCG(CGF);
4194  } else
4195  RCG(CGF);
4196  };
4197 
4198  // Forward the provided action to the privatization codegen.
4199  RegionCodeGenTy PrivRCG(PrivCodeGen);
4200  PrivRCG.setAction(Action);
4201 
4202  // Notwithstanding the body of the region is emitted as inlined directive,
4203  // we don't use an inline scope as changes in the references inside the
4204  // region are expected to be visible outside, so we do not privative them.
4205  OMPLexicalScope Scope(CGF, S);
4206  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
4207  PrivRCG);
4208  };
4209 
4210  RegionCodeGenTy RCG(CodeGen);
4211 
4212  // If we don't have target devices, don't bother emitting the data mapping
4213  // code.
4214  if (CGM.getLangOpts().OMPTargetTriples.empty()) {
4215  RCG(*this);
4216  return;
4217  }
4218 
4219  // Check if we have any if clause associated with the directive.
4220  const Expr *IfCond = nullptr;
4221  if (auto *C = S.getSingleClause<OMPIfClause>())
4222  IfCond = C->getCondition();
4223 
4224  // Check if we have any device clause associated with the directive.
4225  const Expr *Device = nullptr;
4226  if (auto *C = S.getSingleClause<OMPDeviceClause>())
4227  Device = C->getDevice();
4228 
4229  // Set the action to signal privatization of device pointers.
4230  RCG.setAction(PrivAction);
4231 
4232  // Emit region code.
4233  CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
4234  Info);
4235 }
4236 
4238  const OMPTargetEnterDataDirective &S) {
4239  // If we don't have target devices, don't bother emitting the data mapping
4240  // code.
4241  if (CGM.getLangOpts().OMPTargetTriples.empty())
4242  return;
4243 
4244  // Check if we have any if clause associated with the directive.
4245  const Expr *IfCond = nullptr;
4246  if (auto *C = S.getSingleClause<OMPIfClause>())
4247  IfCond = C->getCondition();
4248 
4249  // Check if we have any device clause associated with the directive.
4250  const Expr *Device = nullptr;
4251  if (auto *C = S.getSingleClause<OMPDeviceClause>())
4252  Device = C->getDevice();
4253 
4254  auto &&CodeGen = [&S, IfCond, Device](CodeGenFunction &CGF,
4255  PrePostActionTy &) {
4256  CGF.CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(CGF, S, IfCond,
4257  Device);
4258  };
4259  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
4260  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_target_enter_data,
4261  CodeGen);
4262 }
4263 
4265  const OMPTargetExitDataDirective &S) {
4266  // If we don't have target devices, don't bother emitting the data mapping
4267  // code.
4268  if (CGM.getLangOpts().OMPTargetTriples.empty())
4269  return;
4270 
4271  // Check if we have any if clause associated with the directive.
4272  const Expr *IfCond = nullptr;
4273  if (auto *C = S.getSingleClause<OMPIfClause>())
4274  IfCond = C->getCondition();
4275 
4276  // Check if we have any device clause associated with the directive.
4277  const Expr *Device = nullptr;
4278  if (auto *C = S.getSingleClause<OMPDeviceClause>())
4279  Device = C->getDevice();
4280 
4281  auto &&CodeGen = [&S, IfCond, Device](CodeGenFunction &CGF,
4282  PrePostActionTy &) {
4283  CGF.CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(CGF, S, IfCond,
4284  Device);
4285  };
4286  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
4287  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_target_exit_data,
4288  CodeGen);
4289 }
4290 
4292  const OMPTargetParallelDirective &S,
4293  PrePostActionTy &Action) {
4294  // Get the captured statement associated with the 'parallel' region.
4295  auto *CS = S.getCapturedStmt(OMPD_parallel);
4296  Action.Enter(CGF);
4297  auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &) {
4298  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4299  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4300  CGF.EmitOMPPrivateClause(S, PrivateScope);
4301  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4302  (void)PrivateScope.Privatize();
4303  // TODO: Add support for clauses.
4304  CGF.EmitStmt(CS->getCapturedStmt());
4305  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4306  };
4307  emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
4310  CGF, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
4311 }
4312 
4314  CodeGenModule &CGM, StringRef ParentName,
4315  const OMPTargetParallelDirective &S) {
4316  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4317  emitTargetParallelRegion(CGF, S, Action);
4318  };
4319  llvm::Function *Fn;
4320  llvm::Constant *Addr;
4321  // Emit target region as a standalone region.
4322  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4323  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4324  assert(Fn && Addr && "Target device function emission failed.");
4325 }
4326 
4328  const OMPTargetParallelDirective &S) {
4329  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4330  emitTargetParallelRegion(CGF, S, Action);
4331  };
4332  emitCommonOMPTargetDirective(*this, S, CodeGen);
4333 }
4334 
4337  PrePostActionTy &Action) {
4338  Action.Enter(CGF);
4339  // Emit directive as a combined directive that consists of two implicit
4340  // directives: 'parallel' with 'for' directive.
4341  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4343  CGF, OMPD_target_parallel_for, S.hasCancel());
4346  };
4347  emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
4349 }
4350 
4352  CodeGenModule &CGM, StringRef ParentName,
4353  const OMPTargetParallelForDirective &S) {
4354  // Emit SPMD target parallel for region as a standalone region.
4355  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4356  emitTargetParallelForRegion(CGF, S, Action);
4357  };
4358  llvm::Function *Fn;
4359  llvm::Constant *Addr;
4360  // Emit target region as a standalone region.
4361  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4362  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4363  assert(Fn && Addr && "Target device function emission failed.");
4364 }
4365 
4367  const OMPTargetParallelForDirective &S) {
4368  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4369  emitTargetParallelForRegion(CGF, S, Action);
4370  };
4371  emitCommonOMPTargetDirective(*this, S, CodeGen);
4372 }
4373 
4374 static void
4377  PrePostActionTy &Action) {
4378  Action.Enter(CGF);
4379  // Emit directive as a combined directive that consists of two implicit
4380  // directives: 'parallel' with 'for' directive.
4381  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4384  };
4385  emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
4387 }
4388 
4390  CodeGenModule &CGM, StringRef ParentName,
4392  // Emit SPMD target parallel for region as a standalone region.
4393  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4394  emitTargetParallelForSimdRegion(CGF, S, Action);
4395  };
4396  llvm::Function *Fn;
4397  llvm::Constant *Addr;
4398  // Emit target region as a standalone region.
4399  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4400  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4401  assert(Fn && Addr && "Target device function emission failed.");
4402 }
4403 
4406  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4407  emitTargetParallelForSimdRegion(CGF, S, Action);
4408  };
4409  emitCommonOMPTargetDirective(*this, S, CodeGen);
4410 }
4411 
4412 /// Emit a helper variable and return corresponding lvalue.
4413 static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
4414  const ImplicitParamDecl *PVD,
4416  auto *VDecl = cast<VarDecl>(Helper->getDecl());
4417  Privates.addPrivate(
4418  VDecl, [&CGF, PVD]() -> Address { return CGF.GetAddrOfLocalVar(PVD); });
4419 }
4420 
4423  // Emit outlined function for task construct.
4424  auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
4425  auto CapturedStruct = GenerateCapturedStmtArgument(*CS);
4426  auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
4427  const Expr *IfCond = nullptr;
4428  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4429  if (C->getNameModifier() == OMPD_unknown ||
4430  C->getNameModifier() == OMPD_taskloop) {
4431  IfCond = C->getCondition();
4432  break;
4433  }
4434  }
4435 
4436  OMPTaskDataTy Data;
4437  // Check if taskloop must be emitted without taskgroup.
4439  // TODO: Check if we should emit tied or untied task.
4440  Data.Tied = true;
4441  // Set scheduling for taskloop
4442  if (const auto* Clause = S.getSingleClause<OMPGrainsizeClause>()) {
4443  // grainsize clause
4444  Data.Schedule.setInt(/*IntVal=*/false);
4445  Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
4446  } else if (const auto* Clause = S.getSingleClause<OMPNumTasksClause>()) {
4447  // num_tasks clause
4448  Data.Schedule.setInt(/*IntVal=*/true);
4449  Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
4450  }
4451 
4452  auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
4453  // if (PreCond) {
4454  // for (IV in 0..LastIteration) BODY;
4455  // <Final counter/linear vars updates>;
4456  // }
4457  //
4458 
4459  // Emit: if (PreCond) - begin.
4460  // If the condition constant folds and can be elided, avoid emitting the
4461  // whole loop.
4462  bool CondConstant;
4463  llvm::BasicBlock *ContBlock = nullptr;
4464  OMPLoopScope PreInitScope(CGF, S);
4465  if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
4466  if (!CondConstant)
4467  return;
4468  } else {
4469  auto *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
4470  ContBlock = CGF.createBasicBlock("taskloop.if.end");
4471  emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
4472  CGF.getProfileCount(&S));
4473  CGF.EmitBlock(ThenBlock);
4474  CGF.incrementProfileCounter(&S);
4475  }
4476 
4478  CGF.EmitOMPSimdInit(S);
4479 
4480  OMPPrivateScope LoopScope(CGF);
4481  // Emit helper vars inits.
4482  enum { LowerBound = 5, UpperBound, Stride, LastIter };
4483  auto *I = CS->getCapturedDecl()->param_begin();
4484  auto *LBP = std::next(I, LowerBound);
4485  auto *UBP = std::next(I, UpperBound);
4486  auto *STP = std::next(I, Stride);
4487  auto *LIP = std::next(I, LastIter);
4488  mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
4489  LoopScope);
4490  mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
4491  LoopScope);
4492  mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
4493  mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
4494  LoopScope);
4495  CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
4496  bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4497  (void)LoopScope.Privatize();
4498  // Emit the loop iteration variable.
4499  const Expr *IVExpr = S.getIterationVariable();
4500  const VarDecl *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
4501  CGF.EmitVarDecl(*IVDecl);
4502  CGF.EmitIgnoredExpr(S.getInit());
4503 
4504  // Emit the iterations count variable.
4505  // If it is not a variable, Sema decided to calculate iterations count on
4506  // each iteration (e.g., it is foldable into a constant).
4507  if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
4508  CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
4509  // Emit calculation of the iterations count.
4510  CGF.EmitIgnoredExpr(S.getCalcLastIteration());
4511  }
4512 
4513  CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
4514  S.getInc(),
4515  [&S](CodeGenFunction &CGF) {
4516  CGF.EmitOMPLoopBody(S, JumpDest());
4517  CGF.EmitStopPoint(&S);
4518  },
4519  [](CodeGenFunction &) {});
4520  // Emit: if (PreCond) - end.
4521  if (ContBlock) {
4522  CGF.EmitBranch(ContBlock);
4523  CGF.EmitBlock(ContBlock, true);
4524  }
4525  // Emit final copy of the lastprivate variables if IsLastIter != 0.
4526  if (HasLastprivateClause) {
4527  CGF.EmitOMPLastprivateClauseFinal(
4529  CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
4530  CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
4531  (*LIP)->getType(), S.getLocStart())));
4532  }
4533  };
4534  auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
4535  IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
4536  const OMPTaskDataTy &Data) {
4537  auto &&CodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &) {
4538  OMPLoopScope PreInitScope(CGF, S);
4539  CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getLocStart(), S,
4540  OutlinedFn, SharedsTy,
4541  CapturedStruct, IfCond, Data);
4542  };
4543  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
4544  CodeGen);
4545  };
4546  if (Data.Nogroup)
4547  EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data);
4548  else {
4549  CGM.getOpenMPRuntime().emitTaskgroupRegion(
4550  *this,
4551  [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
4552  PrePostActionTy &Action) {
4553  Action.Enter(CGF);
4554  CGF.EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data);
4555  },
4556  S.getLocStart());
4557  }
4558 }
4559 
4561  EmitOMPTaskLoopBasedDirective(S);
4562 }
4563 
4565  const OMPTaskLoopSimdDirective &S) {
4566  EmitOMPTaskLoopBasedDirective(S);
4567 }
4568 
4569 // Generate the instructions for '#pragma omp target update' directive.
4571  const OMPTargetUpdateDirective &S) {
4572  // If we don't have target devices, don't bother emitting the data mapping
4573  // code.
4574  if (CGM.getLangOpts().OMPTargetTriples.empty())
4575  return;
4576 
4577  // Check if we have any if clause associated with the directive.
4578  const Expr *IfCond = nullptr;
4579  if (auto *C = S.getSingleClause<OMPIfClause>())
4580  IfCond = C->getCondition();
4581 
4582  // Check if we have any device clause associated with the directive.
4583  const Expr *Device = nullptr;
4584  if (auto *C = S.getSingleClause<OMPDeviceClause>())
4585  Device = C->getDevice();
4586 
4587  auto &&CodeGen = [&S, IfCond, Device](CodeGenFunction &CGF,
4588  PrePostActionTy &) {
4589  CGF.CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(CGF, S, IfCond,
4590  Device);
4591  };
4592  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
4593  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_target_update,
4594  CodeGen);
4595 }
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
Definition: CGCall.cpp:640
bool isAggregate() const
Definition: CGValue.h:54
This represents &#39;#pragma omp distribute simd&#39; composite directive.
Definition: StmtOpenMP.h:3190
Expr * getNextUpperBound() const
Definition: StmtOpenMP.h:818
This represents &#39;#pragma omp master&#39; directive.
Definition: StmtOpenMP.h:1373
static Address castValueFromUintptr(CodeGenFunction &CGF, QualType DstType, StringRef Name, LValue AddrLV, bool isReferenceType=false)
This represents &#39;#pragma omp task&#39; directive.
Definition: StmtOpenMP.h:1713
static const Decl * getCanonicalDecl(const Decl *D)
FunctionDecl - An instance of this class is created to represent a function declaration or definition...
Definition: Decl.h:1698
This represents &#39;thread_limit&#39; clause in the &#39;#pragma omp ...&#39; directive.
Expr * getUpperBoundVariable() const
Definition: StmtOpenMP.h:786
Other implicit parameter.
Definition: Decl.h:1474
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
This represents clause &#39;copyin&#39; in the &#39;#pragma omp ...&#39; directives.
Complete object ctor.
Definition: ABI.h:26
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition: Type.h:2285
Scheduling data for loop-based OpenMP directives.
Definition: OpenMPKinds.h:124
A (possibly-)qualified type.
Definition: Type.h:653
capture_init_iterator capture_init_begin()
Retrieve the first initialization argument.
Definition: Stmt.h:2228
bool isArrayType() const
Definition: Type.h:5993
ArrayRef< OMPClause * > clauses()
Definition: StmtOpenMP.h:235
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
Definition: CGDecl.cpp:156
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
Definition: Decl.h:3940
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
static std::pair< LValue, LValue > emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> &CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
static OMPClauseWithPreInit * get(OMPClause *C)
Stmt - This represents one statement.
Definition: Stmt.h:66
This represents clause &#39;in_reduction&#39; in the &#39;#pragma omp task&#39; directives.
Expr * getLowerBoundVariable() const
Definition: StmtOpenMP.h:778
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
Store of global named registers are always calls to intrinsics.
Definition: CGExpr.cpp:2071
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
Definition: Type.cpp:456
bool requiresCleanups() const
Determine whether this scope requires any cleanups.
void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S)
SmallVector< std::pair< OpenMPDependClauseKind, const Expr * >, 4 > Dependences
This represents &#39;#pragma omp for simd&#39; directive.
Definition: StmtOpenMP.h:1123
SmallVector< const Expr *, 4 > LastprivateCopies
QualType getLValueReferenceType(QualType T, bool SpelledAsLValue=true) const
Return the uniqued reference to the type for an lvalue reference to the specified type...
Decl - This represents one declaration (or definition), e.g.
Definition: DeclBase.h:86
This represents &#39;grainsize&#39; clause in the &#39;#pragma omp ...&#39; directive.
This represents &#39;#pragma omp teams distribute parallel for&#39; composite directive.
Definition: StmtOpenMP.h:3601
bool isNothrow() const
Definition: Decl.cpp:4252
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
This represents &#39;if&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:240
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Definition: Type.h:5893
void EmitOMPOrderedDirective(const OMPOrderedDirective &S)
bool isVolatile() const
Definition: CGValue.h:298
This represents &#39;priority&#39; clause in the &#39;#pragma omp ...&#39; directive.
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
The base class of the type hierarchy.
Definition: Type.h:1353
This represents &#39;#pragma omp target teams distribute&#39; combined directive.
Definition: StmtOpenMP.h:3738
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
Definition: CGExpr.cpp:1836
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Definition: Stmt.h:2172
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
const TargetInfo & getTargetInfo() const
Definition: ASTContext.h:671
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters)
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Definition: CGExpr.cpp:2232
Floating point control options.
Definition: LangOptions.h:208
This represents &#39;#pragma omp parallel for&#39; directive.
Definition: StmtOpenMP.h:1494
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value *> &CapturedVars)
This represents &#39;#pragma omp target teams distribute parallel for&#39; combined directive.
Definition: StmtOpenMP.h:3806
Expr * getCombinedEnsureUpperBound() const
Definition: StmtOpenMP.h:870
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
Definition: CGAtomic.cpp:1871
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant, or if it does but contains a label, return false.
Represents a point when we exit a loop.
Definition: ProgramPoint.h:664
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
SmallVector< const Expr *, 4 > ReductionCopies
void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S)
static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, const Twine &Name, llvm::Value *Init=nullptr)
This represents &#39;#pragma omp target exit data&#39; directive.
Definition: StmtOpenMP.h:2405
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
CanQualType getCanonicalParamType(QualType T) const
Return the canonical parameter type corresponding to the specific potentially non-canonical one...
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S)
VarDecl - An instance of this class is created to represent a variable declaration or definition...
Definition: Decl.h:807
This represents clause &#39;private&#39; in the &#39;#pragma omp ...&#39; directives.
void EmitOMPCriticalDirective(const OMPCriticalDirective &S)
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:2588
This represents &#39;num_threads&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:382
const Stmt * getBody() const
Definition: StmtOpenMP.h:900
Extra information about a function prototype.
Definition: Type.h:3389
uint64_t getProfileCount(const Stmt *S)
Get the profiler&#39;s count for the given statement.
Expr * getCombinedUpperBoundVariable() const
Definition: StmtOpenMP.h:864
Expr * getCalcLastIteration() const
Definition: StmtOpenMP.h:746
llvm::Value * getPointer() const
Definition: Address.h:38
unsigned getContextParamPosition() const
Definition: Decl.h:3934
This represents implicit clause &#39;flush&#39; for the &#39;#pragma omp flush&#39; directive.
void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D, bool NoFinals, llvm::Value *IsLastIterCond=nullptr)
Emit final copying of lastprivate values to original variables at the end of the worksharing or simd ...
void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S)
static void emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeDirective &S)
bool isXLHSInRHSPart() const
Return true if helper update expression has form &#39;OpaqueValueExpr(x) binop OpaqueValueExpr(expr)&#39; and...
Definition: StmtOpenMP.h:2202
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
This represents &#39;nogroup&#39; clause in the &#39;#pragma omp ...&#39; directive.
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB, const CodeGenLoopBoundsTy &CodeGenLoopBounds, const CodeGenDispatchBoundsTy &CGDispatchBounds)
Emit code for the worksharing loop-based directive.
This represents &#39;safelen&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:447
void EmitOMPTargetTeamsDistributeParallelForSimdDirective(const OMPTargetTeamsDistributeParallelForSimdDirective &S)
IdentifierInfo * getIdentifier() const
getIdentifier - Get the identifier that names this declaration, if there is one.
Definition: Decl.h:265
Expr * IgnoreImpCasts() LLVM_READONLY
IgnoreImpCasts - Skip past any implicit casts which might surround this expression.
Definition: Expr.h:2865
void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S)
SmallVector< const Expr *, 4 > PrivateVars
RecordDecl - Represents a struct/union/class.
Definition: Decl.h:3482
llvm::DenseMap< const VarDecl *, FieldDecl * > LambdaCaptureFields
One of these records is kept for each identifier that is lexed.
void EmitOMPSimdDirective(const OMPSimdDirective &S)
This represents &#39;#pragma omp parallel&#39; directive.
Definition: StmtOpenMP.h:251
Address getAddress() const
Definition: CGValue.h:324
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:149
This represents &#39;simd&#39; clause in the &#39;#pragma omp ...&#39; directive.
void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S)
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
field_range fields() const
Definition: Decl.h:3613
static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, PrePostActionTy &Action)
Expr * getEnsureUpperBound() const
Definition: StmtOpenMP.h:802
SmallVector< const Expr *, 4 > LastprivateVars
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
Definition: CGExpr.cpp:2210
This represents clause &#39;lastprivate&#39; in the &#39;#pragma omp ...&#39; directives.
void EmitOMPDistributeLoop(const OMPLoopDirective &S, const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr)
Emit code for the distribute loop-based directive.
CharUnits getAlignment() const
Definition: CGValue.h:313
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
Expr * getInc() const
Definition: StmtOpenMP.h:762
static void emitSimdlenSafelenClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, bool IsMonotonic)
ArrayRef< Expr * > updates()
Definition: StmtOpenMP.h:931
bool isReferenceType() const
Definition: Type.h:5956
void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S, OMPPrivateScope &LoopScope)
Emit initial code for loop counters of loop-based directives.
This represents &#39;#pragma omp target simd&#39; directive.
Definition: StmtOpenMP.h:3326
Address getAllocatedAddress() const
Returns the raw, allocated address, which is not necessarily the address of the object itself...
OpenMPDirectiveKind getDirectiveKind() const
Definition: StmtOpenMP.h:221
This represents &#39;#pragma omp barrier&#39; directive.
Definition: StmtOpenMP.h:1825
This is a common base class for loop directives (&#39;omp simd&#39;, &#39;omp for&#39;, &#39;omp for simd&#39; etc...
Definition: StmtOpenMP.h:313
Expr * getNumTeams()
Return NumTeams number.
This represents &#39;#pragma omp critical&#39; directive.
Definition: StmtOpenMP.h:1420
static std::pair< bool, RValue > emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, RValue Update, BinaryOperatorKind BO, llvm::AtomicOrdering AO, bool IsXLHSInRHSPart)
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
Definition: StmtOpenMP.h:2704
IdentifierTable & Idents
Definition: ASTContext.h:537
An r-value expression (a pr-value in the C++11 taxonomy) produces a temporary value.
Definition: Specifiers.h:107
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
void EmitOMPDistributeParallelForSimdDirective(const OMPDistributeParallelForSimdDirective &S)
bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for lastprivate variables.
static void EmitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, bool IsSeqCst, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
EmitAnyExpr - Emit code to compute the specified expression which can have any type.
Definition: CGExpr.cpp:173
This represents clause &#39;copyprivate&#39; in the &#39;#pragma omp ...&#39; directives.
OpenMPDistScheduleClauseKind
OpenMP attributes for &#39;dist_schedule&#39; clause.
Definition: OpenMPKinds.h:100
This represents &#39;#pragma omp distribute parallel for&#39; composite directive.
Definition: StmtOpenMP.h:3041
Stmt * getBody() const override
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
Definition: Decl.cpp:4249
This represents &#39;#pragma omp teams distribute parallel for simd&#39; composite directive.
Definition: StmtOpenMP.h:3530
BinaryOperatorKind
static bool hasScalarEvaluationKind(QualType T)
SourceLocation getLocEnd() const
Returns ending location of directive.
Definition: StmtOpenMP.h:170
Expr *const * const_capture_init_iterator
Const iterator that walks over the capture initialization arguments.
Definition: Stmt.h:2215
void EmitOMPTargetTeamsDistributeSimdDirective(const OMPTargetTeamsDistributeSimdDirective &S)
ArrayRef< Expr * > finals()
Definition: StmtOpenMP.h:937
Expr * getIsLastIterVariable() const
Definition: StmtOpenMP.h:770
void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy, SourceLocation Loc)
OpenMPScheduleClauseKind Schedule
Definition: OpenMPKinds.h:125
CharUnits - This is an opaque type for sizes expressed in character units.
Definition: CharUnits.h:38
static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc, const unsigned IVSize, const bool IVSigned)
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
static void emitDistributeParallelForDistributeInnerBoundParams(CodeGenFunction &CGF, const OMPExecutableDirective &S, llvm::SmallVectorImpl< llvm::Value *> &CapturedVars)
virtual bool hasBuiltinAtomic(uint64_t AtomicSizeInBits, uint64_t AlignmentInBits) const
Returns true if the given target supports lock-free atomic operations at the specified width and alig...
Definition: TargetInfo.h:457
llvm::function_ref< std::pair< LValue, LValue > CodeGenFunction &, const OMPExecutableDirective &S)> CodeGenLoopBoundsTy
Expr * getX()
Get &#39;x&#39; part of the associated expression/statement.
Definition: StmtOpenMP.h:2186
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type...
Definition: Type.h:6356
void EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S)
param_iterator param_end() const
Retrieve an iterator one past the last parameter decl.
Definition: Decl.h:3942
CharUnits getAlignment() const
Return the alignment of this pointer.
Definition: Address.h:67
void EmitOMPTeamsDistributeParallelForSimdDirective(const OMPTeamsDistributeParallelForSimdDirective &S)
void EmitOMPTargetDirective(const OMPTargetDirective &S)
Expr * getIterationVariable() const
Definition: StmtOpenMP.h:738
bool isComplex() const
Definition: CGValue.h:53
A builtin binary operation expression such as "x + y" or "x <= y".
Definition: Expr.h:2985
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
Definition: Type.h:5790
OpenMPScheduleClauseModifier M2
Definition: OpenMPKinds.h:127
void setVectorizeWidth(unsigned W)
Set the vectorize width for the next loop pushed.
Definition: CGLoopInfo.h:147
SmallVector< const Expr *, 4 > PrivateCopies
static llvm::Function * emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S)
void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S)
This represents &#39;#pragma omp cancellation point&#39; directive.
Definition: StmtOpenMP.h:2660
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper)
Emit a helper variable and return corresponding lvalue.
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations...
Scope - A scope is a transient data structure that is used while parsing the program.
Definition: Scope.h:39
field_iterator field_begin() const
Definition: Decl.cpp:3937
This represents &#39;final&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:330
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
void EmitOMPParallelDirective(const OMPParallelDirective &S)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
This represents &#39;#pragma omp teams&#39; directive.
Definition: StmtOpenMP.h:2603
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
Definition: CGExpr.cpp:161
SmallVector< const Expr *, 4 > FirstprivateCopies
Expr * getInit() const
Definition: StmtOpenMP.h:758
This represents clause &#39;reduction&#39; in the &#39;#pragma omp ...&#39; directives.
const VarDecl * getBaseDecl(unsigned N) const
Returns the base declaration of the reduction item.
This represents &#39;#pragma omp teams distribute simd&#39; combined directive.
Definition: StmtOpenMP.h:3460
bool isSimple() const
Definition: CGValue.h:249
An ordinary object is located at an address in memory.
Definition: Specifiers.h:123
SmallVector< const Expr *, 4 > ReductionOps
Controls insertion of cancellation exit blocks in worksharing constructs.
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler&#39;s counter for the given statement by StepV.
llvm::function_ref< std::pair< llvm::Value *, llvm::Value * > CodeGenFunction &, const OMPExecutableDirective &S, Address LB, Address UB)> CodeGenDispatchBoundsTy
This represents the body of a CapturedStmt, and serves as its DeclContext.
Definition: Decl.h:3860
SmallVector< const Expr *, 4 > ReductionVars
static void EmitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, SourceLocation Loc)
static llvm::Value * convertToScalarValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static FunctionDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation NLoc, DeclarationName N, QualType T, TypeSourceInfo *TInfo, StorageClass SC, bool isInlineSpecified=false, bool hasWrittenPrototype=true, bool isConstexprSpecified=false)
Definition: Decl.h:1876
static llvm::Function * emitOutlinedFunctionPrologue(CodeGenFunction &CGF, FunctionArgList &Args, llvm::MapVector< const Decl *, std::pair< const VarDecl *, Address >> &LocalAddrs, llvm::DenseMap< const Decl *, std::pair< const Expr *, llvm::Value *>> &VLASizes, llvm::Value *&CXXThisValue, const FunctionOptions &FO)
SourceLocation getLocStart() const
Returns starting location of directive kind.
Definition: StmtOpenMP.h:168
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
static CodeGenFunction::ComplexPairTy convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
Definition: Decl.cpp:4187
This represents &#39;#pragma omp target parallel for simd&#39; directive.
Definition: StmtOpenMP.h:3258
ArrayRef< Expr * > private_counters()
Definition: StmtOpenMP.h:919
CompoundStmt - This represents a group of statements like { stmt stmt }.
Definition: Stmt.h:595
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
EmitAutoVarAlloca - Emit the alloca and debug information for a local variable.
Definition: CGDecl.cpp:961
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:1303
void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S)
This represents &#39;#pragma omp taskgroup&#39; directive.
Definition: StmtOpenMP.h:1913
RValue - This trivial value class is used to represent the result of an expression that is evaluated...
Definition: CGValue.h:39
This represents clause &#39;aligned&#39; in the &#39;#pragma omp ...&#39; directives.
SourceLocation getLocEnd() const LLVM_READONLY
Definition: Stmt.cpp:290
bool addPrivate(const VarDecl *LocalVD, llvm::function_ref< Address()> PrivateGen)
Registers LocalVD variable as a private and apply PrivateGen function for it to generate correspondin...
Class intended to support codegen of all kind of the reduction clauses.
bool isGlobalVarCaptured(const VarDecl *VD) const
Checks if the global variable is captured in current function.
Expr * getCombinedLowerBoundVariable() const
Definition: StmtOpenMP.h:858
This represents clause &#39;task_reduction&#39; in the &#39;#pragma omp taskgroup&#39; directives.
void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit)
Helper for the OpenMP loop directives.
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Definition: CharUnits.h:179
TypeSourceInfo * getTrivialTypeSourceInfo(QualType T, SourceLocation Loc=SourceLocation()) const
Allocate a TypeSourceInfo where all locations have been initialized to a given location, which defaults to the empty location.
This represents &#39;#pragma omp distribute&#39; directive.
Definition: StmtOpenMP.h:2914
This represents implicit clause &#39;depend&#39; for the &#39;#pragma omp task&#39; directive.
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:1557
This represents &#39;proc_bind&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:675
ValueDecl - Represent the declaration of a variable (in which case it is an lvalue) a function (in wh...
Definition: Decl.h:628
Expr - This represents one expression.
Definition: Expr.h:106
static void emitTargetParallelForRegion(CodeGenFunction &CGF, const OMPTargetParallelForDirective &S, PrePostActionTy &Action)
Emit only debug info necessary for generating line number tables (-gline-tables-only).
void EmitAutoVarInit(const AutoVarEmission &emission)
Definition: CGDecl.cpp:1211
static Address invalid()
Definition: Address.h:35
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited...
const FunctionProtoType * T
This represents &#39;simdlen&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:501
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top, if IgnoreCaptured is true.
Definition: Stmt.cpp:132
llvm::Value * EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified complex type to the specified destination type, where the destination type is an LLVM scalar type.
llvm::function_ref< void(CodeGenFunction &, SourceLocation, const unsigned, const bool)> CodeGenOrderedTy
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
Definition: CGValue.h:66
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
void EmitOMPTeamsDirective(const OMPTeamsDirective &S)
const T * castAs() const
Member-template castAs<specific type>.
Definition: Type.h:6370
void SetInternalFunctionAttributes(const Decl *D, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:1107
static std::pair< llvm::Value *, llvm::Value * > emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
When dealing with dispatch schedules (e.g.
This represents &#39;#pragma omp target teams distribute parallel for simd&#39; combined directive.
Definition: StmtOpenMP.h:3890
static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
OpenMPClauseKind
OpenMP clauses.
Definition: OpenMPKinds.h:33
llvm::PointerType * getType() const
Return the type of the pointer value.
Definition: Address.h:44
static llvm::iterator_range< specific_clause_iterator< SpecificClause > > getClausesOfKind(ArrayRef< OMPClause *> Clauses)
Definition: StmtOpenMP.h:130
This represents &#39;#pragma omp target teams distribute simd&#39; combined directive.
Definition: StmtOpenMP.h:3963
void EmitOMPSimdInit(const OMPLoopDirective &D, bool IsMonotonic=false)
Helpers for the OpenMP loop directives.
This represents &#39;ordered&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:925
Expr * getDistInc() const
Definition: StmtOpenMP.h:846
Expr * getNextLowerBound() const
Definition: StmtOpenMP.h:810
QualType getType() const
Definition: Expr.h:128
Expr * getPrevEnsureUpperBound() const
Definition: StmtOpenMP.h:852
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> &CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
This represents &#39;#pragma omp for&#39; directive.
Definition: StmtOpenMP.h:1046
static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDirective &S)
SmallVector< const Expr *, 4 > FirstprivateVars
void EmitOMPMasterDirective(const OMPMasterDirective &S)
static void EmitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *V, SourceLocation Loc)
static std::pair< llvm::Value *, llvm::Value * > emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
if the &#39;for&#39; loop has a dispatch schedule (e.g.
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
Definition: CharUnits.h:197
void setParallel(bool Enable=true)
Set the next pushed loop as parallel.
Definition: CGLoopInfo.h:127
This represents &#39;#pragma omp target teams&#39; directive.
Definition: StmtOpenMP.h:3679
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
SourceLocation getEnd() const
UnaryOperator - This represents the unary-expression&#39;s (except sizeof and alignof), the postinc/postdec operators from postfix-expression, and various extensions.
Definition: Expr.h:1717
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
QualType getFunctionType(QualType ResultTy, ArrayRef< QualType > Args, const FunctionProtoType::ExtProtoInfo &EPI) const
Return a normal function type with a typed argument list.
Definition: ASTContext.h:1333
void EmitOMPBarrierDirective(const OMPBarrierDirective &S)
bool EmitOMPLinearClauseInit(const OMPLoopDirective &D)
Emit initial code for linear variables.
This represents &#39;#pragma omp cancel&#39; directive.
Definition: StmtOpenMP.h:2718
This represents clause &#39;firstprivate&#39; in the &#39;#pragma omp ...&#39; directives.
ValueDecl * getDecl()
Definition: Expr.h:1041
static void EmitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
const LangOptions & getLangOpts() const
ASTContext & getContext() const
static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, const RegionCodeGenTy &CodeGen)
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
Definition: CGAtomic.cpp:1449
static std::pair< LValue, LValue > emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
The following two functions generate expressions for the loop lower and upper bounds in case of stati...
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
Definition: Decl.cpp:1973
GlobalDecl - represents a global declaration.
Definition: GlobalDecl.h:35
This represents &#39;#pragma omp flush&#39; directive.
Definition: StmtOpenMP.h:1986
bool hasClausesOfKind() const
Returns true if the current directive has one or more clauses of a specific kind. ...
Definition: StmtOpenMP.h:162
This represents &#39;#pragma omp parallel for simd&#39; directive.
Definition: StmtOpenMP.h:1574
This represents &#39;seq_cst&#39; clause in the &#39;#pragma omp atomic&#39; directive.
Definitio