clang  7.0.0svn
CGStmtOpenMP.cpp
Go to the documentation of this file.
1 //===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This contains code to emit OpenMP nodes as LLVM code.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "CGCleanup.h"
15 #include "CGOpenMPRuntime.h"
16 #include "CodeGenFunction.h"
17 #include "CodeGenModule.h"
18 #include "TargetInfo.h"
19 #include "clang/AST/Stmt.h"
20 #include "clang/AST/StmtOpenMP.h"
21 #include "clang/AST/DeclOpenMP.h"
22 #include "llvm/IR/CallSite.h"
23 using namespace clang;
24 using namespace CodeGen;
25 
26 namespace {
27 /// Lexical scope for OpenMP executable constructs, that handles correct codegen
28 /// for captured expressions.
29 class OMPLexicalScope : public CodeGenFunction::LexicalScope {
30  void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
31  for (const auto *C : S.clauses()) {
32  if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
33  if (const auto *PreInit =
34  cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
35  for (const auto *I : PreInit->decls()) {
36  if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
37  CGF.EmitVarDecl(cast<VarDecl>(*I));
38  } else {
40  CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
41  CGF.EmitAutoVarCleanups(Emission);
42  }
43  }
44  }
45  }
46  }
47  }
48  CodeGenFunction::OMPPrivateScope InlinedShareds;
49 
50  static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
51  return CGF.LambdaCaptureFields.lookup(VD) ||
52  (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
53  (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl));
54  }
55 
56 public:
57  OMPLexicalScope(
59  const llvm::Optional<OpenMPDirectiveKind> CapturedRegion = llvm::None,
60  const bool EmitPreInitStmt = true)
61  : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
62  InlinedShareds(CGF) {
63  if (EmitPreInitStmt)
64  emitPreInitStmt(CGF, S);
65  if (!CapturedRegion.hasValue())
66  return;
67  assert(S.hasAssociatedStmt() &&
68  "Expected associated statement for inlined directive.");
69  const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
70  for (const auto &C : CS->captures()) {
71  if (C.capturesVariable() || C.capturesVariableByCopy()) {
72  auto *VD = C.getCapturedVar();
73  assert(VD == VD->getCanonicalDecl() &&
74  "Canonical decl must be captured.");
75  DeclRefExpr DRE(
76  const_cast<VarDecl *>(VD),
77  isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
78  InlinedShareds.isGlobalVarCaptured(VD)),
79  VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
80  InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
81  return CGF.EmitLValue(&DRE).getAddress();
82  });
83  }
84  }
85  (void)InlinedShareds.Privatize();
86  }
87 };
88 
89 /// Lexical scope for OpenMP parallel construct, that handles correct codegen
90 /// for captured expressions.
91 class OMPParallelScope final : public OMPLexicalScope {
92  bool EmitPreInitStmt(const OMPExecutableDirective &S) {
94  return !(isOpenMPTargetExecutionDirective(Kind) ||
97  }
98 
99 public:
100  OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
101  : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
102  EmitPreInitStmt(S)) {}
103 };
104 
105 /// Lexical scope for OpenMP teams construct, that handles correct codegen
106 /// for captured expressions.
107 class OMPTeamsScope final : public OMPLexicalScope {
108  bool EmitPreInitStmt(const OMPExecutableDirective &S) {
110  return !isOpenMPTargetExecutionDirective(Kind) &&
112  }
113 
114 public:
115  OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
116  : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
117  EmitPreInitStmt(S)) {}
118 };
119 
120 /// Private scope for OpenMP loop-based directives, that supports capturing
121 /// of used expression from loop statement.
122 class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
123  void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) {
124  CodeGenFunction::OMPMapVars PreCondVars;
125  for (const auto *E : S.counters()) {
126  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
127  (void)PreCondVars.setVarAddr(
128  CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
129  }
130  (void)PreCondVars.apply(CGF);
131  if (const auto *PreInits = cast_or_null<DeclStmt>(S.getPreInits())) {
132  for (const auto *I : PreInits->decls())
133  CGF.EmitVarDecl(cast<VarDecl>(*I));
134  }
135  PreCondVars.restore(CGF);
136  }
137 
138 public:
139  OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S)
140  : CodeGenFunction::RunCleanupsScope(CGF) {
141  emitPreInitStmt(CGF, S);
142  }
143 };
144 
145 class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
146  CodeGenFunction::OMPPrivateScope InlinedShareds;
147 
148  static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
149  return CGF.LambdaCaptureFields.lookup(VD) ||
150  (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
151  (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) &&
152  cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
153  }
154 
155 public:
156  OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
157  : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
158  InlinedShareds(CGF) {
159  for (const auto *C : S.clauses()) {
160  if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
161  if (const auto *PreInit =
162  cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
163  for (const auto *I : PreInit->decls()) {
164  if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
165  CGF.EmitVarDecl(cast<VarDecl>(*I));
166  } else {
168  CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
169  CGF.EmitAutoVarCleanups(Emission);
170  }
171  }
172  }
173  } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
174  for (const Expr *E : UDP->varlists()) {
175  const Decl *D = cast<DeclRefExpr>(E)->getDecl();
176  if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
177  CGF.EmitVarDecl(*OED);
178  }
179  }
180  }
182  CGF.EmitOMPPrivateClause(S, InlinedShareds);
183  if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
184  if (const Expr *E = TG->getReductionRef())
185  CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
186  }
187  const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
188  while (CS) {
189  for (auto &C : CS->captures()) {
190  if (C.capturesVariable() || C.capturesVariableByCopy()) {
191  auto *VD = C.getCapturedVar();
192  assert(VD == VD->getCanonicalDecl() &&
193  "Canonical decl must be captured.");
194  DeclRefExpr DRE(const_cast<VarDecl *>(VD),
195  isCapturedVar(CGF, VD) ||
196  (CGF.CapturedStmtInfo &&
197  InlinedShareds.isGlobalVarCaptured(VD)),
199  C.getLocation());
200  InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
201  return CGF.EmitLValue(&DRE).getAddress();
202  });
203  }
204  }
205  CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
206  }
207  (void)InlinedShareds.Privatize();
208  }
209 };
210 
211 } // namespace
212 
214  const OMPExecutableDirective &S,
215  const RegionCodeGenTy &CodeGen);
216 
218  if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
219  if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
220  OrigVD = OrigVD->getCanonicalDecl();
221  bool IsCaptured =
222  LambdaCaptureFields.lookup(OrigVD) ||
223  (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
224  (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
225  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), IsCaptured,
226  OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
227  return EmitLValue(&DRE);
228  }
229  }
230  return EmitLValue(E);
231 }
232 
234  ASTContext &C = getContext();
235  llvm::Value *Size = nullptr;
236  auto SizeInChars = C.getTypeSizeInChars(Ty);
237  if (SizeInChars.isZero()) {
238  // getTypeSizeInChars() returns 0 for a VLA.
239  while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) {
240  VlaSizePair VlaSize = getVLASize(VAT);
241  Ty = VlaSize.Type;
242  Size = Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts)
243  : VlaSize.NumElts;
244  }
245  SizeInChars = C.getTypeSizeInChars(Ty);
246  if (SizeInChars.isZero())
247  return llvm::ConstantInt::get(SizeTy, /*V=*/0);
248  return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
249  }
250  return CGM.getSize(SizeInChars);
251 }
252 
254  const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
255  const RecordDecl *RD = S.getCapturedRecordDecl();
256  auto CurField = RD->field_begin();
257  auto CurCap = S.captures().begin();
259  E = S.capture_init_end();
260  I != E; ++I, ++CurField, ++CurCap) {
261  if (CurField->hasCapturedVLAType()) {
262  const VariableArrayType *VAT = CurField->getCapturedVLAType();
263  llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
264  CapturedVars.push_back(Val);
265  } else if (CurCap->capturesThis()) {
266  CapturedVars.push_back(CXXThisValue);
267  } else if (CurCap->capturesVariableByCopy()) {
268  llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
269 
270  // If the field is not a pointer, we need to save the actual value
271  // and load it as a void pointer.
272  if (!CurField->getType()->isAnyPointerType()) {
273  ASTContext &Ctx = getContext();
274  Address DstAddr = CreateMemTemp(
275  Ctx.getUIntPtrType(),
276  Twine(CurCap->getCapturedVar()->getName(), ".casted"));
277  LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
278 
279  llvm::Value *SrcAddrVal = EmitScalarConversion(
280  DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
281  Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
282  LValue SrcLV =
283  MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
284 
285  // Store the value using the source type pointer.
286  EmitStoreThroughLValue(RValue::get(CV), SrcLV);
287 
288  // Load the value using the destination type pointer.
289  CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
290  }
291  CapturedVars.push_back(CV);
292  } else {
293  assert(CurCap->capturesVariable() && "Expected capture by reference.");
294  CapturedVars.push_back(EmitLValue(*I).getAddress().getPointer());
295  }
296  }
297 }
298 
300  QualType DstType, StringRef Name,
301  LValue AddrLV,
302  bool isReferenceType = false) {
303  ASTContext &Ctx = CGF.getContext();
304 
305  llvm::Value *CastedPtr = CGF.EmitScalarConversion(
306  AddrLV.getAddress().getPointer(), Ctx.getUIntPtrType(),
307  Ctx.getPointerType(DstType), Loc);
308  Address TmpAddr =
309  CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType))
310  .getAddress();
311 
312  // If we are dealing with references we need to return the address of the
313  // reference instead of the reference of the value.
314  if (isReferenceType) {
315  QualType RefType = Ctx.getLValueReferenceType(DstType);
316  llvm::Value *RefVal = TmpAddr.getPointer();
317  TmpAddr = CGF.CreateMemTemp(RefType, Twine(Name, ".ref"));
318  LValue TmpLVal = CGF.MakeAddrLValue(TmpAddr, RefType);
319  CGF.EmitStoreThroughLValue(RValue::get(RefVal), TmpLVal, /*isInit=*/true);
320  }
321 
322  return TmpAddr;
323 }
324 
326  if (T->isLValueReferenceType())
327  return C.getLValueReferenceType(
329  /*SpelledAsLValue=*/false);
330  if (T->isPointerType())
332  if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
333  if (const auto *VLA = dyn_cast<VariableArrayType>(A))
334  return getCanonicalParamType(C, VLA->getElementType());
335  if (!A->isVariablyModifiedType())
336  return C.getCanonicalType(T);
337  }
338  return C.getCanonicalParamType(T);
339 }
340 
341 namespace {
342  /// Contains required data for proper outlined function codegen.
343  struct FunctionOptions {
344  /// Captured statement for which the function is generated.
345  const CapturedStmt *S = nullptr;
346  /// true if cast to/from UIntPtr is required for variables captured by
347  /// value.
348  const bool UIntPtrCastRequired = true;
349  /// true if only casted arguments must be registered as local args or VLA
350  /// sizes.
351  const bool RegisterCastedArgsOnly = false;
352  /// Name of the generated function.
353  const StringRef FunctionName;
354  explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
355  bool RegisterCastedArgsOnly,
356  StringRef FunctionName)
357  : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
358  RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
359  FunctionName(FunctionName) {}
360  };
361 }
362 
363 static llvm::Function *emitOutlinedFunctionPrologue(
364  CodeGenFunction &CGF, FunctionArgList &Args,
365  llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
366  &LocalAddrs,
367  llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
368  &VLASizes,
369  llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
370  const CapturedDecl *CD = FO.S->getCapturedDecl();
371  const RecordDecl *RD = FO.S->getCapturedRecordDecl();
372  assert(CD->hasBody() && "missing CapturedDecl body");
373 
374  CXXThisValue = nullptr;
375  // Build the argument list.
376  CodeGenModule &CGM = CGF.CGM;
377  ASTContext &Ctx = CGM.getContext();
378  FunctionArgList TargetArgs;
379  Args.append(CD->param_begin(),
380  std::next(CD->param_begin(), CD->getContextParamPosition()));
381  TargetArgs.append(
382  CD->param_begin(),
383  std::next(CD->param_begin(), CD->getContextParamPosition()));
384  auto I = FO.S->captures().begin();
385  FunctionDecl *DebugFunctionDecl = nullptr;
386  if (!FO.UIntPtrCastRequired) {
388  DebugFunctionDecl = FunctionDecl::Create(
389  Ctx, Ctx.getTranslationUnitDecl(), FO.S->getLocStart(),
392  Ctx.getFunctionType(Ctx.VoidTy, llvm::None, EPI)),
393  SC_Static, /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/false);
394  }
395  for (const FieldDecl *FD : RD->fields()) {
396  QualType ArgType = FD->getType();
397  IdentifierInfo *II = nullptr;
398  VarDecl *CapVar = nullptr;
399 
400  // If this is a capture by copy and the type is not a pointer, the outlined
401  // function argument type should be uintptr and the value properly casted to
402  // uintptr. This is necessary given that the runtime library is only able to
403  // deal with pointers. We can pass in the same way the VLA type sizes to the
404  // outlined function.
405  if (FO.UIntPtrCastRequired &&
406  ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
407  I->capturesVariableArrayType()))
408  ArgType = Ctx.getUIntPtrType();
409 
410  if (I->capturesVariable() || I->capturesVariableByCopy()) {
411  CapVar = I->getCapturedVar();
412  II = CapVar->getIdentifier();
413  } else if (I->capturesThis()) {
414  II = &Ctx.Idents.get("this");
415  } else {
416  assert(I->capturesVariableArrayType());
417  II = &Ctx.Idents.get("vla");
418  }
419  if (ArgType->isVariablyModifiedType())
420  ArgType = getCanonicalParamType(Ctx, ArgType);
421  VarDecl *Arg;
422  if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
423  Arg = ParmVarDecl::Create(
424  Ctx, DebugFunctionDecl,
425  CapVar ? CapVar->getLocStart() : FD->getLocStart(),
426  CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
427  /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
428  } else {
429  Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
430  II, ArgType, ImplicitParamDecl::Other);
431  }
432  Args.emplace_back(Arg);
433  // Do not cast arguments if we emit function with non-original types.
434  TargetArgs.emplace_back(
435  FO.UIntPtrCastRequired
436  ? Arg
437  : CGM.getOpenMPRuntime().translateParameter(FD, Arg));
438  ++I;
439  }
440  Args.append(
441  std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
442  CD->param_end());
443  TargetArgs.append(
444  std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
445  CD->param_end());
446 
447  // Create the function declaration.
448  const CGFunctionInfo &FuncInfo =
449  CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, TargetArgs);
450  llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
451 
452  auto *F =
454  FO.FunctionName, &CGM.getModule());
455  CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
456  if (CD->isNothrow())
457  F->setDoesNotThrow();
458  F->setDoesNotRecurse();
459 
460  // Generate the function.
461  CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
462  FO.S->getLocStart(), CD->getBody()->getLocStart());
463  unsigned Cnt = CD->getContextParamPosition();
464  I = FO.S->captures().begin();
465  for (const FieldDecl *FD : RD->fields()) {
466  // Do not map arguments if we emit function with non-original types.
467  Address LocalAddr(Address::invalid());
468  if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
469  LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
470  TargetArgs[Cnt]);
471  } else {
472  LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
473  }
474  // If we are capturing a pointer by copy we don't need to do anything, just
475  // use the value that we get from the arguments.
476  if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
477  const VarDecl *CurVD = I->getCapturedVar();
478  // If the variable is a reference we need to materialize it here.
479  if (CurVD->getType()->isReferenceType()) {
480  Address RefAddr = CGF.CreateMemTemp(
481  CurVD->getType(), CGM.getPointerAlign(), ".materialized_ref");
482  CGF.EmitStoreOfScalar(LocalAddr.getPointer(), RefAddr,
483  /*Volatile=*/false, CurVD->getType());
484  LocalAddr = RefAddr;
485  }
486  if (!FO.RegisterCastedArgsOnly)
487  LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
488  ++Cnt;
489  ++I;
490  continue;
491  }
492 
493  LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
495  if (FD->hasCapturedVLAType()) {
496  if (FO.UIntPtrCastRequired) {
497  ArgLVal = CGF.MakeAddrLValue(
498  castValueFromUintptr(CGF, I->getLocation(), FD->getType(),
499  Args[Cnt]->getName(), ArgLVal),
501  }
502  llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
503  const VariableArrayType *VAT = FD->getCapturedVLAType();
504  VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg);
505  } else if (I->capturesVariable()) {
506  const VarDecl *Var = I->getCapturedVar();
507  QualType VarTy = Var->getType();
508  Address ArgAddr = ArgLVal.getAddress();
509  if (!VarTy->isReferenceType()) {
510  if (ArgLVal.getType()->isLValueReferenceType()) {
511  ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
512  } else if (!VarTy->isVariablyModifiedType() ||
513  !VarTy->isPointerType()) {
514  assert(ArgLVal.getType()->isPointerType());
515  ArgAddr = CGF.EmitLoadOfPointer(
516  ArgAddr, ArgLVal.getType()->castAs<PointerType>());
517  }
518  }
519  if (!FO.RegisterCastedArgsOnly) {
520  LocalAddrs.insert(
521  {Args[Cnt],
522  {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}});
523  }
524  } else if (I->capturesVariableByCopy()) {
525  assert(!FD->getType()->isAnyPointerType() &&
526  "Not expecting a captured pointer.");
527  const VarDecl *Var = I->getCapturedVar();
528  QualType VarTy = Var->getType();
529  LocalAddrs.insert(
530  {Args[Cnt],
531  {Var, FO.UIntPtrCastRequired
532  ? castValueFromUintptr(CGF, I->getLocation(),
533  FD->getType(), Args[Cnt]->getName(),
534  ArgLVal, VarTy->isReferenceType())
535  : ArgLVal.getAddress()}});
536  } else {
537  // If 'this' is captured, load it into CXXThisValue.
538  assert(I->capturesThis());
539  CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
540  LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress()}});
541  }
542  ++Cnt;
543  ++I;
544  }
545 
546  return F;
547 }
548 
549 llvm::Function *
551  assert(
552  CapturedStmtInfo &&
553  "CapturedStmtInfo should be set when generating the captured function");
554  const CapturedDecl *CD = S.getCapturedDecl();
555  // Build the argument list.
556  bool NeedWrapperFunction =
557  getDebugInfo() &&
558  CGM.getCodeGenOpts().getDebugInfo() >= codegenoptions::LimitedDebugInfo;
559  FunctionArgList Args;
560  llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
561  llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
562  SmallString<256> Buffer;
563  llvm::raw_svector_ostream Out(Buffer);
564  Out << CapturedStmtInfo->getHelperName();
565  if (NeedWrapperFunction)
566  Out << "_debug__";
567  FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
568  Out.str());
569  llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs,
570  VLASizes, CXXThisValue, FO);
571  for (const auto &LocalAddrPair : LocalAddrs) {
572  if (LocalAddrPair.second.first) {
573  setAddrOfLocalVar(LocalAddrPair.second.first,
574  LocalAddrPair.second.second);
575  }
576  }
577  for (const auto &VLASizePair : VLASizes)
578  VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
579  PGO.assignRegionCounters(GlobalDecl(CD), F);
580  CapturedStmtInfo->EmitBody(*this, CD->getBody());
581  FinishFunction(CD->getBodyRBrace());
582  if (!NeedWrapperFunction)
583  return F;
584 
585  FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
586  /*RegisterCastedArgsOnly=*/true,
587  CapturedStmtInfo->getHelperName());
588  CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
589  WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
590  Args.clear();
591  LocalAddrs.clear();
592  VLASizes.clear();
593  llvm::Function *WrapperF =
594  emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
595  WrapperCGF.CXXThisValue, WrapperFO);
597  for (const auto *Arg : Args) {
599  auto I = LocalAddrs.find(Arg);
600  if (I != LocalAddrs.end()) {
601  LValue LV = WrapperCGF.MakeAddrLValue(
602  I->second.second,
603  I->second.first ? I->second.first->getType() : Arg->getType(),
605  CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getLocStart());
606  } else {
607  auto EI = VLASizes.find(Arg);
608  if (EI != VLASizes.end()) {
609  CallArg = EI->second.second;
610  } else {
611  LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
612  Arg->getType(),
614  CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getLocStart());
615  }
616  }
617  CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
618  }
619  CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.getLocStart(),
620  F, CallArgs);
621  WrapperCGF.FinishFunction();
622  return WrapperF;
623 }
624 
625 //===----------------------------------------------------------------------===//
626 // OpenMP Directive Emission
627 //===----------------------------------------------------------------------===//
629  Address DestAddr, Address SrcAddr, QualType OriginalType,
630  const llvm::function_ref<void(Address, Address)> CopyGen) {
631  // Perform element-by-element initialization.
632  QualType ElementTy;
633 
634  // Drill down to the base element type on both arrays.
635  const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
636  llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
637  SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
638 
639  llvm::Value *SrcBegin = SrcAddr.getPointer();
640  llvm::Value *DestBegin = DestAddr.getPointer();
641  // Cast from pointer to array type to pointer to single element.
642  llvm::Value *DestEnd = Builder.CreateGEP(DestBegin, NumElements);
643  // The basic structure here is a while-do loop.
644  llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body");
645  llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done");
646  llvm::Value *IsEmpty =
647  Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
648  Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
649 
650  // Enter the loop body, making that address the current address.
651  llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
652  EmitBlock(BodyBB);
653 
654  CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
655 
656  llvm::PHINode *SrcElementPHI =
657  Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
658  SrcElementPHI->addIncoming(SrcBegin, EntryBB);
659  Address SrcElementCurrent =
660  Address(SrcElementPHI,
661  SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
662 
663  llvm::PHINode *DestElementPHI =
664  Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
665  DestElementPHI->addIncoming(DestBegin, EntryBB);
666  Address DestElementCurrent =
667  Address(DestElementPHI,
668  DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
669 
670  // Emit copy.
671  CopyGen(DestElementCurrent, SrcElementCurrent);
672 
673  // Shift the address forward by one element.
674  llvm::Value *DestElementNext = Builder.CreateConstGEP1_32(
675  DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
676  llvm::Value *SrcElementNext = Builder.CreateConstGEP1_32(
677  SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
678  // Check whether we've reached the end.
679  llvm::Value *Done =
680  Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
681  Builder.CreateCondBr(Done, DoneBB, BodyBB);
682  DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
683  SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
684 
685  // Done.
686  EmitBlock(DoneBB, /*IsFinished=*/true);
687 }
688 
689 void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
690  Address SrcAddr, const VarDecl *DestVD,
691  const VarDecl *SrcVD, const Expr *Copy) {
692  if (OriginalType->isArrayType()) {
693  const auto *BO = dyn_cast<BinaryOperator>(Copy);
694  if (BO && BO->getOpcode() == BO_Assign) {
695  // Perform simple memcpy for simple copying.
696  LValue Dest = MakeAddrLValue(DestAddr, OriginalType);
697  LValue Src = MakeAddrLValue(SrcAddr, OriginalType);
698  EmitAggregateAssign(Dest, Src, OriginalType);
699  } else {
700  // For arrays with complex element types perform element by element
701  // copying.
702  EmitOMPAggregateAssign(
703  DestAddr, SrcAddr, OriginalType,
704  [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
705  // Working with the single array element, so have to remap
706  // destination and source variables to corresponding array
707  // elements.
709  Remap.addPrivate(DestVD, [DestElement]() { return DestElement; });
710  Remap.addPrivate(SrcVD, [SrcElement]() { return SrcElement; });
711  (void)Remap.Privatize();
712  EmitIgnoredExpr(Copy);
713  });
714  }
715  } else {
716  // Remap pseudo source variable to private copy.
718  Remap.addPrivate(SrcVD, [SrcAddr]() { return SrcAddr; });
719  Remap.addPrivate(DestVD, [DestAddr]() { return DestAddr; });
720  (void)Remap.Privatize();
721  // Emit copying of the whole variable.
722  EmitIgnoredExpr(Copy);
723  }
724 }
725 
727  OMPPrivateScope &PrivateScope) {
728  if (!HaveInsertPoint())
729  return false;
730  bool FirstprivateIsLastprivate = false;
731  llvm::DenseSet<const VarDecl *> Lastprivates;
732  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
733  for (const auto *D : C->varlists())
734  Lastprivates.insert(
735  cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl());
736  }
737  llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
739  getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind());
740  // Force emission of the firstprivate copy if the directive does not emit
741  // outlined function, like omp for, omp simd, omp distribute etc.
742  bool MustEmitFirstprivateCopy =
743  CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
744  for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
745  auto IRef = C->varlist_begin();
746  auto InitsRef = C->inits().begin();
747  for (const Expr *IInit : C->private_copies()) {
748  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
749  bool ThisFirstprivateIsLastprivate =
750  Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
751  const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
752  if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
753  !FD->getType()->isReferenceType()) {
754  EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
755  ++IRef;
756  ++InitsRef;
757  continue;
758  }
759  FirstprivateIsLastprivate =
760  FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
761  if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
762  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
763  const auto *VDInit =
764  cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
765  bool IsRegistered;
766  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
767  /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
768  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
769  LValue OriginalLVal = EmitLValue(&DRE);
770  QualType Type = VD->getType();
771  if (Type->isArrayType()) {
772  // Emit VarDecl with copy init for arrays.
773  // Get the address of the original variable captured in current
774  // captured region.
775  IsRegistered = PrivateScope.addPrivate(
776  OrigVD, [this, VD, Type, OriginalLVal, VDInit]() {
777  AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
778  const Expr *Init = VD->getInit();
779  if (!isa<CXXConstructExpr>(Init) ||
780  isTrivialInitializer(Init)) {
781  // Perform simple memcpy.
782  LValue Dest =
783  MakeAddrLValue(Emission.getAllocatedAddress(), Type);
784  EmitAggregateAssign(Dest, OriginalLVal, Type);
785  } else {
786  EmitOMPAggregateAssign(
787  Emission.getAllocatedAddress(), OriginalLVal.getAddress(),
788  Type,
789  [this, VDInit, Init](Address DestElement,
790  Address SrcElement) {
791  // Clean up any temporaries needed by the
792  // initialization.
793  RunCleanupsScope InitScope(*this);
794  // Emit initialization for single element.
795  setAddrOfLocalVar(VDInit, SrcElement);
796  EmitAnyExprToMem(Init, DestElement,
797  Init->getType().getQualifiers(),
798  /*IsInitializer*/ false);
799  LocalDeclMap.erase(VDInit);
800  });
801  }
802  EmitAutoVarCleanups(Emission);
803  return Emission.getAllocatedAddress();
804  });
805  } else {
806  Address OriginalAddr = OriginalLVal.getAddress();
807  IsRegistered = PrivateScope.addPrivate(
808  OrigVD, [this, VDInit, OriginalAddr, VD]() {
809  // Emit private VarDecl with copy init.
810  // Remap temp VDInit variable to the address of the original
811  // variable (for proper handling of captured global variables).
812  setAddrOfLocalVar(VDInit, OriginalAddr);
813  EmitDecl(*VD);
814  LocalDeclMap.erase(VDInit);
815  return GetAddrOfLocalVar(VD);
816  });
817  }
818  assert(IsRegistered &&
819  "firstprivate var already registered as private");
820  // Silence the warning about unused variable.
821  (void)IsRegistered;
822  }
823  ++IRef;
824  ++InitsRef;
825  }
826  }
827  return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
828 }
829 
831  const OMPExecutableDirective &D,
832  CodeGenFunction::OMPPrivateScope &PrivateScope) {
833  if (!HaveInsertPoint())
834  return;
835  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
836  for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
837  auto IRef = C->varlist_begin();
838  for (const Expr *IInit : C->private_copies()) {
839  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
840  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
841  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
842  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
843  // Emit private VarDecl with copy init.
844  EmitDecl(*VD);
845  return GetAddrOfLocalVar(VD);
846  });
847  assert(IsRegistered && "private var already registered as private");
848  // Silence the warning about unused variable.
849  (void)IsRegistered;
850  }
851  ++IRef;
852  }
853  }
854 }
855 
857  if (!HaveInsertPoint())
858  return false;
859  // threadprivate_var1 = master_threadprivate_var1;
860  // operator=(threadprivate_var2, master_threadprivate_var2);
861  // ...
862  // __kmpc_barrier(&loc, global_tid);
864  llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
865  for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
866  auto IRef = C->varlist_begin();
867  auto ISrcRef = C->source_exprs().begin();
868  auto IDestRef = C->destination_exprs().begin();
869  for (const Expr *AssignOp : C->assignment_ops()) {
870  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
871  QualType Type = VD->getType();
872  if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
873  // Get the address of the master variable. If we are emitting code with
874  // TLS support, the address is passed from the master as field in the
875  // captured declaration.
876  Address MasterAddr = Address::invalid();
877  if (getLangOpts().OpenMPUseTLS &&
878  getContext().getTargetInfo().isTLSSupported()) {
879  assert(CapturedStmtInfo->lookup(VD) &&
880  "Copyin threadprivates should have been captured!");
881  DeclRefExpr DRE(const_cast<VarDecl *>(VD), true, (*IRef)->getType(),
882  VK_LValue, (*IRef)->getExprLoc());
883  MasterAddr = EmitLValue(&DRE).getAddress();
884  LocalDeclMap.erase(VD);
885  } else {
886  MasterAddr =
887  Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
888  : CGM.GetAddrOfGlobal(VD),
889  getContext().getDeclAlign(VD));
890  }
891  // Get the address of the threadprivate variable.
892  Address PrivateAddr = EmitLValue(*IRef).getAddress();
893  if (CopiedVars.size() == 1) {
894  // At first check if current thread is a master thread. If it is, no
895  // need to copy data.
896  CopyBegin = createBasicBlock("copyin.not.master");
897  CopyEnd = createBasicBlock("copyin.not.master.end");
898  Builder.CreateCondBr(
899  Builder.CreateICmpNE(
900  Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy),
901  Builder.CreatePtrToInt(PrivateAddr.getPointer(),
902  CGM.IntPtrTy)),
903  CopyBegin, CopyEnd);
904  EmitBlock(CopyBegin);
905  }
906  const auto *SrcVD =
907  cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
908  const auto *DestVD =
909  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
910  EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
911  }
912  ++IRef;
913  ++ISrcRef;
914  ++IDestRef;
915  }
916  }
917  if (CopyEnd) {
918  // Exit out of copying procedure for non-master thread.
919  EmitBlock(CopyEnd, /*IsFinished=*/true);
920  return true;
921  }
922  return false;
923 }
924 
926  const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
927  if (!HaveInsertPoint())
928  return false;
929  bool HasAtLeastOneLastprivate = false;
932  const auto *LoopDirective = cast<OMPLoopDirective>(&D);
933  for (const Expr *C : LoopDirective->counters()) {
934  SIMDLCVs.insert(
935  cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
936  }
937  }
938  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
939  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
940  HasAtLeastOneLastprivate = true;
942  !getLangOpts().OpenMPSimd)
943  break;
944  auto IRef = C->varlist_begin();
945  auto IDestRef = C->destination_exprs().begin();
946  for (const Expr *IInit : C->private_copies()) {
947  // Keep the address of the original variable for future update at the end
948  // of the loop.
949  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
950  // Taskloops do not require additional initialization, it is done in
951  // runtime support library.
952  if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
953  const auto *DestVD =
954  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
955  PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() {
956  DeclRefExpr DRE(
957  const_cast<VarDecl *>(OrigVD),
958  /*RefersToEnclosingVariableOrCapture=*/CapturedStmtInfo->lookup(
959  OrigVD) != nullptr,
960  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
961  return EmitLValue(&DRE).getAddress();
962  });
963  // Check if the variable is also a firstprivate: in this case IInit is
964  // not generated. Initialization of this variable will happen in codegen
965  // for 'firstprivate' clause.
966  if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
967  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
968  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
969  // Emit private VarDecl with copy init.
970  EmitDecl(*VD);
971  return GetAddrOfLocalVar(VD);
972  });
973  assert(IsRegistered &&
974  "lastprivate var already registered as private");
975  (void)IsRegistered;
976  }
977  }
978  ++IRef;
979  ++IDestRef;
980  }
981  }
982  return HasAtLeastOneLastprivate;
983 }
984 
986  const OMPExecutableDirective &D, bool NoFinals,
987  llvm::Value *IsLastIterCond) {
988  if (!HaveInsertPoint())
989  return;
990  // Emit following code:
991  // if (<IsLastIterCond>) {
992  // orig_var1 = private_orig_var1;
993  // ...
994  // orig_varn = private_orig_varn;
995  // }
996  llvm::BasicBlock *ThenBB = nullptr;
997  llvm::BasicBlock *DoneBB = nullptr;
998  if (IsLastIterCond) {
999  ThenBB = createBasicBlock(".omp.lastprivate.then");
1000  DoneBB = createBasicBlock(".omp.lastprivate.done");
1001  Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1002  EmitBlock(ThenBB);
1003  }
1004  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1005  llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1006  if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1007  auto IC = LoopDirective->counters().begin();
1008  for (const Expr *F : LoopDirective->finals()) {
1009  const auto *D =
1010  cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1011  if (NoFinals)
1012  AlreadyEmittedVars.insert(D);
1013  else
1014  LoopCountersAndUpdates[D] = F;
1015  ++IC;
1016  }
1017  }
1018  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1019  auto IRef = C->varlist_begin();
1020  auto ISrcRef = C->source_exprs().begin();
1021  auto IDestRef = C->destination_exprs().begin();
1022  for (const Expr *AssignOp : C->assignment_ops()) {
1023  const auto *PrivateVD =
1024  cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1025  QualType Type = PrivateVD->getType();
1026  const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1027  if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1028  // If lastprivate variable is a loop control variable for loop-based
1029  // directive, update its value before copyin back to original
1030  // variable.
1031  if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1032  EmitIgnoredExpr(FinalExpr);
1033  const auto *SrcVD =
1034  cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1035  const auto *DestVD =
1036  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1037  // Get the address of the original variable.
1038  Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1039  // Get the address of the private variable.
1040  Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
1041  if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1042  PrivateAddr =
1043  Address(Builder.CreateLoad(PrivateAddr),
1044  getNaturalTypeAlignment(RefTy->getPointeeType()));
1045  EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1046  }
1047  ++IRef;
1048  ++ISrcRef;
1049  ++IDestRef;
1050  }
1051  if (const Expr *PostUpdate = C->getPostUpdateExpr())
1052  EmitIgnoredExpr(PostUpdate);
1053  }
1054  if (IsLastIterCond)
1055  EmitBlock(DoneBB, /*IsFinished=*/true);
1056 }
1057 
1059  const OMPExecutableDirective &D,
1060  CodeGenFunction::OMPPrivateScope &PrivateScope) {
1061  if (!HaveInsertPoint())
1062  return;
1065  SmallVector<const Expr *, 4> ReductionOps;
1068  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1069  auto IPriv = C->privates().begin();
1070  auto IRed = C->reduction_ops().begin();
1071  auto ILHS = C->lhs_exprs().begin();
1072  auto IRHS = C->rhs_exprs().begin();
1073  for (const Expr *Ref : C->varlists()) {
1074  Shareds.emplace_back(Ref);
1075  Privates.emplace_back(*IPriv);
1076  ReductionOps.emplace_back(*IRed);
1077  LHSs.emplace_back(*ILHS);
1078  RHSs.emplace_back(*IRHS);
1079  std::advance(IPriv, 1);
1080  std::advance(IRed, 1);
1081  std::advance(ILHS, 1);
1082  std::advance(IRHS, 1);
1083  }
1084  }
1085  ReductionCodeGen RedCG(Shareds, Privates, ReductionOps);
1086  unsigned Count = 0;
1087  auto ILHS = LHSs.begin();
1088  auto IRHS = RHSs.begin();
1089  auto IPriv = Privates.begin();
1090  for (const Expr *IRef : Shareds) {
1091  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1092  // Emit private VarDecl with reduction init.
1093  RedCG.emitSharedLValue(*this, Count);
1094  RedCG.emitAggregateType(*this, Count);
1095  AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);
1096  RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
1097  RedCG.getSharedLValue(Count),
1098  [&Emission](CodeGenFunction &CGF) {
1099  CGF.EmitAutoVarInit(Emission);
1100  return true;
1101  });
1102  EmitAutoVarCleanups(Emission);
1103  Address BaseAddr = RedCG.adjustPrivateAddress(
1104  *this, Count, Emission.getAllocatedAddress());
1105  bool IsRegistered = PrivateScope.addPrivate(
1106  RedCG.getBaseDecl(Count), [BaseAddr]() { return BaseAddr; });
1107  assert(IsRegistered && "private var already registered as private");
1108  // Silence the warning about unused variable.
1109  (void)IsRegistered;
1110 
1111  const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1112  const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1113  QualType Type = PrivateVD->getType();
1114  bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1115  if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1116  // Store the address of the original variable associated with the LHS
1117  // implicit variable.
1118  PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
1119  return RedCG.getSharedLValue(Count).getAddress();
1120  });
1121  PrivateScope.addPrivate(
1122  RHSVD, [this, PrivateVD]() { return GetAddrOfLocalVar(PrivateVD); });
1123  } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1124  isa<ArraySubscriptExpr>(IRef)) {
1125  // Store the address of the original variable associated with the LHS
1126  // implicit variable.
1127  PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
1128  return RedCG.getSharedLValue(Count).getAddress();
1129  });
1130  PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() {
1131  return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD),
1132  ConvertTypeForMem(RHSVD->getType()),
1133  "rhs.begin");
1134  });
1135  } else {
1136  QualType Type = PrivateVD->getType();
1137  bool IsArray = getContext().getAsArrayType(Type) != nullptr;
1138  Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress();
1139  // Store the address of the original variable associated with the LHS
1140  // implicit variable.
1141  if (IsArray) {
1142  OriginalAddr = Builder.CreateElementBitCast(
1143  OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin");
1144  }
1145  PrivateScope.addPrivate(LHSVD, [OriginalAddr]() { return OriginalAddr; });
1146  PrivateScope.addPrivate(
1147  RHSVD, [this, PrivateVD, RHSVD, IsArray]() {
1148  return IsArray
1149  ? Builder.CreateElementBitCast(
1150  GetAddrOfLocalVar(PrivateVD),
1151  ConvertTypeForMem(RHSVD->getType()), "rhs.begin")
1152  : GetAddrOfLocalVar(PrivateVD);
1153  });
1154  }
1155  ++ILHS;
1156  ++IRHS;
1157  ++IPriv;
1158  ++Count;
1159  }
1160 }
1161 
1163  const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1164  if (!HaveInsertPoint())
1165  return;
1170  bool HasAtLeastOneReduction = false;
1171  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1172  HasAtLeastOneReduction = true;
1173  Privates.append(C->privates().begin(), C->privates().end());
1174  LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1175  RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1176  ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1177  }
1178  if (HasAtLeastOneReduction) {
1179  bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1181  ReductionKind == OMPD_simd;
1182  bool SimpleReduction = ReductionKind == OMPD_simd;
1183  // Emit nowait reduction if nowait clause is present or directive is a
1184  // parallel directive (it always has implicit barrier).
1185  CGM.getOpenMPRuntime().emitReduction(
1186  *this, D.getLocEnd(), Privates, LHSExprs, RHSExprs, ReductionOps,
1187  {WithNowait, SimpleReduction, ReductionKind});
1188  }
1189 }
1190 
1192  CodeGenFunction &CGF, const OMPExecutableDirective &D,
1193  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1194  if (!CGF.HaveInsertPoint())
1195  return;
1196  llvm::BasicBlock *DoneBB = nullptr;
1197  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1198  if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
1199  if (!DoneBB) {
1200  if (llvm::Value *Cond = CondGen(CGF)) {
1201  // If the first post-update expression is found, emit conditional
1202  // block if it was requested.
1203  llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1204  DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1205  CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1206  CGF.EmitBlock(ThenBB);
1207  }
1208  }
1209  CGF.EmitIgnoredExpr(PostUpdate);
1210  }
1211  }
1212  if (DoneBB)
1213  CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1214 }
1215 
1216 namespace {
1217 /// Codegen lambda for appending distribute lower and upper bounds to outlined
1218 /// parallel function. This is necessary for combined constructs such as
1219 /// 'distribute parallel for'
1220 typedef llvm::function_ref<void(CodeGenFunction &,
1221  const OMPExecutableDirective &,
1223  CodeGenBoundParametersTy;
1224 } // anonymous namespace
1225 
1227  CodeGenFunction &CGF, const OMPExecutableDirective &S,
1228  OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1229  const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1230  const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1231  llvm::Value *OutlinedFn =
1232  CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
1233  S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
1234  if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
1235  CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1236  llvm::Value *NumThreads =
1237  CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1238  /*IgnoreResultAssign=*/true);
1239  CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
1240  CGF, NumThreads, NumThreadsClause->getLocStart());
1241  }
1242  if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
1243  CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1244  CGF.CGM.getOpenMPRuntime().emitProcBindClause(
1245  CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getLocStart());
1246  }
1247  const Expr *IfCond = nullptr;
1248  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1249  if (C->getNameModifier() == OMPD_unknown ||
1250  C->getNameModifier() == OMPD_parallel) {
1251  IfCond = C->getCondition();
1252  break;
1253  }
1254  }
1255 
1256  OMPParallelScope Scope(CGF, S);
1258  // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
1259  // lower and upper bounds with the pragma 'for' chunking mechanism.
1260  // The following lambda takes care of appending the lower and upper bound
1261  // parameters when necessary
1262  CodeGenBoundParameters(CGF, S, CapturedVars);
1263  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
1264  CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getLocStart(), OutlinedFn,
1265  CapturedVars, IfCond);
1266 }
1267 
1269  const OMPExecutableDirective &,
1271 
1273  // Emit parallel region as a standalone region.
1274  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1275  Action.Enter(CGF);
1276  OMPPrivateScope PrivateScope(CGF);
1277  bool Copyins = CGF.EmitOMPCopyinClause(S);
1278  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1279  if (Copyins) {
1280  // Emit implicit barrier to synchronize threads and avoid data races on
1281  // propagation master's thread values of threadprivate variables to local
1282  // instances of that variables of all other implicit threads.
1283  CGF.CGM.getOpenMPRuntime().emitBarrierCall(
1284  CGF, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
1285  /*ForceSimpleCall=*/true);
1286  }
1287  CGF.EmitOMPPrivateClause(S, PrivateScope);
1288  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1289  (void)PrivateScope.Privatize();
1290  CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1291  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
1292  };
1293  emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
1296  [](CodeGenFunction &) { return nullptr; });
1297 }
1298 
1300  JumpDest LoopExit) {
1301  RunCleanupsScope BodyScope(*this);
1302  // Update counters values on current iteration.
1303  for (const Expr *UE : D.updates())
1304  EmitIgnoredExpr(UE);
1305  // Update the linear variables.
1306  // In distribute directives only loop counters may be marked as linear, no
1307  // need to generate the code for them.
1309  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1310  for (const Expr *UE : C->updates())
1311  EmitIgnoredExpr(UE);
1312  }
1313  }
1314 
1315  // On a continue in the body, jump to the end.
1316  JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue");
1317  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1318  // Emit loop body.
1319  EmitStmt(D.getBody());
1320  // The end (updates/cleanups).
1321  EmitBlock(Continue.getBlock());
1322  BreakContinueStack.pop_back();
1323 }
1324 
1326  const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
1327  const Expr *IncExpr,
1328  const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
1329  const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
1330  auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
1331 
1332  // Start the loop with a block that tests the condition.
1333  auto CondBlock = createBasicBlock("omp.inner.for.cond");
1334  EmitBlock(CondBlock);
1335  const SourceRange R = S.getSourceRange();
1336  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1337  SourceLocToDebugLoc(R.getEnd()));
1338 
1339  // If there are any cleanups between here and the loop-exit scope,
1340  // create a block to stage a loop exit along.
1341  llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1342  if (RequiresCleanup)
1343  ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
1344 
1345  llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body");
1346 
1347  // Emit condition.
1348  EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
1349  if (ExitBlock != LoopExit.getBlock()) {
1350  EmitBlock(ExitBlock);
1351  EmitBranchThroughCleanup(LoopExit);
1352  }
1353 
1354  EmitBlock(LoopBody);
1355  incrementProfileCounter(&S);
1356 
1357  // Create a block for the increment.
1358  JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
1359  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1360 
1361  BodyGen(*this);
1362 
1363  // Emit "IV = IV + 1" and a back-edge to the condition block.
1364  EmitBlock(Continue.getBlock());
1365  EmitIgnoredExpr(IncExpr);
1366  PostIncGen(*this);
1367  BreakContinueStack.pop_back();
1368  EmitBranch(CondBlock);
1369  LoopStack.pop();
1370  // Emit the fall-through block.
1371  EmitBlock(LoopExit.getBlock());
1372 }
1373 
1375  if (!HaveInsertPoint())
1376  return false;
1377  // Emit inits for the linear variables.
1378  bool HasLinears = false;
1379  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1380  for (const Expr *Init : C->inits()) {
1381  HasLinears = true;
1382  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
1383  if (const auto *Ref =
1384  dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
1385  AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
1386  const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
1387  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
1388  CapturedStmtInfo->lookup(OrigVD) != nullptr,
1389  VD->getInit()->getType(), VK_LValue,
1390  VD->getInit()->getExprLoc());
1391  EmitExprAsInit(&DRE, VD, MakeAddrLValue(Emission.getAllocatedAddress(),
1392  VD->getType()),
1393  /*capturedByInit=*/false);
1394  EmitAutoVarCleanups(Emission);
1395  } else {
1396  EmitVarDecl(*VD);
1397  }
1398  }
1399  // Emit the linear steps for the linear clauses.
1400  // If a step is not constant, it is pre-calculated before the loop.
1401  if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
1402  if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
1403  EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
1404  // Emit calculation of the linear step.
1405  EmitIgnoredExpr(CS);
1406  }
1407  }
1408  return HasLinears;
1409 }
1410 
1412  const OMPLoopDirective &D,
1413  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1414  if (!HaveInsertPoint())
1415  return;
1416  llvm::BasicBlock *DoneBB = nullptr;
1417  // Emit the final values of the linear variables.
1418  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1419  auto IC = C->varlist_begin();
1420  for (const Expr *F : C->finals()) {
1421  if (!DoneBB) {
1422  if (llvm::Value *Cond = CondGen(*this)) {
1423  // If the first post-update expression is found, emit conditional
1424  // block if it was requested.
1425  llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu");
1426  DoneBB = createBasicBlock(".omp.linear.pu.done");
1427  Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1428  EmitBlock(ThenBB);
1429  }
1430  }
1431  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
1432  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
1433  CapturedStmtInfo->lookup(OrigVD) != nullptr,
1434  (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
1435  Address OrigAddr = EmitLValue(&DRE).getAddress();
1436  CodeGenFunction::OMPPrivateScope VarScope(*this);
1437  VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1438  (void)VarScope.Privatize();
1439  EmitIgnoredExpr(F);
1440  ++IC;
1441  }
1442  if (const Expr *PostUpdate = C->getPostUpdateExpr())
1443  EmitIgnoredExpr(PostUpdate);
1444  }
1445  if (DoneBB)
1446  EmitBlock(DoneBB, /*IsFinished=*/true);
1447 }
1448 
1450  const OMPExecutableDirective &D) {
1451  if (!CGF.HaveInsertPoint())
1452  return;
1453  for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
1454  unsigned ClauseAlignment = 0;
1455  if (const Expr *AlignmentExpr = Clause->getAlignment()) {
1456  auto *AlignmentCI =
1457  cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
1458  ClauseAlignment = static_cast<unsigned>(AlignmentCI->getZExtValue());
1459  }
1460  for (const Expr *E : Clause->varlists()) {
1461  unsigned Alignment = ClauseAlignment;
1462  if (Alignment == 0) {
1463  // OpenMP [2.8.1, Description]
1464  // If no optional parameter is specified, implementation-defined default
1465  // alignments for SIMD instructions on the target platforms are assumed.
1466  Alignment =
1467  CGF.getContext()
1469  E->getType()->getPointeeType()))
1470  .getQuantity();
1471  }
1472  assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) &&
1473  "alignment is not power of 2");
1474  if (Alignment != 0) {
1475  llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
1476  CGF.EmitAlignmentAssumption(PtrValue, Alignment);
1477  }
1478  }
1479  }
1480 }
1481 
1483  const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
1484  if (!HaveInsertPoint())
1485  return;
1486  auto I = S.private_counters().begin();
1487  for (const Expr *E : S.counters()) {
1488  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1489  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
1490  // Emit var without initialization.
1491  AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD);
1492  EmitAutoVarCleanups(VarEmission);
1493  LocalDeclMap.erase(PrivateVD);
1494  (void)LoopScope.addPrivate(VD, [&VarEmission]() {
1495  return VarEmission.getAllocatedAddress();
1496  });
1497  if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
1498  VD->hasGlobalStorage()) {
1499  (void)LoopScope.addPrivate(PrivateVD, [this, VD, E]() {
1500  DeclRefExpr DRE(const_cast<VarDecl *>(VD),
1501  LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
1502  E->getType(), VK_LValue, E->getExprLoc());
1503  return EmitLValue(&DRE).getAddress();
1504  });
1505  } else {
1506  (void)LoopScope.addPrivate(PrivateVD, [&VarEmission]() {
1507  return VarEmission.getAllocatedAddress();
1508  });
1509  }
1510  ++I;
1511  }
1512 }
1513 
1514 static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
1515  const Expr *Cond, llvm::BasicBlock *TrueBlock,
1516  llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
1517  if (!CGF.HaveInsertPoint())
1518  return;
1519  {
1520  CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
1521  CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
1522  (void)PreCondScope.Privatize();
1523  // Get initial values of real counters.
1524  for (const Expr *I : S.inits()) {
1525  CGF.EmitIgnoredExpr(I);
1526  }
1527  }
1528  // Check that loop is executed at least one time.
1529  CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
1530 }
1531 
1533  const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
1534  if (!HaveInsertPoint())
1535  return;
1538  const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1539  for (const Expr *C : LoopDirective->counters()) {
1540  SIMDLCVs.insert(
1541  cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
1542  }
1543  }
1544  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1545  auto CurPrivate = C->privates().begin();
1546  for (const Expr *E : C->varlists()) {
1547  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1548  const auto *PrivateVD =
1549  cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
1550  if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
1551  bool IsRegistered = PrivateScope.addPrivate(VD, [this, PrivateVD]() {
1552  // Emit private VarDecl with copy init.
1553  EmitVarDecl(*PrivateVD);
1554  return GetAddrOfLocalVar(PrivateVD);
1555  });
1556  assert(IsRegistered && "linear var already registered as private");
1557  // Silence the warning about unused variable.
1558  (void)IsRegistered;
1559  } else {
1560  EmitVarDecl(*PrivateVD);
1561  }
1562  ++CurPrivate;
1563  }
1564  }
1565 }
1566 
1568  const OMPExecutableDirective &D,
1569  bool IsMonotonic) {
1570  if (!CGF.HaveInsertPoint())
1571  return;
1572  if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
1573  RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
1574  /*ignoreResult=*/true);
1575  auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1576  CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1577  // In presence of finite 'safelen', it may be unsafe to mark all
1578  // the memory instructions parallel, because loop-carried
1579  // dependences of 'safelen' iterations are possible.
1580  if (!IsMonotonic)
1582  } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
1583  RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
1584  /*ignoreResult=*/true);
1585  auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1586  CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1587  // In presence of finite 'safelen', it may be unsafe to mark all
1588  // the memory instructions parallel, because loop-carried
1589  // dependences of 'safelen' iterations are possible.
1590  CGF.LoopStack.setParallel(/*Enable=*/false);
1591  }
1592 }
1593 
1595  bool IsMonotonic) {
1596  // Walk clauses and process safelen/lastprivate.
1597  LoopStack.setParallel(!IsMonotonic);
1598  LoopStack.setVectorizeEnable();
1599  emitSimdlenSafelenClause(*this, D, IsMonotonic);
1600 }
1601 
1603  const OMPLoopDirective &D,
1604  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1605  if (!HaveInsertPoint())
1606  return;
1607  llvm::BasicBlock *DoneBB = nullptr;
1608  auto IC = D.counters().begin();
1609  auto IPC = D.private_counters().begin();
1610  for (const Expr *F : D.finals()) {
1611  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
1612  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
1613  const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
1614  if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
1615  OrigVD->hasGlobalStorage() || CED) {
1616  if (!DoneBB) {
1617  if (llvm::Value *Cond = CondGen(*this)) {
1618  // If the first post-update expression is found, emit conditional
1619  // block if it was requested.
1620  llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then");
1621  DoneBB = createBasicBlock(".omp.final.done");
1622  Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1623  EmitBlock(ThenBB);
1624  }
1625  }
1626  Address OrigAddr = Address::invalid();
1627  if (CED) {
1628  OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();
1629  } else {
1630  DeclRefExpr DRE(const_cast<VarDecl *>(PrivateVD),
1631  /*RefersToEnclosingVariableOrCapture=*/false,
1632  (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
1633  OrigAddr = EmitLValue(&DRE).getAddress();
1634  }
1635  OMPPrivateScope VarScope(*this);
1636  VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1637  (void)VarScope.Privatize();
1638  EmitIgnoredExpr(F);
1639  }
1640  ++IC;
1641  ++IPC;
1642  }
1643  if (DoneBB)
1644  EmitBlock(DoneBB, /*IsFinished=*/true);
1645 }
1646 
1648  const OMPLoopDirective &S,
1650  CGF.EmitOMPLoopBody(S, LoopExit);
1651  CGF.EmitStopPoint(&S);
1652 }
1653 
1654 /// Emit a helper variable and return corresponding lvalue.
1656  const DeclRefExpr *Helper) {
1657  auto VDecl = cast<VarDecl>(Helper->getDecl());
1658  CGF.EmitVarDecl(*VDecl);
1659  return CGF.EmitLValue(Helper);
1660 }
1661 
1663  PrePostActionTy &Action) {
1664  Action.Enter(CGF);
1666  "Expected simd directive");
1667  OMPLoopScope PreInitScope(CGF, S);
1668  // if (PreCond) {
1669  // for (IV in 0..LastIteration) BODY;
1670  // <Final counter/linear vars updates>;
1671  // }
1672  //
1676  (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));
1677  (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));
1678  }
1679 
1680  // Emit: if (PreCond) - begin.
1681  // If the condition constant folds and can be elided, avoid emitting the
1682  // whole loop.
1683  bool CondConstant;
1684  llvm::BasicBlock *ContBlock = nullptr;
1685  if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
1686  if (!CondConstant)
1687  return;
1688  } else {
1689  llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");
1690  ContBlock = CGF.createBasicBlock("simd.if.end");
1691  emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
1692  CGF.getProfileCount(&S));
1693  CGF.EmitBlock(ThenBlock);
1694  CGF.incrementProfileCounter(&S);
1695  }
1696 
1697  // Emit the loop iteration variable.
1698  const Expr *IVExpr = S.getIterationVariable();
1699  const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
1700  CGF.EmitVarDecl(*IVDecl);
1701  CGF.EmitIgnoredExpr(S.getInit());
1702 
1703  // Emit the iterations count variable.
1704  // If it is not a variable, Sema decided to calculate iterations count on
1705  // each iteration (e.g., it is foldable into a constant).
1706  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
1707  CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
1708  // Emit calculation of the iterations count.
1710  }
1711 
1712  CGF.EmitOMPSimdInit(S);
1713 
1714  emitAlignedClause(CGF, S);
1715  (void)CGF.EmitOMPLinearClauseInit(S);
1716  {
1717  CodeGenFunction::OMPPrivateScope LoopScope(CGF);
1718  CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
1719  CGF.EmitOMPLinearClause(S, LoopScope);
1720  CGF.EmitOMPPrivateClause(S, LoopScope);
1721  CGF.EmitOMPReductionClauseInit(S, LoopScope);
1722  bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
1723  (void)LoopScope.Privatize();
1724  CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
1725  S.getInc(),
1726  [&S](CodeGenFunction &CGF) {
1728  CGF.EmitStopPoint(&S);
1729  },
1730  [](CodeGenFunction &) {});
1731  CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; });
1732  // Emit final copy of the lastprivate variables at the end of loops.
1733  if (HasLastprivateClause)
1734  CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
1735  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
1737  [](CodeGenFunction &) { return nullptr; });
1738  }
1739  CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; });
1740  // Emit: if (PreCond) - end.
1741  if (ContBlock) {
1742  CGF.EmitBranch(ContBlock);
1743  CGF.EmitBlock(ContBlock, true);
1744  }
1745 }
1746 
1748  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1749  emitOMPSimdRegion(CGF, S, Action);
1750  };
1751  OMPLexicalScope Scope(*this, S, OMPD_unknown);
1752  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
1753 }
1754 
1755 void CodeGenFunction::EmitOMPOuterLoop(
1756  bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
1758  const CodeGenFunction::OMPLoopArguments &LoopArgs,
1759  const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
1760  const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
1761  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1762 
1763  const Expr *IVExpr = S.getIterationVariable();
1764  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1765  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1766 
1767  JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
1768 
1769  // Start the loop with a block that tests the condition.
1770  llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond");
1771  EmitBlock(CondBlock);
1772  const SourceRange R = S.getSourceRange();
1773  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1774  SourceLocToDebugLoc(R.getEnd()));
1775 
1776  llvm::Value *BoolCondVal = nullptr;
1777  if (!DynamicOrOrdered) {
1778  // UB = min(UB, GlobalUB) or
1779  // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
1780  // 'distribute parallel for')
1781  EmitIgnoredExpr(LoopArgs.EUB);
1782  // IV = LB
1783  EmitIgnoredExpr(LoopArgs.Init);
1784  // IV < UB
1785  BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
1786  } else {
1787  BoolCondVal =
1788  RT.emitForNext(*this, S.getLocStart(), IVSize, IVSigned, LoopArgs.IL,
1789  LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
1790  }
1791 
1792  // If there are any cleanups between here and the loop-exit scope,
1793  // create a block to stage a loop exit along.
1794  llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1795  if (LoopScope.requiresCleanups())
1796  ExitBlock = createBasicBlock("omp.dispatch.cleanup");
1797 
1798  llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body");
1799  Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
1800  if (ExitBlock != LoopExit.getBlock()) {
1801  EmitBlock(ExitBlock);
1802  EmitBranchThroughCleanup(LoopExit);
1803  }
1804  EmitBlock(LoopBody);
1805 
1806  // Emit "IV = LB" (in case of static schedule, we have already calculated new
1807  // LB for loop condition and emitted it above).
1808  if (DynamicOrOrdered)
1809  EmitIgnoredExpr(LoopArgs.Init);
1810 
1811  // Create a block for the increment.
1812  JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
1813  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1814 
1815  // Generate !llvm.loop.parallel metadata for loads and stores for loops
1816  // with dynamic/guided scheduling and without ordered clause.
1818  LoopStack.setParallel(!IsMonotonic);
1819  else
1820  EmitOMPSimdInit(S, IsMonotonic);
1821 
1822  SourceLocation Loc = S.getLocStart();
1823 
1824  // when 'distribute' is not combined with a 'for':
1825  // while (idx <= UB) { BODY; ++idx; }
1826  // when 'distribute' is combined with a 'for'
1827  // (e.g. 'distribute parallel for')
1828  // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
1829  EmitOMPInnerLoop(
1830  S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
1831  [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
1832  CodeGenLoop(CGF, S, LoopExit);
1833  },
1834  [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
1835  CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
1836  });
1837 
1838  EmitBlock(Continue.getBlock());
1839  BreakContinueStack.pop_back();
1840  if (!DynamicOrOrdered) {
1841  // Emit "LB = LB + Stride", "UB = UB + Stride".
1842  EmitIgnoredExpr(LoopArgs.NextLB);
1843  EmitIgnoredExpr(LoopArgs.NextUB);
1844  }
1845 
1846  EmitBranch(CondBlock);
1847  LoopStack.pop();
1848  // Emit the fall-through block.
1849  EmitBlock(LoopExit.getBlock());
1850 
1851  // Tell the runtime we are done.
1852  auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) {
1853  if (!DynamicOrOrdered)
1854  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(),
1855  S.getDirectiveKind());
1856  };
1857  OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
1858 }
1859 
1860 void CodeGenFunction::EmitOMPForOuterLoop(
1861  const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
1862  const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
1863  const OMPLoopArguments &LoopArgs,
1864  const CodeGenDispatchBoundsTy &CGDispatchBounds) {
1865  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1866 
1867  // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
1868  const bool DynamicOrOrdered =
1869  Ordered || RT.isDynamic(ScheduleKind.Schedule);
1870 
1871  assert((Ordered ||
1872  !RT.isStaticNonchunked(ScheduleKind.Schedule,
1873  LoopArgs.Chunk != nullptr)) &&
1874  "static non-chunked schedule does not need outer loop");
1875 
1876  // Emit outer loop.
1877  //
1878  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1879  // When schedule(dynamic,chunk_size) is specified, the iterations are
1880  // distributed to threads in the team in chunks as the threads request them.
1881  // Each thread executes a chunk of iterations, then requests another chunk,
1882  // until no chunks remain to be distributed. Each chunk contains chunk_size
1883  // iterations, except for the last chunk to be distributed, which may have
1884  // fewer iterations. When no chunk_size is specified, it defaults to 1.
1885  //
1886  // When schedule(guided,chunk_size) is specified, the iterations are assigned
1887  // to threads in the team in chunks as the executing threads request them.
1888  // Each thread executes a chunk of iterations, then requests another chunk,
1889  // until no chunks remain to be assigned. For a chunk_size of 1, the size of
1890  // each chunk is proportional to the number of unassigned iterations divided
1891  // by the number of threads in the team, decreasing to 1. For a chunk_size
1892  // with value k (greater than 1), the size of each chunk is determined in the
1893  // same way, with the restriction that the chunks do not contain fewer than k
1894  // iterations (except for the last chunk to be assigned, which may have fewer
1895  // than k iterations).
1896  //
1897  // When schedule(auto) is specified, the decision regarding scheduling is
1898  // delegated to the compiler and/or runtime system. The programmer gives the
1899  // implementation the freedom to choose any possible mapping of iterations to
1900  // threads in the team.
1901  //
1902  // When schedule(runtime) is specified, the decision regarding scheduling is
1903  // deferred until run time, and the schedule and chunk size are taken from the
1904  // run-sched-var ICV. If the ICV is set to auto, the schedule is
1905  // implementation defined
1906  //
1907  // while(__kmpc_dispatch_next(&LB, &UB)) {
1908  // idx = LB;
1909  // while (idx <= UB) { BODY; ++idx;
1910  // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
1911  // } // inner loop
1912  // }
1913  //
1914  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1915  // When schedule(static, chunk_size) is specified, iterations are divided into
1916  // chunks of size chunk_size, and the chunks are assigned to the threads in
1917  // the team in a round-robin fashion in the order of the thread number.
1918  //
1919  // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
1920  // while (idx <= UB) { BODY; ++idx; } // inner loop
1921  // LB = LB + ST;
1922  // UB = UB + ST;
1923  // }
1924  //
1925 
1926  const Expr *IVExpr = S.getIterationVariable();
1927  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1928  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1929 
1930  if (DynamicOrOrdered) {
1931  const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
1932  CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
1933  llvm::Value *LBVal = DispatchBounds.first;
1934  llvm::Value *UBVal = DispatchBounds.second;
1935  CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
1936  LoopArgs.Chunk};
1937  RT.emitForDispatchInit(*this, S.getLocStart(), ScheduleKind, IVSize,
1938  IVSigned, Ordered, DipatchRTInputValues);
1939  } else {
1940  CGOpenMPRuntime::StaticRTInput StaticInit(
1941  IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
1942  LoopArgs.ST, LoopArgs.Chunk);
1943  RT.emitForStaticInit(*this, S.getLocStart(), S.getDirectiveKind(),
1944  ScheduleKind, StaticInit);
1945  }
1946 
1947  auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
1948  const unsigned IVSize,
1949  const bool IVSigned) {
1950  if (Ordered) {
1951  CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
1952  IVSigned);
1953  }
1954  };
1955 
1956  OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
1957  LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
1958  OuterLoopArgs.IncExpr = S.getInc();
1959  OuterLoopArgs.Init = S.getInit();
1960  OuterLoopArgs.Cond = S.getCond();
1961  OuterLoopArgs.NextLB = S.getNextLowerBound();
1962  OuterLoopArgs.NextUB = S.getNextUpperBound();
1963  EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
1964  emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
1965 }
1966 
1968  const unsigned IVSize, const bool IVSigned) {}
1969 
1970 void CodeGenFunction::EmitOMPDistributeOuterLoop(
1971  OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
1972  OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
1973  const CodeGenLoopTy &CodeGenLoopContent) {
1974 
1975  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1976 
1977  // Emit outer loop.
1978  // Same behavior as a OMPForOuterLoop, except that schedule cannot be
1979  // dynamic
1980  //
1981 
1982  const Expr *IVExpr = S.getIterationVariable();
1983  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1984  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1985 
1986  CGOpenMPRuntime::StaticRTInput StaticInit(
1987  IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
1988  LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
1989  RT.emitDistributeStaticInit(*this, S.getLocStart(), ScheduleKind, StaticInit);
1990 
1991  // for combined 'distribute' and 'for' the increment expression of distribute
1992  // is store in DistInc. For 'distribute' alone, it is in Inc.
1993  Expr *IncExpr;
1995  IncExpr = S.getDistInc();
1996  else
1997  IncExpr = S.getInc();
1998 
1999  // this routine is shared by 'omp distribute parallel for' and
2000  // 'omp distribute': select the right EUB expression depending on the
2001  // directive
2002  OMPLoopArguments OuterLoopArgs;
2003  OuterLoopArgs.LB = LoopArgs.LB;
2004  OuterLoopArgs.UB = LoopArgs.UB;
2005  OuterLoopArgs.ST = LoopArgs.ST;
2006  OuterLoopArgs.IL = LoopArgs.IL;
2007  OuterLoopArgs.Chunk = LoopArgs.Chunk;
2008  OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2010  : S.getEnsureUpperBound();
2011  OuterLoopArgs.IncExpr = IncExpr;
2012  OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2013  ? S.getCombinedInit()
2014  : S.getInit();
2015  OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2016  ? S.getCombinedCond()
2017  : S.getCond();
2018  OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2020  : S.getNextLowerBound();
2021  OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2023  : S.getNextUpperBound();
2024 
2025  EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
2026  LoopScope, OuterLoopArgs, CodeGenLoopContent,
2028 }
2029 
2030 static std::pair<LValue, LValue>
2032  const OMPExecutableDirective &S) {
2033  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2034  LValue LB =
2035  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2036  LValue UB =
2037  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2038 
2039  // When composing 'distribute' with 'for' (e.g. as in 'distribute
2040  // parallel for') we need to use the 'distribute'
2041  // chunk lower and upper bounds rather than the whole loop iteration
2042  // space. These are parameters to the outlined function for 'parallel'
2043  // and we copy the bounds of the previous schedule into the
2044  // the current ones.
2045  LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
2046  LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
2047  llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
2048  PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());
2049  PrevLBVal = CGF.EmitScalarConversion(
2050  PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
2051  LS.getIterationVariable()->getType(),
2053  llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
2054  PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());
2055  PrevUBVal = CGF.EmitScalarConversion(
2056  PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
2057  LS.getIterationVariable()->getType(),
2059 
2060  CGF.EmitStoreOfScalar(PrevLBVal, LB);
2061  CGF.EmitStoreOfScalar(PrevUBVal, UB);
2062 
2063  return {LB, UB};
2064 }
2065 
2066 /// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
2067 /// we need to use the LB and UB expressions generated by the worksharing
2068 /// code generation support, whereas in non combined situations we would
2069 /// just emit 0 and the LastIteration expression
2070 /// This function is necessary due to the difference of the LB and UB
2071 /// types for the RT emission routines for 'for_static_init' and
2072 /// 'for_dispatch_init'
2073 static std::pair<llvm::Value *, llvm::Value *>
2075  const OMPExecutableDirective &S,
2076  Address LB, Address UB) {
2077  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2078  const Expr *IVExpr = LS.getIterationVariable();
2079  // when implementing a dynamic schedule for a 'for' combined with a
2080  // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
2081  // is not normalized as each team only executes its own assigned
2082  // distribute chunk
2083  QualType IteratorTy = IVExpr->getType();
2084  llvm::Value *LBVal =
2085  CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getLocStart());
2086  llvm::Value *UBVal =
2087  CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getLocStart());
2088  return {LBVal, UBVal};
2089 }
2090 
2092  CodeGenFunction &CGF, const OMPExecutableDirective &S,
2093  llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) {
2094  const auto &Dir = cast<OMPLoopDirective>(S);
2095  LValue LB =
2096  CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
2097  llvm::Value *LBCast = CGF.Builder.CreateIntCast(
2098  CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2099  CapturedVars.push_back(LBCast);
2100  LValue UB =
2101  CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
2102 
2103  llvm::Value *UBCast = CGF.Builder.CreateIntCast(
2104  CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2105  CapturedVars.push_back(UBCast);
2106 }
2107 
2108 static void
2110  const OMPLoopDirective &S,
2112  auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
2113  PrePostActionTy &Action) {
2114  Action.Enter(CGF);
2115  bool HasCancel = false;
2117  if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
2118  HasCancel = D->hasCancel();
2119  else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
2120  HasCancel = D->hasCancel();
2121  else if (const auto *D =
2122  dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
2123  HasCancel = D->hasCancel();
2124  }
2126  HasCancel);
2130  };
2131 
2133  CGF, S,
2134  isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for,
2135  CGInlinedWorksharingLoop,
2137 }
2138 
2141  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2143  S.getDistInc());
2144  };
2145  OMPLexicalScope Scope(*this, S, OMPD_parallel);
2146  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2147 }
2148 
2151  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2153  S.getDistInc());
2154  };
2155  OMPLexicalScope Scope(*this, S, OMPD_parallel);
2156  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2157 }
2158 
2160  const OMPDistributeSimdDirective &S) {
2161  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2163  };
2164  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2165  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2166 }
2167 
2169  CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
2170  // Emit SPMD target parallel for region as a standalone region.
2171  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2172  emitOMPSimdRegion(CGF, S, Action);
2173  };
2174  llvm::Function *Fn;
2175  llvm::Constant *Addr;
2176  // Emit target region as a standalone region.
2177  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
2178  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
2179  assert(Fn && Addr && "Target device function emission failed.");
2180 }
2181 
2183  const OMPTargetSimdDirective &S) {
2184  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2185  emitOMPSimdRegion(CGF, S, Action);
2186  };
2187  emitCommonOMPTargetDirective(*this, S, CodeGen);
2188 }
2189 
2190 namespace {
2191  struct ScheduleKindModifiersTy {
2195  ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
2198  : Kind(Kind), M1(M1), M2(M2) {}
2199  };
2200 } // namespace
2201 
2203  const OMPLoopDirective &S, Expr *EUB,
2204  const CodeGenLoopBoundsTy &CodeGenLoopBounds,
2205  const CodeGenDispatchBoundsTy &CGDispatchBounds) {
2206  // Emit the loop iteration variable.
2207  const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
2208  const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
2209  EmitVarDecl(*IVDecl);
2210 
2211  // Emit the iterations count variable.
2212  // If it is not a variable, Sema decided to calculate iterations count on each
2213  // iteration (e.g., it is foldable into a constant).
2214  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2215  EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2216  // Emit calculation of the iterations count.
2217  EmitIgnoredExpr(S.getCalcLastIteration());
2218  }
2219 
2220  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
2221 
2222  bool HasLastprivateClause;
2223  // Check pre-condition.
2224  {
2225  OMPLoopScope PreInitScope(*this, S);
2226  // Skip the entire loop if we don't meet the precondition.
2227  // If the condition constant folds and can be elided, avoid emitting the
2228  // whole loop.
2229  bool CondConstant;
2230  llvm::BasicBlock *ContBlock = nullptr;
2231  if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2232  if (!CondConstant)
2233  return false;
2234  } else {
2235  llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
2236  ContBlock = createBasicBlock("omp.precond.end");
2237  emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
2238  getProfileCount(&S));
2239  EmitBlock(ThenBlock);
2240  incrementProfileCounter(&S);
2241  }
2242 
2243  RunCleanupsScope DoacrossCleanupScope(*this);
2244  bool Ordered = false;
2245  if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
2246  if (OrderedClause->getNumForLoops())
2247  RT.emitDoacrossInit(*this, S);
2248  else
2249  Ordered = true;
2250  }
2251 
2252  llvm::DenseSet<const Expr *> EmittedFinals;
2253  emitAlignedClause(*this, S);
2254  bool HasLinears = EmitOMPLinearClauseInit(S);
2255  // Emit helper vars inits.
2256 
2257  std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
2258  LValue LB = Bounds.first;
2259  LValue UB = Bounds.second;
2260  LValue ST =
2261  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
2262  LValue IL =
2263  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
2264 
2265  // Emit 'then' code.
2266  {
2267  OMPPrivateScope LoopScope(*this);
2268  if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
2269  // Emit implicit barrier to synchronize threads and avoid data races on
2270  // initialization of firstprivate variables and post-update of
2271  // lastprivate variables.
2272  CGM.getOpenMPRuntime().emitBarrierCall(
2273  *this, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
2274  /*ForceSimpleCall=*/true);
2275  }
2276  EmitOMPPrivateClause(S, LoopScope);
2277  HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
2278  EmitOMPReductionClauseInit(S, LoopScope);
2279  EmitOMPPrivateLoopCounters(S, LoopScope);
2280  EmitOMPLinearClause(S, LoopScope);
2281  (void)LoopScope.Privatize();
2282 
2283  // Detect the loop schedule kind and chunk.
2284  llvm::Value *Chunk = nullptr;
2285  OpenMPScheduleTy ScheduleKind;
2286  if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
2287  ScheduleKind.Schedule = C->getScheduleKind();
2288  ScheduleKind.M1 = C->getFirstScheduleModifier();
2289  ScheduleKind.M2 = C->getSecondScheduleModifier();
2290  if (const Expr *Ch = C->getChunkSize()) {
2291  Chunk = EmitScalarExpr(Ch);
2292  Chunk = EmitScalarConversion(Chunk, Ch->getType(),
2294  S.getLocStart());
2295  }
2296  }
2297  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2298  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2299  // OpenMP 4.5, 2.7.1 Loop Construct, Description.
2300  // If the static schedule kind is specified or if the ordered clause is
2301  // specified, and if no monotonic modifier is specified, the effect will
2302  // be as if the monotonic modifier was specified.
2303  if (RT.isStaticNonchunked(ScheduleKind.Schedule,
2304  /* Chunked */ Chunk != nullptr) &&
2305  !Ordered) {
2307  EmitOMPSimdInit(S, /*IsMonotonic=*/true);
2308  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2309  // When no chunk_size is specified, the iteration space is divided into
2310  // chunks that are approximately equal in size, and at most one chunk is
2311  // distributed to each thread. Note that the size of the chunks is
2312  // unspecified in this case.
2313  CGOpenMPRuntime::StaticRTInput StaticInit(
2314  IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
2315  UB.getAddress(), ST.getAddress());
2316  RT.emitForStaticInit(*this, S.getLocStart(), S.getDirectiveKind(),
2317  ScheduleKind, StaticInit);
2318  JumpDest LoopExit =
2319  getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
2320  // UB = min(UB, GlobalUB);
2321  EmitIgnoredExpr(S.getEnsureUpperBound());
2322  // IV = LB;
2323  EmitIgnoredExpr(S.getInit());
2324  // while (idx <= UB) { BODY; ++idx; }
2325  EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
2326  S.getInc(),
2327  [&S, LoopExit](CodeGenFunction &CGF) {
2328  CGF.EmitOMPLoopBody(S, LoopExit);
2329  CGF.EmitStopPoint(&S);
2330  },
2331  [](CodeGenFunction &) {});
2332  EmitBlock(LoopExit.getBlock());
2333  // Tell the runtime we are done.
2334  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2335  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(),
2336  S.getDirectiveKind());
2337  };
2338  OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
2339  } else {
2340  const bool IsMonotonic =
2341  Ordered || ScheduleKind.Schedule == OMPC_SCHEDULE_static ||
2342  ScheduleKind.Schedule == OMPC_SCHEDULE_unknown ||
2343  ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
2344  ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
2345  // Emit the outer loop, which requests its work chunk [LB..UB] from
2346  // runtime and runs the inner loop to process it.
2347  const OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),
2348  ST.getAddress(), IL.getAddress(),
2349  Chunk, EUB);
2350  EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
2351  LoopArguments, CGDispatchBounds);
2352  }
2354  EmitOMPSimdFinal(S,
2355  [IL, &S](CodeGenFunction &CGF) {
2356  return CGF.Builder.CreateIsNotNull(
2357  CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2358  });
2359  }
2360  EmitOMPReductionClauseFinal(
2361  S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind())
2362  ? /*Parallel and Simd*/ OMPD_parallel_for_simd
2363  : /*Parallel only*/ OMPD_parallel);
2364  // Emit post-update of the reduction variables if IsLastIter != 0.
2366  *this, S, [IL, &S](CodeGenFunction &CGF) {
2367  return CGF.Builder.CreateIsNotNull(
2368  CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2369  });
2370  // Emit final copy of the lastprivate variables if IsLastIter != 0.
2371  if (HasLastprivateClause)
2372  EmitOMPLastprivateClauseFinal(
2374  Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getLocStart())));
2375  }
2376  EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) {
2377  return CGF.Builder.CreateIsNotNull(
2378  CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2379  });
2380  DoacrossCleanupScope.ForceCleanup();
2381  // We're now done with the loop, so jump to the continuation block.
2382  if (ContBlock) {
2383  EmitBranch(ContBlock);
2384  EmitBlock(ContBlock, /*IsFinished=*/true);
2385  }
2386  }
2387  return HasLastprivateClause;
2388 }
2389 
2390 /// The following two functions generate expressions for the loop lower
2391 /// and upper bounds in case of static and dynamic (dispatch) schedule
2392 /// of the associated 'for' or 'distribute' loop.
2393 static std::pair<LValue, LValue>
2395  const auto &LS = cast<OMPLoopDirective>(S);
2396  LValue LB =
2397  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2398  LValue UB =
2399  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2400  return {LB, UB};
2401 }
2402 
2403 /// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
2404 /// consider the lower and upper bound expressions generated by the
2405 /// worksharing loop support, but we use 0 and the iteration space size as
2406 /// constants
2407 static std::pair<llvm::Value *, llvm::Value *>
2409  Address LB, Address UB) {
2410  const auto &LS = cast<OMPLoopDirective>(S);
2411  const Expr *IVExpr = LS.getIterationVariable();
2412  const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
2413  llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
2414  llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
2415  return {LBVal, UBVal};
2416 }
2417 
2419  bool HasLastprivates = false;
2420  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2421  PrePostActionTy &) {
2422  OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel());
2423  HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2426  };
2427  {
2428  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2429  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
2430  S.hasCancel());
2431  }
2432 
2433  // Emit an implicit barrier at the end.
2434  if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
2435  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for);
2436 }
2437 
2439  bool HasLastprivates = false;
2440  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2441  PrePostActionTy &) {
2442  HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2445  };
2446  {
2447  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2448  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2449  }
2450 
2451  // Emit an implicit barrier at the end.
2452  if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
2453  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for);
2454 }
2455 
2457  const Twine &Name,
2458  llvm::Value *Init = nullptr) {
2459  LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
2460  if (Init)
2461  CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
2462  return LVal;
2463 }
2464 
2465 void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
2467  const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
2468  bool HasLastprivates = false;
2469  auto &&CodeGen = [&S, CapturedStmt, CS,
2470  &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
2471  ASTContext &C = CGF.getContext();
2472  QualType KmpInt32Ty =
2473  C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2474  // Emit helper vars inits.
2475  LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
2476  CGF.Builder.getInt32(0));
2477  llvm::ConstantInt *GlobalUBVal = CS != nullptr
2478  ? CGF.Builder.getInt32(CS->size() - 1)
2479  : CGF.Builder.getInt32(0);
2480  LValue UB =
2481  createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
2482  LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
2483  CGF.Builder.getInt32(1));
2484  LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
2485  CGF.Builder.getInt32(0));
2486  // Loop counter.
2487  LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
2488  OpaqueValueExpr IVRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue);
2489  CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
2490  OpaqueValueExpr UBRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue);
2491  CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
2492  // Generate condition for loop.
2493  BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue,
2495  // Increment for loop counter.
2496  UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary,
2497  S.getLocStart(), true);
2498  auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
2499  // Iterate through all sections and emit a switch construct:
2500  // switch (IV) {
2501  // case 0:
2502  // <SectionStmt[0]>;
2503  // break;
2504  // ...
2505  // case <NumSection> - 1:
2506  // <SectionStmt[<NumSection> - 1]>;
2507  // break;
2508  // }
2509  // .omp.sections.exit:
2510  llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
2511  llvm::SwitchInst *SwitchStmt =
2512  CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getLocStart()),
2513  ExitBB, CS == nullptr ? 1 : CS->size());
2514  if (CS) {
2515  unsigned CaseNumber = 0;
2516  for (const Stmt *SubStmt : CS->children()) {
2517  auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2518  CGF.EmitBlock(CaseBB);
2519  SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
2520  CGF.EmitStmt(SubStmt);
2521  CGF.EmitBranch(ExitBB);
2522  ++CaseNumber;
2523  }
2524  } else {
2525  llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");
2526  CGF.EmitBlock(CaseBB);
2527  SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
2528  CGF.EmitStmt(CapturedStmt);
2529  CGF.EmitBranch(ExitBB);
2530  }
2531  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2532  };
2533 
2534  CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2535  if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
2536  // Emit implicit barrier to synchronize threads and avoid data races on
2537  // initialization of firstprivate variables and post-update of lastprivate
2538  // variables.
2539  CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2540  CGF, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
2541  /*ForceSimpleCall=*/true);
2542  }
2543  CGF.EmitOMPPrivateClause(S, LoopScope);
2544  HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2545  CGF.EmitOMPReductionClauseInit(S, LoopScope);
2546  (void)LoopScope.Privatize();
2547 
2548  // Emit static non-chunked loop.
2549  OpenMPScheduleTy ScheduleKind;
2550  ScheduleKind.Schedule = OMPC_SCHEDULE_static;
2551  CGOpenMPRuntime::StaticRTInput StaticInit(
2552  /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(),
2553  LB.getAddress(), UB.getAddress(), ST.getAddress());
2554  CGF.CGM.getOpenMPRuntime().emitForStaticInit(
2555  CGF, S.getLocStart(), S.getDirectiveKind(), ScheduleKind, StaticInit);
2556  // UB = min(UB, GlobalUB);
2557  llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getLocStart());
2558  llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
2559  CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
2560  CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
2561  // IV = LB;
2562  CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getLocStart()), IV);
2563  // while (idx <= UB) { BODY; ++idx; }
2564  CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen,
2565  [](CodeGenFunction &) {});
2566  // Tell the runtime we are done.
2567  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2568  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(),
2569  S.getDirectiveKind());
2570  };
2571  CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
2572  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
2573  // Emit post-update of the reduction variables if IsLastIter != 0.
2574  emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) {
2575  return CGF.Builder.CreateIsNotNull(
2576  CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2577  });
2578 
2579  // Emit final copy of the lastprivate variables if IsLastIter != 0.
2580  if (HasLastprivates)
2582  S, /*NoFinals=*/false,
2583  CGF.Builder.CreateIsNotNull(
2584  CGF.EmitLoadOfScalar(IL, S.getLocStart())));
2585  };
2586 
2587  bool HasCancel = false;
2588  if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
2589  HasCancel = OSD->hasCancel();
2590  else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
2591  HasCancel = OPSD->hasCancel();
2592  OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
2593  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
2594  HasCancel);
2595  // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
2596  // clause. Otherwise the barrier will be generated by the codegen for the
2597  // directive.
2598  if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
2599  // Emit implicit barrier to synchronize threads and avoid data races on
2600  // initialization of firstprivate variables.
2601  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(),
2602  OMPD_unknown);
2603  }
2604 }
2605 
2607  {
2608  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2609  EmitSections(S);
2610  }
2611  // Emit an implicit barrier at the end.
2612  if (!S.getSingleClause<OMPNowaitClause>()) {
2613  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(),
2614  OMPD_sections);
2615  }
2616 }
2617 
2619  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2621  };
2622  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2623  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen,
2624  S.hasCancel());
2625 }
2626 
2628  llvm::SmallVector<const Expr *, 8> CopyprivateVars;
2631  llvm::SmallVector<const Expr *, 8> AssignmentOps;
2632  // Check if there are any 'copyprivate' clauses associated with this
2633  // 'single' construct.
2634  // Build a list of copyprivate variables along with helper expressions
2635  // (<source>, <destination>, <destination>=<source> expressions)
2636  for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
2637  CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
2638  DestExprs.append(C->destination_exprs().begin(),
2639  C->destination_exprs().end());
2640  SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
2641  AssignmentOps.append(C->assignment_ops().begin(),
2642  C->assignment_ops().end());
2643  }
2644  // Emit code for 'single' region along with 'copyprivate' clauses
2645  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2646  Action.Enter(CGF);
2647  OMPPrivateScope SingleScope(CGF);
2648  (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
2649  CGF.EmitOMPPrivateClause(S, SingleScope);
2650  (void)SingleScope.Privatize();
2652  };
2653  {
2654  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2655  CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getLocStart(),
2656  CopyprivateVars, DestExprs,
2657  SrcExprs, AssignmentOps);
2658  }
2659  // Emit an implicit barrier at the end (to avoid data race on firstprivate
2660  // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
2661  if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
2662  CGM.getOpenMPRuntime().emitBarrierCall(
2663  *this, S.getLocStart(),
2664  S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
2665  }
2666 }
2667 
2669  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2670  Action.Enter(CGF);
2672  };
2673  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2674  CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getLocStart());
2675 }
2676 
2678  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2679  Action.Enter(CGF);
2681  };
2682  const Expr *Hint = nullptr;
2683  if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
2684  Hint = HintClause->getHint();
2685  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2686  CGM.getOpenMPRuntime().emitCriticalRegion(*this,
2688  CodeGen, S.getLocStart(), Hint);
2689 }
2690 
2692  const OMPParallelForDirective &S) {
2693  // Emit directive as a combined directive that consists of two implicit
2694  // directives: 'parallel' with 'for' directive.
2695  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2696  Action.Enter(CGF);
2697  OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel());
2700  };
2701  emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
2703 }
2704 
2706  const OMPParallelForSimdDirective &S) {
2707  // Emit directive as a combined directive that consists of two implicit
2708  // directives: 'parallel' with 'for' directive.
2709  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2710  Action.Enter(CGF);
2713  };
2714  emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen,
2716 }
2717 
2719  const OMPParallelSectionsDirective &S) {
2720  // Emit directive as a combined directive that consists of two implicit
2721  // directives: 'parallel' with 'sections' directive.
2722  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2723  Action.Enter(CGF);
2724  CGF.EmitSections(S);
2725  };
2726  emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
2728 }
2729 
2731  const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
2732  const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
2733  OMPTaskDataTy &Data) {
2734  // Emit outlined function for task construct.
2735  const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
2736  auto I = CS->getCapturedDecl()->param_begin();
2737  auto PartId = std::next(I);
2738  auto TaskT = std::next(I, 4);
2739  // Check if the task is final
2740  if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
2741  // If the condition constant folds and can be elided, try to avoid emitting
2742  // the condition and the dead arm of the if/else.
2743  const Expr *Cond = Clause->getCondition();
2744  bool CondConstant;
2745  if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
2746  Data.Final.setInt(CondConstant);
2747  else
2748  Data.Final.setPointer(EvaluateExprAsBool(Cond));
2749  } else {
2750  // By default the task is not final.
2751  Data.Final.setInt(/*IntVal=*/false);
2752  }
2753  // Check if the task has 'priority' clause.
2754  if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
2755  const Expr *Prio = Clause->getPriority();
2756  Data.Priority.setInt(/*IntVal=*/true);
2757  Data.Priority.setPointer(EmitScalarConversion(
2758  EmitScalarExpr(Prio), Prio->getType(),
2759  getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
2760  Prio->getExprLoc()));
2761  }
2762  // The first function argument for tasks is a thread id, the second one is a
2763  // part id (0 for tied tasks, >=0 for untied task).
2764  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
2765  // Get list of private variables.
2766  for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
2767  auto IRef = C->varlist_begin();
2768  for (const Expr *IInit : C->private_copies()) {
2769  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2770  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2771  Data.PrivateVars.push_back(*IRef);
2772  Data.PrivateCopies.push_back(IInit);
2773  }
2774  ++IRef;
2775  }
2776  }
2777  EmittedAsPrivate.clear();
2778  // Get list of firstprivate variables.
2779  for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
2780  auto IRef = C->varlist_begin();
2781  auto IElemInitRef = C->inits().begin();
2782  for (const Expr *IInit : C->private_copies()) {
2783  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2784  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2785  Data.FirstprivateVars.push_back(*IRef);
2786  Data.FirstprivateCopies.push_back(IInit);
2787  Data.FirstprivateInits.push_back(*IElemInitRef);
2788  }
2789  ++IRef;
2790  ++IElemInitRef;
2791  }
2792  }
2793  // Get list of lastprivate variables (for taskloops).
2794  llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
2795  for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
2796  auto IRef = C->varlist_begin();
2797  auto ID = C->destination_exprs().begin();
2798  for (const Expr *IInit : C->private_copies()) {
2799  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2800  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2801  Data.LastprivateVars.push_back(*IRef);
2802  Data.LastprivateCopies.push_back(IInit);
2803  }
2804  LastprivateDstsOrigs.insert(
2805  {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
2806  cast<DeclRefExpr>(*IRef)});
2807  ++IRef;
2808  ++ID;
2809  }
2810  }
2813  for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
2814  auto IPriv = C->privates().begin();
2815  auto IRed = C->reduction_ops().begin();
2816  auto ILHS = C->lhs_exprs().begin();
2817  auto IRHS = C->rhs_exprs().begin();
2818  for (const Expr *Ref : C->varlists()) {
2819  Data.ReductionVars.emplace_back(Ref);
2820  Data.ReductionCopies.emplace_back(*IPriv);
2821  Data.ReductionOps.emplace_back(*IRed);
2822  LHSs.emplace_back(*ILHS);
2823  RHSs.emplace_back(*IRHS);
2824  std::advance(IPriv, 1);
2825  std::advance(IRed, 1);
2826  std::advance(ILHS, 1);
2827  std::advance(IRHS, 1);
2828  }
2829  }
2830  Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
2831  *this, S.getLocStart(), LHSs, RHSs, Data);
2832  // Build list of dependences.
2833  for (const auto *C : S.getClausesOfKind<OMPDependClause>())
2834  for (const Expr *IRef : C->varlists())
2835  Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
2836  auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
2837  CapturedRegion](CodeGenFunction &CGF,
2838  PrePostActionTy &Action) {
2839  // Set proper addresses for generated private copies.
2840  OMPPrivateScope Scope(CGF);
2841  if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
2842  !Data.LastprivateVars.empty()) {
2843  enum { PrivatesParam = 2, CopyFnParam = 3 };
2844  llvm::Value *CopyFn = CGF.Builder.CreateLoad(
2845  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
2846  llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
2847  CS->getCapturedDecl()->getParam(PrivatesParam)));
2848  // Map privates.
2851  CallArgs.push_back(PrivatesPtr);
2852  for (const Expr *E : Data.PrivateVars) {
2853  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2854  Address PrivatePtr = CGF.CreateMemTemp(
2855  CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
2856  PrivatePtrs.emplace_back(VD, PrivatePtr);
2857  CallArgs.push_back(PrivatePtr.getPointer());
2858  }
2859  for (const Expr *E : Data.FirstprivateVars) {
2860  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2861  Address PrivatePtr =
2862  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2863  ".firstpriv.ptr.addr");
2864  PrivatePtrs.emplace_back(VD, PrivatePtr);
2865  CallArgs.push_back(PrivatePtr.getPointer());
2866  }
2867  for (const Expr *E : Data.LastprivateVars) {
2868  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2869  Address PrivatePtr =
2870  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2871  ".lastpriv.ptr.addr");
2872  PrivatePtrs.emplace_back(VD, PrivatePtr);
2873  CallArgs.push_back(PrivatePtr.getPointer());
2874  }
2875  CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(),
2876  CopyFn, CallArgs);
2877  for (const auto &Pair : LastprivateDstsOrigs) {
2878  const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
2879  DeclRefExpr DRE(
2880  const_cast<VarDecl *>(OrigVD),
2881  /*RefersToEnclosingVariableOrCapture=*/CGF.CapturedStmtInfo->lookup(
2882  OrigVD) != nullptr,
2883  Pair.second->getType(), VK_LValue, Pair.second->getExprLoc());
2884  Scope.addPrivate(Pair.first, [&CGF, &DRE]() {
2885  return CGF.EmitLValue(&DRE).getAddress();
2886  });
2887  }
2888  for (const auto &Pair : PrivatePtrs) {
2889  Address Replacement(CGF.Builder.CreateLoad(Pair.second),
2890  CGF.getContext().getDeclAlign(Pair.first));
2891  Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
2892  }
2893  }
2894  if (Data.Reductions) {
2895  OMPLexicalScope LexScope(CGF, S, CapturedRegion);
2897  Data.ReductionOps);
2898  llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
2899  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
2900  for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
2901  RedCG.emitSharedLValue(CGF, Cnt);
2902  RedCG.emitAggregateType(CGF, Cnt);
2903  // FIXME: This must removed once the runtime library is fixed.
2904  // Emit required threadprivate variables for
2905  // initilizer/combiner/finalizer.
2906  CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getLocStart(),
2907  RedCG, Cnt);
2908  Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
2909  CGF, S.getLocStart(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
2910  Replacement =
2911  Address(CGF.EmitScalarConversion(
2912  Replacement.getPointer(), CGF.getContext().VoidPtrTy,
2913  CGF.getContext().getPointerType(
2914  Data.ReductionCopies[Cnt]->getType()),
2915  Data.ReductionCopies[Cnt]->getExprLoc()),
2916  Replacement.getAlignment());
2917  Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
2918  Scope.addPrivate(RedCG.getBaseDecl(Cnt),
2919  [Replacement]() { return Replacement; });
2920  }
2921  }
2922  // Privatize all private variables except for in_reduction items.
2923  (void)Scope.Privatize();
2924  SmallVector<const Expr *, 4> InRedVars;
2925  SmallVector<const Expr *, 4> InRedPrivs;
2927  SmallVector<const Expr *, 4> TaskgroupDescriptors;
2928  for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
2929  auto IPriv = C->privates().begin();
2930  auto IRed = C->reduction_ops().begin();
2931  auto ITD = C->taskgroup_descriptors().begin();
2932  for (const Expr *Ref : C->varlists()) {
2933  InRedVars.emplace_back(Ref);
2934  InRedPrivs.emplace_back(*IPriv);
2935  InRedOps.emplace_back(*IRed);
2936  TaskgroupDescriptors.emplace_back(*ITD);
2937  std::advance(IPriv, 1);
2938  std::advance(IRed, 1);
2939  std::advance(ITD, 1);
2940  }
2941  }
2942  // Privatize in_reduction items here, because taskgroup descriptors must be
2943  // privatized earlier.
2944  OMPPrivateScope InRedScope(CGF);
2945  if (!InRedVars.empty()) {
2946  ReductionCodeGen RedCG(InRedVars, InRedPrivs, InRedOps);
2947  for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
2948  RedCG.emitSharedLValue(CGF, Cnt);
2949  RedCG.emitAggregateType(CGF, Cnt);
2950  // The taskgroup descriptor variable is always implicit firstprivate and
2951  // privatized already during procoessing of the firstprivates.
2952  // FIXME: This must removed once the runtime library is fixed.
2953  // Emit required threadprivate variables for
2954  // initilizer/combiner/finalizer.
2955  CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getLocStart(),
2956  RedCG, Cnt);
2957  llvm::Value *ReductionsPtr =
2958  CGF.EmitLoadOfScalar(CGF.EmitLValue(TaskgroupDescriptors[Cnt]),
2959  TaskgroupDescriptors[Cnt]->getExprLoc());
2960  Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
2961  CGF, S.getLocStart(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
2962  Replacement = Address(
2963  CGF.EmitScalarConversion(
2964  Replacement.getPointer(), CGF.getContext().VoidPtrTy,
2965  CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
2966  InRedPrivs[Cnt]->getExprLoc()),
2967  Replacement.getAlignment());
2968  Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
2969  InRedScope.addPrivate(RedCG.getBaseDecl(Cnt),
2970  [Replacement]() { return Replacement; });
2971  }
2972  }
2973  (void)InRedScope.Privatize();
2974 
2975  Action.Enter(CGF);
2976  BodyGen(CGF);
2977  };
2978  llvm::Value *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
2979  S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied,
2980  Data.NumberOfParts);
2981  OMPLexicalScope Scope(*this, S);
2982  TaskGen(*this, OutlinedFn, Data);
2983 }
2984 
2985 static ImplicitParamDecl *
2987  QualType Ty, CapturedDecl *CD,
2988  SourceLocation Loc) {
2989  auto *OrigVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
2991  auto *OrigRef = DeclRefExpr::Create(
2992  C, NestedNameSpecifierLoc(), SourceLocation(), OrigVD,
2993  /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
2994  auto *PrivateVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
2996  auto *PrivateRef = DeclRefExpr::Create(
2997  C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD,
2998  /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
2999  QualType ElemType = C.getBaseElementType(Ty);
3000  auto *InitVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, ElemType,
3002  auto *InitRef = DeclRefExpr::Create(
3003  C, NestedNameSpecifierLoc(), SourceLocation(), InitVD,
3004  /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue);
3005  PrivateVD->setInitStyle(VarDecl::CInit);
3006  PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue,
3007  InitRef, /*BasePath=*/nullptr,
3008  VK_RValue));
3009  Data.FirstprivateVars.emplace_back(OrigRef);
3010  Data.FirstprivateCopies.emplace_back(PrivateRef);
3011  Data.FirstprivateInits.emplace_back(InitRef);
3012  return OrigVD;
3013 }
3014 
3016  const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,
3017  OMPTargetDataInfo &InputInfo) {
3018  // Emit outlined function for task construct.
3019  const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3020  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3021  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3022  auto I = CS->getCapturedDecl()->param_begin();
3023  auto PartId = std::next(I);
3024  auto TaskT = std::next(I, 4);
3025  OMPTaskDataTy Data;
3026  // The task is not final.
3027  Data.Final.setInt(/*IntVal=*/false);
3028  // Get list of firstprivate variables.
3029  for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
3030  auto IRef = C->varlist_begin();
3031  auto IElemInitRef = C->inits().begin();
3032  for (auto *IInit : C->private_copies()) {
3033  Data.FirstprivateVars.push_back(*IRef);
3034  Data.FirstprivateCopies.push_back(IInit);
3035  Data.FirstprivateInits.push_back(*IElemInitRef);
3036  ++IRef;
3037  ++IElemInitRef;
3038  }
3039  }
3040  OMPPrivateScope TargetScope(*this);
3041  VarDecl *BPVD = nullptr;
3042  VarDecl *PVD = nullptr;
3043  VarDecl *SVD = nullptr;
3044  if (InputInfo.NumberOfTargetItems > 0) {
3045  auto *CD = CapturedDecl::Create(
3046  getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0);
3047  llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);
3048  QualType BaseAndPointersType = getContext().getConstantArrayType(
3049  getContext().VoidPtrTy, ArrSize, ArrayType::Normal,
3050  /*IndexTypeQuals=*/0);
3052  getContext(), Data, BaseAndPointersType, CD, S.getLocStart());
3054  getContext(), Data, BaseAndPointersType, CD, S.getLocStart());
3055  QualType SizesType = getContext().getConstantArrayType(
3056  getContext().getSizeType(), ArrSize, ArrayType::Normal,
3057  /*IndexTypeQuals=*/0);
3058  SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD,
3059  S.getLocStart());
3060  TargetScope.addPrivate(
3061  BPVD, [&InputInfo]() { return InputInfo.BasePointersArray; });
3062  TargetScope.addPrivate(PVD,
3063  [&InputInfo]() { return InputInfo.PointersArray; });
3064  TargetScope.addPrivate(SVD,
3065  [&InputInfo]() { return InputInfo.SizesArray; });
3066  }
3067  (void)TargetScope.Privatize();
3068  // Build list of dependences.
3069  for (const auto *C : S.getClausesOfKind<OMPDependClause>())
3070  for (const Expr *IRef : C->varlists())
3071  Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
3072  auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD,
3073  &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
3074  // Set proper addresses for generated private copies.
3075  OMPPrivateScope Scope(CGF);
3076  if (!Data.FirstprivateVars.empty()) {
3077  enum { PrivatesParam = 2, CopyFnParam = 3 };
3078  llvm::Value *CopyFn = CGF.Builder.CreateLoad(
3079  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
3080  llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
3081  CS->getCapturedDecl()->getParam(PrivatesParam)));
3082  // Map privates.
3085  CallArgs.push_back(PrivatesPtr);
3086  for (const Expr *E : Data.FirstprivateVars) {
3087  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3088  Address PrivatePtr =
3089  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
3090  ".firstpriv.ptr.addr");
3091  PrivatePtrs.emplace_back(VD, PrivatePtr);
3092  CallArgs.push_back(PrivatePtr.getPointer());
3093  }
3094  CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(),
3095  CopyFn, CallArgs);
3096  for (const auto &Pair : PrivatePtrs) {
3097  Address Replacement(CGF.Builder.CreateLoad(Pair.second),
3098  CGF.getContext().getDeclAlign(Pair.first));
3099  Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
3100  }
3101  }
3102  // Privatize all private variables except for in_reduction items.
3103  (void)Scope.Privatize();
3104  if (InputInfo.NumberOfTargetItems > 0) {
3105  InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
3106  CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0, CGF.getPointerSize());
3107  InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
3108  CGF.GetAddrOfLocalVar(PVD), /*Index=*/0, CGF.getPointerSize());
3109  InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
3110  CGF.GetAddrOfLocalVar(SVD), /*Index=*/0, CGF.getSizeSize());
3111  }
3112 
3113  Action.Enter(CGF);
3114  OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);
3115  BodyGen(CGF);
3116  };
3117  llvm::Value *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3118  S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true,
3119  Data.NumberOfParts);
3120  llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
3121  IntegerLiteral IfCond(getContext(), TrueOrFalse,
3122  getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3123  SourceLocation());
3124 
3125  CGM.getOpenMPRuntime().emitTaskCall(*this, S.getLocStart(), S, OutlinedFn,
3126  SharedsTy, CapturedStruct, &IfCond, Data);
3127 }
3128 
3130  // Emit outlined function for task construct.
3131  const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3132  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3133  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3134  const Expr *IfCond = nullptr;
3135  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3136  if (C->getNameModifier() == OMPD_unknown ||
3137  C->getNameModifier() == OMPD_task) {
3138  IfCond = C->getCondition();
3139  break;
3140  }
3141  }
3142 
3143  OMPTaskDataTy Data;
3144  // Check if we should emit tied or untied task.
3145  Data.Tied = !S.getSingleClause<OMPUntiedClause>();
3146  auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
3147  CGF.EmitStmt(CS->getCapturedStmt());
3148  };
3149  auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
3150  IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
3151  const OMPTaskDataTy &Data) {
3152  CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getLocStart(), S, OutlinedFn,
3153  SharedsTy, CapturedStruct, IfCond,
3154  Data);
3155  };
3156  EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data);
3157 }
3158 
3160  const OMPTaskyieldDirective &S) {
3161  CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getLocStart());
3162 }
3163 
3165  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_barrier);
3166 }
3167 
3169  CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getLocStart());
3170 }
3171 
3173  const OMPTaskgroupDirective &S) {
3174  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3175  Action.Enter(CGF);
3176  if (const Expr *E = S.getReductionRef()) {
3179  OMPTaskDataTy Data;
3180  for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
3181  auto IPriv = C->privates().begin();
3182  auto IRed = C->reduction_ops().begin();
3183  auto ILHS = C->lhs_exprs().begin();
3184  auto IRHS = C->rhs_exprs().begin();
3185  for (const Expr *Ref : C->varlists()) {
3186  Data.ReductionVars.emplace_back(Ref);
3187  Data.ReductionCopies.emplace_back(*IPriv);
3188  Data.ReductionOps.emplace_back(*IRed);
3189  LHSs.emplace_back(*ILHS);
3190  RHSs.emplace_back(*IRHS);
3191  std::advance(IPriv, 1);
3192  std::advance(IRed, 1);
3193  std::advance(ILHS, 1);
3194  std::advance(IRHS, 1);
3195  }
3196  }
3197  llvm::Value *ReductionDesc =
3198  CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getLocStart(),
3199  LHSs, RHSs, Data);
3200  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3201  CGF.EmitVarDecl(*VD);
3202  CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),
3203  /*Volatile=*/false, E->getType());
3204  }
3206  };
3207  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3208  CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getLocStart());
3209 }
3210 
3212  CGM.getOpenMPRuntime().emitFlush(
3213  *this,
3214  [&S]() -> ArrayRef<const Expr *> {
3215  if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>())
3216  return llvm::makeArrayRef(FlushClause->varlist_begin(),
3217  FlushClause->varlist_end());
3218  return llvm::None;
3219  }(),
3220  S.getLocStart());
3221 }
3222 
3224  const CodeGenLoopTy &CodeGenLoop,
3225  Expr *IncExpr) {
3226  // Emit the loop iteration variable.
3227  const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3228  const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3229  EmitVarDecl(*IVDecl);
3230 
3231  // Emit the iterations count variable.
3232  // If it is not a variable, Sema decided to calculate iterations count on each
3233  // iteration (e.g., it is foldable into a constant).
3234  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3235  EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3236  // Emit calculation of the iterations count.
3237  EmitIgnoredExpr(S.getCalcLastIteration());
3238  }
3239 
3240  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3241 
3242  bool HasLastprivateClause = false;
3243  // Check pre-condition.
3244  {
3245  OMPLoopScope PreInitScope(*this, S);
3246  // Skip the entire loop if we don't meet the precondition.
3247  // If the condition constant folds and can be elided, avoid emitting the
3248  // whole loop.
3249  bool CondConstant;
3250  llvm::BasicBlock *ContBlock = nullptr;
3251  if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
3252  if (!CondConstant)
3253  return;
3254  } else {
3255  llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
3256  ContBlock = createBasicBlock("omp.precond.end");
3257  emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
3258  getProfileCount(&S));
3259  EmitBlock(ThenBlock);
3260  incrementProfileCounter(&S);
3261  }
3262 
3263  emitAlignedClause(*this, S);
3264  // Emit 'then' code.
3265  {
3266  // Emit helper vars inits.
3267 
3268  LValue LB = EmitOMPHelperVar(
3269  *this, cast<DeclRefExpr>(
3272  : S.getLowerBoundVariable())));
3273  LValue UB = EmitOMPHelperVar(
3274  *this, cast<DeclRefExpr>(
3277  : S.getUpperBoundVariable())));
3278  LValue ST =
3279  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
3280  LValue IL =
3281  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
3282 
3283  OMPPrivateScope LoopScope(*this);
3284  if (EmitOMPFirstprivateClause(S, LoopScope)) {
3285  // Emit implicit barrier to synchronize threads and avoid data races
3286  // on initialization of firstprivate variables and post-update of
3287  // lastprivate variables.
3288  CGM.getOpenMPRuntime().emitBarrierCall(
3289  *this, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
3290  /*ForceSimpleCall=*/true);
3291  }
3292  EmitOMPPrivateClause(S, LoopScope);
3296  EmitOMPReductionClauseInit(S, LoopScope);
3297  HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
3298  EmitOMPPrivateLoopCounters(S, LoopScope);
3299  (void)LoopScope.Privatize();
3300 
3301  // Detect the distribute schedule kind and chunk.
3302  llvm::Value *Chunk = nullptr;
3304  if (const auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
3305  ScheduleKind = C->getDistScheduleKind();
3306  if (const Expr *Ch = C->getChunkSize()) {
3307  Chunk = EmitScalarExpr(Ch);
3308  Chunk = EmitScalarConversion(Chunk, Ch->getType(),
3310  S.getLocStart());
3311  }
3312  }
3313  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3314  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3315 
3316  // OpenMP [2.10.8, distribute Construct, Description]
3317  // If dist_schedule is specified, kind must be static. If specified,
3318  // iterations are divided into chunks of size chunk_size, chunks are
3319  // assigned to the teams of the league in a round-robin fashion in the
3320  // order of the team number. When no chunk_size is specified, the
3321  // iteration space is divided into chunks that are approximately equal
3322  // in size, and at most one chunk is distributed to each team of the
3323  // league. The size of the chunks is unspecified in this case.
3324  if (RT.isStaticNonchunked(ScheduleKind,
3325  /* Chunked */ Chunk != nullptr)) {
3327  EmitOMPSimdInit(S, /*IsMonotonic=*/true);
3328  CGOpenMPRuntime::StaticRTInput StaticInit(
3329  IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(),
3330  LB.getAddress(), UB.getAddress(), ST.getAddress());
3331  RT.emitDistributeStaticInit(*this, S.getLocStart(), ScheduleKind,
3332  StaticInit);
3333  JumpDest LoopExit =
3334  getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
3335  // UB = min(UB, GlobalUB);
3338  : S.getEnsureUpperBound());
3339  // IV = LB;
3341  ? S.getCombinedInit()
3342  : S.getInit());
3343 
3344  const Expr *Cond =
3346  ? S.getCombinedCond()
3347  : S.getCond();
3348 
3349  // for distribute alone, codegen
3350  // while (idx <= UB) { BODY; ++idx; }
3351  // when combined with 'for' (e.g. as in 'distribute parallel for')
3352  // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
3353  EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), Cond, IncExpr,
3354  [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3355  CodeGenLoop(CGF, S, LoopExit);
3356  },
3357  [](CodeGenFunction &) {});
3358  EmitBlock(LoopExit.getBlock());
3359  // Tell the runtime we are done.
3360  RT.emitForStaticFinish(*this, S.getLocStart(), S.getDirectiveKind());
3361  } else {
3362  // Emit the outer loop, which requests its work chunk [LB..UB] from
3363  // runtime and runs the inner loop to process it.
3364  const OMPLoopArguments LoopArguments = {
3365  LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(),
3366  Chunk};
3367  EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
3368  CodeGenLoop);
3369  }
3371  EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
3372  return CGF.Builder.CreateIsNotNull(
3373  CGF.EmitLoadOfScalar(IL, S.getLocStart()));
3374  });
3375  }
3379  OpenMPDirectiveKind ReductionKind = OMPD_unknown;
3382  ReductionKind = OMPD_parallel_for_simd;
3383  } else if (isOpenMPParallelDirective(S.getDirectiveKind())) {
3384  ReductionKind = OMPD_parallel_for;
3385  } else if (isOpenMPSimdDirective(S.getDirectiveKind())) {
3386  ReductionKind = OMPD_simd;
3387  } else if (!isOpenMPTeamsDirective(S.getDirectiveKind()) &&
3389  llvm_unreachable(
3390  "No reduction clauses is allowed in distribute directive.");
3391  }
3392  EmitOMPReductionClauseFinal(S, ReductionKind);
3393  // Emit post-update of the reduction variables if IsLastIter != 0.
3395  *this, S, [IL, &S](CodeGenFunction &CGF) {
3396  return CGF.Builder.CreateIsNotNull(
3397  CGF.EmitLoadOfScalar(IL, S.getLocStart()));
3398  });
3399  }
3400  // Emit final copy of the lastprivate variables if IsLastIter != 0.
3401  if (HasLastprivateClause) {
3402  EmitOMPLastprivateClauseFinal(
3403  S, /*NoFinals=*/false,
3404  Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getLocStart())));
3405  }
3406  }
3407 
3408  // We're now done with the loop, so jump to the continuation block.
3409  if (ContBlock) {
3410  EmitBranch(ContBlock);
3411  EmitBlock(ContBlock, true);
3412  }
3413  }
3414 }
3415 
3417  const OMPDistributeDirective &S) {
3418  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3420  };
3421  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3422  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3423 }
3424 
3425 static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM,
3426  const CapturedStmt *S) {
3427  CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
3429  CGF.CapturedStmtInfo = &CapStmtInfo;
3430  llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S);
3431  Fn->setDoesNotRecurse();
3432  return Fn;
3433 }
3434 
3436  if (S.hasClausesOfKind<OMPDependClause>()) {
3437  assert(!S.getAssociatedStmt() &&
3438  "No associated statement must be in ordered depend construct.");
3439  for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
3440  CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
3441  return;
3442  }
3443  const auto *C = S.getSingleClause<OMPSIMDClause>();
3444  auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
3445  PrePostActionTy &Action) {
3446  const CapturedStmt *CS = S.getInnermostCapturedStmt();
3447  if (C) {
3449  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
3450  llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS);
3451  CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(),
3452  OutlinedFn, CapturedVars);
3453  } else {
3454  Action.Enter(CGF);
3455  CGF.EmitStmt(CS->getCapturedStmt());
3456  }
3457  };
3458  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3459  CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getLocStart(), !C);
3460 }
3461 
3463  QualType SrcType, QualType DestType,
3464  SourceLocation Loc) {
3465  assert(CGF.hasScalarEvaluationKind(DestType) &&
3466  "DestType must have scalar evaluation kind.");
3467  assert(!Val.isAggregate() && "Must be a scalar or complex.");
3468  return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
3469  DestType, Loc)
3471  Val.getComplexVal(), SrcType, DestType, Loc);
3472 }
3473 
3476  QualType DestType, SourceLocation Loc) {
3477  assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
3478  "DestType must have complex evaluation kind.");
3479  CodeGenFunction::ComplexPairTy ComplexVal;
3480  if (Val.isScalar()) {
3481  // Convert the input element to the element type of the complex.
3482  QualType DestElementType =
3483  DestType->castAs<ComplexType>()->getElementType();
3484  llvm::Value *ScalarVal = CGF.EmitScalarConversion(
3485  Val.getScalarVal(), SrcType, DestElementType, Loc);
3486  ComplexVal = CodeGenFunction::ComplexPairTy(
3487  ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
3488  } else {
3489  assert(Val.isComplex() && "Must be a scalar or complex.");
3490  QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
3491  QualType DestElementType =
3492  DestType->castAs<ComplexType>()->getElementType();
3493  ComplexVal.first = CGF.EmitScalarConversion(
3494  Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
3495  ComplexVal.second = CGF.EmitScalarConversion(
3496  Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
3497  }
3498  return ComplexVal;
3499 }
3500 
3501 static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst,
3502  LValue LVal, RValue RVal) {
3503  if (LVal.isGlobalReg()) {
3504  CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
3505  } else {
3506  CGF.EmitAtomicStore(RVal, LVal,
3507  IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3508  : llvm::AtomicOrdering::Monotonic,
3509  LVal.isVolatile(), /*IsInit=*/false);
3510  }
3511 }
3512 
3514  QualType RValTy, SourceLocation Loc) {
3515  switch (getEvaluationKind(LVal.getType())) {
3516  case TEK_Scalar:
3517  EmitStoreThroughLValue(RValue::get(convertToScalarValue(
3518  *this, RVal, RValTy, LVal.getType(), Loc)),
3519  LVal);
3520  break;
3521  case TEK_Complex:
3522  EmitStoreOfComplex(
3523  convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,
3524  /*isInit=*/false);
3525  break;
3526  case TEK_Aggregate:
3527  llvm_unreachable("Must be a scalar or complex.");
3528  }
3529 }
3530 
3531 static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst,
3532  const Expr *X, const Expr *V,
3533  SourceLocation Loc) {
3534  // v = x;
3535  assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
3536  assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
3537  LValue XLValue = CGF.EmitLValue(X);
3538  LValue VLValue = CGF.EmitLValue(V);
3539  RValue Res = XLValue.isGlobalReg()
3540  ? CGF.EmitLoadOfLValue(XLValue, Loc)
3541  : CGF.EmitAtomicLoad(
3542  XLValue, Loc,
3543  IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3544  : llvm::AtomicOrdering::Monotonic,
3545  XLValue.isVolatile());
3546  // OpenMP, 2.12.6, atomic Construct
3547  // Any atomic construct with a seq_cst clause forces the atomically
3548  // performed operation to include an implicit flush operation without a
3549  // list.
3550  if (IsSeqCst)
3551  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3552  CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
3553 }
3554 
3555 static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst,
3556  const Expr *X, const Expr *E,
3557  SourceLocation Loc) {
3558  // x = expr;
3559  assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
3560  emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
3561  // OpenMP, 2.12.6, atomic Construct
3562  // Any atomic construct with a seq_cst clause forces the atomically
3563  // performed operation to include an implicit flush operation without a
3564  // list.
3565  if (IsSeqCst)
3566  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3567 }
3568 
3569 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
3570  RValue Update,
3571  BinaryOperatorKind BO,
3572  llvm::AtomicOrdering AO,
3573  bool IsXLHSInRHSPart) {
3574  ASTContext &Context = CGF.getContext();
3575  // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
3576  // expression is simple and atomic is allowed for the given type for the
3577  // target platform.
3578  if (BO == BO_Comma || !Update.isScalar() ||
3579  !Update.getScalarVal()->getType()->isIntegerTy() ||
3580  !X.isSimple() || (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
3581  (Update.getScalarVal()->getType() !=
3582  X.getAddress().getElementType())) ||
3583  !X.getAddress().getElementType()->isIntegerTy() ||
3584  !Context.getTargetInfo().hasBuiltinAtomic(
3585  Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
3586  return std::make_pair(false, RValue::get(nullptr));
3587 
3588  llvm::AtomicRMWInst::BinOp RMWOp;
3589  switch (BO) {
3590  case BO_Add:
3591  RMWOp = llvm::AtomicRMWInst::Add;
3592  break;
3593  case BO_Sub:
3594  if (!IsXLHSInRHSPart)
3595  return std::make_pair(false, RValue::get(nullptr));
3596  RMWOp = llvm::AtomicRMWInst::Sub;
3597  break;
3598  case BO_And:
3599  RMWOp = llvm::AtomicRMWInst::And;
3600  break;
3601  case BO_Or:
3602  RMWOp = llvm::AtomicRMWInst::Or;
3603  break;
3604  case BO_Xor:
3605  RMWOp = llvm::AtomicRMWInst::Xor;
3606  break;
3607  case BO_LT:
3609  ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
3610  : llvm::AtomicRMWInst::Max)
3611  : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
3612  : llvm::AtomicRMWInst::UMax);
3613  break;
3614  case BO_GT:
3616  ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
3617  : llvm::AtomicRMWInst::Min)
3618  : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
3619  : llvm::AtomicRMWInst::UMin);
3620  break;
3621  case BO_Assign:
3622  RMWOp = llvm::AtomicRMWInst::Xchg;
3623  break;
3624  case BO_Mul:
3625  case BO_Div:
3626  case BO_Rem:
3627  case BO_Shl:
3628  case BO_Shr:
3629  case BO_LAnd:
3630  case BO_LOr:
3631  return std::make_pair(false, RValue::get(nullptr));
3632  case BO_PtrMemD:
3633  case BO_PtrMemI:
3634  case BO_LE:
3635  case BO_GE:
3636  case BO_EQ:
3637  case BO_NE:
3638  case BO_Cmp:
3639  case BO_AddAssign:
3640  case BO_SubAssign:
3641  case BO_AndAssign:
3642  case BO_OrAssign:
3643  case BO_XorAssign:
3644  case BO_MulAssign:
3645  case BO_DivAssign:
3646  case BO_RemAssign:
3647  case BO_ShlAssign:
3648  case BO_ShrAssign:
3649  case BO_Comma:
3650  llvm_unreachable("Unsupported atomic update operation");
3651  }
3652  llvm::Value *UpdateVal = Update.getScalarVal();
3653  if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
3654  UpdateVal = CGF.Builder.CreateIntCast(
3655  IC, X.getAddress().getElementType(),
3657  }
3658  llvm::Value *Res =
3659  CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO);
3660  return std::make_pair(true, RValue::get(Res));
3661 }
3662 
3664  LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3665  llvm::AtomicOrdering AO, SourceLocation Loc,
3666  const llvm::function_ref<RValue(RValue)> CommonGen) {
3667  // Update expressions are allowed to have the following forms:
3668  // x binop= expr; -> xrval + expr;
3669  // x++, ++x -> xrval + 1;
3670  // x--, --x -> xrval - 1;
3671  // x = x binop expr; -> xrval binop expr
3672  // x = expr Op x; - > expr binop xrval;
3673  auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);
3674  if (!Res.first) {
3675  if (X.isGlobalReg()) {
3676  // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
3677  // 'xrval'.
3678  EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
3679  } else {
3680  // Perform compare-and-swap procedure.
3681  EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
3682  }
3683  }
3684  return Res;
3685 }
3686 
3687 static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst,
3688  const Expr *X, const Expr *E,
3689  const Expr *UE, bool IsXLHSInRHSPart,
3690  SourceLocation Loc) {
3691  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3692  "Update expr in 'atomic update' must be a binary operator.");
3693  const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3694  // Update expressions are allowed to have the following forms:
3695  // x binop= expr; -> xrval + expr;
3696  // x++, ++x -> xrval + 1;
3697  // x--, --x -> xrval - 1;
3698  // x = x binop expr; -> xrval binop expr
3699  // x = expr Op x; - > expr binop xrval;
3700  assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
3701  LValue XLValue = CGF.EmitLValue(X);
3702  RValue ExprRValue = CGF.EmitAnyExpr(E);
3703  llvm::AtomicOrdering AO = IsSeqCst
3704  ? llvm::AtomicOrdering::SequentiallyConsistent
3705  : llvm::AtomicOrdering::Monotonic;
3706  const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3707  const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3708  const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3709  const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3710  auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {
3711  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3712  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3713  return CGF.EmitAnyExpr(UE);
3714  };
3716  XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3717  // OpenMP, 2.12.6, atomic Construct
3718  // Any atomic construct with a seq_cst clause forces the atomically
3719  // performed operation to include an implicit flush operation without a
3720  // list.
3721  if (IsSeqCst)
3722  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3723 }
3724 
3726  QualType SourceType, QualType ResType,
3727  SourceLocation Loc) {
3728  switch (CGF.getEvaluationKind(ResType)) {
3729  case TEK_Scalar:
3730  return RValue::get(
3731  convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
3732  case TEK_Complex: {
3733  auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
3734  return RValue::getComplex(Res.first, Res.second);
3735  }
3736  case TEK_Aggregate:
3737  break;
3738  }
3739  llvm_unreachable("Must be a scalar or complex.");
3740 }
3741 
3742 static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst,
3743  bool IsPostfixUpdate, const Expr *V,
3744  const Expr *X, const Expr *E,
3745  const Expr *UE, bool IsXLHSInRHSPart,
3746  SourceLocation Loc) {
3747  assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
3748  assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
3749  RValue NewVVal;
3750  LValue VLValue = CGF.EmitLValue(V);
3751  LValue XLValue = CGF.EmitLValue(X);
3752  RValue ExprRValue = CGF.EmitAnyExpr(E);
3753  llvm::AtomicOrdering AO = IsSeqCst
3754  ? llvm::AtomicOrdering::SequentiallyConsistent
3755  : llvm::AtomicOrdering::Monotonic;
3756  QualType NewVValType;
3757  if (UE) {
3758  // 'x' is updated with some additional value.
3759  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3760  "Update expr in 'atomic capture' must be a binary operator.");
3761  const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3762  // Update expressions are allowed to have the following forms:
3763  // x binop= expr; -> xrval + expr;
3764  // x++, ++x -> xrval + 1;
3765  // x--, --x -> xrval - 1;
3766  // x = x binop expr; -> xrval binop expr
3767  // x = expr Op x; - > expr binop xrval;
3768  const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3769  const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3770  const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3771  NewVValType = XRValExpr->getType();
3772  const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3773  auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
3774  IsPostfixUpdate](RValue XRValue) {
3775  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3776  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3777  RValue Res = CGF.EmitAnyExpr(UE);
3778  NewVVal = IsPostfixUpdate ? XRValue : Res;
3779  return Res;
3780  };
3781  auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3782  XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3783  if (Res.first) {
3784  // 'atomicrmw' instruction was generated.
3785  if (IsPostfixUpdate) {
3786  // Use old value from 'atomicrmw'.
3787  NewVVal = Res.second;
3788  } else {
3789  // 'atomicrmw' does not provide new value, so evaluate it using old
3790  // value of 'x'.
3791  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3792  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
3793  NewVVal = CGF.EmitAnyExpr(UE);
3794  }
3795  }
3796  } else {
3797  // 'x' is simply rewritten with some 'expr'.
3798  NewVValType = X->getType().getNonReferenceType();
3799  ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
3800  X->getType().getNonReferenceType(), Loc);
3801  auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) {
3802  NewVVal = XRValue;
3803  return ExprRValue;
3804  };
3805  // Try to perform atomicrmw xchg, otherwise simple exchange.
3806  auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3807  XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
3808  Loc, Gen);
3809  if (Res.first) {
3810  // 'atomicrmw' instruction was generated.
3811  NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
3812  }
3813  }
3814  // Emit post-update store to 'v' of old/new 'x' value.
3815  CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
3816  // OpenMP, 2.12.6, atomic Construct
3817  // Any atomic construct with a seq_cst clause forces the atomically
3818  // performed operation to include an implicit flush operation without a
3819  // list.
3820  if (IsSeqCst)
3821  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3822 }
3823 
3825  bool IsSeqCst, bool IsPostfixUpdate,
3826  const Expr *X, const Expr *V, const Expr *E,
3827  const Expr *UE, bool IsXLHSInRHSPart,
3828  SourceLocation Loc) {
3829  switch (Kind) {
3830  case OMPC_read:
3831  emitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc);
3832  break;
3833  case OMPC_write:
3834  emitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc);
3835  break;
3836  case OMPC_unknown:
3837  case OMPC_update:
3838  emitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc);
3839  break;
3840  case OMPC_capture:
3841  emitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE,
3842  IsXLHSInRHSPart, Loc);
3843  break;
3844  case OMPC_if:
3845  case OMPC_final:
3846  case OMPC_num_threads:
3847  case OMPC_private:
3848  case OMPC_firstprivate:
3849  case OMPC_lastprivate:
3850  case OMPC_reduction:
3851  case OMPC_task_reduction:
3852  case OMPC_in_reduction:
3853  case OMPC_safelen:
3854  case OMPC_simdlen:
3855  case OMPC_collapse:
3856  case OMPC_default:
3857  case OMPC_seq_cst:
3858  case OMPC_shared:
3859  case OMPC_linear:
3860  case OMPC_aligned:
3861  case OMPC_copyin:
3862  case OMPC_copyprivate:
3863  case OMPC_flush:
3864  case OMPC_proc_bind:
3865  case OMPC_schedule:
3866  case OMPC_ordered:
3867  case OMPC_nowait:
3868  case OMPC_untied:
3869  case OMPC_threadprivate:
3870  case OMPC_depend:
3871  case OMPC_mergeable:
3872  case OMPC_device:
3873  case OMPC_threads:
3874  case OMPC_simd:
3875  case OMPC_map:
3876  case OMPC_num_teams:
3877  case OMPC_thread_limit:
3878  case OMPC_priority:
3879  case OMPC_grainsize:
3880  case OMPC_nogroup:
3881  case OMPC_num_tasks:
3882  case OMPC_hint:
3883  case OMPC_dist_schedule:
3884  case OMPC_defaultmap:
3885  case OMPC_uniform:
3886  case OMPC_to:
3887  case OMPC_from:
3888  case OMPC_use_device_ptr:
3889  case OMPC_is_device_ptr:
3890  llvm_unreachable("Clause is not allowed in 'omp atomic'.");
3891  }
3892 }
3893 
3895  bool IsSeqCst = S.getSingleClause<OMPSeqCstClause>();
3897  for (const OMPClause *C : S.clauses()) {
3898  // Find first clause (skip seq_cst clause, if it is first).
3899  if (C->getClauseKind() != OMPC_seq_cst) {
3900  Kind = C->getClauseKind();
3901  break;
3902  }
3903  }
3904 
3905  const Stmt *CS = S.getInnermostCapturedStmt()->IgnoreContainers();
3906  if (const auto *EWC = dyn_cast<ExprWithCleanups>(CS))
3907  enterFullExpression(EWC);
3908  // Processing for statements under 'atomic capture'.
3909  if (const auto *Compound = dyn_cast<CompoundStmt>(CS)) {
3910  for (const Stmt *C : Compound->body()) {
3911  if (const auto *EWC = dyn_cast<ExprWithCleanups>(C))
3912  enterFullExpression(EWC);
3913  }
3914  }
3915 
3916  auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF,
3917  PrePostActionTy &) {
3918  CGF.EmitStopPoint(CS);
3919  emitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(),
3920  S.getV(), S.getExpr(), S.getUpdateExpr(),
3921  S.isXLHSInRHSPart(), S.getLocStart());
3922  };
3923  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3924  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen);
3925 }
3926 
3928  const OMPExecutableDirective &S,
3929  const RegionCodeGenTy &CodeGen) {
3931  CodeGenModule &CGM = CGF.CGM;
3932 
3933  // On device emit this construct as inlined code.
3934  if (CGM.getLangOpts().OpenMPIsDevice) {
3935  OMPLexicalScope Scope(CGF, S, OMPD_target);
3936  CGM.getOpenMPRuntime().emitInlinedDirective(
3937  CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3939  });
3940  return;
3941  }
3942 
3943  llvm::Function *Fn = nullptr;
3944  llvm::Constant *FnID = nullptr;
3945 
3946  const Expr *IfCond = nullptr;
3947  // Check for the at most one if clause associated with the target region.
3948  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3949  if (C->getNameModifier() == OMPD_unknown ||
3950  C->getNameModifier() == OMPD_target) {
3951  IfCond = C->getCondition();
3952  break;
3953  }
3954  }
3955 
3956  // Check if we have any device clause associated with the directive.
3957  const Expr *Device = nullptr;
3958  if (auto *C = S.getSingleClause<OMPDeviceClause>())
3959  Device = C->getDevice();
3960 
3961  // Check if we have an if clause whose conditional always evaluates to false
3962  // or if we do not have any targets specified. If so the target region is not
3963  // an offload entry point.
3964  bool IsOffloadEntry = true;
3965  if (IfCond) {
3966  bool Val;
3967  if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
3968  IsOffloadEntry = false;
3969  }
3970  if (CGM.getLangOpts().OMPTargetTriples.empty())
3971  IsOffloadEntry = false;
3972 
3973  assert(CGF.CurFuncDecl && "No parent declaration for target region!");
3974  StringRef ParentName;
3975  // In case we have Ctors/Dtors we use the complete type variant to produce
3976  // the mangling of the device outlined kernel.
3977  if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
3978  ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));
3979  else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
3980  ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));
3981  else
3982  ParentName =
3983  CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));
3984 
3985  // Emit target region as a standalone region.
3986  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
3987  IsOffloadEntry, CodeGen);
3988  OMPLexicalScope Scope(CGF, S, OMPD_task);
3989  CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device);
3990 }
3991 
3993  PrePostActionTy &Action) {
3994  Action.Enter(CGF);
3995  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
3996  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
3997  CGF.EmitOMPPrivateClause(S, PrivateScope);
3998  (void)PrivateScope.Privatize();
3999 
4000  CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
4001 }
4002 
4004  StringRef ParentName,
4005  const OMPTargetDirective &S) {
4006  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4007  emitTargetRegion(CGF, S, Action);
4008  };
4009  llvm::Function *Fn;
4010  llvm::Constant *Addr;
4011  // Emit target region as a standalone region.
4012  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4013  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4014  assert(Fn && Addr && "Target device function emission failed.");
4015 }
4016 
4018  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4019  emitTargetRegion(CGF, S, Action);
4020  };
4021  emitCommonOMPTargetDirective(*this, S, CodeGen);
4022 }
4023 
4025  const OMPExecutableDirective &S,
4026  OpenMPDirectiveKind InnermostKind,
4027  const RegionCodeGenTy &CodeGen) {
4028  const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
4029  llvm::Value *OutlinedFn =
4030  CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
4031  S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
4032 
4033  const auto *NT = S.getSingleClause<OMPNumTeamsClause>();
4034  const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
4035  if (NT || TL) {
4036  const Expr *NumTeams = NT ? NT->getNumTeams() : nullptr;
4037  const Expr *ThreadLimit = TL ? TL->getThreadLimit() : nullptr;
4038 
4039  CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
4040  S.getLocStart());
4041  }
4042 
4043  OMPTeamsScope Scope(CGF, S);
4045  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
4046  CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getLocStart(), OutlinedFn,
4047  CapturedVars);
4048 }
4049 
4051  // Emit teams region as a standalone region.
4052  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4053  Action.Enter(CGF);
4054  OMPPrivateScope PrivateScope(CGF);
4055  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4056  CGF.EmitOMPPrivateClause(S, PrivateScope);
4057  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4058  (void)PrivateScope.Privatize();
4059  CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
4060  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4061  };
4062  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4064  [](CodeGenFunction &) { return nullptr; });
4065 }
4066 
4068  const OMPTargetTeamsDirective &S) {
4069  auto *CS = S.getCapturedStmt(OMPD_teams);
4070  Action.Enter(CGF);
4071  // Emit teams region as a standalone region.
4072  auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4073  Action.Enter(CGF);
4074  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4075  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4076  CGF.EmitOMPPrivateClause(S, PrivateScope);
4077  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4078  (void)PrivateScope.Privatize();
4079  CGF.EmitStmt(CS->getCapturedStmt());
4080  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4081  };
4082  emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
4084  [](CodeGenFunction &) { return nullptr; });
4085 }
4086 
4088  CodeGenModule &CGM, StringRef ParentName,
4089  const OMPTargetTeamsDirective &S) {
4090  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4091  emitTargetTeamsRegion(CGF, Action, S);
4092  };
4093  llvm::Function *Fn;
4094  llvm::Constant *Addr;
4095  // Emit target region as a standalone region.
4096  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4097  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4098  assert(Fn && Addr && "Target device function emission failed.");
4099 }
4100 
4102  const OMPTargetTeamsDirective &S) {
4103  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4104  emitTargetTeamsRegion(CGF, Action, S);
4105  };
4106  emitCommonOMPTargetDirective(*this, S, CodeGen);
4107 }
4108 
4109 static void
4112  Action.Enter(CGF);
4113  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4115  };
4116 
4117  // Emit teams region as a standalone region.
4118  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4119  PrePostActionTy &Action) {
4120  Action.Enter(CGF);
4121  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4122  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4123  (void)PrivateScope.Privatize();
4124  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4125  CodeGenDistribute);
4126  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4127  };
4128  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
4130  [](CodeGenFunction &) { return nullptr; });
4131 }
4132 
4134  CodeGenModule &CGM, StringRef ParentName,
4136  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4137  emitTargetTeamsDistributeRegion(CGF, Action, S);
4138  };
4139  llvm::Function *Fn;
4140  llvm::Constant *Addr;
4141  // Emit target region as a standalone region.
4142  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4143  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4144  assert(Fn && Addr && "Target device function emission failed.");
4145 }
4146 
4149  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4150  emitTargetTeamsDistributeRegion(CGF, Action, S);
4151  };
4152  emitCommonOMPTargetDirective(*this, S, CodeGen);
4153 }
4154 
4156  CodeGenFunction &CGF, PrePostActionTy &Action,
4158  Action.Enter(CGF);
4159  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4161  };
4162 
4163  // Emit teams region as a standalone region.
4164  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4165  PrePostActionTy &Action) {
4166  Action.Enter(CGF);
4167  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4168  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4169  (void)PrivateScope.Privatize();
4170  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4171  CodeGenDistribute);
4172  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4173  };
4174  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);
4176  [](CodeGenFunction &) { return nullptr; });
4177 }
4178 
4180  CodeGenModule &CGM, StringRef ParentName,
4182  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4183  emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4184  };
4185  llvm::Function *Fn;
4186  llvm::Constant *Addr;
4187  // Emit target region as a standalone region.
4188  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4189  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4190  assert(Fn && Addr && "Target device function emission failed.");
4191 }
4192 
4195  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4196  emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4197  };
4198  emitCommonOMPTargetDirective(*this, S, CodeGen);
4199 }
4200 
4202  const OMPTeamsDistributeDirective &S) {
4203 
4204  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4206  };
4207 
4208  // Emit teams region as a standalone region.
4209  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4210  PrePostActionTy &Action) {
4211  Action.Enter(CGF);
4212  OMPPrivateScope PrivateScope(CGF);
4213  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4214  (void)PrivateScope.Privatize();
4215  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4216  CodeGenDistribute);
4217  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4218  };
4219  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4221  [](CodeGenFunction &) { return nullptr; });
4222 }
4223 
4226  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4228  };
4229 
4230  // Emit teams region as a standalone region.
4231  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4232  PrePostActionTy &Action) {
4233  Action.Enter(CGF);
4234  OMPPrivateScope PrivateScope(CGF);
4235  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4236  (void)PrivateScope.Privatize();
4237  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
4238  CodeGenDistribute);
4239  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4240  };
4241  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen);
4243  [](CodeGenFunction &) { return nullptr; });
4244 }
4245 
4248  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4250  S.getDistInc());
4251  };
4252 
4253  // Emit teams region as a standalone region.
4254  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4255  PrePostActionTy &Action) {
4256  Action.Enter(CGF);
4257  OMPPrivateScope PrivateScope(CGF);
4258  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4259  (void)PrivateScope.Privatize();
4260  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4261  CodeGenDistribute);
4262  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4263  };
4264  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4266  [](CodeGenFunction &) { return nullptr; });
4267 }
4268 
4271  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4273  S.getDistInc());
4274  };
4275 
4276  // Emit teams region as a standalone region.
4277  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4278  PrePostActionTy &Action) {
4279  Action.Enter(CGF);
4280  OMPPrivateScope PrivateScope(CGF);
4281  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4282  (void)PrivateScope.Privatize();
4283  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4284  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4285  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4286  };
4287  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4289  [](CodeGenFunction &) { return nullptr; });
4290 }
4291 
4294  PrePostActionTy &Action) {
4295  Action.Enter(CGF);
4296  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4298  S.getDistInc());
4299  };
4300 
4301  // Emit teams region as a standalone region.
4302  auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4303  PrePostActionTy &Action) {
4304  Action.Enter(CGF);
4305  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4306  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4307  (void)PrivateScope.Privatize();
4308  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4309  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4310  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4311  };
4312 
4313  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
4314  CodeGenTeams);
4316  [](CodeGenFunction &) { return nullptr; });
4317 }
4318 
4320  CodeGenModule &CGM, StringRef ParentName,
4322  // Emit SPMD target teams distribute parallel for region as a standalone
4323  // region.
4324  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4326  };
4327  llvm::Function *Fn;
4328  llvm::Constant *Addr;
4329  // Emit target region as a standalone region.
4330  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4331  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4332  assert(Fn && Addr && "Target device function emission failed.");
4333 }
4334 
4337  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4339  };
4340  emitCommonOMPTargetDirective(*this, S, CodeGen);
4341 }
4342 
4344  CodeGenFunction &CGF,
4346  PrePostActionTy &Action) {
4347  Action.Enter(CGF);
4348  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4350  S.getDistInc());
4351  };
4352 
4353  // Emit teams region as a standalone region.
4354  auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4355  PrePostActionTy &Action) {
4356  Action.Enter(CGF);
4357  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4358  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4359  (void)PrivateScope.Privatize();
4360  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4361  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4362  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4363  };
4364 
4365  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd,
4366  CodeGenTeams);
4368  [](CodeGenFunction &) { return nullptr; });
4369 }
4370 
4372  CodeGenModule &CGM, StringRef ParentName,
4374  // Emit SPMD target teams distribute parallel for simd region as a standalone
4375  // region.
4376  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4378  };
4379  llvm::Function *Fn;
4380  llvm::Constant *Addr;
4381  // Emit target region as a standalone region.
4382  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4383  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4384  assert(Fn && Addr && "Target device function emission failed.");
4385 }
4386 
4389  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4391  };
4392  emitCommonOMPTargetDirective(*this, S, CodeGen);
4393 }
4394 
4396  const OMPCancellationPointDirective &S) {
4397  CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getLocStart(),
4398  S.getCancelRegion());
4399 }
4400 
4402  const Expr *IfCond = nullptr;
4403  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4404  if (C->getNameModifier() == OMPD_unknown ||
4405  C->getNameModifier() == OMPD_cancel) {
4406  IfCond = C->getCondition();
4407  break;
4408  }
4409  }
4410  CGM.getOpenMPRuntime().emitCancelCall(*this, S.getLocStart(), IfCond,
4411  S.getCancelRegion());
4412 }
4413 
4416  if (Kind == OMPD_parallel || Kind == OMPD_task ||
4417  Kind == OMPD_target_parallel)
4418  return ReturnBlock;
4419  assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
4420  Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
4421  Kind == OMPD_distribute_parallel_for ||
4422  Kind == OMPD_target_parallel_for ||
4423  Kind == OMPD_teams_distribute_parallel_for ||
4424  Kind == OMPD_target_teams_distribute_parallel_for);
4425  return OMPCancelStack.getExitBlock();
4426 }
4427 
4429  const OMPClause &NC, OMPPrivateScope &PrivateScope,
4430  const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) {
4431  const auto &C = cast<OMPUseDevicePtrClause>(NC);
4432  auto OrigVarIt = C.varlist_begin();
4433  auto InitIt = C.inits().begin();
4434  for (const Expr *PvtVarIt : C.private_copies()) {
4435  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl());
4436  const auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl());
4437  const auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl());
4438 
4439  // In order to identify the right initializer we need to match the
4440  // declaration used by the mapping logic. In some cases we may get
4441  // OMPCapturedExprDecl that refers to the original declaration.
4442  const ValueDecl *MatchingVD = OrigVD;
4443  if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
4444  // OMPCapturedExprDecl are used to privative fields of the current
4445  // structure.
4446  const auto *ME = cast<MemberExpr>(OED->getInit());
4447  assert(isa<CXXThisExpr>(ME->getBase()) &&
4448  "Base should be the current struct!");
4449  MatchingVD = ME->getMemberDecl();
4450  }
4451 
4452  // If we don't have information about the current list item, move on to
4453  // the next one.
4454  auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
4455  if (InitAddrIt == CaptureDeviceAddrMap.end())
4456  continue;
4457 
4458  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, OrigVD,
4459  InitAddrIt, InitVD,
4460  PvtVD]() {
4461  // Initialize the temporary initialization variable with the address we
4462  // get from the runtime library. We have to cast the source address
4463  // because it is always a void *. References are materialized in the
4464  // privatization scope, so the initialization here disregards the fact
4465  // the original variable is a reference.
4466  QualType AddrQTy =
4467  getContext().getPointerType(OrigVD->getType().getNonReferenceType());
4468  llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy);
4469  Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy);
4470  setAddrOfLocalVar(InitVD, InitAddr);
4471 
4472  // Emit private declaration, it will be initialized by the value we
4473  // declaration we just added to the local declarations map.
4474  EmitDecl(*PvtVD);
4475 
4476  // The initialization variables reached its purpose in the emission
4477  // of the previous declaration, so we don't need it anymore.
4478  LocalDeclMap.erase(InitVD);
4479 
4480  // Return the address of the private variable.
4481  return GetAddrOfLocalVar(PvtVD);
4482  });
4483  assert(IsRegistered && "firstprivate var already registered as private");
4484  // Silence the warning about unused variable.
4485  (void)IsRegistered;
4486 
4487  ++OrigVarIt;
4488  ++InitIt;
4489  }
4490 }
4491 
4492 // Generate the instructions for '#pragma omp target data' directive.
4494  const OMPTargetDataDirective &S) {
4495  CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true);
4496 
4497  // Create a pre/post action to signal the privatization of the device pointer.
4498  // This action can be replaced by the OpenMP runtime code generation to
4499  // deactivate privatization.
4500  bool PrivatizeDevicePointers = false;
4501  class DevicePointerPrivActionTy : public PrePostActionTy {
4502  bool &PrivatizeDevicePointers;
4503 
4504  public:
4505  explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
4506  : PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {}
4507  void Enter(CodeGenFunction &CGF) override {
4508  PrivatizeDevicePointers = true;
4509  }
4510  };
4511  DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
4512 
4513  auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers](
4514  CodeGenFunction &CGF, PrePostActionTy &Action) {
4515  auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4517  };
4518 
4519  // Codegen that selects whether to generate the privatization code or not.
4520  auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers,
4521  &InnermostCodeGen](CodeGenFunction &CGF,
4522  PrePostActionTy &Action) {
4523  RegionCodeGenTy RCG(InnermostCodeGen);
4524  PrivatizeDevicePointers = false;
4525 
4526  // Call the pre-action to change the status of PrivatizeDevicePointers if
4527  // needed.
4528  Action.Enter(CGF);
4529 
4530  if (PrivatizeDevicePointers) {
4531  OMPPrivateScope PrivateScope(CGF);
4532  // Emit all instances of the use_device_ptr clause.
4533  for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
4534  CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
4535  Info.CaptureDeviceAddrMap);
4536  (void)PrivateScope.Privatize();
4537  RCG(CGF);
4538  } else {
4539  RCG(CGF);
4540  }
4541  };
4542 
4543  // Forward the provided action to the privatization codegen.
4544  RegionCodeGenTy PrivRCG(PrivCodeGen);
4545  PrivRCG.setAction(Action);
4546 
4547  // Notwithstanding the body of the region is emitted as inlined directive,
4548  // we don't use an inline scope as changes in the references inside the
4549  // region are expected to be visible outside, so we do not privative them.
4550  OMPLexicalScope Scope(CGF, S);
4551  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
4552  PrivRCG);
4553  };
4554 
4555  RegionCodeGenTy RCG(CodeGen);
4556 
4557  // If we don't have target devices, don't bother emitting the data mapping
4558  // code.
4559  if (CGM.getLangOpts().OMPTargetTriples.empty()) {
4560  RCG(*this);
4561  return;
4562  }
4563 
4564  // Check if we have any if clause associated with the directive.
4565  const Expr *IfCond = nullptr;
4566  if (const auto *C = S.getSingleClause<OMPIfClause>())
4567  IfCond = C->getCondition();
4568 
4569  // Check if we have any device clause associated with the directive.
4570  const Expr *Device = nullptr;
4571  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4572  Device = C->getDevice();
4573 
4574  // Set the action to signal privatization of device pointers.
4575  RCG.setAction(PrivAction);
4576 
4577  // Emit region code.
4578  CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
4579  Info);
4580 }
4581 
4583  const OMPTargetEnterDataDirective &S) {
4584  // If we don't have target devices, don't bother emitting the data mapping
4585  // code.
4586  if (CGM.getLangOpts().OMPTargetTriples.empty())
4587  return;
4588 
4589  // Check if we have any if clause associated with the directive.
4590  const Expr *IfCond = nullptr;
4591  if (const auto *C = S.getSingleClause<OMPIfClause>())
4592  IfCond = C->getCondition();
4593 
4594  // Check if we have any device clause associated with the directive.
4595  const Expr *Device = nullptr;
4596  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4597  Device = C->getDevice();
4598 
4599  OMPLexicalScope Scope(*this, S, OMPD_task);
4600  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4601 }
4602 
4604  const OMPTargetExitDataDirective &S) {
4605  // If we don't have target devices, don't bother emitting the data mapping
4606  // code.
4607  if (CGM.getLangOpts().OMPTargetTriples.empty())
4608  return;
4609 
4610  // Check if we have any if clause associated with the directive.
4611  const Expr *IfCond = nullptr;
4612  if (const auto *C = S.getSingleClause<OMPIfClause>())
4613  IfCond = C->getCondition();
4614 
4615  // Check if we have any device clause associated with the directive.
4616  const Expr *Device = nullptr;
4617  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4618  Device = C->getDevice();
4619 
4620  OMPLexicalScope Scope(*this, S, OMPD_task);
4621  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4622 }
4623 
4625  const OMPTargetParallelDirective &S,
4626  PrePostActionTy &Action) {
4627  // Get the captured statement associated with the 'parallel' region.
4628  const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
4629  Action.Enter(CGF);
4630  auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4631  Action.Enter(CGF);
4632  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4633  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4634  CGF.EmitOMPPrivateClause(S, PrivateScope);
4635  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4636  (void)PrivateScope.Privatize();
4637  // TODO: Add support for clauses.
4638  CGF.EmitStmt(CS->getCapturedStmt());
4639  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4640  };
4641  emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
4644  [](CodeGenFunction &) { return nullptr; });
4645 }
4646 
4648  CodeGenModule &CGM, StringRef ParentName,
4649  const OMPTargetParallelDirective &S) {
4650  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4651  emitTargetParallelRegion(CGF, S, Action);
4652  };
4653  llvm::Function *Fn;
4654  llvm::Constant *Addr;
4655  // Emit target region as a standalone region.
4656  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4657  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4658  assert(Fn && Addr && "Target device function emission failed.");
4659 }
4660 
4662  const OMPTargetParallelDirective &S) {
4663  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4664  emitTargetParallelRegion(CGF, S, Action);
4665  };
4666  emitCommonOMPTargetDirective(*this, S, CodeGen);
4667 }
4668 
4671  PrePostActionTy &Action) {
4672  Action.Enter(CGF);
4673  // Emit directive as a combined directive that consists of two implicit
4674  // directives: 'parallel' with 'for' directive.
4675  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4676  Action.Enter(CGF);
4678  CGF, OMPD_target_parallel_for, S.hasCancel());
4681  };
4682  emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
4684 }
4685 
4687  CodeGenModule &CGM, StringRef ParentName,
4688  const OMPTargetParallelForDirective &S) {
4689  // Emit SPMD target parallel for region as a standalone region.
4690  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4691  emitTargetParallelForRegion(CGF, S, Action);
4692  };
4693  llvm::Function *Fn;
4694  llvm::Constant *Addr;
4695  // Emit target region as a standalone region.
4696  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4697  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4698  assert(Fn && Addr && "Target device function emission failed.");
4699 }
4700 
4702  const OMPTargetParallelForDirective &S) {
4703  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4704  emitTargetParallelForRegion(CGF, S, Action);
4705  };
4706  emitCommonOMPTargetDirective(*this, S, CodeGen);
4707 }
4708 
4709 static void
4712  PrePostActionTy &Action) {
4713  Action.Enter(CGF);
4714  // Emit directive as a combined directive that consists of two implicit
4715  // directives: 'parallel' with 'for' directive.
4716  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4717  Action.Enter(CGF);
4720  };
4721  emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
4723 }
4724 
4726  CodeGenModule &CGM, StringRef ParentName,
4728  // Emit SPMD target parallel for region as a standalone region.
4729  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4730  emitTargetParallelForSimdRegion(CGF, S, Action);
4731  };
4732  llvm::Function *Fn;
4733  llvm::Constant *Addr;
4734  // Emit target region as a standalone region.
4735  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4736  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4737  assert(Fn && Addr && "Target device function emission failed.");
4738 }
4739 
4742  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4743  emitTargetParallelForSimdRegion(CGF, S, Action);
4744  };
4745  emitCommonOMPTargetDirective(*this, S, CodeGen);
4746 }
4747 
4748 /// Emit a helper variable and return corresponding lvalue.
4749 static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
4750  const ImplicitParamDecl *PVD,
4752  const auto *VDecl = cast<VarDecl>(Helper->getDecl());
4753  Privates.addPrivate(VDecl,
4754  [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); });
4755 }
4756 
4759  // Emit outlined function for task construct.
4760  const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);
4761  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
4762  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
4763  const Expr *IfCond = nullptr;
4764  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4765  if (C->getNameModifier() == OMPD_unknown ||
4766  C->getNameModifier() == OMPD_taskloop) {
4767  IfCond = C->getCondition();
4768  break;
4769  }
4770  }
4771 
4772  OMPTaskDataTy Data;
4773  // Check if taskloop must be emitted without taskgroup.
4775  // TODO: Check if we should emit tied or untied task.
4776  Data.Tied = true;
4777  // Set scheduling for taskloop
4778  if (const auto* Clause = S.getSingleClause<OMPGrainsizeClause>()) {
4779  // grainsize clause
4780  Data.Schedule.setInt(/*IntVal=*/false);
4781  Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
4782  } else if (const auto* Clause = S.getSingleClause<OMPNumTasksClause>()) {
4783  // num_tasks clause
4784  Data.Schedule.setInt(/*IntVal=*/true);
4785  Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
4786  }
4787 
4788  auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
4789  // if (PreCond) {
4790  // for (IV in 0..LastIteration) BODY;
4791  // <Final counter/linear vars updates>;
4792  // }
4793  //
4794 
4795  // Emit: if (PreCond) - begin.
4796  // If the condition constant folds and can be elided, avoid emitting the
4797  // whole loop.
4798  bool CondConstant;
4799  llvm::BasicBlock *ContBlock = nullptr;
4800  OMPLoopScope PreInitScope(CGF, S);
4801  if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
4802  if (!CondConstant)
4803  return;
4804  } else {
4805  llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
4806  ContBlock = CGF.createBasicBlock("taskloop.if.end");
4807  emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
4808  CGF.getProfileCount(&S));
4809  CGF.EmitBlock(ThenBlock);
4810  CGF.incrementProfileCounter(&S);
4811  }
4812 
4814  CGF.EmitOMPSimdInit(S);
4815 
4816  OMPPrivateScope LoopScope(CGF);
4817  // Emit helper vars inits.
4818  enum { LowerBound = 5, UpperBound, Stride, LastIter };
4819  auto *I = CS->getCapturedDecl()->param_begin();
4820  auto *LBP = std::next(I, LowerBound);
4821  auto *UBP = std::next(I, UpperBound);
4822  auto *STP = std::next(I, Stride);
4823  auto *LIP = std::next(I, LastIter);
4824  mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
4825  LoopScope);
4826  mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
4827  LoopScope);
4828  mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
4829  mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
4830  LoopScope);
4831  CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
4832  bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4833  (void)LoopScope.Privatize();
4834  // Emit the loop iteration variable.
4835  const Expr *IVExpr = S.getIterationVariable();
4836  const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
4837  CGF.EmitVarDecl(*IVDecl);
4838  CGF.EmitIgnoredExpr(S.getInit());
4839 
4840  // Emit the iterations count variable.
4841  // If it is not a variable, Sema decided to calculate iterations count on
4842  // each iteration (e.g., it is foldable into a constant).
4843  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
4844  CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
4845  // Emit calculation of the iterations count.
4846  CGF.EmitIgnoredExpr(S.getCalcLastIteration());
4847  }
4848 
4849  CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
4850  S.getInc(),
4851  [&S](CodeGenFunction &CGF) {
4852  CGF.EmitOMPLoopBody(S, JumpDest());
4853  CGF.EmitStopPoint(&S);
4854  },
4855  [](CodeGenFunction &) {});
4856  // Emit: if (PreCond) - end.
4857  if (ContBlock) {
4858  CGF.EmitBranch(ContBlock);
4859  CGF.EmitBlock(ContBlock, true);
4860  }
4861  // Emit final copy of the lastprivate variables if IsLastIter != 0.
4862  if (HasLastprivateClause) {
4863  CGF.EmitOMPLastprivateClauseFinal(
4865  CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
4866  CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
4867  (*LIP)->getType(), S.getLocStart())));
4868  }
4869  };
4870  auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
4871  IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
4872  const OMPTaskDataTy &Data) {
4873  auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
4874  &Data](CodeGenFunction &CGF, PrePostActionTy &) {
4875  OMPLoopScope PreInitScope(CGF, S);
4876  CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getLocStart(), S,
4877  OutlinedFn, SharedsTy,
4878  CapturedStruct, IfCond, Data);
4879  };
4880  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
4881  CodeGen);
4882  };
4883  if (Data.Nogroup) {
4884  EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data);
4885  } else {
4886  CGM.getOpenMPRuntime().emitTaskgroupRegion(
4887  *this,
4888  [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
4889  PrePostActionTy &Action) {
4890  Action.Enter(CGF);
4891  CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen,
4892  Data);
4893  },
4894  S.getLocStart());
4895  }
4896 }
4897 
4899  EmitOMPTaskLoopBasedDirective(S);
4900 }
4901 
4903  const OMPTaskLoopSimdDirective &S) {
4904  EmitOMPTaskLoopBasedDirective(S);
4905 }
4906 
4907 // Generate the instructions for '#pragma omp target update' directive.
4909  const OMPTargetUpdateDirective &S) {
4910  // If we don't have target devices, don't bother emitting the data mapping
4911  // code.
4912  if (CGM.getLangOpts().OMPTargetTriples.empty())
4913  return;
4914 
4915  // Check if we have any if clause associated with the directive.
4916  const Expr *IfCond = nullptr;
4917  if (const auto *C = S.getSingleClause<OMPIfClause>())
4918  IfCond = C->getCondition();
4919 
4920  // Check if we have any device clause associated with the directive.
4921  const Expr *Device = nullptr;
4922  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4923  Device = C->getDevice();
4924 
4925  OMPLexicalScope Scope(*this, S, OMPD_task);
4926  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4927 }
4928 
4930  const OMPExecutableDirective &D) {
4931  if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
4932  return;
4933  auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {
4935  emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action);
4936  } else {
4937  if (const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
4938  for (const Expr *E : LD->counters()) {
4939  if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
4940  cast<DeclRefExpr>(E)->getDecl())) {
4941  // Emit only those that were not explicitly referenced in clauses.
4942  if (!CGF.LocalDeclMap.count(VD))
4943  CGF.EmitVarDecl(*VD);
4944  }
4945  }
4946  }
4948  }
4949  };
4950  OMPSimdLexicalScope Scope(*this, D);
4951  CGM.getOpenMPRuntime().emitInlinedDirective(
4952  *this,
4953  isOpenMPSimdDirective(D.getDirectiveKind()) ? OMPD_simd
4954  : D.getDirectiveKind(),
4955  CodeGen);
4956 }
4957 
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
Definition: CGCall.cpp:653
bool isAggregate() const
Definition: CGValue.h:54
This represents &#39;#pragma omp distribute simd&#39; composite directive.
Definition: StmtOpenMP.h:3212
Expr * getNextUpperBound() const
Definition: StmtOpenMP.h:841
This represents &#39;#pragma omp master&#39; directive.
Definition: StmtOpenMP.h:1395
This represents &#39;#pragma omp task&#39; directive.
Definition: StmtOpenMP.h:1735
static const Decl * getCanonicalDecl(const Decl *D)
Represents a function declaration or definition.
Definition: Decl.h:1714
This represents &#39;thread_limit&#39; clause in the &#39;#pragma omp ...&#39; directive.
Expr * getUpperBoundVariable() const
Definition: StmtOpenMP.h:809
Other implicit parameter.
Definition: Decl.h:1493
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, SourceLocation Loc)
This represents clause &#39;copyin&#39; in the &#39;#pragma omp ...&#39; directives.
Complete object ctor.
Definition: ABI.h:26
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition: Type.h:2375
Scheduling data for loop-based OpenMP directives.
Definition: OpenMPKinds.h:124
A (possibly-)qualified type.
Definition: Type.h:655
capture_init_iterator capture_init_begin()
Retrieve the first initialization argument.
Definition: Stmt.h:2244
bool isArrayType() const
Definition: Type.h:6098
ArrayRef< OMPClause * > clauses()
Definition: StmtOpenMP.h:258
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *V, SourceLocation Loc)
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
Definition: CGDecl.cpp:156
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
Definition: Decl.h:4095
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Definition: CGExpr.cpp:139
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
Definition: Dominators.h:30
static std::pair< LValue, LValue > emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static OMPClauseWithPreInit * get(OMPClause *C)
Stmt - This represents one statement.
Definition: Stmt.h:66
This represents clause &#39;in_reduction&#39; in the &#39;#pragma omp task&#39; directives.
Expr * getLowerBoundVariable() const
Definition: StmtOpenMP.h:801
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
Store of global named registers are always calls to intrinsics.
Definition: CGExpr.cpp:2100
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
Definition: Type.cpp:460
bool requiresCleanups() const
Determine whether this scope requires any cleanups.
void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S)
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
SmallVector< std::pair< OpenMPDependClauseKind, const Expr * >, 4 > Dependences
This represents &#39;#pragma omp for simd&#39; directive.
Definition: StmtOpenMP.h:1145
SmallVector< const Expr *, 4 > LastprivateCopies
QualType getLValueReferenceType(QualType T, bool SpelledAsLValue=true) const
Return the uniqued reference to the type for an lvalue reference to the specified type...
Decl - This represents one declaration (or definition), e.g.
Definition: DeclBase.h:86
This represents &#39;grainsize&#39; clause in the &#39;#pragma omp ...&#39; directive.
This represents &#39;#pragma omp teams distribute parallel for&#39; composite directive.
Definition: StmtOpenMP.h:3623
bool isNothrow() const
Definition: Decl.cpp:4342
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
This represents &#39;if&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:240
CapturedStmt * getInnermostCapturedStmt()
Get innermost captured statement for the construct.
Definition: StmtOpenMP.h:226
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Definition: Type.h:5998
void EmitOMPOrderedDirective(const OMPOrderedDirective &S)
bool isVolatile() const
Definition: CGValue.h:301
This represents &#39;priority&#39; clause in the &#39;#pragma omp ...&#39; directive.
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
The base class of the type hierarchy.
Definition: Type.h:1421
This represents &#39;#pragma omp target teams distribute&#39; combined directive.
Definition: StmtOpenMP.h:3760
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
Definition: CGExpr.cpp:1865
static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, bool IsSeqCst, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Definition: Stmt.h:2188
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Definition: Type.h:2650
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
const TargetInfo & getTargetInfo() const
Definition: ASTContext.h:672
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters)
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Definition: CGExpr.cpp:2277
Floating point control options.
Definition: LangOptions.h:263
This represents &#39;#pragma omp parallel for&#39; directive.
Definition: StmtOpenMP.h:1516
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value *> &CapturedVars)
This represents &#39;#pragma omp target teams distribute parallel for&#39; combined directive.
Definition: StmtOpenMP.h:3828
Expr * getCombinedEnsureUpperBound() const
Definition: StmtOpenMP.h:893
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
Definition: CGAtomic.cpp:1885
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant, or if it does but contains a label, return false.
Represents a point when we exit a loop.
Definition: ProgramPoint.h:683
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
SmallVector< const Expr *, 4 > ReductionCopies
void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S)
static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, const Twine &Name, llvm::Value *Init=nullptr)
This represents &#39;#pragma omp target exit data&#39; directive.
Definition: StmtOpenMP.h:2427
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
CanQualType getCanonicalParamType(QualType T) const
Return the canonical parameter type corresponding to the specific potentially non-canonical one...
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S)
Represents a variable declaration or definition.
Definition: Decl.h:812
This represents clause &#39;private&#39; in the &#39;#pragma omp ...&#39; directives.
void EmitOMPCriticalDirective(const OMPCriticalDirective &S)
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:2610
This represents &#39;num_threads&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:382
const Stmt * getBody() const
Definition: StmtOpenMP.h:923
Extra information about a function prototype.
Definition: Type.h:3488
uint64_t getProfileCount(const Stmt *S)
Get the profiler&#39;s count for the given statement.
Expr * getCombinedUpperBoundVariable() const
Definition: StmtOpenMP.h:887
Expr * getCalcLastIteration() const
Definition: StmtOpenMP.h:769
llvm::Value * getPointer() const
Definition: Address.h:38
unsigned getContextParamPosition() const
Definition: Decl.h:4089
void EmitOMPInnerLoop(const Stmt &S, bool RequiresCleanup, const Expr *LoopCond, const Expr *IncExpr, const llvm::function_ref< void(CodeGenFunction &)> BodyGen, const llvm::function_ref< void(CodeGenFunction &)> PostIncGen)
Emit inner loop of the worksharing/simd construct.
This represents implicit clause &#39;flush&#39; for the &#39;#pragma omp flush&#39; directive.
void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D, bool NoFinals, llvm::Value *IsLastIterCond=nullptr)
Emit final copying of lastprivate values to original variables at the end of the worksharing or simd ...
void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S)
static void emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeDirective &S)
bool isXLHSInRHSPart() const
Return true if helper update expression has form &#39;OpaqueValueExpr(x) binop OpaqueValueExpr(expr)&#39; and...
Definition: StmtOpenMP.h:2224
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
This represents &#39;nogroup&#39; clause in the &#39;#pragma omp ...&#39; directive.
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB, const CodeGenLoopBoundsTy &CodeGenLoopBounds, const CodeGenDispatchBoundsTy &CGDispatchBounds)
Emit code for the worksharing loop-based directive.
This represents &#39;safelen&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:447
void EmitOMPTargetTeamsDistributeParallelForSimdDirective(const OMPTargetTeamsDistributeParallelForSimdDirective &S)
IdentifierInfo * getIdentifier() const
Get the identifier that names this declaration, if there is one.
Definition: Decl.h:269
Expr * IgnoreImpCasts() LLVM_READONLY
IgnoreImpCasts - Skip past any implicit casts which might surround this expression.
Definition: Expr.h:2945
void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S)
SmallVector< const Expr *, 4 > PrivateVars
Represents a struct/union/class.
Definition: Decl.h:3548
llvm::DenseMap< const VarDecl *, FieldDecl * > LambdaCaptureFields
One of these records is kept for each identifier that is lexed.
void EmitOMPSimdDirective(const OMPSimdDirective &S)
This represents &#39;#pragma omp parallel&#39; directive.
Definition: StmtOpenMP.h:274
Address getAddress() const
Definition: CGValue.h:327
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:150
A C++ nested-name-specifier augmented with source location information.
This represents &#39;simd&#39; clause in the &#39;#pragma omp ...&#39; directive.
void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S)