clang  8.0.0svn
CGStmtOpenMP.cpp
Go to the documentation of this file.
1 //===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This contains code to emit OpenMP nodes as LLVM code.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "CGCleanup.h"
15 #include "CGOpenMPRuntime.h"
16 #include "CodeGenFunction.h"
17 #include "CodeGenModule.h"
18 #include "TargetInfo.h"
19 #include "clang/AST/Stmt.h"
20 #include "clang/AST/StmtOpenMP.h"
21 #include "clang/AST/DeclOpenMP.h"
22 #include "llvm/IR/CallSite.h"
23 using namespace clang;
24 using namespace CodeGen;
25 
26 namespace {
27 /// Lexical scope for OpenMP executable constructs, that handles correct codegen
28 /// for captured expressions.
29 class OMPLexicalScope : public CodeGenFunction::LexicalScope {
30  void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
31  for (const auto *C : S.clauses()) {
32  if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
33  if (const auto *PreInit =
34  cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
35  for (const auto *I : PreInit->decls()) {
36  if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
37  CGF.EmitVarDecl(cast<VarDecl>(*I));
38  } else {
40  CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
41  CGF.EmitAutoVarCleanups(Emission);
42  }
43  }
44  }
45  }
46  }
47  }
48  CodeGenFunction::OMPPrivateScope InlinedShareds;
49 
50  static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
51  return CGF.LambdaCaptureFields.lookup(VD) ||
52  (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
53  (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl));
54  }
55 
56 public:
57  OMPLexicalScope(
59  const llvm::Optional<OpenMPDirectiveKind> CapturedRegion = llvm::None,
60  const bool EmitPreInitStmt = true)
61  : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
62  InlinedShareds(CGF) {
63  if (EmitPreInitStmt)
64  emitPreInitStmt(CGF, S);
65  if (!CapturedRegion.hasValue())
66  return;
67  assert(S.hasAssociatedStmt() &&
68  "Expected associated statement for inlined directive.");
69  const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
70  for (const auto &C : CS->captures()) {
71  if (C.capturesVariable() || C.capturesVariableByCopy()) {
72  auto *VD = C.getCapturedVar();
73  assert(VD == VD->getCanonicalDecl() &&
74  "Canonical decl must be captured.");
75  DeclRefExpr DRE(
76  const_cast<VarDecl *>(VD),
77  isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
78  InlinedShareds.isGlobalVarCaptured(VD)),
79  VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
80  InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
81  return CGF.EmitLValue(&DRE).getAddress();
82  });
83  }
84  }
85  (void)InlinedShareds.Privatize();
86  }
87 };
88 
89 /// Lexical scope for OpenMP parallel construct, that handles correct codegen
90 /// for captured expressions.
91 class OMPParallelScope final : public OMPLexicalScope {
92  bool EmitPreInitStmt(const OMPExecutableDirective &S) {
94  return !(isOpenMPTargetExecutionDirective(Kind) ||
97  }
98 
99 public:
100  OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
101  : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
102  EmitPreInitStmt(S)) {}
103 };
104 
105 /// Lexical scope for OpenMP teams construct, that handles correct codegen
106 /// for captured expressions.
107 class OMPTeamsScope final : public OMPLexicalScope {
108  bool EmitPreInitStmt(const OMPExecutableDirective &S) {
110  return !isOpenMPTargetExecutionDirective(Kind) &&
112  }
113 
114 public:
115  OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
116  : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
117  EmitPreInitStmt(S)) {}
118 };
119 
120 /// Private scope for OpenMP loop-based directives, that supports capturing
121 /// of used expression from loop statement.
122 class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
123  void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) {
124  CodeGenFunction::OMPMapVars PreCondVars;
125  for (const auto *E : S.counters()) {
126  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
127  (void)PreCondVars.setVarAddr(
128  CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
129  }
130  (void)PreCondVars.apply(CGF);
131  if (const auto *PreInits = cast_or_null<DeclStmt>(S.getPreInits())) {
132  for (const auto *I : PreInits->decls())
133  CGF.EmitVarDecl(cast<VarDecl>(*I));
134  }
135  PreCondVars.restore(CGF);
136  }
137 
138 public:
139  OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S)
140  : CodeGenFunction::RunCleanupsScope(CGF) {
141  emitPreInitStmt(CGF, S);
142  }
143 };
144 
145 class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
146  CodeGenFunction::OMPPrivateScope InlinedShareds;
147 
148  static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
149  return CGF.LambdaCaptureFields.lookup(VD) ||
150  (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
151  (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) &&
152  cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
153  }
154 
155 public:
156  OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
157  : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
158  InlinedShareds(CGF) {
159  for (const auto *C : S.clauses()) {
160  if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
161  if (const auto *PreInit =
162  cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
163  for (const auto *I : PreInit->decls()) {
164  if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
165  CGF.EmitVarDecl(cast<VarDecl>(*I));
166  } else {
168  CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
169  CGF.EmitAutoVarCleanups(Emission);
170  }
171  }
172  }
173  } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
174  for (const Expr *E : UDP->varlists()) {
175  const Decl *D = cast<DeclRefExpr>(E)->getDecl();
176  if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
177  CGF.EmitVarDecl(*OED);
178  }
179  }
180  }
182  CGF.EmitOMPPrivateClause(S, InlinedShareds);
183  if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
184  if (const Expr *E = TG->getReductionRef())
185  CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
186  }
187  const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
188  while (CS) {
189  for (auto &C : CS->captures()) {
190  if (C.capturesVariable() || C.capturesVariableByCopy()) {
191  auto *VD = C.getCapturedVar();
192  assert(VD == VD->getCanonicalDecl() &&
193  "Canonical decl must be captured.");
194  DeclRefExpr DRE(const_cast<VarDecl *>(VD),
195  isCapturedVar(CGF, VD) ||
196  (CGF.CapturedStmtInfo &&
197  InlinedShareds.isGlobalVarCaptured(VD)),
199  C.getLocation());
200  InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
201  return CGF.EmitLValue(&DRE).getAddress();
202  });
203  }
204  }
205  CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
206  }
207  (void)InlinedShareds.Privatize();
208  }
209 };
210 
211 } // namespace
212 
214  const OMPExecutableDirective &S,
215  const RegionCodeGenTy &CodeGen);
216 
218  if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
219  if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
220  OrigVD = OrigVD->getCanonicalDecl();
221  bool IsCaptured =
222  LambdaCaptureFields.lookup(OrigVD) ||
223  (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
224  (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
225  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), IsCaptured,
226  OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
227  return EmitLValue(&DRE);
228  }
229  }
230  return EmitLValue(E);
231 }
232 
234  ASTContext &C = getContext();
235  llvm::Value *Size = nullptr;
236  auto SizeInChars = C.getTypeSizeInChars(Ty);
237  if (SizeInChars.isZero()) {
238  // getTypeSizeInChars() returns 0 for a VLA.
239  while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) {
240  VlaSizePair VlaSize = getVLASize(VAT);
241  Ty = VlaSize.Type;
242  Size = Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts)
243  : VlaSize.NumElts;
244  }
245  SizeInChars = C.getTypeSizeInChars(Ty);
246  if (SizeInChars.isZero())
247  return llvm::ConstantInt::get(SizeTy, /*V=*/0);
248  return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
249  }
250  return CGM.getSize(SizeInChars);
251 }
252 
254  const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
255  const RecordDecl *RD = S.getCapturedRecordDecl();
256  auto CurField = RD->field_begin();
257  auto CurCap = S.captures().begin();
259  E = S.capture_init_end();
260  I != E; ++I, ++CurField, ++CurCap) {
261  if (CurField->hasCapturedVLAType()) {
262  const VariableArrayType *VAT = CurField->getCapturedVLAType();
263  llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
264  CapturedVars.push_back(Val);
265  } else if (CurCap->capturesThis()) {
266  CapturedVars.push_back(CXXThisValue);
267  } else if (CurCap->capturesVariableByCopy()) {
268  llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
269 
270  // If the field is not a pointer, we need to save the actual value
271  // and load it as a void pointer.
272  if (!CurField->getType()->isAnyPointerType()) {
273  ASTContext &Ctx = getContext();
274  Address DstAddr = CreateMemTemp(
275  Ctx.getUIntPtrType(),
276  Twine(CurCap->getCapturedVar()->getName(), ".casted"));
277  LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
278 
279  llvm::Value *SrcAddrVal = EmitScalarConversion(
280  DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
281  Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
282  LValue SrcLV =
283  MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
284 
285  // Store the value using the source type pointer.
286  EmitStoreThroughLValue(RValue::get(CV), SrcLV);
287 
288  // Load the value using the destination type pointer.
289  CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
290  }
291  CapturedVars.push_back(CV);
292  } else {
293  assert(CurCap->capturesVariable() && "Expected capture by reference.");
294  CapturedVars.push_back(EmitLValue(*I).getAddress().getPointer());
295  }
296  }
297 }
298 
300  QualType DstType, StringRef Name,
301  LValue AddrLV,
302  bool isReferenceType = false) {
303  ASTContext &Ctx = CGF.getContext();
304 
305  llvm::Value *CastedPtr = CGF.EmitScalarConversion(
306  AddrLV.getAddress().getPointer(), Ctx.getUIntPtrType(),
307  Ctx.getPointerType(DstType), Loc);
308  Address TmpAddr =
309  CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType))
310  .getAddress();
311 
312  // If we are dealing with references we need to return the address of the
313  // reference instead of the reference of the value.
314  if (isReferenceType) {
315  QualType RefType = Ctx.getLValueReferenceType(DstType);
316  llvm::Value *RefVal = TmpAddr.getPointer();
317  TmpAddr = CGF.CreateMemTemp(RefType, Twine(Name, ".ref"));
318  LValue TmpLVal = CGF.MakeAddrLValue(TmpAddr, RefType);
319  CGF.EmitStoreThroughLValue(RValue::get(RefVal), TmpLVal, /*isInit=*/true);
320  }
321 
322  return TmpAddr;
323 }
324 
326  if (T->isLValueReferenceType())
327  return C.getLValueReferenceType(
329  /*SpelledAsLValue=*/false);
330  if (T->isPointerType())
332  if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
333  if (const auto *VLA = dyn_cast<VariableArrayType>(A))
334  return getCanonicalParamType(C, VLA->getElementType());
335  if (!A->isVariablyModifiedType())
336  return C.getCanonicalType(T);
337  }
338  return C.getCanonicalParamType(T);
339 }
340 
341 namespace {
342  /// Contains required data for proper outlined function codegen.
343  struct FunctionOptions {
344  /// Captured statement for which the function is generated.
345  const CapturedStmt *S = nullptr;
346  /// true if cast to/from UIntPtr is required for variables captured by
347  /// value.
348  const bool UIntPtrCastRequired = true;
349  /// true if only casted arguments must be registered as local args or VLA
350  /// sizes.
351  const bool RegisterCastedArgsOnly = false;
352  /// Name of the generated function.
353  const StringRef FunctionName;
354  explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
355  bool RegisterCastedArgsOnly,
356  StringRef FunctionName)
357  : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
358  RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
359  FunctionName(FunctionName) {}
360  };
361 }
362 
363 static llvm::Function *emitOutlinedFunctionPrologue(
364  CodeGenFunction &CGF, FunctionArgList &Args,
365  llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
366  &LocalAddrs,
367  llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
368  &VLASizes,
369  llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
370  const CapturedDecl *CD = FO.S->getCapturedDecl();
371  const RecordDecl *RD = FO.S->getCapturedRecordDecl();
372  assert(CD->hasBody() && "missing CapturedDecl body");
373 
374  CXXThisValue = nullptr;
375  // Build the argument list.
376  CodeGenModule &CGM = CGF.CGM;
377  ASTContext &Ctx = CGM.getContext();
378  FunctionArgList TargetArgs;
379  Args.append(CD->param_begin(),
380  std::next(CD->param_begin(), CD->getContextParamPosition()));
381  TargetArgs.append(
382  CD->param_begin(),
383  std::next(CD->param_begin(), CD->getContextParamPosition()));
384  auto I = FO.S->captures().begin();
385  FunctionDecl *DebugFunctionDecl = nullptr;
386  if (!FO.UIntPtrCastRequired) {
388  QualType FunctionTy = Ctx.getFunctionType(Ctx.VoidTy, llvm::None, EPI);
389  DebugFunctionDecl = FunctionDecl::Create(
390  Ctx, Ctx.getTranslationUnitDecl(), FO.S->getBeginLoc(),
391  SourceLocation(), DeclarationName(), FunctionTy,
392  Ctx.getTrivialTypeSourceInfo(FunctionTy), SC_Static,
393  /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/false);
394  }
395  for (const FieldDecl *FD : RD->fields()) {
396  QualType ArgType = FD->getType();
397  IdentifierInfo *II = nullptr;
398  VarDecl *CapVar = nullptr;
399 
400  // If this is a capture by copy and the type is not a pointer, the outlined
401  // function argument type should be uintptr and the value properly casted to
402  // uintptr. This is necessary given that the runtime library is only able to
403  // deal with pointers. We can pass in the same way the VLA type sizes to the
404  // outlined function.
405  if (FO.UIntPtrCastRequired &&
406  ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
407  I->capturesVariableArrayType()))
408  ArgType = Ctx.getUIntPtrType();
409 
410  if (I->capturesVariable() || I->capturesVariableByCopy()) {
411  CapVar = I->getCapturedVar();
412  II = CapVar->getIdentifier();
413  } else if (I->capturesThis()) {
414  II = &Ctx.Idents.get("this");
415  } else {
416  assert(I->capturesVariableArrayType());
417  II = &Ctx.Idents.get("vla");
418  }
419  if (ArgType->isVariablyModifiedType())
420  ArgType = getCanonicalParamType(Ctx, ArgType);
421  VarDecl *Arg;
422  if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
423  Arg = ParmVarDecl::Create(
424  Ctx, DebugFunctionDecl,
425  CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),
426  CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
427  /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
428  } else {
429  Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
430  II, ArgType, ImplicitParamDecl::Other);
431  }
432  Args.emplace_back(Arg);
433  // Do not cast arguments if we emit function with non-original types.
434  TargetArgs.emplace_back(
435  FO.UIntPtrCastRequired
436  ? Arg
437  : CGM.getOpenMPRuntime().translateParameter(FD, Arg));
438  ++I;
439  }
440  Args.append(
441  std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
442  CD->param_end());
443  TargetArgs.append(
444  std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
445  CD->param_end());
446 
447  // Create the function declaration.
448  const CGFunctionInfo &FuncInfo =
449  CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, TargetArgs);
450  llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
451 
452  auto *F =
454  FO.FunctionName, &CGM.getModule());
455  CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
456  if (CD->isNothrow())
457  F->setDoesNotThrow();
458  F->setDoesNotRecurse();
459 
460  // Generate the function.
461  CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
462  FO.S->getBeginLoc(), CD->getBody()->getBeginLoc());
463  unsigned Cnt = CD->getContextParamPosition();
464  I = FO.S->captures().begin();
465  for (const FieldDecl *FD : RD->fields()) {
466  // Do not map arguments if we emit function with non-original types.
467  Address LocalAddr(Address::invalid());
468  if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
469  LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
470  TargetArgs[Cnt]);
471  } else {
472  LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
473  }
474  // If we are capturing a pointer by copy we don't need to do anything, just
475  // use the value that we get from the arguments.
476  if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
477  const VarDecl *CurVD = I->getCapturedVar();
478  // If the variable is a reference we need to materialize it here.
479  if (CurVD->getType()->isReferenceType()) {
480  Address RefAddr = CGF.CreateMemTemp(
481  CurVD->getType(), CGM.getPointerAlign(), ".materialized_ref");
482  CGF.EmitStoreOfScalar(LocalAddr.getPointer(), RefAddr,
483  /*Volatile=*/false, CurVD->getType());
484  LocalAddr = RefAddr;
485  }
486  if (!FO.RegisterCastedArgsOnly)
487  LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
488  ++Cnt;
489  ++I;
490  continue;
491  }
492 
493  LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
495  if (FD->hasCapturedVLAType()) {
496  if (FO.UIntPtrCastRequired) {
497  ArgLVal = CGF.MakeAddrLValue(
498  castValueFromUintptr(CGF, I->getLocation(), FD->getType(),
499  Args[Cnt]->getName(), ArgLVal),
501  }
502  llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
503  const VariableArrayType *VAT = FD->getCapturedVLAType();
504  VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg);
505  } else if (I->capturesVariable()) {
506  const VarDecl *Var = I->getCapturedVar();
507  QualType VarTy = Var->getType();
508  Address ArgAddr = ArgLVal.getAddress();
509  if (!VarTy->isReferenceType()) {
510  if (ArgLVal.getType()->isLValueReferenceType()) {
511  ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
512  } else if (!VarTy->isVariablyModifiedType() ||
513  !VarTy->isPointerType()) {
514  assert(ArgLVal.getType()->isPointerType());
515  ArgAddr = CGF.EmitLoadOfPointer(
516  ArgAddr, ArgLVal.getType()->castAs<PointerType>());
517  }
518  }
519  if (!FO.RegisterCastedArgsOnly) {
520  LocalAddrs.insert(
521  {Args[Cnt],
522  {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}});
523  }
524  } else if (I->capturesVariableByCopy()) {
525  assert(!FD->getType()->isAnyPointerType() &&
526  "Not expecting a captured pointer.");
527  const VarDecl *Var = I->getCapturedVar();
528  QualType VarTy = Var->getType();
529  LocalAddrs.insert(
530  {Args[Cnt],
531  {Var, FO.UIntPtrCastRequired
532  ? castValueFromUintptr(CGF, I->getLocation(),
533  FD->getType(), Args[Cnt]->getName(),
534  ArgLVal, VarTy->isReferenceType())
535  : ArgLVal.getAddress()}});
536  } else {
537  // If 'this' is captured, load it into CXXThisValue.
538  assert(I->capturesThis());
539  CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
540  LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress()}});
541  }
542  ++Cnt;
543  ++I;
544  }
545 
546  return F;
547 }
548 
549 llvm::Function *
551  assert(
552  CapturedStmtInfo &&
553  "CapturedStmtInfo should be set when generating the captured function");
554  const CapturedDecl *CD = S.getCapturedDecl();
555  // Build the argument list.
556  bool NeedWrapperFunction =
557  getDebugInfo() &&
558  CGM.getCodeGenOpts().getDebugInfo() >= codegenoptions::LimitedDebugInfo;
559  FunctionArgList Args;
560  llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
561  llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
562  SmallString<256> Buffer;
563  llvm::raw_svector_ostream Out(Buffer);
564  Out << CapturedStmtInfo->getHelperName();
565  if (NeedWrapperFunction)
566  Out << "_debug__";
567  FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
568  Out.str());
569  llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs,
570  VLASizes, CXXThisValue, FO);
571  for (const auto &LocalAddrPair : LocalAddrs) {
572  if (LocalAddrPair.second.first) {
573  setAddrOfLocalVar(LocalAddrPair.second.first,
574  LocalAddrPair.second.second);
575  }
576  }
577  for (const auto &VLASizePair : VLASizes)
578  VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
579  PGO.assignRegionCounters(GlobalDecl(CD), F);
580  CapturedStmtInfo->EmitBody(*this, CD->getBody());
581  FinishFunction(CD->getBodyRBrace());
582  if (!NeedWrapperFunction)
583  return F;
584 
585  FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
586  /*RegisterCastedArgsOnly=*/true,
587  CapturedStmtInfo->getHelperName());
588  CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
589  WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
590  Args.clear();
591  LocalAddrs.clear();
592  VLASizes.clear();
593  llvm::Function *WrapperF =
594  emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
595  WrapperCGF.CXXThisValue, WrapperFO);
597  for (const auto *Arg : Args) {
599  auto I = LocalAddrs.find(Arg);
600  if (I != LocalAddrs.end()) {
601  LValue LV = WrapperCGF.MakeAddrLValue(
602  I->second.second,
603  I->second.first ? I->second.first->getType() : Arg->getType(),
605  CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
606  } else {
607  auto EI = VLASizes.find(Arg);
608  if (EI != VLASizes.end()) {
609  CallArg = EI->second.second;
610  } else {
611  LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
612  Arg->getType(),
614  CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
615  }
616  }
617  CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
618  }
619  CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.getBeginLoc(),
620  F, CallArgs);
621  WrapperCGF.FinishFunction();
622  return WrapperF;
623 }
624 
625 //===----------------------------------------------------------------------===//
626 // OpenMP Directive Emission
627 //===----------------------------------------------------------------------===//
629  Address DestAddr, Address SrcAddr, QualType OriginalType,
630  const llvm::function_ref<void(Address, Address)> CopyGen) {
631  // Perform element-by-element initialization.
632  QualType ElementTy;
633 
634  // Drill down to the base element type on both arrays.
635  const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
636  llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
637  SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
638 
639  llvm::Value *SrcBegin = SrcAddr.getPointer();
640  llvm::Value *DestBegin = DestAddr.getPointer();
641  // Cast from pointer to array type to pointer to single element.
642  llvm::Value *DestEnd = Builder.CreateGEP(DestBegin, NumElements);
643  // The basic structure here is a while-do loop.
644  llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body");
645  llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done");
646  llvm::Value *IsEmpty =
647  Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
648  Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
649 
650  // Enter the loop body, making that address the current address.
651  llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
652  EmitBlock(BodyBB);
653 
654  CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
655 
656  llvm::PHINode *SrcElementPHI =
657  Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
658  SrcElementPHI->addIncoming(SrcBegin, EntryBB);
659  Address SrcElementCurrent =
660  Address(SrcElementPHI,
661  SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
662 
663  llvm::PHINode *DestElementPHI =
664  Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
665  DestElementPHI->addIncoming(DestBegin, EntryBB);
666  Address DestElementCurrent =
667  Address(DestElementPHI,
668  DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
669 
670  // Emit copy.
671  CopyGen(DestElementCurrent, SrcElementCurrent);
672 
673  // Shift the address forward by one element.
674  llvm::Value *DestElementNext = Builder.CreateConstGEP1_32(
675  DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
676  llvm::Value *SrcElementNext = Builder.CreateConstGEP1_32(
677  SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
678  // Check whether we've reached the end.
679  llvm::Value *Done =
680  Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
681  Builder.CreateCondBr(Done, DoneBB, BodyBB);
682  DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
683  SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
684 
685  // Done.
686  EmitBlock(DoneBB, /*IsFinished=*/true);
687 }
688 
689 void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
690  Address SrcAddr, const VarDecl *DestVD,
691  const VarDecl *SrcVD, const Expr *Copy) {
692  if (OriginalType->isArrayType()) {
693  const auto *BO = dyn_cast<BinaryOperator>(Copy);
694  if (BO && BO->getOpcode() == BO_Assign) {
695  // Perform simple memcpy for simple copying.
696  LValue Dest = MakeAddrLValue(DestAddr, OriginalType);
697  LValue Src = MakeAddrLValue(SrcAddr, OriginalType);
698  EmitAggregateAssign(Dest, Src, OriginalType);
699  } else {
700  // For arrays with complex element types perform element by element
701  // copying.
702  EmitOMPAggregateAssign(
703  DestAddr, SrcAddr, OriginalType,
704  [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
705  // Working with the single array element, so have to remap
706  // destination and source variables to corresponding array
707  // elements.
709  Remap.addPrivate(DestVD, [DestElement]() { return DestElement; });
710  Remap.addPrivate(SrcVD, [SrcElement]() { return SrcElement; });
711  (void)Remap.Privatize();
712  EmitIgnoredExpr(Copy);
713  });
714  }
715  } else {
716  // Remap pseudo source variable to private copy.
718  Remap.addPrivate(SrcVD, [SrcAddr]() { return SrcAddr; });
719  Remap.addPrivate(DestVD, [DestAddr]() { return DestAddr; });
720  (void)Remap.Privatize();
721  // Emit copying of the whole variable.
722  EmitIgnoredExpr(Copy);
723  }
724 }
725 
727  OMPPrivateScope &PrivateScope) {
728  if (!HaveInsertPoint())
729  return false;
730  bool FirstprivateIsLastprivate = false;
731  llvm::DenseSet<const VarDecl *> Lastprivates;
732  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
733  for (const auto *D : C->varlists())
734  Lastprivates.insert(
735  cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl());
736  }
737  llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
739  getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind());
740  // Force emission of the firstprivate copy if the directive does not emit
741  // outlined function, like omp for, omp simd, omp distribute etc.
742  bool MustEmitFirstprivateCopy =
743  CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
744  for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
745  auto IRef = C->varlist_begin();
746  auto InitsRef = C->inits().begin();
747  for (const Expr *IInit : C->private_copies()) {
748  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
749  bool ThisFirstprivateIsLastprivate =
750  Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
751  const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
752  if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
753  !FD->getType()->isReferenceType()) {
754  EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
755  ++IRef;
756  ++InitsRef;
757  continue;
758  }
759  FirstprivateIsLastprivate =
760  FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
761  if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
762  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
763  const auto *VDInit =
764  cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
765  bool IsRegistered;
766  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
767  /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
768  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
769  LValue OriginalLVal = EmitLValue(&DRE);
770  QualType Type = VD->getType();
771  if (Type->isArrayType()) {
772  // Emit VarDecl with copy init for arrays.
773  // Get the address of the original variable captured in current
774  // captured region.
775  IsRegistered = PrivateScope.addPrivate(
776  OrigVD, [this, VD, Type, OriginalLVal, VDInit]() {
777  AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
778  const Expr *Init = VD->getInit();
779  if (!isa<CXXConstructExpr>(Init) ||
780  isTrivialInitializer(Init)) {
781  // Perform simple memcpy.
782  LValue Dest =
783  MakeAddrLValue(Emission.getAllocatedAddress(), Type);
784  EmitAggregateAssign(Dest, OriginalLVal, Type);
785  } else {
786  EmitOMPAggregateAssign(
787  Emission.getAllocatedAddress(), OriginalLVal.getAddress(),
788  Type,
789  [this, VDInit, Init](Address DestElement,
790  Address SrcElement) {
791  // Clean up any temporaries needed by the
792  // initialization.
793  RunCleanupsScope InitScope(*this);
794  // Emit initialization for single element.
795  setAddrOfLocalVar(VDInit, SrcElement);
796  EmitAnyExprToMem(Init, DestElement,
797  Init->getType().getQualifiers(),
798  /*IsInitializer*/ false);
799  LocalDeclMap.erase(VDInit);
800  });
801  }
802  EmitAutoVarCleanups(Emission);
803  return Emission.getAllocatedAddress();
804  });
805  } else {
806  Address OriginalAddr = OriginalLVal.getAddress();
807  IsRegistered = PrivateScope.addPrivate(
808  OrigVD, [this, VDInit, OriginalAddr, VD]() {
809  // Emit private VarDecl with copy init.
810  // Remap temp VDInit variable to the address of the original
811  // variable (for proper handling of captured global variables).
812  setAddrOfLocalVar(VDInit, OriginalAddr);
813  EmitDecl(*VD);
814  LocalDeclMap.erase(VDInit);
815  return GetAddrOfLocalVar(VD);
816  });
817  }
818  assert(IsRegistered &&
819  "firstprivate var already registered as private");
820  // Silence the warning about unused variable.
821  (void)IsRegistered;
822  }
823  ++IRef;
824  ++InitsRef;
825  }
826  }
827  return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
828 }
829 
831  const OMPExecutableDirective &D,
832  CodeGenFunction::OMPPrivateScope &PrivateScope) {
833  if (!HaveInsertPoint())
834  return;
835  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
836  for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
837  auto IRef = C->varlist_begin();
838  for (const Expr *IInit : C->private_copies()) {
839  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
840  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
841  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
842  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
843  // Emit private VarDecl with copy init.
844  EmitDecl(*VD);
845  return GetAddrOfLocalVar(VD);
846  });
847  assert(IsRegistered && "private var already registered as private");
848  // Silence the warning about unused variable.
849  (void)IsRegistered;
850  }
851  ++IRef;
852  }
853  }
854 }
855 
857  if (!HaveInsertPoint())
858  return false;
859  // threadprivate_var1 = master_threadprivate_var1;
860  // operator=(threadprivate_var2, master_threadprivate_var2);
861  // ...
862  // __kmpc_barrier(&loc, global_tid);
864  llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
865  for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
866  auto IRef = C->varlist_begin();
867  auto ISrcRef = C->source_exprs().begin();
868  auto IDestRef = C->destination_exprs().begin();
869  for (const Expr *AssignOp : C->assignment_ops()) {
870  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
871  QualType Type = VD->getType();
872  if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
873  // Get the address of the master variable. If we are emitting code with
874  // TLS support, the address is passed from the master as field in the
875  // captured declaration.
876  Address MasterAddr = Address::invalid();
877  if (getLangOpts().OpenMPUseTLS &&
878  getContext().getTargetInfo().isTLSSupported()) {
879  assert(CapturedStmtInfo->lookup(VD) &&
880  "Copyin threadprivates should have been captured!");
881  DeclRefExpr DRE(const_cast<VarDecl *>(VD), true, (*IRef)->getType(),
882  VK_LValue, (*IRef)->getExprLoc());
883  MasterAddr = EmitLValue(&DRE).getAddress();
884  LocalDeclMap.erase(VD);
885  } else {
886  MasterAddr =
887  Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
888  : CGM.GetAddrOfGlobal(VD),
889  getContext().getDeclAlign(VD));
890  }
891  // Get the address of the threadprivate variable.
892  Address PrivateAddr = EmitLValue(*IRef).getAddress();
893  if (CopiedVars.size() == 1) {
894  // At first check if current thread is a master thread. If it is, no
895  // need to copy data.
896  CopyBegin = createBasicBlock("copyin.not.master");
897  CopyEnd = createBasicBlock("copyin.not.master.end");
898  Builder.CreateCondBr(
899  Builder.CreateICmpNE(
900  Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy),
901  Builder.CreatePtrToInt(PrivateAddr.getPointer(),
902  CGM.IntPtrTy)),
903  CopyBegin, CopyEnd);
904  EmitBlock(CopyBegin);
905  }
906  const auto *SrcVD =
907  cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
908  const auto *DestVD =
909  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
910  EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
911  }
912  ++IRef;
913  ++ISrcRef;
914  ++IDestRef;
915  }
916  }
917  if (CopyEnd) {
918  // Exit out of copying procedure for non-master thread.
919  EmitBlock(CopyEnd, /*IsFinished=*/true);
920  return true;
921  }
922  return false;
923 }
924 
926  const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
927  if (!HaveInsertPoint())
928  return false;
929  bool HasAtLeastOneLastprivate = false;
932  const auto *LoopDirective = cast<OMPLoopDirective>(&D);
933  for (const Expr *C : LoopDirective->counters()) {
934  SIMDLCVs.insert(
935  cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
936  }
937  }
938  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
939  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
940  HasAtLeastOneLastprivate = true;
942  !getLangOpts().OpenMPSimd)
943  break;
944  auto IRef = C->varlist_begin();
945  auto IDestRef = C->destination_exprs().begin();
946  for (const Expr *IInit : C->private_copies()) {
947  // Keep the address of the original variable for future update at the end
948  // of the loop.
949  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
950  // Taskloops do not require additional initialization, it is done in
951  // runtime support library.
952  if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
953  const auto *DestVD =
954  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
955  PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() {
956  DeclRefExpr DRE(
957  const_cast<VarDecl *>(OrigVD),
958  /*RefersToEnclosingVariableOrCapture=*/CapturedStmtInfo->lookup(
959  OrigVD) != nullptr,
960  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
961  return EmitLValue(&DRE).getAddress();
962  });
963  // Check if the variable is also a firstprivate: in this case IInit is
964  // not generated. Initialization of this variable will happen in codegen
965  // for 'firstprivate' clause.
966  if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
967  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
968  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
969  // Emit private VarDecl with copy init.
970  EmitDecl(*VD);
971  return GetAddrOfLocalVar(VD);
972  });
973  assert(IsRegistered &&
974  "lastprivate var already registered as private");
975  (void)IsRegistered;
976  }
977  }
978  ++IRef;
979  ++IDestRef;
980  }
981  }
982  return HasAtLeastOneLastprivate;
983 }
984 
986  const OMPExecutableDirective &D, bool NoFinals,
987  llvm::Value *IsLastIterCond) {
988  if (!HaveInsertPoint())
989  return;
990  // Emit following code:
991  // if (<IsLastIterCond>) {
992  // orig_var1 = private_orig_var1;
993  // ...
994  // orig_varn = private_orig_varn;
995  // }
996  llvm::BasicBlock *ThenBB = nullptr;
997  llvm::BasicBlock *DoneBB = nullptr;
998  if (IsLastIterCond) {
999  ThenBB = createBasicBlock(".omp.lastprivate.then");
1000  DoneBB = createBasicBlock(".omp.lastprivate.done");
1001  Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1002  EmitBlock(ThenBB);
1003  }
1004  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1005  llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1006  if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1007  auto IC = LoopDirective->counters().begin();
1008  for (const Expr *F : LoopDirective->finals()) {
1009  const auto *D =
1010  cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1011  if (NoFinals)
1012  AlreadyEmittedVars.insert(D);
1013  else
1014  LoopCountersAndUpdates[D] = F;
1015  ++IC;
1016  }
1017  }
1018  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1019  auto IRef = C->varlist_begin();
1020  auto ISrcRef = C->source_exprs().begin();
1021  auto IDestRef = C->destination_exprs().begin();
1022  for (const Expr *AssignOp : C->assignment_ops()) {
1023  const auto *PrivateVD =
1024  cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1025  QualType Type = PrivateVD->getType();
1026  const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1027  if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1028  // If lastprivate variable is a loop control variable for loop-based
1029  // directive, update its value before copyin back to original
1030  // variable.
1031  if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1032  EmitIgnoredExpr(FinalExpr);
1033  const auto *SrcVD =
1034  cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1035  const auto *DestVD =
1036  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1037  // Get the address of the original variable.
1038  Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1039  // Get the address of the private variable.
1040  Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
1041  if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1042  PrivateAddr =
1043  Address(Builder.CreateLoad(PrivateAddr),
1044  getNaturalTypeAlignment(RefTy->getPointeeType()));
1045  EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1046  }
1047  ++IRef;
1048  ++ISrcRef;
1049  ++IDestRef;
1050  }
1051  if (const Expr *PostUpdate = C->getPostUpdateExpr())
1052  EmitIgnoredExpr(PostUpdate);
1053  }
1054  if (IsLastIterCond)
1055  EmitBlock(DoneBB, /*IsFinished=*/true);
1056 }
1057 
1059  const OMPExecutableDirective &D,
1060  CodeGenFunction::OMPPrivateScope &PrivateScope) {
1061  if (!HaveInsertPoint())
1062  return;
1065  SmallVector<const Expr *, 4> ReductionOps;
1068  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1069  auto IPriv = C->privates().begin();
1070  auto IRed = C->reduction_ops().begin();
1071  auto ILHS = C->lhs_exprs().begin();
1072  auto IRHS = C->rhs_exprs().begin();
1073  for (const Expr *Ref : C->varlists()) {
1074  Shareds.emplace_back(Ref);
1075  Privates.emplace_back(*IPriv);
1076  ReductionOps.emplace_back(*IRed);
1077  LHSs.emplace_back(*ILHS);
1078  RHSs.emplace_back(*IRHS);
1079  std::advance(IPriv, 1);
1080  std::advance(IRed, 1);
1081  std::advance(ILHS, 1);
1082  std::advance(IRHS, 1);
1083  }
1084  }
1085  ReductionCodeGen RedCG(Shareds, Privates, ReductionOps);
1086  unsigned Count = 0;
1087  auto ILHS = LHSs.begin();
1088  auto IRHS = RHSs.begin();
1089  auto IPriv = Privates.begin();
1090  for (const Expr *IRef : Shareds) {
1091  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1092  // Emit private VarDecl with reduction init.
1093  RedCG.emitSharedLValue(*this, Count);
1094  RedCG.emitAggregateType(*this, Count);
1095  AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);
1096  RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
1097  RedCG.getSharedLValue(Count),
1098  [&Emission](CodeGenFunction &CGF) {
1099  CGF.EmitAutoVarInit(Emission);
1100  return true;
1101  });
1102  EmitAutoVarCleanups(Emission);
1103  Address BaseAddr = RedCG.adjustPrivateAddress(
1104  *this, Count, Emission.getAllocatedAddress());
1105  bool IsRegistered = PrivateScope.addPrivate(
1106  RedCG.getBaseDecl(Count), [BaseAddr]() { return BaseAddr; });
1107  assert(IsRegistered && "private var already registered as private");
1108  // Silence the warning about unused variable.
1109  (void)IsRegistered;
1110 
1111  const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1112  const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1113  QualType Type = PrivateVD->getType();
1114  bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1115  if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1116  // Store the address of the original variable associated with the LHS
1117  // implicit variable.
1118  PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
1119  return RedCG.getSharedLValue(Count).getAddress();
1120  });
1121  PrivateScope.addPrivate(
1122  RHSVD, [this, PrivateVD]() { return GetAddrOfLocalVar(PrivateVD); });
1123  } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1124  isa<ArraySubscriptExpr>(IRef)) {
1125  // Store the address of the original variable associated with the LHS
1126  // implicit variable.
1127  PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
1128  return RedCG.getSharedLValue(Count).getAddress();
1129  });
1130  PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() {
1131  return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD),
1132  ConvertTypeForMem(RHSVD->getType()),
1133  "rhs.begin");
1134  });
1135  } else {
1136  QualType Type = PrivateVD->getType();
1137  bool IsArray = getContext().getAsArrayType(Type) != nullptr;
1138  Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress();
1139  // Store the address of the original variable associated with the LHS
1140  // implicit variable.
1141  if (IsArray) {
1142  OriginalAddr = Builder.CreateElementBitCast(
1143  OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin");
1144  }
1145  PrivateScope.addPrivate(LHSVD, [OriginalAddr]() { return OriginalAddr; });
1146  PrivateScope.addPrivate(
1147  RHSVD, [this, PrivateVD, RHSVD, IsArray]() {
1148  return IsArray
1149  ? Builder.CreateElementBitCast(
1150  GetAddrOfLocalVar(PrivateVD),
1151  ConvertTypeForMem(RHSVD->getType()), "rhs.begin")
1152  : GetAddrOfLocalVar(PrivateVD);
1153  });
1154  }
1155  ++ILHS;
1156  ++IRHS;
1157  ++IPriv;
1158  ++Count;
1159  }
1160 }
1161 
1163  const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1164  if (!HaveInsertPoint())
1165  return;
1170  bool HasAtLeastOneReduction = false;
1171  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1172  HasAtLeastOneReduction = true;
1173  Privates.append(C->privates().begin(), C->privates().end());
1174  LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1175  RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1176  ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1177  }
1178  if (HasAtLeastOneReduction) {
1179  bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1181  ReductionKind == OMPD_simd;
1182  bool SimpleReduction = ReductionKind == OMPD_simd;
1183  // Emit nowait reduction if nowait clause is present or directive is a
1184  // parallel directive (it always has implicit barrier).
1185  CGM.getOpenMPRuntime().emitReduction(
1186  *this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1187  {WithNowait, SimpleReduction, ReductionKind});
1188  }
1189 }
1190 
1192  CodeGenFunction &CGF, const OMPExecutableDirective &D,
1193  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1194  if (!CGF.HaveInsertPoint())
1195  return;
1196  llvm::BasicBlock *DoneBB = nullptr;
1197  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1198  if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
1199  if (!DoneBB) {
1200  if (llvm::Value *Cond = CondGen(CGF)) {
1201  // If the first post-update expression is found, emit conditional
1202  // block if it was requested.
1203  llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1204  DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1205  CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1206  CGF.EmitBlock(ThenBB);
1207  }
1208  }
1209  CGF.EmitIgnoredExpr(PostUpdate);
1210  }
1211  }
1212  if (DoneBB)
1213  CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1214 }
1215 
1216 namespace {
1217 /// Codegen lambda for appending distribute lower and upper bounds to outlined
1218 /// parallel function. This is necessary for combined constructs such as
1219 /// 'distribute parallel for'
1220 typedef llvm::function_ref<void(CodeGenFunction &,
1221  const OMPExecutableDirective &,
1223  CodeGenBoundParametersTy;
1224 } // anonymous namespace
1225 
1227  CodeGenFunction &CGF, const OMPExecutableDirective &S,
1228  OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1229  const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1230  const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1231  llvm::Value *OutlinedFn =
1232  CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
1233  S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
1234  if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
1235  CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1236  llvm::Value *NumThreads =
1237  CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1238  /*IgnoreResultAssign=*/true);
1239  CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
1240  CGF, NumThreads, NumThreadsClause->getBeginLoc());
1241  }
1242  if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
1243  CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1244  CGF.CGM.getOpenMPRuntime().emitProcBindClause(
1245  CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1246  }
1247  const Expr *IfCond = nullptr;
1248  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1249  if (C->getNameModifier() == OMPD_unknown ||
1250  C->getNameModifier() == OMPD_parallel) {
1251  IfCond = C->getCondition();
1252  break;
1253  }
1254  }
1255 
1256  OMPParallelScope Scope(CGF, S);
1258  // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
1259  // lower and upper bounds with the pragma 'for' chunking mechanism.
1260  // The following lambda takes care of appending the lower and upper bound
1261  // parameters when necessary
1262  CodeGenBoundParameters(CGF, S, CapturedVars);
1263  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
1264  CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn,
1265  CapturedVars, IfCond);
1266 }
1267 
1269  const OMPExecutableDirective &,
1271 
1273  // Emit parallel region as a standalone region.
1274  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1275  Action.Enter(CGF);
1276  OMPPrivateScope PrivateScope(CGF);
1277  bool Copyins = CGF.EmitOMPCopyinClause(S);
1278  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1279  if (Copyins) {
1280  // Emit implicit barrier to synchronize threads and avoid data races on
1281  // propagation master's thread values of threadprivate variables to local
1282  // instances of that variables of all other implicit threads.
1283  CGF.CGM.getOpenMPRuntime().emitBarrierCall(
1284  CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
1285  /*ForceSimpleCall=*/true);
1286  }
1287  CGF.EmitOMPPrivateClause(S, PrivateScope);
1288  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1289  (void)PrivateScope.Privatize();
1290  CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1291  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
1292  };
1293  emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
1296  [](CodeGenFunction &) { return nullptr; });
1297 }
1298 
1300  JumpDest LoopExit) {
1301  RunCleanupsScope BodyScope(*this);
1302  // Update counters values on current iteration.
1303  for (const Expr *UE : D.updates())
1304  EmitIgnoredExpr(UE);
1305  // Update the linear variables.
1306  // In distribute directives only loop counters may be marked as linear, no
1307  // need to generate the code for them.
1309  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1310  for (const Expr *UE : C->updates())
1311  EmitIgnoredExpr(UE);
1312  }
1313  }
1314 
1315  // On a continue in the body, jump to the end.
1316  JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue");
1317  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1318  // Emit loop body.
1319  EmitStmt(D.getBody());
1320  // The end (updates/cleanups).
1321  EmitBlock(Continue.getBlock());
1322  BreakContinueStack.pop_back();
1323 }
1324 
1326  const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
1327  const Expr *IncExpr,
1328  const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
1329  const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
1330  auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
1331 
1332  // Start the loop with a block that tests the condition.
1333  auto CondBlock = createBasicBlock("omp.inner.for.cond");
1334  EmitBlock(CondBlock);
1335  const SourceRange R = S.getSourceRange();
1336  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1337  SourceLocToDebugLoc(R.getEnd()));
1338 
1339  // If there are any cleanups between here and the loop-exit scope,
1340  // create a block to stage a loop exit along.
1341  llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1342  if (RequiresCleanup)
1343  ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
1344 
1345  llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body");
1346 
1347  // Emit condition.
1348  EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
1349  if (ExitBlock != LoopExit.getBlock()) {
1350  EmitBlock(ExitBlock);
1351  EmitBranchThroughCleanup(LoopExit);
1352  }
1353 
1354  EmitBlock(LoopBody);
1355  incrementProfileCounter(&S);
1356 
1357  // Create a block for the increment.
1358  JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
1359  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1360 
1361  BodyGen(*this);
1362 
1363  // Emit "IV = IV + 1" and a back-edge to the condition block.
1364  EmitBlock(Continue.getBlock());
1365  EmitIgnoredExpr(IncExpr);
1366  PostIncGen(*this);
1367  BreakContinueStack.pop_back();
1368  EmitBranch(CondBlock);
1369  LoopStack.pop();
1370  // Emit the fall-through block.
1371  EmitBlock(LoopExit.getBlock());
1372 }
1373 
1375  if (!HaveInsertPoint())
1376  return false;
1377  // Emit inits for the linear variables.
1378  bool HasLinears = false;
1379  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1380  for (const Expr *Init : C->inits()) {
1381  HasLinears = true;
1382  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
1383  if (const auto *Ref =
1384  dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
1385  AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
1386  const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
1387  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
1388  CapturedStmtInfo->lookup(OrigVD) != nullptr,
1389  VD->getInit()->getType(), VK_LValue,
1390  VD->getInit()->getExprLoc());
1391  EmitExprAsInit(&DRE, VD, MakeAddrLValue(Emission.getAllocatedAddress(),
1392  VD->getType()),
1393  /*capturedByInit=*/false);
1394  EmitAutoVarCleanups(Emission);
1395  } else {
1396  EmitVarDecl(*VD);
1397  }
1398  }
1399  // Emit the linear steps for the linear clauses.
1400  // If a step is not constant, it is pre-calculated before the loop.
1401  if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
1402  if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
1403  EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
1404  // Emit calculation of the linear step.
1405  EmitIgnoredExpr(CS);
1406  }
1407  }
1408  return HasLinears;
1409 }
1410 
1412  const OMPLoopDirective &D,
1413  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1414  if (!HaveInsertPoint())
1415  return;
1416  llvm::BasicBlock *DoneBB = nullptr;
1417  // Emit the final values of the linear variables.
1418  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1419  auto IC = C->varlist_begin();
1420  for (const Expr *F : C->finals()) {
1421  if (!DoneBB) {
1422  if (llvm::Value *Cond = CondGen(*this)) {
1423  // If the first post-update expression is found, emit conditional
1424  // block if it was requested.
1425  llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu");
1426  DoneBB = createBasicBlock(".omp.linear.pu.done");
1427  Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1428  EmitBlock(ThenBB);
1429  }
1430  }
1431  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
1432  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
1433  CapturedStmtInfo->lookup(OrigVD) != nullptr,
1434  (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
1435  Address OrigAddr = EmitLValue(&DRE).getAddress();
1436  CodeGenFunction::OMPPrivateScope VarScope(*this);
1437  VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1438  (void)VarScope.Privatize();
1439  EmitIgnoredExpr(F);
1440  ++IC;
1441  }
1442  if (const Expr *PostUpdate = C->getPostUpdateExpr())
1443  EmitIgnoredExpr(PostUpdate);
1444  }
1445  if (DoneBB)
1446  EmitBlock(DoneBB, /*IsFinished=*/true);
1447 }
1448 
1450  const OMPExecutableDirective &D) {
1451  if (!CGF.HaveInsertPoint())
1452  return;
1453  for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
1454  unsigned ClauseAlignment = 0;
1455  if (const Expr *AlignmentExpr = Clause->getAlignment()) {
1456  auto *AlignmentCI =
1457  cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
1458  ClauseAlignment = static_cast<unsigned>(AlignmentCI->getZExtValue());
1459  }
1460  for (const Expr *E : Clause->varlists()) {
1461  unsigned Alignment = ClauseAlignment;
1462  if (Alignment == 0) {
1463  // OpenMP [2.8.1, Description]
1464  // If no optional parameter is specified, implementation-defined default
1465  // alignments for SIMD instructions on the target platforms are assumed.
1466  Alignment =
1467  CGF.getContext()
1469  E->getType()->getPointeeType()))
1470  .getQuantity();
1471  }
1472  assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) &&
1473  "alignment is not power of 2");
1474  if (Alignment != 0) {
1475  llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
1476  CGF.EmitAlignmentAssumption(PtrValue, Alignment);
1477  }
1478  }
1479  }
1480 }
1481 
1483  const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
1484  if (!HaveInsertPoint())
1485  return;
1486  auto I = S.private_counters().begin();
1487  for (const Expr *E : S.counters()) {
1488  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1489  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
1490  // Emit var without initialization.
1491  AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD);
1492  EmitAutoVarCleanups(VarEmission);
1493  LocalDeclMap.erase(PrivateVD);
1494  (void)LoopScope.addPrivate(VD, [&VarEmission]() {
1495  return VarEmission.getAllocatedAddress();
1496  });
1497  if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
1498  VD->hasGlobalStorage()) {
1499  (void)LoopScope.addPrivate(PrivateVD, [this, VD, E]() {
1500  DeclRefExpr DRE(const_cast<VarDecl *>(VD),
1501  LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
1502  E->getType(), VK_LValue, E->getExprLoc());
1503  return EmitLValue(&DRE).getAddress();
1504  });
1505  } else {
1506  (void)LoopScope.addPrivate(PrivateVD, [&VarEmission]() {
1507  return VarEmission.getAllocatedAddress();
1508  });
1509  }
1510  ++I;
1511  }
1512  // Privatize extra loop counters used in loops for ordered(n) clauses.
1513  for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) {
1514  if (!C->getNumForLoops())
1515  continue;
1516  for (unsigned I = S.getCollapsedNumber(),
1517  E = C->getLoopNumIterations().size();
1518  I < E; ++I) {
1519  const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I));
1520  const auto *VD = cast<VarDecl>(DRE->getDecl());
1521  // Override only those variables that are really emitted already.
1522  if (LocalDeclMap.count(VD)) {
1523  (void)LoopScope.addPrivate(VD, [this, DRE, VD]() {
1524  return CreateMemTemp(DRE->getType(), VD->getName());
1525  });
1526  }
1527  }
1528  }
1529 }
1530 
1531 static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
1532  const Expr *Cond, llvm::BasicBlock *TrueBlock,
1533  llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
1534  if (!CGF.HaveInsertPoint())
1535  return;
1536  {
1537  CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
1538  CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
1539  (void)PreCondScope.Privatize();
1540  // Get initial values of real counters.
1541  for (const Expr *I : S.inits()) {
1542  CGF.EmitIgnoredExpr(I);
1543  }
1544  }
1545  // Check that loop is executed at least one time.
1546  CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
1547 }
1548 
1550  const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
1551  if (!HaveInsertPoint())
1552  return;
1555  const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1556  for (const Expr *C : LoopDirective->counters()) {
1557  SIMDLCVs.insert(
1558  cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
1559  }
1560  }
1561  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1562  auto CurPrivate = C->privates().begin();
1563  for (const Expr *E : C->varlists()) {
1564  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1565  const auto *PrivateVD =
1566  cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
1567  if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
1568  bool IsRegistered = PrivateScope.addPrivate(VD, [this, PrivateVD]() {
1569  // Emit private VarDecl with copy init.
1570  EmitVarDecl(*PrivateVD);
1571  return GetAddrOfLocalVar(PrivateVD);
1572  });
1573  assert(IsRegistered && "linear var already registered as private");
1574  // Silence the warning about unused variable.
1575  (void)IsRegistered;
1576  } else {
1577  EmitVarDecl(*PrivateVD);
1578  }
1579  ++CurPrivate;
1580  }
1581  }
1582 }
1583 
1585  const OMPExecutableDirective &D,
1586  bool IsMonotonic) {
1587  if (!CGF.HaveInsertPoint())
1588  return;
1589  if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
1590  RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
1591  /*ignoreResult=*/true);
1592  auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1593  CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1594  // In presence of finite 'safelen', it may be unsafe to mark all
1595  // the memory instructions parallel, because loop-carried
1596  // dependences of 'safelen' iterations are possible.
1597  if (!IsMonotonic)
1599  } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
1600  RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
1601  /*ignoreResult=*/true);
1602  auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1603  CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1604  // In presence of finite 'safelen', it may be unsafe to mark all
1605  // the memory instructions parallel, because loop-carried
1606  // dependences of 'safelen' iterations are possible.
1607  CGF.LoopStack.setParallel(/*Enable=*/false);
1608  }
1609 }
1610 
1612  bool IsMonotonic) {
1613  // Walk clauses and process safelen/lastprivate.
1614  LoopStack.setParallel(!IsMonotonic);
1615  LoopStack.setVectorizeEnable();
1616  emitSimdlenSafelenClause(*this, D, IsMonotonic);
1617 }
1618 
1620  const OMPLoopDirective &D,
1621  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1622  if (!HaveInsertPoint())
1623  return;
1624  llvm::BasicBlock *DoneBB = nullptr;
1625  auto IC = D.counters().begin();
1626  auto IPC = D.private_counters().begin();
1627  for (const Expr *F : D.finals()) {
1628  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
1629  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
1630  const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
1631  if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
1632  OrigVD->hasGlobalStorage() || CED) {
1633  if (!DoneBB) {
1634  if (llvm::Value *Cond = CondGen(*this)) {
1635  // If the first post-update expression is found, emit conditional
1636  // block if it was requested.
1637  llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then");
1638  DoneBB = createBasicBlock(".omp.final.done");
1639  Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1640  EmitBlock(ThenBB);
1641  }
1642  }
1643  Address OrigAddr = Address::invalid();
1644  if (CED) {
1645  OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();
1646  } else {
1647  DeclRefExpr DRE(const_cast<VarDecl *>(PrivateVD),
1648  /*RefersToEnclosingVariableOrCapture=*/false,
1649  (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
1650  OrigAddr = EmitLValue(&DRE).getAddress();
1651  }
1652  OMPPrivateScope VarScope(*this);
1653  VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1654  (void)VarScope.Privatize();
1655  EmitIgnoredExpr(F);
1656  }
1657  ++IC;
1658  ++IPC;
1659  }
1660  if (DoneBB)
1661  EmitBlock(DoneBB, /*IsFinished=*/true);
1662 }
1663 
1665  const OMPLoopDirective &S,
1667  CGF.EmitOMPLoopBody(S, LoopExit);
1668  CGF.EmitStopPoint(&S);
1669 }
1670 
1671 /// Emit a helper variable and return corresponding lvalue.
1673  const DeclRefExpr *Helper) {
1674  auto VDecl = cast<VarDecl>(Helper->getDecl());
1675  CGF.EmitVarDecl(*VDecl);
1676  return CGF.EmitLValue(Helper);
1677 }
1678 
1680  PrePostActionTy &Action) {
1681  Action.Enter(CGF);
1683  "Expected simd directive");
1684  OMPLoopScope PreInitScope(CGF, S);
1685  // if (PreCond) {
1686  // for (IV in 0..LastIteration) BODY;
1687  // <Final counter/linear vars updates>;
1688  // }
1689  //
1693  (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));
1694  (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));
1695  }
1696 
1697  // Emit: if (PreCond) - begin.
1698  // If the condition constant folds and can be elided, avoid emitting the
1699  // whole loop.
1700  bool CondConstant;
1701  llvm::BasicBlock *ContBlock = nullptr;
1702  if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
1703  if (!CondConstant)
1704  return;
1705  } else {
1706  llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");
1707  ContBlock = CGF.createBasicBlock("simd.if.end");
1708  emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
1709  CGF.getProfileCount(&S));
1710  CGF.EmitBlock(ThenBlock);
1711  CGF.incrementProfileCounter(&S);
1712  }
1713 
1714  // Emit the loop iteration variable.
1715  const Expr *IVExpr = S.getIterationVariable();
1716  const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
1717  CGF.EmitVarDecl(*IVDecl);
1718  CGF.EmitIgnoredExpr(S.getInit());
1719 
1720  // Emit the iterations count variable.
1721  // If it is not a variable, Sema decided to calculate iterations count on
1722  // each iteration (e.g., it is foldable into a constant).
1723  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
1724  CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
1725  // Emit calculation of the iterations count.
1727  }
1728 
1729  CGF.EmitOMPSimdInit(S);
1730 
1731  emitAlignedClause(CGF, S);
1732  (void)CGF.EmitOMPLinearClauseInit(S);
1733  {
1734  CodeGenFunction::OMPPrivateScope LoopScope(CGF);
1735  CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
1736  CGF.EmitOMPLinearClause(S, LoopScope);
1737  CGF.EmitOMPPrivateClause(S, LoopScope);
1738  CGF.EmitOMPReductionClauseInit(S, LoopScope);
1739  bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
1740  (void)LoopScope.Privatize();
1742  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
1743  CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
1744  S.getInc(),
1745  [&S](CodeGenFunction &CGF) {
1747  CGF.EmitStopPoint(&S);
1748  },
1749  [](CodeGenFunction &) {});
1750  CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; });
1751  // Emit final copy of the lastprivate variables at the end of loops.
1752  if (HasLastprivateClause)
1753  CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
1754  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
1756  [](CodeGenFunction &) { return nullptr; });
1757  }
1758  CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; });
1759  // Emit: if (PreCond) - end.
1760  if (ContBlock) {
1761  CGF.EmitBranch(ContBlock);
1762  CGF.EmitBlock(ContBlock, true);
1763  }
1764 }
1765 
1767  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1768  emitOMPSimdRegion(CGF, S, Action);
1769  };
1770  OMPLexicalScope Scope(*this, S, OMPD_unknown);
1771  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
1772 }
1773 
1774 void CodeGenFunction::EmitOMPOuterLoop(
1775  bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
1777  const CodeGenFunction::OMPLoopArguments &LoopArgs,
1778  const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
1779  const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
1780  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1781 
1782  const Expr *IVExpr = S.getIterationVariable();
1783  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1784  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1785 
1786  JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
1787 
1788  // Start the loop with a block that tests the condition.
1789  llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond");
1790  EmitBlock(CondBlock);
1791  const SourceRange R = S.getSourceRange();
1792  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1793  SourceLocToDebugLoc(R.getEnd()));
1794 
1795  llvm::Value *BoolCondVal = nullptr;
1796  if (!DynamicOrOrdered) {
1797  // UB = min(UB, GlobalUB) or
1798  // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
1799  // 'distribute parallel for')
1800  EmitIgnoredExpr(LoopArgs.EUB);
1801  // IV = LB
1802  EmitIgnoredExpr(LoopArgs.Init);
1803  // IV < UB
1804  BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
1805  } else {
1806  BoolCondVal =
1807  RT.emitForNext(*this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
1808  LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
1809  }
1810 
1811  // If there are any cleanups between here and the loop-exit scope,
1812  // create a block to stage a loop exit along.
1813  llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1814  if (LoopScope.requiresCleanups())
1815  ExitBlock = createBasicBlock("omp.dispatch.cleanup");
1816 
1817  llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body");
1818  Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
1819  if (ExitBlock != LoopExit.getBlock()) {
1820  EmitBlock(ExitBlock);
1821  EmitBranchThroughCleanup(LoopExit);
1822  }
1823  EmitBlock(LoopBody);
1824 
1825  // Emit "IV = LB" (in case of static schedule, we have already calculated new
1826  // LB for loop condition and emitted it above).
1827  if (DynamicOrOrdered)
1828  EmitIgnoredExpr(LoopArgs.Init);
1829 
1830  // Create a block for the increment.
1831  JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
1832  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1833 
1834  // Generate !llvm.loop.parallel metadata for loads and stores for loops
1835  // with dynamic/guided scheduling and without ordered clause.
1837  LoopStack.setParallel(!IsMonotonic);
1838  else
1839  EmitOMPSimdInit(S, IsMonotonic);
1840 
1841  SourceLocation Loc = S.getBeginLoc();
1842 
1843  // when 'distribute' is not combined with a 'for':
1844  // while (idx <= UB) { BODY; ++idx; }
1845  // when 'distribute' is combined with a 'for'
1846  // (e.g. 'distribute parallel for')
1847  // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
1848  EmitOMPInnerLoop(
1849  S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
1850  [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
1851  CodeGenLoop(CGF, S, LoopExit);
1852  },
1853  [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
1854  CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
1855  });
1856 
1857  EmitBlock(Continue.getBlock());
1858  BreakContinueStack.pop_back();
1859  if (!DynamicOrOrdered) {
1860  // Emit "LB = LB + Stride", "UB = UB + Stride".
1861  EmitIgnoredExpr(LoopArgs.NextLB);
1862  EmitIgnoredExpr(LoopArgs.NextUB);
1863  }
1864 
1865  EmitBranch(CondBlock);
1866  LoopStack.pop();
1867  // Emit the fall-through block.
1868  EmitBlock(LoopExit.getBlock());
1869 
1870  // Tell the runtime we are done.
1871  auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) {
1872  if (!DynamicOrOrdered)
1873  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
1874  S.getDirectiveKind());
1875  };
1876  OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
1877 }
1878 
1879 void CodeGenFunction::EmitOMPForOuterLoop(
1880  const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
1881  const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
1882  const OMPLoopArguments &LoopArgs,
1883  const CodeGenDispatchBoundsTy &CGDispatchBounds) {
1884  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1885 
1886  // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
1887  const bool DynamicOrOrdered =
1888  Ordered || RT.isDynamic(ScheduleKind.Schedule);
1889 
1890  assert((Ordered ||
1891  !RT.isStaticNonchunked(ScheduleKind.Schedule,
1892  LoopArgs.Chunk != nullptr)) &&
1893  "static non-chunked schedule does not need outer loop");
1894 
1895  // Emit outer loop.
1896  //
1897  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1898  // When schedule(dynamic,chunk_size) is specified, the iterations are
1899  // distributed to threads in the team in chunks as the threads request them.
1900  // Each thread executes a chunk of iterations, then requests another chunk,
1901  // until no chunks remain to be distributed. Each chunk contains chunk_size
1902  // iterations, except for the last chunk to be distributed, which may have
1903  // fewer iterations. When no chunk_size is specified, it defaults to 1.
1904  //
1905  // When schedule(guided,chunk_size) is specified, the iterations are assigned
1906  // to threads in the team in chunks as the executing threads request them.
1907  // Each thread executes a chunk of iterations, then requests another chunk,
1908  // until no chunks remain to be assigned. For a chunk_size of 1, the size of
1909  // each chunk is proportional to the number of unassigned iterations divided
1910  // by the number of threads in the team, decreasing to 1. For a chunk_size
1911  // with value k (greater than 1), the size of each chunk is determined in the
1912  // same way, with the restriction that the chunks do not contain fewer than k
1913  // iterations (except for the last chunk to be assigned, which may have fewer
1914  // than k iterations).
1915  //
1916  // When schedule(auto) is specified, the decision regarding scheduling is
1917  // delegated to the compiler and/or runtime system. The programmer gives the
1918  // implementation the freedom to choose any possible mapping of iterations to
1919  // threads in the team.
1920  //
1921  // When schedule(runtime) is specified, the decision regarding scheduling is
1922  // deferred until run time, and the schedule and chunk size are taken from the
1923  // run-sched-var ICV. If the ICV is set to auto, the schedule is
1924  // implementation defined
1925  //
1926  // while(__kmpc_dispatch_next(&LB, &UB)) {
1927  // idx = LB;
1928  // while (idx <= UB) { BODY; ++idx;
1929  // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
1930  // } // inner loop
1931  // }
1932  //
1933  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1934  // When schedule(static, chunk_size) is specified, iterations are divided into
1935  // chunks of size chunk_size, and the chunks are assigned to the threads in
1936  // the team in a round-robin fashion in the order of the thread number.
1937  //
1938  // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
1939  // while (idx <= UB) { BODY; ++idx; } // inner loop
1940  // LB = LB + ST;
1941  // UB = UB + ST;
1942  // }
1943  //
1944 
1945  const Expr *IVExpr = S.getIterationVariable();
1946  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1947  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1948 
1949  if (DynamicOrOrdered) {
1950  const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
1951  CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
1952  llvm::Value *LBVal = DispatchBounds.first;
1953  llvm::Value *UBVal = DispatchBounds.second;
1954  CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
1955  LoopArgs.Chunk};
1956  RT.emitForDispatchInit(*this, S.getBeginLoc(), ScheduleKind, IVSize,
1957  IVSigned, Ordered, DipatchRTInputValues);
1958  } else {
1959  CGOpenMPRuntime::StaticRTInput StaticInit(
1960  IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
1961  LoopArgs.ST, LoopArgs.Chunk);
1962  RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
1963  ScheduleKind, StaticInit);
1964  }
1965 
1966  auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
1967  const unsigned IVSize,
1968  const bool IVSigned) {
1969  if (Ordered) {
1970  CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
1971  IVSigned);
1972  }
1973  };
1974 
1975  OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
1976  LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
1977  OuterLoopArgs.IncExpr = S.getInc();
1978  OuterLoopArgs.Init = S.getInit();
1979  OuterLoopArgs.Cond = S.getCond();
1980  OuterLoopArgs.NextLB = S.getNextLowerBound();
1981  OuterLoopArgs.NextUB = S.getNextUpperBound();
1982  EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
1983  emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
1984 }
1985 
1987  const unsigned IVSize, const bool IVSigned) {}
1988 
1989 void CodeGenFunction::EmitOMPDistributeOuterLoop(
1990  OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
1991  OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
1992  const CodeGenLoopTy &CodeGenLoopContent) {
1993 
1994  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1995 
1996  // Emit outer loop.
1997  // Same behavior as a OMPForOuterLoop, except that schedule cannot be
1998  // dynamic
1999  //
2000 
2001  const Expr *IVExpr = S.getIterationVariable();
2002  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2003  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2004 
2005  CGOpenMPRuntime::StaticRTInput StaticInit(
2006  IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
2007  LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
2008  RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, StaticInit);
2009 
2010  // for combined 'distribute' and 'for' the increment expression of distribute
2011  // is stored in DistInc. For 'distribute' alone, it is in Inc.
2012  Expr *IncExpr;
2014  IncExpr = S.getDistInc();
2015  else
2016  IncExpr = S.getInc();
2017 
2018  // this routine is shared by 'omp distribute parallel for' and
2019  // 'omp distribute': select the right EUB expression depending on the
2020  // directive
2021  OMPLoopArguments OuterLoopArgs;
2022  OuterLoopArgs.LB = LoopArgs.LB;
2023  OuterLoopArgs.UB = LoopArgs.UB;
2024  OuterLoopArgs.ST = LoopArgs.ST;
2025  OuterLoopArgs.IL = LoopArgs.IL;
2026  OuterLoopArgs.Chunk = LoopArgs.Chunk;
2027  OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2029  : S.getEnsureUpperBound();
2030  OuterLoopArgs.IncExpr = IncExpr;
2031  OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2032  ? S.getCombinedInit()
2033  : S.getInit();
2034  OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2035  ? S.getCombinedCond()
2036  : S.getCond();
2037  OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2039  : S.getNextLowerBound();
2040  OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2042  : S.getNextUpperBound();
2043 
2044  EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
2045  LoopScope, OuterLoopArgs, CodeGenLoopContent,
2047 }
2048 
2049 static std::pair<LValue, LValue>
2051  const OMPExecutableDirective &S) {
2052  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2053  LValue LB =
2054  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2055  LValue UB =
2056  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2057 
2058  // When composing 'distribute' with 'for' (e.g. as in 'distribute
2059  // parallel for') we need to use the 'distribute'
2060  // chunk lower and upper bounds rather than the whole loop iteration
2061  // space. These are parameters to the outlined function for 'parallel'
2062  // and we copy the bounds of the previous schedule into the
2063  // the current ones.
2064  LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
2065  LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
2066  llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
2067  PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());
2068  PrevLBVal = CGF.EmitScalarConversion(
2069  PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
2070  LS.getIterationVariable()->getType(),
2072  llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
2073  PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());
2074  PrevUBVal = CGF.EmitScalarConversion(
2075  PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
2076  LS.getIterationVariable()->getType(),
2078 
2079  CGF.EmitStoreOfScalar(PrevLBVal, LB);
2080  CGF.EmitStoreOfScalar(PrevUBVal, UB);
2081 
2082  return {LB, UB};
2083 }
2084 
2085 /// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
2086 /// we need to use the LB and UB expressions generated by the worksharing
2087 /// code generation support, whereas in non combined situations we would
2088 /// just emit 0 and the LastIteration expression
2089 /// This function is necessary due to the difference of the LB and UB
2090 /// types for the RT emission routines for 'for_static_init' and
2091 /// 'for_dispatch_init'
2092 static std::pair<llvm::Value *, llvm::Value *>
2094  const OMPExecutableDirective &S,
2095  Address LB, Address UB) {
2096  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2097  const Expr *IVExpr = LS.getIterationVariable();
2098  // when implementing a dynamic schedule for a 'for' combined with a
2099  // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
2100  // is not normalized as each team only executes its own assigned
2101  // distribute chunk
2102  QualType IteratorTy = IVExpr->getType();
2103  llvm::Value *LBVal =
2104  CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
2105  llvm::Value *UBVal =
2106  CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
2107  return {LBVal, UBVal};
2108 }
2109 
2111  CodeGenFunction &CGF, const OMPExecutableDirective &S,
2112  llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) {
2113  const auto &Dir = cast<OMPLoopDirective>(S);
2114  LValue LB =
2115  CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
2116  llvm::Value *LBCast = CGF.Builder.CreateIntCast(
2117  CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2118  CapturedVars.push_back(LBCast);
2119  LValue UB =
2120  CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
2121 
2122  llvm::Value *UBCast = CGF.Builder.CreateIntCast(
2123  CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2124  CapturedVars.push_back(UBCast);
2125 }
2126 
2127 static void
2129  const OMPLoopDirective &S,
2131  auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
2132  PrePostActionTy &Action) {
2133  Action.Enter(CGF);
2134  bool HasCancel = false;
2136  if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
2137  HasCancel = D->hasCancel();
2138  else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
2139  HasCancel = D->hasCancel();
2140  else if (const auto *D =
2141  dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
2142  HasCancel = D->hasCancel();
2143  }
2145  HasCancel);
2149  };
2150 
2152  CGF, S,
2153  isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for,
2154  CGInlinedWorksharingLoop,
2156 }
2157 
2160  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2162  S.getDistInc());
2163  };
2164  OMPLexicalScope Scope(*this, S, OMPD_parallel);
2165  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2166 }
2167 
2170  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2172  S.getDistInc());
2173  };
2174  OMPLexicalScope Scope(*this, S, OMPD_parallel);
2175  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2176 }
2177 
2179  const OMPDistributeSimdDirective &S) {
2180  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2182  };
2183  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2184  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2185 }
2186 
2188  CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
2189  // Emit SPMD target parallel for region as a standalone region.
2190  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2191  emitOMPSimdRegion(CGF, S, Action);
2192  };
2193  llvm::Function *Fn;
2194  llvm::Constant *Addr;
2195  // Emit target region as a standalone region.
2196  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
2197  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
2198  assert(Fn && Addr && "Target device function emission failed.");
2199 }
2200 
2202  const OMPTargetSimdDirective &S) {
2203  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2204  emitOMPSimdRegion(CGF, S, Action);
2205  };
2206  emitCommonOMPTargetDirective(*this, S, CodeGen);
2207 }
2208 
2209 namespace {
2210  struct ScheduleKindModifiersTy {
2214  ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
2217  : Kind(Kind), M1(M1), M2(M2) {}
2218  };
2219 } // namespace
2220 
2222  const OMPLoopDirective &S, Expr *EUB,
2223  const CodeGenLoopBoundsTy &CodeGenLoopBounds,
2224  const CodeGenDispatchBoundsTy &CGDispatchBounds) {
2225  // Emit the loop iteration variable.
2226  const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
2227  const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
2228  EmitVarDecl(*IVDecl);
2229 
2230  // Emit the iterations count variable.
2231  // If it is not a variable, Sema decided to calculate iterations count on each
2232  // iteration (e.g., it is foldable into a constant).
2233  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2234  EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2235  // Emit calculation of the iterations count.
2236  EmitIgnoredExpr(S.getCalcLastIteration());
2237  }
2238 
2239  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
2240 
2241  bool HasLastprivateClause;
2242  // Check pre-condition.
2243  {
2244  OMPLoopScope PreInitScope(*this, S);
2245  // Skip the entire loop if we don't meet the precondition.
2246  // If the condition constant folds and can be elided, avoid emitting the
2247  // whole loop.
2248  bool CondConstant;
2249  llvm::BasicBlock *ContBlock = nullptr;
2250  if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2251  if (!CondConstant)
2252  return false;
2253  } else {
2254  llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
2255  ContBlock = createBasicBlock("omp.precond.end");
2256  emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
2257  getProfileCount(&S));
2258  EmitBlock(ThenBlock);
2259  incrementProfileCounter(&S);
2260  }
2261 
2262  RunCleanupsScope DoacrossCleanupScope(*this);
2263  bool Ordered = false;
2264  if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
2265  if (OrderedClause->getNumForLoops())
2266  RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations());
2267  else
2268  Ordered = true;
2269  }
2270 
2271  llvm::DenseSet<const Expr *> EmittedFinals;
2272  emitAlignedClause(*this, S);
2273  bool HasLinears = EmitOMPLinearClauseInit(S);
2274  // Emit helper vars inits.
2275 
2276  std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
2277  LValue LB = Bounds.first;
2278  LValue UB = Bounds.second;
2279  LValue ST =
2280  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
2281  LValue IL =
2282  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
2283 
2284  // Emit 'then' code.
2285  {
2286  OMPPrivateScope LoopScope(*this);
2287  if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
2288  // Emit implicit barrier to synchronize threads and avoid data races on
2289  // initialization of firstprivate variables and post-update of
2290  // lastprivate variables.
2291  CGM.getOpenMPRuntime().emitBarrierCall(
2292  *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
2293  /*ForceSimpleCall=*/true);
2294  }
2295  EmitOMPPrivateClause(S, LoopScope);
2296  HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
2297  EmitOMPReductionClauseInit(S, LoopScope);
2298  EmitOMPPrivateLoopCounters(S, LoopScope);
2299  EmitOMPLinearClause(S, LoopScope);
2300  (void)LoopScope.Privatize();
2302  CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
2303 
2304  // Detect the loop schedule kind and chunk.
2305  const Expr *ChunkExpr = nullptr;
2306  OpenMPScheduleTy ScheduleKind;
2307  if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
2308  ScheduleKind.Schedule = C->getScheduleKind();
2309  ScheduleKind.M1 = C->getFirstScheduleModifier();
2310  ScheduleKind.M2 = C->getSecondScheduleModifier();
2311  ChunkExpr = C->getChunkSize();
2312  } else {
2313  // Default behaviour for schedule clause.
2314  CGM.getOpenMPRuntime().getDefaultScheduleAndChunk(
2315  *this, S, ScheduleKind.Schedule, ChunkExpr);
2316  }
2317  bool HasChunkSizeOne = false;
2318  llvm::Value *Chunk = nullptr;
2319  if (ChunkExpr) {
2320  Chunk = EmitScalarExpr(ChunkExpr);
2321  Chunk = EmitScalarConversion(Chunk, ChunkExpr->getType(),
2323  S.getBeginLoc());
2324  Expr::EvalResult Result;
2325  if (ChunkExpr->EvaluateAsInt(Result, getContext())) {
2326  llvm::APSInt EvaluatedChunk = Result.Val.getInt();
2327  HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
2328  }
2329  }
2330  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2331  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2332  // OpenMP 4.5, 2.7.1 Loop Construct, Description.
2333  // If the static schedule kind is specified or if the ordered clause is
2334  // specified, and if no monotonic modifier is specified, the effect will
2335  // be as if the monotonic modifier was specified.
2336  bool StaticChunkedOne = RT.isStaticChunked(ScheduleKind.Schedule,
2337  /* Chunked */ Chunk != nullptr) && HasChunkSizeOne &&
2339  if ((RT.isStaticNonchunked(ScheduleKind.Schedule,
2340  /* Chunked */ Chunk != nullptr) ||
2341  StaticChunkedOne) &&
2342  !Ordered) {
2344  EmitOMPSimdInit(S, /*IsMonotonic=*/true);
2345  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2346  // When no chunk_size is specified, the iteration space is divided into
2347  // chunks that are approximately equal in size, and at most one chunk is
2348  // distributed to each thread. Note that the size of the chunks is
2349  // unspecified in this case.
2350  CGOpenMPRuntime::StaticRTInput StaticInit(
2351  IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
2352  UB.getAddress(), ST.getAddress(),
2353  StaticChunkedOne ? Chunk : nullptr);
2354  RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
2355  ScheduleKind, StaticInit);
2356  JumpDest LoopExit =
2357  getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
2358  // UB = min(UB, GlobalUB);
2359  if (!StaticChunkedOne)
2360  EmitIgnoredExpr(S.getEnsureUpperBound());
2361  // IV = LB;
2362  EmitIgnoredExpr(S.getInit());
2363  // For unchunked static schedule generate:
2364  //
2365  // while (idx <= UB) {
2366  // BODY;
2367  // ++idx;
2368  // }
2369  //
2370  // For static schedule with chunk one:
2371  //
2372  // while (IV <= PrevUB) {
2373  // BODY;
2374  // IV += ST;
2375  // }
2376  EmitOMPInnerLoop(S, LoopScope.requiresCleanups(),
2377  StaticChunkedOne ? S.getCombinedParForInDistCond() : S.getCond(),
2378  StaticChunkedOne ? S.getDistInc() : S.getInc(),
2379  [&S, LoopExit](CodeGenFunction &CGF) {
2380  CGF.EmitOMPLoopBody(S, LoopExit);
2381  CGF.EmitStopPoint(&S);
2382  },
2383  [](CodeGenFunction &) {});
2384  EmitBlock(LoopExit.getBlock());
2385  // Tell the runtime we are done.
2386  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2387  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2388  S.getDirectiveKind());
2389  };
2390  OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
2391  } else {
2392  const bool IsMonotonic =
2393  Ordered || ScheduleKind.Schedule == OMPC_SCHEDULE_static ||
2394  ScheduleKind.Schedule == OMPC_SCHEDULE_unknown ||
2395  ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
2396  ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
2397  // Emit the outer loop, which requests its work chunk [LB..UB] from
2398  // runtime and runs the inner loop to process it.
2399  const OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),
2400  ST.getAddress(), IL.getAddress(),
2401  Chunk, EUB);
2402  EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
2403  LoopArguments, CGDispatchBounds);
2404  }
2406  EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
2407  return CGF.Builder.CreateIsNotNull(
2408  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2409  });
2410  }
2411  EmitOMPReductionClauseFinal(
2412  S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind())
2413  ? /*Parallel and Simd*/ OMPD_parallel_for_simd
2414  : /*Parallel only*/ OMPD_parallel);
2415  // Emit post-update of the reduction variables if IsLastIter != 0.
2417  *this, S, [IL, &S](CodeGenFunction &CGF) {
2418  return CGF.Builder.CreateIsNotNull(
2419  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2420  });
2421  // Emit final copy of the lastprivate variables if IsLastIter != 0.
2422  if (HasLastprivateClause)
2423  EmitOMPLastprivateClauseFinal(
2425  Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
2426  }
2427  EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) {
2428  return CGF.Builder.CreateIsNotNull(
2429  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2430  });
2431  DoacrossCleanupScope.ForceCleanup();
2432  // We're now done with the loop, so jump to the continuation block.
2433  if (ContBlock) {
2434  EmitBranch(ContBlock);
2435  EmitBlock(ContBlock, /*IsFinished=*/true);
2436  }
2437  }
2438  return HasLastprivateClause;
2439 }
2440 
2441 /// The following two functions generate expressions for the loop lower
2442 /// and upper bounds in case of static and dynamic (dispatch) schedule
2443 /// of the associated 'for' or 'distribute' loop.
2444 static std::pair<LValue, LValue>
2446  const auto &LS = cast<OMPLoopDirective>(S);
2447  LValue LB =
2448  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2449  LValue UB =
2450  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2451  return {LB, UB};
2452 }
2453 
2454 /// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
2455 /// consider the lower and upper bound expressions generated by the
2456 /// worksharing loop support, but we use 0 and the iteration space size as
2457 /// constants
2458 static std::pair<llvm::Value *, llvm::Value *>
2460  Address LB, Address UB) {
2461  const auto &LS = cast<OMPLoopDirective>(S);
2462  const Expr *IVExpr = LS.getIterationVariable();
2463  const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
2464  llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
2465  llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
2466  return {LBVal, UBVal};
2467 }
2468 
2470  bool HasLastprivates = false;
2471  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2472  PrePostActionTy &) {
2473  OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel());
2474  HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2477  };
2478  {
2479  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2480  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
2481  S.hasCancel());
2482  }
2483 
2484  // Emit an implicit barrier at the end.
2485  if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
2486  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
2487 }
2488 
2490  bool HasLastprivates = false;
2491  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2492  PrePostActionTy &) {
2493  HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2496  };
2497  {
2498  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2499  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2500  }
2501 
2502  // Emit an implicit barrier at the end.
2503  if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
2504  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
2505 }
2506 
2508  const Twine &Name,
2509  llvm::Value *Init = nullptr) {
2510  LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
2511  if (Init)
2512  CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
2513  return LVal;
2514 }
2515 
2516 void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
2518  const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
2519  bool HasLastprivates = false;
2520  auto &&CodeGen = [&S, CapturedStmt, CS,
2521  &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
2522  ASTContext &C = CGF.getContext();
2523  QualType KmpInt32Ty =
2524  C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2525  // Emit helper vars inits.
2526  LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
2527  CGF.Builder.getInt32(0));
2528  llvm::ConstantInt *GlobalUBVal = CS != nullptr
2529  ? CGF.Builder.getInt32(CS->size() - 1)
2530  : CGF.Builder.getInt32(0);
2531  LValue UB =
2532  createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
2533  LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
2534  CGF.Builder.getInt32(1));
2535  LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
2536  CGF.Builder.getInt32(0));
2537  // Loop counter.
2538  LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
2539  OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
2540  CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
2541  OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
2542  CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
2543  // Generate condition for loop.
2544  BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue,
2546  // Increment for loop counter.
2547  UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary,
2548  S.getBeginLoc(), true);
2549  auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
2550  // Iterate through all sections and emit a switch construct:
2551  // switch (IV) {
2552  // case 0:
2553  // <SectionStmt[0]>;
2554  // break;
2555  // ...
2556  // case <NumSection> - 1:
2557  // <SectionStmt[<NumSection> - 1]>;
2558  // break;
2559  // }
2560  // .omp.sections.exit:
2561  llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
2562  llvm::SwitchInst *SwitchStmt =
2563  CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
2564  ExitBB, CS == nullptr ? 1 : CS->size());
2565  if (CS) {
2566  unsigned CaseNumber = 0;
2567  for (const Stmt *SubStmt : CS->children()) {
2568  auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2569  CGF.EmitBlock(CaseBB);
2570  SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
2571  CGF.EmitStmt(SubStmt);
2572  CGF.EmitBranch(ExitBB);
2573  ++CaseNumber;
2574  }
2575  } else {
2576  llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");
2577  CGF.EmitBlock(CaseBB);
2578  SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
2579  CGF.EmitStmt(CapturedStmt);
2580  CGF.EmitBranch(ExitBB);
2581  }
2582  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2583  };
2584 
2585  CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2586  if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
2587  // Emit implicit barrier to synchronize threads and avoid data races on
2588  // initialization of firstprivate variables and post-update of lastprivate
2589  // variables.
2590  CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2591  CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
2592  /*ForceSimpleCall=*/true);
2593  }
2594  CGF.EmitOMPPrivateClause(S, LoopScope);
2595  HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2596  CGF.EmitOMPReductionClauseInit(S, LoopScope);
2597  (void)LoopScope.Privatize();
2598  if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
2599  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
2600 
2601  // Emit static non-chunked loop.
2602  OpenMPScheduleTy ScheduleKind;
2603  ScheduleKind.Schedule = OMPC_SCHEDULE_static;
2604  CGOpenMPRuntime::StaticRTInput StaticInit(
2605  /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(),
2606  LB.getAddress(), UB.getAddress(), ST.getAddress());
2607  CGF.CGM.getOpenMPRuntime().emitForStaticInit(
2608  CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit);
2609  // UB = min(UB, GlobalUB);
2610  llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
2611  llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
2612  CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
2613  CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
2614  // IV = LB;
2615  CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
2616  // while (idx <= UB) { BODY; ++idx; }
2617  CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen,
2618  [](CodeGenFunction &) {});
2619  // Tell the runtime we are done.
2620  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2621  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2622  S.getDirectiveKind());
2623  };
2624  CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
2625  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
2626  // Emit post-update of the reduction variables if IsLastIter != 0.
2627  emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) {
2628  return CGF.Builder.CreateIsNotNull(
2629  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2630  });
2631 
2632  // Emit final copy of the lastprivate variables if IsLastIter != 0.
2633  if (HasLastprivates)
2635  S, /*NoFinals=*/false,
2636  CGF.Builder.CreateIsNotNull(
2637  CGF.EmitLoadOfScalar(IL, S.getBeginLoc())));
2638  };
2639 
2640  bool HasCancel = false;
2641  if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
2642  HasCancel = OSD->hasCancel();
2643  else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
2644  HasCancel = OPSD->hasCancel();
2645  OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
2646  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
2647  HasCancel);
2648  // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
2649  // clause. Otherwise the barrier will be generated by the codegen for the
2650  // directive.
2651  if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
2652  // Emit implicit barrier to synchronize threads and avoid data races on
2653  // initialization of firstprivate variables.
2654  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
2655  OMPD_unknown);
2656  }
2657 }
2658 
2660  {
2661  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2662  EmitSections(S);
2663  }
2664  // Emit an implicit barrier at the end.
2665  if (!S.getSingleClause<OMPNowaitClause>()) {
2666  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
2667  OMPD_sections);
2668  }
2669 }
2670 
2672  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2674  };
2675  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2676  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen,
2677  S.hasCancel());
2678 }
2679 
2681  llvm::SmallVector<const Expr *, 8> CopyprivateVars;
2684  llvm::SmallVector<const Expr *, 8> AssignmentOps;
2685  // Check if there are any 'copyprivate' clauses associated with this
2686  // 'single' construct.
2687  // Build a list of copyprivate variables along with helper expressions
2688  // (<source>, <destination>, <destination>=<source> expressions)
2689  for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
2690  CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
2691  DestExprs.append(C->destination_exprs().begin(),
2692  C->destination_exprs().end());
2693  SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
2694  AssignmentOps.append(C->assignment_ops().begin(),
2695  C->assignment_ops().end());
2696  }
2697  // Emit code for 'single' region along with 'copyprivate' clauses
2698  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2699  Action.Enter(CGF);
2700  OMPPrivateScope SingleScope(CGF);
2701  (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
2702  CGF.EmitOMPPrivateClause(S, SingleScope);
2703  (void)SingleScope.Privatize();
2705  };
2706  {
2707  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2708  CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getBeginLoc(),
2709  CopyprivateVars, DestExprs,
2710  SrcExprs, AssignmentOps);
2711  }
2712  // Emit an implicit barrier at the end (to avoid data race on firstprivate
2713  // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
2714  if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
2715  CGM.getOpenMPRuntime().emitBarrierCall(
2716  *this, S.getBeginLoc(),
2717  S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
2718  }
2719 }
2720 
2722  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2723  Action.Enter(CGF);
2725  };
2726  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2727  CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
2728 }
2729 
2731  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2732  Action.Enter(CGF);
2734  };
2735  const Expr *Hint = nullptr;
2736  if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
2737  Hint = HintClause->getHint();
2738  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2739  CGM.getOpenMPRuntime().emitCriticalRegion(*this,
2741  CodeGen, S.getBeginLoc(), Hint);
2742 }
2743 
2745  const OMPParallelForDirective &S) {
2746  // Emit directive as a combined directive that consists of two implicit
2747  // directives: 'parallel' with 'for' directive.
2748  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2749  Action.Enter(CGF);
2750  OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel());
2753  };
2754  emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
2756 }
2757 
2759  const OMPParallelForSimdDirective &S) {
2760  // Emit directive as a combined directive that consists of two implicit
2761  // directives: 'parallel' with 'for' directive.
2762  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2763  Action.Enter(CGF);
2766  };
2767  emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen,
2769 }
2770 
2772  const OMPParallelSectionsDirective &S) {
2773  // Emit directive as a combined directive that consists of two implicit
2774  // directives: 'parallel' with 'sections' directive.
2775  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2776  Action.Enter(CGF);
2777  CGF.EmitSections(S);
2778  };
2779  emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
2781 }
2782 
2784  const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
2785  const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
2786  OMPTaskDataTy &Data) {
2787  // Emit outlined function for task construct.
2788  const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
2789  auto I = CS->getCapturedDecl()->param_begin();
2790  auto PartId = std::next(I);
2791  auto TaskT = std::next(I, 4);
2792  // Check if the task is final
2793  if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
2794  // If the condition constant folds and can be elided, try to avoid emitting
2795  // the condition and the dead arm of the if/else.
2796  const Expr *Cond = Clause->getCondition();
2797  bool CondConstant;
2798  if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
2799  Data.Final.setInt(CondConstant);
2800  else
2801  Data.Final.setPointer(EvaluateExprAsBool(Cond));
2802  } else {
2803  // By default the task is not final.
2804  Data.Final.setInt(/*IntVal=*/false);
2805  }
2806  // Check if the task has 'priority' clause.
2807  if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
2808  const Expr *Prio = Clause->getPriority();
2809  Data.Priority.setInt(/*IntVal=*/true);
2810  Data.Priority.setPointer(EmitScalarConversion(
2811  EmitScalarExpr(Prio), Prio->getType(),
2812  getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
2813  Prio->getExprLoc()));
2814  }
2815  // The first function argument for tasks is a thread id, the second one is a
2816  // part id (0 for tied tasks, >=0 for untied task).
2817  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
2818  // Get list of private variables.
2819  for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
2820  auto IRef = C->varlist_begin();
2821  for (const Expr *IInit : C->private_copies()) {
2822  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2823  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2824  Data.PrivateVars.push_back(*IRef);
2825  Data.PrivateCopies.push_back(IInit);
2826  }
2827  ++IRef;
2828  }
2829  }
2830  EmittedAsPrivate.clear();
2831  // Get list of firstprivate variables.
2832  for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
2833  auto IRef = C->varlist_begin();
2834  auto IElemInitRef = C->inits().begin();
2835  for (const Expr *IInit : C->private_copies()) {
2836  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2837  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2838  Data.FirstprivateVars.push_back(*IRef);
2839  Data.FirstprivateCopies.push_back(IInit);
2840  Data.FirstprivateInits.push_back(*IElemInitRef);
2841  }
2842  ++IRef;
2843  ++IElemInitRef;
2844  }
2845  }
2846  // Get list of lastprivate variables (for taskloops).
2847  llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
2848  for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
2849  auto IRef = C->varlist_begin();
2850  auto ID = C->destination_exprs().begin();
2851  for (const Expr *IInit : C->private_copies()) {
2852  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2853  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2854  Data.LastprivateVars.push_back(*IRef);
2855  Data.LastprivateCopies.push_back(IInit);
2856  }
2857  LastprivateDstsOrigs.insert(
2858  {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
2859  cast<DeclRefExpr>(*IRef)});
2860  ++IRef;
2861  ++ID;
2862  }
2863  }
2866  for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
2867  auto IPriv = C->privates().begin();
2868  auto IRed = C->reduction_ops().begin();
2869  auto ILHS = C->lhs_exprs().begin();
2870  auto IRHS = C->rhs_exprs().begin();
2871  for (const Expr *Ref : C->varlists()) {
2872  Data.ReductionVars.emplace_back(Ref);
2873  Data.ReductionCopies.emplace_back(*IPriv);
2874  Data.ReductionOps.emplace_back(*IRed);
2875  LHSs.emplace_back(*ILHS);
2876  RHSs.emplace_back(*IRHS);
2877  std::advance(IPriv, 1);
2878  std::advance(IRed, 1);
2879  std::advance(ILHS, 1);
2880  std::advance(IRHS, 1);
2881  }
2882  }
2883  Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
2884  *this, S.getBeginLoc(), LHSs, RHSs, Data);
2885  // Build list of dependences.
2886  for (const auto *C : S.getClausesOfKind<OMPDependClause>())
2887  for (const Expr *IRef : C->varlists())
2888  Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
2889  auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
2890  CapturedRegion](CodeGenFunction &CGF,
2891  PrePostActionTy &Action) {
2892  // Set proper addresses for generated private copies.
2893  OMPPrivateScope Scope(CGF);
2894  if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
2895  !Data.LastprivateVars.empty()) {
2896  enum { PrivatesParam = 2, CopyFnParam = 3 };
2897  llvm::Value *CopyFn = CGF.Builder.CreateLoad(
2898  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
2899  llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
2900  CS->getCapturedDecl()->getParam(PrivatesParam)));
2901  // Map privates.
2904  CallArgs.push_back(PrivatesPtr);
2905  for (const Expr *E : Data.PrivateVars) {
2906  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2907  Address PrivatePtr = CGF.CreateMemTemp(
2908  CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
2909  PrivatePtrs.emplace_back(VD, PrivatePtr);
2910  CallArgs.push_back(PrivatePtr.getPointer());
2911  }
2912  for (const Expr *E : Data.FirstprivateVars) {
2913  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2914  Address PrivatePtr =
2915  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2916  ".firstpriv.ptr.addr");
2917  PrivatePtrs.emplace_back(VD, PrivatePtr);
2918  CallArgs.push_back(PrivatePtr.getPointer());
2919  }
2920  for (const Expr *E : Data.LastprivateVars) {
2921  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2922  Address PrivatePtr =
2923  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2924  ".lastpriv.ptr.addr");
2925  PrivatePtrs.emplace_back(VD, PrivatePtr);
2926  CallArgs.push_back(PrivatePtr.getPointer());
2927  }
2928  CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
2929  CopyFn, CallArgs);
2930  for (const auto &Pair : LastprivateDstsOrigs) {
2931  const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
2932  DeclRefExpr DRE(
2933  const_cast<VarDecl *>(OrigVD),
2934  /*RefersToEnclosingVariableOrCapture=*/CGF.CapturedStmtInfo->lookup(
2935  OrigVD) != nullptr,
2936  Pair.second->getType(), VK_LValue, Pair.second->getExprLoc());
2937  Scope.addPrivate(Pair.first, [&CGF, &DRE]() {
2938  return CGF.EmitLValue(&DRE).getAddress();
2939  });
2940  }
2941  for (const auto &Pair : PrivatePtrs) {
2942  Address Replacement(CGF.Builder.CreateLoad(Pair.second),
2943  CGF.getContext().getDeclAlign(Pair.first));
2944  Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
2945  }
2946  }
2947  if (Data.Reductions) {
2948  OMPLexicalScope LexScope(CGF, S, CapturedRegion);
2950  Data.ReductionOps);
2951  llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
2952  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
2953  for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
2954  RedCG.emitSharedLValue(CGF, Cnt);
2955  RedCG.emitAggregateType(CGF, Cnt);
2956  // FIXME: This must removed once the runtime library is fixed.
2957  // Emit required threadprivate variables for
2958  // initializer/combiner/finalizer.
2959  CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
2960  RedCG, Cnt);
2961  Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
2962  CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
2963  Replacement =
2964  Address(CGF.EmitScalarConversion(
2965  Replacement.getPointer(), CGF.getContext().VoidPtrTy,
2966  CGF.getContext().getPointerType(
2967  Data.ReductionCopies[Cnt]->getType()),
2968  Data.ReductionCopies[Cnt]->getExprLoc()),
2969  Replacement.getAlignment());
2970  Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
2971  Scope.addPrivate(RedCG.getBaseDecl(Cnt),
2972  [Replacement]() { return Replacement; });
2973  }
2974  }
2975  // Privatize all private variables except for in_reduction items.
2976  (void)Scope.Privatize();
2977  SmallVector<const Expr *, 4> InRedVars;
2978  SmallVector<const Expr *, 4> InRedPrivs;
2980  SmallVector<const Expr *, 4> TaskgroupDescriptors;
2981  for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
2982  auto IPriv = C->privates().begin();
2983  auto IRed = C->reduction_ops().begin();
2984  auto ITD = C->taskgroup_descriptors().begin();
2985  for (const Expr *Ref : C->varlists()) {
2986  InRedVars.emplace_back(Ref);
2987  InRedPrivs.emplace_back(*IPriv);
2988  InRedOps.emplace_back(*IRed);
2989  TaskgroupDescriptors.emplace_back(*ITD);
2990  std::advance(IPriv, 1);
2991  std::advance(IRed, 1);
2992  std::advance(ITD, 1);
2993  }
2994  }
2995  // Privatize in_reduction items here, because taskgroup descriptors must be
2996  // privatized earlier.
2997  OMPPrivateScope InRedScope(CGF);
2998  if (!InRedVars.empty()) {
2999  ReductionCodeGen RedCG(InRedVars, InRedPrivs, InRedOps);
3000  for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
3001  RedCG.emitSharedLValue(CGF, Cnt);
3002  RedCG.emitAggregateType(CGF, Cnt);
3003  // The taskgroup descriptor variable is always implicit firstprivate and
3004  // privatized already during processing of the firstprivates.
3005  // FIXME: This must removed once the runtime library is fixed.
3006  // Emit required threadprivate variables for
3007  // initializer/combiner/finalizer.
3008  CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
3009  RedCG, Cnt);
3010  llvm::Value *ReductionsPtr =
3011  CGF.EmitLoadOfScalar(CGF.EmitLValue(TaskgroupDescriptors[Cnt]),
3012  TaskgroupDescriptors[Cnt]->getExprLoc());
3013  Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
3014  CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
3015  Replacement = Address(
3016  CGF.EmitScalarConversion(
3017  Replacement.getPointer(), CGF.getContext().VoidPtrTy,
3018  CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
3019  InRedPrivs[Cnt]->getExprLoc()),
3020  Replacement.getAlignment());
3021  Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
3022  InRedScope.addPrivate(RedCG.getBaseDecl(Cnt),
3023  [Replacement]() { return Replacement; });
3024  }
3025  }
3026  (void)InRedScope.Privatize();
3027 
3028  Action.Enter(CGF);
3029  BodyGen(CGF);
3030  };
3031  llvm::Value *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3032  S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied,
3033  Data.NumberOfParts);
3034  OMPLexicalScope Scope(*this, S);
3035  TaskGen(*this, OutlinedFn, Data);
3036 }
3037 
3038 static ImplicitParamDecl *
3040  QualType Ty, CapturedDecl *CD,
3041  SourceLocation Loc) {
3042  auto *OrigVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3044  auto *OrigRef = DeclRefExpr::Create(
3045  C, NestedNameSpecifierLoc(), SourceLocation(), OrigVD,
3046  /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
3047  auto *PrivateVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3049  auto *PrivateRef = DeclRefExpr::Create(
3050  C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD,
3051  /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
3052  QualType ElemType = C.getBaseElementType(Ty);
3053  auto *InitVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, ElemType,
3055  auto *InitRef = DeclRefExpr::Create(
3056  C, NestedNameSpecifierLoc(), SourceLocation(), InitVD,
3057  /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue);
3058  PrivateVD->setInitStyle(VarDecl::CInit);
3059  PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue,
3060  InitRef, /*BasePath=*/nullptr,
3061  VK_RValue));
3062  Data.FirstprivateVars.emplace_back(OrigRef);
3063  Data.FirstprivateCopies.emplace_back(PrivateRef);
3064  Data.FirstprivateInits.emplace_back(InitRef);
3065  return OrigVD;
3066 }
3067 
3069  const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,
3070  OMPTargetDataInfo &InputInfo) {
3071  // Emit outlined function for task construct.
3072  const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3073  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3074  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3075  auto I = CS->getCapturedDecl()->param_begin();
3076  auto PartId = std::next(I);
3077  auto TaskT = std::next(I, 4);
3078  OMPTaskDataTy Data;
3079  // The task is not final.
3080  Data.Final.setInt(/*IntVal=*/false);
3081  // Get list of firstprivate variables.
3082  for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
3083  auto IRef = C->varlist_begin();
3084  auto IElemInitRef = C->inits().begin();
3085  for (auto *IInit : C->private_copies()) {
3086  Data.FirstprivateVars.push_back(*IRef);
3087  Data.FirstprivateCopies.push_back(IInit);
3088  Data.FirstprivateInits.push_back(*IElemInitRef);
3089  ++IRef;
3090  ++IElemInitRef;
3091  }
3092  }
3093  OMPPrivateScope TargetScope(*this);
3094  VarDecl *BPVD = nullptr;
3095  VarDecl *PVD = nullptr;
3096  VarDecl *SVD = nullptr;
3097  if (InputInfo.NumberOfTargetItems > 0) {
3098  auto *CD = CapturedDecl::Create(
3099  getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0);
3100  llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);
3101  QualType BaseAndPointersType = getContext().getConstantArrayType(
3102  getContext().VoidPtrTy, ArrSize, ArrayType::Normal,
3103  /*IndexTypeQuals=*/0);
3105  getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
3107  getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
3108  QualType SizesType = getContext().getConstantArrayType(
3109  getContext().getSizeType(), ArrSize, ArrayType::Normal,
3110  /*IndexTypeQuals=*/0);
3111  SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD,
3112  S.getBeginLoc());
3113  TargetScope.addPrivate(
3114  BPVD, [&InputInfo]() { return InputInfo.BasePointersArray; });
3115  TargetScope.addPrivate(PVD,
3116  [&InputInfo]() { return InputInfo.PointersArray; });
3117  TargetScope.addPrivate(SVD,
3118  [&InputInfo]() { return InputInfo.SizesArray; });
3119  }
3120  (void)TargetScope.Privatize();
3121  // Build list of dependences.
3122  for (const auto *C : S.getClausesOfKind<OMPDependClause>())
3123  for (const Expr *IRef : C->varlists())
3124  Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
3125  auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD,
3126  &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
3127  // Set proper addresses for generated private copies.
3128  OMPPrivateScope Scope(CGF);
3129  if (!Data.FirstprivateVars.empty()) {
3130  enum { PrivatesParam = 2, CopyFnParam = 3 };
3131  llvm::Value *CopyFn = CGF.Builder.CreateLoad(
3132  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
3133  llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
3134  CS->getCapturedDecl()->getParam(PrivatesParam)));
3135  // Map privates.
3138  CallArgs.push_back(PrivatesPtr);
3139  for (const Expr *E : Data.FirstprivateVars) {
3140  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3141  Address PrivatePtr =
3142  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
3143  ".firstpriv.ptr.addr");
3144  PrivatePtrs.emplace_back(VD, PrivatePtr);
3145  CallArgs.push_back(PrivatePtr.getPointer());
3146  }
3147  CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
3148  CopyFn, CallArgs);
3149  for (const auto &Pair : PrivatePtrs) {
3150  Address Replacement(CGF.Builder.CreateLoad(Pair.second),
3151  CGF.getContext().getDeclAlign(Pair.first));
3152  Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
3153  }
3154  }
3155  // Privatize all private variables except for in_reduction items.
3156  (void)Scope.Privatize();
3157  if (InputInfo.NumberOfTargetItems > 0) {
3158  InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
3159  CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0, CGF.getPointerSize());
3160  InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
3161  CGF.GetAddrOfLocalVar(PVD), /*Index=*/0, CGF.getPointerSize());
3162  InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
3163  CGF.GetAddrOfLocalVar(SVD), /*Index=*/0, CGF.getSizeSize());
3164  }
3165 
3166  Action.Enter(CGF);
3167  OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);
3168  BodyGen(CGF);
3169  };
3170  llvm::Value *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3171  S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true,
3172  Data.NumberOfParts);
3173  llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
3174  IntegerLiteral IfCond(getContext(), TrueOrFalse,
3175  getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3176  SourceLocation());
3177 
3178  CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn,
3179  SharedsTy, CapturedStruct, &IfCond, Data);
3180 }
3181 
3183  // Emit outlined function for task construct.
3184  const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3185  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3186  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3187  const Expr *IfCond = nullptr;
3188  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3189  if (C->getNameModifier() == OMPD_unknown ||
3190  C->getNameModifier() == OMPD_task) {
3191  IfCond = C->getCondition();
3192  break;
3193  }
3194  }
3195 
3196  OMPTaskDataTy Data;
3197  // Check if we should emit tied or untied task.
3198  Data.Tied = !S.getSingleClause<OMPUntiedClause>();
3199  auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
3200  CGF.EmitStmt(CS->getCapturedStmt());
3201  };
3202  auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
3203  IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
3204  const OMPTaskDataTy &Data) {
3205  CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn,
3206  SharedsTy, CapturedStruct, IfCond,
3207  Data);
3208  };
3209  EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data);
3210 }
3211 
3213  const OMPTaskyieldDirective &S) {
3214  CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getBeginLoc());
3215 }
3216 
3218  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_barrier);
3219 }
3220 
3222  CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc());
3223 }
3224 
3226  const OMPTaskgroupDirective &S) {
3227  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3228  Action.Enter(CGF);
3229  if (const Expr *E = S.getReductionRef()) {
3232  OMPTaskDataTy Data;
3233  for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
3234  auto IPriv = C->privates().begin();
3235  auto IRed = C->reduction_ops().begin();
3236  auto ILHS = C->lhs_exprs().begin();
3237  auto IRHS = C->rhs_exprs().begin();
3238  for (const Expr *Ref : C->varlists()) {
3239  Data.ReductionVars.emplace_back(Ref);
3240  Data.ReductionCopies.emplace_back(*IPriv);
3241  Data.ReductionOps.emplace_back(*IRed);
3242  LHSs.emplace_back(*ILHS);
3243  RHSs.emplace_back(*IRHS);
3244  std::advance(IPriv, 1);
3245  std::advance(IRed, 1);
3246  std::advance(ILHS, 1);
3247  std::advance(IRHS, 1);
3248  }
3249  }
3250  llvm::Value *ReductionDesc =
3251  CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(),
3252  LHSs, RHSs, Data);
3253  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3254  CGF.EmitVarDecl(*VD);
3255  CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),
3256  /*Volatile=*/false, E->getType());
3257  }
3259  };
3260  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3261  CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getBeginLoc());
3262 }
3263 
3265  CGM.getOpenMPRuntime().emitFlush(
3266  *this,
3267  [&S]() -> ArrayRef<const Expr *> {
3268  if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>())
3269  return llvm::makeArrayRef(FlushClause->varlist_begin(),
3270  FlushClause->varlist_end());
3271  return llvm::None;
3272  }(),
3273  S.getBeginLoc());
3274 }
3275 
3277  const CodeGenLoopTy &CodeGenLoop,
3278  Expr *IncExpr) {
3279  // Emit the loop iteration variable.
3280  const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3281  const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3282  EmitVarDecl(*IVDecl);
3283 
3284  // Emit the iterations count variable.
3285  // If it is not a variable, Sema decided to calculate iterations count on each
3286  // iteration (e.g., it is foldable into a constant).
3287  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3288  EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3289  // Emit calculation of the iterations count.
3290  EmitIgnoredExpr(S.getCalcLastIteration());
3291  }
3292 
3293  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3294 
3295  bool HasLastprivateClause = false;
3296  // Check pre-condition.
3297  {
3298  OMPLoopScope PreInitScope(*this, S);
3299  // Skip the entire loop if we don't meet the precondition.
3300  // If the condition constant folds and can be elided, avoid emitting the
3301  // whole loop.
3302  bool CondConstant;
3303  llvm::BasicBlock *ContBlock = nullptr;
3304  if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
3305  if (!CondConstant)
3306  return;
3307  } else {
3308  llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
3309  ContBlock = createBasicBlock("omp.precond.end");
3310  emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
3311  getProfileCount(&S));
3312  EmitBlock(ThenBlock);
3313  incrementProfileCounter(&S);
3314  }
3315 
3316  emitAlignedClause(*this, S);
3317  // Emit 'then' code.
3318  {
3319  // Emit helper vars inits.
3320 
3321  LValue LB = EmitOMPHelperVar(
3322  *this, cast<DeclRefExpr>(
3325  : S.getLowerBoundVariable())));
3326  LValue UB = EmitOMPHelperVar(
3327  *this, cast<DeclRefExpr>(
3330  : S.getUpperBoundVariable())));
3331  LValue ST =
3332  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
3333  LValue IL =
3334  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
3335 
3336  OMPPrivateScope LoopScope(*this);
3337  if (EmitOMPFirstprivateClause(S, LoopScope)) {
3338  // Emit implicit barrier to synchronize threads and avoid data races
3339  // on initialization of firstprivate variables and post-update of
3340  // lastprivate variables.
3341  CGM.getOpenMPRuntime().emitBarrierCall(
3342  *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
3343  /*ForceSimpleCall=*/true);
3344  }
3345  EmitOMPPrivateClause(S, LoopScope);
3349  EmitOMPReductionClauseInit(S, LoopScope);
3350  HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
3351  EmitOMPPrivateLoopCounters(S, LoopScope);
3352  (void)LoopScope.Privatize();
3354  CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
3355 
3356  // Detect the distribute schedule kind and chunk.
3357  llvm::Value *Chunk = nullptr;
3359  if (const auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
3360  ScheduleKind = C->getDistScheduleKind();
3361  if (const Expr *Ch = C->getChunkSize()) {
3362  Chunk = EmitScalarExpr(Ch);
3363  Chunk = EmitScalarConversion(Chunk, Ch->getType(),
3365  S.getBeginLoc());
3366  }
3367  } else {
3368  // Default behaviour for dist_schedule clause.
3369  CGM.getOpenMPRuntime().getDefaultDistScheduleAndChunk(
3370  *this, S, ScheduleKind, Chunk);
3371  }
3372  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3373  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3374 
3375  // OpenMP [2.10.8, distribute Construct, Description]
3376  // If dist_schedule is specified, kind must be static. If specified,
3377  // iterations are divided into chunks of size chunk_size, chunks are
3378  // assigned to the teams of the league in a round-robin fashion in the
3379  // order of the team number. When no chunk_size is specified, the
3380  // iteration space is divided into chunks that are approximately equal
3381  // in size, and at most one chunk is distributed to each team of the
3382  // league. The size of the chunks is unspecified in this case.
3383  bool StaticChunked = RT.isStaticChunked(
3384  ScheduleKind, /* Chunked */ Chunk != nullptr) &&
3386  if (RT.isStaticNonchunked(ScheduleKind,
3387  /* Chunked */ Chunk != nullptr) ||
3388  StaticChunked) {
3390  EmitOMPSimdInit(S, /*IsMonotonic=*/true);
3391  CGOpenMPRuntime::StaticRTInput StaticInit(
3392  IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(),
3393  LB.getAddress(), UB.getAddress(), ST.getAddress(),
3394  StaticChunked ? Chunk : nullptr);
3395  RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind,
3396  StaticInit);
3397  JumpDest LoopExit =
3398  getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
3399  // UB = min(UB, GlobalUB);
3402  : S.getEnsureUpperBound());
3403  // IV = LB;
3405  ? S.getCombinedInit()
3406  : S.getInit());
3407 
3408  const Expr *Cond =
3410  ? S.getCombinedCond()
3411  : S.getCond();
3412 
3413  if (StaticChunked)
3414  Cond = S.getCombinedDistCond();
3415 
3416  // For static unchunked schedules generate:
3417  //
3418  // 1. For distribute alone, codegen
3419  // while (idx <= UB) {
3420  // BODY;
3421  // ++idx;
3422  // }
3423  //
3424  // 2. When combined with 'for' (e.g. as in 'distribute parallel for')
3425  // while (idx <= UB) {
3426  // <CodeGen rest of pragma>(LB, UB);
3427  // idx += ST;
3428  // }
3429  //
3430  // For static chunk one schedule generate:
3431  //
3432  // while (IV <= GlobalUB) {
3433  // <CodeGen rest of pragma>(LB, UB);
3434  // LB += ST;
3435  // UB += ST;
3436  // UB = min(UB, GlobalUB);
3437  // IV = LB;
3438  // }
3439  //
3440  EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), Cond, IncExpr,
3441  [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3442  CodeGenLoop(CGF, S, LoopExit);
3443  },
3444  [&S, StaticChunked](CodeGenFunction &CGF) {
3445  if (StaticChunked) {
3446  CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());
3447  CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());
3448  CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());
3449  CGF.EmitIgnoredExpr(S.getCombinedInit());
3450  }
3451  });
3452  EmitBlock(LoopExit.getBlock());
3453  // Tell the runtime we are done.
3454  RT.emitForStaticFinish(*this, S.getBeginLoc(), S.getDirectiveKind());
3455  } else {
3456  // Emit the outer loop, which requests its work chunk [LB..UB] from
3457  // runtime and runs the inner loop to process it.
3458  const OMPLoopArguments LoopArguments = {
3459  LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(),
3460  Chunk};
3461  EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
3462  CodeGenLoop);
3463  }
3465  EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
3466  return CGF.Builder.CreateIsNotNull(
3467  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3468  });
3469  }
3473  EmitOMPReductionClauseFinal(S, OMPD_simd);
3474  // Emit post-update of the reduction variables if IsLastIter != 0.
3476  *this, S, [IL, &S](CodeGenFunction &CGF) {
3477  return CGF.Builder.CreateIsNotNull(
3478  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3479  });
3480  }
3481  // Emit final copy of the lastprivate variables if IsLastIter != 0.
3482  if (HasLastprivateClause) {
3483  EmitOMPLastprivateClauseFinal(
3484  S, /*NoFinals=*/false,
3485  Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
3486  }
3487  }
3488 
3489  // We're now done with the loop, so jump to the continuation block.
3490  if (ContBlock) {
3491  EmitBranch(ContBlock);
3492  EmitBlock(ContBlock, true);
3493  }
3494  }
3495 }
3496 
3498  const OMPDistributeDirective &S) {
3499  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3501  };
3502  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3503  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3504 }
3505 
3506 static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM,
3507  const CapturedStmt *S) {
3508  CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
3510  CGF.CapturedStmtInfo = &CapStmtInfo;
3511  llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S);
3512  Fn->setDoesNotRecurse();
3513  return Fn;
3514 }
3515 
3517  if (S.hasClausesOfKind<OMPDependClause>()) {
3518  assert(!S.getAssociatedStmt() &&
3519  "No associated statement must be in ordered depend construct.");
3520  for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
3521  CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
3522  return;
3523  }
3524  const auto *C = S.getSingleClause<OMPSIMDClause>();
3525  auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
3526  PrePostActionTy &Action) {
3527  const CapturedStmt *CS = S.getInnermostCapturedStmt();
3528  if (C) {
3530  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
3531  llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS);
3532  CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
3533  OutlinedFn, CapturedVars);
3534  } else {
3535  Action.Enter(CGF);
3536  CGF.EmitStmt(CS->getCapturedStmt());
3537  }
3538  };
3539  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3540  CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getBeginLoc(), !C);
3541 }
3542 
3544  QualType SrcType, QualType DestType,
3545  SourceLocation Loc) {
3546  assert(CGF.hasScalarEvaluationKind(DestType) &&
3547  "DestType must have scalar evaluation kind.");
3548  assert(!Val.isAggregate() && "Must be a scalar or complex.");
3549  return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
3550  DestType, Loc)
3552  Val.getComplexVal(), SrcType, DestType, Loc);
3553 }
3554 
3557  QualType DestType, SourceLocation Loc) {
3558  assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
3559  "DestType must have complex evaluation kind.");
3560  CodeGenFunction::ComplexPairTy ComplexVal;
3561  if (Val.isScalar()) {
3562  // Convert the input element to the element type of the complex.
3563  QualType DestElementType =
3564  DestType->castAs<ComplexType>()->getElementType();
3565  llvm::Value *ScalarVal = CGF.EmitScalarConversion(
3566  Val.getScalarVal(), SrcType, DestElementType, Loc);
3567  ComplexVal = CodeGenFunction::ComplexPairTy(
3568  ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
3569  } else {
3570  assert(Val.isComplex() && "Must be a scalar or complex.");
3571  QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
3572  QualType DestElementType =
3573  DestType->castAs<ComplexType>()->getElementType();
3574  ComplexVal.first = CGF.EmitScalarConversion(
3575  Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
3576  ComplexVal.second = CGF.EmitScalarConversion(
3577  Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
3578  }
3579  return ComplexVal;
3580 }
3581 
3582 static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst,
3583  LValue LVal, RValue RVal) {
3584  if (LVal.isGlobalReg()) {
3585  CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
3586  } else {
3587  CGF.EmitAtomicStore(RVal, LVal,
3588  IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3589  : llvm::AtomicOrdering::Monotonic,
3590  LVal.isVolatile(), /*IsInit=*/false);
3591  }
3592 }
3593 
3595  QualType RValTy, SourceLocation Loc) {
3596  switch (getEvaluationKind(LVal.getType())) {
3597  case TEK_Scalar:
3598  EmitStoreThroughLValue(RValue::get(convertToScalarValue(
3599  *this, RVal, RValTy, LVal.getType(), Loc)),
3600  LVal);
3601  break;
3602  case TEK_Complex:
3603  EmitStoreOfComplex(
3604  convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,
3605  /*isInit=*/false);
3606  break;
3607  case TEK_Aggregate:
3608  llvm_unreachable("Must be a scalar or complex.");
3609  }
3610 }
3611 
3612 static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst,
3613  const Expr *X, const Expr *V,
3614  SourceLocation Loc) {
3615  // v = x;
3616  assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
3617  assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
3618  LValue XLValue = CGF.EmitLValue(X);
3619  LValue VLValue = CGF.EmitLValue(V);
3620  RValue Res = XLValue.isGlobalReg()
3621  ? CGF.EmitLoadOfLValue(XLValue, Loc)
3622  : CGF.EmitAtomicLoad(
3623  XLValue, Loc,
3624  IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3625  : llvm::AtomicOrdering::Monotonic,
3626  XLValue.isVolatile());
3627  // OpenMP, 2.12.6, atomic Construct
3628  // Any atomic construct with a seq_cst clause forces the atomically
3629  // performed operation to include an implicit flush operation without a
3630  // list.
3631  if (IsSeqCst)
3632  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3633  CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
3634 }
3635 
3636 static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst,
3637  const Expr *X, const Expr *E,
3638  SourceLocation Loc) {
3639  // x = expr;
3640  assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
3641  emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
3642  // OpenMP, 2.12.6, atomic Construct
3643  // Any atomic construct with a seq_cst clause forces the atomically
3644  // performed operation to include an implicit flush operation without a
3645  // list.
3646  if (IsSeqCst)
3647  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3648 }
3649 
3650 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
3651  RValue Update,
3652  BinaryOperatorKind BO,
3653  llvm::AtomicOrdering AO,
3654  bool IsXLHSInRHSPart) {
3655  ASTContext &Context = CGF.getContext();
3656  // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
3657  // expression is simple and atomic is allowed for the given type for the
3658  // target platform.
3659  if (BO == BO_Comma || !Update.isScalar() ||
3660  !Update.getScalarVal()->getType()->isIntegerTy() ||
3661  !X.isSimple() || (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
3662  (Update.getScalarVal()->getType() !=
3663  X.getAddress().getElementType())) ||
3664  !X.getAddress().getElementType()->isIntegerTy() ||
3665  !Context.getTargetInfo().hasBuiltinAtomic(
3666  Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
3667  return std::make_pair(false, RValue::get(nullptr));
3668 
3669  llvm::AtomicRMWInst::BinOp RMWOp;
3670  switch (BO) {
3671  case BO_Add:
3672  RMWOp = llvm::AtomicRMWInst::Add;
3673  break;
3674  case BO_Sub:
3675  if (!IsXLHSInRHSPart)
3676  return std::make_pair(false, RValue::get(nullptr));
3677  RMWOp = llvm::AtomicRMWInst::Sub;
3678  break;
3679  case BO_And:
3680  RMWOp = llvm::AtomicRMWInst::And;
3681  break;
3682  case BO_Or:
3683  RMWOp = llvm::AtomicRMWInst::Or;
3684  break;
3685  case BO_Xor:
3686  RMWOp = llvm::AtomicRMWInst::Xor;
3687  break;
3688  case BO_LT:
3690  ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
3691  : llvm::AtomicRMWInst::Max)
3692  : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
3693  : llvm::AtomicRMWInst::UMax);
3694  break;
3695  case BO_GT:
3697  ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
3698  : llvm::AtomicRMWInst::Min)
3699  : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
3700  : llvm::AtomicRMWInst::UMin);
3701  break;
3702  case BO_Assign:
3703  RMWOp = llvm::AtomicRMWInst::Xchg;
3704  break;
3705  case BO_Mul:
3706  case BO_Div:
3707  case BO_Rem:
3708  case BO_Shl:
3709  case BO_Shr:
3710  case BO_LAnd:
3711  case BO_LOr:
3712  return std::make_pair(false, RValue::get(nullptr));
3713  case BO_PtrMemD:
3714  case BO_PtrMemI:
3715  case BO_LE:
3716  case BO_GE:
3717  case BO_EQ:
3718  case BO_NE:
3719  case BO_Cmp:
3720  case BO_AddAssign:
3721  case BO_SubAssign:
3722  case BO_AndAssign:
3723  case BO_OrAssign:
3724  case BO_XorAssign:
3725  case BO_MulAssign:
3726  case BO_DivAssign:
3727  case BO_RemAssign:
3728  case BO_ShlAssign:
3729  case BO_ShrAssign:
3730  case BO_Comma:
3731  llvm_unreachable("Unsupported atomic update operation");
3732  }
3733  llvm::Value *UpdateVal = Update.getScalarVal();
3734  if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
3735  UpdateVal = CGF.Builder.CreateIntCast(
3736  IC, X.getAddress().getElementType(),
3738  }
3739  llvm::Value *Res =
3740  CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO);
3741  return std::make_pair(true, RValue::get(Res));
3742 }
3743 
3745  LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3746  llvm::AtomicOrdering AO, SourceLocation Loc,
3747  const llvm::function_ref<RValue(RValue)> CommonGen) {
3748  // Update expressions are allowed to have the following forms:
3749  // x binop= expr; -> xrval + expr;
3750  // x++, ++x -> xrval + 1;
3751  // x--, --x -> xrval - 1;
3752  // x = x binop expr; -> xrval binop expr
3753  // x = expr Op x; - > expr binop xrval;
3754  auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);
3755  if (!Res.first) {
3756  if (X.isGlobalReg()) {
3757  // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
3758  // 'xrval'.
3759  EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
3760  } else {
3761  // Perform compare-and-swap procedure.
3762  EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
3763  }
3764  }
3765  return Res;
3766 }
3767 
3768 static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst,
3769  const Expr *X, const Expr *E,
3770  const Expr *UE, bool IsXLHSInRHSPart,
3771  SourceLocation Loc) {
3772  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3773  "Update expr in 'atomic update' must be a binary operator.");
3774  const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3775  // Update expressions are allowed to have the following forms:
3776  // x binop= expr; -> xrval + expr;
3777  // x++, ++x -> xrval + 1;
3778  // x--, --x -> xrval - 1;
3779  // x = x binop expr; -> xrval binop expr
3780  // x = expr Op x; - > expr binop xrval;
3781  assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
3782  LValue XLValue = CGF.EmitLValue(X);
3783  RValue ExprRValue = CGF.EmitAnyExpr(E);
3784  llvm::AtomicOrdering AO = IsSeqCst
3785  ? llvm::AtomicOrdering::SequentiallyConsistent
3786  : llvm::AtomicOrdering::Monotonic;
3787  const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3788  const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3789  const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3790  const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3791  auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {
3792  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3793  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3794  return CGF.EmitAnyExpr(UE);
3795  };
3797  XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3798  // OpenMP, 2.12.6, atomic Construct
3799  // Any atomic construct with a seq_cst clause forces the atomically
3800  // performed operation to include an implicit flush operation without a
3801  // list.
3802  if (IsSeqCst)
3803  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3804 }
3805 
3807  QualType SourceType, QualType ResType,
3808  SourceLocation Loc) {
3809  switch (CGF.getEvaluationKind(ResType)) {
3810  case TEK_Scalar:
3811  return RValue::get(
3812  convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
3813  case TEK_Complex: {
3814  auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
3815  return RValue::getComplex(Res.first, Res.second);
3816  }
3817  case TEK_Aggregate:
3818  break;
3819  }
3820  llvm_unreachable("Must be a scalar or complex.");
3821 }
3822 
3823 static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst,
3824  bool IsPostfixUpdate, const Expr *V,
3825  const Expr *X, const Expr *E,
3826  const Expr *UE, bool IsXLHSInRHSPart,
3827  SourceLocation Loc) {
3828  assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
3829  assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
3830  RValue NewVVal;
3831  LValue VLValue = CGF.EmitLValue(V);
3832  LValue XLValue = CGF.EmitLValue(X);
3833  RValue ExprRValue = CGF.EmitAnyExpr(E);
3834  llvm::AtomicOrdering AO = IsSeqCst
3835  ? llvm::AtomicOrdering::SequentiallyConsistent
3836  : llvm::AtomicOrdering::Monotonic;
3837  QualType NewVValType;
3838  if (UE) {
3839  // 'x' is updated with some additional value.
3840  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3841  "Update expr in 'atomic capture' must be a binary operator.");
3842  const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3843  // Update expressions are allowed to have the following forms:
3844  // x binop= expr; -> xrval + expr;
3845  // x++, ++x -> xrval + 1;
3846  // x--, --x -> xrval - 1;
3847  // x = x binop expr; -> xrval binop expr
3848  // x = expr Op x; - > expr binop xrval;
3849  const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3850  const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3851  const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3852  NewVValType = XRValExpr->getType();
3853  const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3854  auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
3855  IsPostfixUpdate](RValue XRValue) {
3856  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3857  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3858  RValue Res = CGF.EmitAnyExpr(UE);
3859  NewVVal = IsPostfixUpdate ? XRValue : Res;
3860  return Res;
3861  };
3862  auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3863  XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3864  if (Res.first) {
3865  // 'atomicrmw' instruction was generated.
3866  if (IsPostfixUpdate) {
3867  // Use old value from 'atomicrmw'.
3868  NewVVal = Res.second;
3869  } else {
3870  // 'atomicrmw' does not provide new value, so evaluate it using old
3871  // value of 'x'.
3872  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3873  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
3874  NewVVal = CGF.EmitAnyExpr(UE);
3875  }
3876  }
3877  } else {
3878  // 'x' is simply rewritten with some 'expr'.
3879  NewVValType = X->getType().getNonReferenceType();
3880  ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
3881  X->getType().getNonReferenceType(), Loc);
3882  auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) {
3883  NewVVal = XRValue;
3884  return ExprRValue;
3885  };
3886  // Try to perform atomicrmw xchg, otherwise simple exchange.
3887  auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3888  XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
3889  Loc, Gen);
3890  if (Res.first) {
3891  // 'atomicrmw' instruction was generated.
3892  NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
3893  }
3894  }
3895  // Emit post-update store to 'v' of old/new 'x' value.
3896  CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
3897  // OpenMP, 2.12.6, atomic Construct
3898  // Any atomic construct with a seq_cst clause forces the atomically
3899  // performed operation to include an implicit flush operation without a
3900  // list.
3901  if (IsSeqCst)
3902  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3903 }
3904 
3906  bool IsSeqCst, bool IsPostfixUpdate,
3907  const Expr *X, const Expr *V, const Expr *E,
3908  const Expr *UE, bool IsXLHSInRHSPart,
3909  SourceLocation Loc) {
3910  switch (Kind) {
3911  case OMPC_read:
3912  emitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc);
3913  break;
3914  case OMPC_write:
3915  emitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc);
3916  break;
3917  case OMPC_unknown:
3918  case OMPC_update:
3919  emitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc);
3920  break;
3921  case OMPC_capture:
3922  emitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE,
3923  IsXLHSInRHSPart, Loc);
3924  break;
3925  case OMPC_if:
3926  case OMPC_final:
3927  case OMPC_num_threads:
3928  case OMPC_private:
3929  case OMPC_firstprivate:
3930  case OMPC_lastprivate:
3931  case OMPC_reduction:
3932  case OMPC_task_reduction:
3933  case OMPC_in_reduction:
3934  case OMPC_safelen:
3935  case OMPC_simdlen:
3936  case OMPC_collapse:
3937  case OMPC_default:
3938  case OMPC_seq_cst:
3939  case OMPC_shared:
3940  case OMPC_linear:
3941  case OMPC_aligned:
3942  case OMPC_copyin:
3943  case OMPC_copyprivate:
3944  case OMPC_flush:
3945  case OMPC_proc_bind:
3946  case OMPC_schedule:
3947  case OMPC_ordered:
3948  case OMPC_nowait:
3949  case OMPC_untied:
3950  case OMPC_threadprivate:
3951  case OMPC_depend:
3952  case OMPC_mergeable:
3953  case OMPC_device:
3954  case OMPC_threads:
3955  case OMPC_simd:
3956  case OMPC_map:
3957  case OMPC_num_teams:
3958  case OMPC_thread_limit:
3959  case OMPC_priority:
3960  case OMPC_grainsize:
3961  case OMPC_nogroup:
3962  case OMPC_num_tasks:
3963  case OMPC_hint:
3964  case OMPC_dist_schedule:
3965  case OMPC_defaultmap:
3966  case OMPC_uniform:
3967  case OMPC_to:
3968  case OMPC_from:
3969  case OMPC_use_device_ptr:
3970  case OMPC_is_device_ptr:
3971  case OMPC_unified_address:
3972  case OMPC_unified_shared_memory:
3973  case OMPC_reverse_offload:
3974  case OMPC_dynamic_allocators:
3975  case OMPC_atomic_default_mem_order:
3976  llvm_unreachable("Clause is not allowed in 'omp atomic'.");
3977  }
3978 }
3979 
3981  bool IsSeqCst = S.getSingleClause<OMPSeqCstClause>();
3983  for (const OMPClause *C : S.clauses()) {
3984  // Find first clause (skip seq_cst clause, if it is first).
3985  if (C->getClauseKind() != OMPC_seq_cst) {
3986  Kind = C->getClauseKind();
3987  break;
3988  }
3989  }
3990 
3991  const Stmt *CS = S.getInnermostCapturedStmt()->IgnoreContainers();
3992  if (const auto *FE = dyn_cast<FullExpr>(CS))
3993  enterFullExpression(FE);
3994  // Processing for statements under 'atomic capture'.
3995  if (const auto *Compound = dyn_cast<CompoundStmt>(CS)) {
3996  for (const Stmt *C : Compound->body()) {
3997  if (const auto *FE = dyn_cast<FullExpr>(C))
3998  enterFullExpression(FE);
3999  }
4000  }
4001 
4002  auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF,
4003  PrePostActionTy &) {
4004  CGF.EmitStopPoint(CS);
4005  emitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(),
4006  S.getV(), S.getExpr(), S.getUpdateExpr(),
4007  S.isXLHSInRHSPart(), S.getBeginLoc());
4008  };
4009  OMPLexicalScope Scope(*this, S, OMPD_unknown);
4010  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen);
4011 }
4012 
4014  const OMPExecutableDirective &S,
4015  const RegionCodeGenTy &CodeGen) {
4017  CodeGenModule &CGM = CGF.CGM;
4018 
4019  // On device emit this construct as inlined code.
4020  if (CGM.getLangOpts().OpenMPIsDevice) {
4021  OMPLexicalScope Scope(CGF, S, OMPD_target);
4022  CGM.getOpenMPRuntime().emitInlinedDirective(
4023  CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4025  });
4026  return;
4027  }
4028 
4029  llvm::Function *Fn = nullptr;
4030  llvm::Constant *FnID = nullptr;
4031 
4032  const Expr *IfCond = nullptr;
4033  // Check for the at most one if clause associated with the target region.
4034  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4035  if (C->getNameModifier() == OMPD_unknown ||
4036  C->getNameModifier() == OMPD_target) {
4037  IfCond = C->getCondition();
4038  break;
4039  }
4040  }
4041 
4042  // Check if we have any device clause associated with the directive.
4043  const Expr *Device = nullptr;
4044  if (auto *C = S.getSingleClause<OMPDeviceClause>())
4045  Device = C->getDevice();
4046 
4047  // Check if we have an if clause whose conditional always evaluates to false
4048  // or if we do not have any targets specified. If so the target region is not
4049  // an offload entry point.
4050  bool IsOffloadEntry = true;
4051  if (IfCond) {
4052  bool Val;
4053  if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
4054  IsOffloadEntry = false;
4055  }
4056  if (CGM.getLangOpts().OMPTargetTriples.empty())
4057  IsOffloadEntry = false;
4058 
4059  assert(CGF.CurFuncDecl && "No parent declaration for target region!");
4060  StringRef ParentName;
4061  // In case we have Ctors/Dtors we use the complete type variant to produce
4062  // the mangling of the device outlined kernel.
4063  if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
4064  ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));
4065  else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
4066  ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));
4067  else
4068  ParentName =
4069  CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));
4070 
4071  // Emit target region as a standalone region.
4072  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
4073  IsOffloadEntry, CodeGen);
4074  OMPLexicalScope Scope(CGF, S, OMPD_task);
4075  CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device);
4076 }
4077 
4079  PrePostActionTy &Action) {
4080  Action.Enter(CGF);
4081  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4082  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4083  CGF.EmitOMPPrivateClause(S, PrivateScope);
4084  (void)PrivateScope.Privatize();
4086  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4087 
4088  CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
4089 }
4090 
4092  StringRef ParentName,
4093  const OMPTargetDirective &S) {
4094  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4095  emitTargetRegion(CGF, S, Action);
4096  };
4097  llvm::Function *Fn;
4098  llvm::Constant *Addr;
4099  // Emit target region as a standalone region.
4100  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4101  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4102  assert(Fn && Addr && "Target device function emission failed.");
4103 }
4104 
4106  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4107  emitTargetRegion(CGF, S, Action);
4108  };
4109  emitCommonOMPTargetDirective(*this, S, CodeGen);
4110 }
4111 
4113  const OMPExecutableDirective &S,
4114  OpenMPDirectiveKind InnermostKind,
4115  const RegionCodeGenTy &CodeGen) {
4116  const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
4117  llvm::Value *OutlinedFn =
4118  CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
4119  S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
4120 
4121  const auto *NT = S.getSingleClause<OMPNumTeamsClause>();
4122  const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
4123  if (NT || TL) {
4124  const Expr *NumTeams = NT ? NT->getNumTeams() : nullptr;
4125  const Expr *ThreadLimit = TL ? TL->getThreadLimit() : nullptr;
4126 
4127  CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
4128  S.getBeginLoc());
4129  }
4130 
4131  OMPTeamsScope Scope(CGF, S);
4133  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
4134  CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn,
4135  CapturedVars);
4136 }
4137 
4139  // Emit teams region as a standalone region.
4140  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4141  Action.Enter(CGF);
4142  OMPPrivateScope PrivateScope(CGF);
4143  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4144  CGF.EmitOMPPrivateClause(S, PrivateScope);
4145  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4146  (void)PrivateScope.Privatize();
4147  CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
4148  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4149  };
4150  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4152  [](CodeGenFunction &) { return nullptr; });
4153 }
4154 
4156  const OMPTargetTeamsDirective &S) {
4157  auto *CS = S.getCapturedStmt(OMPD_teams);
4158  Action.Enter(CGF);
4159  // Emit teams region as a standalone region.
4160  auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4161  Action.Enter(CGF);
4162  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4163  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4164  CGF.EmitOMPPrivateClause(S, PrivateScope);
4165  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4166  (void)PrivateScope.Privatize();
4168  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4169  CGF.EmitStmt(CS->getCapturedStmt());
4170  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4171  };
4172  emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
4174  [](CodeGenFunction &) { return nullptr; });
4175 }
4176 
4178  CodeGenModule &CGM, StringRef ParentName,
4179  const OMPTargetTeamsDirective &S) {
4180  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4181  emitTargetTeamsRegion(CGF, Action, S);
4182  };
4183  llvm::Function *Fn;
4184  llvm::Constant *Addr;
4185  // Emit target region as a standalone region.
4186  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4187  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4188  assert(Fn && Addr && "Target device function emission failed.");
4189 }
4190 
4192  const OMPTargetTeamsDirective &S) {
4193  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4194  emitTargetTeamsRegion(CGF, Action, S);
4195  };
4196  emitCommonOMPTargetDirective(*this, S, CodeGen);
4197 }
4198 
4199 static void
4202  Action.Enter(CGF);
4203  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4205  };
4206 
4207  // Emit teams region as a standalone region.
4208  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4209  PrePostActionTy &Action) {
4210  Action.Enter(CGF);
4211  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4212  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4213  (void)PrivateScope.Privatize();
4214  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4215  CodeGenDistribute);
4216  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4217  };
4218  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
4220  [](CodeGenFunction &) { return nullptr; });
4221 }
4222 
4224  CodeGenModule &CGM, StringRef ParentName,
4226  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4227  emitTargetTeamsDistributeRegion(CGF, Action, S);
4228  };
4229  llvm::Function *Fn;
4230  llvm::Constant *Addr;
4231  // Emit target region as a standalone region.
4232  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4233  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4234  assert(Fn && Addr && "Target device function emission failed.");
4235 }
4236 
4239  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4240  emitTargetTeamsDistributeRegion(CGF, Action, S);
4241  };
4242  emitCommonOMPTargetDirective(*this, S, CodeGen);
4243 }
4244 
4246  CodeGenFunction &CGF, PrePostActionTy &Action,
4248  Action.Enter(CGF);
4249  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4251  };
4252 
4253  // Emit teams region as a standalone region.
4254  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4255  PrePostActionTy &Action) {
4256  Action.Enter(CGF);
4257  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4258  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4259  (void)PrivateScope.Privatize();
4260  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4261  CodeGenDistribute);
4262  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4263  };
4264  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);
4266  [](CodeGenFunction &) { return nullptr; });
4267 }
4268 
4270  CodeGenModule &CGM, StringRef ParentName,
4272  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4273  emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4274  };
4275  llvm::Function *Fn;
4276  llvm::Constant *Addr;
4277  // Emit target region as a standalone region.
4278  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4279  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4280  assert(Fn && Addr && "Target device function emission failed.");
4281 }
4282 
4285  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4286  emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4287  };
4288  emitCommonOMPTargetDirective(*this, S, CodeGen);
4289 }
4290 
4292  const OMPTeamsDistributeDirective &S) {
4293 
4294  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4296  };
4297 
4298  // Emit teams region as a standalone region.
4299  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4300  PrePostActionTy &Action) {
4301  Action.Enter(CGF);
4302  OMPPrivateScope PrivateScope(CGF);
4303  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4304  (void)PrivateScope.Privatize();
4305  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4306  CodeGenDistribute);
4307  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4308  };
4309  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4311  [](CodeGenFunction &) { return nullptr; });
4312 }
4313 
4316  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4318  };
4319 
4320  // Emit teams region as a standalone region.
4321  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4322  PrePostActionTy &Action) {
4323  Action.Enter(CGF);
4324  OMPPrivateScope PrivateScope(CGF);
4325  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4326  (void)PrivateScope.Privatize();
4327  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
4328  CodeGenDistribute);
4329  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4330  };
4331  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen);
4333  [](CodeGenFunction &) { return nullptr; });
4334 }
4335 
4338  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4340  S.getDistInc());
4341  };
4342 
4343  // Emit teams region as a standalone region.
4344  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4345  PrePostActionTy &Action) {
4346  Action.Enter(CGF);
4347  OMPPrivateScope PrivateScope(CGF);
4348  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4349  (void)PrivateScope.Privatize();
4350  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4351  CodeGenDistribute);
4352  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4353  };
4354  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4356  [](CodeGenFunction &) { return nullptr; });
4357 }
4358 
4361  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4363  S.getDistInc());
4364  };
4365 
4366  // Emit teams region as a standalone region.
4367  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4368  PrePostActionTy &Action) {
4369  Action.Enter(CGF);
4370  OMPPrivateScope PrivateScope(CGF);
4371  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4372  (void)PrivateScope.Privatize();
4373  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4374  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4375  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4376  };
4377  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4379  [](CodeGenFunction &) { return nullptr; });
4380 }
4381 
4384  PrePostActionTy &Action) {
4385  Action.Enter(CGF);
4386  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4388  S.getDistInc());
4389  };
4390 
4391  // Emit teams region as a standalone region.
4392  auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4393  PrePostActionTy &Action) {
4394  Action.Enter(CGF);
4395  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4396  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4397  (void)PrivateScope.Privatize();
4398  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4399  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4400  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4401  };
4402 
4403  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
4404  CodeGenTeams);
4406  [](CodeGenFunction &) { return nullptr; });
4407 }
4408 
4410  CodeGenModule &CGM, StringRef ParentName,
4412  // Emit SPMD target teams distribute parallel for region as a standalone
4413  // region.
4414  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4416  };
4417  llvm::Function *Fn;
4418  llvm::Constant *Addr;
4419  // Emit target region as a standalone region.
4420  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4421  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4422  assert(Fn && Addr && "Target device function emission failed.");
4423 }
4424 
4427  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4429  };
4430  emitCommonOMPTargetDirective(*this, S, CodeGen);
4431 }
4432 
4434  CodeGenFunction &CGF,
4436  PrePostActionTy &Action) {
4437  Action.Enter(CGF);
4438  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4440  S.getDistInc());
4441  };
4442 
4443  // Emit teams region as a standalone region.
4444  auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4445  PrePostActionTy &Action) {
4446  Action.Enter(CGF);
4447  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4448  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4449  (void)PrivateScope.Privatize();
4450  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4451  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4452  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4453  };
4454 
4455  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd,
4456  CodeGenTeams);
4458  [](CodeGenFunction &) { return nullptr; });
4459 }
4460 
4462  CodeGenModule &CGM, StringRef ParentName,
4464  // Emit SPMD target teams distribute parallel for simd region as a standalone
4465  // region.
4466  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4468  };
4469  llvm::Function *Fn;
4470  llvm::Constant *Addr;
4471  // Emit target region as a standalone region.
4472  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4473  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4474  assert(Fn && Addr && "Target device function emission failed.");
4475 }
4476 
4479  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4481  };
4482  emitCommonOMPTargetDirective(*this, S, CodeGen);
4483 }
4484 
4486  const OMPCancellationPointDirective &S) {
4487  CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getBeginLoc(),
4488  S.getCancelRegion());
4489 }
4490 
4492  const Expr *IfCond = nullptr;
4493  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4494  if (C->getNameModifier() == OMPD_unknown ||
4495  C->getNameModifier() == OMPD_cancel) {
4496  IfCond = C->getCondition();
4497  break;
4498  }
4499  }
4500  CGM.getOpenMPRuntime().emitCancelCall(*this, S.getBeginLoc(), IfCond,
4501  S.getCancelRegion());
4502 }
4503 
4506  if (Kind == OMPD_parallel || Kind == OMPD_task ||
4507  Kind == OMPD_target_parallel)
4508  return ReturnBlock;
4509  assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
4510  Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
4511  Kind == OMPD_distribute_parallel_for ||
4512  Kind == OMPD_target_parallel_for ||
4513  Kind == OMPD_teams_distribute_parallel_for ||
4514  Kind == OMPD_target_teams_distribute_parallel_for);
4515  return OMPCancelStack.getExitBlock();
4516 }
4517 
4519  const OMPClause &NC, OMPPrivateScope &PrivateScope,
4520  const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) {
4521  const auto &C = cast<OMPUseDevicePtrClause>(NC);
4522  auto OrigVarIt = C.varlist_begin();
4523  auto InitIt = C.inits().begin();
4524  for (const Expr *PvtVarIt : C.private_copies()) {
4525  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl());
4526  const auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl());
4527  const auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl());
4528 
4529  // In order to identify the right initializer we need to match the
4530  // declaration used by the mapping logic. In some cases we may get
4531  // OMPCapturedExprDecl that refers to the original declaration.
4532  const ValueDecl *MatchingVD = OrigVD;
4533  if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
4534  // OMPCapturedExprDecl are used to privative fields of the current
4535  // structure.
4536  const auto *ME = cast<MemberExpr>(OED->getInit());
4537  assert(isa<CXXThisExpr>(ME->getBase()) &&
4538  "Base should be the current struct!");
4539  MatchingVD = ME->getMemberDecl();
4540  }
4541 
4542  // If we don't have information about the current list item, move on to
4543  // the next one.
4544  auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
4545  if (InitAddrIt == CaptureDeviceAddrMap.end())
4546  continue;
4547 
4548  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, OrigVD,
4549  InitAddrIt, InitVD,
4550  PvtVD]() {
4551  // Initialize the temporary initialization variable with the address we
4552  // get from the runtime library. We have to cast the source address
4553  // because it is always a void *. References are materialized in the
4554  // privatization scope, so the initialization here disregards the fact
4555  // the original variable is a reference.
4556  QualType AddrQTy =
4557  getContext().getPointerType(OrigVD->getType().getNonReferenceType());
4558  llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy);
4559  Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy);
4560  setAddrOfLocalVar(InitVD, InitAddr);
4561 
4562  // Emit private declaration, it will be initialized by the value we
4563  // declaration we just added to the local declarations map.
4564  EmitDecl(*PvtVD);
4565 
4566  // The initialization variables reached its purpose in the emission
4567  // of the previous declaration, so we don't need it anymore.
4568  LocalDeclMap.erase(InitVD);
4569 
4570  // Return the address of the private variable.
4571  return GetAddrOfLocalVar(PvtVD);
4572  });
4573  assert(IsRegistered && "firstprivate var already registered as private");
4574  // Silence the warning about unused variable.
4575  (void)IsRegistered;
4576 
4577  ++OrigVarIt;
4578  ++InitIt;
4579  }
4580 }
4581 
4582 // Generate the instructions for '#pragma omp target data' directive.
4584  const OMPTargetDataDirective &S) {
4585  CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true);
4586 
4587  // Create a pre/post action to signal the privatization of the device pointer.
4588  // This action can be replaced by the OpenMP runtime code generation to
4589  // deactivate privatization.
4590  bool PrivatizeDevicePointers = false;
4591  class DevicePointerPrivActionTy : public PrePostActionTy {
4592  bool &PrivatizeDevicePointers;
4593 
4594  public:
4595  explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
4596  : PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {}
4597  void Enter(CodeGenFunction &CGF) override {
4598  PrivatizeDevicePointers = true;
4599  }
4600  };
4601  DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
4602 
4603  auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers](
4604  CodeGenFunction &CGF, PrePostActionTy &Action) {
4605  auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4607  };
4608 
4609  // Codegen that selects whether to generate the privatization code or not.
4610  auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers,
4611  &InnermostCodeGen](CodeGenFunction &CGF,
4612  PrePostActionTy &Action) {
4613  RegionCodeGenTy RCG(InnermostCodeGen);
4614  PrivatizeDevicePointers = false;
4615 
4616  // Call the pre-action to change the status of PrivatizeDevicePointers if
4617  // needed.
4618  Action.Enter(CGF);
4619 
4620  if (PrivatizeDevicePointers) {
4621  OMPPrivateScope PrivateScope(CGF);
4622  // Emit all instances of the use_device_ptr clause.
4623  for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
4624  CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
4625  Info.CaptureDeviceAddrMap);
4626  (void)PrivateScope.Privatize();
4627  RCG(CGF);
4628  } else {
4629  RCG(CGF);
4630  }
4631  };
4632 
4633  // Forward the provided action to the privatization codegen.
4634  RegionCodeGenTy PrivRCG(PrivCodeGen);
4635  PrivRCG.setAction(Action);
4636 
4637  // Notwithstanding the body of the region is emitted as inlined directive,
4638  // we don't use an inline scope as changes in the references inside the
4639  // region are expected to be visible outside, so we do not privative them.
4640  OMPLexicalScope Scope(CGF, S);
4641  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
4642  PrivRCG);
4643  };
4644 
4645  RegionCodeGenTy RCG(CodeGen);
4646 
4647  // If we don't have target devices, don't bother emitting the data mapping
4648  // code.
4649  if (CGM.getLangOpts().OMPTargetTriples.empty()) {
4650  RCG(*this);
4651  return;
4652  }
4653 
4654  // Check if we have any if clause associated with the directive.
4655  const Expr *IfCond = nullptr;
4656  if (const auto *C = S.getSingleClause<OMPIfClause>())
4657  IfCond = C->getCondition();
4658 
4659  // Check if we have any device clause associated with the directive.
4660  const Expr *Device = nullptr;
4661  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4662  Device = C->getDevice();
4663 
4664  // Set the action to signal privatization of device pointers.
4665  RCG.setAction(PrivAction);
4666 
4667  // Emit region code.
4668  CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
4669  Info);
4670 }
4671 
4673  const OMPTargetEnterDataDirective &S) {
4674  // If we don't have target devices, don't bother emitting the data mapping
4675  // code.
4676  if (CGM.getLangOpts().OMPTargetTriples.empty())
4677  return;
4678 
4679  // Check if we have any if clause associated with the directive.
4680  const Expr *IfCond = nullptr;
4681  if (const auto *C = S.getSingleClause<OMPIfClause>())
4682  IfCond = C->getCondition();
4683 
4684  // Check if we have any device clause associated with the directive.
4685  const Expr *Device = nullptr;
4686  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4687  Device = C->getDevice();
4688 
4689  OMPLexicalScope Scope(*this, S, OMPD_task);
4690  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4691 }
4692 
4694  const OMPTargetExitDataDirective &S) {
4695  // If we don't have target devices, don't bother emitting the data mapping
4696  // code.
4697  if (CGM.getLangOpts().OMPTargetTriples.empty())
4698  return;
4699 
4700  // Check if we have any if clause associated with the directive.
4701  const Expr *IfCond = nullptr;
4702  if (const auto *C = S.getSingleClause<OMPIfClause>())
4703  IfCond = C->getCondition();
4704 
4705  // Check if we have any device clause associated with the directive.
4706  const Expr *Device = nullptr;
4707  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4708  Device = C->getDevice();
4709 
4710  OMPLexicalScope Scope(*this, S, OMPD_task);
4711  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4712 }
4713 
4715  const OMPTargetParallelDirective &S,
4716  PrePostActionTy &Action) {
4717  // Get the captured statement associated with the 'parallel' region.
4718  const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
4719  Action.Enter(CGF);
4720  auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4721  Action.Enter(CGF);
4722  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4723  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4724  CGF.EmitOMPPrivateClause(S, PrivateScope);
4725  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4726  (void)PrivateScope.Privatize();
4728  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4729  // TODO: Add support for clauses.
4730  CGF.EmitStmt(CS->getCapturedStmt());
4731  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4732  };
4733  emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
4736  [](CodeGenFunction &) { return nullptr; });
4737 }
4738 
4740  CodeGenModule &CGM, StringRef ParentName,
4741  const OMPTargetParallelDirective &S) {
4742  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4743  emitTargetParallelRegion(CGF, S, Action);
4744  };
4745  llvm::Function *Fn;
4746  llvm::Constant *Addr;
4747  // Emit target region as a standalone region.
4748  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4749  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4750  assert(Fn && Addr && "Target device function emission failed.");
4751 }
4752 
4754  const OMPTargetParallelDirective &S) {
4755  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4756  emitTargetParallelRegion(CGF, S, Action);
4757  };
4758  emitCommonOMPTargetDirective(*this, S, CodeGen);
4759 }
4760 
4763  PrePostActionTy &Action) {
4764  Action.Enter(CGF);
4765  // Emit directive as a combined directive that consists of two implicit
4766  // directives: 'parallel' with 'for' directive.
4767  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4768  Action.Enter(CGF);
4770  CGF, OMPD_target_parallel_for, S.hasCancel());
4773  };
4774  emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
4776 }
4777 
4779  CodeGenModule &CGM, StringRef ParentName,
4780  const OMPTargetParallelForDirective &S) {
4781  // Emit SPMD target parallel for region as a standalone region.
4782  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4783  emitTargetParallelForRegion(CGF, S, Action);
4784  };
4785  llvm::Function *Fn;
4786  llvm::Constant *Addr;
4787  // Emit target region as a standalone region.
4788  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4789  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4790  assert(Fn && Addr && "Target device function emission failed.");
4791 }
4792 
4794  const OMPTargetParallelForDirective &S) {
4795  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4796  emitTargetParallelForRegion(CGF, S, Action);
4797  };
4798  emitCommonOMPTargetDirective(*this, S, CodeGen);
4799 }
4800 
4801 static void
4804  PrePostActionTy &Action) {
4805  Action.Enter(CGF);
4806  // Emit directive as a combined directive that consists of two implicit
4807  // directives: 'parallel' with 'for' directive.
4808  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4809  Action.Enter(CGF);
4812  };
4813  emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
4815 }
4816 
4818  CodeGenModule &CGM, StringRef ParentName,
4820  // Emit SPMD target parallel for region as a standalone region.
4821  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4822  emitTargetParallelForSimdRegion(CGF, S, Action);
4823  };
4824  llvm::Function *Fn;
4825  llvm::Constant *Addr;
4826  // Emit target region as a standalone region.
4827  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4828  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4829  assert(Fn && Addr && "Target device function emission failed.");
4830 }
4831 
4834  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4835  emitTargetParallelForSimdRegion(CGF, S, Action);
4836  };
4837  emitCommonOMPTargetDirective(*this, S, CodeGen);
4838 }
4839 
4840 /// Emit a helper variable and return corresponding lvalue.
4841 static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
4842  const ImplicitParamDecl *PVD,
4844  const auto *VDecl = cast<VarDecl>(Helper->getDecl());
4845  Privates.addPrivate(VDecl,
4846  [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); });
4847 }
4848 
4851  // Emit outlined function for task construct.
4852  const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);
4853  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
4854  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
4855  const Expr *IfCond = nullptr;
4856  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4857  if (C->getNameModifier() == OMPD_unknown ||
4858  C->getNameModifier() == OMPD_taskloop) {
4859  IfCond = C->getCondition();
4860  break;
4861  }
4862  }
4863 
4864  OMPTaskDataTy Data;
4865  // Check if taskloop must be emitted without taskgroup.
4867  // TODO: Check if we should emit tied or untied task.
4868  Data.Tied = true;
4869  // Set scheduling for taskloop
4870  if (const auto* Clause = S.getSingleClause<OMPGrainsizeClause>()) {
4871  // grainsize clause
4872  Data.Schedule.setInt(/*IntVal=*/false);
4873  Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
4874  } else if (const auto* Clause = S.getSingleClause<OMPNumTasksClause>()) {
4875  // num_tasks clause
4876  Data.Schedule.setInt(/*IntVal=*/true);
4877  Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
4878  }
4879 
4880  auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
4881  // if (PreCond) {
4882  // for (IV in 0..LastIteration) BODY;
4883  // <Final counter/linear vars updates>;
4884  // }
4885  //
4886 
4887  // Emit: if (PreCond) - begin.
4888  // If the condition constant folds and can be elided, avoid emitting the
4889  // whole loop.
4890  bool CondConstant;
4891  llvm::BasicBlock *ContBlock = nullptr;
4892  OMPLoopScope PreInitScope(CGF, S);
4893  if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
4894  if (!CondConstant)
4895  return;
4896  } else {
4897  llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
4898  ContBlock = CGF.createBasicBlock("taskloop.if.end");
4899  emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
4900  CGF.getProfileCount(&S));
4901  CGF.EmitBlock(ThenBlock);
4902  CGF.incrementProfileCounter(&S);
4903  }
4904 
4906  CGF.EmitOMPSimdInit(S);
4907 
4908  OMPPrivateScope LoopScope(CGF);
4909  // Emit helper vars inits.
4910  enum { LowerBound = 5, UpperBound, Stride, LastIter };
4911  auto *I = CS->getCapturedDecl()->param_begin();
4912  auto *LBP = std::next(I, LowerBound);
4913  auto *UBP = std::next(I, UpperBound);
4914  auto *STP = std::next(I, Stride);
4915  auto *LIP = std::next(I, LastIter);
4916  mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
4917  LoopScope);
4918  mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
4919  LoopScope);
4920  mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
4921  mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
4922  LoopScope);
4923  CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
4924  bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4925  (void)LoopScope.Privatize();
4926  // Emit the loop iteration variable.
4927  const Expr *IVExpr = S.getIterationVariable();
4928  const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
4929  CGF.EmitVarDecl(*IVDecl);
4930  CGF.EmitIgnoredExpr(S.getInit());
4931 
4932  // Emit the iterations count variable.
4933  // If it is not a variable, Sema decided to calculate iterations count on
4934  // each iteration (e.g., it is foldable into a constant).
4935  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
4936  CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
4937  // Emit calculation of the iterations count.
4938  CGF.EmitIgnoredExpr(S.getCalcLastIteration());
4939  }
4940 
4941  CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
4942  S.getInc(),
4943  [&S](CodeGenFunction &CGF) {
4944  CGF.EmitOMPLoopBody(S, JumpDest());
4945  CGF.EmitStopPoint(&S);
4946  },
4947  [](CodeGenFunction &) {});
4948  // Emit: if (PreCond) - end.
4949  if (ContBlock) {
4950  CGF.EmitBranch(ContBlock);
4951  CGF.EmitBlock(ContBlock, true);
4952  }
4953  // Emit final copy of the lastprivate variables if IsLastIter != 0.
4954  if (HasLastprivateClause) {
4955  CGF.EmitOMPLastprivateClauseFinal(
4957  CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
4958  CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
4959  (*LIP)->getType(), S.getBeginLoc())));
4960  }
4961  };
4962  auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
4963  IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
4964  const OMPTaskDataTy &Data) {
4965  auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
4966  &Data](CodeGenFunction &CGF, PrePostActionTy &) {
4967  OMPLoopScope PreInitScope(CGF, S);
4968  CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
4969  OutlinedFn, SharedsTy,
4970  CapturedStruct, IfCond, Data);
4971  };
4972  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
4973  CodeGen);
4974  };
4975  if (Data.Nogroup) {
4976  EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data);
4977  } else {
4978  CGM.getOpenMPRuntime().emitTaskgroupRegion(
4979  *this,
4980  [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
4981  PrePostActionTy &Action) {
4982  Action.Enter(CGF);
4983  CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen,
4984  Data);
4985  },
4986  S.getBeginLoc());
4987  }
4988 }
4989 
4991  EmitOMPTaskLoopBasedDirective(S);
4992 }
4993 
4995  const OMPTaskLoopSimdDirective &S) {
4996  EmitOMPTaskLoopBasedDirective(S);
4997 }
4998 
4999 // Generate the instructions for '#pragma omp target update' directive.
5001  const OMPTargetUpdateDirective &S) {
5002  // If we don't have target devices, don't bother emitting the data mapping
5003  // code.
5004  if (CGM.getLangOpts().OMPTargetTriples.empty())
5005  return;
5006 
5007  // Check if we have any if clause associated with the directive.
5008  const Expr *IfCond = nullptr;
5009  if (const auto *C = S.getSingleClause<OMPIfClause>())
5010  IfCond = C->getCondition();
5011 
5012  // Check if we have any device clause associated with the directive.
5013  const Expr *Device = nullptr;
5014  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
5015  Device = C->getDevice();
5016 
5017  OMPLexicalScope Scope(*this, S, OMPD_task);
5018  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
5019 }
5020 
5022  const OMPExecutableDirective &D) {
5023  if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
5024  return;
5025  auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {
5027  emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action);
5028  } else {
5029  OMPPrivateScope LoopGlobals(CGF);
5030  if (const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
5031  for (const Expr *E : LD->counters()) {
5032  const auto *VD = dyn_cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5033  if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) {
5034  LValue GlobLVal = CGF.EmitLValue(E);
5035  LoopGlobals.addPrivate(
5036  VD, [&GlobLVal]() { return GlobLVal.getAddress(); });
5037  }
5038  if (isa<OMPCapturedExprDecl>(VD)) {
5039  // Emit only those that were not explicitly referenced in clauses.
5040  if (!CGF.LocalDeclMap.count(VD))
5041  CGF.EmitVarDecl(*VD);
5042  }
5043  }
5044  for (const auto *C : D.getClausesOfKind<OMPOrderedClause>()) {
5045  if (!C->getNumForLoops())
5046  continue;
5047  for (unsigned I = LD->getCollapsedNumber(),
5048  E = C->getLoopNumIterations().size();
5049  I < E; ++I) {
5050  if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
5051  cast<DeclRefExpr>(C->getLoopCounter(I))->getDecl())) {
5052  // Emit only those that were not explicitly referenced in clauses.
5053  if (!CGF.LocalDeclMap.count(VD))
5054  CGF.EmitVarDecl(*VD);
5055  }
5056  }
5057  }
5058  }
5059  LoopGlobals.Privatize();
5061  }
5062  };
5063  OMPSimdLexicalScope Scope(*this, D);
5064  CGM.getOpenMPRuntime().emitInlinedDirective(
5065  *this,
5066  isOpenMPSimdDirective(D.getDirectiveKind()) ? OMPD_simd
5067  : D.getDirectiveKind(),
5068  CodeGen);
5069 }
5070 
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
Definition: CGCall.cpp:659
bool isAggregate() const
Definition: CGValue.h:54
This represents &#39;#pragma omp distribute simd&#39; composite directive.
Definition: StmtOpenMP.h:3248
Expr * getNextUpperBound() const
Definition: StmtOpenMP.h:865
This represents &#39;#pragma omp master&#39; directive.
Definition: StmtOpenMP.h:1431
This represents &#39;#pragma omp task&#39; directive.
Definition: StmtOpenMP.h:1771
static const Decl * getCanonicalDecl(const Decl *D)
Represents a function declaration or definition.
Definition: Decl.h:1739
This represents &#39;thread_limit&#39; clause in the &#39;#pragma omp ...&#39; directive.
Expr * getUpperBoundVariable() const
Definition: StmtOpenMP.h:833
Other implicit parameter.
Definition: Decl.h:1511
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, SourceLocation Loc)
This represents clause &#39;copyin&#39; in the &#39;#pragma omp ...&#39; directives.
Complete object ctor.
Definition: ABI.h:26
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition: Type.h:2541
Scheduling data for loop-based OpenMP directives.
Definition: OpenMPKinds.h:132
A (possibly-)qualified type.
Definition: Type.h:638
capture_init_iterator capture_init_begin()
Retrieve the first initialization argument.
Definition: Stmt.h:3062
bool isArrayType() const
Definition: Type.h:6349
ArrayRef< OMPClause * > clauses()
Definition: StmtOpenMP.h:260
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *V, SourceLocation Loc)
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
Definition: CGDecl.cpp:158
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
Definition: Decl.h:4133
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Definition: CGExpr.cpp:139
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
Definition: Dominators.h:30
static std::pair< LValue, LValue > emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer...
static OMPClauseWithPreInit * get(OMPClause *C)
Stmt - This represents one statement.
Definition: Stmt.h:66
This represents clause &#39;in_reduction&#39; in the &#39;#pragma omp task&#39; directives.
Expr * getLowerBoundVariable() const
Definition: StmtOpenMP.h:825
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
Store of global named registers are always calls to intrinsics.
Definition: CGExpr.cpp:2145
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
Definition: Type.cpp:505
bool requiresCleanups() const
Determine whether this scope requires any cleanups.
void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S)
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
SmallVector< std::pair< OpenMPDependClauseKind, const Expr * >, 4 > Dependences
This represents &#39;#pragma omp for simd&#39; directive.
Definition: StmtOpenMP.h:1181
SmallVector< const Expr *, 4 > LastprivateCopies
QualType getLValueReferenceType(QualType T, bool SpelledAsLValue=true) const
Return the uniqued reference to the type for an lvalue reference to the specified type...
Decl - This represents one declaration (or definition), e.g.
Definition: DeclBase.h:87
This represents &#39;grainsize&#39; clause in the &#39;#pragma omp ...&#39; directive.
This represents &#39;#pragma omp teams distribute parallel for&#39; composite directive.
Definition: StmtOpenMP.h:3659
SourceLocation getBeginLoc() const
Returns starting location of directive kind.
Definition: StmtOpenMP.h:168
bool isNothrow() const
Definition: Decl.cpp:4483
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
This represents &#39;if&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:240
CapturedStmt * getInnermostCapturedStmt()
Get innermost captured statement for the construct.
Definition: StmtOpenMP.h:226
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Definition: Type.h:6249
void EmitOMPOrderedDirective(const OMPOrderedDirective &S)
bool isVolatile() const
Definition: CGValue.h:301
This represents &#39;priority&#39; clause in the &#39;#pragma omp ...&#39; directive.
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
The base class of the type hierarchy.
Definition: Type.h:1411
This represents &#39;#pragma omp target teams distribute&#39; combined directive.
Definition: StmtOpenMP.h:3796
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
Definition: CGExpr.cpp:1910
static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, bool IsSeqCst, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Definition: Stmt.h:3006
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Definition: Type.h:2816
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
const TargetInfo & getTargetInfo() const
Definition: ASTContext.h:689
Expr * getCombinedParForInDistCond() const
Definition: StmtOpenMP.h:953
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters)
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Definition: CGExpr.cpp:2318
Floating point control options.
Definition: LangOptions.h:299
This represents &#39;#pragma omp parallel for&#39; directive.
Definition: StmtOpenMP.h:1552
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value *> &CapturedVars)
This represents &#39;#pragma omp target teams distribute parallel for&#39; combined directive.
Definition: StmtOpenMP.h:3864
Expr * getCombinedEnsureUpperBound() const
Definition: StmtOpenMP.h:917
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
Definition: CGAtomic.cpp:1895
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant, or if it does but contains a label, return false.
Represents a point when we exit a loop.
Definition: ProgramPoint.h:715
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
SmallVector< const Expr *, 4 > ReductionCopies
SourceLocation getEndLoc() const
Returns ending location of directive.
Definition: StmtOpenMP.h:170
void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S)
static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, const Twine &Name, llvm::Value *Init=nullptr)
This represents &#39;#pragma omp target exit data&#39; directive.
Definition: StmtOpenMP.h:2463
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
Definition: Code