clang  8.0.0svn
CGStmtOpenMP.cpp
Go to the documentation of this file.
1 //===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This contains code to emit OpenMP nodes as LLVM code.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "CGCleanup.h"
15 #include "CGOpenMPRuntime.h"
16 #include "CodeGenFunction.h"
17 #include "CodeGenModule.h"
18 #include "TargetInfo.h"
19 #include "clang/AST/Stmt.h"
20 #include "clang/AST/StmtOpenMP.h"
21 #include "clang/AST/DeclOpenMP.h"
22 #include "llvm/IR/CallSite.h"
23 using namespace clang;
24 using namespace CodeGen;
25 
26 namespace {
27 /// Lexical scope for OpenMP executable constructs, that handles correct codegen
28 /// for captured expressions.
29 class OMPLexicalScope : public CodeGenFunction::LexicalScope {
30  void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
31  for (const auto *C : S.clauses()) {
32  if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
33  if (const auto *PreInit =
34  cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
35  for (const auto *I : PreInit->decls()) {
36  if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
37  CGF.EmitVarDecl(cast<VarDecl>(*I));
38  } else {
40  CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
41  CGF.EmitAutoVarCleanups(Emission);
42  }
43  }
44  }
45  }
46  }
47  }
48  CodeGenFunction::OMPPrivateScope InlinedShareds;
49 
50  static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
51  return CGF.LambdaCaptureFields.lookup(VD) ||
52  (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
53  (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl));
54  }
55 
56 public:
57  OMPLexicalScope(
59  const llvm::Optional<OpenMPDirectiveKind> CapturedRegion = llvm::None,
60  const bool EmitPreInitStmt = true)
61  : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
62  InlinedShareds(CGF) {
63  if (EmitPreInitStmt)
64  emitPreInitStmt(CGF, S);
65  if (!CapturedRegion.hasValue())
66  return;
67  assert(S.hasAssociatedStmt() &&
68  "Expected associated statement for inlined directive.");
69  const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
70  for (const auto &C : CS->captures()) {
71  if (C.capturesVariable() || C.capturesVariableByCopy()) {
72  auto *VD = C.getCapturedVar();
73  assert(VD == VD->getCanonicalDecl() &&
74  "Canonical decl must be captured.");
75  DeclRefExpr DRE(
76  const_cast<VarDecl *>(VD),
77  isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
78  InlinedShareds.isGlobalVarCaptured(VD)),
79  VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
80  InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
81  return CGF.EmitLValue(&DRE).getAddress();
82  });
83  }
84  }
85  (void)InlinedShareds.Privatize();
86  }
87 };
88 
89 /// Lexical scope for OpenMP parallel construct, that handles correct codegen
90 /// for captured expressions.
91 class OMPParallelScope final : public OMPLexicalScope {
92  bool EmitPreInitStmt(const OMPExecutableDirective &S) {
94  return !(isOpenMPTargetExecutionDirective(Kind) ||
97  }
98 
99 public:
100  OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
101  : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
102  EmitPreInitStmt(S)) {}
103 };
104 
105 /// Lexical scope for OpenMP teams construct, that handles correct codegen
106 /// for captured expressions.
107 class OMPTeamsScope final : public OMPLexicalScope {
108  bool EmitPreInitStmt(const OMPExecutableDirective &S) {
110  return !isOpenMPTargetExecutionDirective(Kind) &&
112  }
113 
114 public:
115  OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
116  : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
117  EmitPreInitStmt(S)) {}
118 };
119 
120 /// Private scope for OpenMP loop-based directives, that supports capturing
121 /// of used expression from loop statement.
122 class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
123  void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) {
124  CodeGenFunction::OMPMapVars PreCondVars;
125  for (const auto *E : S.counters()) {
126  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
127  (void)PreCondVars.setVarAddr(
128  CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
129  }
130  (void)PreCondVars.apply(CGF);
131  if (const auto *PreInits = cast_or_null<DeclStmt>(S.getPreInits())) {
132  for (const auto *I : PreInits->decls())
133  CGF.EmitVarDecl(cast<VarDecl>(*I));
134  }
135  PreCondVars.restore(CGF);
136  }
137 
138 public:
139  OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S)
140  : CodeGenFunction::RunCleanupsScope(CGF) {
141  emitPreInitStmt(CGF, S);
142  }
143 };
144 
145 class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
146  CodeGenFunction::OMPPrivateScope InlinedShareds;
147 
148  static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
149  return CGF.LambdaCaptureFields.lookup(VD) ||
150  (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
151  (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) &&
152  cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
153  }
154 
155 public:
156  OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
157  : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
158  InlinedShareds(CGF) {
159  for (const auto *C : S.clauses()) {
160  if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
161  if (const auto *PreInit =
162  cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
163  for (const auto *I : PreInit->decls()) {
164  if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
165  CGF.EmitVarDecl(cast<VarDecl>(*I));
166  } else {
168  CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
169  CGF.EmitAutoVarCleanups(Emission);
170  }
171  }
172  }
173  } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
174  for (const Expr *E : UDP->varlists()) {
175  const Decl *D = cast<DeclRefExpr>(E)->getDecl();
176  if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
177  CGF.EmitVarDecl(*OED);
178  }
179  }
180  }
182  CGF.EmitOMPPrivateClause(S, InlinedShareds);
183  if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
184  if (const Expr *E = TG->getReductionRef())
185  CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
186  }
187  const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
188  while (CS) {
189  for (auto &C : CS->captures()) {
190  if (C.capturesVariable() || C.capturesVariableByCopy()) {
191  auto *VD = C.getCapturedVar();
192  assert(VD == VD->getCanonicalDecl() &&
193  "Canonical decl must be captured.");
194  DeclRefExpr DRE(const_cast<VarDecl *>(VD),
195  isCapturedVar(CGF, VD) ||
196  (CGF.CapturedStmtInfo &&
197  InlinedShareds.isGlobalVarCaptured(VD)),
199  C.getLocation());
200  InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
201  return CGF.EmitLValue(&DRE).getAddress();
202  });
203  }
204  }
205  CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
206  }
207  (void)InlinedShareds.Privatize();
208  }
209 };
210 
211 } // namespace
212 
214  const OMPExecutableDirective &S,
215  const RegionCodeGenTy &CodeGen);
216 
218  if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
219  if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
220  OrigVD = OrigVD->getCanonicalDecl();
221  bool IsCaptured =
222  LambdaCaptureFields.lookup(OrigVD) ||
223  (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
224  (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
225  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), IsCaptured,
226  OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
227  return EmitLValue(&DRE);
228  }
229  }
230  return EmitLValue(E);
231 }
232 
234  ASTContext &C = getContext();
235  llvm::Value *Size = nullptr;
236  auto SizeInChars = C.getTypeSizeInChars(Ty);
237  if (SizeInChars.isZero()) {
238  // getTypeSizeInChars() returns 0 for a VLA.
239  while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) {
240  VlaSizePair VlaSize = getVLASize(VAT);
241  Ty = VlaSize.Type;
242  Size = Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts)
243  : VlaSize.NumElts;
244  }
245  SizeInChars = C.getTypeSizeInChars(Ty);
246  if (SizeInChars.isZero())
247  return llvm::ConstantInt::get(SizeTy, /*V=*/0);
248  return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
249  }
250  return CGM.getSize(SizeInChars);
251 }
252 
254  const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
255  const RecordDecl *RD = S.getCapturedRecordDecl();
256  auto CurField = RD->field_begin();
257  auto CurCap = S.captures().begin();
259  E = S.capture_init_end();
260  I != E; ++I, ++CurField, ++CurCap) {
261  if (CurField->hasCapturedVLAType()) {
262  const VariableArrayType *VAT = CurField->getCapturedVLAType();
263  llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
264  CapturedVars.push_back(Val);
265  } else if (CurCap->capturesThis()) {
266  CapturedVars.push_back(CXXThisValue);
267  } else if (CurCap->capturesVariableByCopy()) {
268  llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
269 
270  // If the field is not a pointer, we need to save the actual value
271  // and load it as a void pointer.
272  if (!CurField->getType()->isAnyPointerType()) {
273  ASTContext &Ctx = getContext();
274  Address DstAddr = CreateMemTemp(
275  Ctx.getUIntPtrType(),
276  Twine(CurCap->getCapturedVar()->getName(), ".casted"));
277  LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
278 
279  llvm::Value *SrcAddrVal = EmitScalarConversion(
280  DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
281  Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
282  LValue SrcLV =
283  MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
284 
285  // Store the value using the source type pointer.
286  EmitStoreThroughLValue(RValue::get(CV), SrcLV);
287 
288  // Load the value using the destination type pointer.
289  CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
290  }
291  CapturedVars.push_back(CV);
292  } else {
293  assert(CurCap->capturesVariable() && "Expected capture by reference.");
294  CapturedVars.push_back(EmitLValue(*I).getAddress().getPointer());
295  }
296  }
297 }
298 
300  QualType DstType, StringRef Name,
301  LValue AddrLV,
302  bool isReferenceType = false) {
303  ASTContext &Ctx = CGF.getContext();
304 
305  llvm::Value *CastedPtr = CGF.EmitScalarConversion(
306  AddrLV.getAddress().getPointer(), Ctx.getUIntPtrType(),
307  Ctx.getPointerType(DstType), Loc);
308  Address TmpAddr =
309  CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType))
310  .getAddress();
311 
312  // If we are dealing with references we need to return the address of the
313  // reference instead of the reference of the value.
314  if (isReferenceType) {
315  QualType RefType = Ctx.getLValueReferenceType(DstType);
316  llvm::Value *RefVal = TmpAddr.getPointer();
317  TmpAddr = CGF.CreateMemTemp(RefType, Twine(Name, ".ref"));
318  LValue TmpLVal = CGF.MakeAddrLValue(TmpAddr, RefType);
319  CGF.EmitStoreThroughLValue(RValue::get(RefVal), TmpLVal, /*isInit=*/true);
320  }
321 
322  return TmpAddr;
323 }
324 
326  if (T->isLValueReferenceType())
327  return C.getLValueReferenceType(
329  /*SpelledAsLValue=*/false);
330  if (T->isPointerType())
332  if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
333  if (const auto *VLA = dyn_cast<VariableArrayType>(A))
334  return getCanonicalParamType(C, VLA->getElementType());
335  if (!A->isVariablyModifiedType())
336  return C.getCanonicalType(T);
337  }
338  return C.getCanonicalParamType(T);
339 }
340 
341 namespace {
342  /// Contains required data for proper outlined function codegen.
343  struct FunctionOptions {
344  /// Captured statement for which the function is generated.
345  const CapturedStmt *S = nullptr;
346  /// true if cast to/from UIntPtr is required for variables captured by
347  /// value.
348  const bool UIntPtrCastRequired = true;
349  /// true if only casted arguments must be registered as local args or VLA
350  /// sizes.
351  const bool RegisterCastedArgsOnly = false;
352  /// Name of the generated function.
353  const StringRef FunctionName;
354  explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
355  bool RegisterCastedArgsOnly,
356  StringRef FunctionName)
357  : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
358  RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
359  FunctionName(FunctionName) {}
360  };
361 }
362 
363 static llvm::Function *emitOutlinedFunctionPrologue(
364  CodeGenFunction &CGF, FunctionArgList &Args,
365  llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
366  &LocalAddrs,
367  llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
368  &VLASizes,
369  llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
370  const CapturedDecl *CD = FO.S->getCapturedDecl();
371  const RecordDecl *RD = FO.S->getCapturedRecordDecl();
372  assert(CD->hasBody() && "missing CapturedDecl body");
373 
374  CXXThisValue = nullptr;
375  // Build the argument list.
376  CodeGenModule &CGM = CGF.CGM;
377  ASTContext &Ctx = CGM.getContext();
378  FunctionArgList TargetArgs;
379  Args.append(CD->param_begin(),
380  std::next(CD->param_begin(), CD->getContextParamPosition()));
381  TargetArgs.append(
382  CD->param_begin(),
383  std::next(CD->param_begin(), CD->getContextParamPosition()));
384  auto I = FO.S->captures().begin();
385  FunctionDecl *DebugFunctionDecl = nullptr;
386  if (!FO.UIntPtrCastRequired) {
388  DebugFunctionDecl = FunctionDecl::Create(
389  Ctx, Ctx.getTranslationUnitDecl(), FO.S->getBeginLoc(),
392  Ctx.getFunctionType(Ctx.VoidTy, llvm::None, EPI)),
393  SC_Static, /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/false);
394  }
395  for (const FieldDecl *FD : RD->fields()) {
396  QualType ArgType = FD->getType();
397  IdentifierInfo *II = nullptr;
398  VarDecl *CapVar = nullptr;
399 
400  // If this is a capture by copy and the type is not a pointer, the outlined
401  // function argument type should be uintptr and the value properly casted to
402  // uintptr. This is necessary given that the runtime library is only able to
403  // deal with pointers. We can pass in the same way the VLA type sizes to the
404  // outlined function.
405  if (FO.UIntPtrCastRequired &&
406  ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
407  I->capturesVariableArrayType()))
408  ArgType = Ctx.getUIntPtrType();
409 
410  if (I->capturesVariable() || I->capturesVariableByCopy()) {
411  CapVar = I->getCapturedVar();
412  II = CapVar->getIdentifier();
413  } else if (I->capturesThis()) {
414  II = &Ctx.Idents.get("this");
415  } else {
416  assert(I->capturesVariableArrayType());
417  II = &Ctx.Idents.get("vla");
418  }
419  if (ArgType->isVariablyModifiedType())
420  ArgType = getCanonicalParamType(Ctx, ArgType);
421  VarDecl *Arg;
422  if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
423  Arg = ParmVarDecl::Create(
424  Ctx, DebugFunctionDecl,
425  CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),
426  CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
427  /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
428  } else {
429  Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
430  II, ArgType, ImplicitParamDecl::Other);
431  }
432  Args.emplace_back(Arg);
433  // Do not cast arguments if we emit function with non-original types.
434  TargetArgs.emplace_back(
435  FO.UIntPtrCastRequired
436  ? Arg
437  : CGM.getOpenMPRuntime().translateParameter(FD, Arg));
438  ++I;
439  }
440  Args.append(
441  std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
442  CD->param_end());
443  TargetArgs.append(
444  std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
445  CD->param_end());
446 
447  // Create the function declaration.
448  const CGFunctionInfo &FuncInfo =
449  CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, TargetArgs);
450  llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
451 
452  auto *F =
454  FO.FunctionName, &CGM.getModule());
455  CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
456  if (CD->isNothrow())
457  F->setDoesNotThrow();
458  F->setDoesNotRecurse();
459 
460  // Generate the function.
461  CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
462  FO.S->getBeginLoc(), CD->getBody()->getBeginLoc());
463  unsigned Cnt = CD->getContextParamPosition();
464  I = FO.S->captures().begin();
465  for (const FieldDecl *FD : RD->fields()) {
466  // Do not map arguments if we emit function with non-original types.
467  Address LocalAddr(Address::invalid());
468  if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
469  LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
470  TargetArgs[Cnt]);
471  } else {
472  LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
473  }
474  // If we are capturing a pointer by copy we don't need to do anything, just
475  // use the value that we get from the arguments.
476  if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
477  const VarDecl *CurVD = I->getCapturedVar();
478  // If the variable is a reference we need to materialize it here.
479  if (CurVD->getType()->isReferenceType()) {
480  Address RefAddr = CGF.CreateMemTemp(
481  CurVD->getType(), CGM.getPointerAlign(), ".materialized_ref");
482  CGF.EmitStoreOfScalar(LocalAddr.getPointer(), RefAddr,
483  /*Volatile=*/false, CurVD->getType());
484  LocalAddr = RefAddr;
485  }
486  if (!FO.RegisterCastedArgsOnly)
487  LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
488  ++Cnt;
489  ++I;
490  continue;
491  }
492 
493  LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
495  if (FD->hasCapturedVLAType()) {
496  if (FO.UIntPtrCastRequired) {
497  ArgLVal = CGF.MakeAddrLValue(
498  castValueFromUintptr(CGF, I->getLocation(), FD->getType(),
499  Args[Cnt]->getName(), ArgLVal),
501  }
502  llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
503  const VariableArrayType *VAT = FD->getCapturedVLAType();
504  VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg);
505  } else if (I->capturesVariable()) {
506  const VarDecl *Var = I->getCapturedVar();
507  QualType VarTy = Var->getType();
508  Address ArgAddr = ArgLVal.getAddress();
509  if (!VarTy->isReferenceType()) {
510  if (ArgLVal.getType()->isLValueReferenceType()) {
511  ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
512  } else if (!VarTy->isVariablyModifiedType() ||
513  !VarTy->isPointerType()) {
514  assert(ArgLVal.getType()->isPointerType());
515  ArgAddr = CGF.EmitLoadOfPointer(
516  ArgAddr, ArgLVal.getType()->castAs<PointerType>());
517  }
518  }
519  if (!FO.RegisterCastedArgsOnly) {
520  LocalAddrs.insert(
521  {Args[Cnt],
522  {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}});
523  }
524  } else if (I->capturesVariableByCopy()) {
525  assert(!FD->getType()->isAnyPointerType() &&
526  "Not expecting a captured pointer.");
527  const VarDecl *Var = I->getCapturedVar();
528  QualType VarTy = Var->getType();
529  LocalAddrs.insert(
530  {Args[Cnt],
531  {Var, FO.UIntPtrCastRequired
532  ? castValueFromUintptr(CGF, I->getLocation(),
533  FD->getType(), Args[Cnt]->getName(),
534  ArgLVal, VarTy->isReferenceType())
535  : ArgLVal.getAddress()}});
536  } else {
537  // If 'this' is captured, load it into CXXThisValue.
538  assert(I->capturesThis());
539  CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
540  LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress()}});
541  }
542  ++Cnt;
543  ++I;
544  }
545 
546  return F;
547 }
548 
549 llvm::Function *
551  assert(
552  CapturedStmtInfo &&
553  "CapturedStmtInfo should be set when generating the captured function");
554  const CapturedDecl *CD = S.getCapturedDecl();
555  // Build the argument list.
556  bool NeedWrapperFunction =
557  getDebugInfo() &&
558  CGM.getCodeGenOpts().getDebugInfo() >= codegenoptions::LimitedDebugInfo;
559  FunctionArgList Args;
560  llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
561  llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
562  SmallString<256> Buffer;
563  llvm::raw_svector_ostream Out(Buffer);
564  Out << CapturedStmtInfo->getHelperName();
565  if (NeedWrapperFunction)
566  Out << "_debug__";
567  FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
568  Out.str());
569  llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs,
570  VLASizes, CXXThisValue, FO);
571  for (const auto &LocalAddrPair : LocalAddrs) {
572  if (LocalAddrPair.second.first) {
573  setAddrOfLocalVar(LocalAddrPair.second.first,
574  LocalAddrPair.second.second);
575  }
576  }
577  for (const auto &VLASizePair : VLASizes)
578  VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
579  PGO.assignRegionCounters(GlobalDecl(CD), F);
580  CapturedStmtInfo->EmitBody(*this, CD->getBody());
581  FinishFunction(CD->getBodyRBrace());
582  if (!NeedWrapperFunction)
583  return F;
584 
585  FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
586  /*RegisterCastedArgsOnly=*/true,
587  CapturedStmtInfo->getHelperName());
588  CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
589  WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
590  Args.clear();
591  LocalAddrs.clear();
592  VLASizes.clear();
593  llvm::Function *WrapperF =
594  emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
595  WrapperCGF.CXXThisValue, WrapperFO);
597  for (const auto *Arg : Args) {
599  auto I = LocalAddrs.find(Arg);
600  if (I != LocalAddrs.end()) {
601  LValue LV = WrapperCGF.MakeAddrLValue(
602  I->second.second,
603  I->second.first ? I->second.first->getType() : Arg->getType(),
605  CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
606  } else {
607  auto EI = VLASizes.find(Arg);
608  if (EI != VLASizes.end()) {
609  CallArg = EI->second.second;
610  } else {
611  LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
612  Arg->getType(),
614  CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
615  }
616  }
617  CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
618  }
619  CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.getBeginLoc(),
620  F, CallArgs);
621  WrapperCGF.FinishFunction();
622  return WrapperF;
623 }
624 
625 //===----------------------------------------------------------------------===//
626 // OpenMP Directive Emission
627 //===----------------------------------------------------------------------===//
629  Address DestAddr, Address SrcAddr, QualType OriginalType,
630  const llvm::function_ref<void(Address, Address)> CopyGen) {
631  // Perform element-by-element initialization.
632  QualType ElementTy;
633 
634  // Drill down to the base element type on both arrays.
635  const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
636  llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
637  SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
638 
639  llvm::Value *SrcBegin = SrcAddr.getPointer();
640  llvm::Value *DestBegin = DestAddr.getPointer();
641  // Cast from pointer to array type to pointer to single element.
642  llvm::Value *DestEnd = Builder.CreateGEP(DestBegin, NumElements);
643  // The basic structure here is a while-do loop.
644  llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body");
645  llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done");
646  llvm::Value *IsEmpty =
647  Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
648  Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
649 
650  // Enter the loop body, making that address the current address.
651  llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
652  EmitBlock(BodyBB);
653 
654  CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
655 
656  llvm::PHINode *SrcElementPHI =
657  Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
658  SrcElementPHI->addIncoming(SrcBegin, EntryBB);
659  Address SrcElementCurrent =
660  Address(SrcElementPHI,
661  SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
662 
663  llvm::PHINode *DestElementPHI =
664  Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
665  DestElementPHI->addIncoming(DestBegin, EntryBB);
666  Address DestElementCurrent =
667  Address(DestElementPHI,
668  DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
669 
670  // Emit copy.
671  CopyGen(DestElementCurrent, SrcElementCurrent);
672 
673  // Shift the address forward by one element.
674  llvm::Value *DestElementNext = Builder.CreateConstGEP1_32(
675  DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
676  llvm::Value *SrcElementNext = Builder.CreateConstGEP1_32(
677  SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
678  // Check whether we've reached the end.
679  llvm::Value *Done =
680  Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
681  Builder.CreateCondBr(Done, DoneBB, BodyBB);
682  DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
683  SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
684 
685  // Done.
686  EmitBlock(DoneBB, /*IsFinished=*/true);
687 }
688 
689 void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
690  Address SrcAddr, const VarDecl *DestVD,
691  const VarDecl *SrcVD, const Expr *Copy) {
692  if (OriginalType->isArrayType()) {
693  const auto *BO = dyn_cast<BinaryOperator>(Copy);
694  if (BO && BO->getOpcode() == BO_Assign) {
695  // Perform simple memcpy for simple copying.
696  LValue Dest = MakeAddrLValue(DestAddr, OriginalType);
697  LValue Src = MakeAddrLValue(SrcAddr, OriginalType);
698  EmitAggregateAssign(Dest, Src, OriginalType);
699  } else {
700  // For arrays with complex element types perform element by element
701  // copying.
702  EmitOMPAggregateAssign(
703  DestAddr, SrcAddr, OriginalType,
704  [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
705  // Working with the single array element, so have to remap
706  // destination and source variables to corresponding array
707  // elements.
709  Remap.addPrivate(DestVD, [DestElement]() { return DestElement; });
710  Remap.addPrivate(SrcVD, [SrcElement]() { return SrcElement; });
711  (void)Remap.Privatize();
712  EmitIgnoredExpr(Copy);
713  });
714  }
715  } else {
716  // Remap pseudo source variable to private copy.
718  Remap.addPrivate(SrcVD, [SrcAddr]() { return SrcAddr; });
719  Remap.addPrivate(DestVD, [DestAddr]() { return DestAddr; });
720  (void)Remap.Privatize();
721  // Emit copying of the whole variable.
722  EmitIgnoredExpr(Copy);
723  }
724 }
725 
727  OMPPrivateScope &PrivateScope) {
728  if (!HaveInsertPoint())
729  return false;
730  bool FirstprivateIsLastprivate = false;
731  llvm::DenseSet<const VarDecl *> Lastprivates;
732  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
733  for (const auto *D : C->varlists())
734  Lastprivates.insert(
735  cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl());
736  }
737  llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
739  getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind());
740  // Force emission of the firstprivate copy if the directive does not emit
741  // outlined function, like omp for, omp simd, omp distribute etc.
742  bool MustEmitFirstprivateCopy =
743  CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
744  for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
745  auto IRef = C->varlist_begin();
746  auto InitsRef = C->inits().begin();
747  for (const Expr *IInit : C->private_copies()) {
748  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
749  bool ThisFirstprivateIsLastprivate =
750  Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
751  const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
752  if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
753  !FD->getType()->isReferenceType()) {
754  EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
755  ++IRef;
756  ++InitsRef;
757  continue;
758  }
759  FirstprivateIsLastprivate =
760  FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
761  if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
762  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
763  const auto *VDInit =
764  cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
765  bool IsRegistered;
766  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
767  /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
768  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
769  LValue OriginalLVal = EmitLValue(&DRE);
770  QualType Type = VD->getType();
771  if (Type->isArrayType()) {
772  // Emit VarDecl with copy init for arrays.
773  // Get the address of the original variable captured in current
774  // captured region.
775  IsRegistered = PrivateScope.addPrivate(
776  OrigVD, [this, VD, Type, OriginalLVal, VDInit]() {
777  AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
778  const Expr *Init = VD->getInit();
779  if (!isa<CXXConstructExpr>(Init) ||
780  isTrivialInitializer(Init)) {
781  // Perform simple memcpy.
782  LValue Dest =
783  MakeAddrLValue(Emission.getAllocatedAddress(), Type);
784  EmitAggregateAssign(Dest, OriginalLVal, Type);
785  } else {
786  EmitOMPAggregateAssign(
787  Emission.getAllocatedAddress(), OriginalLVal.getAddress(),
788  Type,
789  [this, VDInit, Init](Address DestElement,
790  Address SrcElement) {
791  // Clean up any temporaries needed by the
792  // initialization.
793  RunCleanupsScope InitScope(*this);
794  // Emit initialization for single element.
795  setAddrOfLocalVar(VDInit, SrcElement);
796  EmitAnyExprToMem(Init, DestElement,
797  Init->getType().getQualifiers(),
798  /*IsInitializer*/ false);
799  LocalDeclMap.erase(VDInit);
800  });
801  }
802  EmitAutoVarCleanups(Emission);
803  return Emission.getAllocatedAddress();
804  });
805  } else {
806  Address OriginalAddr = OriginalLVal.getAddress();
807  IsRegistered = PrivateScope.addPrivate(
808  OrigVD, [this, VDInit, OriginalAddr, VD]() {
809  // Emit private VarDecl with copy init.
810  // Remap temp VDInit variable to the address of the original
811  // variable (for proper handling of captured global variables).
812  setAddrOfLocalVar(VDInit, OriginalAddr);
813  EmitDecl(*VD);
814  LocalDeclMap.erase(VDInit);
815  return GetAddrOfLocalVar(VD);
816  });
817  }
818  assert(IsRegistered &&
819  "firstprivate var already registered as private");
820  // Silence the warning about unused variable.
821  (void)IsRegistered;
822  }
823  ++IRef;
824  ++InitsRef;
825  }
826  }
827  return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
828 }
829 
831  const OMPExecutableDirective &D,
832  CodeGenFunction::OMPPrivateScope &PrivateScope) {
833  if (!HaveInsertPoint())
834  return;
835  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
836  for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
837  auto IRef = C->varlist_begin();
838  for (const Expr *IInit : C->private_copies()) {
839  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
840  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
841  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
842  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
843  // Emit private VarDecl with copy init.
844  EmitDecl(*VD);
845  return GetAddrOfLocalVar(VD);
846  });
847  assert(IsRegistered && "private var already registered as private");
848  // Silence the warning about unused variable.
849  (void)IsRegistered;
850  }
851  ++IRef;
852  }
853  }
854 }
855 
857  if (!HaveInsertPoint())
858  return false;
859  // threadprivate_var1 = master_threadprivate_var1;
860  // operator=(threadprivate_var2, master_threadprivate_var2);
861  // ...
862  // __kmpc_barrier(&loc, global_tid);
864  llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
865  for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
866  auto IRef = C->varlist_begin();
867  auto ISrcRef = C->source_exprs().begin();
868  auto IDestRef = C->destination_exprs().begin();
869  for (const Expr *AssignOp : C->assignment_ops()) {
870  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
871  QualType Type = VD->getType();
872  if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
873  // Get the address of the master variable. If we are emitting code with
874  // TLS support, the address is passed from the master as field in the
875  // captured declaration.
876  Address MasterAddr = Address::invalid();
877  if (getLangOpts().OpenMPUseTLS &&
878  getContext().getTargetInfo().isTLSSupported()) {
879  assert(CapturedStmtInfo->lookup(VD) &&
880  "Copyin threadprivates should have been captured!");
881  DeclRefExpr DRE(const_cast<VarDecl *>(VD), true, (*IRef)->getType(),
882  VK_LValue, (*IRef)->getExprLoc());
883  MasterAddr = EmitLValue(&DRE).getAddress();
884  LocalDeclMap.erase(VD);
885  } else {
886  MasterAddr =
887  Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
888  : CGM.GetAddrOfGlobal(VD),
889  getContext().getDeclAlign(VD));
890  }
891  // Get the address of the threadprivate variable.
892  Address PrivateAddr = EmitLValue(*IRef).getAddress();
893  if (CopiedVars.size() == 1) {
894  // At first check if current thread is a master thread. If it is, no
895  // need to copy data.
896  CopyBegin = createBasicBlock("copyin.not.master");
897  CopyEnd = createBasicBlock("copyin.not.master.end");
898  Builder.CreateCondBr(
899  Builder.CreateICmpNE(
900  Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy),
901  Builder.CreatePtrToInt(PrivateAddr.getPointer(),
902  CGM.IntPtrTy)),
903  CopyBegin, CopyEnd);
904  EmitBlock(CopyBegin);
905  }
906  const auto *SrcVD =
907  cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
908  const auto *DestVD =
909  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
910  EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
911  }
912  ++IRef;
913  ++ISrcRef;
914  ++IDestRef;
915  }
916  }
917  if (CopyEnd) {
918  // Exit out of copying procedure for non-master thread.
919  EmitBlock(CopyEnd, /*IsFinished=*/true);
920  return true;
921  }
922  return false;
923 }
924 
926  const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
927  if (!HaveInsertPoint())
928  return false;
929  bool HasAtLeastOneLastprivate = false;
932  const auto *LoopDirective = cast<OMPLoopDirective>(&D);
933  for (const Expr *C : LoopDirective->counters()) {
934  SIMDLCVs.insert(
935  cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
936  }
937  }
938  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
939  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
940  HasAtLeastOneLastprivate = true;
942  !getLangOpts().OpenMPSimd)
943  break;
944  auto IRef = C->varlist_begin();
945  auto IDestRef = C->destination_exprs().begin();
946  for (const Expr *IInit : C->private_copies()) {
947  // Keep the address of the original variable for future update at the end
948  // of the loop.
949  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
950  // Taskloops do not require additional initialization, it is done in
951  // runtime support library.
952  if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
953  const auto *DestVD =
954  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
955  PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() {
956  DeclRefExpr DRE(
957  const_cast<VarDecl *>(OrigVD),
958  /*RefersToEnclosingVariableOrCapture=*/CapturedStmtInfo->lookup(
959  OrigVD) != nullptr,
960  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
961  return EmitLValue(&DRE).getAddress();
962  });
963  // Check if the variable is also a firstprivate: in this case IInit is
964  // not generated. Initialization of this variable will happen in codegen
965  // for 'firstprivate' clause.
966  if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
967  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
968  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
969  // Emit private VarDecl with copy init.
970  EmitDecl(*VD);
971  return GetAddrOfLocalVar(VD);
972  });
973  assert(IsRegistered &&
974  "lastprivate var already registered as private");
975  (void)IsRegistered;
976  }
977  }
978  ++IRef;
979  ++IDestRef;
980  }
981  }
982  return HasAtLeastOneLastprivate;
983 }
984 
986  const OMPExecutableDirective &D, bool NoFinals,
987  llvm::Value *IsLastIterCond) {
988  if (!HaveInsertPoint())
989  return;
990  // Emit following code:
991  // if (<IsLastIterCond>) {
992  // orig_var1 = private_orig_var1;
993  // ...
994  // orig_varn = private_orig_varn;
995  // }
996  llvm::BasicBlock *ThenBB = nullptr;
997  llvm::BasicBlock *DoneBB = nullptr;
998  if (IsLastIterCond) {
999  ThenBB = createBasicBlock(".omp.lastprivate.then");
1000  DoneBB = createBasicBlock(".omp.lastprivate.done");
1001  Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1002  EmitBlock(ThenBB);
1003  }
1004  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1005  llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1006  if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1007  auto IC = LoopDirective->counters().begin();
1008  for (const Expr *F : LoopDirective->finals()) {
1009  const auto *D =
1010  cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1011  if (NoFinals)
1012  AlreadyEmittedVars.insert(D);
1013  else
1014  LoopCountersAndUpdates[D] = F;
1015  ++IC;
1016  }
1017  }
1018  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1019  auto IRef = C->varlist_begin();
1020  auto ISrcRef = C->source_exprs().begin();
1021  auto IDestRef = C->destination_exprs().begin();
1022  for (const Expr *AssignOp : C->assignment_ops()) {
1023  const auto *PrivateVD =
1024  cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1025  QualType Type = PrivateVD->getType();
1026  const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1027  if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1028  // If lastprivate variable is a loop control variable for loop-based
1029  // directive, update its value before copyin back to original
1030  // variable.
1031  if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1032  EmitIgnoredExpr(FinalExpr);
1033  const auto *SrcVD =
1034  cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1035  const auto *DestVD =
1036  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1037  // Get the address of the original variable.
1038  Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1039  // Get the address of the private variable.
1040  Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
1041  if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1042  PrivateAddr =
1043  Address(Builder.CreateLoad(PrivateAddr),
1044  getNaturalTypeAlignment(RefTy->getPointeeType()));
1045  EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1046  }
1047  ++IRef;
1048  ++ISrcRef;
1049  ++IDestRef;
1050  }
1051  if (const Expr *PostUpdate = C->getPostUpdateExpr())
1052  EmitIgnoredExpr(PostUpdate);
1053  }
1054  if (IsLastIterCond)
1055  EmitBlock(DoneBB, /*IsFinished=*/true);
1056 }
1057 
1059  const OMPExecutableDirective &D,
1060  CodeGenFunction::OMPPrivateScope &PrivateScope) {
1061  if (!HaveInsertPoint())
1062  return;
1065  SmallVector<const Expr *, 4> ReductionOps;
1068  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1069  auto IPriv = C->privates().begin();
1070  auto IRed = C->reduction_ops().begin();
1071  auto ILHS = C->lhs_exprs().begin();
1072  auto IRHS = C->rhs_exprs().begin();
1073  for (const Expr *Ref : C->varlists()) {
1074  Shareds.emplace_back(Ref);
1075  Privates.emplace_back(*IPriv);
1076  ReductionOps.emplace_back(*IRed);
1077  LHSs.emplace_back(*ILHS);
1078  RHSs.emplace_back(*IRHS);
1079  std::advance(IPriv, 1);
1080  std::advance(IRed, 1);
1081  std::advance(ILHS, 1);
1082  std::advance(IRHS, 1);
1083  }
1084  }
1085  ReductionCodeGen RedCG(Shareds, Privates, ReductionOps);
1086  unsigned Count = 0;
1087  auto ILHS = LHSs.begin();
1088  auto IRHS = RHSs.begin();
1089  auto IPriv = Privates.begin();
1090  for (const Expr *IRef : Shareds) {
1091  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1092  // Emit private VarDecl with reduction init.
1093  RedCG.emitSharedLValue(*this, Count);
1094  RedCG.emitAggregateType(*this, Count);
1095  AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);
1096  RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
1097  RedCG.getSharedLValue(Count),
1098  [&Emission](CodeGenFunction &CGF) {
1099  CGF.EmitAutoVarInit(Emission);
1100  return true;
1101  });
1102  EmitAutoVarCleanups(Emission);
1103  Address BaseAddr = RedCG.adjustPrivateAddress(
1104  *this, Count, Emission.getAllocatedAddress());
1105  bool IsRegistered = PrivateScope.addPrivate(
1106  RedCG.getBaseDecl(Count), [BaseAddr]() { return BaseAddr; });
1107  assert(IsRegistered && "private var already registered as private");
1108  // Silence the warning about unused variable.
1109  (void)IsRegistered;
1110 
1111  const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1112  const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1113  QualType Type = PrivateVD->getType();
1114  bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1115  if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1116  // Store the address of the original variable associated with the LHS
1117  // implicit variable.
1118  PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
1119  return RedCG.getSharedLValue(Count).getAddress();
1120  });
1121  PrivateScope.addPrivate(
1122  RHSVD, [this, PrivateVD]() { return GetAddrOfLocalVar(PrivateVD); });
1123  } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1124  isa<ArraySubscriptExpr>(IRef)) {
1125  // Store the address of the original variable associated with the LHS
1126  // implicit variable.
1127  PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
1128  return RedCG.getSharedLValue(Count).getAddress();
1129  });
1130  PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() {
1131  return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD),
1132  ConvertTypeForMem(RHSVD->getType()),
1133  "rhs.begin");
1134  });
1135  } else {
1136  QualType Type = PrivateVD->getType();
1137  bool IsArray = getContext().getAsArrayType(Type) != nullptr;
1138  Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress();
1139  // Store the address of the original variable associated with the LHS
1140  // implicit variable.
1141  if (IsArray) {
1142  OriginalAddr = Builder.CreateElementBitCast(
1143  OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin");
1144  }
1145  PrivateScope.addPrivate(LHSVD, [OriginalAddr]() { return OriginalAddr; });
1146  PrivateScope.addPrivate(
1147  RHSVD, [this, PrivateVD, RHSVD, IsArray]() {
1148  return IsArray
1149  ? Builder.CreateElementBitCast(
1150  GetAddrOfLocalVar(PrivateVD),
1151  ConvertTypeForMem(RHSVD->getType()), "rhs.begin")
1152  : GetAddrOfLocalVar(PrivateVD);
1153  });
1154  }
1155  ++ILHS;
1156  ++IRHS;
1157  ++IPriv;
1158  ++Count;
1159  }
1160 }
1161 
1163  const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1164  if (!HaveInsertPoint())
1165  return;
1170  bool HasAtLeastOneReduction = false;
1171  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1172  HasAtLeastOneReduction = true;
1173  Privates.append(C->privates().begin(), C->privates().end());
1174  LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1175  RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1176  ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1177  }
1178  if (HasAtLeastOneReduction) {
1179  bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1181  ReductionKind == OMPD_simd;
1182  bool SimpleReduction = ReductionKind == OMPD_simd;
1183  // Emit nowait reduction if nowait clause is present or directive is a
1184  // parallel directive (it always has implicit barrier).
1185  CGM.getOpenMPRuntime().emitReduction(
1186  *this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1187  {WithNowait, SimpleReduction, ReductionKind});
1188  }
1189 }
1190 
1192  CodeGenFunction &CGF, const OMPExecutableDirective &D,
1193  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1194  if (!CGF.HaveInsertPoint())
1195  return;
1196  llvm::BasicBlock *DoneBB = nullptr;
1197  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1198  if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
1199  if (!DoneBB) {
1200  if (llvm::Value *Cond = CondGen(CGF)) {
1201  // If the first post-update expression is found, emit conditional
1202  // block if it was requested.
1203  llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1204  DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1205  CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1206  CGF.EmitBlock(ThenBB);
1207  }
1208  }
1209  CGF.EmitIgnoredExpr(PostUpdate);
1210  }
1211  }
1212  if (DoneBB)
1213  CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1214 }
1215 
1216 namespace {
1217 /// Codegen lambda for appending distribute lower and upper bounds to outlined
1218 /// parallel function. This is necessary for combined constructs such as
1219 /// 'distribute parallel for'
1220 typedef llvm::function_ref<void(CodeGenFunction &,
1221  const OMPExecutableDirective &,
1223  CodeGenBoundParametersTy;
1224 } // anonymous namespace
1225 
1227  CodeGenFunction &CGF, const OMPExecutableDirective &S,
1228  OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1229  const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1230  const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1231  llvm::Value *OutlinedFn =
1232  CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
1233  S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
1234  if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
1235  CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1236  llvm::Value *NumThreads =
1237  CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1238  /*IgnoreResultAssign=*/true);
1239  CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
1240  CGF, NumThreads, NumThreadsClause->getBeginLoc());
1241  }
1242  if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
1243  CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1244  CGF.CGM.getOpenMPRuntime().emitProcBindClause(
1245  CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1246  }
1247  const Expr *IfCond = nullptr;
1248  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1249  if (C->getNameModifier() == OMPD_unknown ||
1250  C->getNameModifier() == OMPD_parallel) {
1251  IfCond = C->getCondition();
1252  break;
1253  }
1254  }
1255 
1256  OMPParallelScope Scope(CGF, S);
1258  // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
1259  // lower and upper bounds with the pragma 'for' chunking mechanism.
1260  // The following lambda takes care of appending the lower and upper bound
1261  // parameters when necessary
1262  CodeGenBoundParameters(CGF, S, CapturedVars);
1263  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
1264  CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn,
1265  CapturedVars, IfCond);
1266 }
1267 
1269  const OMPExecutableDirective &,
1271 
1273  // Emit parallel region as a standalone region.
1274  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1275  Action.Enter(CGF);
1276  OMPPrivateScope PrivateScope(CGF);
1277  bool Copyins = CGF.EmitOMPCopyinClause(S);
1278  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1279  if (Copyins) {
1280  // Emit implicit barrier to synchronize threads and avoid data races on
1281  // propagation master's thread values of threadprivate variables to local
1282  // instances of that variables of all other implicit threads.
1283  CGF.CGM.getOpenMPRuntime().emitBarrierCall(
1284  CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
1285  /*ForceSimpleCall=*/true);
1286  }
1287  CGF.EmitOMPPrivateClause(S, PrivateScope);
1288  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1289  (void)PrivateScope.Privatize();
1290  CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1291  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
1292  };
1293  emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
1296  [](CodeGenFunction &) { return nullptr; });
1297 }
1298 
1300  JumpDest LoopExit) {
1301  RunCleanupsScope BodyScope(*this);
1302  // Update counters values on current iteration.
1303  for (const Expr *UE : D.updates())
1304  EmitIgnoredExpr(UE);
1305  // Update the linear variables.
1306  // In distribute directives only loop counters may be marked as linear, no
1307  // need to generate the code for them.
1309  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1310  for (const Expr *UE : C->updates())
1311  EmitIgnoredExpr(UE);
1312  }
1313  }
1314 
1315  // On a continue in the body, jump to the end.
1316  JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue");
1317  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1318  // Emit loop body.
1319  EmitStmt(D.getBody());
1320  // The end (updates/cleanups).
1321  EmitBlock(Continue.getBlock());
1322  BreakContinueStack.pop_back();
1323 }
1324 
1326  const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
1327  const Expr *IncExpr,
1328  const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
1329  const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
1330  auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
1331 
1332  // Start the loop with a block that tests the condition.
1333  auto CondBlock = createBasicBlock("omp.inner.for.cond");
1334  EmitBlock(CondBlock);
1335  const SourceRange R = S.getSourceRange();
1336  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1337  SourceLocToDebugLoc(R.getEnd()));
1338 
1339  // If there are any cleanups between here and the loop-exit scope,
1340  // create a block to stage a loop exit along.
1341  llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1342  if (RequiresCleanup)
1343  ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
1344 
1345  llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body");
1346 
1347  // Emit condition.
1348  EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
1349  if (ExitBlock != LoopExit.getBlock()) {
1350  EmitBlock(ExitBlock);
1351  EmitBranchThroughCleanup(LoopExit);
1352  }
1353 
1354  EmitBlock(LoopBody);
1355  incrementProfileCounter(&S);
1356 
1357  // Create a block for the increment.
1358  JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
1359  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1360 
1361  BodyGen(*this);
1362 
1363  // Emit "IV = IV + 1" and a back-edge to the condition block.
1364  EmitBlock(Continue.getBlock());
1365  EmitIgnoredExpr(IncExpr);
1366  PostIncGen(*this);
1367  BreakContinueStack.pop_back();
1368  EmitBranch(CondBlock);
1369  LoopStack.pop();
1370  // Emit the fall-through block.
1371  EmitBlock(LoopExit.getBlock());
1372 }
1373 
1375  if (!HaveInsertPoint())
1376  return false;
1377  // Emit inits for the linear variables.
1378  bool HasLinears = false;
1379  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1380  for (const Expr *Init : C->inits()) {
1381  HasLinears = true;
1382  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
1383  if (const auto *Ref =
1384  dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
1385  AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
1386  const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
1387  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
1388  CapturedStmtInfo->lookup(OrigVD) != nullptr,
1389  VD->getInit()->getType(), VK_LValue,
1390  VD->getInit()->getExprLoc());
1391  EmitExprAsInit(&DRE, VD, MakeAddrLValue(Emission.getAllocatedAddress(),
1392  VD->getType()),
1393  /*capturedByInit=*/false);
1394  EmitAutoVarCleanups(Emission);
1395  } else {
1396  EmitVarDecl(*VD);
1397  }
1398  }
1399  // Emit the linear steps for the linear clauses.
1400  // If a step is not constant, it is pre-calculated before the loop.
1401  if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
1402  if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
1403  EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
1404  // Emit calculation of the linear step.
1405  EmitIgnoredExpr(CS);
1406  }
1407  }
1408  return HasLinears;
1409 }
1410 
1412  const OMPLoopDirective &D,
1413  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1414  if (!HaveInsertPoint())
1415  return;
1416  llvm::BasicBlock *DoneBB = nullptr;
1417  // Emit the final values of the linear variables.
1418  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1419  auto IC = C->varlist_begin();
1420  for (const Expr *F : C->finals()) {
1421  if (!DoneBB) {
1422  if (llvm::Value *Cond = CondGen(*this)) {
1423  // If the first post-update expression is found, emit conditional
1424  // block if it was requested.
1425  llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu");
1426  DoneBB = createBasicBlock(".omp.linear.pu.done");
1427  Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1428  EmitBlock(ThenBB);
1429  }
1430  }
1431  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
1432  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
1433  CapturedStmtInfo->lookup(OrigVD) != nullptr,
1434  (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
1435  Address OrigAddr = EmitLValue(&DRE).getAddress();
1436  CodeGenFunction::OMPPrivateScope VarScope(*this);
1437  VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1438  (void)VarScope.Privatize();
1439  EmitIgnoredExpr(F);
1440  ++IC;
1441  }
1442  if (const Expr *PostUpdate = C->getPostUpdateExpr())
1443  EmitIgnoredExpr(PostUpdate);
1444  }
1445  if (DoneBB)
1446  EmitBlock(DoneBB, /*IsFinished=*/true);
1447 }
1448 
1450  const OMPExecutableDirective &D) {
1451  if (!CGF.HaveInsertPoint())
1452  return;
1453  for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
1454  unsigned ClauseAlignment = 0;
1455  if (const Expr *AlignmentExpr = Clause->getAlignment()) {
1456  auto *AlignmentCI =
1457  cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
1458  ClauseAlignment = static_cast<unsigned>(AlignmentCI->getZExtValue());
1459  }
1460  for (const Expr *E : Clause->varlists()) {
1461  unsigned Alignment = ClauseAlignment;
1462  if (Alignment == 0) {
1463  // OpenMP [2.8.1, Description]
1464  // If no optional parameter is specified, implementation-defined default
1465  // alignments for SIMD instructions on the target platforms are assumed.
1466  Alignment =
1467  CGF.getContext()
1469  E->getType()->getPointeeType()))
1470  .getQuantity();
1471  }
1472  assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) &&
1473  "alignment is not power of 2");
1474  if (Alignment != 0) {
1475  llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
1476  CGF.EmitAlignmentAssumption(PtrValue, Alignment);
1477  }
1478  }
1479  }
1480 }
1481 
1483  const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
1484  if (!HaveInsertPoint())
1485  return;
1486  auto I = S.private_counters().begin();
1487  for (const Expr *E : S.counters()) {
1488  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1489  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
1490  // Emit var without initialization.
1491  AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD);
1492  EmitAutoVarCleanups(VarEmission);
1493  LocalDeclMap.erase(PrivateVD);
1494  (void)LoopScope.addPrivate(VD, [&VarEmission]() {
1495  return VarEmission.getAllocatedAddress();
1496  });
1497  if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
1498  VD->hasGlobalStorage()) {
1499  (void)LoopScope.addPrivate(PrivateVD, [this, VD, E]() {
1500  DeclRefExpr DRE(const_cast<VarDecl *>(VD),
1501  LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
1502  E->getType(), VK_LValue, E->getExprLoc());
1503  return EmitLValue(&DRE).getAddress();
1504  });
1505  } else {
1506  (void)LoopScope.addPrivate(PrivateVD, [&VarEmission]() {
1507  return VarEmission.getAllocatedAddress();
1508  });
1509  }
1510  ++I;
1511  }
1512  // Privatize extra loop counters used in loops for ordered(n) clauses.
1513  for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) {
1514  if (!C->getNumForLoops())
1515  continue;
1516  for (unsigned I = S.getCollapsedNumber(),
1517  E = C->getLoopNumIterations().size();
1518  I < E; ++I) {
1519  const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I));
1520  const auto *VD = cast<VarDecl>(DRE->getDecl());
1521  // Override only those variables that are really emitted already.
1522  if (LocalDeclMap.count(VD)) {
1523  (void)LoopScope.addPrivate(VD, [this, DRE, VD]() {
1524  return CreateMemTemp(DRE->getType(), VD->getName());
1525  });
1526  }
1527  }
1528  }
1529 }
1530 
1531 static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
1532  const Expr *Cond, llvm::BasicBlock *TrueBlock,
1533  llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
1534  if (!CGF.HaveInsertPoint())
1535  return;
1536  {
1537  CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
1538  CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
1539  (void)PreCondScope.Privatize();
1540  // Get initial values of real counters.
1541  for (const Expr *I : S.inits()) {
1542  CGF.EmitIgnoredExpr(I);
1543  }
1544  }
1545  // Check that loop is executed at least one time.
1546  CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
1547 }
1548 
1550  const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
1551  if (!HaveInsertPoint())
1552  return;
1555  const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1556  for (const Expr *C : LoopDirective->counters()) {
1557  SIMDLCVs.insert(
1558  cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
1559  }
1560  }
1561  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1562  auto CurPrivate = C->privates().begin();
1563  for (const Expr *E : C->varlists()) {
1564  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1565  const auto *PrivateVD =
1566  cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
1567  if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
1568  bool IsRegistered = PrivateScope.addPrivate(VD, [this, PrivateVD]() {
1569  // Emit private VarDecl with copy init.
1570  EmitVarDecl(*PrivateVD);
1571  return GetAddrOfLocalVar(PrivateVD);
1572  });
1573  assert(IsRegistered && "linear var already registered as private");
1574  // Silence the warning about unused variable.
1575  (void)IsRegistered;
1576  } else {
1577  EmitVarDecl(*PrivateVD);
1578  }
1579  ++CurPrivate;
1580  }
1581  }
1582 }
1583 
1585  const OMPExecutableDirective &D,
1586  bool IsMonotonic) {
1587  if (!CGF.HaveInsertPoint())
1588  return;
1589  if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
1590  RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
1591  /*ignoreResult=*/true);
1592  auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1593  CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1594  // In presence of finite 'safelen', it may be unsafe to mark all
1595  // the memory instructions parallel, because loop-carried
1596  // dependences of 'safelen' iterations are possible.
1597  if (!IsMonotonic)
1599  } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
1600  RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
1601  /*ignoreResult=*/true);
1602  auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1603  CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1604  // In presence of finite 'safelen', it may be unsafe to mark all
1605  // the memory instructions parallel, because loop-carried
1606  // dependences of 'safelen' iterations are possible.
1607  CGF.LoopStack.setParallel(/*Enable=*/false);
1608  }
1609 }
1610 
1612  bool IsMonotonic) {
1613  // Walk clauses and process safelen/lastprivate.
1614  LoopStack.setParallel(!IsMonotonic);
1615  LoopStack.setVectorizeEnable();
1616  emitSimdlenSafelenClause(*this, D, IsMonotonic);
1617 }
1618 
1620  const OMPLoopDirective &D,
1621  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1622  if (!HaveInsertPoint())
1623  return;
1624  llvm::BasicBlock *DoneBB = nullptr;
1625  auto IC = D.counters().begin();
1626  auto IPC = D.private_counters().begin();
1627  for (const Expr *F : D.finals()) {
1628  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
1629  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
1630  const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
1631  if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
1632  OrigVD->hasGlobalStorage() || CED) {
1633  if (!DoneBB) {
1634  if (llvm::Value *Cond = CondGen(*this)) {
1635  // If the first post-update expression is found, emit conditional
1636  // block if it was requested.
1637  llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then");
1638  DoneBB = createBasicBlock(".omp.final.done");
1639  Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1640  EmitBlock(ThenBB);
1641  }
1642  }
1643  Address OrigAddr = Address::invalid();
1644  if (CED) {
1645  OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();
1646  } else {
1647  DeclRefExpr DRE(const_cast<VarDecl *>(PrivateVD),
1648  /*RefersToEnclosingVariableOrCapture=*/false,
1649  (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
1650  OrigAddr = EmitLValue(&DRE).getAddress();
1651  }
1652  OMPPrivateScope VarScope(*this);
1653  VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1654  (void)VarScope.Privatize();
1655  EmitIgnoredExpr(F);
1656  }
1657  ++IC;
1658  ++IPC;
1659  }
1660  if (DoneBB)
1661  EmitBlock(DoneBB, /*IsFinished=*/true);
1662 }
1663 
1665  const OMPLoopDirective &S,
1667  CGF.EmitOMPLoopBody(S, LoopExit);
1668  CGF.EmitStopPoint(&S);
1669 }
1670 
1671 /// Emit a helper variable and return corresponding lvalue.
1673  const DeclRefExpr *Helper) {
1674  auto VDecl = cast<VarDecl>(Helper->getDecl());
1675  CGF.EmitVarDecl(*VDecl);
1676  return CGF.EmitLValue(Helper);
1677 }
1678 
1680  PrePostActionTy &Action) {
1681  Action.Enter(CGF);
1683  "Expected simd directive");
1684  OMPLoopScope PreInitScope(CGF, S);
1685  // if (PreCond) {
1686  // for (IV in 0..LastIteration) BODY;
1687  // <Final counter/linear vars updates>;
1688  // }
1689  //
1693  (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));
1694  (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));
1695  }
1696 
1697  // Emit: if (PreCond) - begin.
1698  // If the condition constant folds and can be elided, avoid emitting the
1699  // whole loop.
1700  bool CondConstant;
1701  llvm::BasicBlock *ContBlock = nullptr;
1702  if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
1703  if (!CondConstant)
1704  return;
1705  } else {
1706  llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");
1707  ContBlock = CGF.createBasicBlock("simd.if.end");
1708  emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
1709  CGF.getProfileCount(&S));
1710  CGF.EmitBlock(ThenBlock);
1711  CGF.incrementProfileCounter(&S);
1712  }
1713 
1714  // Emit the loop iteration variable.
1715  const Expr *IVExpr = S.getIterationVariable();
1716  const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
1717  CGF.EmitVarDecl(*IVDecl);
1718  CGF.EmitIgnoredExpr(S.getInit());
1719 
1720  // Emit the iterations count variable.
1721  // If it is not a variable, Sema decided to calculate iterations count on
1722  // each iteration (e.g., it is foldable into a constant).
1723  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
1724  CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
1725  // Emit calculation of the iterations count.
1727  }
1728 
1729  CGF.EmitOMPSimdInit(S);
1730 
1731  emitAlignedClause(CGF, S);
1732  (void)CGF.EmitOMPLinearClauseInit(S);
1733  {
1734  CodeGenFunction::OMPPrivateScope LoopScope(CGF);
1735  CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
1736  CGF.EmitOMPLinearClause(S, LoopScope);
1737  CGF.EmitOMPPrivateClause(S, LoopScope);
1738  CGF.EmitOMPReductionClauseInit(S, LoopScope);
1739  bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
1740  (void)LoopScope.Privatize();
1741  CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
1742  S.getInc(),
1743  [&S](CodeGenFunction &CGF) {
1745  CGF.EmitStopPoint(&S);
1746  },
1747  [](CodeGenFunction &) {});
1748  CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; });
1749  // Emit final copy of the lastprivate variables at the end of loops.
1750  if (HasLastprivateClause)
1751  CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
1752  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
1754  [](CodeGenFunction &) { return nullptr; });
1755  }
1756  CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; });
1757  // Emit: if (PreCond) - end.
1758  if (ContBlock) {
1759  CGF.EmitBranch(ContBlock);
1760  CGF.EmitBlock(ContBlock, true);
1761  }
1762 }
1763 
1765  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1766  emitOMPSimdRegion(CGF, S, Action);
1767  };
1768  OMPLexicalScope Scope(*this, S, OMPD_unknown);
1769  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
1770 }
1771 
1772 void CodeGenFunction::EmitOMPOuterLoop(
1773  bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
1775  const CodeGenFunction::OMPLoopArguments &LoopArgs,
1776  const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
1777  const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
1778  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1779 
1780  const Expr *IVExpr = S.getIterationVariable();
1781  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1782  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1783 
1784  JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
1785 
1786  // Start the loop with a block that tests the condition.
1787  llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond");
1788  EmitBlock(CondBlock);
1789  const SourceRange R = S.getSourceRange();
1790  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1791  SourceLocToDebugLoc(R.getEnd()));
1792 
1793  llvm::Value *BoolCondVal = nullptr;
1794  if (!DynamicOrOrdered) {
1795  // UB = min(UB, GlobalUB) or
1796  // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
1797  // 'distribute parallel for')
1798  EmitIgnoredExpr(LoopArgs.EUB);
1799  // IV = LB
1800  EmitIgnoredExpr(LoopArgs.Init);
1801  // IV < UB
1802  BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
1803  } else {
1804  BoolCondVal =
1805  RT.emitForNext(*this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
1806  LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
1807  }
1808 
1809  // If there are any cleanups between here and the loop-exit scope,
1810  // create a block to stage a loop exit along.
1811  llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1812  if (LoopScope.requiresCleanups())
1813  ExitBlock = createBasicBlock("omp.dispatch.cleanup");
1814 
1815  llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body");
1816  Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
1817  if (ExitBlock != LoopExit.getBlock()) {
1818  EmitBlock(ExitBlock);
1819  EmitBranchThroughCleanup(LoopExit);
1820  }
1821  EmitBlock(LoopBody);
1822 
1823  // Emit "IV = LB" (in case of static schedule, we have already calculated new
1824  // LB for loop condition and emitted it above).
1825  if (DynamicOrOrdered)
1826  EmitIgnoredExpr(LoopArgs.Init);
1827 
1828  // Create a block for the increment.
1829  JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
1830  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1831 
1832  // Generate !llvm.loop.parallel metadata for loads and stores for loops
1833  // with dynamic/guided scheduling and without ordered clause.
1835  LoopStack.setParallel(!IsMonotonic);
1836  else
1837  EmitOMPSimdInit(S, IsMonotonic);
1838 
1839  SourceLocation Loc = S.getBeginLoc();
1840 
1841  // when 'distribute' is not combined with a 'for':
1842  // while (idx <= UB) { BODY; ++idx; }
1843  // when 'distribute' is combined with a 'for'
1844  // (e.g. 'distribute parallel for')
1845  // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
1846  EmitOMPInnerLoop(
1847  S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
1848  [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
1849  CodeGenLoop(CGF, S, LoopExit);
1850  },
1851  [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
1852  CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
1853  });
1854 
1855  EmitBlock(Continue.getBlock());
1856  BreakContinueStack.pop_back();
1857  if (!DynamicOrOrdered) {
1858  // Emit "LB = LB + Stride", "UB = UB + Stride".
1859  EmitIgnoredExpr(LoopArgs.NextLB);
1860  EmitIgnoredExpr(LoopArgs.NextUB);
1861  }
1862 
1863  EmitBranch(CondBlock);
1864  LoopStack.pop();
1865  // Emit the fall-through block.
1866  EmitBlock(LoopExit.getBlock());
1867 
1868  // Tell the runtime we are done.
1869  auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) {
1870  if (!DynamicOrOrdered)
1871  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
1872  S.getDirectiveKind());
1873  };
1874  OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
1875 }
1876 
1877 void CodeGenFunction::EmitOMPForOuterLoop(
1878  const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
1879  const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
1880  const OMPLoopArguments &LoopArgs,
1881  const CodeGenDispatchBoundsTy &CGDispatchBounds) {
1882  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1883 
1884  // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
1885  const bool DynamicOrOrdered =
1886  Ordered || RT.isDynamic(ScheduleKind.Schedule);
1887 
1888  assert((Ordered ||
1889  !RT.isStaticNonchunked(ScheduleKind.Schedule,
1890  LoopArgs.Chunk != nullptr)) &&
1891  "static non-chunked schedule does not need outer loop");
1892 
1893  // Emit outer loop.
1894  //
1895  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1896  // When schedule(dynamic,chunk_size) is specified, the iterations are
1897  // distributed to threads in the team in chunks as the threads request them.
1898  // Each thread executes a chunk of iterations, then requests another chunk,
1899  // until no chunks remain to be distributed. Each chunk contains chunk_size
1900  // iterations, except for the last chunk to be distributed, which may have
1901  // fewer iterations. When no chunk_size is specified, it defaults to 1.
1902  //
1903  // When schedule(guided,chunk_size) is specified, the iterations are assigned
1904  // to threads in the team in chunks as the executing threads request them.
1905  // Each thread executes a chunk of iterations, then requests another chunk,
1906  // until no chunks remain to be assigned. For a chunk_size of 1, the size of
1907  // each chunk is proportional to the number of unassigned iterations divided
1908  // by the number of threads in the team, decreasing to 1. For a chunk_size
1909  // with value k (greater than 1), the size of each chunk is determined in the
1910  // same way, with the restriction that the chunks do not contain fewer than k
1911  // iterations (except for the last chunk to be assigned, which may have fewer
1912  // than k iterations).
1913  //
1914  // When schedule(auto) is specified, the decision regarding scheduling is
1915  // delegated to the compiler and/or runtime system. The programmer gives the
1916  // implementation the freedom to choose any possible mapping of iterations to
1917  // threads in the team.
1918  //
1919  // When schedule(runtime) is specified, the decision regarding scheduling is
1920  // deferred until run time, and the schedule and chunk size are taken from the
1921  // run-sched-var ICV. If the ICV is set to auto, the schedule is
1922  // implementation defined
1923  //
1924  // while(__kmpc_dispatch_next(&LB, &UB)) {
1925  // idx = LB;
1926  // while (idx <= UB) { BODY; ++idx;
1927  // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
1928  // } // inner loop
1929  // }
1930  //
1931  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1932  // When schedule(static, chunk_size) is specified, iterations are divided into
1933  // chunks of size chunk_size, and the chunks are assigned to the threads in
1934  // the team in a round-robin fashion in the order of the thread number.
1935  //
1936  // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
1937  // while (idx <= UB) { BODY; ++idx; } // inner loop
1938  // LB = LB + ST;
1939  // UB = UB + ST;
1940  // }
1941  //
1942 
1943  const Expr *IVExpr = S.getIterationVariable();
1944  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1945  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1946 
1947  if (DynamicOrOrdered) {
1948  const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
1949  CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
1950  llvm::Value *LBVal = DispatchBounds.first;
1951  llvm::Value *UBVal = DispatchBounds.second;
1952  CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
1953  LoopArgs.Chunk};
1954  RT.emitForDispatchInit(*this, S.getBeginLoc(), ScheduleKind, IVSize,
1955  IVSigned, Ordered, DipatchRTInputValues);
1956  } else {
1957  CGOpenMPRuntime::StaticRTInput StaticInit(
1958  IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
1959  LoopArgs.ST, LoopArgs.Chunk);
1960  RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
1961  ScheduleKind, StaticInit);
1962  }
1963 
1964  auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
1965  const unsigned IVSize,
1966  const bool IVSigned) {
1967  if (Ordered) {
1968  CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
1969  IVSigned);
1970  }
1971  };
1972 
1973  OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
1974  LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
1975  OuterLoopArgs.IncExpr = S.getInc();
1976  OuterLoopArgs.Init = S.getInit();
1977  OuterLoopArgs.Cond = S.getCond();
1978  OuterLoopArgs.NextLB = S.getNextLowerBound();
1979  OuterLoopArgs.NextUB = S.getNextUpperBound();
1980  EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
1981  emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
1982 }
1983 
1985  const unsigned IVSize, const bool IVSigned) {}
1986 
1987 void CodeGenFunction::EmitOMPDistributeOuterLoop(
1988  OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
1989  OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
1990  const CodeGenLoopTy &CodeGenLoopContent) {
1991 
1992  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1993 
1994  // Emit outer loop.
1995  // Same behavior as a OMPForOuterLoop, except that schedule cannot be
1996  // dynamic
1997  //
1998 
1999  const Expr *IVExpr = S.getIterationVariable();
2000  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2001  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2002 
2003  CGOpenMPRuntime::StaticRTInput StaticInit(
2004  IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
2005  LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
2006  RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, StaticInit);
2007 
2008  // for combined 'distribute' and 'for' the increment expression of distribute
2009  // is store in DistInc. For 'distribute' alone, it is in Inc.
2010  Expr *IncExpr;
2012  IncExpr = S.getDistInc();
2013  else
2014  IncExpr = S.getInc();
2015 
2016  // this routine is shared by 'omp distribute parallel for' and
2017  // 'omp distribute': select the right EUB expression depending on the
2018  // directive
2019  OMPLoopArguments OuterLoopArgs;
2020  OuterLoopArgs.LB = LoopArgs.LB;
2021  OuterLoopArgs.UB = LoopArgs.UB;
2022  OuterLoopArgs.ST = LoopArgs.ST;
2023  OuterLoopArgs.IL = LoopArgs.IL;
2024  OuterLoopArgs.Chunk = LoopArgs.Chunk;
2025  OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2027  : S.getEnsureUpperBound();
2028  OuterLoopArgs.IncExpr = IncExpr;
2029  OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2030  ? S.getCombinedInit()
2031  : S.getInit();
2032  OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2033  ? S.getCombinedCond()
2034  : S.getCond();
2035  OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2037  : S.getNextLowerBound();
2038  OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2040  : S.getNextUpperBound();
2041 
2042  EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
2043  LoopScope, OuterLoopArgs, CodeGenLoopContent,
2045 }
2046 
2047 static std::pair<LValue, LValue>
2049  const OMPExecutableDirective &S) {
2050  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2051  LValue LB =
2052  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2053  LValue UB =
2054  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2055 
2056  // When composing 'distribute' with 'for' (e.g. as in 'distribute
2057  // parallel for') we need to use the 'distribute'
2058  // chunk lower and upper bounds rather than the whole loop iteration
2059  // space. These are parameters to the outlined function for 'parallel'
2060  // and we copy the bounds of the previous schedule into the
2061  // the current ones.
2062  LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
2063  LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
2064  llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
2065  PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());
2066  PrevLBVal = CGF.EmitScalarConversion(
2067  PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
2068  LS.getIterationVariable()->getType(),
2070  llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
2071  PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());
2072  PrevUBVal = CGF.EmitScalarConversion(
2073  PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
2074  LS.getIterationVariable()->getType(),
2076 
2077  CGF.EmitStoreOfScalar(PrevLBVal, LB);
2078  CGF.EmitStoreOfScalar(PrevUBVal, UB);
2079 
2080  return {LB, UB};
2081 }
2082 
2083 /// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
2084 /// we need to use the LB and UB expressions generated by the worksharing
2085 /// code generation support, whereas in non combined situations we would
2086 /// just emit 0 and the LastIteration expression
2087 /// This function is necessary due to the difference of the LB and UB
2088 /// types for the RT emission routines for 'for_static_init' and
2089 /// 'for_dispatch_init'
2090 static std::pair<llvm::Value *, llvm::Value *>
2092  const OMPExecutableDirective &S,
2093  Address LB, Address UB) {
2094  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2095  const Expr *IVExpr = LS.getIterationVariable();
2096  // when implementing a dynamic schedule for a 'for' combined with a
2097  // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
2098  // is not normalized as each team only executes its own assigned
2099  // distribute chunk
2100  QualType IteratorTy = IVExpr->getType();
2101  llvm::Value *LBVal =
2102  CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
2103  llvm::Value *UBVal =
2104  CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
2105  return {LBVal, UBVal};
2106 }
2107 
2109  CodeGenFunction &CGF, const OMPExecutableDirective &S,
2110  llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) {
2111  const auto &Dir = cast<OMPLoopDirective>(S);
2112  LValue LB =
2113  CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
2114  llvm::Value *LBCast = CGF.Builder.CreateIntCast(
2115  CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2116  CapturedVars.push_back(LBCast);
2117  LValue UB =
2118  CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
2119 
2120  llvm::Value *UBCast = CGF.Builder.CreateIntCast(
2121  CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2122  CapturedVars.push_back(UBCast);
2123 }
2124 
2125 static void
2127  const OMPLoopDirective &S,
2129  auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
2130  PrePostActionTy &Action) {
2131  Action.Enter(CGF);
2132  bool HasCancel = false;
2134  if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
2135  HasCancel = D->hasCancel();
2136  else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
2137  HasCancel = D->hasCancel();
2138  else if (const auto *D =
2139  dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
2140  HasCancel = D->hasCancel();
2141  }
2143  HasCancel);
2147  };
2148 
2150  CGF, S,
2151  isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for,
2152  CGInlinedWorksharingLoop,
2154 }
2155 
2158  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2160  S.getDistInc());
2161  };
2162  OMPLexicalScope Scope(*this, S, OMPD_parallel);
2163  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2164 }
2165 
2168  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2170  S.getDistInc());
2171  };
2172  OMPLexicalScope Scope(*this, S, OMPD_parallel);
2173  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2174 }
2175 
2177  const OMPDistributeSimdDirective &S) {
2178  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2180  };
2181  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2182  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2183 }
2184 
2186  CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
2187  // Emit SPMD target parallel for region as a standalone region.
2188  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2189  emitOMPSimdRegion(CGF, S, Action);
2190  };
2191  llvm::Function *Fn;
2192  llvm::Constant *Addr;
2193  // Emit target region as a standalone region.
2194  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
2195  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
2196  assert(Fn && Addr && "Target device function emission failed.");
2197 }
2198 
2200  const OMPTargetSimdDirective &S) {
2201  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2202  emitOMPSimdRegion(CGF, S, Action);
2203  };
2204  emitCommonOMPTargetDirective(*this, S, CodeGen);
2205 }
2206 
2207 namespace {
2208  struct ScheduleKindModifiersTy {
2212  ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
2215  : Kind(Kind), M1(M1), M2(M2) {}
2216  };
2217 } // namespace
2218 
2220  const OMPLoopDirective &S, Expr *EUB,
2221  const CodeGenLoopBoundsTy &CodeGenLoopBounds,
2222  const CodeGenDispatchBoundsTy &CGDispatchBounds) {
2223  // Emit the loop iteration variable.
2224  const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
2225  const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
2226  EmitVarDecl(*IVDecl);
2227 
2228  // Emit the iterations count variable.
2229  // If it is not a variable, Sema decided to calculate iterations count on each
2230  // iteration (e.g., it is foldable into a constant).
2231  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2232  EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2233  // Emit calculation of the iterations count.
2234  EmitIgnoredExpr(S.getCalcLastIteration());
2235  }
2236 
2237  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
2238 
2239  bool HasLastprivateClause;
2240  // Check pre-condition.
2241  {
2242  OMPLoopScope PreInitScope(*this, S);
2243  // Skip the entire loop if we don't meet the precondition.
2244  // If the condition constant folds and can be elided, avoid emitting the
2245  // whole loop.
2246  bool CondConstant;
2247  llvm::BasicBlock *ContBlock = nullptr;
2248  if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2249  if (!CondConstant)
2250  return false;
2251  } else {
2252  llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
2253  ContBlock = createBasicBlock("omp.precond.end");
2254  emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
2255  getProfileCount(&S));
2256  EmitBlock(ThenBlock);
2257  incrementProfileCounter(&S);
2258  }
2259 
2260  RunCleanupsScope DoacrossCleanupScope(*this);
2261  bool Ordered = false;
2262  if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
2263  if (OrderedClause->getNumForLoops())
2264  RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations());
2265  else
2266  Ordered = true;
2267  }
2268 
2269  llvm::DenseSet<const Expr *> EmittedFinals;
2270  emitAlignedClause(*this, S);
2271  bool HasLinears = EmitOMPLinearClauseInit(S);
2272  // Emit helper vars inits.
2273 
2274  std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
2275  LValue LB = Bounds.first;
2276  LValue UB = Bounds.second;
2277  LValue ST =
2278  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
2279  LValue IL =
2280  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
2281 
2282  // Emit 'then' code.
2283  {
2284  OMPPrivateScope LoopScope(*this);
2285  if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
2286  // Emit implicit barrier to synchronize threads and avoid data races on
2287  // initialization of firstprivate variables and post-update of
2288  // lastprivate variables.
2289  CGM.getOpenMPRuntime().emitBarrierCall(
2290  *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
2291  /*ForceSimpleCall=*/true);
2292  }
2293  EmitOMPPrivateClause(S, LoopScope);
2294  HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
2295  EmitOMPReductionClauseInit(S, LoopScope);
2296  EmitOMPPrivateLoopCounters(S, LoopScope);
2297  EmitOMPLinearClause(S, LoopScope);
2298  (void)LoopScope.Privatize();
2299 
2300  // Detect the loop schedule kind and chunk.
2301  llvm::Value *Chunk = nullptr;
2302  OpenMPScheduleTy ScheduleKind;
2303  if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
2304  ScheduleKind.Schedule = C->getScheduleKind();
2305  ScheduleKind.M1 = C->getFirstScheduleModifier();
2306  ScheduleKind.M2 = C->getSecondScheduleModifier();
2307  if (const Expr *Ch = C->getChunkSize()) {
2308  Chunk = EmitScalarExpr(Ch);
2309  Chunk = EmitScalarConversion(Chunk, Ch->getType(),
2311  S.getBeginLoc());
2312  }
2313  }
2314  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2315  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2316  // OpenMP 4.5, 2.7.1 Loop Construct, Description.
2317  // If the static schedule kind is specified or if the ordered clause is
2318  // specified, and if no monotonic modifier is specified, the effect will
2319  // be as if the monotonic modifier was specified.
2320  if (RT.isStaticNonchunked(ScheduleKind.Schedule,
2321  /* Chunked */ Chunk != nullptr) &&
2322  !Ordered) {
2324  EmitOMPSimdInit(S, /*IsMonotonic=*/true);
2325  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2326  // When no chunk_size is specified, the iteration space is divided into
2327  // chunks that are approximately equal in size, and at most one chunk is
2328  // distributed to each thread. Note that the size of the chunks is
2329  // unspecified in this case.
2330  CGOpenMPRuntime::StaticRTInput StaticInit(
2331  IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
2332  UB.getAddress(), ST.getAddress());
2333  RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
2334  ScheduleKind, StaticInit);
2335  JumpDest LoopExit =
2336  getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
2337  // UB = min(UB, GlobalUB);
2338  EmitIgnoredExpr(S.getEnsureUpperBound());
2339  // IV = LB;
2340  EmitIgnoredExpr(S.getInit());
2341  // while (idx <= UB) { BODY; ++idx; }
2342  EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
2343  S.getInc(),
2344  [&S, LoopExit](CodeGenFunction &CGF) {
2345  CGF.EmitOMPLoopBody(S, LoopExit);
2346  CGF.EmitStopPoint(&S);
2347  },
2348  [](CodeGenFunction &) {});
2349  EmitBlock(LoopExit.getBlock());
2350  // Tell the runtime we are done.
2351  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2352  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2353  S.getDirectiveKind());
2354  };
2355  OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
2356  } else {
2357  const bool IsMonotonic =
2358  Ordered || ScheduleKind.Schedule == OMPC_SCHEDULE_static ||
2359  ScheduleKind.Schedule == OMPC_SCHEDULE_unknown ||
2360  ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
2361  ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
2362  // Emit the outer loop, which requests its work chunk [LB..UB] from
2363  // runtime and runs the inner loop to process it.
2364  const OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),
2365  ST.getAddress(), IL.getAddress(),
2366  Chunk, EUB);
2367  EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
2368  LoopArguments, CGDispatchBounds);
2369  }
2371  EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
2372  return CGF.Builder.CreateIsNotNull(
2373  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2374  });
2375  }
2376  EmitOMPReductionClauseFinal(
2377  S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind())
2378  ? /*Parallel and Simd*/ OMPD_parallel_for_simd
2379  : /*Parallel only*/ OMPD_parallel);
2380  // Emit post-update of the reduction variables if IsLastIter != 0.
2382  *this, S, [IL, &S](CodeGenFunction &CGF) {
2383  return CGF.Builder.CreateIsNotNull(
2384  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2385  });
2386  // Emit final copy of the lastprivate variables if IsLastIter != 0.
2387  if (HasLastprivateClause)
2388  EmitOMPLastprivateClauseFinal(
2390  Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
2391  }
2392  EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) {
2393  return CGF.Builder.CreateIsNotNull(
2394  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2395  });
2396  DoacrossCleanupScope.ForceCleanup();
2397  // We're now done with the loop, so jump to the continuation block.
2398  if (ContBlock) {
2399  EmitBranch(ContBlock);
2400  EmitBlock(ContBlock, /*IsFinished=*/true);
2401  }
2402  }
2403  return HasLastprivateClause;
2404 }
2405 
2406 /// The following two functions generate expressions for the loop lower
2407 /// and upper bounds in case of static and dynamic (dispatch) schedule
2408 /// of the associated 'for' or 'distribute' loop.
2409 static std::pair<LValue, LValue>
2411  const auto &LS = cast<OMPLoopDirective>(S);
2412  LValue LB =
2413  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2414  LValue UB =
2415  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2416  return {LB, UB};
2417 }
2418 
2419 /// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
2420 /// consider the lower and upper bound expressions generated by the
2421 /// worksharing loop support, but we use 0 and the iteration space size as
2422 /// constants
2423 static std::pair<llvm::Value *, llvm::Value *>
2425  Address LB, Address UB) {
2426  const auto &LS = cast<OMPLoopDirective>(S);
2427  const Expr *IVExpr = LS.getIterationVariable();
2428  const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
2429  llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
2430  llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
2431  return {LBVal, UBVal};
2432 }
2433 
2435  bool HasLastprivates = false;
2436  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2437  PrePostActionTy &) {
2438  OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel());
2439  HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2442  };
2443  {
2444  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2445  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
2446  S.hasCancel());
2447  }
2448 
2449  // Emit an implicit barrier at the end.
2450  if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
2451  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
2452 }
2453 
2455  bool HasLastprivates = false;
2456  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2457  PrePostActionTy &) {
2458  HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2461  };
2462  {
2463  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2464  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2465  }
2466 
2467  // Emit an implicit barrier at the end.
2468  if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
2469  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
2470 }
2471 
2473  const Twine &Name,
2474  llvm::Value *Init = nullptr) {
2475  LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
2476  if (Init)
2477  CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
2478  return LVal;
2479 }
2480 
2481 void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
2483  const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
2484  bool HasLastprivates = false;
2485  auto &&CodeGen = [&S, CapturedStmt, CS,
2486  &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
2487  ASTContext &C = CGF.getContext();
2488  QualType KmpInt32Ty =
2489  C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2490  // Emit helper vars inits.
2491  LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
2492  CGF.Builder.getInt32(0));
2493  llvm::ConstantInt *GlobalUBVal = CS != nullptr
2494  ? CGF.Builder.getInt32(CS->size() - 1)
2495  : CGF.Builder.getInt32(0);
2496  LValue UB =
2497  createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
2498  LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
2499  CGF.Builder.getInt32(1));
2500  LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
2501  CGF.Builder.getInt32(0));
2502  // Loop counter.
2503  LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
2504  OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
2505  CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
2506  OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
2507  CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
2508  // Generate condition for loop.
2509  BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue,
2511  // Increment for loop counter.
2512  UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary,
2513  S.getBeginLoc(), true);
2514  auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
2515  // Iterate through all sections and emit a switch construct:
2516  // switch (IV) {
2517  // case 0:
2518  // <SectionStmt[0]>;
2519  // break;
2520  // ...
2521  // case <NumSection> - 1:
2522  // <SectionStmt[<NumSection> - 1]>;
2523  // break;
2524  // }
2525  // .omp.sections.exit:
2526  llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
2527  llvm::SwitchInst *SwitchStmt =
2528  CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
2529  ExitBB, CS == nullptr ? 1 : CS->size());
2530  if (CS) {
2531  unsigned CaseNumber = 0;
2532  for (const Stmt *SubStmt : CS->children()) {
2533  auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2534  CGF.EmitBlock(CaseBB);
2535  SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
2536  CGF.EmitStmt(SubStmt);
2537  CGF.EmitBranch(ExitBB);
2538  ++CaseNumber;
2539  }
2540  } else {
2541  llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");
2542  CGF.EmitBlock(CaseBB);
2543  SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
2544  CGF.EmitStmt(CapturedStmt);
2545  CGF.EmitBranch(ExitBB);
2546  }
2547  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2548  };
2549 
2550  CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2551  if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
2552  // Emit implicit barrier to synchronize threads and avoid data races on
2553  // initialization of firstprivate variables and post-update of lastprivate
2554  // variables.
2555  CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2556  CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
2557  /*ForceSimpleCall=*/true);
2558  }
2559  CGF.EmitOMPPrivateClause(S, LoopScope);
2560  HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2561  CGF.EmitOMPReductionClauseInit(S, LoopScope);
2562  (void)LoopScope.Privatize();
2563 
2564  // Emit static non-chunked loop.
2565  OpenMPScheduleTy ScheduleKind;
2566  ScheduleKind.Schedule = OMPC_SCHEDULE_static;
2567  CGOpenMPRuntime::StaticRTInput StaticInit(
2568  /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(),
2569  LB.getAddress(), UB.getAddress(), ST.getAddress());
2570  CGF.CGM.getOpenMPRuntime().emitForStaticInit(
2571  CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit);
2572  // UB = min(UB, GlobalUB);
2573  llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
2574  llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
2575  CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
2576  CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
2577  // IV = LB;
2578  CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
2579  // while (idx <= UB) { BODY; ++idx; }
2580  CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen,
2581  [](CodeGenFunction &) {});
2582  // Tell the runtime we are done.
2583  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2584  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2585  S.getDirectiveKind());
2586  };
2587  CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
2588  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
2589  // Emit post-update of the reduction variables if IsLastIter != 0.
2590  emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) {
2591  return CGF.Builder.CreateIsNotNull(
2592  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2593  });
2594 
2595  // Emit final copy of the lastprivate variables if IsLastIter != 0.
2596  if (HasLastprivates)
2598  S, /*NoFinals=*/false,
2599  CGF.Builder.CreateIsNotNull(
2600  CGF.EmitLoadOfScalar(IL, S.getBeginLoc())));
2601  };
2602 
2603  bool HasCancel = false;
2604  if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
2605  HasCancel = OSD->hasCancel();
2606  else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
2607  HasCancel = OPSD->hasCancel();
2608  OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
2609  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
2610  HasCancel);
2611  // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
2612  // clause. Otherwise the barrier will be generated by the codegen for the
2613  // directive.
2614  if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
2615  // Emit implicit barrier to synchronize threads and avoid data races on
2616  // initialization of firstprivate variables.
2617  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
2618  OMPD_unknown);
2619  }
2620 }
2621 
2623  {
2624  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2625  EmitSections(S);
2626  }
2627  // Emit an implicit barrier at the end.
2628  if (!S.getSingleClause<OMPNowaitClause>()) {
2629  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
2630  OMPD_sections);
2631  }
2632 }
2633 
2635  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2637  };
2638  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2639  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen,
2640  S.hasCancel());
2641 }
2642 
2644  llvm::SmallVector<const Expr *, 8> CopyprivateVars;
2647  llvm::SmallVector<const Expr *, 8> AssignmentOps;
2648  // Check if there are any 'copyprivate' clauses associated with this
2649  // 'single' construct.
2650  // Build a list of copyprivate variables along with helper expressions
2651  // (<source>, <destination>, <destination>=<source> expressions)
2652  for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
2653  CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
2654  DestExprs.append(C->destination_exprs().begin(),
2655  C->destination_exprs().end());
2656  SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
2657  AssignmentOps.append(C->assignment_ops().begin(),
2658  C->assignment_ops().end());
2659  }
2660  // Emit code for 'single' region along with 'copyprivate' clauses
2661  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2662  Action.Enter(CGF);
2663  OMPPrivateScope SingleScope(CGF);
2664  (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
2665  CGF.EmitOMPPrivateClause(S, SingleScope);
2666  (void)SingleScope.Privatize();
2668  };
2669  {
2670  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2671  CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getBeginLoc(),
2672  CopyprivateVars, DestExprs,
2673  SrcExprs, AssignmentOps);
2674  }
2675  // Emit an implicit barrier at the end (to avoid data race on firstprivate
2676  // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
2677  if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
2678  CGM.getOpenMPRuntime().emitBarrierCall(
2679  *this, S.getBeginLoc(),
2680  S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
2681  }
2682 }
2683 
2685  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2686  Action.Enter(CGF);
2688  };
2689  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2690  CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
2691 }
2692 
2694  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2695  Action.Enter(CGF);
2697  };
2698  const Expr *Hint = nullptr;
2699  if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
2700  Hint = HintClause->getHint();
2701  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2702  CGM.getOpenMPRuntime().emitCriticalRegion(*this,
2704  CodeGen, S.getBeginLoc(), Hint);
2705 }
2706 
2708  const OMPParallelForDirective &S) {
2709  // Emit directive as a combined directive that consists of two implicit
2710  // directives: 'parallel' with 'for' directive.
2711  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2712  Action.Enter(CGF);
2713  OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel());
2716  };
2717  emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
2719 }
2720 
2722  const OMPParallelForSimdDirective &S) {
2723  // Emit directive as a combined directive that consists of two implicit
2724  // directives: 'parallel' with 'for' directive.
2725  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2726  Action.Enter(CGF);
2729  };
2730  emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen,
2732 }
2733 
2735  const OMPParallelSectionsDirective &S) {
2736  // Emit directive as a combined directive that consists of two implicit
2737  // directives: 'parallel' with 'sections' directive.
2738  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2739  Action.Enter(CGF);
2740  CGF.EmitSections(S);
2741  };
2742  emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
2744 }
2745 
2747  const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
2748  const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
2749  OMPTaskDataTy &Data) {
2750  // Emit outlined function for task construct.
2751  const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
2752  auto I = CS->getCapturedDecl()->param_begin();
2753  auto PartId = std::next(I);
2754  auto TaskT = std::next(I, 4);
2755  // Check if the task is final
2756  if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
2757  // If the condition constant folds and can be elided, try to avoid emitting
2758  // the condition and the dead arm of the if/else.
2759  const Expr *Cond = Clause->getCondition();
2760  bool CondConstant;
2761  if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
2762  Data.Final.setInt(CondConstant);
2763  else
2764  Data.Final.setPointer(EvaluateExprAsBool(Cond));
2765  } else {
2766  // By default the task is not final.
2767  Data.Final.setInt(/*IntVal=*/false);
2768  }
2769  // Check if the task has 'priority' clause.
2770  if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
2771  const Expr *Prio = Clause->getPriority();
2772  Data.Priority.setInt(/*IntVal=*/true);
2773  Data.Priority.setPointer(EmitScalarConversion(
2774  EmitScalarExpr(Prio), Prio->getType(),
2775  getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
2776  Prio->getExprLoc()));
2777  }
2778  // The first function argument for tasks is a thread id, the second one is a
2779  // part id (0 for tied tasks, >=0 for untied task).
2780  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
2781  // Get list of private variables.
2782  for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
2783  auto IRef = C->varlist_begin();
2784  for (const Expr *IInit : C->private_copies()) {
2785  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2786  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2787  Data.PrivateVars.push_back(*IRef);
2788  Data.PrivateCopies.push_back(IInit);
2789  }
2790  ++IRef;
2791  }
2792  }
2793  EmittedAsPrivate.clear();
2794  // Get list of firstprivate variables.
2795  for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
2796  auto IRef = C->varlist_begin();
2797  auto IElemInitRef = C->inits().begin();
2798  for (const Expr *IInit : C->private_copies()) {
2799  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2800  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2801  Data.FirstprivateVars.push_back(*IRef);
2802  Data.FirstprivateCopies.push_back(IInit);
2803  Data.FirstprivateInits.push_back(*IElemInitRef);
2804  }
2805  ++IRef;
2806  ++IElemInitRef;
2807  }
2808  }
2809  // Get list of lastprivate variables (for taskloops).
2810  llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
2811  for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
2812  auto IRef = C->varlist_begin();
2813  auto ID = C->destination_exprs().begin();
2814  for (const Expr *IInit : C->private_copies()) {
2815  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2816  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2817  Data.LastprivateVars.push_back(*IRef);
2818  Data.LastprivateCopies.push_back(IInit);
2819  }
2820  LastprivateDstsOrigs.insert(
2821  {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
2822  cast<DeclRefExpr>(*IRef)});
2823  ++IRef;
2824  ++ID;
2825  }
2826  }
2829  for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
2830  auto IPriv = C->privates().begin();
2831  auto IRed = C->reduction_ops().begin();
2832  auto ILHS = C->lhs_exprs().begin();
2833  auto IRHS = C->rhs_exprs().begin();
2834  for (const Expr *Ref : C->varlists()) {
2835  Data.ReductionVars.emplace_back(Ref);
2836  Data.ReductionCopies.emplace_back(*IPriv);
2837  Data.ReductionOps.emplace_back(*IRed);
2838  LHSs.emplace_back(*ILHS);
2839  RHSs.emplace_back(*IRHS);
2840  std::advance(IPriv, 1);
2841  std::advance(IRed, 1);
2842  std::advance(ILHS, 1);
2843  std::advance(IRHS, 1);
2844  }
2845  }
2846  Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
2847  *this, S.getBeginLoc(), LHSs, RHSs, Data);
2848  // Build list of dependences.
2849  for (const auto *C : S.getClausesOfKind<OMPDependClause>())
2850  for (const Expr *IRef : C->varlists())
2851  Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
2852  auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
2853  CapturedRegion](CodeGenFunction &CGF,
2854  PrePostActionTy &Action) {
2855  // Set proper addresses for generated private copies.
2856  OMPPrivateScope Scope(CGF);
2857  if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
2858  !Data.LastprivateVars.empty()) {
2859  enum { PrivatesParam = 2, CopyFnParam = 3 };
2860  llvm::Value *CopyFn = CGF.Builder.CreateLoad(
2861  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
2862  llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
2863  CS->getCapturedDecl()->getParam(PrivatesParam)));
2864  // Map privates.
2867  CallArgs.push_back(PrivatesPtr);
2868  for (const Expr *E : Data.PrivateVars) {
2869  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2870  Address PrivatePtr = CGF.CreateMemTemp(
2871  CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
2872  PrivatePtrs.emplace_back(VD, PrivatePtr);
2873  CallArgs.push_back(PrivatePtr.getPointer());
2874  }
2875  for (const Expr *E : Data.FirstprivateVars) {
2876  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2877  Address PrivatePtr =
2878  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2879  ".firstpriv.ptr.addr");
2880  PrivatePtrs.emplace_back(VD, PrivatePtr);
2881  CallArgs.push_back(PrivatePtr.getPointer());
2882  }
2883  for (const Expr *E : Data.LastprivateVars) {
2884  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2885  Address PrivatePtr =
2886  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2887  ".lastpriv.ptr.addr");
2888  PrivatePtrs.emplace_back(VD, PrivatePtr);
2889  CallArgs.push_back(PrivatePtr.getPointer());
2890  }
2891  CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
2892  CopyFn, CallArgs);
2893  for (const auto &Pair : LastprivateDstsOrigs) {
2894  const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
2895  DeclRefExpr DRE(
2896  const_cast<VarDecl *>(OrigVD),
2897  /*RefersToEnclosingVariableOrCapture=*/CGF.CapturedStmtInfo->lookup(
2898  OrigVD) != nullptr,
2899  Pair.second->getType(), VK_LValue, Pair.second->getExprLoc());
2900  Scope.addPrivate(Pair.first, [&CGF, &DRE]() {
2901  return CGF.EmitLValue(&DRE).getAddress();
2902  });
2903  }
2904  for (const auto &Pair : PrivatePtrs) {
2905  Address Replacement(CGF.Builder.CreateLoad(Pair.second),
2906  CGF.getContext().getDeclAlign(Pair.first));
2907  Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
2908  }
2909  }
2910  if (Data.Reductions) {
2911  OMPLexicalScope LexScope(CGF, S, CapturedRegion);
2913  Data.ReductionOps);
2914  llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
2915  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
2916  for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
2917  RedCG.emitSharedLValue(CGF, Cnt);
2918  RedCG.emitAggregateType(CGF, Cnt);
2919  // FIXME: This must removed once the runtime library is fixed.
2920  // Emit required threadprivate variables for
2921  // initilizer/combiner/finalizer.
2922  CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
2923  RedCG, Cnt);
2924  Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
2925  CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
2926  Replacement =
2927  Address(CGF.EmitScalarConversion(
2928  Replacement.getPointer(), CGF.getContext().VoidPtrTy,
2929  CGF.getContext().getPointerType(
2930  Data.ReductionCopies[Cnt]->getType()),
2931  Data.ReductionCopies[Cnt]->getExprLoc()),
2932  Replacement.getAlignment());
2933  Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
2934  Scope.addPrivate(RedCG.getBaseDecl(Cnt),
2935  [Replacement]() { return Replacement; });
2936  }
2937  }
2938  // Privatize all private variables except for in_reduction items.
2939  (void)Scope.Privatize();
2940  SmallVector<const Expr *, 4> InRedVars;
2941  SmallVector<const Expr *, 4> InRedPrivs;
2943  SmallVector<const Expr *, 4> TaskgroupDescriptors;
2944  for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
2945  auto IPriv = C->privates().begin();
2946  auto IRed = C->reduction_ops().begin();
2947  auto ITD = C->taskgroup_descriptors().begin();
2948  for (const Expr *Ref : C->varlists()) {
2949  InRedVars.emplace_back(Ref);
2950  InRedPrivs.emplace_back(*IPriv);
2951  InRedOps.emplace_back(*IRed);
2952  TaskgroupDescriptors.emplace_back(*ITD);
2953  std::advance(IPriv, 1);
2954  std::advance(IRed, 1);
2955  std::advance(ITD, 1);
2956  }
2957  }
2958  // Privatize in_reduction items here, because taskgroup descriptors must be
2959  // privatized earlier.
2960  OMPPrivateScope InRedScope(CGF);
2961  if (!InRedVars.empty()) {
2962  ReductionCodeGen RedCG(InRedVars, InRedPrivs, InRedOps);
2963  for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
2964  RedCG.emitSharedLValue(CGF, Cnt);
2965  RedCG.emitAggregateType(CGF, Cnt);
2966  // The taskgroup descriptor variable is always implicit firstprivate and
2967  // privatized already during procoessing of the firstprivates.
2968  // FIXME: This must removed once the runtime library is fixed.
2969  // Emit required threadprivate variables for
2970  // initilizer/combiner/finalizer.
2971  CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
2972  RedCG, Cnt);
2973  llvm::Value *ReductionsPtr =
2974  CGF.EmitLoadOfScalar(CGF.EmitLValue(TaskgroupDescriptors[Cnt]),
2975  TaskgroupDescriptors[Cnt]->getExprLoc());
2976  Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
2977  CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
2978  Replacement = Address(
2979  CGF.EmitScalarConversion(
2980  Replacement.getPointer(), CGF.getContext().VoidPtrTy,
2981  CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
2982  InRedPrivs[Cnt]->getExprLoc()),
2983  Replacement.getAlignment());
2984  Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
2985  InRedScope.addPrivate(RedCG.getBaseDecl(Cnt),
2986  [Replacement]() { return Replacement; });
2987  }
2988  }
2989  (void)InRedScope.Privatize();
2990 
2991  Action.Enter(CGF);
2992  BodyGen(CGF);
2993  };
2994  llvm::Value *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
2995  S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied,
2996  Data.NumberOfParts);
2997  OMPLexicalScope Scope(*this, S);
2998  TaskGen(*this, OutlinedFn, Data);
2999 }
3000 
3001 static ImplicitParamDecl *
3003  QualType Ty, CapturedDecl *CD,
3004  SourceLocation Loc) {
3005  auto *OrigVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3007  auto *OrigRef = DeclRefExpr::Create(
3008  C, NestedNameSpecifierLoc(), SourceLocation(), OrigVD,
3009  /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
3010  auto *PrivateVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3012  auto *PrivateRef = DeclRefExpr::Create(
3013  C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD,
3014  /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
3015  QualType ElemType = C.getBaseElementType(Ty);
3016  auto *InitVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, ElemType,
3018  auto *InitRef = DeclRefExpr::Create(
3019  C, NestedNameSpecifierLoc(), SourceLocation(), InitVD,
3020  /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue);
3021  PrivateVD->setInitStyle(VarDecl::CInit);
3022  PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue,
3023  InitRef, /*BasePath=*/nullptr,
3024  VK_RValue));
3025  Data.FirstprivateVars.emplace_back(OrigRef);
3026  Data.FirstprivateCopies.emplace_back(PrivateRef);
3027  Data.FirstprivateInits.emplace_back(InitRef);
3028  return OrigVD;
3029 }
3030 
3032  const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,
3033  OMPTargetDataInfo &InputInfo) {
3034  // Emit outlined function for task construct.
3035  const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3036  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3037  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3038  auto I = CS->getCapturedDecl()->param_begin();
3039  auto PartId = std::next(I);
3040  auto TaskT = std::next(I, 4);
3041  OMPTaskDataTy Data;
3042  // The task is not final.
3043  Data.Final.setInt(/*IntVal=*/false);
3044  // Get list of firstprivate variables.
3045  for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
3046  auto IRef = C->varlist_begin();
3047  auto IElemInitRef = C->inits().begin();
3048  for (auto *IInit : C->private_copies()) {
3049  Data.FirstprivateVars.push_back(*IRef);
3050  Data.FirstprivateCopies.push_back(IInit);
3051  Data.FirstprivateInits.push_back(*IElemInitRef);
3052  ++IRef;
3053  ++IElemInitRef;
3054  }
3055  }
3056  OMPPrivateScope TargetScope(*this);
3057  VarDecl *BPVD = nullptr;
3058  VarDecl *PVD = nullptr;
3059  VarDecl *SVD = nullptr;
3060  if (InputInfo.NumberOfTargetItems > 0) {
3061  auto *CD = CapturedDecl::Create(
3062  getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0);
3063  llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);
3064  QualType BaseAndPointersType = getContext().getConstantArrayType(
3065  getContext().VoidPtrTy, ArrSize, ArrayType::Normal,
3066  /*IndexTypeQuals=*/0);
3068  getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
3070  getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
3071  QualType SizesType = getContext().getConstantArrayType(
3072  getContext().getSizeType(), ArrSize, ArrayType::Normal,
3073  /*IndexTypeQuals=*/0);
3074  SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD,
3075  S.getBeginLoc());
3076  TargetScope.addPrivate(
3077  BPVD, [&InputInfo]() { return InputInfo.BasePointersArray; });
3078  TargetScope.addPrivate(PVD,
3079  [&InputInfo]() { return InputInfo.PointersArray; });
3080  TargetScope.addPrivate(SVD,
3081  [&InputInfo]() { return InputInfo.SizesArray; });
3082  }
3083  (void)TargetScope.Privatize();
3084  // Build list of dependences.
3085  for (const auto *C : S.getClausesOfKind<OMPDependClause>())
3086  for (const Expr *IRef : C->varlists())
3087  Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
3088  auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD,
3089  &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
3090  // Set proper addresses for generated private copies.
3091  OMPPrivateScope Scope(CGF);
3092  if (!Data.FirstprivateVars.empty()) {
3093  enum { PrivatesParam = 2, CopyFnParam = 3 };
3094  llvm::Value *CopyFn = CGF.Builder.CreateLoad(
3095  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
3096  llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
3097  CS->getCapturedDecl()->getParam(PrivatesParam)));
3098  // Map privates.
3101  CallArgs.push_back(PrivatesPtr);
3102  for (const Expr *E : Data.FirstprivateVars) {
3103  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3104  Address PrivatePtr =
3105  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
3106  ".firstpriv.ptr.addr");
3107  PrivatePtrs.emplace_back(VD, PrivatePtr);
3108  CallArgs.push_back(PrivatePtr.getPointer());
3109  }
3110  CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
3111  CopyFn, CallArgs);
3112  for (const auto &Pair : PrivatePtrs) {
3113  Address Replacement(CGF.Builder.CreateLoad(Pair.second),
3114  CGF.getContext().getDeclAlign(Pair.first));
3115  Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
3116  }
3117  }
3118  // Privatize all private variables except for in_reduction items.
3119  (void)Scope.Privatize();
3120  if (InputInfo.NumberOfTargetItems > 0) {
3121  InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
3122  CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0, CGF.getPointerSize());
3123  InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
3124  CGF.GetAddrOfLocalVar(PVD), /*Index=*/0, CGF.getPointerSize());
3125  InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
3126  CGF.GetAddrOfLocalVar(SVD), /*Index=*/0, CGF.getSizeSize());
3127  }
3128 
3129  Action.Enter(CGF);
3130  OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);
3131  BodyGen(CGF);
3132  };
3133  llvm::Value *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3134  S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true,
3135  Data.NumberOfParts);
3136  llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
3137  IntegerLiteral IfCond(getContext(), TrueOrFalse,
3138  getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3139  SourceLocation());
3140 
3141  CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn,
3142  SharedsTy, CapturedStruct, &IfCond, Data);
3143 }
3144 
3146  // Emit outlined function for task construct.
3147  const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3148  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3149  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3150  const Expr *IfCond = nullptr;
3151  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3152  if (C->getNameModifier() == OMPD_unknown ||
3153  C->getNameModifier() == OMPD_task) {
3154  IfCond = C->getCondition();
3155  break;
3156  }
3157  }
3158 
3159  OMPTaskDataTy Data;
3160  // Check if we should emit tied or untied task.
3161  Data.Tied = !S.getSingleClause<OMPUntiedClause>();
3162  auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
3163  CGF.EmitStmt(CS->getCapturedStmt());
3164  };
3165  auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
3166  IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
3167  const OMPTaskDataTy &Data) {
3168  CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn,
3169  SharedsTy, CapturedStruct, IfCond,
3170  Data);
3171  };
3172  EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data);
3173 }
3174 
3176  const OMPTaskyieldDirective &S) {
3177  CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getBeginLoc());
3178 }
3179 
3181  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_barrier);
3182 }
3183 
3185  CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc());
3186 }
3187 
3189  const OMPTaskgroupDirective &S) {
3190  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3191  Action.Enter(CGF);
3192  if (const Expr *E = S.getReductionRef()) {
3195  OMPTaskDataTy Data;
3196  for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
3197  auto IPriv = C->privates().begin();
3198  auto IRed = C->reduction_ops().begin();
3199  auto ILHS = C->lhs_exprs().begin();
3200  auto IRHS = C->rhs_exprs().begin();
3201  for (const Expr *Ref : C->varlists()) {
3202  Data.ReductionVars.emplace_back(Ref);
3203  Data.ReductionCopies.emplace_back(*IPriv);
3204  Data.ReductionOps.emplace_back(*IRed);
3205  LHSs.emplace_back(*ILHS);
3206  RHSs.emplace_back(*IRHS);
3207  std::advance(IPriv, 1);
3208  std::advance(IRed, 1);
3209  std::advance(ILHS, 1);
3210  std::advance(IRHS, 1);
3211  }
3212  }
3213  llvm::Value *ReductionDesc =
3214  CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(),
3215  LHSs, RHSs, Data);
3216  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3217  CGF.EmitVarDecl(*VD);
3218  CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),
3219  /*Volatile=*/false, E->getType());
3220  }
3222  };
3223  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3224  CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getBeginLoc());
3225 }
3226 
3228  CGM.getOpenMPRuntime().emitFlush(
3229  *this,
3230  [&S]() -> ArrayRef<const Expr *> {
3231  if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>())
3232  return llvm::makeArrayRef(FlushClause->varlist_begin(),
3233  FlushClause->varlist_end());
3234  return llvm::None;
3235  }(),
3236  S.getBeginLoc());
3237 }
3238 
3240  const CodeGenLoopTy &CodeGenLoop,
3241  Expr *IncExpr) {
3242  // Emit the loop iteration variable.
3243  const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3244  const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3245  EmitVarDecl(*IVDecl);
3246 
3247  // Emit the iterations count variable.
3248  // If it is not a variable, Sema decided to calculate iterations count on each
3249  // iteration (e.g., it is foldable into a constant).
3250  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3251  EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3252  // Emit calculation of the iterations count.
3253  EmitIgnoredExpr(S.getCalcLastIteration());
3254  }
3255 
3256  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3257 
3258  bool HasLastprivateClause = false;
3259  // Check pre-condition.
3260  {
3261  OMPLoopScope PreInitScope(*this, S);
3262  // Skip the entire loop if we don't meet the precondition.
3263  // If the condition constant folds and can be elided, avoid emitting the
3264  // whole loop.
3265  bool CondConstant;
3266  llvm::BasicBlock *ContBlock = nullptr;
3267  if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
3268  if (!CondConstant)
3269  return;
3270  } else {
3271  llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
3272  ContBlock = createBasicBlock("omp.precond.end");
3273  emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
3274  getProfileCount(&S));
3275  EmitBlock(ThenBlock);
3276  incrementProfileCounter(&S);
3277  }
3278 
3279  emitAlignedClause(*this, S);
3280  // Emit 'then' code.
3281  {
3282  // Emit helper vars inits.
3283 
3284  LValue LB = EmitOMPHelperVar(
3285  *this, cast<DeclRefExpr>(
3288  : S.getLowerBoundVariable())));
3289  LValue UB = EmitOMPHelperVar(
3290  *this, cast<DeclRefExpr>(
3293  : S.getUpperBoundVariable())));
3294  LValue ST =
3295  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
3296  LValue IL =
3297  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
3298 
3299  OMPPrivateScope LoopScope(*this);
3300  if (EmitOMPFirstprivateClause(S, LoopScope)) {
3301  // Emit implicit barrier to synchronize threads and avoid data races
3302  // on initialization of firstprivate variables and post-update of
3303  // lastprivate variables.
3304  CGM.getOpenMPRuntime().emitBarrierCall(
3305  *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
3306  /*ForceSimpleCall=*/true);
3307  }
3308  EmitOMPPrivateClause(S, LoopScope);
3312  EmitOMPReductionClauseInit(S, LoopScope);
3313  HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
3314  EmitOMPPrivateLoopCounters(S, LoopScope);
3315  (void)LoopScope.Privatize();
3316 
3317  // Detect the distribute schedule kind and chunk.
3318  llvm::Value *Chunk = nullptr;
3320  if (const auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
3321  ScheduleKind = C->getDistScheduleKind();
3322  if (const Expr *Ch = C->getChunkSize()) {
3323  Chunk = EmitScalarExpr(Ch);
3324  Chunk = EmitScalarConversion(Chunk, Ch->getType(),
3326  S.getBeginLoc());
3327  }
3328  }
3329  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3330  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3331 
3332  // OpenMP [2.10.8, distribute Construct, Description]
3333  // If dist_schedule is specified, kind must be static. If specified,
3334  // iterations are divided into chunks of size chunk_size, chunks are
3335  // assigned to the teams of the league in a round-robin fashion in the
3336  // order of the team number. When no chunk_size is specified, the
3337  // iteration space is divided into chunks that are approximately equal
3338  // in size, and at most one chunk is distributed to each team of the
3339  // league. The size of the chunks is unspecified in this case.
3340  if (RT.isStaticNonchunked(ScheduleKind,
3341  /* Chunked */ Chunk != nullptr)) {
3343  EmitOMPSimdInit(S, /*IsMonotonic=*/true);
3344  CGOpenMPRuntime::StaticRTInput StaticInit(
3345  IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(),
3346  LB.getAddress(), UB.getAddress(), ST.getAddress());
3347  RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind,
3348  StaticInit);
3349  JumpDest LoopExit =
3350  getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
3351  // UB = min(UB, GlobalUB);
3354  : S.getEnsureUpperBound());
3355  // IV = LB;
3357  ? S.getCombinedInit()
3358  : S.getInit());
3359 
3360  const Expr *Cond =
3362  ? S.getCombinedCond()
3363  : S.getCond();
3364 
3365  // for distribute alone, codegen
3366  // while (idx <= UB) { BODY; ++idx; }
3367  // when combined with 'for' (e.g. as in 'distribute parallel for')
3368  // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
3369  EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), Cond, IncExpr,
3370  [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3371  CodeGenLoop(CGF, S, LoopExit);
3372  },
3373  [](CodeGenFunction &) {});
3374  EmitBlock(LoopExit.getBlock());
3375  // Tell the runtime we are done.
3376  RT.emitForStaticFinish(*this, S.getBeginLoc(), S.getDirectiveKind());
3377  } else {
3378  // Emit the outer loop, which requests its work chunk [LB..UB] from
3379  // runtime and runs the inner loop to process it.
3380  const OMPLoopArguments LoopArguments = {
3381  LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(),
3382  Chunk};
3383  EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
3384  CodeGenLoop);
3385  }
3387  EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
3388  return CGF.Builder.CreateIsNotNull(
3389  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3390  });
3391  }
3395  OpenMPDirectiveKind ReductionKind = OMPD_unknown;
3398  ReductionKind = OMPD_parallel_for_simd;
3399  } else if (isOpenMPParallelDirective(S.getDirectiveKind())) {
3400  ReductionKind = OMPD_parallel_for;
3401  } else if (isOpenMPSimdDirective(S.getDirectiveKind())) {
3402  ReductionKind = OMPD_simd;
3403  } else if (!isOpenMPTeamsDirective(S.getDirectiveKind()) &&
3405  llvm_unreachable(
3406  "No reduction clauses is allowed in distribute directive.");
3407  }
3408  EmitOMPReductionClauseFinal(S, ReductionKind);
3409  // Emit post-update of the reduction variables if IsLastIter != 0.
3411  *this, S, [IL, &S](CodeGenFunction &CGF) {
3412  return CGF.Builder.CreateIsNotNull(
3413  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3414  });
3415  }
3416  // Emit final copy of the lastprivate variables if IsLastIter != 0.
3417  if (HasLastprivateClause) {
3418  EmitOMPLastprivateClauseFinal(
3419  S, /*NoFinals=*/false,
3420  Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
3421  }
3422  }
3423 
3424  // We're now done with the loop, so jump to the continuation block.
3425  if (ContBlock) {
3426  EmitBranch(ContBlock);
3427  EmitBlock(ContBlock, true);
3428  }
3429  }
3430 }
3431 
3433  const OMPDistributeDirective &S) {
3434  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3436  };
3437  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3438  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3439 }
3440 
3441 static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM,
3442  const CapturedStmt *S) {
3443  CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
3445  CGF.CapturedStmtInfo = &CapStmtInfo;
3446  llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S);
3447  Fn->setDoesNotRecurse();
3448  return Fn;
3449 }
3450 
3452  if (S.hasClausesOfKind<OMPDependClause>()) {
3453  assert(!S.getAssociatedStmt() &&
3454  "No associated statement must be in ordered depend construct.");
3455  for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
3456  CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
3457  return;
3458  }
3459  const auto *C = S.getSingleClause<OMPSIMDClause>();
3460  auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
3461  PrePostActionTy &Action) {
3462  const CapturedStmt *CS = S.getInnermostCapturedStmt();
3463  if (C) {
3465  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
3466  llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS);
3467  CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
3468  OutlinedFn, CapturedVars);
3469  } else {
3470  Action.Enter(CGF);
3471  CGF.EmitStmt(CS->getCapturedStmt());
3472  }
3473  };
3474  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3475  CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getBeginLoc(), !C);
3476 }
3477 
3479  QualType SrcType, QualType DestType,
3480  SourceLocation Loc) {
3481  assert(CGF.hasScalarEvaluationKind(DestType) &&
3482  "DestType must have scalar evaluation kind.");
3483  assert(!Val.isAggregate() && "Must be a scalar or complex.");
3484  return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
3485  DestType, Loc)
3487  Val.getComplexVal(), SrcType, DestType, Loc);
3488 }
3489 
3492  QualType DestType, SourceLocation Loc) {
3493  assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
3494  "DestType must have complex evaluation kind.");
3495  CodeGenFunction::ComplexPairTy ComplexVal;
3496  if (Val.isScalar()) {
3497  // Convert the input element to the element type of the complex.
3498  QualType DestElementType =
3499  DestType->castAs<ComplexType>()->getElementType();
3500  llvm::Value *ScalarVal = CGF.EmitScalarConversion(
3501  Val.getScalarVal(), SrcType, DestElementType, Loc);
3502  ComplexVal = CodeGenFunction::ComplexPairTy(
3503  ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
3504  } else {
3505  assert(Val.isComplex() && "Must be a scalar or complex.");
3506  QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
3507  QualType DestElementType =
3508  DestType->castAs<ComplexType>()->getElementType();
3509  ComplexVal.first = CGF.EmitScalarConversion(
3510  Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
3511  ComplexVal.second = CGF.EmitScalarConversion(
3512  Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
3513  }
3514  return ComplexVal;
3515 }
3516 
3517 static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst,
3518  LValue LVal, RValue RVal) {
3519  if (LVal.isGlobalReg()) {
3520  CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
3521  } else {
3522  CGF.EmitAtomicStore(RVal, LVal,
3523  IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3524  : llvm::AtomicOrdering::Monotonic,
3525  LVal.isVolatile(), /*IsInit=*/false);
3526  }
3527 }
3528 
3530  QualType RValTy, SourceLocation Loc) {
3531  switch (getEvaluationKind(LVal.getType())) {
3532  case TEK_Scalar:
3533  EmitStoreThroughLValue(RValue::get(convertToScalarValue(
3534  *this, RVal, RValTy, LVal.getType(), Loc)),
3535  LVal);
3536  break;
3537  case TEK_Complex:
3538  EmitStoreOfComplex(
3539  convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,
3540  /*isInit=*/false);
3541  break;
3542  case TEK_Aggregate:
3543  llvm_unreachable("Must be a scalar or complex.");
3544  }
3545 }
3546 
3547 static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst,
3548  const Expr *X, const Expr *V,
3549  SourceLocation Loc) {
3550  // v = x;
3551  assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
3552  assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
3553  LValue XLValue = CGF.EmitLValue(X);
3554  LValue VLValue = CGF.EmitLValue(V);
3555  RValue Res = XLValue.isGlobalReg()
3556  ? CGF.EmitLoadOfLValue(XLValue, Loc)
3557  : CGF.EmitAtomicLoad(
3558  XLValue, Loc,
3559  IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3560  : llvm::AtomicOrdering::Monotonic,
3561  XLValue.isVolatile());
3562  // OpenMP, 2.12.6, atomic Construct
3563  // Any atomic construct with a seq_cst clause forces the atomically
3564  // performed operation to include an implicit flush operation without a
3565  // list.
3566  if (IsSeqCst)
3567  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3568  CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
3569 }
3570 
3571 static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst,
3572  const Expr *X, const Expr *E,
3573  SourceLocation Loc) {
3574  // x = expr;
3575  assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
3576  emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
3577  // OpenMP, 2.12.6, atomic Construct
3578  // Any atomic construct with a seq_cst clause forces the atomically
3579  // performed operation to include an implicit flush operation without a
3580  // list.
3581  if (IsSeqCst)
3582  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3583 }
3584 
3585 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
3586  RValue Update,
3587  BinaryOperatorKind BO,
3588  llvm::AtomicOrdering AO,
3589  bool IsXLHSInRHSPart) {
3590  ASTContext &Context = CGF.getContext();
3591  // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
3592  // expression is simple and atomic is allowed for the given type for the
3593  // target platform.
3594  if (BO == BO_Comma || !Update.isScalar() ||
3595  !Update.getScalarVal()->getType()->isIntegerTy() ||
3596  !X.isSimple() || (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
3597  (Update.getScalarVal()->getType() !=
3598  X.getAddress().getElementType())) ||
3599  !X.getAddress().getElementType()->isIntegerTy() ||
3600  !Context.getTargetInfo().hasBuiltinAtomic(
3601  Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
3602  return std::make_pair(false, RValue::get(nullptr));
3603 
3604  llvm::AtomicRMWInst::BinOp RMWOp;
3605  switch (BO) {
3606  case BO_Add:
3607  RMWOp = llvm::AtomicRMWInst::Add;
3608  break;
3609  case BO_Sub:
3610  if (!IsXLHSInRHSPart)
3611  return std::make_pair(false, RValue::get(nullptr));
3612  RMWOp = llvm::AtomicRMWInst::Sub;
3613  break;
3614  case BO_And:
3615  RMWOp = llvm::AtomicRMWInst::And;
3616  break;
3617  case BO_Or:
3618  RMWOp = llvm::AtomicRMWInst::Or;
3619  break;
3620  case BO_Xor:
3621  RMWOp = llvm::AtomicRMWInst::Xor;
3622  break;
3623  case BO_LT:
3625  ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
3626  : llvm::AtomicRMWInst::Max)
3627  : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
3628  : llvm::AtomicRMWInst::UMax);
3629  break;
3630  case BO_GT:
3632  ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
3633  : llvm::AtomicRMWInst::Min)
3634  : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
3635  : llvm::AtomicRMWInst::UMin);
3636  break;
3637  case BO_Assign:
3638  RMWOp = llvm::AtomicRMWInst::Xchg;
3639  break;
3640  case BO_Mul:
3641  case BO_Div:
3642  case BO_Rem:
3643  case BO_Shl:
3644  case BO_Shr:
3645  case BO_LAnd:
3646  case BO_LOr:
3647  return std::make_pair(false, RValue::get(nullptr));
3648  case BO_PtrMemD:
3649  case BO_PtrMemI:
3650  case BO_LE:
3651  case BO_GE:
3652  case BO_EQ:
3653  case BO_NE:
3654  case BO_Cmp:
3655  case BO_AddAssign:
3656  case BO_SubAssign:
3657  case BO_AndAssign:
3658  case BO_OrAssign:
3659  case BO_XorAssign:
3660  case BO_MulAssign:
3661  case BO_DivAssign:
3662  case BO_RemAssign:
3663  case BO_ShlAssign:
3664  case BO_ShrAssign:
3665  case BO_Comma:
3666  llvm_unreachable("Unsupported atomic update operation");
3667  }
3668  llvm::Value *UpdateVal = Update.getScalarVal();
3669  if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
3670  UpdateVal = CGF.Builder.CreateIntCast(
3671  IC, X.getAddress().getElementType(),
3673  }
3674  llvm::Value *Res =
3675  CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO);
3676  return std::make_pair(true, RValue::get(Res));
3677 }
3678 
3680  LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3681  llvm::AtomicOrdering AO, SourceLocation Loc,
3682  const llvm::function_ref<RValue(RValue)> CommonGen) {
3683  // Update expressions are allowed to have the following forms:
3684  // x binop= expr; -> xrval + expr;
3685  // x++, ++x -> xrval + 1;
3686  // x--, --x -> xrval - 1;
3687  // x = x binop expr; -> xrval binop expr
3688  // x = expr Op x; - > expr binop xrval;
3689  auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);
3690  if (!Res.first) {
3691  if (X.isGlobalReg()) {
3692  // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
3693  // 'xrval'.
3694  EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
3695  } else {
3696  // Perform compare-and-swap procedure.
3697  EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
3698  }
3699  }
3700  return Res;
3701 }
3702 
3703 static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst,
3704  const Expr *X, const Expr *E,
3705  const Expr *UE, bool IsXLHSInRHSPart,
3706  SourceLocation Loc) {
3707  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3708  "Update expr in 'atomic update' must be a binary operator.");
3709  const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3710  // Update expressions are allowed to have the following forms:
3711  // x binop= expr; -> xrval + expr;
3712  // x++, ++x -> xrval + 1;
3713  // x--, --x -> xrval - 1;
3714  // x = x binop expr; -> xrval binop expr
3715  // x = expr Op x; - > expr binop xrval;
3716  assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
3717  LValue XLValue = CGF.EmitLValue(X);
3718  RValue ExprRValue = CGF.EmitAnyExpr(E);
3719  llvm::AtomicOrdering AO = IsSeqCst
3720  ? llvm::AtomicOrdering::SequentiallyConsistent
3721  : llvm::AtomicOrdering::Monotonic;
3722  const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3723  const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3724  const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3725  const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3726  auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {
3727  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3728  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3729  return CGF.EmitAnyExpr(UE);
3730  };
3732  XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3733  // OpenMP, 2.12.6, atomic Construct
3734  // Any atomic construct with a seq_cst clause forces the atomically
3735  // performed operation to include an implicit flush operation without a
3736  // list.
3737  if (IsSeqCst)
3738  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3739 }
3740 
3742  QualType SourceType, QualType ResType,
3743  SourceLocation Loc) {
3744  switch (CGF.getEvaluationKind(ResType)) {
3745  case TEK_Scalar:
3746  return RValue::get(
3747  convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
3748  case TEK_Complex: {
3749  auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
3750  return RValue::getComplex(Res.first, Res.second);
3751  }
3752  case TEK_Aggregate:
3753  break;
3754  }
3755  llvm_unreachable("Must be a scalar or complex.");
3756 }
3757 
3758 static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst,
3759  bool IsPostfixUpdate, const Expr *V,
3760  const Expr *X, const Expr *E,
3761  const Expr *UE, bool IsXLHSInRHSPart,
3762  SourceLocation Loc) {
3763  assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
3764  assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
3765  RValue NewVVal;
3766  LValue VLValue = CGF.EmitLValue(V);
3767  LValue XLValue = CGF.EmitLValue(X);
3768  RValue ExprRValue = CGF.EmitAnyExpr(E);
3769  llvm::AtomicOrdering AO = IsSeqCst
3770  ? llvm::AtomicOrdering::SequentiallyConsistent
3771  : llvm::AtomicOrdering::Monotonic;
3772  QualType NewVValType;
3773  if (UE) {
3774  // 'x' is updated with some additional value.
3775  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3776  "Update expr in 'atomic capture' must be a binary operator.");
3777  const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3778  // Update expressions are allowed to have the following forms:
3779  // x binop= expr; -> xrval + expr;
3780  // x++, ++x -> xrval + 1;
3781  // x--, --x -> xrval - 1;
3782  // x = x binop expr; -> xrval binop expr
3783  // x = expr Op x; - > expr binop xrval;
3784  const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3785  const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3786  const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3787  NewVValType = XRValExpr->getType();
3788  const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3789  auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
3790  IsPostfixUpdate](RValue XRValue) {
3791  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3792  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3793  RValue Res = CGF.EmitAnyExpr(UE);
3794  NewVVal = IsPostfixUpdate ? XRValue : Res;
3795  return Res;
3796  };
3797  auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3798  XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3799  if (Res.first) {
3800  // 'atomicrmw' instruction was generated.
3801  if (IsPostfixUpdate) {
3802  // Use old value from 'atomicrmw'.
3803  NewVVal = Res.second;
3804  } else {
3805  // 'atomicrmw' does not provide new value, so evaluate it using old
3806  // value of 'x'.
3807  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3808  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
3809  NewVVal = CGF.EmitAnyExpr(UE);
3810  }
3811  }
3812  } else {
3813  // 'x' is simply rewritten with some 'expr'.
3814  NewVValType = X->getType().getNonReferenceType();
3815  ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
3816  X->getType().getNonReferenceType(), Loc);
3817  auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) {
3818  NewVVal = XRValue;
3819  return ExprRValue;
3820  };
3821  // Try to perform atomicrmw xchg, otherwise simple exchange.
3822  auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3823  XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
3824  Loc, Gen);
3825  if (Res.first) {
3826  // 'atomicrmw' instruction was generated.
3827  NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
3828  }
3829  }
3830  // Emit post-update store to 'v' of old/new 'x' value.
3831  CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
3832  // OpenMP, 2.12.6, atomic Construct
3833  // Any atomic construct with a seq_cst clause forces the atomically
3834  // performed operation to include an implicit flush operation without a
3835  // list.
3836  if (IsSeqCst)
3837  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3838 }
3839 
3841  bool IsSeqCst, bool IsPostfixUpdate,
3842  const Expr *X, const Expr *V, const Expr *E,
3843  const Expr *UE, bool IsXLHSInRHSPart,
3844  SourceLocation Loc) {
3845  switch (Kind) {
3846  case OMPC_read:
3847  emitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc);
3848  break;
3849  case OMPC_write:
3850  emitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc);
3851  break;
3852  case OMPC_unknown:
3853  case OMPC_update:
3854  emitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc);
3855  break;
3856  case OMPC_capture:
3857  emitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE,
3858  IsXLHSInRHSPart, Loc);
3859  break;
3860  case OMPC_if:
3861  case OMPC_final:
3862  case OMPC_num_threads:
3863  case OMPC_private:
3864  case OMPC_firstprivate:
3865  case OMPC_lastprivate:
3866  case OMPC_reduction:
3867  case OMPC_task_reduction:
3868  case OMPC_in_reduction:
3869  case OMPC_safelen:
3870  case OMPC_simdlen:
3871  case OMPC_collapse:
3872  case OMPC_default:
3873  case OMPC_seq_cst:
3874  case OMPC_shared:
3875  case OMPC_linear:
3876  case OMPC_aligned:
3877  case OMPC_copyin:
3878  case OMPC_copyprivate:
3879  case OMPC_flush:
3880  case OMPC_proc_bind:
3881  case OMPC_schedule:
3882  case OMPC_ordered:
3883  case OMPC_nowait:
3884  case OMPC_untied:
3885  case OMPC_threadprivate:
3886  case OMPC_depend:
3887  case OMPC_mergeable:
3888  case OMPC_device:
3889  case OMPC_threads:
3890  case OMPC_simd:
3891  case OMPC_map:
3892  case OMPC_num_teams:
3893  case OMPC_thread_limit:
3894  case OMPC_priority:
3895  case OMPC_grainsize:
3896  case OMPC_nogroup:
3897  case OMPC_num_tasks:
3898  case OMPC_hint:
3899  case OMPC_dist_schedule:
3900  case OMPC_defaultmap:
3901  case OMPC_uniform:
3902  case OMPC_to:
3903  case OMPC_from:
3904  case OMPC_use_device_ptr:
3905  case OMPC_is_device_ptr:
3906  llvm_unreachable("Clause is not allowed in 'omp atomic'.");
3907  }
3908 }
3909 
3911  bool IsSeqCst = S.getSingleClause<OMPSeqCstClause>();
3913  for (const OMPClause *C : S.clauses()) {
3914  // Find first clause (skip seq_cst clause, if it is first).
3915  if (C->getClauseKind() != OMPC_seq_cst) {
3916  Kind = C->getClauseKind();
3917  break;
3918  }
3919  }
3920 
3921  const Stmt *CS = S.getInnermostCapturedStmt()->IgnoreContainers();
3922  if (const auto *EWC = dyn_cast<ExprWithCleanups>(CS))
3923  enterFullExpression(EWC);
3924  // Processing for statements under 'atomic capture'.
3925  if (const auto *Compound = dyn_cast<CompoundStmt>(CS)) {
3926  for (const Stmt *C : Compound->body()) {
3927  if (const auto *EWC = dyn_cast<ExprWithCleanups>(C))
3928  enterFullExpression(EWC);
3929  }
3930  }
3931 
3932  auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF,
3933  PrePostActionTy &) {
3934  CGF.EmitStopPoint(CS);
3935  emitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(),
3936  S.getV(), S.getExpr(), S.getUpdateExpr(),
3937  S.isXLHSInRHSPart(), S.getBeginLoc());
3938  };
3939  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3940  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen);
3941 }
3942 
3944  const OMPExecutableDirective &S,
3945  const RegionCodeGenTy &CodeGen) {
3947  CodeGenModule &CGM = CGF.CGM;
3948 
3949  // On device emit this construct as inlined code.
3950  if (CGM.getLangOpts().OpenMPIsDevice) {
3951  OMPLexicalScope Scope(CGF, S, OMPD_target);
3952  CGM.getOpenMPRuntime().emitInlinedDirective(
3953  CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3955  });
3956  return;
3957  }
3958 
3959  llvm::Function *Fn = nullptr;
3960  llvm::Constant *FnID = nullptr;
3961 
3962  const Expr *IfCond = nullptr;
3963  // Check for the at most one if clause associated with the target region.
3964  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3965  if (C->getNameModifier() == OMPD_unknown ||
3966  C->getNameModifier() == OMPD_target) {
3967  IfCond = C->getCondition();
3968  break;
3969  }
3970  }
3971 
3972  // Check if we have any device clause associated with the directive.
3973  const Expr *Device = nullptr;
3974  if (auto *C = S.getSingleClause<OMPDeviceClause>())
3975  Device = C->getDevice();
3976 
3977  // Check if we have an if clause whose conditional always evaluates to false
3978  // or if we do not have any targets specified. If so the target region is not
3979  // an offload entry point.
3980  bool IsOffloadEntry = true;
3981  if (IfCond) {
3982  bool Val;
3983  if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
3984  IsOffloadEntry = false;
3985  }
3986  if (CGM.getLangOpts().OMPTargetTriples.empty())
3987  IsOffloadEntry = false;
3988 
3989  assert(CGF.CurFuncDecl && "No parent declaration for target region!");
3990  StringRef ParentName;
3991  // In case we have Ctors/Dtors we use the complete type variant to produce
3992  // the mangling of the device outlined kernel.
3993  if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
3994  ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));
3995  else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
3996  ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));
3997  else
3998  ParentName =
3999  CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));
4000 
4001  // Emit target region as a standalone region.
4002  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
4003  IsOffloadEntry, CodeGen);
4004  OMPLexicalScope Scope(CGF, S, OMPD_task);
4005  CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device);
4006 }
4007 
4009  PrePostActionTy &Action) {
4010  Action.Enter(CGF);
4011  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4012  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4013  CGF.EmitOMPPrivateClause(S, PrivateScope);
4014  (void)PrivateScope.Privatize();
4015 
4016  CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
4017 }
4018 
4020  StringRef ParentName,
4021  const OMPTargetDirective &S) {
4022  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4023  emitTargetRegion(CGF, S, Action);
4024  };
4025  llvm::Function *Fn;
4026  llvm::Constant *Addr;
4027  // Emit target region as a standalone region.
4028  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4029  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4030  assert(Fn && Addr && "Target device function emission failed.");
4031 }
4032 
4034  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4035  emitTargetRegion(CGF, S, Action);
4036  };
4037  emitCommonOMPTargetDirective(*this, S, CodeGen);
4038 }
4039 
4041  const OMPExecutableDirective &S,
4042  OpenMPDirectiveKind InnermostKind,
4043  const RegionCodeGenTy &CodeGen) {
4044  const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
4045  llvm::Value *OutlinedFn =
4046  CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
4047  S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
4048 
4049  const auto *NT = S.getSingleClause<OMPNumTeamsClause>();
4050  const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
4051  if (NT || TL) {
4052  const Expr *NumTeams = NT ? NT->getNumTeams() : nullptr;
4053  const Expr *ThreadLimit = TL ? TL->getThreadLimit() : nullptr;
4054 
4055  CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
4056  S.getBeginLoc());
4057  }
4058 
4059  OMPTeamsScope Scope(CGF, S);
4061  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
4062  CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn,
4063  CapturedVars);
4064 }
4065 
4067  // Emit teams region as a standalone region.
4068  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4069  Action.Enter(CGF);
4070  OMPPrivateScope PrivateScope(CGF);
4071  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4072  CGF.EmitOMPPrivateClause(S, PrivateScope);
4073  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4074  (void)PrivateScope.Privatize();
4075  CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
4076  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4077  };
4078  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4080  [](CodeGenFunction &) { return nullptr; });
4081 }
4082 
4084  const OMPTargetTeamsDirective &S) {
4085  auto *CS = S.getCapturedStmt(OMPD_teams);
4086  Action.Enter(CGF);
4087  // Emit teams region as a standalone region.
4088  auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4089  Action.Enter(CGF);
4090  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4091  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4092  CGF.EmitOMPPrivateClause(S, PrivateScope);
4093  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4094  (void)PrivateScope.Privatize();
4095  CGF.EmitStmt(CS->getCapturedStmt());
4096  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4097  };
4098  emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
4100  [](CodeGenFunction &) { return nullptr; });
4101 }
4102 
4104  CodeGenModule &CGM, StringRef ParentName,
4105  const OMPTargetTeamsDirective &S) {
4106  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4107  emitTargetTeamsRegion(CGF, Action, S);
4108  };
4109  llvm::Function *Fn;
4110  llvm::Constant *Addr;
4111  // Emit target region as a standalone region.
4112  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4113  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4114  assert(Fn && Addr && "Target device function emission failed.");
4115 }
4116 
4118  const OMPTargetTeamsDirective &S) {
4119  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4120  emitTargetTeamsRegion(CGF, Action, S);
4121  };
4122  emitCommonOMPTargetDirective(*this, S, CodeGen);
4123 }
4124 
4125 static void
4128  Action.Enter(CGF);
4129  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4131  };
4132 
4133  // Emit teams region as a standalone region.
4134  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4135  PrePostActionTy &Action) {
4136  Action.Enter(CGF);
4137  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4138  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4139  (void)PrivateScope.Privatize();
4140  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4141  CodeGenDistribute);
4142  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4143  };
4144  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
4146  [](CodeGenFunction &) { return nullptr; });
4147 }
4148 
4150  CodeGenModule &CGM, StringRef ParentName,
4152  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4153  emitTargetTeamsDistributeRegion(CGF, Action, S);
4154  };
4155  llvm::Function *Fn;
4156  llvm::Constant *Addr;
4157  // Emit target region as a standalone region.
4158  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4159  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4160  assert(Fn && Addr && "Target device function emission failed.");
4161 }
4162 
4165  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4166  emitTargetTeamsDistributeRegion(CGF, Action, S);
4167  };
4168  emitCommonOMPTargetDirective(*this, S, CodeGen);
4169 }
4170 
4172  CodeGenFunction &CGF, PrePostActionTy &Action,
4174  Action.Enter(CGF);
4175  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4177  };
4178 
4179  // Emit teams region as a standalone region.
4180  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4181  PrePostActionTy &Action) {
4182  Action.Enter(CGF);
4183  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4184  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4185  (void)PrivateScope.Privatize();
4186  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4187  CodeGenDistribute);
4188  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4189  };
4190  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);
4192  [](CodeGenFunction &) { return nullptr; });
4193 }
4194 
4196  CodeGenModule &CGM, StringRef ParentName,
4198  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4199  emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4200  };
4201  llvm::Function *Fn;
4202  llvm::Constant *Addr;
4203  // Emit target region as a standalone region.
4204  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4205  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4206  assert(Fn && Addr && "Target device function emission failed.");
4207 }
4208 
4211  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4212  emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4213  };
4214  emitCommonOMPTargetDirective(*this, S, CodeGen);
4215 }
4216 
4218  const OMPTeamsDistributeDirective &S) {
4219 
4220  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4222  };
4223 
4224  // Emit teams region as a standalone region.
4225  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4226  PrePostActionTy &Action) {
4227  Action.Enter(CGF);
4228  OMPPrivateScope PrivateScope(CGF);
4229  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4230  (void)PrivateScope.Privatize();
4231  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4232  CodeGenDistribute);
4233  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4234  };
4235  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4237  [](CodeGenFunction &) { return nullptr; });
4238 }
4239 
4242  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4244  };
4245 
4246  // Emit teams region as a standalone region.
4247  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4248  PrePostActionTy &Action) {
4249  Action.Enter(CGF);
4250  OMPPrivateScope PrivateScope(CGF);
4251  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4252  (void)PrivateScope.Privatize();
4253  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
4254  CodeGenDistribute);
4255  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4256  };
4257  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen);
4259  [](CodeGenFunction &) { return nullptr; });
4260 }
4261 
4264  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4266  S.getDistInc());
4267  };
4268 
4269  // Emit teams region as a standalone region.
4270  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4271  PrePostActionTy &Action) {
4272  Action.Enter(CGF);
4273  OMPPrivateScope PrivateScope(CGF);
4274  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4275  (void)PrivateScope.Privatize();
4276  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4277  CodeGenDistribute);
4278  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4279  };
4280  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4282  [](CodeGenFunction &) { return nullptr; });
4283 }
4284 
4287  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4289  S.getDistInc());
4290  };
4291 
4292  // Emit teams region as a standalone region.
4293  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4294  PrePostActionTy &Action) {
4295  Action.Enter(CGF);
4296  OMPPrivateScope PrivateScope(CGF);
4297  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4298  (void)PrivateScope.Privatize();
4299  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4300  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4301  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4302  };
4303  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4305  [](CodeGenFunction &) { return nullptr; });
4306 }
4307 
4310  PrePostActionTy &Action) {
4311  Action.Enter(CGF);
4312  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4314  S.getDistInc());
4315  };
4316 
4317  // Emit teams region as a standalone region.
4318  auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4319  PrePostActionTy &Action) {
4320  Action.Enter(CGF);
4321  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4322  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4323  (void)PrivateScope.Privatize();
4324  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4325  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4326  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4327  };
4328 
4329  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
4330  CodeGenTeams);
4332  [](CodeGenFunction &) { return nullptr; });
4333 }
4334 
4336  CodeGenModule &CGM, StringRef ParentName,
4338  // Emit SPMD target teams distribute parallel for region as a standalone
4339  // region.
4340  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4342  };
4343  llvm::Function *Fn;
4344  llvm::Constant *Addr;
4345  // Emit target region as a standalone region.
4346  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4347  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4348  assert(Fn && Addr && "Target device function emission failed.");
4349 }
4350 
4353  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4355  };
4356  emitCommonOMPTargetDirective(*this, S, CodeGen);
4357 }
4358 
4360  CodeGenFunction &CGF,
4362  PrePostActionTy &Action) {
4363  Action.Enter(CGF);
4364  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4366  S.getDistInc());
4367  };
4368 
4369  // Emit teams region as a standalone region.
4370  auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4371  PrePostActionTy &Action) {
4372  Action.Enter(CGF);
4373  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4374  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4375  (void)PrivateScope.Privatize();
4376  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4377  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4378  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4379  };
4380 
4381  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd,
4382  CodeGenTeams);
4384  [](CodeGenFunction &) { return nullptr; });
4385 }
4386 
4388  CodeGenModule &CGM, StringRef ParentName,
4390  // Emit SPMD target teams distribute parallel for simd region as a standalone
4391  // region.
4392  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4394  };
4395  llvm::Function *Fn;
4396  llvm::Constant *Addr;
4397  // Emit target region as a standalone region.
4398  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4399  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4400  assert(Fn && Addr && "Target device function emission failed.");
4401 }
4402 
4405  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4407  };
4408  emitCommonOMPTargetDirective(*this, S, CodeGen);
4409 }
4410 
4412  const OMPCancellationPointDirective &S) {
4413  CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getBeginLoc(),
4414  S.getCancelRegion());
4415 }
4416 
4418  const Expr *IfCond = nullptr;
4419  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4420  if (C->getNameModifier() == OMPD_unknown ||
4421  C->getNameModifier() == OMPD_cancel) {
4422  IfCond = C->getCondition();
4423  break;
4424  }
4425  }
4426  CGM.getOpenMPRuntime().emitCancelCall(*this, S.getBeginLoc(), IfCond,
4427  S.getCancelRegion());
4428 }
4429 
4432  if (Kind == OMPD_parallel || Kind == OMPD_task ||
4433  Kind == OMPD_target_parallel)
4434  return ReturnBlock;
4435  assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
4436  Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
4437  Kind == OMPD_distribute_parallel_for ||
4438  Kind == OMPD_target_parallel_for ||
4439  Kind == OMPD_teams_distribute_parallel_for ||
4440  Kind == OMPD_target_teams_distribute_parallel_for);
4441  return OMPCancelStack.getExitBlock();
4442 }
4443 
4445  const OMPClause &NC, OMPPrivateScope &PrivateScope,
4446  const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) {
4447  const auto &C = cast<OMPUseDevicePtrClause>(NC);
4448  auto OrigVarIt = C.varlist_begin();
4449  auto InitIt = C.inits().begin();
4450  for (const Expr *PvtVarIt : C.private_copies()) {
4451  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl());
4452  const auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl());
4453  const auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl());
4454 
4455  // In order to identify the right initializer we need to match the
4456  // declaration used by the mapping logic. In some cases we may get
4457  // OMPCapturedExprDecl that refers to the original declaration.
4458  const ValueDecl *MatchingVD = OrigVD;
4459  if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
4460  // OMPCapturedExprDecl are used to privative fields of the current
4461  // structure.
4462  const auto *ME = cast<MemberExpr>(OED->getInit());
4463  assert(isa<CXXThisExpr>(ME->getBase()) &&
4464  "Base should be the current struct!");
4465  MatchingVD = ME->getMemberDecl();
4466  }
4467 
4468  // If we don't have information about the current list item, move on to
4469  // the next one.
4470  auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
4471  if (InitAddrIt == CaptureDeviceAddrMap.end())
4472  continue;
4473 
4474  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, OrigVD,
4475  InitAddrIt, InitVD,
4476  PvtVD]() {
4477  // Initialize the temporary initialization variable with the address we
4478  // get from the runtime library. We have to cast the source address
4479  // because it is always a void *. References are materialized in the
4480  // privatization scope, so the initialization here disregards the fact
4481  // the original variable is a reference.
4482  QualType AddrQTy =
4483  getContext().getPointerType(OrigVD->getType().getNonReferenceType());
4484  llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy);
4485  Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy);
4486  setAddrOfLocalVar(InitVD, InitAddr);
4487 
4488  // Emit private declaration, it will be initialized by the value we
4489  // declaration we just added to the local declarations map.
4490  EmitDecl(*PvtVD);
4491 
4492  // The initialization variables reached its purpose in the emission
4493  // of the previous declaration, so we don't need it anymore.
4494  LocalDeclMap.erase(InitVD);
4495 
4496  // Return the address of the private variable.
4497  return GetAddrOfLocalVar(PvtVD);
4498  });
4499  assert(IsRegistered && "firstprivate var already registered as private");
4500  // Silence the warning about unused variable.
4501  (void)IsRegistered;
4502 
4503  ++OrigVarIt;
4504  ++InitIt;
4505  }
4506 }
4507 
4508 // Generate the instructions for '#pragma omp target data' directive.
4510  const OMPTargetDataDirective &S) {
4511  CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true);
4512 
4513  // Create a pre/post action to signal the privatization of the device pointer.
4514  // This action can be replaced by the OpenMP runtime code generation to
4515  // deactivate privatization.
4516  bool PrivatizeDevicePointers = false;
4517  class DevicePointerPrivActionTy : public PrePostActionTy {
4518  bool &PrivatizeDevicePointers;
4519 
4520  public:
4521  explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
4522  : PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {}
4523  void Enter(CodeGenFunction &CGF) override {
4524  PrivatizeDevicePointers = true;
4525  }
4526  };
4527  DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
4528 
4529  auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers](
4530  CodeGenFunction &CGF, PrePostActionTy &Action) {
4531  auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4533  };
4534 
4535  // Codegen that selects whether to generate the privatization code or not.
4536  auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers,
4537  &InnermostCodeGen](CodeGenFunction &CGF,
4538  PrePostActionTy &Action) {
4539  RegionCodeGenTy RCG(InnermostCodeGen);
4540  PrivatizeDevicePointers = false;
4541 
4542  // Call the pre-action to change the status of PrivatizeDevicePointers if
4543  // needed.
4544  Action.Enter(CGF);
4545 
4546  if (PrivatizeDevicePointers) {
4547  OMPPrivateScope PrivateScope(CGF);
4548  // Emit all instances of the use_device_ptr clause.
4549  for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
4550  CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
4551  Info.CaptureDeviceAddrMap);
4552  (void)PrivateScope.Privatize();
4553  RCG(CGF);
4554  } else {
4555  RCG(CGF);
4556  }
4557  };
4558 
4559  // Forward the provided action to the privatization codegen.
4560  RegionCodeGenTy PrivRCG(PrivCodeGen);
4561  PrivRCG.setAction(Action);
4562 
4563  // Notwithstanding the body of the region is emitted as inlined directive,
4564  // we don't use an inline scope as changes in the references inside the
4565  // region are expected to be visible outside, so we do not privative them.
4566  OMPLexicalScope Scope(CGF, S);
4567  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
4568  PrivRCG);
4569  };
4570 
4571  RegionCodeGenTy RCG(CodeGen);
4572 
4573  // If we don't have target devices, don't bother emitting the data mapping
4574  // code.
4575  if (CGM.getLangOpts().OMPTargetTriples.empty()) {
4576  RCG(*this);
4577  return;
4578  }
4579 
4580  // Check if we have any if clause associated with the directive.
4581  const Expr *IfCond = nullptr;
4582  if (const auto *C = S.getSingleClause<OMPIfClause>())
4583  IfCond = C->getCondition();
4584 
4585  // Check if we have any device clause associated with the directive.
4586  const Expr *Device = nullptr;
4587  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4588  Device = C->getDevice();
4589 
4590  // Set the action to signal privatization of device pointers.
4591  RCG.setAction(PrivAction);
4592 
4593  // Emit region code.
4594  CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
4595  Info);
4596 }
4597 
4599  const OMPTargetEnterDataDirective &S) {
4600  // If we don't have target devices, don't bother emitting the data mapping
4601  // code.
4602  if (CGM.getLangOpts().OMPTargetTriples.empty())
4603  return;
4604 
4605  // Check if we have any if clause associated with the directive.
4606  const Expr *IfCond = nullptr;
4607  if (const auto *C = S.getSingleClause<OMPIfClause>())
4608  IfCond = C->getCondition();
4609 
4610  // Check if we have any device clause associated with the directive.
4611  const Expr *Device = nullptr;
4612  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4613  Device = C->getDevice();
4614 
4615  OMPLexicalScope Scope(*this, S, OMPD_task);
4616  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4617 }
4618 
4620  const OMPTargetExitDataDirective &S) {
4621  // If we don't have target devices, don't bother emitting the data mapping
4622  // code.
4623  if (CGM.getLangOpts().OMPTargetTriples.empty())
4624  return;
4625 
4626  // Check if we have any if clause associated with the directive.
4627  const Expr *IfCond = nullptr;
4628  if (const auto *C = S.getSingleClause<OMPIfClause>())
4629  IfCond = C->getCondition();
4630 
4631  // Check if we have any device clause associated with the directive.
4632  const Expr *Device = nullptr;
4633  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4634  Device = C->getDevice();
4635 
4636  OMPLexicalScope Scope(*this, S, OMPD_task);
4637  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4638 }
4639 
4641  const OMPTargetParallelDirective &S,
4642  PrePostActionTy &Action) {
4643  // Get the captured statement associated with the 'parallel' region.
4644  const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
4645  Action.Enter(CGF);
4646  auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4647  Action.Enter(CGF);
4648  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4649  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4650  CGF.EmitOMPPrivateClause(S, PrivateScope);
4651  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4652  (void)PrivateScope.Privatize();
4653  // TODO: Add support for clauses.
4654  CGF.EmitStmt(CS->getCapturedStmt());
4655  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4656  };
4657  emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
4660  [](CodeGenFunction &) { return nullptr; });
4661 }
4662 
4664  CodeGenModule &CGM, StringRef ParentName,
4665  const OMPTargetParallelDirective &S) {
4666  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4667  emitTargetParallelRegion(CGF, S, Action);
4668  };
4669  llvm::Function *Fn;
4670  llvm::Constant *Addr;
4671  // Emit target region as a standalone region.
4672  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4673  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4674  assert(Fn && Addr && "Target device function emission failed.");
4675 }
4676 
4678  const OMPTargetParallelDirective &S) {
4679  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4680  emitTargetParallelRegion(CGF, S, Action);
4681  };
4682  emitCommonOMPTargetDirective(*this, S, CodeGen);
4683 }
4684 
4687  PrePostActionTy &Action) {
4688  Action.Enter(CGF);
4689  // Emit directive as a combined directive that consists of two implicit
4690  // directives: 'parallel' with 'for' directive.
4691  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4692  Action.Enter(CGF);
4694  CGF, OMPD_target_parallel_for, S.hasCancel());
4697  };
4698  emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
4700 }
4701 
4703  CodeGenModule &CGM, StringRef ParentName,
4704  const OMPTargetParallelForDirective &S) {
4705  // Emit SPMD target parallel for region as a standalone region.
4706  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4707  emitTargetParallelForRegion(CGF, S, Action);
4708  };
4709  llvm::Function *Fn;
4710  llvm::Constant *Addr;
4711  // Emit target region as a standalone region.
4712  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4713  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4714  assert(Fn && Addr && "Target device function emission failed.");
4715 }
4716 
4718  const OMPTargetParallelForDirective &S) {
4719  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4720  emitTargetParallelForRegion(CGF, S, Action);
4721  };
4722  emitCommonOMPTargetDirective(*this, S, CodeGen);
4723 }
4724 
4725 static void
4728  PrePostActionTy &Action) {
4729  Action.Enter(CGF);
4730  // Emit directive as a combined directive that consists of two implicit
4731  // directives: 'parallel' with 'for' directive.
4732  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4733  Action.Enter(CGF);
4736  };
4737  emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
4739 }
4740 
4742  CodeGenModule &CGM, StringRef ParentName,
4744  // Emit SPMD target parallel for region as a standalone region.
4745  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4746  emitTargetParallelForSimdRegion(CGF, S, Action);
4747  };
4748  llvm::Function *Fn;
4749  llvm::Constant *Addr;
4750  // Emit target region as a standalone region.
4751  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4752  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4753  assert(Fn && Addr && "Target device function emission failed.");
4754 }
4755 
4758  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4759  emitTargetParallelForSimdRegion(CGF, S, Action);
4760  };
4761  emitCommonOMPTargetDirective(*this, S, CodeGen);
4762 }
4763 
4764 /// Emit a helper variable and return corresponding lvalue.
4765 static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
4766  const ImplicitParamDecl *PVD,
4768  const auto *VDecl = cast<VarDecl>(Helper->getDecl());
4769  Privates.addPrivate(VDecl,
4770  [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); });
4771 }
4772 
4775  // Emit outlined function for task construct.
4776  const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);
4777  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
4778  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
4779  const Expr *IfCond = nullptr;
4780  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4781  if (C->getNameModifier() == OMPD_unknown ||
4782  C->getNameModifier() == OMPD_taskloop) {
4783  IfCond = C->getCondition();
4784  break;
4785  }
4786  }
4787 
4788  OMPTaskDataTy Data;
4789  // Check if taskloop must be emitted without taskgroup.
4791  // TODO: Check if we should emit tied or untied task.
4792  Data.Tied = true;
4793  // Set scheduling for taskloop
4794  if (const auto* Clause = S.getSingleClause<OMPGrainsizeClause>()) {
4795  // grainsize clause
4796  Data.Schedule.setInt(/*IntVal=*/false);
4797  Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
4798  } else if (const auto* Clause = S.getSingleClause<OMPNumTasksClause>()) {
4799  // num_tasks clause
4800  Data.Schedule.setInt(/*IntVal=*/true);
4801  Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
4802  }
4803 
4804  auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
4805  // if (PreCond) {
4806  // for (IV in 0..LastIteration) BODY;
4807  // <Final counter/linear vars updates>;
4808  // }
4809  //
4810 
4811  // Emit: if (PreCond) - begin.
4812  // If the condition constant folds and can be elided, avoid emitting the
4813  // whole loop.
4814  bool CondConstant;
4815  llvm::BasicBlock *ContBlock = nullptr;
4816  OMPLoopScope PreInitScope(CGF, S);
4817  if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
4818  if (!CondConstant)
4819  return;
4820  } else {
4821  llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
4822  ContBlock = CGF.createBasicBlock("taskloop.if.end");
4823  emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
4824  CGF.getProfileCount(&S));
4825  CGF.EmitBlock(ThenBlock);
4826  CGF.incrementProfileCounter(&S);
4827  }
4828 
4830  CGF.EmitOMPSimdInit(S);
4831 
4832  OMPPrivateScope LoopScope(CGF);
4833  // Emit helper vars inits.
4834  enum { LowerBound = 5, UpperBound, Stride, LastIter };
4835  auto *I = CS->getCapturedDecl()->param_begin();
4836  auto *LBP = std::next(I, LowerBound);
4837  auto *UBP = std::next(I, UpperBound);
4838  auto *STP = std::next(I, Stride);
4839  auto *LIP = std::next(I, LastIter);
4840  mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
4841  LoopScope);
4842  mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
4843  LoopScope);
4844  mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
4845  mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
4846  LoopScope);
4847  CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
4848  bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4849  (void)LoopScope.Privatize();
4850  // Emit the loop iteration variable.
4851  const Expr *IVExpr = S.getIterationVariable();
4852  const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
4853  CGF.EmitVarDecl(*IVDecl);
4854  CGF.EmitIgnoredExpr(S.getInit());
4855 
4856  // Emit the iterations count variable.
4857  // If it is not a variable, Sema decided to calculate iterations count on
4858  // each iteration (e.g., it is foldable into a constant).
4859  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
4860  CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
4861  // Emit calculation of the iterations count.
4862  CGF.EmitIgnoredExpr(S.getCalcLastIteration());
4863  }
4864 
4865  CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
4866  S.getInc(),
4867  [&S](CodeGenFunction &CGF) {
4868  CGF.EmitOMPLoopBody(S, JumpDest());
4869  CGF.EmitStopPoint(&S);
4870  },
4871  [](CodeGenFunction &) {});
4872  // Emit: if (PreCond) - end.
4873  if (ContBlock) {
4874  CGF.EmitBranch(ContBlock);
4875  CGF.EmitBlock(ContBlock, true);
4876  }
4877  // Emit final copy of the lastprivate variables if IsLastIter != 0.
4878  if (HasLastprivateClause) {
4879  CGF.EmitOMPLastprivateClauseFinal(
4881  CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
4882  CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
4883  (*LIP)->getType(), S.getBeginLoc())));
4884  }
4885  };
4886  auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
4887  IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
4888  const OMPTaskDataTy &Data) {
4889  auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
4890  &Data](CodeGenFunction &CGF, PrePostActionTy &) {
4891  OMPLoopScope PreInitScope(CGF, S);
4892  CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
4893  OutlinedFn, SharedsTy,
4894  CapturedStruct, IfCond, Data);
4895  };
4896  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
4897  CodeGen);
4898  };
4899  if (Data.Nogroup) {
4900  EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data);
4901  } else {
4902  CGM.getOpenMPRuntime().emitTaskgroupRegion(
4903  *this,
4904  [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
4905  PrePostActionTy &Action) {
4906  Action.Enter(CGF);
4907  CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen,
4908  Data);
4909  },
4910  S.getBeginLoc());
4911  }
4912 }
4913 
4915  EmitOMPTaskLoopBasedDirective(S);
4916 }
4917 
4919  const OMPTaskLoopSimdDirective &S) {
4920  EmitOMPTaskLoopBasedDirective(S);
4921 }
4922 
4923 // Generate the instructions for '#pragma omp target update' directive.
4925  const OMPTargetUpdateDirective &S) {
4926  // If we don't have target devices, don't bother emitting the data mapping
4927  // code.
4928  if (CGM.getLangOpts().OMPTargetTriples.empty())
4929  return;
4930 
4931  // Check if we have any if clause associated with the directive.
4932  const Expr *IfCond = nullptr;
4933  if (const auto *C = S.getSingleClause<OMPIfClause>())
4934  IfCond = C->getCondition();
4935 
4936  // Check if we have any device clause associated with the directive.
4937  const Expr *Device = nullptr;
4938  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4939  Device = C->getDevice();
4940 
4941  OMPLexicalScope Scope(*this, S, OMPD_task);
4942  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4943 }
4944 
4946  const OMPExecutableDirective &D) {
4947  if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
4948  return;
4949  auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {
4951  emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action);
4952  } else {
4953  if (const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
4954  for (const Expr *E : LD->counters()) {
4955  if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
4956  cast<DeclRefExpr>(E)->getDecl())) {
4957  // Emit only those that were not explicitly referenced in clauses.
4958  if (!CGF.LocalDeclMap.count(VD))
4959  CGF.EmitVarDecl(*VD);
4960  }
4961  }
4962  for (const auto *C : D.getClausesOfKind<OMPOrderedClause>()) {
4963  if (!C->getNumForLoops())
4964  continue;
4965  for (unsigned I = LD->getCollapsedNumber(),
4966  E = C->getLoopNumIterations().size();
4967  I < E; ++I) {
4968  if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
4969  cast<DeclRefExpr>(C->getLoopCounter(I))->getDecl())) {
4970  // Emit only those that were not explicitly referenced in clauses.
4971  if (!CGF.LocalDeclMap.count(VD))
4972  CGF.EmitVarDecl(*VD);
4973  }
4974  }
4975  }
4976  }
4978  }
4979  };
4980  OMPSimdLexicalScope Scope(*this, D);
4981  CGM.getOpenMPRuntime().emitInlinedDirective(
4982  *this,
4983  isOpenMPSimdDirective(D.getDirectiveKind()) ? OMPD_simd
4984  : D.getDirectiveKind(),
4985  CodeGen);
4986 }
4987 
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
Definition: CGCall.cpp:653
bool isAggregate() const
Definition: CGValue.h:54
This represents &#39;#pragma omp distribute simd&#39; composite directive.
Definition: StmtOpenMP.h:3214
Expr * getNextUpperBound() const
Definition: StmtOpenMP.h:843
This represents &#39;#pragma omp master&#39; directive.
Definition: StmtOpenMP.h:1397
This represents &#39;#pragma omp task&#39; directive.
Definition: StmtOpenMP.h:1737
static const Decl * getCanonicalDecl(const Decl *D)
Represents a function declaration or definition.
Definition: Decl.h:1717
This represents &#39;thread_limit&#39; clause in the &#39;#pragma omp ...&#39; directive.
Expr * getUpperBoundVariable() const
Definition: StmtOpenMP.h:811
Other implicit parameter.
Definition: Decl.h:1496
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, SourceLocation Loc)
This represents clause &#39;copyin&#39; in the &#39;#pragma omp ...&#39; directives.
Complete object ctor.
Definition: ABI.h:26
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition: Type.h:2511
Scheduling data for loop-based OpenMP directives.
Definition: OpenMPKinds.h:124
A (possibly-)qualified type.
Definition: Type.h:642
capture_init_iterator capture_init_begin()
Retrieve the first initialization argument.
Definition: Stmt.h:2251
bool isArrayType() const
Definition: Type.h:6226
ArrayRef< OMPClause * > clauses()
Definition: StmtOpenMP.h:260
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *V, SourceLocation Loc)
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
Definition: CGDecl.cpp:157
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
Definition: Decl.h:4093
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Definition: CGExpr.cpp:139
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
Definition: Dominators.h:30
static std::pair< LValue, LValue > emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static OMPClauseWithPreInit * get(OMPClause *C)
Stmt - This represents one statement.
Definition: Stmt.h:66
This represents clause &#39;in_reduction&#39; in the &#39;#pragma omp task&#39; directives.
Expr * getLowerBoundVariable() const
Definition: StmtOpenMP.h:803
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
Store of global named registers are always calls to intrinsics.
Definition: CGExpr.cpp:2133
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
Definition: Type.cpp:497
bool requiresCleanups() const
Determine whether this scope requires any cleanups.
void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S)
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
SmallVector< std::pair< OpenMPDependClauseKind, const Expr * >, 4 > Dependences
This represents &#39;#pragma omp for simd&#39; directive.
Definition: StmtOpenMP.h:1147
SmallVector< const Expr *, 4 > LastprivateCopies
QualType getLValueReferenceType(QualType T, bool SpelledAsLValue=true) const
Return the uniqued reference to the type for an lvalue reference to the specified type...
Decl - This represents one declaration (or definition), e.g.
Definition: DeclBase.h:87
This represents &#39;grainsize&#39; clause in the &#39;#pragma omp ...&#39; directive.
This represents &#39;#pragma omp teams distribute parallel for&#39; composite directive.
Definition: StmtOpenMP.h:3625
SourceLocation getBeginLoc() const
Returns starting location of directive kind.
Definition: StmtOpenMP.h:168
bool isNothrow() const
Definition: Decl.cpp:4447
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
This represents &#39;if&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:240
CapturedStmt * getInnermostCapturedStmt()
Get innermost captured statement for the construct.
Definition: StmtOpenMP.h:226
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Definition: Type.h:6126
void EmitOMPOrderedDirective(const OMPOrderedDirective &S)
bool isVolatile() const
Definition: CGValue.h:301
This represents &#39;priority&#39; clause in the &#39;#pragma omp ...&#39; directive.
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
The base class of the type hierarchy.
Definition: Type.h:1415
This represents &#39;#pragma omp target teams distribute&#39; combined directive.
Definition: StmtOpenMP.h:3762
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
Definition: CGExpr.cpp:1898
static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, bool IsSeqCst, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Definition: Stmt.h:2195
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Definition: Type.h:2786
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
const TargetInfo & getTargetInfo() const
Definition: ASTContext.h:699
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters)
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Definition: CGExpr.cpp:2306
Floating point control options.
Definition: LangOptions.h:274
This represents &#39;#pragma omp parallel for&#39; directive.
Definition: StmtOpenMP.h:1518
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value *> &CapturedVars)
This represents &#39;#pragma omp target teams distribute parallel for&#39; combined directive.
Definition: StmtOpenMP.h:3830
Expr * getCombinedEnsureUpperBound() const
Definition: StmtOpenMP.h:895
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
Definition: CGAtomic.cpp:1895
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant, or if it does but contains a label, return false.
Represents a point when we exit a loop.
Definition: ProgramPoint.h:711
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
SmallVector< const Expr *, 4 > ReductionCopies
SourceLocation getEndLoc() const
Returns ending location of directive.
Definition: StmtOpenMP.h:170
void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S)
static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, const Twine &Name, llvm::Value *Init=nullptr)
This represents &#39;#pragma omp target exit data&#39; directive.
Definition: StmtOpenMP.h:2429
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
CanQualType getCanonicalParamType(QualType T) const
Return the canonical parameter type corresponding to the specific potentially non-canonical one...
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S)
Represents a variable declaration or definition.
Definition: Decl.h:812
This represents clause &#39;private&#39; in the &#39;#pragma omp ...&#39; directives.
void EmitOMPCriticalDirective(const OMPCriticalDirective &S)
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:2612
This represents &#39;num_threads&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:382
const Stmt * getBody() const
Definition: StmtOpenMP.h:925
Extra information about a function prototype.
Definition: Type.h:3669
uint64_t getProfileCount(const Stmt *S)
Get the profiler&#39;s count for the given statement.
Expr * getCombinedUpperBoundVariable() const
Definition: StmtOpenMP.h:889
Expr * getCalcLastIteration() const
Definition: StmtOpenMP.h:771
llvm::Value * getPointer() const
Definition: Address.h:38
unsigned getContextParamPosition() const
Definition: Decl.h:4087
void EmitOMPInnerLoop(const Stmt &S, bool RequiresCleanup, const Expr *LoopCond, const Expr *IncExpr, const llvm::function_ref< void(CodeGenFunction &)> BodyGen, const llvm::function_ref< void(CodeGenFunction &)> PostIncGen)
Emit inner loop of the worksharing/simd construct.
This represents implicit clause &#39;flush&#39; for the &#39;#pragma omp flush&#39; directive.
void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D, bool NoFinals, llvm::Value *IsLastIterCond=nullptr)
Emit final copying of lastprivate values to original variables at the end of the worksharing or simd ...
void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S)
static void emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeDirective &S)
bool isXLHSInRHSPart() const
Return true if helper update expression has form &#39;OpaqueValueExpr(x) binop OpaqueValueExpr(expr)&#39; and...
Definition: StmtOpenMP.h:2226
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
This represents &#39;nogroup&#39; clause in the &#39;#pragma omp ...&#39; directive.
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB, const CodeGenLoopBoundsTy &CodeGenLoopBounds, const CodeGenDispatchBoundsTy &CGDispatchBounds)
Emit code for the worksharing loop-based directive.
This represents &#39;safelen&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:447
void EmitOMPTargetTeamsDistributeParallelForSimdDirective(const OMPTargetTeamsDistributeParallelForSimdDirective &S)
IdentifierInfo * getIdentifier() const
Get the identifier that names this declaration, if there is one.
Definition: Decl.h:269
Expr * IgnoreImpCasts() LLVM_READONLY
IgnoreImpCasts - Skip past any implicit casts which might surround this expression.
Definition: Expr.h:2977
void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S)
SmallVector< const Expr *, 4 > PrivateVars
Definition: CGOpenMPR