clang 24.0.0git
CGOpenMPRuntime.cpp
Go to the documentation of this file.
1//===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for OpenMP runtime code generation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGOpenMPRuntime.h"
14#include "ABIInfoImpl.h"
15#include "CGCXXABI.h"
16#include "CGCleanup.h"
17#include "CGDebugInfo.h"
18#include "CGRecordLayout.h"
19#include "CodeGenFunction.h"
20#include "TargetInfo.h"
21#include "clang/AST/APValue.h"
22#include "clang/AST/Attr.h"
23#include "clang/AST/Decl.h"
32#include "llvm/ADT/ArrayRef.h"
33#include "llvm/ADT/STLExtras.h"
34#include "llvm/ADT/SmallSet.h"
35#include "llvm/ADT/SmallVector.h"
36#include "llvm/ADT/StringExtras.h"
37#include "llvm/Analysis/ValueTracking.h"
38#include "llvm/Bitcode/BitcodeReader.h"
39#include "llvm/IR/Constants.h"
40#include "llvm/IR/DerivedTypes.h"
41#include "llvm/IR/GlobalValue.h"
42#include "llvm/IR/InstrTypes.h"
43#include "llvm/IR/Value.h"
44#include "llvm/Support/AtomicOrdering.h"
45#include "llvm/Support/VirtualFileSystem.h"
46#include "llvm/Support/raw_ostream.h"
47#include <cassert>
48#include <cstdint>
49#include <numeric>
50#include <optional>
51
52using namespace clang;
53using namespace CodeGen;
54using namespace llvm::omp;
55
56namespace {
57/// Base class for handling code generation inside OpenMP regions.
58class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
59public:
60 /// Kinds of OpenMP regions used in codegen.
61 enum CGOpenMPRegionKind {
62 /// Region with outlined function for standalone 'parallel'
63 /// directive.
64 ParallelOutlinedRegion,
65 /// Region with outlined function for standalone 'task' directive.
66 TaskOutlinedRegion,
67 /// Region for constructs that do not require function outlining,
68 /// like 'for', 'sections', 'atomic' etc. directives.
69 InlinedRegion,
70 /// Region with outlined function for standalone 'target' directive.
71 TargetRegion,
72 };
73
74 CGOpenMPRegionInfo(const CapturedStmt &CS,
75 const CGOpenMPRegionKind RegionKind,
76 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
77 bool HasCancel)
78 : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
79 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
80
81 CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
82 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
83 bool HasCancel)
84 : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
85 Kind(Kind), HasCancel(HasCancel) {}
86
87 /// Get a variable or parameter for storing global thread id
88 /// inside OpenMP construct.
89 virtual const VarDecl *getThreadIDVariable() const = 0;
90
91 /// Emit the captured statement body.
92 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
93
94 /// Get an LValue for the current ThreadID variable.
95 /// \return LValue for thread id variable. This LValue always has type int32*.
96 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
97
98 virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
99
100 CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
101
102 OpenMPDirectiveKind getDirectiveKind() const { return Kind; }
103
104 bool hasCancel() const { return HasCancel; }
105
106 static bool classof(const CGCapturedStmtInfo *Info) {
107 return Info->getKind() == CR_OpenMP;
108 }
109
110 ~CGOpenMPRegionInfo() override = default;
111
112protected:
113 CGOpenMPRegionKind RegionKind;
114 RegionCodeGenTy CodeGen;
116 bool HasCancel;
117};
118
119/// API for captured statement code generation in OpenMP constructs.
120class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo {
121public:
122 CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
123 const RegionCodeGenTy &CodeGen,
124 OpenMPDirectiveKind Kind, bool HasCancel,
125 StringRef HelperName)
126 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
127 HasCancel),
128 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
129 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
130 }
131
132 /// Get a variable or parameter for storing global thread id
133 /// inside OpenMP construct.
134 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
135
136 /// Get the name of the capture helper.
137 StringRef getHelperName() const override { return HelperName; }
138
139 static bool classof(const CGCapturedStmtInfo *Info) {
140 return CGOpenMPRegionInfo::classof(Info) &&
141 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
142 ParallelOutlinedRegion;
143 }
144
145private:
146 /// A variable or parameter storing global thread id for OpenMP
147 /// constructs.
148 const VarDecl *ThreadIDVar;
149 StringRef HelperName;
150};
151
152/// API for captured statement code generation in OpenMP constructs.
153class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo {
154public:
155 class UntiedTaskActionTy final : public PrePostActionTy {
156 bool Untied;
157 const VarDecl *PartIDVar;
158 const RegionCodeGenTy UntiedCodeGen;
159 llvm::SwitchInst *UntiedSwitch = nullptr;
160
161 public:
162 UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar,
163 const RegionCodeGenTy &UntiedCodeGen)
164 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
165 void Enter(CodeGenFunction &CGF) override {
166 if (Untied) {
167 // Emit task switching point.
168 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
169 CGF.GetAddrOfLocalVar(PartIDVar),
170 PartIDVar->getType()->castAs<PointerType>());
171 llvm::Value *Res =
172 CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation());
173 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done.");
174 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB);
175 CGF.EmitBlock(DoneBB);
177 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
178 UntiedSwitch->addCase(CGF.Builder.getInt32(0),
179 CGF.Builder.GetInsertBlock());
180 emitUntiedSwitch(CGF);
181 }
182 }
183 void emitUntiedSwitch(CodeGenFunction &CGF) const {
184 if (Untied) {
185 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
186 CGF.GetAddrOfLocalVar(PartIDVar),
187 PartIDVar->getType()->castAs<PointerType>());
188 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
189 PartIdLVal);
190 UntiedCodeGen(CGF);
191 CodeGenFunction::JumpDest CurPoint =
192 CGF.getJumpDestInCurrentScope(".untied.next.");
194 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
195 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
196 CGF.Builder.GetInsertBlock());
197 CGF.EmitBranchThroughCleanup(CurPoint);
198 CGF.EmitBlock(CurPoint.getBlock());
199 }
200 }
201 unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); }
202 };
203 CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
204 const VarDecl *ThreadIDVar,
205 const RegionCodeGenTy &CodeGen,
206 OpenMPDirectiveKind Kind, bool HasCancel,
207 const UntiedTaskActionTy &Action)
208 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
209 ThreadIDVar(ThreadIDVar), Action(Action) {
210 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
211 }
212
213 /// Get a variable or parameter for storing global thread id
214 /// inside OpenMP construct.
215 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
216
217 /// Get an LValue for the current ThreadID variable.
218 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
219
220 /// Get the name of the capture helper.
221 StringRef getHelperName() const override { return ".omp_outlined."; }
222
223 void emitUntiedSwitch(CodeGenFunction &CGF) override {
224 Action.emitUntiedSwitch(CGF);
225 }
226
227 static bool classof(const CGCapturedStmtInfo *Info) {
228 return CGOpenMPRegionInfo::classof(Info) &&
229 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
230 TaskOutlinedRegion;
231 }
232
233private:
234 /// A variable or parameter storing global thread id for OpenMP
235 /// constructs.
236 const VarDecl *ThreadIDVar;
237 /// Action for emitting code for untied tasks.
238 const UntiedTaskActionTy &Action;
239};
240
241/// API for inlined captured statement code generation in OpenMP
242/// constructs.
243class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
244public:
245 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
246 const RegionCodeGenTy &CodeGen,
247 OpenMPDirectiveKind Kind, bool HasCancel)
248 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
249 OldCSI(OldCSI),
250 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
251
252 // Retrieve the value of the context parameter.
253 llvm::Value *getContextValue() const override {
254 if (OuterRegionInfo)
255 return OuterRegionInfo->getContextValue();
256 llvm_unreachable("No context value for inlined OpenMP region");
257 }
258
259 void setContextValue(llvm::Value *V) override {
260 if (OuterRegionInfo) {
261 OuterRegionInfo->setContextValue(V);
262 return;
263 }
264 llvm_unreachable("No context value for inlined OpenMP region");
265 }
266
267 /// Lookup the captured field decl for a variable.
268 const FieldDecl *lookup(const VarDecl *VD) const override {
269 if (OuterRegionInfo)
270 return OuterRegionInfo->lookup(VD);
271 // If there is no outer outlined region,no need to lookup in a list of
272 // captured variables, we can use the original one.
273 return nullptr;
274 }
275
276 FieldDecl *getThisFieldDecl() const override {
277 if (OuterRegionInfo)
278 return OuterRegionInfo->getThisFieldDecl();
279 return nullptr;
280 }
281
282 /// Get a variable or parameter for storing global thread id
283 /// inside OpenMP construct.
284 const VarDecl *getThreadIDVariable() const override {
285 if (OuterRegionInfo)
286 return OuterRegionInfo->getThreadIDVariable();
287 return nullptr;
288 }
289
290 /// Get an LValue for the current ThreadID variable.
291 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {
292 if (OuterRegionInfo)
293 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
294 llvm_unreachable("No LValue for inlined OpenMP construct");
295 }
296
297 /// Get the name of the capture helper.
298 StringRef getHelperName() const override {
299 if (auto *OuterRegionInfo = getOldCSI())
300 return OuterRegionInfo->getHelperName();
301 llvm_unreachable("No helper name for inlined OpenMP construct");
302 }
303
304 void emitUntiedSwitch(CodeGenFunction &CGF) override {
305 if (OuterRegionInfo)
306 OuterRegionInfo->emitUntiedSwitch(CGF);
307 }
308
309 CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
310
311 static bool classof(const CGCapturedStmtInfo *Info) {
312 return CGOpenMPRegionInfo::classof(Info) &&
313 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
314 }
315
316 ~CGOpenMPInlinedRegionInfo() override = default;
317
318private:
319 /// CodeGen info about outer OpenMP region.
320 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
321 CGOpenMPRegionInfo *OuterRegionInfo;
322};
323
324/// API for captured statement code generation in OpenMP target
325/// constructs. For this captures, implicit parameters are used instead of the
326/// captured fields. The name of the target region has to be unique in a given
327/// application so it is provided by the client, because only the client has
328/// the information to generate that.
329class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {
330public:
331 CGOpenMPTargetRegionInfo(const CapturedStmt &CS,
332 const RegionCodeGenTy &CodeGen, StringRef HelperName)
333 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
334 /*HasCancel=*/false),
335 HelperName(HelperName) {}
336
337 /// This is unused for target regions because each starts executing
338 /// with a single thread.
339 const VarDecl *getThreadIDVariable() const override { return nullptr; }
340
341 /// Get the name of the capture helper.
342 StringRef getHelperName() const override { return HelperName; }
343
344 static bool classof(const CGCapturedStmtInfo *Info) {
345 return CGOpenMPRegionInfo::classof(Info) &&
346 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
347 }
348
349private:
350 StringRef HelperName;
351};
352
353static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
354 llvm_unreachable("No codegen for expressions");
355}
356/// API for generation of expressions captured in a innermost OpenMP
357/// region.
358class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo {
359public:
360 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
361 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
362 OMPD_unknown,
363 /*HasCancel=*/false),
364 PrivScope(CGF) {
365 // Make sure the globals captured in the provided statement are local by
366 // using the privatization logic. We assume the same variable is not
367 // captured more than once.
368 for (const auto &C : CS.captures()) {
369 if (!C.capturesVariable() && !C.capturesVariableByCopy())
370 continue;
371
372 const VarDecl *VD = C.getCapturedVar();
373 if (VD->isLocalVarDeclOrParm())
374 continue;
375
376 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
377 /*RefersToEnclosingVariableOrCapture=*/false,
378 VD->getType().getNonReferenceType(), VK_LValue,
379 C.getLocation());
380 PrivScope.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress());
381 }
382 (void)PrivScope.Privatize();
383 }
384
385 /// Lookup the captured field decl for a variable.
386 const FieldDecl *lookup(const VarDecl *VD) const override {
387 if (const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
388 return FD;
389 return nullptr;
390 }
391
392 /// Emit the captured statement body.
393 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
394 llvm_unreachable("No body for expressions");
395 }
396
397 /// Get a variable or parameter for storing global thread id
398 /// inside OpenMP construct.
399 const VarDecl *getThreadIDVariable() const override {
400 llvm_unreachable("No thread id for expressions");
401 }
402
403 /// Get the name of the capture helper.
404 StringRef getHelperName() const override {
405 llvm_unreachable("No helper name for expressions");
406 }
407
408 static bool classof(const CGCapturedStmtInfo *Info) { return false; }
409
410private:
411 /// Private scope to capture global variables.
412 CodeGenFunction::OMPPrivateScope PrivScope;
413};
414
415/// RAII for emitting code of OpenMP constructs.
416class InlinedOpenMPRegionRAII {
417 CodeGenFunction &CGF;
418 llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
419 FieldDecl *LambdaThisCaptureField = nullptr;
420 const CodeGen::CGBlockInfo *BlockInfo = nullptr;
421 bool NoInheritance = false;
422
423public:
424 /// Constructs region for combined constructs.
425 /// \param CodeGen Code generation sequence for combined directives. Includes
426 /// a list of functions used for code generation of implicitly inlined
427 /// regions.
428 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
429 OpenMPDirectiveKind Kind, bool HasCancel,
430 bool NoInheritance = true)
431 : CGF(CGF), NoInheritance(NoInheritance) {
432 // Start emission for the construct.
433 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
434 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
435 if (NoInheritance) {
436 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
437 LambdaThisCaptureField = CGF.LambdaThisCaptureField;
438 CGF.LambdaThisCaptureField = nullptr;
439 BlockInfo = CGF.BlockInfo;
440 CGF.BlockInfo = nullptr;
441 }
442 }
443
444 ~InlinedOpenMPRegionRAII() {
445 // Restore original CapturedStmtInfo only if we're done with code emission.
446 auto *OldCSI =
447 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
448 delete CGF.CapturedStmtInfo;
449 CGF.CapturedStmtInfo = OldCSI;
450 if (NoInheritance) {
451 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
452 CGF.LambdaThisCaptureField = LambdaThisCaptureField;
453 CGF.BlockInfo = BlockInfo;
454 }
455 }
456};
457
458/// Values for bit flags used in the ident_t to describe the fields.
459/// All enumeric elements are named and described in accordance with the code
460/// from https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h
461enum OpenMPLocationFlags : unsigned {
462 /// Use trampoline for internal microtask.
463 OMP_IDENT_IMD = 0x01,
464 /// Use c-style ident structure.
465 OMP_IDENT_KMPC = 0x02,
466 /// Atomic reduction option for kmpc_reduce.
467 OMP_ATOMIC_REDUCE = 0x10,
468 /// Explicit 'barrier' directive.
469 OMP_IDENT_BARRIER_EXPL = 0x20,
470 /// Implicit barrier in code.
471 OMP_IDENT_BARRIER_IMPL = 0x40,
472 /// Implicit barrier in 'for' directive.
473 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
474 /// Implicit barrier in 'sections' directive.
475 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
476 /// Implicit barrier in 'single' directive.
477 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
478 /// Call of __kmp_for_static_init for static loop.
479 OMP_IDENT_WORK_LOOP = 0x200,
480 /// Call of __kmp_for_static_init for sections.
481 OMP_IDENT_WORK_SECTIONS = 0x400,
482 /// Call of __kmp_for_static_init for distribute.
483 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
484 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE)
485};
486
487/// Describes ident structure that describes a source location.
488/// All descriptions are taken from
489/// https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h
490/// Original structure:
491/// typedef struct ident {
492/// kmp_int32 reserved_1; /**< might be used in Fortran;
493/// see above */
494/// kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags;
495/// KMP_IDENT_KMPC identifies this union
496/// member */
497/// kmp_int32 reserved_2; /**< not really used in Fortran any more;
498/// see above */
499///#if USE_ITT_BUILD
500/// /* but currently used for storing
501/// region-specific ITT */
502/// /* contextual information. */
503///#endif /* USE_ITT_BUILD */
504/// kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for
505/// C++ */
506/// char const *psource; /**< String describing the source location.
507/// The string is composed of semi-colon separated
508// fields which describe the source file,
509/// the function and a pair of line numbers that
510/// delimit the construct.
511/// */
512/// } ident_t;
513enum IdentFieldIndex {
514 /// might be used in Fortran
515 IdentField_Reserved_1,
516 /// OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
517 IdentField_Flags,
518 /// Not really used in Fortran any more
519 IdentField_Reserved_2,
520 /// Source[4] in Fortran, do not use for C++
521 IdentField_Reserved_3,
522 /// String describing the source location. The string is composed of
523 /// semi-colon separated fields which describe the source file, the function
524 /// and a pair of line numbers that delimit the construct.
525 IdentField_PSource
526};
527
528/// Schedule types for 'omp for' loops (these enumerators are taken from
529/// the enum sched_type in kmp.h).
530enum OpenMPSchedType {
531 /// Lower bound for default (unordered) versions.
532 OMP_sch_lower = 32,
533 OMP_sch_static_chunked = 33,
534 OMP_sch_static = 34,
535 OMP_sch_dynamic_chunked = 35,
536 OMP_sch_guided_chunked = 36,
537 OMP_sch_runtime = 37,
538 OMP_sch_auto = 38,
539 /// static with chunk adjustment (e.g., simd)
540 OMP_sch_static_balanced_chunked = 45,
541 /// Lower bound for 'ordered' versions.
542 OMP_ord_lower = 64,
543 OMP_ord_static_chunked = 65,
544 OMP_ord_static = 66,
545 OMP_ord_dynamic_chunked = 67,
546 OMP_ord_guided_chunked = 68,
547 OMP_ord_runtime = 69,
548 OMP_ord_auto = 70,
549 OMP_sch_default = OMP_sch_static,
550 /// dist_schedule types
551 OMP_dist_sch_static_chunked = 91,
552 OMP_dist_sch_static = 92,
553 /// Fused distribute+for static schedule (entityId = team*nthreads + tid,
554 /// num_entities = nteams*nthreads). One for_static_init call, no
555 /// surrounding distribute_static_init. Matches
556 /// kmp_sched_distr_static_chunk_sched_static_chunkone in the device RTL
557 /// (openmp/device/include/DeviceTypes.h).
558 OMP_dist_sch_static_chunked_sch_static_chunkone = 93,
559 /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
560 /// Set if the monotonic schedule modifier was present.
561 OMP_sch_modifier_monotonic = (1 << 29),
562 /// Set if the nonmonotonic schedule modifier was present.
563 OMP_sch_modifier_nonmonotonic = (1 << 30),
564};
565
566/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
567/// region.
568class CleanupTy final : public EHScopeStack::Cleanup {
569 PrePostActionTy *Action;
570
571public:
572 explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
573 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
574 if (!CGF.HaveInsertPoint())
575 return;
576 Action->Exit(CGF);
577 }
578};
579
580} // anonymous namespace
581
584 if (PrePostAction) {
585 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction);
586 Callback(CodeGen, CGF, *PrePostAction);
587 } else {
588 PrePostActionTy Action;
589 Callback(CodeGen, CGF, Action);
590 }
591}
592
593/// Check if the combiner is a call to UDR combiner and if it is so return the
594/// UDR decl used for reduction.
595static const OMPDeclareReductionDecl *
596getReductionInit(const Expr *ReductionOp) {
597 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
598 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
599 if (const auto *DRE =
600 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
601 if (const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
602 return DRD;
603 return nullptr;
604}
605
607 const OMPDeclareReductionDecl *DRD,
608 const Expr *InitOp,
609 Address Private, Address Original,
610 QualType Ty) {
611 if (DRD->getInitializer()) {
612 std::pair<llvm::Function *, llvm::Function *> Reduction =
614 const auto *CE = cast<CallExpr>(InitOp);
615 const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
616 const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts();
617 const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts();
618 const auto *LHSDRE =
619 cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
620 const auto *RHSDRE =
621 cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
622 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
623 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()), Private);
624 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()), Original);
625 (void)PrivateScope.Privatize();
628 CGF.EmitIgnoredExpr(InitOp);
629 } else {
630 llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty);
631 std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"});
632 auto *GV = new llvm::GlobalVariable(
633 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
634 llvm::GlobalValue::PrivateLinkage, Init, Name);
635 LValue LV = CGF.MakeNaturalAlignRawAddrLValue(GV, Ty);
636 RValue InitRVal;
637 switch (CGF.getEvaluationKind(Ty)) {
638 case TEK_Scalar:
639 InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation());
640 break;
641 case TEK_Complex:
642 InitRVal =
644 break;
645 case TEK_Aggregate: {
646 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_LValue);
647 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, LV);
648 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
649 /*IsInitializer=*/false);
650 return;
651 }
652 }
653 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_PRValue);
654 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
655 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
656 /*IsInitializer=*/false);
657 }
658}
659
660/// Emit initialization of arrays of complex types.
661/// \param DestAddr Address of the array.
662/// \param Type Type of array.
663/// \param Init Initial expression of array.
664/// \param SrcAddr Address of the original array, required only if the declare
665/// reduction has an initializer.
667 QualType Type, bool EmitDeclareReductionInit,
668 const Expr *Init,
669 const OMPDeclareReductionDecl *DRD,
670 Address SrcAddr = Address::invalid()) {
671 // Perform element-by-element initialization.
672 QualType ElementTy;
673
674 // Only the initializer of a declare reduction reads the original item.
675 bool UseSrc = DRD && DRD->getInitializer();
676 assert((!UseSrc || SrcAddr.isValid()) &&
677 "Declare reduction initializer requires the original item address.");
678
679 // Drill down to the base element type on both arrays.
680 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
681 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);
682 if (UseSrc)
683 SrcAddr = SrcAddr.withElementType(DestAddr.getElementType());
684
685 llvm::Value *SrcBegin = nullptr;
686 if (UseSrc)
687 SrcBegin = SrcAddr.emitRawPointer(CGF);
688 llvm::Value *DestBegin = DestAddr.emitRawPointer(CGF);
689 // Cast from pointer to array type to pointer to single element.
690 llvm::Value *DestEnd =
691 CGF.Builder.CreateGEP(DestAddr.getElementType(), DestBegin, NumElements);
692 // The basic structure here is a while-do loop.
693 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body");
694 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done");
695 llvm::Value *IsEmpty =
696 CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty");
697 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
698
699 // Enter the loop body, making that address the current address.
700 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
701 CGF.EmitBlock(BodyBB);
702
703 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
704
705 llvm::PHINode *SrcElementPHI = nullptr;
706 Address SrcElementCurrent = Address::invalid();
707 if (UseSrc) {
708 SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2,
709 "omp.arraycpy.srcElementPast");
710 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
711 SrcElementCurrent =
712 Address(SrcElementPHI, SrcAddr.getElementType(),
713 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
714 }
715 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
716 DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
717 DestElementPHI->addIncoming(DestBegin, EntryBB);
718 Address DestElementCurrent =
719 Address(DestElementPHI, DestAddr.getElementType(),
720 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
721
722 // Emit copy.
723 {
725 if (EmitDeclareReductionInit) {
726 emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent,
727 SrcElementCurrent, ElementTy);
728 } else
729 CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),
730 /*IsInitializer=*/false);
731 }
732
733 if (UseSrc) {
734 // Shift the address forward by one element.
735 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32(
736 SrcAddr.getElementType(), SrcElementPHI, /*Idx0=*/1,
737 "omp.arraycpy.dest.element");
738 SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock());
739 }
740
741 // Shift the address forward by one element.
742 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32(
743 DestAddr.getElementType(), DestElementPHI, /*Idx0=*/1,
744 "omp.arraycpy.dest.element");
745 // Check whether we've reached the end.
746 llvm::Value *Done =
747 CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
748 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
749 DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock());
750
751 // Done.
752 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
753}
754
755LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {
756 return CGF.EmitOMPSharedLValue(E);
757}
758
759LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
760 const Expr *E) {
761 if (const auto *OASE = dyn_cast<ArraySectionExpr>(E))
762 return CGF.EmitArraySectionExpr(OASE, /*IsLowerBound=*/false);
763 return LValue();
764}
765
766void ReductionCodeGen::emitAggregateInitialization(
767 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr,
768 const OMPDeclareReductionDecl *DRD) {
769 // Emit VarDecl with copy init for arrays.
770 // Get the address of the original variable captured in current
771 // captured region.
772 const auto *PrivateVD =
773 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
774 bool EmitDeclareReductionInit =
775 DRD && (DRD->getInitializer() || !PrivateVD->hasInit());
776 EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(),
777 EmitDeclareReductionInit,
778 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
779 : PrivateVD->getInit(),
780 DRD, SharedAddr);
781}
782
786 ArrayRef<const Expr *> ReductionOps) {
787 ClausesData.reserve(Shareds.size());
788 SharedAddresses.reserve(Shareds.size());
789 Sizes.reserve(Shareds.size());
790 BaseDecls.reserve(Shareds.size());
791 const auto *IOrig = Origs.begin();
792 const auto *IPriv = Privates.begin();
793 const auto *IRed = ReductionOps.begin();
794 for (const Expr *Ref : Shareds) {
795 ClausesData.emplace_back(Ref, *IOrig, *IPriv, *IRed);
796 std::advance(IOrig, 1);
797 std::advance(IPriv, 1);
798 std::advance(IRed, 1);
799 }
800}
801
803 assert(SharedAddresses.size() == N && OrigAddresses.size() == N &&
804 "Number of generated lvalues must be exactly N.");
805 LValue First = emitSharedLValue(CGF, ClausesData[N].Shared);
806 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Shared);
807 SharedAddresses.emplace_back(First, Second);
808 if (ClausesData[N].Shared == ClausesData[N].Ref) {
809 OrigAddresses.emplace_back(First, Second);
810 } else {
811 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
812 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
813 OrigAddresses.emplace_back(First, Second);
814 }
815}
816
818 QualType PrivateType = getPrivateType(N);
819 bool AsArraySection = isa<ArraySectionExpr>(ClausesData[N].Ref);
820 if (!PrivateType->isVariablyModifiedType()) {
821 Sizes.emplace_back(
822 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()),
823 nullptr);
824 return;
825 }
826 llvm::Value *Size;
827 llvm::Value *SizeInChars;
828 auto *ElemType = OrigAddresses[N].first.getAddress().getElementType();
829 auto *ElemSizeOf = llvm::ConstantInt::get(
830 CGF.SizeTy, CGF.CGM.getDataLayout().getTypeAllocSize(ElemType));
831 if (AsArraySection) {
832 SizeInChars =
833 CGF.Builder.CreatePtrDiff(OrigAddresses[N].second.getPointer(CGF),
834 OrigAddresses[N].first.getPointer(CGF));
835 SizeInChars = CGF.Builder.CreateNUWAdd(SizeInChars, ElemSizeOf);
836 } else {
837 SizeInChars =
838 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType());
839 }
840 Size = ElemSizeOf->isOne()
841 ? SizeInChars
842 : CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
843 Sizes.emplace_back(SizeInChars, Size);
845 CGF,
847 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
848 RValue::get(Size));
849 CGF.EmitVariablyModifiedType(PrivateType);
850}
851
853 llvm::Value *Size) {
854 QualType PrivateType = getPrivateType(N);
855 if (!PrivateType->isVariablyModifiedType()) {
856 assert(!Size && !Sizes[N].second &&
857 "Size should be nullptr for non-variably modified reduction "
858 "items.");
859 return;
860 }
862 CGF,
864 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
865 RValue::get(Size));
866 CGF.EmitVariablyModifiedType(PrivateType);
867}
868
870 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr,
871 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) {
872 assert(SharedAddresses.size() > N && "No variable was generated");
873 const auto *PrivateVD =
874 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
875 const OMPDeclareReductionDecl *DRD =
876 getReductionInit(ClausesData[N].ReductionOp);
877 if (CGF.getContext().getAsArrayType(PrivateVD->getType())) {
878 if (DRD && DRD->getInitializer())
879 (void)DefaultInit(CGF);
880 emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD);
881 } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
882 (void)DefaultInit(CGF);
883 QualType SharedType = SharedAddresses[N].first.getType();
884 emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp,
885 PrivateAddr, SharedAddr, SharedType);
886 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
887 !CGF.isTrivialInitializer(PrivateVD->getInit())) {
888 CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr,
889 PrivateVD->getType().getQualifiers(),
890 /*IsInitializer=*/false);
891 }
892}
893
895 QualType PrivateType = getPrivateType(N);
896 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
897 return DTorKind != QualType::DK_none;
898}
899
901 Address PrivateAddr) {
902 QualType PrivateType = getPrivateType(N);
903 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
904 if (needCleanups(N)) {
905 PrivateAddr =
906 PrivateAddr.withElementType(CGF.ConvertTypeForMem(PrivateType));
907 CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType);
908 }
909}
910
911static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
912 LValue BaseLV) {
913 BaseTy = BaseTy.getNonReferenceType();
914 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
915 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
916 if (const auto *PtrTy = BaseTy->getAs<PointerType>()) {
917 BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(), PtrTy);
918 } else {
919 LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(), BaseTy);
920 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);
921 }
922 BaseTy = BaseTy->getPointeeType();
923 }
924 return CGF.MakeAddrLValue(
925 BaseLV.getAddress().withElementType(CGF.ConvertTypeForMem(ElTy)),
926 BaseLV.getType(), BaseLV.getBaseInfo(),
927 CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType()));
928}
929
931 Address OriginalBaseAddress, llvm::Value *Addr) {
933 Address TopTmp = Address::invalid();
934 Address MostTopTmp = Address::invalid();
935 BaseTy = BaseTy.getNonReferenceType();
936 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
937 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
938 Tmp = CGF.CreateMemTempWithoutCast(BaseTy);
939 if (TopTmp.isValid())
940 CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp);
941 else
942 MostTopTmp = Tmp;
943 TopTmp = Tmp;
944 BaseTy = BaseTy->getPointeeType();
945 }
946
947 if (Tmp.isValid()) {
949 Addr, Tmp.getElementType());
950 CGF.Builder.CreateStore(Addr, Tmp);
951 return MostTopTmp;
952 }
953
955 Addr, OriginalBaseAddress.getType());
956 return OriginalBaseAddress.withPointer(Addr, NotKnownNonNull);
957}
958
959static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) {
960 const VarDecl *OrigVD = nullptr;
961 if (const auto *OASE = dyn_cast<ArraySectionExpr>(Ref)) {
962 const Expr *Base = OASE->getBase()->IgnoreParenImpCasts();
963 while (const auto *TempOASE = dyn_cast<ArraySectionExpr>(Base))
964 Base = TempOASE->getBase()->IgnoreParenImpCasts();
965 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
966 Base = TempASE->getBase()->IgnoreParenImpCasts();
968 OrigVD = cast<VarDecl>(DE->getDecl());
969 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
970 const Expr *Base = ASE->getBase()->IgnoreParenImpCasts();
971 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
972 Base = TempASE->getBase()->IgnoreParenImpCasts();
974 OrigVD = cast<VarDecl>(DE->getDecl());
975 }
976 return OrigVD;
977}
978
980 Address PrivateAddr) {
981 const DeclRefExpr *DE;
982 if (const VarDecl *OrigVD = ::getBaseDecl(ClausesData[N].Ref, DE)) {
983 BaseDecls.emplace_back(OrigVD);
984 LValue OriginalBaseLValue = CGF.EmitLValue(DE);
985 LValue BaseLValue =
986 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
987 OriginalBaseLValue);
988 Address SharedAddr = SharedAddresses[N].first.getAddress();
989 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
990 SharedAddr.getElementType(), BaseLValue.getPointer(CGF),
991 SharedAddr.emitRawPointer(CGF));
992 llvm::Value *PrivatePointer =
994 PrivateAddr.emitRawPointer(CGF), SharedAddr.getType());
995 llvm::Value *Ptr = CGF.Builder.CreateGEP(
996 SharedAddr.getElementType(), PrivatePointer, Adjustment);
997 return castToBase(CGF, OrigVD->getType(),
998 SharedAddresses[N].first.getType(),
999 OriginalBaseLValue.getAddress(), Ptr);
1000 }
1001 BaseDecls.emplace_back(
1002 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
1003 return PrivateAddr;
1004}
1005
1007 const OMPDeclareReductionDecl *DRD =
1008 getReductionInit(ClausesData[N].ReductionOp);
1009 return DRD && DRD->getInitializer();
1010}
1011
1012LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
1013 return CGF.EmitLoadOfPointerLValue(
1014 CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1015 getThreadIDVariable()->getType()->castAs<PointerType>());
1016}
1017
1018void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt *S) {
1019 if (!CGF.HaveInsertPoint())
1020 return;
1021 // 1.2.2 OpenMP Language Terminology
1022 // Structured block - An executable statement with a single entry at the
1023 // top and a single exit at the bottom.
1024 // The point of exit cannot be a branch out of the structured block.
1025 // longjmp() and throw() must not violate the entry/exit criteria.
1026 CGF.EHStack.pushTerminate();
1027 if (S)
1029 CodeGen(CGF);
1030 CGF.EHStack.popTerminate();
1031}
1032
1033LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1034 CodeGenFunction &CGF) {
1035 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1036 getThreadIDVariable()->getType(),
1038}
1039
1041 QualType FieldTy) {
1042 auto *Field = FieldDecl::Create(
1043 C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy,
1044 C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()),
1045 /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
1046 Field->setAccess(AS_public);
1047 DC->addDecl(Field);
1048 return Field;
1049}
1050
1052 : CGM(CGM), OMPBuilder(CGM.getModule()) {
1053 KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8);
1054 llvm::OpenMPIRBuilderConfig Config(
1055 CGM.getLangOpts().OpenMPIsTargetDevice, isGPU(),
1056 CGM.getLangOpts().OpenMPOffloadMandatory,
1057 /*HasRequiresReverseOffload*/ false, /*HasRequiresUnifiedAddress*/ false,
1058 hasRequiresUnifiedSharedMemory(), /*HasRequiresDynamicAllocators*/ false);
1059 Config.setDefaultTargetAS(
1060 CGM.getContext().getTargetInfo().getTargetAddressSpace(LangAS::Default));
1061 Config.setRuntimeCC(CGM.getRuntimeCC());
1062
1063 OMPBuilder.setConfig(Config);
1064 OMPBuilder.initialize();
1065 OMPBuilder.loadOffloadInfoMetadata(*CGM.getFileSystem(),
1066 CGM.getLangOpts().OpenMPIsTargetDevice
1067 ? CGM.getLangOpts().OMPHostIRFile
1068 : StringRef{});
1069
1070 // The user forces the compiler to behave as if omp requires
1071 // unified_shared_memory was given.
1072 if (CGM.getLangOpts().OpenMPForceUSM) {
1074 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(true);
1075 }
1076}
1077
1079 InternalVars.clear();
1080 // Clean non-target variable declarations possibly used only in debug info.
1081 for (const auto &Data : EmittedNonTargetVariables) {
1082 if (!Data.getValue().pointsToAliveValue())
1083 continue;
1084 auto *GV = dyn_cast<llvm::GlobalVariable>(Data.getValue());
1085 if (!GV)
1086 continue;
1087 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1088 continue;
1089 GV->eraseFromParent();
1090 }
1091}
1092
1094 return OMPBuilder.createPlatformSpecificName(Parts);
1095}
1096
1097static llvm::Function *
1099 const Expr *CombinerInitializer, const VarDecl *In,
1100 const VarDecl *Out, bool IsCombiner) {
1101 // void .omp_combiner.(Ty *in, Ty *out);
1102 ASTContext &C = CGM.getContext();
1103 QualType PtrTy = C.getPointerType(Ty).withRestrict();
1104 auto *OmpOutParm = ImplicitParamDecl::Create(
1105 C, /*DC=*/nullptr, Out->getLocation(),
1106 /*Id=*/nullptr, PtrTy, ImplicitParamKind::Other);
1107 auto *OmpInParm = ImplicitParamDecl::Create(
1108 C, /*DC=*/nullptr, In->getLocation(),
1109 /*Id=*/nullptr, PtrTy, ImplicitParamKind::Other);
1110 FunctionArgList Args{OmpOutParm, OmpInParm};
1111 const CGFunctionInfo &FnInfo =
1112 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
1113 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
1114 std::string Name = CGM.getOpenMPRuntime().getName(
1115 {IsCombiner ? "omp_combiner" : "omp_initializer", ""});
1116 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1117 Name, &CGM.getModule());
1118 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
1119 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
1120 Fn->addFnAttr("sample-profile-suffix-elision-policy", "selected");
1121 if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
1122 Fn->removeFnAttr(llvm::Attribute::NoInline);
1123 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1124 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1125 }
1126 CodeGenFunction CGF(CGM);
1127 // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions.
1128 // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions.
1129 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(),
1130 Out->getLocation());
1132 Address AddrIn = CGF.GetAddrOfLocalVar(OmpInParm);
1133 Scope.addPrivate(
1134 In, CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>())
1135 .getAddress());
1136 Address AddrOut = CGF.GetAddrOfLocalVar(OmpOutParm);
1137 Scope.addPrivate(
1138 Out, CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>())
1139 .getAddress());
1140 (void)Scope.Privatize();
1141 if (!IsCombiner && Out->hasInit() &&
1142 !CGF.isTrivialInitializer(Out->getInit())) {
1143 CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out),
1144 Out->getType().getQualifiers(),
1145 /*IsInitializer=*/true);
1146 }
1147 if (CombinerInitializer)
1148 CGF.EmitIgnoredExpr(CombinerInitializer);
1149 Scope.ForceCleanup();
1150 CGF.FinishFunction();
1151 return Fn;
1152}
1153
1156 if (UDRMap.count(D) > 0)
1157 return;
1158 llvm::Function *Combiner = emitCombinerOrInitializer(
1159 CGM, D->getType(), D->getCombiner(),
1162 /*IsCombiner=*/true);
1163 llvm::Function *Initializer = nullptr;
1164 if (const Expr *Init = D->getInitializer()) {
1166 CGM, D->getType(),
1168 : nullptr,
1171 /*IsCombiner=*/false);
1172 }
1173 UDRMap.try_emplace(D, Combiner, Initializer);
1174 if (CGF)
1175 FunctionUDRMap[CGF->CurFn].push_back(D);
1176}
1177
1178std::pair<llvm::Function *, llvm::Function *>
1180 auto I = UDRMap.find(D);
1181 if (I != UDRMap.end())
1182 return I->second;
1183 emitUserDefinedReduction(/*CGF=*/nullptr, D);
1184 return UDRMap.lookup(D);
1185}
1186
1187namespace {
1188// Temporary RAII solution to perform a push/pop stack event on the OpenMP IR
1189// Builder if one is present.
1190struct PushAndPopStackRAII {
1191 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder, CodeGenFunction &CGF,
1192 bool HasCancel, llvm::omp::Directive Kind)
1193 : OMPBuilder(OMPBuilder) {
1194 if (!OMPBuilder)
1195 return;
1196
1197 // The following callback is the crucial part of clangs cleanup process.
1198 //
1199 // NOTE:
1200 // Once the OpenMPIRBuilder is used to create parallel regions (and
1201 // similar), the cancellation destination (Dest below) is determined via
1202 // IP. That means if we have variables to finalize we split the block at IP,
1203 // use the new block (=BB) as destination to build a JumpDest (via
1204 // getJumpDestInCurrentScope(BB)) which then is fed to
1205 // EmitBranchThroughCleanup. Furthermore, there will not be the need
1206 // to push & pop an FinalizationInfo object.
1207 // The FiniCB will still be needed but at the point where the
1208 // OpenMPIRBuilder is asked to construct a parallel (or similar) construct.
1209 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1210 assert(IP == IP.getNodeParent()->end() &&
1211 "Clang CG should cause non-terminated block!");
1212 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1213 CGF.Builder.restoreIP(IP);
1215 CGF.getOMPCancelDestination(OMPD_parallel);
1216 CGF.EmitBranchThroughCleanup(Dest);
1217 return llvm::Error::success();
1218 };
1219
1220 // TODO: Remove this once we emit parallel regions through the
1221 // OpenMPIRBuilder as it can do this setup internally.
1222 llvm::OpenMPIRBuilder::FinalizationInfo FI({FiniCB, Kind, HasCancel});
1223 OMPBuilder->pushFinalizationCB(std::move(FI));
1224 }
1225 ~PushAndPopStackRAII() {
1226 if (OMPBuilder)
1227 OMPBuilder->popFinalizationCB();
1228 }
1229 llvm::OpenMPIRBuilder *OMPBuilder;
1230};
1231} // namespace
1232
1234 CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS,
1235 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1236 const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) {
1237 assert(ThreadIDVar->getType()->isPointerType() &&
1238 "thread id variable must be of type kmp_int32 *");
1239 CodeGenFunction CGF(CGM, true);
1240 bool HasCancel = false;
1241 if (const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1242 HasCancel = OPD->hasCancel();
1243 else if (const auto *OPD = dyn_cast<OMPTargetParallelDirective>(&D))
1244 HasCancel = OPD->hasCancel();
1245 else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1246 HasCancel = OPSD->hasCancel();
1247 else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1248 HasCancel = OPFD->hasCancel();
1249 else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1250 HasCancel = OPFD->hasCancel();
1251 else if (const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1252 HasCancel = OPFD->hasCancel();
1253 else if (const auto *OPFD =
1254 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1255 HasCancel = OPFD->hasCancel();
1256 else if (const auto *OPFD =
1257 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1258 HasCancel = OPFD->hasCancel();
1259
1260 // TODO: Temporarily inform the OpenMPIRBuilder, if any, about the new
1261 // parallel region to make cancellation barriers work properly.
1262 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1263 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind);
1264 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1265 HasCancel, OutlinedHelperName);
1266 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1267 return CGF.GenerateOpenMPCapturedStmtFunction(*CS, D);
1268}
1269
1270std::string CGOpenMPRuntime::getOutlinedHelperName(StringRef Name) const {
1271 std::string Suffix = getName({"omp_outlined"});
1272 return (Name + Suffix).str();
1273}
1274
1276 return getOutlinedHelperName(CGF.CurFn->getName());
1277}
1278
1279std::string CGOpenMPRuntime::getReductionFuncName(StringRef Name) const {
1280 std::string Suffix = getName({"omp", "reduction", "reduction_func"});
1281 return (Name + Suffix).str();
1282}
1283
1286 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1287 const RegionCodeGenTy &CodeGen) {
1288 const CapturedStmt *CS = D.getCapturedStmt(OMPD_parallel);
1290 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(CGF),
1291 CodeGen);
1292}
1293
1296 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1297 const RegionCodeGenTy &CodeGen) {
1298 const CapturedStmt *CS = D.getCapturedStmt(OMPD_teams);
1299 llvm::Function *OutlinedFn = emitParallelOrTeamsOutlinedFunction(
1300 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(CGF),
1301 CodeGen);
1302 // A teams body is called once per team and is not handed back to the runtime
1303 // as a callback, so unlike a parallel body it cannot be re-entered while a
1304 // call to it is live.
1305 OutlinedFn->setDoesNotRecurse();
1306 return OutlinedFn;
1307}
1308
1310 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1311 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1312 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1313 bool Tied, unsigned &NumberOfParts) {
1314 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
1315 PrePostActionTy &) {
1316 llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc());
1317 llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc());
1318 llvm::Value *TaskArgs[] = {
1319 UpLoc, ThreadID,
1320 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1321 TaskTVar->getType()->castAs<PointerType>())
1322 .getPointer(CGF)};
1323 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1324 CGM.getModule(), OMPRTL___kmpc_omp_task),
1325 TaskArgs);
1326 };
1327 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1328 UntiedCodeGen);
1329 CodeGen.setAction(Action);
1330 assert(!ThreadIDVar->getType()->isPointerType() &&
1331 "thread id variable must be of type kmp_int32 for tasks");
1332 const OpenMPDirectiveKind Region =
1333 isOpenMPTaskLoopDirective(D.getDirectiveKind()) ? OMPD_taskloop
1334 : OMPD_task;
1335 const CapturedStmt *CS = D.getCapturedStmt(Region);
1336 bool HasCancel = false;
1337 if (const auto *TD = dyn_cast<OMPTaskDirective>(&D))
1338 HasCancel = TD->hasCancel();
1339 else if (const auto *TD = dyn_cast<OMPTaskLoopDirective>(&D))
1340 HasCancel = TD->hasCancel();
1341 else if (const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(&D))
1342 HasCancel = TD->hasCancel();
1343 else if (const auto *TD = dyn_cast<OMPMaskedTaskLoopDirective>(&D))
1344 HasCancel = TD->hasCancel();
1345 else if (const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(&D))
1346 HasCancel = TD->hasCancel();
1347 else if (const auto *TD = dyn_cast<OMPParallelMaskedTaskLoopDirective>(&D))
1348 HasCancel = TD->hasCancel();
1349
1350 CodeGenFunction CGF(CGM, true);
1351 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1352 InnermostKind, HasCancel, Action);
1353 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1354 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1355 if (!Tied)
1356 NumberOfParts = Action.getNumberOfParts();
1357 return Res;
1358}
1359
1361 bool AtCurrentPoint) {
1362 auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn];
1363 assert(!Elem.ServiceInsertPt && "Insert point is set already.");
1364
1365 llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty);
1366 if (AtCurrentPoint) {
1367 Elem.ServiceInsertPt = new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt",
1368 CGF.Builder.GetInsertBlock());
1369 } else {
1370 Elem.ServiceInsertPt = new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt");
1371 Elem.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt->getIterator());
1372 }
1373}
1374
1376 auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn];
1377 if (Elem.ServiceInsertPt) {
1378 llvm::Instruction *Ptr = Elem.ServiceInsertPt;
1379 Elem.ServiceInsertPt = nullptr;
1380 Ptr->eraseFromParent();
1381 }
1382}
1383
1385 SourceLocation Loc,
1386 SmallString<128> &Buffer) {
1387 llvm::raw_svector_ostream OS(Buffer);
1388 // Build debug location
1390 OS << ";";
1391 if (auto *DbgInfo = CGF.getDebugInfo())
1392 OS << DbgInfo->remapDIPath(PLoc.getFilename());
1393 else
1394 OS << PLoc.getFilename();
1395 OS << ";";
1396 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1397 OS << FD->getQualifiedNameAsString();
1398 OS << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
1399 return OS.str();
1400}
1401
1403 SourceLocation Loc,
1404 unsigned Flags, bool EmitLoc) {
1405 uint32_t SrcLocStrSize;
1406 llvm::Constant *SrcLocStr;
1407 if ((!EmitLoc && CGM.getCodeGenOpts().getDebugInfo() ==
1408 llvm::codegenoptions::NoDebugInfo) ||
1409 Loc.isInvalid()) {
1410 SrcLocStr = OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1411 } else {
1412 std::string FunctionName;
1413 std::string FileName;
1414 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1415 FunctionName = FD->getQualifiedNameAsString();
1417 if (auto *DbgInfo = CGF.getDebugInfo())
1418 FileName = DbgInfo->remapDIPath(PLoc.getFilename());
1419 else
1420 FileName = PLoc.getFilename();
1421 unsigned Line = PLoc.getLine();
1422 unsigned Column = PLoc.getColumn();
1423 SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(FunctionName, FileName, Line,
1424 Column, SrcLocStrSize);
1425 }
1426 unsigned Reserved2Flags = getDefaultLocationReserved2Flags();
1427 return OMPBuilder.getOrCreateIdent(
1428 SrcLocStr, SrcLocStrSize, llvm::omp::IdentFlag(Flags), Reserved2Flags);
1429}
1430
1432 SourceLocation Loc) {
1433 assert(CGF.CurFn && "No function in current CodeGenFunction.");
1434 // If the OpenMPIRBuilder is used we need to use it for all thread id calls as
1435 // the clang invariants used below might be broken.
1436 if (CGM.getLangOpts().OpenMPIRBuilder) {
1437 SmallString<128> Buffer;
1438 OMPBuilder.updateToLocation(CGF.Builder);
1439 uint32_t SrcLocStrSize;
1440 auto *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(
1441 getIdentStringFromSourceLocation(CGF, Loc, Buffer), SrcLocStrSize);
1442 return OMPBuilder.getOrCreateThreadID(
1443 OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize));
1444 }
1445
1446 llvm::Value *ThreadID = nullptr;
1447 // Check whether we've already cached a load of the thread id in this
1448 // function.
1449 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1450 if (I != OpenMPLocThreadIDMap.end()) {
1451 ThreadID = I->second.ThreadID;
1452 if (ThreadID != nullptr)
1453 return ThreadID;
1454 }
1455 // If exceptions are enabled, do not use parameter to avoid possible crash.
1456 if (auto *OMPRegionInfo =
1457 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
1458 if (OMPRegionInfo->getThreadIDVariable()) {
1459 // Check if this an outlined function with thread id passed as argument.
1460 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1461 llvm::BasicBlock *TopBlock = CGF.AllocaInsertPt->getParent();
1462 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||
1463 !CGF.getLangOpts().CXXExceptions ||
1464 CGF.Builder.GetInsertBlock() == TopBlock ||
1465 !isa<llvm::Instruction>(LVal.getPointer(CGF)) ||
1466 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==
1467 TopBlock ||
1468 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==
1469 CGF.Builder.GetInsertBlock()) {
1470 ThreadID = CGF.EmitLoadOfScalar(LVal, Loc);
1471 // If value loaded in entry block, cache it and use it everywhere in
1472 // function.
1473 if (CGF.Builder.GetInsertBlock() == TopBlock)
1474 OpenMPLocThreadIDMap[CGF.CurFn].ThreadID = ThreadID;
1475 return ThreadID;
1476 }
1477 }
1478 }
1479
1480 // This is not an outlined function region - need to call __kmpc_int32
1481 // kmpc_global_thread_num(ident_t *loc).
1482 // Generate thread id value and cache this value for use across the
1483 // function.
1484 auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn];
1485 if (!Elem.ServiceInsertPt)
1487 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1488 CGF.Builder.SetInsertPoint(Elem.ServiceInsertPt);
1490 llvm::CallInst *Call = CGF.Builder.CreateCall(
1491 OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
1492 OMPRTL___kmpc_global_thread_num),
1493 emitUpdateLocation(CGF, Loc));
1494 Call->setCallingConv(CGF.getRuntimeCC());
1495 Elem.ThreadID = Call;
1496 return Call;
1497}
1498
1500 assert(CGF.CurFn && "No function in current CodeGenFunction.");
1501 if (OpenMPLocThreadIDMap.count(CGF.CurFn)) {
1503 OpenMPLocThreadIDMap.erase(CGF.CurFn);
1504 }
1505 if (auto I = FunctionUDRMap.find(CGF.CurFn); I != FunctionUDRMap.end()) {
1506 for (const auto *D : I->second)
1507 UDRMap.erase(D);
1508 FunctionUDRMap.erase(I);
1509 }
1510 if (auto I = FunctionUDMMap.find(CGF.CurFn); I != FunctionUDMMap.end()) {
1511 for (const auto *D : I->second)
1512 UDMMap.erase(D);
1513 FunctionUDMMap.erase(I);
1514 }
1517}
1518
1520 return OMPBuilder.IdentPtr;
1521}
1522
1523static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind
1525 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
1526 OMPDeclareTargetDeclAttr::getDeviceType(VD);
1527 if (!DevTy)
1528 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1529
1530 switch ((int)*DevTy) { // Avoid -Wcovered-switch-default
1531 case OMPDeclareTargetDeclAttr::DT_Host:
1532 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseHost;
1533 break;
1534 case OMPDeclareTargetDeclAttr::DT_NoHost:
1535 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNoHost;
1536 break;
1537 case OMPDeclareTargetDeclAttr::DT_Any:
1538 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny;
1539 break;
1540 default:
1541 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1542 break;
1543 }
1544}
1545
1546static llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind
1548 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> MapType =
1549 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1550 if (!MapType)
1551 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1552 switch ((int)*MapType) { // Avoid -Wcovered-switch-default
1553 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_To:
1554 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
1555 break;
1556 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Enter:
1557 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter;
1558 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Link:
1559 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
1560 break;
1561 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Local:
1562 // MT_Local variables don't need offload entry (device-local).
1563 llvm_unreachable("MT_Local should not reach convertCaptureClause");
1564 break;
1565 default:
1566 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1567 break;
1568 }
1569}
1570
1571static llvm::TargetRegionEntryInfo getEntryInfoFromPresumedLoc(
1572 CodeGenModule &CGM, llvm::OpenMPIRBuilder &OMPBuilder,
1573 SourceLocation BeginLoc, llvm::StringRef ParentName = "") {
1574
1575 auto FileInfoCallBack = [&]() {
1577 PresumedLoc PLoc = SM.getPresumedLoc(BeginLoc);
1578
1579 if (!CGM.getFileSystem()->exists(PLoc.getFilename()))
1580 PLoc = SM.getPresumedLoc(BeginLoc, /*UseLineDirectives=*/false);
1581
1582 return std::pair<std::string, uint64_t>(PLoc.getFilename(), PLoc.getLine());
1583 };
1584
1585 return OMPBuilder.getTargetEntryUniqueInfo(FileInfoCallBack,
1586 *CGM.getFileSystem(), ParentName);
1587}
1588
1590 auto AddrOfGlobal = [&VD, this]() { return CGM.GetAddrOfGlobal(VD); };
1591
1592 auto LinkageForVariable = [&VD, this]() {
1593 return CGM.getLLVMLinkageVarDefinition(VD);
1594 };
1595
1596 std::vector<llvm::GlobalVariable *> GeneratedRefs;
1597
1598 llvm::Type *LlvmPtrTy = CGM.getTypes().ConvertTypeForMem(
1599 CGM.getContext().getPointerType(VD->getType()));
1600 llvm::Constant *addr = OMPBuilder.getAddrOfDeclareTargetVar(
1602 VD->hasDefinition(CGM.getContext()) == VarDecl::DeclarationOnly,
1603 VD->isExternallyVisible(),
1605 VD->getCanonicalDecl()->getBeginLoc()),
1606 CGM.getMangledName(VD), GeneratedRefs, CGM.getLangOpts().OpenMPSimd,
1607 CGM.getLangOpts().OMPTargetTriples, LlvmPtrTy, AddrOfGlobal,
1608 LinkageForVariable);
1609
1610 if (!addr)
1611 return ConstantAddress::invalid();
1612 return ConstantAddress(addr, LlvmPtrTy, CGM.getContext().getDeclAlign(VD));
1613}
1614
1615llvm::Constant *
1617 assert(!CGM.getLangOpts().OpenMPUseTLS ||
1618 !CGM.getContext().getTargetInfo().isTLSSupported());
1619 // Lookup the entry, lazily creating it if necessary.
1620 std::string Suffix = getName({"cache", ""});
1621 return OMPBuilder.getOrCreateInternalVariable(
1622 CGM.Int8PtrPtrTy, Twine(CGM.getMangledName(VD)).concat(Suffix).str());
1623}
1624
1626 const VarDecl *VD,
1627 Address VDAddr,
1628 SourceLocation Loc) {
1629 if (CGM.getLangOpts().OpenMPUseTLS &&
1630 CGM.getContext().getTargetInfo().isTLSSupported())
1631 return VDAddr;
1632
1633 llvm::Type *VarTy = VDAddr.getElementType();
1634 llvm::Value *Args[] = {
1635 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
1636 CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.Int8PtrTy),
1637 CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
1639 return Address(
1640 CGF.EmitRuntimeCall(
1641 OMPBuilder.getOrCreateRuntimeFunction(
1642 CGM.getModule(), OMPRTL___kmpc_threadprivate_cached),
1643 Args),
1644 CGF.Int8Ty, VDAddr.getAlignment());
1645}
1646
1648 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
1649 llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
1650 // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
1651 // library.
1652 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);
1653 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1654 CGM.getModule(), OMPRTL___kmpc_global_thread_num),
1655 OMPLoc);
1656 // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
1657 // to register constructor/destructor for variable.
1658 llvm::Value *Args[] = {
1659 OMPLoc,
1660 CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.VoidPtrTy),
1661 Ctor, CopyCtor, Dtor};
1662 CGF.EmitRuntimeCall(
1663 OMPBuilder.getOrCreateRuntimeFunction(
1664 CGM.getModule(), OMPRTL___kmpc_threadprivate_register),
1665 Args);
1666}
1667
1669 const VarDecl *VD, Address VDAddr, SourceLocation Loc,
1670 bool PerformInit, CodeGenFunction *CGF) {
1671 if (CGM.getLangOpts().OpenMPUseTLS &&
1672 CGM.getContext().getTargetInfo().isTLSSupported())
1673 return nullptr;
1674
1675 VD = VD->getDefinition(CGM.getContext());
1676 if (VD && ThreadPrivateWithDefinition.insert(CGM.getMangledName(VD)).second) {
1677 QualType ASTTy = VD->getType();
1678
1679 llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
1680 const Expr *Init = VD->getAnyInitializer();
1681 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
1682 // Generate function that re-emits the declaration's initializer into the
1683 // threadprivate copy of the variable VD
1684 CodeGenFunction CtorCGF(CGM);
1685 auto *Dst = ImplicitParamDecl::Create(
1686 CGM.getContext(), /*DC=*/nullptr, Loc,
1687 /*Id=*/nullptr, CGM.getContext().VoidPtrTy, ImplicitParamKind::Other);
1688
1689 FunctionArgList Args{Dst};
1690 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1691 CGM.getContext().VoidPtrTy, Args);
1692 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
1693 std::string Name = getName({"__kmpc_global_ctor_", ""});
1694 llvm::Function *Fn =
1695 CGM.CreateGlobalInitOrCleanUpFunction(FTy, Name, FI, Loc);
1696 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI,
1697 Args, Loc, Loc);
1698 llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar(
1699 CtorCGF.GetAddrOfLocalVar(Dst), /*Volatile=*/false,
1700 CGM.getContext().VoidPtrTy, Dst->getLocation());
1701 Address Arg(ArgVal, CtorCGF.ConvertTypeForMem(ASTTy),
1702 VDAddr.getAlignment());
1703 CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(),
1704 /*IsInitializer=*/true);
1705 ArgVal = CtorCGF.EmitLoadOfScalar(
1706 CtorCGF.GetAddrOfLocalVar(Dst), /*Volatile=*/false,
1707 CGM.getContext().VoidPtrTy, Dst->getLocation());
1708 CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue);
1709 CtorCGF.FinishFunction();
1710 Ctor = Fn;
1711 }
1713 // Generate function that emits destructor call for the threadprivate copy
1714 // of the variable VD
1715 CodeGenFunction DtorCGF(CGM);
1716 auto *Dst = ImplicitParamDecl::Create(
1717 CGM.getContext(), /*DC=*/nullptr, Loc,
1718 /*Id=*/nullptr, CGM.getContext().VoidPtrTy, ImplicitParamKind::Other);
1719
1720 FunctionArgList Args{Dst};
1721 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1722 CGM.getContext().VoidTy, Args);
1723 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
1724 std::string Name = getName({"__kmpc_global_dtor_", ""});
1725 llvm::Function *Fn =
1726 CGM.CreateGlobalInitOrCleanUpFunction(FTy, Name, FI, Loc);
1727 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
1728 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args,
1729 Loc, Loc);
1730 // Create a scope with an artificial location for the body of this function.
1731 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
1732 llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar(
1733 DtorCGF.GetAddrOfLocalVar(Dst),
1734 /*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst->getLocation());
1735 DtorCGF.emitDestroy(
1736 Address(ArgVal, DtorCGF.Int8Ty, VDAddr.getAlignment()), ASTTy,
1737 DtorCGF.getDestroyer(ASTTy.isDestructedType()),
1738 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
1739 DtorCGF.FinishFunction();
1740 Dtor = Fn;
1741 }
1742 // Do not emit init function if it is not required.
1743 if (!Ctor && !Dtor)
1744 return nullptr;
1745
1746 // Copying constructor for the threadprivate variable.
1747 // Must be NULL - reserved by runtime, but currently it requires that this
1748 // parameter is always NULL. Otherwise it fires assertion.
1749 CopyCtor = llvm::Constant::getNullValue(CGM.DefaultPtrTy);
1750 if (Ctor == nullptr) {
1751 Ctor = llvm::Constant::getNullValue(CGM.DefaultPtrTy);
1752 }
1753 if (Dtor == nullptr) {
1754 Dtor = llvm::Constant::getNullValue(CGM.DefaultPtrTy);
1755 }
1756 if (!CGF) {
1757 auto *InitFunctionTy =
1758 llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false);
1759 std::string Name = getName({"__omp_threadprivate_init_", ""});
1760 llvm::Function *InitFunction = CGM.CreateGlobalInitOrCleanUpFunction(
1761 InitFunctionTy, Name, CGM.getTypes().arrangeNullaryFunction());
1762 CodeGenFunction InitCGF(CGM);
1763 FunctionArgList ArgList;
1764 InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction,
1765 CGM.getTypes().arrangeNullaryFunction(), ArgList,
1766 Loc, Loc);
1767 emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1768 InitCGF.FinishFunction();
1769 return InitFunction;
1770 }
1771 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1772 }
1773 return nullptr;
1774}
1775
1777 llvm::GlobalValue *GV) {
1778 std::optional<OMPDeclareTargetDeclAttr *> ActiveAttr =
1779 OMPDeclareTargetDeclAttr::getActiveAttr(FD);
1780
1781 // We only need to handle active 'indirect' declare target functions.
1782 if (!ActiveAttr || !(*ActiveAttr)->getIndirect())
1783 return;
1784
1785 // Get a mangled name to store the new device global in.
1786 llvm::TargetRegionEntryInfo EntryInfo = getEntryInfoFromPresumedLoc(
1788 SmallString<128> Name;
1789 OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(Name, EntryInfo);
1790
1791 // We need to generate a new global to hold the address of the indirectly
1792 // called device function. Doing this allows us to keep the visibility and
1793 // linkage of the associated function unchanged while allowing the runtime to
1794 // access its value.
1795 llvm::GlobalValue *Addr = GV;
1796 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
1797 llvm::PointerType *FnPtrTy = llvm::PointerType::get(
1798 CGM.getLLVMContext(),
1799 CGM.getModule().getDataLayout().getProgramAddressSpace());
1800 Addr = new llvm::GlobalVariable(
1801 CGM.getModule(), FnPtrTy,
1802 /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, GV, Name,
1803 nullptr, llvm::GlobalValue::NotThreadLocal,
1804 CGM.getModule().getDataLayout().getDefaultGlobalsAddressSpace());
1805 Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1806 }
1807
1808 // Register the indirect Vtable:
1809 // This is similar to OMPTargetGlobalVarEntryIndirect, except that the
1810 // size field refers to the size of memory pointed to, not the size of
1811 // the pointer symbol itself (which is implicitly the size of a pointer).
1812 OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(
1813 Name, Addr, CGM.GetTargetTypeStoreSize(CGM.VoidPtrTy).getQuantity(),
1814 llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect,
1815 llvm::GlobalValue::WeakODRLinkage);
1816}
1817
1818void CGOpenMPRuntime::registerVTableOffloadEntry(llvm::GlobalVariable *VTable,
1819 const VarDecl *VD) {
1820 // TODO: add logic to avoid duplicate vtable registrations per
1821 // translation unit; though for external linkage, this should no
1822 // longer be an issue - or at least we can avoid the issue by
1823 // checking for an existing offloading entry. But, perhaps the
1824 // better approach is to defer emission of the vtables and offload
1825 // entries until later (by tracking a list of items that need to be
1826 // emitted).
1827
1828 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1829
1830 // Generate a new externally visible global to point to the
1831 // internally visible vtable. Doing this allows us to keep the
1832 // visibility and linkage of the associated vtable unchanged while
1833 // allowing the runtime to access its value. The externally
1834 // visible global var needs to be emitted with a unique mangled
1835 // name that won't conflict with similarly named (internal)
1836 // vtables in other translation units.
1837
1838 // Register vtable with source location of dynamic object in map
1839 // clause.
1840 llvm::TargetRegionEntryInfo EntryInfo = getEntryInfoFromPresumedLoc(
1842 VTable->getName());
1843
1844 llvm::GlobalVariable *Addr = VTable;
1845 SmallString<128> AddrName;
1846 OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(AddrName, EntryInfo);
1847 AddrName.append("addr");
1848
1849 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
1850 Addr = new llvm::GlobalVariable(
1851 CGM.getModule(), VTable->getType(),
1852 /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, VTable,
1853 AddrName,
1854 /*InsertBefore=*/nullptr, llvm::GlobalValue::NotThreadLocal,
1855 CGM.getModule().getDataLayout().getDefaultGlobalsAddressSpace());
1856 Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1857 }
1858 OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(
1859 AddrName, VTable,
1860 CGM.getDataLayout().getTypeAllocSize(VTable->getInitializer()->getType()),
1861 llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirectVTable,
1862 llvm::GlobalValue::WeakODRLinkage);
1863}
1864
1867 const VarDecl *VD) {
1868 // Register C++ VTable to OpenMP Offload Entry if it's a new
1869 // CXXRecordDecl.
1870 if (CXXRecord && CXXRecord->hasDefinition() && CXXRecord->isDynamicClass() &&
1871 !CGM.getOpenMPRuntime().VTableDeclMap.contains(CXXRecord)) {
1872 auto Res = CGM.getOpenMPRuntime().VTableDeclMap.try_emplace(CXXRecord, VD);
1873 if (Res.second) {
1874 CGM.EmitVTable(CXXRecord);
1875 CodeGenVTables VTables = CGM.getVTables();
1876 llvm::GlobalVariable *VTablesAddr = VTables.GetAddrOfVTable(CXXRecord);
1877 assert(VTablesAddr && "Expected non-null VTable address");
1878 // Must set VTables to weak since we're emitting them in multiple TUs now
1879 if (VTablesAddr->hasExternalLinkage())
1880 VTablesAddr->setLinkage(llvm::GlobalValue::WeakODRLinkage);
1881 CGM.getOpenMPRuntime().registerVTableOffloadEntry(VTablesAddr, VD);
1882 // Emit VTable for all the fields containing dynamic CXXRecord
1883 for (const FieldDecl *Field : CXXRecord->fields()) {
1884 if (CXXRecordDecl *RecordDecl = Field->getType()->getAsCXXRecordDecl())
1886 }
1887 // Emit VTable for all dynamic parent class
1888 for (CXXBaseSpecifier &Base : CXXRecord->bases()) {
1889 if (CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl())
1890 emitAndRegisterVTable(CGM, BaseDecl, VD);
1891 }
1892 }
1893 }
1894}
1895
1897 // Register VTable by scanning through the map clause of OpenMP target region.
1898 // Get CXXRecordDecl and VarDecl from Expr.
1899 auto GetVTableDecl = [](const Expr *E) {
1900 QualType VDTy = E->getType();
1901 CXXRecordDecl *CXXRecord = nullptr;
1902 if (const auto *RefType = VDTy->getAs<LValueReferenceType>())
1903 VDTy = RefType->getPointeeType();
1904 if (VDTy->isPointerType())
1906 else
1907 CXXRecord = VDTy->getAsCXXRecordDecl();
1908
1909 const VarDecl *VD = nullptr;
1910 if (auto *DRE = dyn_cast<DeclRefExpr>(E)) {
1911 // Handle BindingDecls by redirecting to their DecompositionDecl.
1912 if (auto *BD = dyn_cast<BindingDecl>(DRE->getDecl()))
1913 VD = cast<VarDecl>(BD->getDecomposedDecl());
1914 else
1915 VD = cast<VarDecl>(DRE->getDecl());
1916 } else if (auto *MRE = dyn_cast<MemberExpr>(E)) {
1917 if (auto *BaseDRE = dyn_cast<DeclRefExpr>(MRE->getBase())) {
1918 if (auto *BaseVD = dyn_cast<VarDecl>(BaseDRE->getDecl()))
1919 VD = BaseVD;
1920 }
1921 }
1922 return std::pair<CXXRecordDecl *, const VarDecl *>(CXXRecord, VD);
1923 };
1924 // Collect VTable from OpenMP map clause.
1925 for (const auto *C : D.getClausesOfKind<OMPMapClause>()) {
1926 for (const auto *E : C->varlist()) {
1927 auto DeclPair = GetVTableDecl(E);
1928 // Ensure VD is not null
1929 if (DeclPair.second)
1930 emitAndRegisterVTable(CGM, DeclPair.first, DeclPair.second);
1931 }
1932 }
1933}
1934
1936 QualType VarType,
1937 StringRef Name) {
1938 std::string Suffix = getName({"artificial", ""});
1939 llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType);
1940 llvm::GlobalVariable *GAddr = OMPBuilder.getOrCreateInternalVariable(
1941 VarLVType, Twine(Name).concat(Suffix).str());
1942 if (CGM.getLangOpts().OpenMP && CGM.getLangOpts().OpenMPUseTLS &&
1943 CGM.getTarget().isTLSSupported()) {
1944 GAddr->setThreadLocal(/*Val=*/true);
1945 return Address(GAddr, GAddr->getValueType(),
1946 CGM.getContext().getTypeAlignInChars(VarType));
1947 }
1948 std::string CacheSuffix = getName({"cache", ""});
1949 llvm::Value *Args[] = {
1952 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy),
1953 CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy,
1954 /*isSigned=*/false),
1955 OMPBuilder.getOrCreateInternalVariable(
1956 CGM.VoidPtrPtrTy,
1957 Twine(Name).concat(Suffix).concat(CacheSuffix).str())};
1958 return Address(
1960 CGF.EmitRuntimeCall(
1961 OMPBuilder.getOrCreateRuntimeFunction(
1962 CGM.getModule(), OMPRTL___kmpc_threadprivate_cached),
1963 Args),
1964 CGF.Builder.getPtrTy(0)),
1965 VarLVType, CGM.getContext().getTypeAlignInChars(VarType));
1966}
1967
1969 const RegionCodeGenTy &ThenGen,
1970 const RegionCodeGenTy &ElseGen) {
1971 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
1972
1973 // If the condition constant folds and can be elided, try to avoid emitting
1974 // the condition and the dead arm of the if/else.
1975 bool CondConstant;
1976 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) {
1977 if (CondConstant)
1978 ThenGen(CGF);
1979 else
1980 ElseGen(CGF);
1981 return;
1982 }
1983
1984 // Otherwise, the condition did not fold, or we couldn't elide it. Just
1985 // emit the conditional branch.
1986 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then");
1987 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else");
1988 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end");
1989 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0);
1990
1991 // Emit the 'then' code.
1992 CGF.EmitBlock(ThenBlock);
1993 ThenGen(CGF);
1994 CGF.EmitBranch(ContBlock);
1995 // Emit the 'else' code if present.
1996 // There is no need to emit line number for unconditional branch.
1998 CGF.EmitBlock(ElseBlock);
1999 ElseGen(CGF);
2000 // There is no need to emit line number for unconditional branch.
2002 CGF.EmitBranch(ContBlock);
2003 // Emit the continuation block for code after the if.
2004 CGF.EmitBlock(ContBlock, /*IsFinished=*/true);
2005}
2006
2008 CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn,
2009 ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond,
2010 llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier,
2011 OpenMPSeverityClauseKind Severity, const Expr *Message) {
2012 if (!CGF.HaveInsertPoint())
2013 return;
2014
2015 // The caller waits for the parallel region to finish, so its captured stack
2016 // variables remain alive until the region completes. Global variables also
2017 // remain alive. Record this lifetime guarantee on the outlined function's
2018 // parameters, even if other code has access to the same storage.
2019 // The storage may still be modified or accessed through other pointers.
2020 // For a captured pointer variable, this guarantees the lifetime of the
2021 // variable itself, not the object it points to.
2022 if (!CGM.getLangOpts().OpenMPIsTargetDevice) {
2023 assert(OutlinedFn->arg_size() == CapturedVars.size() + 2 &&
2024 "Expected tid and bound tid before the captured arguments");
2025 for (auto [I, Captured] : llvm::enumerate(CapturedVars)) {
2026 if (Captured->getType()->isPointerTy() &&
2027 llvm::isa<llvm::AllocaInst, llvm::GlobalVariable>(
2028 llvm::getUnderlyingObject(Captured)))
2029 OutlinedFn->addParamAttr(I + 2, llvm::Attribute::NoFreeObj);
2030 }
2031 }
2032
2033 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
2034 auto &M = CGM.getModule();
2035 auto &&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc,
2036 this](CodeGenFunction &CGF, PrePostActionTy &) {
2037 // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
2038 llvm::Value *Args[] = {
2039 RTLoc,
2040 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
2041 OutlinedFn};
2043 RealArgs.append(std::begin(Args), std::end(Args));
2044 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
2045
2046 llvm::FunctionCallee RTLFn =
2047 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_fork_call);
2048 CGF.EmitRuntimeCall(RTLFn, RealArgs);
2049 };
2050 auto &&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc, Loc,
2051 this](CodeGenFunction &CGF, PrePostActionTy &) {
2053 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);
2054 // Build calls:
2055 // __kmpc_serialized_parallel(&Loc, GTid);
2056 llvm::Value *Args[] = {RTLoc, ThreadID};
2057 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2058 M, OMPRTL___kmpc_serialized_parallel),
2059 Args);
2060
2061 // OutlinedFn(&GTid, &zero_bound, CapturedStruct);
2062 Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc);
2063 RawAddress ZeroAddrBound =
2065 /*Name=*/".bound.zero.addr");
2066 CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddrBound);
2068 // ThreadId for serialized parallels is 0.
2069 OutlinedFnArgs.push_back(ThreadIDAddr.emitRawPointer(CGF));
2070 OutlinedFnArgs.push_back(ZeroAddrBound.getPointer());
2071 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
2072
2073 // Ensure we do not inline the function. This is trivially true for the ones
2074 // passed to __kmpc_fork_call but the ones called in serialized regions
2075 // could be inlined. This is not a perfect but it is closer to the invariant
2076 // we want, namely, every data environment starts with a new function.
2077 // TODO: We should pass the if condition to the runtime function and do the
2078 // handling there. Much cleaner code.
2079 OutlinedFn->removeFnAttr(llvm::Attribute::AlwaysInline);
2080 OutlinedFn->addFnAttr(llvm::Attribute::NoInline);
2081 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
2082
2083 // __kmpc_end_serialized_parallel(&Loc, GTid);
2084 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
2085 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2086 M, OMPRTL___kmpc_end_serialized_parallel),
2087 EndArgs);
2088 };
2089 if (IfCond) {
2090 emitIfClause(CGF, IfCond, ThenGen, ElseGen);
2091 } else {
2092 RegionCodeGenTy ThenRCG(ThenGen);
2093 ThenRCG(CGF);
2094 }
2095}
2096
2097// If we're inside an (outlined) parallel region, use the region info's
2098// thread-ID variable (it is passed in a first argument of the outlined function
2099// as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
2100// regular serial code region, get thread ID by calling kmp_int32
2101// kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
2102// return the address of that temp.
2104 SourceLocation Loc) {
2105 if (auto *OMPRegionInfo =
2106 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
2107 if (OMPRegionInfo->getThreadIDVariable())
2108 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
2109
2110 llvm::Value *ThreadID = getThreadID(CGF, Loc);
2111 QualType Int32Ty =
2112 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
2113 Address ThreadIDTemp =
2114 CGF.CreateMemTempWithoutCast(Int32Ty, /*Name*/ ".threadid_temp.");
2115 CGF.EmitStoreOfScalar(ThreadID,
2116 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty));
2117
2118 return ThreadIDTemp;
2119}
2120
2121llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {
2122 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
2123 std::string Name = getName({Prefix, "var"});
2124 llvm::GlobalVariable *GV =
2125 OMPBuilder.getOrCreateInternalVariable(KmpCriticalNameTy, Name);
2126 CGM.setDSOLocal(GV);
2127 return GV;
2128}
2129
2130namespace {
2131/// Common pre(post)-action for different OpenMP constructs.
2132class CommonActionTy final : public PrePostActionTy {
2133 llvm::FunctionCallee EnterCallee;
2134 ArrayRef<llvm::Value *> EnterArgs;
2135 llvm::FunctionCallee ExitCallee;
2136 ArrayRef<llvm::Value *> ExitArgs;
2137 bool Conditional;
2138 llvm::BasicBlock *ContBlock = nullptr;
2139
2140public:
2141 CommonActionTy(llvm::FunctionCallee EnterCallee,
2142 ArrayRef<llvm::Value *> EnterArgs,
2143 llvm::FunctionCallee ExitCallee,
2144 ArrayRef<llvm::Value *> ExitArgs, bool Conditional = false)
2145 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
2146 ExitArgs(ExitArgs), Conditional(Conditional) {}
2147 void Enter(CodeGenFunction &CGF) override {
2148 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
2149 if (Conditional) {
2150 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
2151 auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
2152 ContBlock = CGF.createBasicBlock("omp_if.end");
2153 // Generate the branch (If-stmt)
2154 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
2155 CGF.EmitBlock(ThenBlock);
2156 }
2157 }
2158 void Done(CodeGenFunction &CGF) {
2159 // Emit the rest of blocks/branches
2160 CGF.EmitBranch(ContBlock);
2161 CGF.EmitBlock(ContBlock, true);
2162 }
2163 void Exit(CodeGenFunction &CGF) override {
2164 CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
2165 }
2166};
2167} // anonymous namespace
2168
2170 StringRef CriticalName,
2171 const RegionCodeGenTy &CriticalOpGen,
2172 SourceLocation Loc, const Expr *Hint) {
2173 // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]);
2174 // CriticalOpGen();
2175 // __kmpc_end_critical(ident_t *, gtid, Lock);
2176 // Prepare arguments and build a call to __kmpc_critical
2177 if (!CGF.HaveInsertPoint())
2178 return;
2179 llvm::FunctionCallee RuntimeFcn = OMPBuilder.getOrCreateRuntimeFunction(
2180 CGM.getModule(),
2181 Hint ? OMPRTL___kmpc_critical_with_hint : OMPRTL___kmpc_critical);
2182 llvm::Value *LockVar = getCriticalRegionLock(CriticalName);
2183 unsigned LockVarArgIdx = 2;
2184 if (cast<llvm::GlobalVariable>(LockVar)->getAddressSpace() !=
2185 RuntimeFcn.getFunctionType()
2186 ->getParamType(LockVarArgIdx)
2187 ->getPointerAddressSpace())
2188 LockVar = CGF.Builder.CreateAddrSpaceCast(
2189 LockVar, RuntimeFcn.getFunctionType()->getParamType(LockVarArgIdx));
2190 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2191 LockVar};
2192 llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args),
2193 std::end(Args));
2194 if (Hint) {
2195 EnterArgs.push_back(CGF.Builder.CreateIntCast(
2196 CGF.EmitScalarExpr(Hint), CGM.Int32Ty, /*isSigned=*/false));
2197 }
2198 CommonActionTy Action(RuntimeFcn, EnterArgs,
2199 OMPBuilder.getOrCreateRuntimeFunction(
2200 CGM.getModule(), OMPRTL___kmpc_end_critical),
2201 Args);
2202 CriticalOpGen.setAction(Action);
2203 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);
2204}
2205
2207 const RegionCodeGenTy &MasterOpGen,
2208 SourceLocation Loc) {
2209 if (!CGF.HaveInsertPoint())
2210 return;
2211 // if(__kmpc_master(ident_t *, gtid)) {
2212 // MasterOpGen();
2213 // __kmpc_end_master(ident_t *, gtid);
2214 // }
2215 // Prepare arguments and build a call to __kmpc_master
2216 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2217 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2218 CGM.getModule(), OMPRTL___kmpc_master),
2219 Args,
2220 OMPBuilder.getOrCreateRuntimeFunction(
2221 CGM.getModule(), OMPRTL___kmpc_end_master),
2222 Args,
2223 /*Conditional=*/true);
2224 MasterOpGen.setAction(Action);
2225 emitInlinedDirective(CGF, OMPD_master, MasterOpGen);
2226 Action.Done(CGF);
2227}
2228
2230 const RegionCodeGenTy &MaskedOpGen,
2231 SourceLocation Loc, const Expr *Filter) {
2232 if (!CGF.HaveInsertPoint())
2233 return;
2234 // if(__kmpc_masked(ident_t *, gtid, filter)) {
2235 // MaskedOpGen();
2236 // __kmpc_end_masked(iden_t *, gtid);
2237 // }
2238 // Prepare arguments and build a call to __kmpc_masked
2239 llvm::Value *FilterVal = Filter
2240 ? CGF.EmitScalarExpr(Filter, CGF.Int32Ty)
2241 : llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/0);
2242 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2243 FilterVal};
2244 llvm::Value *ArgsEnd[] = {emitUpdateLocation(CGF, Loc),
2245 getThreadID(CGF, Loc)};
2246 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2247 CGM.getModule(), OMPRTL___kmpc_masked),
2248 Args,
2249 OMPBuilder.getOrCreateRuntimeFunction(
2250 CGM.getModule(), OMPRTL___kmpc_end_masked),
2251 ArgsEnd,
2252 /*Conditional=*/true);
2253 MaskedOpGen.setAction(Action);
2254 emitInlinedDirective(CGF, OMPD_masked, MaskedOpGen);
2255 Action.Done(CGF);
2256}
2257
2259 SourceLocation Loc) {
2260 if (!CGF.HaveInsertPoint())
2261 return;
2262 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2263 OMPBuilder.createTaskyield(CGF.Builder);
2264 } else {
2265 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
2266 llvm::Value *Args[] = {
2267 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2268 llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)};
2269 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2270 CGM.getModule(), OMPRTL___kmpc_omp_taskyield),
2271 Args);
2272 }
2273
2274 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
2275 Region->emitUntiedSwitch(CGF);
2276}
2277
2279 const RegionCodeGenTy &TaskgroupOpGen,
2280 SourceLocation Loc) {
2281 if (!CGF.HaveInsertPoint())
2282 return;
2283 // __kmpc_taskgroup(ident_t *, gtid);
2284 // TaskgroupOpGen();
2285 // __kmpc_end_taskgroup(ident_t *, gtid);
2286 // Prepare arguments and build a call to __kmpc_taskgroup
2287 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2288 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2289 CGM.getModule(), OMPRTL___kmpc_taskgroup),
2290 Args,
2291 OMPBuilder.getOrCreateRuntimeFunction(
2292 CGM.getModule(), OMPRTL___kmpc_end_taskgroup),
2293 Args);
2294 TaskgroupOpGen.setAction(Action);
2295 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);
2296}
2297
2298/// Given an array of pointers to variables, project the address of a
2299/// given variable.
2301 unsigned Index, const VarDecl *Var) {
2302 // Pull out the pointer to the variable.
2303 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index);
2304 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr);
2305
2306 llvm::Type *ElemTy = CGF.ConvertTypeForMem(Var->getType());
2307 return Address(Ptr, ElemTy, CGF.getContext().getDeclAlign(Var));
2308}
2309
2311 CodeGenModule &CGM, llvm::Type *ArgsElemType,
2312 ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
2313 ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,
2314 SourceLocation Loc) {
2315 ASTContext &C = CGM.getContext();
2316 // void copy_func(void *LHSArg, void *RHSArg);
2317
2318 auto *LHSArg =
2319 ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
2320 C.VoidPtrTy, ImplicitParamKind::Other);
2321 auto *RHSArg =
2322 ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
2323 C.VoidPtrTy, ImplicitParamKind::Other);
2324 FunctionArgList Args{LHSArg, RHSArg};
2325 const auto &CGFI =
2326 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
2327 std::string Name =
2328 CGM.getOpenMPRuntime().getName({"omp", "copyprivate", "copy_func"});
2329 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
2330 llvm::GlobalValue::InternalLinkage, Name,
2331 &CGM.getModule());
2333 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
2334 Fn->addFnAttr("sample-profile-suffix-elision-policy", "selected");
2335 Fn->setDoesNotRecurse();
2336 CodeGenFunction CGF(CGM);
2337 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
2338 // Dest = (void*[n])(LHSArg);
2339 // Src = (void*[n])(RHSArg);
2341 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(LHSArg)),
2342 CGF.Builder.getPtrTy(0)),
2343 ArgsElemType, CGF.getPointerAlign());
2345 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(RHSArg)),
2346 CGF.Builder.getPtrTy(0)),
2347 ArgsElemType, CGF.getPointerAlign());
2348 // *(Type0*)Dst[0] = *(Type0*)Src[0];
2349 // *(Type1*)Dst[1] = *(Type1*)Src[1];
2350 // ...
2351 // *(Typen*)Dst[n] = *(Typen*)Src[n];
2352 for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
2353 const auto *DestVar =
2354 cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
2355 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar);
2356
2357 const auto *SrcVar =
2358 cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
2359 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar);
2360
2361 const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
2362 QualType Type = VD->getType();
2363 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
2364 }
2365 CGF.FinishFunction();
2366 return Fn;
2367}
2368
2370 const RegionCodeGenTy &SingleOpGen,
2371 SourceLocation Loc,
2372 ArrayRef<const Expr *> CopyprivateVars,
2373 ArrayRef<const Expr *> SrcExprs,
2374 ArrayRef<const Expr *> DstExprs,
2375 ArrayRef<const Expr *> AssignmentOps) {
2376 if (!CGF.HaveInsertPoint())
2377 return;
2378 assert(CopyprivateVars.size() == SrcExprs.size() &&
2379 CopyprivateVars.size() == DstExprs.size() &&
2380 CopyprivateVars.size() == AssignmentOps.size());
2381 ASTContext &C = CGM.getContext();
2382 // int32 did_it = 0;
2383 // if(__kmpc_single(ident_t *, gtid)) {
2384 // SingleOpGen();
2385 // __kmpc_end_single(ident_t *, gtid);
2386 // did_it = 1;
2387 // }
2388 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
2389 // <copy_func>, did_it);
2390
2391 Address DidIt = Address::invalid();
2392 if (!CopyprivateVars.empty()) {
2393 // int32 did_it = 0;
2394 QualType KmpInt32Ty =
2395 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2396 DidIt = CGF.CreateMemTempWithoutCast(KmpInt32Ty, ".omp.copyprivate.did_it");
2397 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt);
2398 }
2399 // Prepare arguments and build a call to __kmpc_single
2400 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2401 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2402 CGM.getModule(), OMPRTL___kmpc_single),
2403 Args,
2404 OMPBuilder.getOrCreateRuntimeFunction(
2405 CGM.getModule(), OMPRTL___kmpc_end_single),
2406 Args,
2407 /*Conditional=*/true);
2408 SingleOpGen.setAction(Action);
2409 emitInlinedDirective(CGF, OMPD_single, SingleOpGen);
2410 if (DidIt.isValid()) {
2411 // did_it = 1;
2412 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt);
2413 }
2414 Action.Done(CGF);
2415 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
2416 // <copy_func>, did_it);
2417 if (DidIt.isValid()) {
2418 llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
2419 QualType CopyprivateArrayTy = C.getConstantArrayType(
2420 C.VoidPtrTy, ArraySize, nullptr, ArraySizeModifier::Normal,
2421 /*IndexTypeQuals=*/0);
2422 // Create a list of all private variables for copyprivate.
2423 Address CopyprivateList = CGF.CreateMemTempWithoutCast(
2424 CopyprivateArrayTy, ".omp.copyprivate.cpr_list");
2425 for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
2426 Address Elem = CGF.Builder.CreateConstArrayGEP(CopyprivateList, I);
2427 CGF.Builder.CreateStore(
2429 CGF.EmitLValue(CopyprivateVars[I]).getPointer(CGF),
2430 CGF.VoidPtrTy),
2431 Elem);
2432 }
2433 // Build function that copies private values from single region to all other
2434 // threads in the corresponding parallel region.
2435 llvm::Value *CpyFn = emitCopyprivateCopyFunction(
2436 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy), CopyprivateVars,
2437 SrcExprs, DstExprs, AssignmentOps, Loc);
2438 llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy);
2440 CopyprivateList, CGF.VoidPtrTy, CGF.Int8Ty);
2441 llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt);
2442 llvm::Value *Args[] = {
2443 emitUpdateLocation(CGF, Loc), // ident_t *<loc>
2444 getThreadID(CGF, Loc), // i32 <gtid>
2445 BufSize, // size_t <buf_size>
2446 CL.emitRawPointer(CGF), // void *<copyprivate list>
2447 CpyFn, // void (*) (void *, void *) <copy_func>
2448 DidItVal // i32 did_it
2449 };
2450 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2451 CGM.getModule(), OMPRTL___kmpc_copyprivate),
2452 Args);
2453 }
2454}
2455
2457 const RegionCodeGenTy &OrderedOpGen,
2458 SourceLocation Loc, bool IsThreads) {
2459 if (!CGF.HaveInsertPoint())
2460 return;
2461 // __kmpc_ordered(ident_t *, gtid);
2462 // OrderedOpGen();
2463 // __kmpc_end_ordered(ident_t *, gtid);
2464 // Prepare arguments and build a call to __kmpc_ordered
2465 if (IsThreads) {
2466 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2467 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2468 CGM.getModule(), OMPRTL___kmpc_ordered),
2469 Args,
2470 OMPBuilder.getOrCreateRuntimeFunction(
2471 CGM.getModule(), OMPRTL___kmpc_end_ordered),
2472 Args);
2473 OrderedOpGen.setAction(Action);
2474 emitInlinedDirective(CGF, OMPD_ordered_blockassoc, OrderedOpGen);
2475 return;
2476 }
2477 emitInlinedDirective(CGF, OMPD_ordered_blockassoc, OrderedOpGen);
2478}
2479
2481 unsigned Flags;
2482 if (Kind == OMPD_for)
2483 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
2484 else if (Kind == OMPD_sections)
2485 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
2486 else if (Kind == OMPD_single)
2487 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
2488 else if (Kind == OMPD_barrier)
2489 Flags = OMP_IDENT_BARRIER_EXPL;
2490 else
2491 Flags = OMP_IDENT_BARRIER_IMPL;
2492 return Flags;
2493}
2494
2496 CodeGenFunction &CGF, const OMPLoopDirective &S,
2497 OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const {
2498 // Check if the loop directive is actually a doacross loop directive. In this
2499 // case choose static, 1 schedule.
2500 if (llvm::any_of(
2501 S.getClausesOfKind<OMPOrderedClause>(),
2502 [](const OMPOrderedClause *C) { return C->getNumForLoops(); })) {
2503 ScheduleKind = OMPC_SCHEDULE_static;
2504 // Chunk size is 1 in this case.
2505 llvm::APInt ChunkSize(32, 1);
2506 ChunkExpr = IntegerLiteral::Create(
2507 CGF.getContext(), ChunkSize,
2508 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
2509 SourceLocation());
2510 }
2511}
2512
2514 OpenMPDirectiveKind Kind, bool EmitChecks,
2515 bool ForceSimpleCall) {
2516 // Check if we should use the OMPBuilder
2517 auto *OMPRegionInfo =
2518 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo);
2519 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2520 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
2521 cantFail(OMPBuilder.createBarrier(CGF.Builder, Kind, ForceSimpleCall,
2522 EmitChecks));
2523 CGF.Builder.restoreIP(AfterIP);
2524 return;
2525 }
2526
2527 if (!CGF.HaveInsertPoint())
2528 return;
2529 // Build call __kmpc_cancel_barrier(loc, thread_id);
2530 // Build call __kmpc_barrier(loc, thread_id);
2531 unsigned Flags = getDefaultFlagsForBarriers(Kind);
2532 // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,
2533 // thread_id);
2534 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
2535 getThreadID(CGF, Loc)};
2536 if (OMPRegionInfo) {
2537 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
2538 llvm::Value *Result = CGF.EmitRuntimeCall(
2539 OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
2540 OMPRTL___kmpc_cancel_barrier),
2541 Args);
2542 if (EmitChecks) {
2543 // if (__kmpc_cancel_barrier()) {
2544 // exit from construct;
2545 // }
2546 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
2547 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
2548 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
2549 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
2550 CGF.EmitBlock(ExitBB);
2551 // exit from construct;
2552 CodeGenFunction::JumpDest CancelDestination =
2553 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
2554 CGF.EmitBranchThroughCleanup(CancelDestination);
2555 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
2556 }
2557 return;
2558 }
2559 }
2560 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2561 CGM.getModule(), OMPRTL___kmpc_barrier),
2562 Args);
2563}
2564
2566 Expr *ME, bool IsFatal) {
2567 llvm::Value *MVL = ME ? CGF.EmitScalarExpr(ME)
2568 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
2569 // Build call void __kmpc_error(ident_t *loc, int severity, const char
2570 // *message)
2571 llvm::Value *Args[] = {
2572 emitUpdateLocation(CGF, Loc, /*Flags=*/0, /*GenLoc=*/true),
2573 llvm::ConstantInt::get(CGM.Int32Ty, IsFatal ? 2 : 1),
2574 CGF.Builder.CreatePointerCast(MVL, CGM.Int8PtrTy)};
2575 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2576 CGM.getModule(), OMPRTL___kmpc_error),
2577 Args);
2578}
2579
2580/// Map the OpenMP loop schedule to the runtime enumeration.
2581static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind,
2582 bool Chunked, bool Ordered) {
2583 switch (ScheduleKind) {
2584 case OMPC_SCHEDULE_static:
2585 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
2586 : (Ordered ? OMP_ord_static : OMP_sch_static);
2587 case OMPC_SCHEDULE_dynamic:
2588 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
2589 case OMPC_SCHEDULE_guided:
2590 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
2591 case OMPC_SCHEDULE_runtime:
2592 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
2593 case OMPC_SCHEDULE_auto:
2594 return Ordered ? OMP_ord_auto : OMP_sch_auto;
2596 assert(!Chunked && "chunk was specified but schedule kind not known");
2597 return Ordered ? OMP_ord_static : OMP_sch_static;
2598 }
2599 llvm_unreachable("Unexpected runtime schedule");
2600}
2601
2602/// Map the OpenMP distribute schedule to the runtime enumeration.
2603static OpenMPSchedType
2605 // only static is allowed for dist_schedule
2606 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
2607}
2608
2610 bool Chunked) const {
2611 OpenMPSchedType Schedule =
2612 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
2613 return Schedule == OMP_sch_static;
2614}
2615
2617 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
2618 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
2619 return Schedule == OMP_dist_sch_static;
2620}
2621
2623 bool Chunked) const {
2624 OpenMPSchedType Schedule =
2625 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
2626 return Schedule == OMP_sch_static_chunked;
2627}
2628
2630 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
2631 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
2632 return Schedule == OMP_dist_sch_static_chunked;
2633}
2634
2636 OpenMPSchedType Schedule =
2637 getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);
2638 assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here");
2639 return Schedule != OMP_sch_static;
2640}
2641
2642static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule,
2645 int Modifier = 0;
2646 switch (M1) {
2647 case OMPC_SCHEDULE_MODIFIER_monotonic:
2648 Modifier = OMP_sch_modifier_monotonic;
2649 break;
2650 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2651 Modifier = OMP_sch_modifier_nonmonotonic;
2652 break;
2653 case OMPC_SCHEDULE_MODIFIER_simd:
2654 if (Schedule == OMP_sch_static_chunked)
2655 Schedule = OMP_sch_static_balanced_chunked;
2656 break;
2659 break;
2660 }
2661 switch (M2) {
2662 case OMPC_SCHEDULE_MODIFIER_monotonic:
2663 Modifier = OMP_sch_modifier_monotonic;
2664 break;
2665 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2666 Modifier = OMP_sch_modifier_nonmonotonic;
2667 break;
2668 case OMPC_SCHEDULE_MODIFIER_simd:
2669 if (Schedule == OMP_sch_static_chunked)
2670 Schedule = OMP_sch_static_balanced_chunked;
2671 break;
2674 break;
2675 }
2676 // OpenMP 5.0, 2.9.2 Worksharing-Loop Construct, Desription.
2677 // If the static schedule kind is specified or if the ordered clause is
2678 // specified, and if the nonmonotonic modifier is not specified, the effect is
2679 // as if the monotonic modifier is specified. Otherwise, unless the monotonic
2680 // modifier is specified, the effect is as if the nonmonotonic modifier is
2681 // specified.
2682 if (CGM.getLangOpts().OpenMP >= 50 && Modifier == 0) {
2683 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
2684 Schedule == OMP_sch_static_balanced_chunked ||
2685 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||
2686 Schedule == OMP_dist_sch_static_chunked ||
2687 Schedule == OMP_dist_sch_static ||
2688 Schedule == OMP_dist_sch_static_chunked_sch_static_chunkone))
2689 Modifier = OMP_sch_modifier_nonmonotonic;
2690 }
2691 return Schedule | Modifier;
2692}
2693
2696 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
2697 bool Ordered, const DispatchRTInput &DispatchValues) {
2698 if (!CGF.HaveInsertPoint())
2699 return;
2700 OpenMPSchedType Schedule = getRuntimeSchedule(
2701 ScheduleKind.Schedule, DispatchValues.Chunk != nullptr, Ordered);
2702 assert(Ordered ||
2703 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
2704 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&
2705 Schedule != OMP_sch_static_balanced_chunked));
2706 // Call __kmpc_dispatch_init(
2707 // ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
2708 // kmp_int[32|64] lower, kmp_int[32|64] upper,
2709 // kmp_int[32|64] stride, kmp_int[32|64] chunk);
2710
2711 // If the Chunk was not specified in the clause - use default value 1.
2712 llvm::Value *Chunk = DispatchValues.Chunk ? DispatchValues.Chunk
2713 : CGF.Builder.getIntN(IVSize, 1);
2714 llvm::Value *Args[] = {
2715 emitUpdateLocation(CGF, Loc),
2716 getThreadID(CGF, Loc),
2717 CGF.Builder.getInt32(addMonoNonMonoModifier(
2718 CGM, Schedule, ScheduleKind.M1, ScheduleKind.M2)), // Schedule type
2719 DispatchValues.LB, // Lower
2720 DispatchValues.UB, // Upper
2721 CGF.Builder.getIntN(IVSize, 1), // Stride
2722 Chunk // Chunk
2723 };
2724 CGF.EmitRuntimeCall(OMPBuilder.createDispatchInitFunction(IVSize, IVSigned),
2725 Args);
2726}
2727
2729 SourceLocation Loc) {
2730 if (!CGF.HaveInsertPoint())
2731 return;
2732 // Call __kmpc_dispatch_deinit(ident_t *loc, kmp_int32 tid);
2733 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2734 CGF.EmitRuntimeCall(OMPBuilder.createDispatchDeinitFunction(), Args);
2735}
2736
2738 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
2739 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
2741 const CGOpenMPRuntime::StaticRTInput &Values) {
2742 if (!CGF.HaveInsertPoint())
2743 return;
2744
2745 assert(!Values.Ordered);
2746 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||
2747 Schedule == OMP_sch_static_balanced_chunked ||
2748 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||
2749 Schedule == OMP_dist_sch_static ||
2750 Schedule == OMP_dist_sch_static_chunked ||
2751 Schedule == OMP_dist_sch_static_chunked_sch_static_chunkone);
2752
2753 // Call __kmpc_for_static_init(
2754 // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
2755 // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
2756 // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
2757 // kmp_int[32|64] incr, kmp_int[32|64] chunk);
2758 llvm::Value *Chunk = Values.Chunk;
2759 if (Chunk == nullptr) {
2760 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
2761 Schedule == OMP_dist_sch_static) &&
2762 "expected static non-chunked schedule");
2763 // If the Chunk was not specified in the clause - use default value 1.
2764 Chunk = CGF.Builder.getIntN(Values.IVSize, 1);
2765 } else {
2766 assert((Schedule == OMP_sch_static_chunked ||
2767 Schedule == OMP_sch_static_balanced_chunked ||
2768 Schedule == OMP_ord_static_chunked ||
2769 Schedule == OMP_dist_sch_static_chunked ||
2770 Schedule == OMP_dist_sch_static_chunked_sch_static_chunkone) &&
2771 "expected static chunked schedule");
2772 }
2773 llvm::Value *Args[] = {
2774 UpdateLocation,
2775 ThreadId,
2776 CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1,
2777 M2)), // Schedule type
2778 Values.IL.emitRawPointer(CGF), // &isLastIter
2779 Values.LB.emitRawPointer(CGF), // &LB
2780 Values.UB.emitRawPointer(CGF), // &UB
2781 Values.ST.emitRawPointer(CGF), // &Stride
2782 CGF.Builder.getIntN(Values.IVSize, 1), // Incr
2783 Chunk // Chunk
2784 };
2785 CGF.EmitRuntimeCall(ForStaticInitFunction, Args);
2786}
2787
2789 SourceLocation Loc,
2790 OpenMPDirectiveKind DKind,
2791 const OpenMPScheduleTy &ScheduleKind,
2792 const StaticRTInput &Values) {
2793 OpenMPSchedType ScheduleNum =
2794 ScheduleKind.UseFusedDistChunkSchedule
2795 ? OMP_dist_sch_static_chunked_sch_static_chunkone
2796 : getRuntimeSchedule(ScheduleKind.Schedule, Values.Chunk != nullptr,
2797 Values.Ordered);
2798 assert((isOpenMPWorksharingDirective(DKind) || (DKind == OMPD_loop)) &&
2799 "Expected loop-based or sections-based directive.");
2800 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,
2802 ? OMP_IDENT_WORK_LOOP
2803 : OMP_IDENT_WORK_SECTIONS);
2804 llvm::Value *ThreadId = getThreadID(CGF, Loc);
2805 llvm::FunctionCallee StaticInitFunction =
2806 OMPBuilder.createForStaticInitFunction(Values.IVSize, Values.IVSigned,
2807 false);
2809 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
2810 ScheduleNum, ScheduleKind.M1, ScheduleKind.M2, Values);
2811}
2812
2816 const CGOpenMPRuntime::StaticRTInput &Values) {
2817 OpenMPSchedType ScheduleNum =
2818 getRuntimeSchedule(SchedKind, Values.Chunk != nullptr);
2819 llvm::Value *UpdatedLocation =
2820 emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE);
2821 llvm::Value *ThreadId = getThreadID(CGF, Loc);
2822 llvm::FunctionCallee StaticInitFunction;
2823 bool isGPUDistribute =
2824 CGM.getLangOpts().OpenMPIsTargetDevice && CGM.getTriple().isGPU();
2825 StaticInitFunction = OMPBuilder.createForStaticInitFunction(
2826 Values.IVSize, Values.IVSigned, isGPUDistribute);
2827
2828 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
2829 ScheduleNum, OMPC_SCHEDULE_MODIFIER_unknown,
2831}
2832
2834 SourceLocation Loc,
2835 OpenMPDirectiveKind DKind) {
2836 assert((DKind == OMPD_distribute || DKind == OMPD_for ||
2837 DKind == OMPD_sections) &&
2838 "Expected distribute, for, or sections directive kind");
2839 if (!CGF.HaveInsertPoint())
2840 return;
2841 // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
2842 llvm::Value *Args[] = {
2843 emitUpdateLocation(CGF, Loc,
2845 (DKind == OMPD_target_teams_loop)
2846 ? OMP_IDENT_WORK_DISTRIBUTE
2847 : isOpenMPLoopDirective(DKind)
2848 ? OMP_IDENT_WORK_LOOP
2849 : OMP_IDENT_WORK_SECTIONS),
2850 getThreadID(CGF, Loc)};
2852 if (isOpenMPDistributeDirective(DKind) &&
2853 CGM.getLangOpts().OpenMPIsTargetDevice && CGM.getTriple().isGPU())
2854 CGF.EmitRuntimeCall(
2855 OMPBuilder.getOrCreateRuntimeFunction(
2856 CGM.getModule(), OMPRTL___kmpc_distribute_static_fini),
2857 Args);
2858 else
2859 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2860 CGM.getModule(), OMPRTL___kmpc_for_static_fini),
2861 Args);
2862}
2863
2865 SourceLocation Loc,
2866 unsigned IVSize,
2867 bool IVSigned) {
2868 if (!CGF.HaveInsertPoint())
2869 return;
2870 // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
2871 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2872 CGF.EmitRuntimeCall(OMPBuilder.createDispatchFiniFunction(IVSize, IVSigned),
2873 Args);
2874}
2875
2877 SourceLocation Loc, unsigned IVSize,
2878 bool IVSigned, Address IL,
2879 Address LB, Address UB,
2880 Address ST) {
2881 // Call __kmpc_dispatch_next(
2882 // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
2883 // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
2884 // kmp_int[32|64] *p_stride);
2885 llvm::Value *Args[] = {
2886 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2887 IL.emitRawPointer(CGF), // &isLastIter
2888 LB.emitRawPointer(CGF), // &Lower
2889 UB.emitRawPointer(CGF), // &Upper
2890 ST.emitRawPointer(CGF) // &Stride
2891 };
2892 llvm::Value *Call = CGF.EmitRuntimeCall(
2893 OMPBuilder.createDispatchNextFunction(IVSize, IVSigned), Args);
2894 return CGF.EmitScalarConversion(
2895 Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1),
2896 CGF.getContext().BoolTy, Loc);
2897}
2898
2900 const Expr *Message,
2901 SourceLocation Loc) {
2902 if (!Message)
2903 return llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
2904 return CGF.EmitScalarExpr(Message);
2905}
2906
2907llvm::Value *
2909 SourceLocation Loc) {
2910 // OpenMP 6.0, 10.4: "If no severity clause is specified then the effect is
2911 // as if sev-level is fatal."
2912 return llvm::ConstantInt::get(CGM.Int32Ty,
2913 Severity == OMPC_SEVERITY_warning ? 1 : 2);
2914}
2915
2917 CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc,
2919 SourceLocation SeverityLoc, const Expr *Message,
2920 SourceLocation MessageLoc) {
2921 if (!CGF.HaveInsertPoint())
2922 return;
2924 {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2925 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)});
2926 // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
2927 // or __kmpc_push_num_threads_strict(&loc, global_tid, num_threads, severity,
2928 // messsage) if strict modifier is used.
2929 RuntimeFunction FnID = OMPRTL___kmpc_push_num_threads;
2930 if (Modifier == OMPC_NUMTHREADS_strict) {
2931 FnID = OMPRTL___kmpc_push_num_threads_strict;
2932 Args.push_back(emitSeverityClause(Severity, SeverityLoc));
2933 Args.push_back(emitMessageClause(CGF, Message, MessageLoc));
2934 }
2935 CGF.EmitRuntimeCall(
2936 OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(), FnID), Args);
2937}
2938
2940 ProcBindKind ProcBind,
2941 SourceLocation Loc) {
2942 if (!CGF.HaveInsertPoint())
2943 return;
2944 assert(ProcBind != OMP_PROC_BIND_unknown && "Unsupported proc_bind value.");
2945 // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)
2946 llvm::Value *Args[] = {
2947 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2948 llvm::ConstantInt::get(CGM.IntTy, unsigned(ProcBind), /*isSigned=*/true)};
2949 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2950 CGM.getModule(), OMPRTL___kmpc_push_proc_bind),
2951 Args);
2952}
2953
2955 SourceLocation Loc, llvm::AtomicOrdering AO) {
2956 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2957 OMPBuilder.createFlush(CGF.Builder);
2958 } else {
2959 if (!CGF.HaveInsertPoint())
2960 return;
2961 // Build call void __kmpc_flush(ident_t *loc)
2962 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2963 CGM.getModule(), OMPRTL___kmpc_flush),
2964 emitUpdateLocation(CGF, Loc));
2965 }
2966}
2967
2968namespace {
2969/// Indexes of fields for type kmp_task_t.
2970enum KmpTaskTFields {
2971 /// List of shared variables.
2972 KmpTaskTShareds,
2973 /// Task routine.
2974 KmpTaskTRoutine,
2975 /// Partition id for the untied tasks.
2976 KmpTaskTPartId,
2977 /// Function with call of destructors for private variables.
2978 Data1,
2979 /// Task priority.
2980 Data2,
2981 /// (Taskloops only) Lower bound.
2982 KmpTaskTLowerBound,
2983 /// (Taskloops only) Upper bound.
2984 KmpTaskTUpperBound,
2985 /// (Taskloops only) Stride.
2986 KmpTaskTStride,
2987 /// (Taskloops only) Is last iteration flag.
2988 KmpTaskTLastIter,
2989 /// (Taskloops only) Reduction data.
2990 KmpTaskTReductions,
2991};
2992} // anonymous namespace
2993
2995 // If we are in simd mode or there are no entries, we don't need to do
2996 // anything.
2997 if (CGM.getLangOpts().OpenMPSimd || OMPBuilder.OffloadInfoManager.empty())
2998 return;
2999
3000 llvm::OpenMPIRBuilder::EmitMetadataErrorReportFunctionTy &&ErrorReportFn =
3001 [this](llvm::OpenMPIRBuilder::EmitMetadataErrorKind Kind,
3002 const llvm::TargetRegionEntryInfo &EntryInfo) -> void {
3003 SourceLocation Loc;
3004 if (Kind != llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR) {
3005 for (auto I = CGM.getContext().getSourceManager().fileinfo_begin(),
3006 E = CGM.getContext().getSourceManager().fileinfo_end();
3007 I != E; ++I) {
3008 if (I->getFirst().getUniqueID().getDevice() == EntryInfo.DeviceID &&
3009 I->getFirst().getUniqueID().getFile() == EntryInfo.FileID) {
3010 Loc = CGM.getContext().getSourceManager().translateFileLineCol(
3011 I->getFirst(), EntryInfo.Line, 1);
3012 break;
3013 }
3014 }
3015 }
3016 switch (Kind) {
3017 case llvm::OpenMPIRBuilder::EMIT_MD_TARGET_REGION_ERROR: {
3018 CGM.getDiags().Report(Loc,
3019 diag::err_target_region_offloading_entry_incorrect)
3020 << EntryInfo.ParentName;
3021 } break;
3022 case llvm::OpenMPIRBuilder::EMIT_MD_DECLARE_TARGET_ERROR: {
3023 CGM.getDiags().Report(
3024 Loc, diag::err_target_var_offloading_entry_incorrect_with_parent)
3025 << EntryInfo.ParentName;
3026 } break;
3027 case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR: {
3028 CGM.getDiags().Report(diag::err_target_var_offloading_entry_incorrect);
3029 } break;
3030 case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_INDIRECT_ERROR: {
3031 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3032 DiagnosticsEngine::Error, "Offloading entry for indirect declare "
3033 "target variable is incorrect: the "
3034 "address is invalid.");
3035 CGM.getDiags().Report(DiagID);
3036 } break;
3037 }
3038 };
3039
3040 OMPBuilder.createOffloadEntriesAndInfoMetadata(ErrorReportFn);
3041}
3042
3044 if (!KmpRoutineEntryPtrTy) {
3045 // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.
3046 ASTContext &C = CGM.getContext();
3047 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
3049 KmpRoutineEntryPtrQTy = C.getPointerType(
3050 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
3051 KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy);
3052 }
3053}
3054
3055namespace {
3056struct PrivateHelpersTy {
3057 PrivateHelpersTy(const Expr *OriginalRef, const VarDecl *Original,
3058 const VarDecl *PrivateCopy, const VarDecl *PrivateElemInit)
3059 : OriginalRef(OriginalRef), Original(Original), PrivateCopy(PrivateCopy),
3060 PrivateElemInit(PrivateElemInit) {}
3061 PrivateHelpersTy(const VarDecl *Original) : Original(Original) {}
3062 const Expr *OriginalRef = nullptr;
3063 const VarDecl *Original = nullptr;
3064 const VarDecl *PrivateCopy = nullptr;
3065 const VarDecl *PrivateElemInit = nullptr;
3066 bool isLocalPrivate() const {
3067 return !OriginalRef && !PrivateCopy && !PrivateElemInit;
3068 }
3069};
3070typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
3071} // anonymous namespace
3072
3073/// For BindingDecls, returns the DecomposedDecl as the original VarDecl.
3074/// For regular VarDecls, returns the VarDecl itself.
3076 if (const auto *BD = dyn_cast<BindingDecl>(Decl))
3077 return cast<VarDecl>(BD->getDecomposedDecl());
3078 return cast<VarDecl>(Decl);
3079}
3080
3081static bool isAllocatableDecl(const VarDecl *VD) {
3082 const VarDecl *CVD = VD->getCanonicalDecl();
3083 if (!CVD->hasAttr<OMPAllocateDeclAttr>())
3084 return false;
3085 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
3086 // Use the default allocation.
3087 return !(AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc &&
3088 !AA->getAllocator());
3089}
3090
3091static RecordDecl *
3093 if (!Privates.empty()) {
3094 ASTContext &C = CGM.getContext();
3095 // Build struct .kmp_privates_t. {
3096 // /* private vars */
3097 // };
3098 RecordDecl *RD = C.buildImplicitRecord(".kmp_privates.t");
3099 RD->startDefinition();
3100 for (const auto &Pair : Privates) {
3101 const VarDecl *VD = Pair.second.Original;
3102 const VarDecl *PrivateCopy = Pair.second.PrivateCopy;
3103 // For BindingDecls, use PrivateCopy type (binding's actual type).
3104 // For regular variables, use Original type to preserve qualifiers.
3105 // Check OriginalRef to detect BindingDecls since Original may be the
3106 // DecompositionDecl.
3107 bool IsBinding =
3108 Pair.second.OriginalRef &&
3110 cast<DeclRefExpr>(Pair.second.OriginalRef)->getDecl());
3111 QualType Type = IsBinding ? PrivateCopy->getType().getNonReferenceType()
3112 : VD->getType().getNonReferenceType();
3113
3114 // If the private variable is a local variable with lvalue ref type,
3115 // allocate the pointer instead of the pointee type.
3116 if (Pair.second.isLocalPrivate()) {
3117 if (VD->getType()->isLValueReferenceType())
3118 Type = C.getPointerType(Type);
3119 if (isAllocatableDecl(VD))
3120 Type = C.getPointerType(Type);
3121 }
3123 if (VD->hasAttrs()) {
3124 for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
3125 E(VD->getAttrs().end());
3126 I != E; ++I)
3127 FD->addAttr(*I);
3128 }
3129 }
3130 RD->completeDefinition();
3131 return RD;
3132 }
3133 return nullptr;
3134}
3135
3136static RecordDecl *
3138 QualType KmpInt32Ty,
3139 QualType KmpRoutineEntryPointerQTy) {
3140 ASTContext &C = CGM.getContext();
3141 // Build struct kmp_task_t {
3142 // void * shareds;
3143 // kmp_routine_entry_t routine;
3144 // kmp_int32 part_id;
3145 // kmp_cmplrdata_t data1;
3146 // kmp_cmplrdata_t data2;
3147 // For taskloops additional fields:
3148 // kmp_uint64 lb;
3149 // kmp_uint64 ub;
3150 // kmp_int64 st;
3151 // kmp_int32 liter;
3152 // void * reductions;
3153 // };
3154 RecordDecl *UD = C.buildImplicitRecord("kmp_cmplrdata_t", TagTypeKind::Union);
3155 UD->startDefinition();
3156 addFieldToRecordDecl(C, UD, KmpInt32Ty);
3157 addFieldToRecordDecl(C, UD, KmpRoutineEntryPointerQTy);
3158 UD->completeDefinition();
3159 CanQualType KmpCmplrdataTy = C.getCanonicalTagType(UD);
3160 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t");
3161 RD->startDefinition();
3162 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
3163 addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy);
3164 addFieldToRecordDecl(C, RD, KmpInt32Ty);
3165 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
3166 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
3167 if (isOpenMPTaskLoopDirective(Kind)) {
3168 QualType KmpUInt64Ty =
3169 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0);
3170 QualType KmpInt64Ty =
3171 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
3172 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
3173 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
3174 addFieldToRecordDecl(C, RD, KmpInt64Ty);
3175 addFieldToRecordDecl(C, RD, KmpInt32Ty);
3176 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
3177 }
3178 RD->completeDefinition();
3179 return RD;
3180}
3181
3182static RecordDecl *
3185 ASTContext &C = CGM.getContext();
3186 // Build struct kmp_task_t_with_privates {
3187 // kmp_task_t task_data;
3188 // .kmp_privates_t. privates;
3189 // };
3190 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t_with_privates");
3191 RD->startDefinition();
3192 addFieldToRecordDecl(C, RD, KmpTaskTQTy);
3193 if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates))
3194 addFieldToRecordDecl(C, RD, C.getCanonicalTagType(PrivateRD));
3195 RD->completeDefinition();
3196 return RD;
3197}
3198
3199/// Emit a proxy function which accepts kmp_task_t as the second
3200/// argument.
3201/// \code
3202/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
3203/// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt,
3204/// For taskloops:
3205/// tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
3206/// tt->reductions, tt->shareds);
3207/// return 0;
3208/// }
3209/// \endcode
3210static llvm::Function *
3212 OpenMPDirectiveKind Kind, QualType KmpInt32Ty,
3213 QualType KmpTaskTWithPrivatesPtrQTy,
3214 QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
3215 QualType SharedsPtrTy, llvm::Function *TaskFunction,
3216 llvm::Value *TaskPrivatesMap) {
3217 ASTContext &C = CGM.getContext();
3218 auto *GtidArg =
3219 ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3220 KmpInt32Ty, ImplicitParamKind::Other);
3221 auto *TaskTypeArg = ImplicitParamDecl::Create(
3222 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3223 KmpTaskTWithPrivatesPtrQTy.withRestrict(), ImplicitParamKind::Other);
3224 FunctionArgList Args{GtidArg, TaskTypeArg};
3225 const auto &TaskEntryFnInfo =
3226 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
3227 llvm::FunctionType *TaskEntryTy =
3228 CGM.getTypes().GetFunctionType(TaskEntryFnInfo);
3229 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_entry", ""});
3230 auto *TaskEntry = llvm::Function::Create(
3231 TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
3232 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskEntry, TaskEntryFnInfo);
3233 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3234 TaskEntry->addFnAttr("sample-profile-suffix-elision-policy", "selected");
3235 TaskEntry->setDoesNotRecurse();
3236 CodeGenFunction CGF(CGM);
3237 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args,
3238 Loc, Loc);
3239
3240 // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
3241 // tt,
3242 // For taskloops:
3243 // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
3244 // tt->task_data.shareds);
3245 llvm::Value *GtidParam = CGF.EmitLoadOfScalar(
3246 CGF.GetAddrOfLocalVar(GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc);
3247 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3248 CGF.GetAddrOfLocalVar(TaskTypeArg),
3249 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3250 const auto *KmpTaskTWithPrivatesQTyRD =
3251 KmpTaskTWithPrivatesQTy->castAsRecordDecl();
3252 LValue Base =
3253 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
3254 const auto *KmpTaskTQTyRD = KmpTaskTQTy->castAsRecordDecl();
3255 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
3256 LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI);
3257 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF);
3258
3259 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
3260 LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI);
3261 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3262 CGF.EmitLoadOfScalar(SharedsLVal, Loc),
3263 CGF.ConvertTypeForMem(SharedsPtrTy));
3264
3265 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
3266 llvm::Value *PrivatesParam;
3267 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
3268 LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);
3269 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3270 PrivatesLVal.getPointer(CGF), CGF.VoidPtrTy);
3271 } else {
3272 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
3273 }
3274
3275 llvm::Value *CommonArgs[] = {
3276 GtidParam, PartidParam, PrivatesParam, TaskPrivatesMap,
3277 CGF.Builder
3278 .CreatePointerBitCastOrAddrSpaceCast(TDBase.getAddress(),
3279 CGF.VoidPtrTy, CGF.Int8Ty)
3280 .emitRawPointer(CGF)};
3281 SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs),
3282 std::end(CommonArgs));
3283 if (isOpenMPTaskLoopDirective(Kind)) {
3284 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
3285 LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI);
3286 llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc);
3287 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
3288 LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI);
3289 llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc);
3290 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
3291 LValue StLVal = CGF.EmitLValueForField(Base, *StFI);
3292 llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc);
3293 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
3294 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
3295 llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc);
3296 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
3297 LValue RLVal = CGF.EmitLValueForField(Base, *RFI);
3298 llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc);
3299 CallArgs.push_back(LBParam);
3300 CallArgs.push_back(UBParam);
3301 CallArgs.push_back(StParam);
3302 CallArgs.push_back(LIParam);
3303 CallArgs.push_back(RParam);
3304 }
3305 CallArgs.push_back(SharedsParam);
3306
3307 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction,
3308 CallArgs);
3309 CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)),
3310 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty));
3311 CGF.FinishFunction();
3312 return TaskEntry;
3313}
3314
3316 SourceLocation Loc,
3317 QualType KmpInt32Ty,
3318 QualType KmpTaskTWithPrivatesPtrQTy,
3319 QualType KmpTaskTWithPrivatesQTy) {
3320 ASTContext &C = CGM.getContext();
3321 auto *GtidArg =
3322 ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3323 KmpInt32Ty, ImplicitParamKind::Other);
3324 auto *TaskTypeArg = ImplicitParamDecl::Create(
3325 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3326 KmpTaskTWithPrivatesPtrQTy.withRestrict(), ImplicitParamKind::Other);
3327 FunctionArgList Args{GtidArg, TaskTypeArg};
3328 const auto &DestructorFnInfo =
3329 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
3330 llvm::FunctionType *DestructorFnTy =
3331 CGM.getTypes().GetFunctionType(DestructorFnInfo);
3332 std::string Name =
3333 CGM.getOpenMPRuntime().getName({"omp_task_destructor", ""});
3334 auto *DestructorFn =
3335 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
3336 Name, &CGM.getModule());
3337 CGM.SetInternalFunctionAttributes(GlobalDecl(), DestructorFn,
3338 DestructorFnInfo);
3339 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3340 DestructorFn->addFnAttr("sample-profile-suffix-elision-policy", "selected");
3341 DestructorFn->setDoesNotRecurse();
3342 CodeGenFunction CGF(CGM);
3343 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,
3344 Args, Loc, Loc);
3345
3346 LValue Base = CGF.EmitLoadOfPointerLValue(
3347 CGF.GetAddrOfLocalVar(TaskTypeArg),
3348 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3349 const auto *KmpTaskTWithPrivatesQTyRD =
3350 KmpTaskTWithPrivatesQTy->castAsRecordDecl();
3351 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
3352 Base = CGF.EmitLValueForField(Base, *FI);
3353 for (const auto *Field : FI->getType()->castAsRecordDecl()->fields()) {
3354 if (QualType::DestructionKind DtorKind =
3355 Field->getType().isDestructedType()) {
3356 LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
3357 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType());
3358 }
3359 }
3360 CGF.FinishFunction();
3361 return DestructorFn;
3362}
3363
3364/// Emit a privates mapping function for correct handling of private and
3365/// firstprivate variables.
3366/// \code
3367/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
3368/// **noalias priv1,..., <tyn> **noalias privn) {
3369/// *priv1 = &.privates.priv1;
3370/// ...;
3371/// *privn = &.privates.privn;
3372/// }
3373/// \endcode
3374static llvm::Value *
3376 const OMPTaskDataTy &Data, QualType PrivatesQTy,
3378 ASTContext &C = CGM.getContext();
3379 FunctionArgList Args;
3380 auto *TaskPrivatesArg = ImplicitParamDecl::Create(
3381 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3382 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
3384 Args.push_back(TaskPrivatesArg);
3385 llvm::SmallDenseMap<CanonicalDeclPtr<const VarDecl>, unsigned> PrivateVarsPos;
3386 // Track BindingDecl positions separately since BindingDecl is not a VarDecl.
3387 llvm::SmallDenseMap<const BindingDecl *, unsigned> BindingDeclPos;
3388 unsigned Counter = 1;
3389 for (const Expr *E : Data.PrivateVars) {
3390 Args.push_back(ImplicitParamDecl::Create(
3391 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3392 C.getPointerType(C.getPointerType(E->getType()))
3393 .withConst()
3394 .withRestrict(),
3396 const ValueDecl *VD = cast<DeclRefExpr>(E)->getDecl();
3397 if (const auto *BD = dyn_cast<BindingDecl>(VD))
3398 BindingDeclPos[cast<BindingDecl>(BD->getCanonicalDecl())] = Counter;
3399 else
3400 PrivateVarsPos[cast<VarDecl>(VD)] = Counter;
3401 ++Counter;
3402 }
3403 for (const Expr *E : Data.FirstprivateVars) {
3404 Args.push_back(ImplicitParamDecl::Create(
3405 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3406 C.getPointerType(C.getPointerType(E->getType()))
3407 .withConst()
3408 .withRestrict(),
3410 const ValueDecl *VD = cast<DeclRefExpr>(E)->getDecl();
3411 if (const auto *BD = dyn_cast<BindingDecl>(VD))
3412 BindingDeclPos[cast<BindingDecl>(BD->getCanonicalDecl())] = Counter;
3413 else
3414 PrivateVarsPos[cast<VarDecl>(VD)] = Counter;
3415 ++Counter;
3416 }
3417 for (const Expr *E : Data.LastprivateVars) {
3418 Args.push_back(ImplicitParamDecl::Create(
3419 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3420 C.getPointerType(C.getPointerType(E->getType()))
3421 .withConst()
3422 .withRestrict(),
3424 const ValueDecl *VD = cast<DeclRefExpr>(E)->getDecl();
3425 if (const auto *BD = dyn_cast<BindingDecl>(VD))
3426 BindingDeclPos[cast<BindingDecl>(BD->getCanonicalDecl())] = Counter;
3427 else
3428 PrivateVarsPos[cast<VarDecl>(VD)] = Counter;
3429 ++Counter;
3430 }
3431 for (const VarDecl *VD : Data.PrivateLocals) {
3433 if (VD->getType()->isLValueReferenceType())
3434 Ty = C.getPointerType(Ty);
3435 if (isAllocatableDecl(VD))
3436 Ty = C.getPointerType(Ty);
3437 Args.push_back(ImplicitParamDecl::Create(
3438 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3439 C.getPointerType(C.getPointerType(Ty)).withConst().withRestrict(),
3441 PrivateVarsPos[VD] = Counter;
3442 ++Counter;
3443 }
3444 const auto &TaskPrivatesMapFnInfo =
3445 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3446 llvm::FunctionType *TaskPrivatesMapTy =
3447 CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo);
3448 std::string Name =
3449 CGM.getOpenMPRuntime().getName({"omp_task_privates_map", ""});
3450 auto *TaskPrivatesMap = llvm::Function::Create(
3451 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
3452 &CGM.getModule());
3453 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskPrivatesMap,
3454 TaskPrivatesMapFnInfo);
3455 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3456 TaskPrivatesMap->addFnAttr("sample-profile-suffix-elision-policy",
3457 "selected");
3458 if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
3459 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
3460 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
3461 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
3462 }
3463 CodeGenFunction CGF(CGM);
3464 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,
3465 TaskPrivatesMapFnInfo, Args, Loc, Loc);
3466
3467 // *privi = &.privates.privi;
3468 LValue Base = CGF.EmitLoadOfPointerLValue(
3469 CGF.GetAddrOfLocalVar(TaskPrivatesArg),
3470 TaskPrivatesArg->getType()->castAs<PointerType>());
3471 const auto *PrivatesQTyRD = PrivatesQTy->castAsRecordDecl();
3472 Counter = 0;
3473 for (const FieldDecl *Field : PrivatesQTyRD->fields()) {
3474 LValue FieldLVal = CGF.EmitLValueForField(Base, Field);
3475 // Lookup by the original declaration (BindingDecl or VarDecl).
3476 const ValueDecl *LookupVD;
3477 if (Privates[Counter].second.OriginalRef) {
3478 LookupVD =
3479 cast<DeclRefExpr>(Privates[Counter].second.OriginalRef)->getDecl();
3480 } else {
3481 LookupVD = Privates[Counter].second.Original;
3482 }
3483
3484 // For BindingDecls, the privates record now stores each binding's type
3485 // directly (not the full DecompositionDecl), so FieldLVal is already
3486 // correct.
3487
3488 unsigned Position;
3489 if (const auto *BD = dyn_cast<BindingDecl>(LookupVD)) {
3490 Position =
3491 BindingDeclPos.lookup(cast<BindingDecl>(BD->getCanonicalDecl()));
3492 assert(Position && "binding not in privates mapping");
3493 } else {
3494 Position = PrivateVarsPos[cast<VarDecl>(LookupVD)];
3495 }
3496 const VarDecl *VD = Args[Position];
3497 LValue RefLVal =
3498 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
3499 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue(
3500 RefLVal.getAddress(), RefLVal.getType()->castAs<PointerType>());
3501 CGF.EmitStoreOfScalar(FieldLVal.getPointer(CGF), RefLoadLVal);
3502 ++Counter;
3503 }
3504 CGF.FinishFunction();
3505 return TaskPrivatesMap;
3506}
3507
3508/// Emit initialization for private variables in task-based directives.
3510 const OMPExecutableDirective &D,
3511 Address KmpTaskSharedsPtr, LValue TDBase,
3512 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3513 QualType SharedsTy, QualType SharedsPtrTy,
3514 const OMPTaskDataTy &Data,
3515 ArrayRef<PrivateDataTy> Privates, bool ForDup) {
3516 ASTContext &C = CGF.getContext();
3517 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
3518 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI);
3519 OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(D.getDirectiveKind())
3520 ? OMPD_taskloop
3521 : OMPD_task;
3522 const CapturedStmt &CS = *D.getCapturedStmt(Kind);
3523 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
3524 LValue SrcBase;
3525 bool IsTargetTask =
3526 isOpenMPTargetDataManagementDirective(D.getDirectiveKind()) ||
3527 isOpenMPTargetExecutionDirective(D.getDirectiveKind());
3528 // For target-based directives skip 4 firstprivate arrays BasePointersArray,
3529 // PointersArray, SizesArray, and MappersArray. The original variables for
3530 // these arrays are not captured and we get their addresses explicitly.
3531 if ((!IsTargetTask && !Data.FirstprivateVars.empty() && ForDup) ||
3532 (IsTargetTask && KmpTaskSharedsPtr.isValid())) {
3533 SrcBase = CGF.MakeAddrLValue(
3535 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy),
3536 CGF.ConvertTypeForMem(SharedsTy)),
3537 SharedsTy);
3538 }
3539 FI = FI->getType()->castAsRecordDecl()->field_begin();
3540 for (const PrivateDataTy &Pair : Privates) {
3541 // Do not initialize private locals.
3542 if (Pair.second.isLocalPrivate()) {
3543 ++FI;
3544 continue;
3545 }
3546 const VarDecl *VD = Pair.second.PrivateCopy;
3547 const Expr *Init = VD->getAnyInitializer();
3548 if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&
3549 !CGF.isTrivialInitializer(Init)))) {
3550 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
3551 if (const VarDecl *Elem = Pair.second.PrivateElemInit) {
3552 const VarDecl *OriginalVD = Pair.second.Original;
3553 // Check if the variable is the target-based BasePointersArray,
3554 // PointersArray, SizesArray, or MappersArray.
3555 LValue SharedRefLValue;
3556 QualType Type = PrivateLValue.getType();
3557 const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);
3558 if (IsTargetTask && !SharedField) {
3559 assert(isa<ImplicitParamDecl>(OriginalVD) &&
3560 isa<CapturedDecl>(OriginalVD->getDeclContext()) &&
3561 cast<CapturedDecl>(OriginalVD->getDeclContext())
3562 ->getNumParams() == 0 &&
3564 cast<CapturedDecl>(OriginalVD->getDeclContext())
3565 ->getDeclContext()) &&
3566 "Expected artificial target data variable.");
3567 SharedRefLValue =
3568 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type);
3569 } else if (ForDup) {
3570 SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
3571 // For BindingDecls, access the specific binding field within the
3572 // captured DecompositionDecl.
3573 if (Pair.second.OriginalRef) {
3574 if (const auto *DRE =
3575 dyn_cast<DeclRefExpr>(Pair.second.OriginalRef)) {
3576 if (const auto *BD = dyn_cast<BindingDecl>(DRE->getDecl())) {
3577 // Emit the binding subobject (member or array element) with
3578 // the decomposed decl temporarily mapped to the capture.
3579 const VarDecl *DD = cast<VarDecl>(BD->getDecomposedDecl());
3580 auto It = CGF.findLocalDecl(DD);
3581 bool WasMapped = It != CGF.localDeclMapEnd();
3582 Address Saved = WasMapped ? It->second : Address::invalid();
3583 if (WasMapped)
3584 It->second = SharedRefLValue.getAddress();
3585 else
3586 CGF.insertLocalDecl(DD, SharedRefLValue.getAddress());
3587 SharedRefLValue = CGF.EmitLValue(BD->getBinding());
3588 if (WasMapped) {
3589 auto RestoreIt = CGF.findLocalDecl(DD);
3590 RestoreIt->second = Saved;
3591 } else {
3592 CGF.eraseLocalDecl(DD);
3593 }
3594 }
3595 }
3596 }
3597 bool IsBinding =
3598 Pair.second.OriginalRef &&
3600 cast<DeclRefExpr>(Pair.second.OriginalRef)->getDecl());
3601 SharedRefLValue = CGF.MakeAddrLValue(
3602 SharedRefLValue.getAddress().withAlignment(
3603 IsBinding ? C.toCharUnitsFromBits(
3604 C.getTypeAlign(SharedRefLValue.getType()))
3605 : C.getDeclAlign(OriginalVD)),
3606 SharedRefLValue.getType(), LValueBaseInfo(AlignmentSource::Decl),
3607 SharedRefLValue.getTBAAInfo());
3608 } else if (CGF.LambdaCaptureFields.count(
3609 Pair.second.Original->getCanonicalDecl()) > 0 ||
3610 isa_and_nonnull<BlockDecl>(CGF.CurCodeDecl)) {
3611 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef);
3612 } else {
3613 // Processing for implicitly captured variables.
3614 InlinedOpenMPRegionRAII Region(
3615 CGF, [](CodeGenFunction &, PrePostActionTy &) {}, OMPD_unknown,
3616 /*HasCancel=*/false, /*NoInheritance=*/true);
3617 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef);
3618 }
3619 if (Type->isArrayType()) {
3620 // Initialize firstprivate array.
3622 // Perform simple memcpy.
3623 CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type);
3624 } else {
3625 // Initialize firstprivate array using element-by-element
3626 // initialization.
3628 PrivateLValue.getAddress(), SharedRefLValue.getAddress(), Type,
3629 [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
3630 Address SrcElement) {
3631 // Clean up any temporaries needed by the initialization.
3632 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3633 InitScope.addPrivate(Elem, SrcElement);
3634 (void)InitScope.Privatize();
3635 // Emit initialization for single element.
3636 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
3637 CGF, &CapturesInfo);
3638 CGF.EmitAnyExprToMem(Init, DestElement,
3639 Init->getType().getQualifiers(),
3640 /*IsInitializer=*/false);
3641 });
3642 }
3643 } else {
3644 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3645 InitScope.addPrivate(Elem, SharedRefLValue.getAddress());
3646 (void)InitScope.Privatize();
3647 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
3648 CGF.EmitExprAsInit(Init, VD, PrivateLValue,
3649 /*capturedByInit=*/false);
3650 }
3651 } else {
3652 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
3653 }
3654 } else if (const VarDecl *OriginalVD = Pair.second.Original) {
3655 // Handle array bindings without initializers (firstprivate only).
3656 // For private clause (PrivateElemInit is nullptr), skip initialization.
3657 // Check if OriginalRef is a BindingDecl with array type.
3658 const BindingDecl *BD = nullptr;
3659 if (Pair.second.OriginalRef) {
3660 if (const auto *DRE = dyn_cast<DeclRefExpr>(Pair.second.OriginalRef)) {
3661 BD = dyn_cast<BindingDecl>(DRE->getDecl());
3662 }
3663 }
3664 if (BD && BD->getType()->isArrayType() && Pair.second.PrivateElemInit) {
3665 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
3666 QualType Type = PrivateLValue.getType();
3667 const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);
3668
3669 // Lambda to temporarily map DecompositionDecl and emit the binding
3670 // expression.
3671 auto EmitBindingWithTempMap = [&CGF](const BindingDecl *BD,
3672 Address DDAddr) -> LValue {
3673 const VarDecl *DD = cast<VarDecl>(BD->getDecomposedDecl());
3674 auto It = CGF.findLocalDecl(DD);
3675 bool WasMapped = It != CGF.localDeclMapEnd();
3676 Address Saved = WasMapped ? It->second : Address::invalid();
3677 if (WasMapped)
3678 It->second = DDAddr;
3679 else
3680 CGF.insertLocalDecl(DD, DDAddr);
3681
3682 LValue Result = CGF.EmitLValue(BD->getBinding());
3683
3684 // Restore the mapping
3685 if (WasMapped) {
3686 auto RestoreIt = CGF.findLocalDecl(DD);
3687 RestoreIt->second = Saved;
3688 } else {
3689 CGF.eraseLocalDecl(DD);
3690 }
3691
3692 return Result;
3693 };
3694
3695 LValue SharedRefLValue;
3696 if (ForDup) {
3697 SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
3698 SharedRefLValue =
3699 EmitBindingWithTempMap(BD, SharedRefLValue.getAddress());
3700 SharedRefLValue = CGF.MakeAddrLValue(
3701 SharedRefLValue.getAddress().withAlignment(
3702 C.getDeclAlign(OriginalVD)),
3703 SharedRefLValue.getType(), LValueBaseInfo(AlignmentSource::Decl),
3704 SharedRefLValue.getTBAAInfo());
3705 } else {
3706 // For !ForDup (first task), emit binding from parent scope using
3707 // InlinedOpenMPRegionRAII to access the correct scope
3708 InlinedOpenMPRegionRAII Region(
3709 CGF, [](CodeGenFunction &, PrePostActionTy &) {}, OMPD_unknown,
3710 /*HasCancel=*/false, /*NoInheritance=*/true);
3711
3712 // Get the decomposed decl address from parent scope
3713 const VarDecl *DD = cast<VarDecl>(BD->getDecomposedDecl());
3714 Address DDAddr = CGF.GetAddrOfLocalVar(DD);
3715 SharedRefLValue = EmitBindingWithTempMap(BD, DDAddr);
3716 }
3717 // Perform simple memcpy for array binding
3718 CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type);
3719 }
3720 }
3721 ++FI;
3722 }
3723}
3724
3725/// Check if duplication function is required for taskloops.
3728 bool InitRequired = false;
3729 for (const PrivateDataTy &Pair : Privates) {
3730 if (Pair.second.isLocalPrivate())
3731 continue;
3732 const VarDecl *VD = Pair.second.PrivateCopy;
3733 const Expr *Init = VD->getAnyInitializer();
3734 InitRequired = InitRequired || (isa_and_nonnull<CXXConstructExpr>(Init) &&
3736 if (InitRequired)
3737 break;
3738 }
3739 return InitRequired;
3740}
3741
3742
3743/// Emit task_dup function (for initialization of
3744/// private/firstprivate/lastprivate vars and last_iter flag)
3745/// \code
3746/// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int
3747/// lastpriv) {
3748/// // setup lastprivate flag
3749/// task_dst->last = lastpriv;
3750/// // could be constructor calls here...
3751/// }
3752/// \endcode
3753static llvm::Value *
3755 const OMPExecutableDirective &D,
3756 QualType KmpTaskTWithPrivatesPtrQTy,
3757 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3758 const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy,
3759 QualType SharedsPtrTy, const OMPTaskDataTy &Data,
3760 ArrayRef<PrivateDataTy> Privates, bool WithLastIter) {
3761 ASTContext &C = CGM.getContext();
3762 auto *DstArg = ImplicitParamDecl::Create(
3763 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpTaskTWithPrivatesPtrQTy,
3765 auto *SrcArg = ImplicitParamDecl::Create(
3766 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpTaskTWithPrivatesPtrQTy,
3768 auto *LastprivArg =
3769 ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy,
3771 FunctionArgList Args{DstArg, SrcArg, LastprivArg};
3772 const auto &TaskDupFnInfo =
3773 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3774 llvm::FunctionType *TaskDupTy = CGM.getTypes().GetFunctionType(TaskDupFnInfo);
3775 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_dup", ""});
3776 auto *TaskDup = llvm::Function::Create(
3777 TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
3778 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskDup, TaskDupFnInfo);
3779 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3780 TaskDup->addFnAttr("sample-profile-suffix-elision-policy", "selected");
3781 TaskDup->setDoesNotRecurse();
3782 CodeGenFunction CGF(CGM);
3783 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc,
3784 Loc);
3785
3786 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3787 CGF.GetAddrOfLocalVar(DstArg),
3788 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3789 // task_dst->liter = lastpriv;
3790 if (WithLastIter) {
3791 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
3792 LValue Base = CGF.EmitLValueForField(
3793 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
3794 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
3795 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
3796 CGF.GetAddrOfLocalVar(LastprivArg), /*Volatile=*/false, C.IntTy, Loc);
3797 CGF.EmitStoreOfScalar(Lastpriv, LILVal);
3798 }
3799
3800 // Emit initial values for private copies (if any).
3801 assert(!Privates.empty());
3802 Address KmpTaskSharedsPtr = Address::invalid();
3803 if (!Data.FirstprivateVars.empty()) {
3804 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3805 CGF.GetAddrOfLocalVar(SrcArg),
3806 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3807 LValue Base = CGF.EmitLValueForField(
3808 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
3809 KmpTaskSharedsPtr = Address(
3811 Base, *std::next(KmpTaskTQTyRD->field_begin(),
3812 KmpTaskTShareds)),
3813 Loc),
3814 CGF.Int8Ty, CGM.getNaturalTypeAlignment(SharedsTy));
3815 }
3816 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
3817 SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);
3818 CGF.FinishFunction();
3819 return TaskDup;
3820}
3821
3822/// Checks if destructor function is required to be generated.
3823/// \return true if cleanups are required, false otherwise.
3824static bool
3825checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3827 for (const PrivateDataTy &P : Privates) {
3828 if (P.second.isLocalPrivate())
3829 continue;
3830 QualType Ty = P.second.Original->getType().getNonReferenceType();
3831 if (Ty.isDestructedType())
3832 return true;
3833 }
3834 return false;
3835}
3836
3837namespace {
3838/// Loop generator for OpenMP iterator expression.
3839class OMPIteratorGeneratorScope final
3841 CodeGenFunction &CGF;
3842 const OMPIteratorExpr *E = nullptr;
3843 SmallVector<CodeGenFunction::JumpDest, 4> ContDests;
3844 SmallVector<CodeGenFunction::JumpDest, 4> ExitDests;
3845 OMPIteratorGeneratorScope() = delete;
3846 OMPIteratorGeneratorScope(OMPIteratorGeneratorScope &) = delete;
3847
3848public:
3849 OMPIteratorGeneratorScope(CodeGenFunction &CGF, const OMPIteratorExpr *E)
3850 : CodeGenFunction::OMPPrivateScope(CGF), CGF(CGF), E(E) {
3851 if (!E)
3852 return;
3853 SmallVector<llvm::Value *, 4> Uppers;
3854 for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) {
3855 Uppers.push_back(CGF.EmitScalarExpr(E->getHelper(I).Upper));
3856 const auto *VD = cast<VarDecl>(E->getIteratorDecl(I));
3857 addPrivate(VD, CGF.CreateMemTemp(VD->getType(), VD->getName()));
3858 const OMPIteratorHelperData &HelperData = E->getHelper(I);
3859 addPrivate(
3860 HelperData.CounterVD,
3861 CGF.CreateMemTemp(HelperData.CounterVD->getType(), "counter.addr"));
3862 }
3863 Privatize();
3864
3865 for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) {
3866 const OMPIteratorHelperData &HelperData = E->getHelper(I);
3867 LValue CLVal =
3868 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(HelperData.CounterVD),
3869 HelperData.CounterVD->getType());
3870 // Counter = 0;
3871 CGF.EmitStoreOfScalar(
3872 llvm::ConstantInt::get(CLVal.getAddress().getElementType(), 0),
3873 CLVal);
3874 CodeGenFunction::JumpDest &ContDest =
3875 ContDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.cont"));
3876 CodeGenFunction::JumpDest &ExitDest =
3877 ExitDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.exit"));
3878 // N = <number-of_iterations>;
3879 llvm::Value *N = Uppers[I];
3880 // cont:
3881 // if (Counter < N) goto body; else goto exit;
3882 CGF.EmitBlock(ContDest.getBlock());
3883 auto *CVal =
3884 CGF.EmitLoadOfScalar(CLVal, HelperData.CounterVD->getLocation());
3885 llvm::Value *Cmp =
3886 HelperData.CounterVD->getType()->isSignedIntegerOrEnumerationType()
3887 ? CGF.Builder.CreateICmpSLT(CVal, N)
3888 : CGF.Builder.CreateICmpULT(CVal, N);
3889 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("iter.body");
3890 CGF.Builder.CreateCondBr(Cmp, BodyBB, ExitDest.getBlock());
3891 // body:
3892 CGF.EmitBlock(BodyBB);
3893 // Iteri = Begini + Counter * Stepi;
3894 CGF.EmitIgnoredExpr(HelperData.Update);
3895 }
3896 }
3897 ~OMPIteratorGeneratorScope() {
3898 if (!E)
3899 return;
3900 for (unsigned I = E->numOfIterators(); I > 0; --I) {
3901 // Counter = Counter + 1;
3902 const OMPIteratorHelperData &HelperData = E->getHelper(I - 1);
3903 CGF.EmitIgnoredExpr(HelperData.CounterUpdate);
3904 // goto cont;
3905 CGF.EmitBranchThroughCleanup(ContDests[I - 1]);
3906 // exit:
3907 CGF.EmitBlock(ExitDests[I - 1].getBlock(), /*IsFinished=*/I == 1);
3908 }
3909 }
3910};
3911} // namespace
3912
3913static std::pair<llvm::Value *, llvm::Value *>
3915 const auto *OASE = dyn_cast<OMPArrayShapingExpr>(E);
3916 llvm::Value *Addr;
3917 if (OASE) {
3918 const Expr *Base = OASE->getBase();
3919 Addr = CGF.EmitScalarExpr(Base);
3920 } else {
3921 Addr = CGF.EmitLValue(E).getPointer(CGF);
3922 }
3923 llvm::Value *SizeVal;
3924 QualType Ty = E->getType();
3925 if (OASE) {
3926 SizeVal = CGF.getTypeSize(OASE->getBase()->getType()->getPointeeType());
3927 for (const Expr *SE : OASE->getDimensions()) {
3928 llvm::Value *Sz = CGF.EmitScalarExpr(SE);
3929 Sz = CGF.EmitScalarConversion(
3930 Sz, SE->getType(), CGF.getContext().getSizeType(), SE->getExprLoc());
3931 SizeVal = CGF.Builder.CreateNUWMul(SizeVal, Sz);
3932 }
3933 } else if (const auto *ASE =
3934 dyn_cast<ArraySectionExpr>(E->IgnoreParenImpCasts())) {
3935 LValue UpAddrLVal = CGF.EmitArraySectionExpr(ASE, /*IsLowerBound=*/false);
3936 Address UpAddrAddress = UpAddrLVal.getAddress();
3937 llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32(
3938 UpAddrAddress.getElementType(), UpAddrAddress.emitRawPointer(CGF),
3939 /*Idx0=*/1);
3940 SizeVal = CGF.Builder.CreatePtrDiff(UpAddr, Addr, "", /*IsNUW=*/true);
3941 } else {
3942 SizeVal = CGF.getTypeSize(Ty);
3943 }
3944 return std::make_pair(Addr, SizeVal);
3945}
3946
3947/// Builds kmp_depend_info, if it is not built yet, and builds flags type.
3948static void getKmpAffinityType(ASTContext &C, QualType &KmpTaskAffinityInfoTy) {
3949 QualType FlagsTy = C.getIntTypeForBitwidth(32, /*Signed=*/false);
3950 if (KmpTaskAffinityInfoTy.isNull()) {
3951 RecordDecl *KmpAffinityInfoRD =
3952 C.buildImplicitRecord("kmp_task_affinity_info_t");
3953 KmpAffinityInfoRD->startDefinition();
3954 addFieldToRecordDecl(C, KmpAffinityInfoRD, C.getIntPtrType());
3955 addFieldToRecordDecl(C, KmpAffinityInfoRD, C.getSizeType());
3956 addFieldToRecordDecl(C, KmpAffinityInfoRD, FlagsTy);
3957 KmpAffinityInfoRD->completeDefinition();
3958 KmpTaskAffinityInfoTy = C.getCanonicalTagType(KmpAffinityInfoRD);
3959 }
3960}
3961
3964 const OMPExecutableDirective &D,
3965 llvm::Function *TaskFunction, QualType SharedsTy,
3966 Address Shareds, const OMPTaskDataTy &Data) {
3967 ASTContext &C = CGM.getContext();
3969 // Aggregate privates and sort them by the alignment.
3970 const auto *I = Data.PrivateCopies.begin();
3971 for (const Expr *E : Data.PrivateVars) {
3972 const auto *Decl = cast<DeclRefExpr>(E)->getDecl();
3973 const auto *VD = getOriginalVarDecl(Decl);
3974 const auto *CopyVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
3975 Privates.emplace_back(C.getDeclAlign(VD),
3976 PrivateHelpersTy(E, VD, CopyVD,
3977 /*PrivateElemInit=*/nullptr));
3978 ++I;
3979 }
3980 I = Data.FirstprivateCopies.begin();
3981 const auto *IElemInitRef = Data.FirstprivateInits.begin();
3982 for (const Expr *E : Data.FirstprivateVars) {
3983 const auto *Decl = cast<DeclRefExpr>(E)->getDecl();
3984 const auto *VD = getOriginalVarDecl(Decl);
3985 const auto *CopyVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
3986 const auto *InitVD =
3987 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl());
3988 Privates.emplace_back(C.getDeclAlign(VD),
3989 PrivateHelpersTy(E, VD, CopyVD, InitVD));
3990 ++I;
3991 ++IElemInitRef;
3992 }
3993 I = Data.LastprivateCopies.begin();
3994 for (const Expr *E : Data.LastprivateVars) {
3995 const auto *Decl = cast<DeclRefExpr>(E)->getDecl();
3996 const auto *VD = getOriginalVarDecl(Decl);
3997 const auto *CopyVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
3998 Privates.emplace_back(C.getDeclAlign(VD),
3999 PrivateHelpersTy(E, VD, CopyVD,
4000 /*PrivateElemInit=*/nullptr));
4001 ++I;
4002 }
4003 for (const VarDecl *VD : Data.PrivateLocals) {
4004 if (isAllocatableDecl(VD))
4005 Privates.emplace_back(CGM.getPointerAlign(), PrivateHelpersTy(VD));
4006 else
4007 Privates.emplace_back(C.getDeclAlign(VD), PrivateHelpersTy(VD));
4008 }
4009 llvm::stable_sort(Privates,
4010 [](const PrivateDataTy &L, const PrivateDataTy &R) {
4011 return L.first > R.first;
4012 });
4013 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
4014 // Build type kmp_routine_entry_t (if not built yet).
4015 emitKmpRoutineEntryT(KmpInt32Ty);
4016 // Build type kmp_task_t (if not built yet).
4017 if (isOpenMPTaskLoopDirective(D.getDirectiveKind())) {
4018 if (SavedKmpTaskloopTQTy.isNull()) {
4019 SavedKmpTaskloopTQTy = C.getCanonicalTagType(createKmpTaskTRecordDecl(
4020 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
4021 }
4023 } else {
4024 assert((D.getDirectiveKind() == OMPD_task ||
4025 isOpenMPTargetExecutionDirective(D.getDirectiveKind()) ||
4026 isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) &&
4027 "Expected taskloop, task or target directive");
4028 if (SavedKmpTaskTQTy.isNull()) {
4029 SavedKmpTaskTQTy = C.getCanonicalTagType(createKmpTaskTRecordDecl(
4030 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
4031 }
4033 }
4034 const auto *KmpTaskTQTyRD = KmpTaskTQTy->castAsRecordDecl();
4035 // Build particular struct kmp_task_t for the given task.
4036 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
4038 CanQualType KmpTaskTWithPrivatesQTy =
4039 C.getCanonicalTagType(KmpTaskTWithPrivatesQTyRD);
4040 QualType KmpTaskTWithPrivatesPtrQTy =
4041 C.getPointerType(KmpTaskTWithPrivatesQTy);
4042 llvm::Type *KmpTaskTWithPrivatesPtrTy = CGF.Builder.getPtrTy(0);
4043 llvm::Value *KmpTaskTWithPrivatesTySize =
4044 CGF.getTypeSize(KmpTaskTWithPrivatesQTy);
4045 QualType SharedsPtrTy = C.getPointerType(SharedsTy);
4046
4047 // Emit initial values for private copies (if any).
4048 llvm::Value *TaskPrivatesMap = nullptr;
4049 llvm::Type *TaskPrivatesMapTy =
4050 std::next(TaskFunction->arg_begin(), 3)->getType();
4051 if (!Privates.empty()) {
4052 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4053 TaskPrivatesMap =
4054 emitTaskPrivateMappingFunction(CGM, Loc, Data, FI->getType(), Privates);
4055 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4056 TaskPrivatesMap, TaskPrivatesMapTy);
4057 } else {
4058 TaskPrivatesMap = llvm::ConstantPointerNull::get(
4059 cast<llvm::PointerType>(TaskPrivatesMapTy));
4060 }
4061 // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,
4062 // kmp_task_t *tt);
4063 llvm::Function *TaskEntry = emitProxyTaskFunction(
4064 CGM, Loc, D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4065 KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction,
4066 TaskPrivatesMap);
4067
4068 // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
4069 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
4070 // kmp_routine_entry_t *task_entry);
4071 // Task flags. Format is taken from
4072 // https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h,
4073 // description of kmp_tasking_flags struct.
4074 enum {
4075 TiedFlag = 0x1,
4076 FinalFlag = 0x2,
4077 DestructorsFlag = 0x8,
4078 PriorityFlag = 0x20,
4079 DetachableFlag = 0x40,
4080 FreeAgentFlag = 0x80,
4081 TransparentFlag = 0x100,
4082 };
4083 unsigned Flags = Data.Tied ? TiedFlag : 0;
4084 bool NeedsCleanup = false;
4085 if (!Privates.empty()) {
4086 NeedsCleanup =
4087 checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD, Privates);
4088 if (NeedsCleanup)
4089 Flags = Flags | DestructorsFlag;
4090 }
4091 if (const auto *Clause = D.getSingleClause<OMPThreadsetClause>()) {
4092 OpenMPThreadsetKind Kind = Clause->getThreadsetKind();
4093 if (Kind == OMPC_THREADSET_omp_pool)
4094 Flags = Flags | FreeAgentFlag;
4095 }
4096 if (D.getSingleClause<OMPTransparentClause>())
4097 Flags |= TransparentFlag;
4098
4099 if (Data.Priority.getInt())
4100 Flags = Flags | PriorityFlag;
4101 if (D.hasClausesOfKind<OMPDetachClause>())
4102 Flags = Flags | DetachableFlag;
4103 llvm::Value *TaskFlags =
4104 Data.Final.getPointer()
4105 ? CGF.Builder.CreateSelect(Data.Final.getPointer(),
4106 CGF.Builder.getInt32(FinalFlag),
4107 CGF.Builder.getInt32(/*C=*/0))
4108 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0);
4109 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags));
4110 llvm::Value *SharedsSize = CGM.getSize(C.getTypeSizeInChars(SharedsTy));
4112 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
4114 TaskEntry, KmpRoutineEntryPtrTy)};
4115 llvm::Value *NewTask;
4116 if (D.hasClausesOfKind<OMPNowaitClause>()) {
4117 // Check if we have any device clause associated with the directive.
4118 const Expr *Device = nullptr;
4119 if (auto *C = D.getSingleClause<OMPDeviceClause>())
4120 Device = C->getDevice();
4121 // Emit device ID if any otherwise use default value.
4122 llvm::Value *DeviceID;
4123 if (Device)
4124 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
4125 CGF.Int64Ty, /*isSigned=*/true);
4126 else
4127 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
4128 AllocArgs.push_back(DeviceID);
4129 NewTask = CGF.EmitRuntimeCall(
4130 OMPBuilder.getOrCreateRuntimeFunction(
4131 CGM.getModule(), OMPRTL___kmpc_omp_target_task_alloc),
4132 AllocArgs);
4133 } else {
4134 NewTask =
4135 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
4136 CGM.getModule(), OMPRTL___kmpc_omp_task_alloc),
4137 AllocArgs);
4138 }
4139 // Emit detach clause initialization.
4140 // evt = (typeof(evt))__kmpc_task_allow_completion_event(loc, tid,
4141 // task_descriptor);
4142 if (const auto *DC = D.getSingleClause<OMPDetachClause>()) {
4143 const Expr *Evt = DC->getEventHandler()->IgnoreParenImpCasts();
4144 LValue EvtLVal = CGF.EmitLValue(Evt);
4145
4146 // Build kmp_event_t *__kmpc_task_allow_completion_event(ident_t *loc_ref,
4147 // int gtid, kmp_task_t *task);
4148 llvm::Value *Loc = emitUpdateLocation(CGF, DC->getBeginLoc());
4149 llvm::Value *Tid = getThreadID(CGF, DC->getBeginLoc());
4150 Tid = CGF.Builder.CreateIntCast(Tid, CGF.IntTy, /*isSigned=*/false);
4151 llvm::Value *EvtVal = CGF.EmitRuntimeCall(
4152 OMPBuilder.getOrCreateRuntimeFunction(
4153 CGM.getModule(), OMPRTL___kmpc_task_allow_completion_event),
4154 {Loc, Tid, NewTask});
4155 EvtVal = CGF.EmitScalarConversion(EvtVal, C.VoidPtrTy, Evt->getType(),
4156 Evt->getExprLoc());
4157 CGF.EmitStoreOfScalar(EvtVal, EvtLVal);
4158 }
4159 // Process affinity clauses.
4160 if (D.hasClausesOfKind<OMPAffinityClause>()) {
4161 // Process list of affinity data.
4162 ASTContext &C = CGM.getContext();
4163 Address AffinitiesArray = Address::invalid();
4164 // Calculate number of elements to form the array of affinity data.
4165 llvm::Value *NumOfElements = nullptr;
4166 unsigned NumAffinities = 0;
4167 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4168 if (const Expr *Modifier = C->getModifier()) {
4169 const auto *IE = cast<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts());
4170 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4171 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper);
4172 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false);
4173 NumOfElements =
4174 NumOfElements ? CGF.Builder.CreateNUWMul(NumOfElements, Sz) : Sz;
4175 }
4176 } else {
4177 NumAffinities += C->varlist_size();
4178 }
4179 }
4181 // Fields ids in kmp_task_affinity_info record.
4182 enum RTLAffinityInfoFieldsTy { BaseAddr, Len, Flags };
4183
4184 QualType KmpTaskAffinityInfoArrayTy;
4185 if (NumOfElements) {
4186 NumOfElements = CGF.Builder.CreateNUWAdd(
4187 llvm::ConstantInt::get(CGF.SizeTy, NumAffinities), NumOfElements);
4188 auto *OVE = new (C) OpaqueValueExpr(
4189 Loc,
4190 C.getIntTypeForBitwidth(C.getTypeSize(C.getSizeType()), /*Signed=*/0),
4191 VK_PRValue);
4192 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE,
4193 RValue::get(NumOfElements));
4194 KmpTaskAffinityInfoArrayTy = C.getVariableArrayType(
4196 /*IndexTypeQuals=*/0);
4197 // Properly emit variable-sized array.
4198 auto *PD = ImplicitParamDecl::Create(C, KmpTaskAffinityInfoArrayTy,
4200 CGF.EmitVarDecl(*PD);
4201 AffinitiesArray = CGF.GetAddrOfLocalVar(PD);
4202 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty,
4203 /*isSigned=*/false);
4204 } else {
4205 KmpTaskAffinityInfoArrayTy = C.getConstantArrayType(
4207 llvm::APInt(C.getTypeSize(C.getSizeType()), NumAffinities), nullptr,
4208 ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
4209 AffinitiesArray = CGF.CreateMemTempWithoutCast(KmpTaskAffinityInfoArrayTy,
4210 ".affs.arr.addr");
4211 AffinitiesArray = CGF.Builder.CreateConstArrayGEP(AffinitiesArray, 0);
4212 NumOfElements = llvm::ConstantInt::get(CGM.Int32Ty, NumAffinities,
4213 /*isSigned=*/false);
4214 }
4215
4216 const auto *KmpAffinityInfoRD = KmpTaskAffinityInfoTy->getAsRecordDecl();
4217 // Fill array by elements without iterators.
4218 unsigned Pos = 0;
4219 bool HasIterator = false;
4220 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4221 if (C->getModifier()) {
4222 HasIterator = true;
4223 continue;
4224 }
4225 for (const Expr *E : C->varlist()) {
4226 llvm::Value *Addr;
4227 llvm::Value *Size;
4228 std::tie(Addr, Size) = getPointerAndSize(CGF, E);
4229 LValue Base =
4230 CGF.MakeAddrLValue(CGF.Builder.CreateConstGEP(AffinitiesArray, Pos),
4232 // affs[i].base_addr = &<Affinities[i].second>;
4233 LValue BaseAddrLVal = CGF.EmitLValueForField(
4234 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
4235 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy),
4236 BaseAddrLVal);
4237 // affs[i].len = sizeof(<Affinities[i].second>);
4238 LValue LenLVal = CGF.EmitLValueForField(
4239 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
4240 CGF.EmitStoreOfScalar(Size, LenLVal);
4241 ++Pos;
4242 }
4243 }
4244 LValue PosLVal;
4245 if (HasIterator) {
4246 PosLVal = CGF.MakeAddrLValue(
4247 CGF.CreateMemTempWithoutCast(C.getSizeType(), "affs.counter.addr"),
4248 C.getSizeType());
4249 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal);
4250 }
4251 // Process elements with iterators.
4252 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4253 const Expr *Modifier = C->getModifier();
4254 if (!Modifier)
4255 continue;
4256 OMPIteratorGeneratorScope IteratorScope(
4257 CGF, cast_or_null<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts()));
4258 for (const Expr *E : C->varlist()) {
4259 llvm::Value *Addr;
4260 llvm::Value *Size;
4261 std::tie(Addr, Size) = getPointerAndSize(CGF, E);
4262 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4263 LValue Base =
4264 CGF.MakeAddrLValue(CGF.Builder.CreateGEP(CGF, AffinitiesArray, Idx),
4266 // affs[i].base_addr = &<Affinities[i].second>;
4267 LValue BaseAddrLVal = CGF.EmitLValueForField(
4268 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
4269 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy),
4270 BaseAddrLVal);
4271 // affs[i].len = sizeof(<Affinities[i].second>);
4272 LValue LenLVal = CGF.EmitLValueForField(
4273 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));
4274 CGF.EmitStoreOfScalar(Size, LenLVal);
4275 Idx = CGF.Builder.CreateNUWAdd(
4276 Idx, llvm::ConstantInt::get(Idx->getType(), 1));
4277 CGF.EmitStoreOfScalar(Idx, PosLVal);
4278 }
4279 }
4280 // Call to kmp_int32 __kmpc_omp_reg_task_with_affinity(ident_t *loc_ref,
4281 // kmp_int32 gtid, kmp_task_t *new_task, kmp_int32
4282 // naffins, kmp_task_affinity_info_t *affin_list);
4283 llvm::Value *LocRef = emitUpdateLocation(CGF, Loc);
4284 llvm::Value *GTid = getThreadID(CGF, Loc);
4285 llvm::Value *AffinListPtr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4286 AffinitiesArray.emitRawPointer(CGF), CGM.VoidPtrTy);
4287 // FIXME: Emit the function and ignore its result for now unless the
4288 // runtime function is properly implemented.
4289 (void)CGF.EmitRuntimeCall(
4290 OMPBuilder.getOrCreateRuntimeFunction(
4291 CGM.getModule(), OMPRTL___kmpc_omp_reg_task_with_affinity),
4292 {LocRef, GTid, NewTask, NumOfElements, AffinListPtr});
4293 }
4294 llvm::Value *NewTaskNewTaskTTy =
4296 NewTask, KmpTaskTWithPrivatesPtrTy);
4297 LValue Base = CGF.MakeNaturalAlignRawAddrLValue(NewTaskNewTaskTTy,
4298 KmpTaskTWithPrivatesQTy);
4299 LValue TDBase =
4300 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
4301 // Fill the data in the resulting kmp_task_t record.
4302 // Copy shareds if there are any.
4303 Address KmpTaskSharedsPtr = Address::invalid();
4304 if (!SharedsTy->castAsRecordDecl()->field_empty()) {
4305 KmpTaskSharedsPtr = Address(
4306 CGF.EmitLoadOfScalar(
4308 TDBase,
4309 *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds)),
4310 Loc),
4311 CGF.Int8Ty, CGM.getNaturalTypeAlignment(SharedsTy));
4312 LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy);
4313 LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy);
4314 CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap);
4315 }
4316 // Emit initial values for private copies (if any).
4318 if (!Privates.empty()) {
4319 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
4320 SharedsTy, SharedsPtrTy, Data, Privates,
4321 /*ForDup=*/false);
4322 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
4323 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
4324 Result.TaskDupFn = emitTaskDupFunction(
4325 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
4326 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
4327 /*WithLastIter=*/!Data.LastprivateVars.empty());
4328 }
4329 }
4330 // Fields of union "kmp_cmplrdata_t" for destructors and priority.
4331 enum { Priority = 0, Destructors = 1 };
4332 // Provide pointer to function with destructors for privates.
4333 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
4334 const auto *KmpCmplrdataUD = (*FI)->getType()->castAsRecordDecl();
4335 assert(KmpCmplrdataUD->isUnion());
4336 if (NeedsCleanup) {
4337 llvm::Value *DestructorFn = emitDestructorsFunction(
4338 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4339 KmpTaskTWithPrivatesQTy);
4340 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI);
4341 LValue DestructorsLV = CGF.EmitLValueForField(
4342 Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
4344 DestructorFn, KmpRoutineEntryPtrTy),
4345 DestructorsLV);
4346 }
4347 // Set priority.
4348 if (Data.Priority.getInt()) {
4349 LValue Data2LV = CGF.EmitLValueForField(
4350 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
4351 LValue PriorityLV = CGF.EmitLValueForField(
4352 Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority));
4353 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV);
4354 }
4355 Result.NewTask = NewTask;
4356 Result.TaskEntry = TaskEntry;
4357 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
4358 Result.TDBase = TDBase;
4359 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
4360 return Result;
4361}
4362
4363/// Translates internal dependency kind into the runtime kind.
4365 RTLDependenceKindTy DepKind;
4366 switch (K) {
4367 case OMPC_DEPEND_in:
4368 DepKind = RTLDependenceKindTy::DepIn;
4369 break;
4370 // Out and InOut dependencies must use the same code.
4371 case OMPC_DEPEND_out:
4372 case OMPC_DEPEND_inout:
4373 DepKind = RTLDependenceKindTy::DepInOut;
4374 break;
4375 case OMPC_DEPEND_mutexinoutset:
4376 DepKind = RTLDependenceKindTy::DepMutexInOutSet;
4377 break;
4378 case OMPC_DEPEND_inoutset:
4379 DepKind = RTLDependenceKindTy::DepInOutSet;
4380 break;
4381 case OMPC_DEPEND_outallmemory:
4382 DepKind = RTLDependenceKindTy::DepOmpAllMem;
4383 break;
4384 case OMPC_DEPEND_source:
4385 case OMPC_DEPEND_sink:
4386 case OMPC_DEPEND_depobj:
4387 case OMPC_DEPEND_inoutallmemory:
4389 llvm_unreachable("Unknown task dependence type");
4390 }
4391 return DepKind;
4392}
4393
4394/// Builds kmp_depend_info, if it is not built yet, and builds flags type.
4395static void getDependTypes(ASTContext &C, QualType &KmpDependInfoTy,
4396 QualType &FlagsTy) {
4397 FlagsTy = C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy), /*Signed=*/false);
4398 if (KmpDependInfoTy.isNull()) {
4399 RecordDecl *KmpDependInfoRD = C.buildImplicitRecord("kmp_depend_info");
4400 KmpDependInfoRD->startDefinition();
4401 addFieldToRecordDecl(C, KmpDependInfoRD, C.getIntPtrType());
4402 addFieldToRecordDecl(C, KmpDependInfoRD, C.getSizeType());
4403 addFieldToRecordDecl(C, KmpDependInfoRD, FlagsTy);
4404 KmpDependInfoRD->completeDefinition();
4405 KmpDependInfoTy = C.getCanonicalTagType(KmpDependInfoRD);
4406 }
4407}
4408
4409std::pair<llvm::Value *, LValue>
4411 SourceLocation Loc) {
4412 ASTContext &C = CGM.getContext();
4413 QualType FlagsTy;
4414 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4415 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();
4416 QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);
4418 DepobjLVal.getAddress().withElementType(
4419 CGF.ConvertTypeForMem(KmpDependInfoPtrTy)),
4420 KmpDependInfoPtrTy->castAs<PointerType>());
4421 Address DepObjAddr = CGF.Builder.CreateGEP(
4422 CGF, Base.getAddress(),
4423 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
4424 LValue NumDepsBase = CGF.MakeAddrLValue(
4425 DepObjAddr, KmpDependInfoTy, Base.getBaseInfo(), Base.getTBAAInfo());
4426 // NumDeps = deps[i].base_addr;
4427 LValue BaseAddrLVal = CGF.EmitLValueForField(
4428 NumDepsBase,
4429 *std::next(KmpDependInfoRD->field_begin(),
4430 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4431 llvm::Value *NumDeps = CGF.EmitLoadOfScalar(BaseAddrLVal, Loc);
4432 return std::make_pair(NumDeps, Base);
4433}
4434
4435static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4436 llvm::PointerUnion<unsigned *, LValue *> Pos,
4438 Address DependenciesArray) {
4439 CodeGenModule &CGM = CGF.CGM;
4440 ASTContext &C = CGM.getContext();
4441 QualType FlagsTy;
4442 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4443 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();
4444 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
4445
4446 OMPIteratorGeneratorScope IteratorScope(
4447 CGF, cast_or_null<OMPIteratorExpr>(
4448 Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4449 : nullptr));
4450 for (const Expr *E : Data.DepExprs) {
4451 llvm::Value *Addr;
4452 llvm::Value *Size;
4453
4454 // The expression will be a nullptr in the 'omp_all_memory' case.
4455 if (E) {
4456 std::tie(Addr, Size) = getPointerAndSize(CGF, E);
4457 Addr = CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy);
4458 } else {
4459 Addr = llvm::ConstantInt::get(CGF.IntPtrTy, 0);
4460 Size = llvm::ConstantInt::get(CGF.SizeTy, 0);
4461 }
4462 LValue Base;
4463 if (unsigned *P = dyn_cast<unsigned *>(Pos)) {
4464 Base = CGF.MakeAddrLValue(
4465 CGF.Builder.CreateConstGEP(DependenciesArray, *P), KmpDependInfoTy);
4466 } else {
4467 assert(E && "Expected a non-null expression");
4468 LValue &PosLVal = *cast<LValue *>(Pos);
4469 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4470 Base = CGF.MakeAddrLValue(
4471 CGF.Builder.CreateGEP(CGF, DependenciesArray, Idx), KmpDependInfoTy);
4472 }
4473 // deps[i].base_addr = &<Dependencies[i].second>;
4474 LValue BaseAddrLVal = CGF.EmitLValueForField(
4475 Base,
4476 *std::next(KmpDependInfoRD->field_begin(),
4477 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4478 CGF.EmitStoreOfScalar(Addr, BaseAddrLVal);
4479 // deps[i].len = sizeof(<Dependencies[i].second>);
4480 LValue LenLVal = CGF.EmitLValueForField(
4481 Base, *std::next(KmpDependInfoRD->field_begin(),
4482 static_cast<unsigned int>(RTLDependInfoFields::Len)));
4483 CGF.EmitStoreOfScalar(Size, LenLVal);
4484 // deps[i].flags = <Dependencies[i].first>;
4485 RTLDependenceKindTy DepKind = translateDependencyKind(Data.DepKind);
4486 LValue FlagsLVal = CGF.EmitLValueForField(
4487 Base,
4488 *std::next(KmpDependInfoRD->field_begin(),
4489 static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4491 llvm::ConstantInt::get(LLVMFlagsTy, static_cast<unsigned int>(DepKind)),
4492 FlagsLVal);
4493 if (unsigned *P = dyn_cast<unsigned *>(Pos)) {
4494 ++(*P);
4495 } else {
4496 LValue &PosLVal = *cast<LValue *>(Pos);
4497 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4498 Idx = CGF.Builder.CreateNUWAdd(Idx,
4499 llvm::ConstantInt::get(Idx->getType(), 1));
4500 CGF.EmitStoreOfScalar(Idx, PosLVal);
4501 }
4502 }
4503}
4504
4508 assert(Data.DepKind == OMPC_DEPEND_depobj &&
4509 "Expected depobj dependency kind.");
4511 SmallVector<LValue, 4> SizeLVals;
4512 ASTContext &C = CGF.getContext();
4513 {
4514 OMPIteratorGeneratorScope IteratorScope(
4515 CGF, cast_or_null<OMPIteratorExpr>(
4516 Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4517 : nullptr));
4518 for (const Expr *E : Data.DepExprs) {
4519 llvm::Value *NumDeps;
4520 LValue Base;
4521 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts());
4522 std::tie(NumDeps, Base) =
4523 getDepobjElements(CGF, DepobjLVal, E->getExprLoc());
4524 LValue NumLVal = CGF.MakeAddrLValue(
4525 CGF.CreateMemTempWithoutCast(C.getUIntPtrType(), "depobj.size.addr"),
4526 C.getUIntPtrType());
4527 CGF.Builder.CreateStore(llvm::ConstantInt::get(CGF.IntPtrTy, 0),
4528 NumLVal.getAddress());
4529 llvm::Value *PrevVal = CGF.EmitLoadOfScalar(NumLVal, E->getExprLoc());
4530 llvm::Value *Add = CGF.Builder.CreateNUWAdd(PrevVal, NumDeps);
4531 CGF.EmitStoreOfScalar(Add, NumLVal);
4532 SizeLVals.push_back(NumLVal);
4533 }
4534 }
4535 for (unsigned I = 0, E = SizeLVals.size(); I < E; ++I) {
4536 llvm::Value *Size =
4537 CGF.EmitLoadOfScalar(SizeLVals[I], Data.DepExprs[I]->getExprLoc());
4538 Sizes.push_back(Size);
4539 }
4540 return Sizes;
4541}
4542
4545 LValue PosLVal,
4547 Address DependenciesArray) {
4548 assert(Data.DepKind == OMPC_DEPEND_depobj &&
4549 "Expected depobj dependency kind.");
4550 llvm::Value *ElSize = CGF.getTypeSize(KmpDependInfoTy);
4551 {
4552 OMPIteratorGeneratorScope IteratorScope(
4553 CGF, cast_or_null<OMPIteratorExpr>(
4554 Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4555 : nullptr));
4556 for (const Expr *E : Data.DepExprs) {
4557 llvm::Value *NumDeps;
4558 LValue Base;
4559 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts());
4560 std::tie(NumDeps, Base) =
4561 getDepobjElements(CGF, DepobjLVal, E->getExprLoc());
4562
4563 // memcopy dependency data.
4564 llvm::Value *Size = CGF.Builder.CreateNUWMul(
4565 ElSize,
4566 CGF.Builder.CreateIntCast(NumDeps, CGF.SizeTy, /*isSigned=*/false));
4567 llvm::Value *Pos = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
4568 Address DepAddr = CGF.Builder.CreateGEP(CGF, DependenciesArray, Pos);
4569 CGF.Builder.CreateMemCpy(DepAddr, Base.getAddress(), Size);
4570
4571 // Increase pos.
4572 // pos += size;
4573 llvm::Value *Add = CGF.Builder.CreateNUWAdd(Pos, NumDeps);
4574 CGF.EmitStoreOfScalar(Add, PosLVal);
4575 }
4576 }
4577}
4578
4579std::pair<llvm::Value *, Address> CGOpenMPRuntime::emitDependClause(
4581 SourceLocation Loc) {
4582 if (llvm::all_of(Dependencies, [](const OMPTaskDataTy::DependData &D) {
4583 return D.DepExprs.empty();
4584 }))
4585 return std::make_pair(nullptr, Address::invalid());
4586 // Process list of dependencies.
4587 ASTContext &C = CGM.getContext();
4588 Address DependenciesArray = Address::invalid();
4589 llvm::Value *NumOfElements = nullptr;
4590 unsigned NumDependencies = std::accumulate(
4591 Dependencies.begin(), Dependencies.end(), 0,
4592 [](unsigned V, const OMPTaskDataTy::DependData &D) {
4593 return D.DepKind == OMPC_DEPEND_depobj
4594 ? V
4595 : (V + (D.IteratorExpr ? 0 : D.DepExprs.size()));
4596 });
4597 QualType FlagsTy;
4598 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4599 bool HasDepobjDeps = false;
4600 bool HasRegularWithIterators = false;
4601 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.IntPtrTy, 0);
4602 llvm::Value *NumOfRegularWithIterators =
4603 llvm::ConstantInt::get(CGF.IntPtrTy, 0);
4604 // Calculate number of depobj dependencies and regular deps with the
4605 // iterators.
4606 for (const OMPTaskDataTy::DependData &D : Dependencies) {
4607 if (D.DepKind == OMPC_DEPEND_depobj) {
4610 for (llvm::Value *Size : Sizes) {
4611 NumOfDepobjElements =
4612 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, Size);
4613 }
4614 HasDepobjDeps = true;
4615 continue;
4616 }
4617 // Include number of iterations, if any.
4618
4619 if (const auto *IE = cast_or_null<OMPIteratorExpr>(D.IteratorExpr)) {
4620 llvm::Value *ClauseIteratorSpace =
4621 llvm::ConstantInt::get(CGF.IntPtrTy, 1);
4622 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4623 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper);
4624 Sz = CGF.Builder.CreateIntCast(Sz, CGF.IntPtrTy, /*isSigned=*/false);
4625 ClauseIteratorSpace = CGF.Builder.CreateNUWMul(Sz, ClauseIteratorSpace);
4626 }
4627 llvm::Value *NumClauseDeps = CGF.Builder.CreateNUWMul(
4628 ClauseIteratorSpace,
4629 llvm::ConstantInt::get(CGF.IntPtrTy, D.DepExprs.size()));
4630 NumOfRegularWithIterators =
4631 CGF.Builder.CreateNUWAdd(NumOfRegularWithIterators, NumClauseDeps);
4632 HasRegularWithIterators = true;
4633 continue;
4634 }
4635 }
4636
4637 QualType KmpDependInfoArrayTy;
4638 if (HasDepobjDeps || HasRegularWithIterators) {
4639 NumOfElements = llvm::ConstantInt::get(CGM.IntPtrTy, NumDependencies,
4640 /*isSigned=*/false);
4641 if (HasDepobjDeps) {
4642 NumOfElements =
4643 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, NumOfElements);
4644 }
4645 if (HasRegularWithIterators) {
4646 NumOfElements =
4647 CGF.Builder.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements);
4648 }
4649 auto *OVE = new (C) OpaqueValueExpr(
4650 Loc, C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0),
4651 VK_PRValue);
4652 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE,
4653 RValue::get(NumOfElements));
4654 KmpDependInfoArrayTy =
4655 C.getVariableArrayType(KmpDependInfoTy, OVE, ArraySizeModifier::Normal,
4656 /*IndexTypeQuals=*/0);
4657 // CGF.EmitVariablyModifiedType(KmpDependInfoArrayTy);
4658 // Properly emit variable-sized array.
4659 auto *PD = ImplicitParamDecl::Create(C, KmpDependInfoArrayTy,
4661 CGF.EmitVarDecl(*PD);
4662 DependenciesArray = CGF.GetAddrOfLocalVar(PD);
4663 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty,
4664 /*isSigned=*/false);
4665 } else {
4666 KmpDependInfoArrayTy = C.getConstantArrayType(
4667 KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies), nullptr,
4668 ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
4669 DependenciesArray =
4670 CGF.CreateMemTempWithoutCast(KmpDependInfoArrayTy, ".dep.arr.addr");
4671 DependenciesArray = CGF.Builder.CreateConstArrayGEP(DependenciesArray, 0);
4672 NumOfElements = llvm::ConstantInt::get(CGM.Int32Ty, NumDependencies,
4673 /*isSigned=*/false);
4674 }
4675 unsigned Pos = 0;
4676 for (const OMPTaskDataTy::DependData &Dep : Dependencies) {
4677 if (Dep.DepKind == OMPC_DEPEND_depobj || Dep.IteratorExpr)
4678 continue;
4679 emitDependData(CGF, KmpDependInfoTy, &Pos, Dep, DependenciesArray);
4680 }
4681 // Copy regular dependencies with iterators.
4682 LValue PosLVal = CGF.MakeAddrLValue(
4683 CGF.CreateMemTempWithoutCast(C.getSizeType(), "dep.counter.addr"),
4684 C.getSizeType());
4685 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal);
4686 for (const OMPTaskDataTy::DependData &Dep : Dependencies) {
4687 if (Dep.DepKind == OMPC_DEPEND_depobj || !Dep.IteratorExpr)
4688 continue;
4689 emitDependData(CGF, KmpDependInfoTy, &PosLVal, Dep, DependenciesArray);
4690 }
4691 // Copy final depobj arrays without iterators.
4692 if (HasDepobjDeps) {
4693 for (const OMPTaskDataTy::DependData &Dep : Dependencies) {
4694 if (Dep.DepKind != OMPC_DEPEND_depobj)
4695 continue;
4696 emitDepobjElements(CGF, KmpDependInfoTy, PosLVal, Dep, DependenciesArray);
4697 }
4698 }
4699 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4700 DependenciesArray, CGF.VoidPtrTy, CGF.Int8Ty);
4701 return std::make_pair(NumOfElements, DependenciesArray);
4702}
4703
4705 CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies,
4706 SourceLocation Loc) {
4707 if (Dependencies.DepExprs.empty())
4708 return Address::invalid();
4709 // Process list of dependencies.
4710 ASTContext &C = CGM.getContext();
4711 Address DependenciesArray = Address::invalid();
4712 unsigned NumDependencies = Dependencies.DepExprs.size();
4713 QualType FlagsTy;
4714 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4715 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();
4716
4717 llvm::Value *Size;
4718 // Define type kmp_depend_info[<Dependencies.size()>];
4719 // For depobj reserve one extra element to store the number of elements.
4720 // It is required to handle depobj(x) update(in) construct.
4721 // kmp_depend_info[<Dependencies.size()>] deps;
4722 llvm::Value *NumDepsVal;
4723 CharUnits Align = C.getTypeAlignInChars(KmpDependInfoTy);
4724 if (const auto *IE =
4725 cast_or_null<OMPIteratorExpr>(Dependencies.IteratorExpr)) {
4726 NumDepsVal = llvm::ConstantInt::get(CGF.SizeTy, 1);
4727 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4728 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper);
4729 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false);
4730 NumDepsVal = CGF.Builder.CreateNUWMul(NumDepsVal, Sz);
4731 }
4732 Size = CGF.Builder.CreateNUWAdd(llvm::ConstantInt::get(CGF.SizeTy, 1),
4733 NumDepsVal);
4734 CharUnits SizeInBytes =
4735 C.getTypeSizeInChars(KmpDependInfoTy).alignTo(Align);
4736 llvm::Value *RecSize = CGM.getSize(SizeInBytes);
4737 Size = CGF.Builder.CreateNUWMul(Size, RecSize);
4738 NumDepsVal =
4739 CGF.Builder.CreateIntCast(NumDepsVal, CGF.IntPtrTy, /*isSigned=*/false);
4740 } else {
4741 QualType KmpDependInfoArrayTy = C.getConstantArrayType(
4742 KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies + 1),
4743 nullptr, ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
4744 CharUnits Sz = C.getTypeSizeInChars(KmpDependInfoArrayTy);
4745 Size = CGM.getSize(Sz.alignTo(Align));
4746 NumDepsVal = llvm::ConstantInt::get(CGF.IntPtrTy, NumDependencies);
4747 }
4748 // Need to allocate on the dynamic memory.
4749 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4750 // Use default allocator.
4751 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4752 llvm::Value *Args[] = {ThreadID, Size, Allocator};
4753
4754 llvm::Value *Addr =
4755 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
4756 CGM.getModule(), OMPRTL___kmpc_alloc),
4757 Args, ".dep.arr.addr");
4758 llvm::Type *KmpDependInfoLlvmTy = CGF.ConvertTypeForMem(KmpDependInfoTy);
4760 Addr, CGF.Builder.getPtrTy(0));
4761 DependenciesArray = Address(Addr, KmpDependInfoLlvmTy, Align);
4762 // Write number of elements in the first element of array for depobj.
4763 LValue Base = CGF.MakeAddrLValue(DependenciesArray, KmpDependInfoTy);
4764 // deps[i].base_addr = NumDependencies;
4765 LValue BaseAddrLVal = CGF.EmitLValueForField(
4766 Base,
4767 *std::next(KmpDependInfoRD->field_begin(),
4768 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4769 CGF.EmitStoreOfScalar(NumDepsVal, BaseAddrLVal);
4770 llvm::PointerUnion<unsigned *, LValue *> Pos;
4771 unsigned Idx = 1;
4772 LValue PosLVal;
4773 if (Dependencies.IteratorExpr) {
4774 PosLVal = CGF.MakeAddrLValue(
4775 CGF.CreateMemTempWithoutCast(C.getSizeType(), "iterator.counter.addr"),
4776 C.getSizeType());
4777 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Idx), PosLVal,
4778 /*IsInit=*/true);
4779 Pos = &PosLVal;
4780 } else {
4781 Pos = &Idx;
4782 }
4783 emitDependData(CGF, KmpDependInfoTy, Pos, Dependencies, DependenciesArray);
4784 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4785 CGF.Builder.CreateConstGEP(DependenciesArray, 1), CGF.VoidPtrTy,
4786 CGF.Int8Ty);
4787 return DependenciesArray;
4788}
4789
4791 SourceLocation Loc) {
4792 ASTContext &C = CGM.getContext();
4793 QualType FlagsTy;
4794 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4795 LValue Base = CGF.EmitLoadOfPointerLValue(DepobjLVal.getAddress(),
4796 C.VoidPtrTy.castAs<PointerType>());
4797 QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);
4799 Base.getAddress(), CGF.ConvertTypeForMem(KmpDependInfoPtrTy),
4801 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
4802 Addr.getElementType(), Addr.emitRawPointer(CGF),
4803 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
4804 DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(DepObjAddr,
4805 CGF.VoidPtrTy);
4806 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4807 // Use default allocator.
4808 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4809 llvm::Value *Args[] = {ThreadID, DepObjAddr, Allocator};
4810
4811 // _kmpc_free(gtid, addr, nullptr);
4812 (void)CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
4813 CGM.getModule(), OMPRTL___kmpc_free),
4814 Args);
4815}
4816
4818 CodeGenFunction &CGF, LValue DepobjLVal, OpenMPDependClauseKind NewDepKind,
4819 SourceLocation Loc) {
4820 ASTContext &C = CGM.getContext();
4821 QualType FlagsTy;
4822 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4823 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();
4824 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
4825 llvm::Value *NumDeps;
4826 LValue Base;
4827 std::tie(NumDeps, Base) = getDepobjElements(CGF, DepobjLVal, Loc);
4828
4829 Address Begin = Base.getAddress();
4830 // Cast from pointer to array type to pointer to single element.
4831 llvm::Value *End = CGF.Builder.CreateGEP(Begin.getElementType(),
4832 Begin.emitRawPointer(CGF), NumDeps);
4833 // The basic structure here is a while-do loop.
4834 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.body");
4835 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.done");
4836 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
4837 CGF.EmitBlock(BodyBB);
4838 llvm::PHINode *ElementPHI =
4839 CGF.Builder.CreatePHI(Begin.getType(), 2, "omp.elementPast");
4840 ElementPHI->addIncoming(Begin.emitRawPointer(CGF), EntryBB);
4841 Begin = Begin.withPointer(ElementPHI, KnownNonNull);
4842 Base = CGF.MakeAddrLValue(Begin, KmpDependInfoTy, Base.getBaseInfo(),
4843 Base.getTBAAInfo());
4844 // deps[i].flags = NewDepKind;
4845 RTLDependenceKindTy DepKind = translateDependencyKind(NewDepKind);
4846 LValue FlagsLVal = CGF.EmitLValueForField(
4847 Base, *std::next(KmpDependInfoRD->field_begin(),
4848 static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4850 llvm::ConstantInt::get(LLVMFlagsTy, static_cast<unsigned int>(DepKind)),
4851 FlagsLVal);
4852
4853 // Shift the address forward by one element.
4854 llvm::Value *ElementNext =
4855 CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext")
4856 .emitRawPointer(CGF);
4857 ElementPHI->addIncoming(ElementNext, CGF.Builder.GetInsertBlock());
4858 llvm::Value *IsEmpty =
4859 CGF.Builder.CreateICmpEQ(ElementNext, End, "omp.isempty");
4860 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
4861 // Done.
4862 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
4863}
4864
4866 const OMPExecutableDirective &D,
4867 llvm::Function *TaskFunction,
4868 QualType SharedsTy, Address Shareds,
4869 const Expr *IfCond,
4870 const OMPTaskDataTy &Data) {
4871 if (!CGF.HaveInsertPoint())
4872 return;
4873
4875 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
4876 llvm::Value *NewTask = Result.NewTask;
4877 llvm::Function *TaskEntry = Result.TaskEntry;
4878 llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
4879 LValue TDBase = Result.TDBase;
4880 const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
4881 // Process list of dependences.
4882 Address DependenciesArray = Address::invalid();
4883 llvm::Value *NumOfElements;
4884 std::tie(NumOfElements, DependenciesArray) =
4885 emitDependClause(CGF, Data.Dependences, Loc);
4886
4887 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
4888 // libcall.
4889 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
4890 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
4891 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
4892 // list is not empty
4893 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4894 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
4895 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
4896 llvm::Value *DepTaskArgs[7];
4897 if (!Data.Dependences.empty()) {
4898 DepTaskArgs[0] = UpLoc;
4899 DepTaskArgs[1] = ThreadID;
4900 DepTaskArgs[2] = NewTask;
4901 DepTaskArgs[3] = NumOfElements;
4902 DepTaskArgs[4] = DependenciesArray.emitRawPointer(CGF);
4903 DepTaskArgs[5] = CGF.Builder.getInt32(0);
4904 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4905 }
4906 auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, &TaskArgs,
4907 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
4908 if (!Data.Tied) {
4909 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
4910 LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
4911 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
4912 }
4913 if (!Data.Dependences.empty()) {
4914 CGF.EmitRuntimeCall(
4915 OMPBuilder.getOrCreateRuntimeFunction(
4916 CGM.getModule(), OMPRTL___kmpc_omp_task_with_deps),
4917 DepTaskArgs);
4918 } else {
4919 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
4920 CGM.getModule(), OMPRTL___kmpc_omp_task),
4921 TaskArgs);
4922 }
4923 // Check if parent region is untied and build return for untied task;
4924 if (auto *Region =
4925 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
4926 Region->emitUntiedSwitch(CGF);
4927 };
4928
4929 llvm::Value *DepWaitTaskArgs[7];
4930 if (!Data.Dependences.empty()) {
4931 DepWaitTaskArgs[0] = UpLoc;
4932 DepWaitTaskArgs[1] = ThreadID;
4933 DepWaitTaskArgs[2] = NumOfElements;
4934 DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
4935 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
4936 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4937 DepWaitTaskArgs[6] =
4938 llvm::ConstantInt::get(CGF.Int32Ty, Data.HasNowaitClause);
4939 }
4940 auto &M = CGM.getModule();
4941 auto &&ElseCodeGen = [this, &M, &TaskArgs, ThreadID, NewTaskNewTaskTTy,
4942 TaskEntry, &Data, &DepWaitTaskArgs,
4943 Loc](CodeGenFunction &CGF, PrePostActionTy &) {
4944 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
4945 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
4946 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
4947 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
4948 // is specified.
4949 if (!Data.Dependences.empty())
4950 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
4951 M, OMPRTL___kmpc_omp_taskwait_deps_51),
4952 DepWaitTaskArgs);
4953 // Call proxy_task_entry(gtid, new_task);
4954 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
4955 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
4956 Action.Enter(CGF);
4957 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
4958 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
4959 OutlinedFnArgs);
4960 };
4961
4962 // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
4963 // kmp_task_t *new_task);
4964 // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
4965 // kmp_task_t *new_task);
4967 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
4968 M, OMPRTL___kmpc_omp_task_begin_if0),
4969 TaskArgs,
4970 OMPBuilder.getOrCreateRuntimeFunction(
4971 M, OMPRTL___kmpc_omp_task_complete_if0),
4972 TaskArgs);
4973 RCG.setAction(Action);
4974 RCG(CGF);
4975 };
4976
4977 if (IfCond) {
4978 emitIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
4979 } else {
4980 RegionCodeGenTy ThenRCG(ThenCodeGen);
4981 ThenRCG(CGF);
4982 }
4983}
4984
4986 const OMPLoopDirective &D,
4987 llvm::Function *TaskFunction,
4988 QualType SharedsTy, Address Shareds,
4989 const Expr *IfCond,
4990 const OMPTaskDataTy &Data) {
4991 if (!CGF.HaveInsertPoint())
4992 return;
4994 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
4995 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
4996 // libcall.
4997 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
4998 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
4999 // sched, kmp_uint64 grainsize, void *task_dup);
5000 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5001 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5002 llvm::Value *IfVal;
5003 if (IfCond) {
5004 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy,
5005 /*isSigned=*/true);
5006 } else {
5007 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1);
5008 }
5009
5010 LValue LBLVal = CGF.EmitLValueForField(
5011 Result.TDBase,
5012 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
5013 const auto *LBVar =
5015 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(), LBLVal.getQuals(),
5016 /*IsInitializer=*/true);
5017 LValue UBLVal = CGF.EmitLValueForField(
5018 Result.TDBase,
5019 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
5020 const auto *UBVar =
5022 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(), UBLVal.getQuals(),
5023 /*IsInitializer=*/true);
5024 LValue StLVal = CGF.EmitLValueForField(
5025 Result.TDBase,
5026 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
5027 const auto *StVar =
5029 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(), StLVal.getQuals(),
5030 /*IsInitializer=*/true);
5031 // Store reductions address.
5032 LValue RedLVal = CGF.EmitLValueForField(
5033 Result.TDBase,
5034 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
5035 if (Data.Reductions) {
5036 CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);
5037 } else {
5038 CGF.EmitNullInitialization(RedLVal.getAddress(),
5039 CGF.getContext().VoidPtrTy);
5040 }
5041 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
5043 UpLoc,
5044 ThreadID,
5045 Result.NewTask,
5046 IfVal,
5047 LBLVal.getPointer(CGF),
5048 UBLVal.getPointer(CGF),
5049 CGF.EmitLoadOfScalar(StLVal, Loc),
5050 llvm::ConstantInt::getSigned(
5051 CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler
5052 llvm::ConstantInt::getSigned(
5053 CGF.IntTy, Data.Schedule.getPointer()
5054 ? Data.Schedule.getInt() ? NumTasks : Grainsize
5055 : NoSchedule),
5056 Data.Schedule.getPointer()
5057 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
5058 /*isSigned=*/false)
5059 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0)};
5060 if (Data.HasModifier)
5061 TaskArgs.push_back(llvm::ConstantInt::get(CGF.Int32Ty, 1));
5062
5063 TaskArgs.push_back(Result.TaskDupFn
5065 Result.TaskDupFn, CGF.VoidPtrTy)
5066 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy));
5067 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
5068 CGM.getModule(), Data.HasModifier
5069 ? OMPRTL___kmpc_taskloop_5
5070 : OMPRTL___kmpc_taskloop),
5071 TaskArgs);
5072}
5073
5074/// Emit reduction operation for each element of array (required for
5075/// array sections) LHS op = RHS.
5076/// \param Type Type of array.
5077/// \param LHSVar Variable on the left side of the reduction operation
5078/// (references element of array in original variable).
5079/// \param RHSVar Variable on the right side of the reduction operation
5080/// (references element of array in original variable).
5081/// \param RedOpGen Generator of reduction operation with use of LHSVar and
5082/// RHSVar.
5084 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
5085 const VarDecl *RHSVar,
5086 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
5087 const Expr *, const Expr *)> &RedOpGen,
5088 const Expr *XExpr = nullptr, const Expr *EExpr = nullptr,
5089 const Expr *UpExpr = nullptr) {
5090 // Perform element-by-element initialization.
5091 QualType ElementTy;
5092 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar);
5093 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar);
5094
5095 // Drill down to the base element type on both arrays.
5096 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
5097 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr);
5098
5099 llvm::Value *RHSBegin = RHSAddr.emitRawPointer(CGF);
5100 llvm::Value *LHSBegin = LHSAddr.emitRawPointer(CGF);
5101 // Cast from pointer to array type to pointer to single element.
5102 llvm::Value *LHSEnd =
5103 CGF.Builder.CreateGEP(LHSAddr.getElementType(), LHSBegin, NumElements);
5104 // The basic structure here is a while-do loop.
5105 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body");
5106 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done");
5107 llvm::Value *IsEmpty =
5108 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty");
5109 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
5110
5111 // Enter the loop body, making that address the current address.
5112 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
5113 CGF.EmitBlock(BodyBB);
5114
5115 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
5116
5117 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
5118 RHSBegin->getType(), 2, "omp.arraycpy.srcElementPast");
5119 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
5120 Address RHSElementCurrent(
5121 RHSElementPHI, RHSAddr.getElementType(),
5122 RHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5123
5124 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
5125 LHSBegin->getType(), 2, "omp.arraycpy.destElementPast");
5126 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
5127 Address LHSElementCurrent(
5128 LHSElementPHI, LHSAddr.getElementType(),
5129 LHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5130
5131 // Emit copy.
5133 Scope.addPrivate(LHSVar, LHSElementCurrent);
5134 Scope.addPrivate(RHSVar, RHSElementCurrent);
5135 Scope.Privatize();
5136 RedOpGen(CGF, XExpr, EExpr, UpExpr);
5137 Scope.ForceCleanup();
5138
5139 // Shift the address forward by one element.
5140 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32(
5141 LHSAddr.getElementType(), LHSElementPHI, /*Idx0=*/1,
5142 "omp.arraycpy.dest.element");
5143 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32(
5144 RHSAddr.getElementType(), RHSElementPHI, /*Idx0=*/1,
5145 "omp.arraycpy.src.element");
5146 // Check whether we've reached the end.
5147 llvm::Value *Done =
5148 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done");
5149 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
5150 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock());
5151 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock());
5152
5153 // Done.
5154 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
5155}
5156
5157/// Emit reduction combiner. If the combiner is a simple expression emit it as
5158/// is, otherwise consider it as combiner of UDR decl and emit it as a call of
5159/// UDR combiner function.
5161 const Expr *ReductionOp) {
5162 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
5163 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
5164 if (const auto *DRE =
5165 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
5166 if (const auto *DRD =
5167 dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
5168 std::pair<llvm::Function *, llvm::Function *> Reduction =
5172 CGF.EmitIgnoredExpr(ReductionOp);
5173 return;
5174 }
5175 CGF.EmitIgnoredExpr(ReductionOp);
5176}
5177
5179 StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType,
5181 ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps) {
5182 ASTContext &C = CGM.getContext();
5183
5184 // void reduction_func(void *LHSArg, void *RHSArg);
5185 auto *LHSArg =
5186 ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
5187 C.VoidPtrTy, ImplicitParamKind::Other);
5188 auto *RHSArg =
5189 ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
5190 C.VoidPtrTy, ImplicitParamKind::Other);
5191 FunctionArgList Args{LHSArg, RHSArg};
5192 const auto &CGFI =
5193 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5194 std::string Name = getReductionFuncName(ReducerName);
5195 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
5196 llvm::GlobalValue::InternalLinkage, Name,
5197 &CGM.getModule());
5198 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
5199 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
5200 Fn->addFnAttr("sample-profile-suffix-elision-policy", "selected");
5201 Fn->setDoesNotRecurse();
5202 CodeGenFunction CGF(CGM);
5203 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
5204
5205 // Dst = (void*[n])(LHSArg);
5206 // Src = (void*[n])(RHSArg);
5208 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(LHSArg)),
5209 CGF.Builder.getPtrTy(0)),
5210 ArgsElemType, CGF.getPointerAlign());
5212 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(RHSArg)),
5213 CGF.Builder.getPtrTy(0)),
5214 ArgsElemType, CGF.getPointerAlign());
5215
5216 // ...
5217 // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
5218 // ...
5220 const auto *IPriv = Privates.begin();
5221 unsigned Idx = 0;
5222 for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
5223 const auto *RHSVar =
5224 cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl());
5225 Scope.addPrivate(RHSVar, emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar));
5226 const auto *LHSVar =
5227 cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl());
5228 Scope.addPrivate(LHSVar, emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar));
5229 QualType PrivTy = (*IPriv)->getType();
5230 if (PrivTy->isVariablyModifiedType()) {
5231 // Get array size and emit VLA type.
5232 ++Idx;
5233 Address Elem = CGF.Builder.CreateConstArrayGEP(LHS, Idx);
5234 llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem);
5235 const VariableArrayType *VLA =
5236 CGF.getContext().getAsVariableArrayType(PrivTy);
5237 const auto *OVE = cast<OpaqueValueExpr>(VLA->getSizeExpr());
5239 CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy)));
5240 CGF.EmitVariablyModifiedType(PrivTy);
5241 }
5242 }
5243 Scope.Privatize();
5244 IPriv = Privates.begin();
5245 const auto *ILHS = LHSExprs.begin();
5246 const auto *IRHS = RHSExprs.begin();
5247 for (const Expr *E : ReductionOps) {
5248 if ((*IPriv)->getType()->isArrayType()) {
5249 // Emit reduction for array section.
5250 const auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5251 const auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5253 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5254 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5255 emitReductionCombiner(CGF, E);
5256 });
5257 } else {
5258 // Emit reduction for array subscript or single variable.
5259 emitReductionCombiner(CGF, E);
5260 }
5261 ++IPriv;
5262 ++ILHS;
5263 ++IRHS;
5264 }
5265 Scope.ForceCleanup();
5266 CGF.FinishFunction();
5267 return Fn;
5268}
5269
5271 const Expr *ReductionOp,
5272 const Expr *PrivateRef,
5273 const DeclRefExpr *LHS,
5274 const DeclRefExpr *RHS) {
5275 if (PrivateRef->getType()->isArrayType()) {
5276 // Emit reduction for array section.
5277 const auto *LHSVar = cast<VarDecl>(LHS->getDecl());
5278 const auto *RHSVar = cast<VarDecl>(RHS->getDecl());
5280 CGF, PrivateRef->getType(), LHSVar, RHSVar,
5281 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5282 emitReductionCombiner(CGF, ReductionOp);
5283 });
5284 } else {
5285 // Emit reduction for array subscript or single variable.
5286 emitReductionCombiner(CGF, ReductionOp);
5287 }
5288}
5289
5290static std::string generateUniqueName(CodeGenModule &CGM,
5291 llvm::StringRef Prefix, const Expr *Ref);
5292
5294 CodeGenFunction &CGF, SourceLocation Loc, const Expr *Privates,
5295 const Expr *LHSExprs, const Expr *RHSExprs, const Expr *ReductionOps) {
5296
5297 // Create a shared global variable (__shared_reduction_var) to accumulate the
5298 // final result.
5299 //
5300 // Call __kmpc_barrier to synchronize threads before initialization.
5301 //
5302 // The master thread (thread_id == 0) initializes __shared_reduction_var
5303 // with the identity value or initializer.
5304 //
5305 // Call __kmpc_barrier to synchronize before combining.
5306 // For each i:
5307 // - Thread enters critical section.
5308 // - Reads its private value from LHSExprs[i].
5309 // - Updates __shared_reduction_var[i] = RedOp_i(__shared_reduction_var[i],
5310 // Privates[i]).
5311 // - Exits critical section.
5312 //
5313 // Call __kmpc_barrier after combining.
5314 //
5315 // Each thread copies __shared_reduction_var[i] back to RHSExprs[i].
5316 //
5317 // Final __kmpc_barrier to synchronize after broadcasting
5318 QualType PrivateType = Privates->getType();
5319 llvm::Type *LLVMType = CGF.ConvertTypeForMem(PrivateType);
5320
5321 const OMPDeclareReductionDecl *UDR = getReductionInit(ReductionOps);
5322 std::string ReductionVarNameStr;
5323 if (const auto *DRE = dyn_cast<DeclRefExpr>(Privates->IgnoreParenCasts()))
5324 ReductionVarNameStr =
5325 generateUniqueName(CGM, DRE->getDecl()->getNameAsString(), Privates);
5326 else
5327 ReductionVarNameStr = "unnamed_priv_var";
5328
5329 // Create an internal shared variable
5330 std::string SharedName =
5331 CGM.getOpenMPRuntime().getName({"internal_pivate_", ReductionVarNameStr});
5332 llvm::GlobalVariable *SharedVar = OMPBuilder.getOrCreateInternalVariable(
5333 LLVMType, ".omp.reduction." + SharedName);
5334
5335 SharedVar->setAlignment(
5336 llvm::MaybeAlign(CGF.getContext().getTypeAlign(PrivateType) / 8));
5337
5338 Address SharedResult =
5339 CGF.MakeNaturalAlignRawAddrLValue(SharedVar, PrivateType).getAddress();
5340
5341 llvm::Value *ThreadId = getThreadID(CGF, Loc);
5342 llvm::Value *BarrierLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE);
5343 llvm::Value *BarrierArgs[] = {BarrierLoc, ThreadId};
5344
5345 llvm::BasicBlock *InitBB = CGF.createBasicBlock("init");
5346 llvm::BasicBlock *InitEndBB = CGF.createBasicBlock("init.end");
5347
5348 llvm::Value *IsWorker = CGF.Builder.CreateICmpEQ(
5349 ThreadId, llvm::ConstantInt::get(ThreadId->getType(), 0));
5350 CGF.Builder.CreateCondBr(IsWorker, InitBB, InitEndBB);
5351
5352 CGF.EmitBlock(InitBB);
5353
5354 auto EmitSharedInit = [&]() {
5355 if (UDR) { // Check if it's a User-Defined Reduction
5356 if (const Expr *UDRInitExpr = UDR->getInitializer()) {
5357 std::pair<llvm::Function *, llvm::Function *> FnPair =
5359 llvm::Function *InitializerFn = FnPair.second;
5360 if (InitializerFn) {
5361 if (const auto *CE =
5362 dyn_cast<CallExpr>(UDRInitExpr->IgnoreParenImpCasts())) {
5363 const auto *OutDRE = cast<DeclRefExpr>(
5364 cast<UnaryOperator>(CE->getArg(0)->IgnoreParenImpCasts())
5365 ->getSubExpr());
5366 const VarDecl *OutVD = cast<VarDecl>(OutDRE->getDecl());
5367
5368 CodeGenFunction::OMPPrivateScope LocalScope(CGF);
5369 LocalScope.addPrivate(OutVD, SharedResult);
5370
5371 (void)LocalScope.Privatize();
5372 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(
5373 CE->getCallee()->IgnoreParenImpCasts())) {
5375 CGF, OVE, RValue::get(InitializerFn));
5376 CGF.EmitIgnoredExpr(CE);
5377 } else {
5378 CGF.EmitAnyExprToMem(UDRInitExpr, SharedResult,
5379 PrivateType.getQualifiers(),
5380 /*IsInitializer=*/true);
5381 }
5382 } else {
5383 CGF.EmitAnyExprToMem(UDRInitExpr, SharedResult,
5384 PrivateType.getQualifiers(),
5385 /*IsInitializer=*/true);
5386 }
5387 } else {
5388 CGF.EmitAnyExprToMem(UDRInitExpr, SharedResult,
5389 PrivateType.getQualifiers(),
5390 /*IsInitializer=*/true);
5391 }
5392 } else {
5393 // EmitNullInitialization handles default construction for C++ classes
5394 // and zeroing for scalars, which is a reasonable default.
5395 CGF.EmitNullInitialization(SharedResult, PrivateType);
5396 }
5397 return; // UDR initialization handled
5398 }
5399 if (const auto *DRE = dyn_cast<DeclRefExpr>(Privates)) {
5400 if (const auto *VD = dyn_cast<VarDecl>(DRE->getDecl())) {
5401 if (const Expr *InitExpr = VD->getInit()) {
5402 CGF.EmitAnyExprToMem(InitExpr, SharedResult,
5403 PrivateType.getQualifiers(), true);
5404 return;
5405 }
5406 }
5407 }
5408 CGF.EmitNullInitialization(SharedResult, PrivateType);
5409 };
5410 EmitSharedInit();
5411 CGF.Builder.CreateBr(InitEndBB);
5412 CGF.EmitBlock(InitEndBB);
5413
5414 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
5415 CGM.getModule(), OMPRTL___kmpc_barrier),
5416 BarrierArgs);
5417
5418 const Expr *ReductionOp = ReductionOps;
5419 const OMPDeclareReductionDecl *CurrentUDR = getReductionInit(ReductionOp);
5420 LValue SharedLV = CGF.MakeAddrLValue(SharedResult, PrivateType);
5421 LValue LHSLV = CGF.EmitLValue(Privates);
5422
5423 auto EmitCriticalReduction = [&](auto ReductionGen) {
5424 std::string CriticalName = getName({"reduction_critical"});
5425 emitCriticalRegion(CGF, CriticalName, ReductionGen, Loc);
5426 };
5427
5428 if (CurrentUDR) {
5429 // Handle user-defined reduction.
5430 auto ReductionGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
5431 Action.Enter(CGF);
5432 std::pair<llvm::Function *, llvm::Function *> FnPair =
5433 getUserDefinedReduction(CurrentUDR);
5434 if (FnPair.first) {
5435 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp)) {
5436 const auto *OutDRE = cast<DeclRefExpr>(
5437 cast<UnaryOperator>(CE->getArg(0)->IgnoreParenImpCasts())
5438 ->getSubExpr());
5439 const auto *InDRE = cast<DeclRefExpr>(
5440 cast<UnaryOperator>(CE->getArg(1)->IgnoreParenImpCasts())
5441 ->getSubExpr());
5442 CodeGenFunction::OMPPrivateScope LocalScope(CGF);
5443 LocalScope.addPrivate(cast<VarDecl>(OutDRE->getDecl()),
5444 SharedLV.getAddress());
5445 LocalScope.addPrivate(cast<VarDecl>(InDRE->getDecl()),
5446 LHSLV.getAddress());
5447 (void)LocalScope.Privatize();
5448 emitReductionCombiner(CGF, ReductionOp);
5449 }
5450 }
5451 };
5452 EmitCriticalReduction(ReductionGen);
5453 } else {
5454 // Handle built-in reduction operations.
5455#ifndef NDEBUG
5456 const Expr *ReductionClauseExpr = ReductionOp->IgnoreParenCasts();
5457 if (const auto *Cleanup = dyn_cast<ExprWithCleanups>(ReductionClauseExpr))
5458 ReductionClauseExpr = Cleanup->getSubExpr()->IgnoreParenCasts();
5459
5460 const Expr *AssignRHS = nullptr;
5461 if (const auto *BinOp = dyn_cast<BinaryOperator>(ReductionClauseExpr)) {
5462 if (BinOp->getOpcode() == BO_Assign)
5463 AssignRHS = BinOp->getRHS();
5464 } else if (const auto *OpCall =
5465 dyn_cast<CXXOperatorCallExpr>(ReductionClauseExpr)) {
5466 if (OpCall->getOperator() == OO_Equal)
5467 AssignRHS = OpCall->getArg(1);
5468 }
5469
5470 assert(AssignRHS &&
5471 "Private Variable Reduction : Invalid ReductionOp expression");
5472#endif
5473
5474 auto ReductionGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
5475 Action.Enter(CGF);
5476 const auto *OmpOutDRE =
5477 dyn_cast<DeclRefExpr>(LHSExprs->IgnoreParenImpCasts());
5478 const auto *OmpInDRE =
5479 dyn_cast<DeclRefExpr>(RHSExprs->IgnoreParenImpCasts());
5480 assert(
5481 OmpOutDRE && OmpInDRE &&
5482 "Private Variable Reduction : LHSExpr/RHSExpr must be DeclRefExprs");
5483 const VarDecl *OmpOutVD = cast<VarDecl>(OmpOutDRE->getDecl());
5484 const VarDecl *OmpInVD = cast<VarDecl>(OmpInDRE->getDecl());
5485 CodeGenFunction::OMPPrivateScope LocalScope(CGF);
5486 LocalScope.addPrivate(OmpOutVD, SharedLV.getAddress());
5487 LocalScope.addPrivate(OmpInVD, LHSLV.getAddress());
5488 (void)LocalScope.Privatize();
5489 // Emit the actual reduction operation
5490 CGF.EmitIgnoredExpr(ReductionOp);
5491 };
5492 EmitCriticalReduction(ReductionGen);
5493 }
5494
5495 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
5496 CGM.getModule(), OMPRTL___kmpc_barrier),
5497 BarrierArgs);
5498
5499 // Broadcast final result
5500 bool IsAggregate = PrivateType->isAggregateType();
5501 LValue SharedLV1 = CGF.MakeAddrLValue(SharedResult, PrivateType);
5502 llvm::Value *FinalResultVal = nullptr;
5503 Address FinalResultAddr = Address::invalid();
5504
5505 if (IsAggregate)
5506 FinalResultAddr = SharedResult;
5507 else
5508 FinalResultVal = CGF.EmitLoadOfScalar(SharedLV1, Loc);
5509
5510 LValue TargetLHSLV = CGF.EmitLValue(RHSExprs);
5511 if (IsAggregate) {
5512 CGF.EmitAggregateCopy(TargetLHSLV,
5513 CGF.MakeAddrLValue(FinalResultAddr, PrivateType),
5514 PrivateType, AggValueSlot::DoesNotOverlap, false);
5515 } else {
5516 CGF.EmitStoreOfScalar(FinalResultVal, TargetLHSLV);
5517 }
5518 // Final synchronization barrier
5519 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
5520 CGM.getModule(), OMPRTL___kmpc_barrier),
5521 BarrierArgs);
5522
5523 // Combiner with original list item
5524 auto OriginalListCombiner = [&](CodeGenFunction &CGF,
5525 PrePostActionTy &Action) {
5526 Action.Enter(CGF);
5527 emitSingleReductionCombiner(CGF, ReductionOps, Privates,
5528 cast<DeclRefExpr>(LHSExprs),
5529 cast<DeclRefExpr>(RHSExprs));
5530 };
5531 EmitCriticalReduction(OriginalListCombiner);
5532}
5533
5535 ArrayRef<const Expr *> OrgPrivates,
5536 ArrayRef<const Expr *> OrgLHSExprs,
5537 ArrayRef<const Expr *> OrgRHSExprs,
5538 ArrayRef<const Expr *> OrgReductionOps,
5539 ReductionOptionsTy Options) {
5540 if (!CGF.HaveInsertPoint())
5541 return;
5542
5543 bool WithNowait = Options.WithNowait;
5544 bool SimpleReduction = Options.SimpleReduction;
5545
5546 // Next code should be emitted for reduction:
5547 //
5548 // static kmp_critical_name lock = { 0 };
5549 //
5550 // void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
5551 // *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]);
5552 // ...
5553 // *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1],
5554 // *(Type<n>-1*)rhs[<n>-1]);
5555 // }
5556 //
5557 // ...
5558 // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
5559 // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5560 // RedList, reduce_func, &<lock>)) {
5561 // case 1:
5562 // ...
5563 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5564 // ...
5565 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5566 // break;
5567 // case 2:
5568 // ...
5569 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5570 // ...
5571 // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);]
5572 // break;
5573 // default:;
5574 // }
5575 //
5576 // if SimpleReduction is true, only the next code is generated:
5577 // ...
5578 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5579 // ...
5580
5581 ASTContext &C = CGM.getContext();
5582
5583 if (SimpleReduction) {
5585 const auto *IPriv = OrgPrivates.begin();
5586 const auto *ILHS = OrgLHSExprs.begin();
5587 const auto *IRHS = OrgRHSExprs.begin();
5588 for (const Expr *E : OrgReductionOps) {
5589 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5590 cast<DeclRefExpr>(*IRHS));
5591 ++IPriv;
5592 ++ILHS;
5593 ++IRHS;
5594 }
5595 return;
5596 }
5597
5598 // Filter out shared reduction variables based on IsPrivateVarReduction flag.
5599 // Only keep entries where the corresponding variable is not private.
5600 SmallVector<const Expr *> FilteredPrivates, FilteredLHSExprs,
5601 FilteredRHSExprs, FilteredReductionOps;
5602 for (unsigned I : llvm::seq<unsigned>(
5603 std::min(OrgReductionOps.size(), OrgLHSExprs.size()))) {
5604 if (!Options.IsPrivateVarReduction[I]) {
5605 FilteredPrivates.emplace_back(OrgPrivates[I]);
5606 FilteredLHSExprs.emplace_back(OrgLHSExprs[I]);
5607 FilteredRHSExprs.emplace_back(OrgRHSExprs[I]);
5608 FilteredReductionOps.emplace_back(OrgReductionOps[I]);
5609 }
5610 }
5611 // Wrap filtered vectors in ArrayRef for downstream shared reduction
5612 // processing.
5613 ArrayRef<const Expr *> Privates = FilteredPrivates;
5614 ArrayRef<const Expr *> LHSExprs = FilteredLHSExprs;
5615 ArrayRef<const Expr *> RHSExprs = FilteredRHSExprs;
5616 ArrayRef<const Expr *> ReductionOps = FilteredReductionOps;
5617
5618 // 1. Build a list of reduction variables.
5619 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
5620 auto Size = RHSExprs.size();
5621 for (const Expr *E : Privates) {
5622 if (E->getType()->isVariablyModifiedType())
5623 // Reserve place for array size.
5624 ++Size;
5625 }
5626 llvm::APInt ArraySize(/*unsigned int numBits=*/32, Size);
5627 QualType ReductionArrayTy = C.getConstantArrayType(
5628 C.VoidPtrTy, ArraySize, nullptr, ArraySizeModifier::Normal,
5629 /*IndexTypeQuals=*/0);
5630 RawAddress ReductionList =
5631 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
5632 const auto *IPriv = Privates.begin();
5633 unsigned Idx = 0;
5634 for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
5635 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
5636 CGF.Builder.CreateStore(
5638 CGF.EmitLValue(RHSExprs[I]).getPointer(CGF), CGF.VoidPtrTy),
5639 Elem);
5640 if ((*IPriv)->getType()->isVariablyModifiedType()) {
5641 // Store array size.
5642 ++Idx;
5643 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
5644 llvm::Value *Size = CGF.Builder.CreateIntCast(
5645 CGF.getVLASize(
5646 CGF.getContext().getAsVariableArrayType((*IPriv)->getType()))
5647 .NumElts,
5648 CGF.SizeTy, /*isSigned=*/false);
5649 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy),
5650 Elem);
5651 }
5652 }
5653
5654 // 2. Emit reduce_func().
5655 llvm::Function *ReductionFn = emitReductionFunction(
5656 CGF.CurFn->getName(), Loc, CGF.ConvertTypeForMem(ReductionArrayTy),
5657 Privates, LHSExprs, RHSExprs, ReductionOps);
5658
5659 // 3. Create static kmp_critical_name lock = { 0 };
5660 std::string Name = getName({"reduction"});
5661 llvm::Value *Lock = getCriticalRegionLock(Name);
5662
5663 // 4. Build res = __kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5664 // RedList, reduce_func, &<lock>);
5665 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE);
5666 llvm::Value *ThreadId = getThreadID(CGF, Loc);
5667 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy);
5668 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5669 ReductionList.getPointer(), CGF.VoidPtrTy);
5670 llvm::Value *Args[] = {
5671 IdentTLoc, // ident_t *<loc>
5672 ThreadId, // i32 <gtid>
5673 CGF.Builder.getInt32(RHSExprs.size()), // i32 <n>
5674 ReductionArrayTySize, // size_type sizeof(RedList)
5675 RL, // void *RedList
5676 ReductionFn, // void (*) (void *, void *) <reduce_func>
5677 Lock // kmp_critical_name *&<lock>
5678 };
5679 llvm::Value *Res = CGF.EmitRuntimeCall(
5680 OMPBuilder.getOrCreateRuntimeFunction(
5681 CGM.getModule(),
5682 WithNowait ? OMPRTL___kmpc_reduce_nowait : OMPRTL___kmpc_reduce),
5683 Args);
5684
5685 // 5. Build switch(res)
5686 llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(".omp.reduction.default");
5687 llvm::SwitchInst *SwInst =
5688 CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2);
5689
5690 // 6. Build case 1:
5691 // ...
5692 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5693 // ...
5694 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5695 // break;
5696 llvm::BasicBlock *Case1BB = CGF.createBasicBlock(".omp.reduction.case1");
5697 SwInst->addCase(CGF.Builder.getInt32(1), Case1BB);
5698 CGF.EmitBlock(Case1BB);
5699
5700 // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5701 llvm::Value *EndArgs[] = {
5702 IdentTLoc, // ident_t *<loc>
5703 ThreadId, // i32 <gtid>
5704 Lock // kmp_critical_name *&<lock>
5705 };
5706 auto &&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps](
5707 CodeGenFunction &CGF, PrePostActionTy &Action) {
5709 const auto *IPriv = Privates.begin();
5710 const auto *ILHS = LHSExprs.begin();
5711 const auto *IRHS = RHSExprs.begin();
5712 for (const Expr *E : ReductionOps) {
5713 RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5714 cast<DeclRefExpr>(*IRHS));
5715 ++IPriv;
5716 ++ILHS;
5717 ++IRHS;
5718 }
5719 };
5721 CommonActionTy Action(
5722 nullptr, {},
5723 OMPBuilder.getOrCreateRuntimeFunction(
5724 CGM.getModule(), WithNowait ? OMPRTL___kmpc_end_reduce_nowait
5725 : OMPRTL___kmpc_end_reduce),
5726 EndArgs);
5727 RCG.setAction(Action);
5728 RCG(CGF);
5729
5730 CGF.EmitBranch(DefaultBB);
5731
5732 // 7. Build case 2:
5733 // ...
5734 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5735 // ...
5736 // break;
5737 llvm::BasicBlock *Case2BB = CGF.createBasicBlock(".omp.reduction.case2");
5738 SwInst->addCase(CGF.Builder.getInt32(2), Case2BB);
5739 CGF.EmitBlock(Case2BB);
5740
5741 auto &&AtomicCodeGen = [Loc, Privates, LHSExprs, RHSExprs, ReductionOps](
5742 CodeGenFunction &CGF, PrePostActionTy &Action) {
5743 const auto *ILHS = LHSExprs.begin();
5744 const auto *IRHS = RHSExprs.begin();
5745 const auto *IPriv = Privates.begin();
5746 for (const Expr *E : ReductionOps) {
5747 const Expr *XExpr = nullptr;
5748 const Expr *EExpr = nullptr;
5749 const Expr *UpExpr = nullptr;
5750 BinaryOperatorKind BO = BO_Comma;
5751 if (const auto *BO = dyn_cast<BinaryOperator>(E)) {
5752 if (BO->getOpcode() == BO_Assign) {
5753 XExpr = BO->getLHS();
5754 UpExpr = BO->getRHS();
5755 }
5756 }
5757 // Try to emit update expression as a simple atomic.
5758 const Expr *RHSExpr = UpExpr;
5759 if (RHSExpr) {
5760 // Analyze RHS part of the whole expression.
5761 if (const auto *ACO = dyn_cast<AbstractConditionalOperator>(
5762 RHSExpr->IgnoreParenImpCasts())) {
5763 // If this is a conditional operator, analyze its condition for
5764 // min/max reduction operator.
5765 RHSExpr = ACO->getCond();
5766 }
5767 if (const auto *BORHS =
5768 dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) {
5769 EExpr = BORHS->getRHS();
5770 BO = BORHS->getOpcode();
5771 }
5772 }
5773 if (XExpr) {
5774 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5775 auto &&AtomicRedGen = [BO, VD,
5776 Loc](CodeGenFunction &CGF, const Expr *XExpr,
5777 const Expr *EExpr, const Expr *UpExpr) {
5778 LValue X = CGF.EmitLValue(XExpr);
5779 RValue E;
5780 if (EExpr)
5781 E = CGF.EmitAnyExpr(EExpr);
5782 CGF.EmitOMPAtomicSimpleUpdateExpr(
5783 X, E, BO, /*IsXLHSInRHSPart=*/true,
5784 llvm::AtomicOrdering::Monotonic, Loc,
5785 [&CGF, UpExpr, VD, Loc](RValue XRValue) {
5786 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5787 Address LHSTemp = CGF.CreateMemTemp(VD->getType());
5788 CGF.emitOMPSimpleStore(
5789 CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue,
5790 VD->getType().getNonReferenceType(), Loc);
5791 PrivateScope.addPrivate(VD, LHSTemp);
5792 (void)PrivateScope.Privatize();
5793 return CGF.EmitAnyExpr(UpExpr);
5794 });
5795 };
5796 if ((*IPriv)->getType()->isArrayType()) {
5797 // Emit atomic reduction for array section.
5798 const auto *RHSVar =
5799 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5800 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar,
5801 AtomicRedGen, XExpr, EExpr, UpExpr);
5802 } else {
5803 // Emit atomic reduction for array subscript or single variable.
5804 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5805 }
5806 } else {
5807 // Emit as a critical region.
5808 auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *,
5809 const Expr *, const Expr *) {
5810 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5811 std::string Name = RT.getName({"atomic_reduction"});
5813 CGF, Name,
5814 [=](CodeGenFunction &CGF, PrePostActionTy &Action) {
5815 Action.Enter(CGF);
5816 emitReductionCombiner(CGF, E);
5817 },
5818 Loc);
5819 };
5820 if ((*IPriv)->getType()->isArrayType()) {
5821 const auto *LHSVar =
5822 cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5823 const auto *RHSVar =
5824 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5825 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5826 CritRedGen);
5827 } else {
5828 CritRedGen(CGF, nullptr, nullptr, nullptr);
5829 }
5830 }
5831 ++ILHS;
5832 ++IRHS;
5833 ++IPriv;
5834 }
5835 };
5836 RegionCodeGenTy AtomicRCG(AtomicCodeGen);
5837 if (!WithNowait) {
5838 // Add emission of __kmpc_end_reduce(<loc>, <gtid>, &<lock>);
5839 llvm::Value *EndArgs[] = {
5840 IdentTLoc, // ident_t *<loc>
5841 ThreadId, // i32 <gtid>
5842 Lock // kmp_critical_name *&<lock>
5843 };
5844 CommonActionTy Action(nullptr, {},
5845 OMPBuilder.getOrCreateRuntimeFunction(
5846 CGM.getModule(), OMPRTL___kmpc_end_reduce),
5847 EndArgs);
5848 AtomicRCG.setAction(Action);
5849 AtomicRCG(CGF);
5850 } else {
5851 AtomicRCG(CGF);
5852 }
5853
5854 CGF.EmitBranch(DefaultBB);
5855 CGF.EmitBlock(DefaultBB, /*IsFinished=*/true);
5856 assert(OrgLHSExprs.size() == OrgPrivates.size() &&
5857 "PrivateVarReduction: Privates size mismatch");
5858 assert(OrgLHSExprs.size() == OrgReductionOps.size() &&
5859 "PrivateVarReduction: ReductionOps size mismatch");
5860 for (unsigned I : llvm::seq<unsigned>(
5861 std::min(OrgReductionOps.size(), OrgLHSExprs.size()))) {
5862 if (Options.IsPrivateVarReduction[I])
5863 emitPrivateReduction(CGF, Loc, OrgPrivates[I], OrgLHSExprs[I],
5864 OrgRHSExprs[I], OrgReductionOps[I]);
5865 }
5866}
5867
5868/// Generates unique name for artificial threadprivate variables.
5869/// Format is: <Prefix> "." <Decl_mangled_name> "_" "<Decl_start_loc_raw_enc>"
5870static std::string generateUniqueName(CodeGenModule &CGM, StringRef Prefix,
5871 const Expr *Ref) {
5872 SmallString<256> Buffer;
5873 llvm::raw_svector_ostream Out(Buffer);
5874 const clang::DeclRefExpr *DE;
5875 const VarDecl *D = ::getBaseDecl(Ref, DE);
5876 if (!D) {
5877 auto *DRE = cast<DeclRefExpr>(Ref);
5878 if (const auto *BD = dyn_cast<BindingDecl>(DRE->getDecl())) {
5879 // For BindingDecls, use the decomposed declaration as the base.
5880 D = cast<VarDecl>(BD->getDecomposedDecl());
5881 } else {
5882 D = cast<VarDecl>(DRE->getDecl());
5883 }
5884 }
5885 D = D->getCanonicalDecl();
5886 std::string Name = CGM.getOpenMPRuntime().getName(
5887 {D->isLocalVarDeclOrParm() ? D->getName() : CGM.getMangledName(D)});
5888 Out << Prefix << Name << "_"
5890 return std::string(Out.str());
5891}
5892
5893/// Emits reduction initializer function:
5894/// \code
5895/// void @.red_init(void* %arg, void* %orig) {
5896/// %0 = bitcast void* %arg to <type>*
5897/// store <type> <init>, <type>* %0
5898/// ret void
5899/// }
5900/// \endcode
5901static llvm::Value *emitReduceInitFunction(CodeGenModule &CGM,
5902 SourceLocation Loc,
5903 ReductionCodeGen &RCG, unsigned N) {
5904 ASTContext &C = CGM.getContext();
5905 QualType VoidPtrTy = C.VoidPtrTy;
5906 VoidPtrTy.addRestrict();
5907 FunctionArgList Args;
5908 auto *Param =
5909 ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
5910 VoidPtrTy, ImplicitParamKind::Other);
5911 auto *ParamOrig =
5912 ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
5913 VoidPtrTy, ImplicitParamKind::Other);
5914 Args.emplace_back(Param);
5915 Args.emplace_back(ParamOrig);
5916 const auto &FnInfo =
5917 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5918 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
5919 std::string Name = CGM.getOpenMPRuntime().getName({"red_init", ""});
5920 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5921 Name, &CGM.getModule());
5922 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
5923 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
5924 Fn->addFnAttr("sample-profile-suffix-elision-policy", "selected");
5925 Fn->setDoesNotRecurse();
5926 CodeGenFunction CGF(CGM);
5927 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
5928 QualType PrivateType = RCG.getPrivateType(N);
5929 Address PrivateAddr = CGF.EmitLoadOfPointer(
5930 CGF.GetAddrOfLocalVar(Param).withElementType(CGF.Builder.getPtrTy(0)),
5931 C.getPointerType(PrivateType)->castAs<PointerType>());
5932 llvm::Value *Size = nullptr;
5933 // If the size of the reduction item is non-constant, load it from global
5934 // threadprivate variable.
5935 if (RCG.getSizes(N).second) {
5937 CGF, CGM.getContext().getSizeType(),
5938 generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N)));
5939 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,
5940 CGM.getContext().getSizeType(), Loc);
5941 }
5942 RCG.emitAggregateType(CGF, N, Size);
5943 Address OrigAddr = Address::invalid();
5944 // If initializer uses initializer from declare reduction construct, emit a
5945 // pointer to the address of the original reduction item (reuired by reduction
5946 // initializer)
5947 if (RCG.usesReductionInitializer(N)) {
5948 Address SharedAddr = CGF.GetAddrOfLocalVar(ParamOrig);
5949 OrigAddr = CGF.EmitLoadOfPointer(
5950 SharedAddr,
5951 CGM.getContext().VoidPtrTy.castAs<PointerType>()->getTypePtr());
5952 }
5953 // Emit the initializer:
5954 // %0 = bitcast void* %arg to <type>*
5955 // store <type> <init>, <type>* %0
5956 RCG.emitInitialization(CGF, N, PrivateAddr, OrigAddr,
5957 [](CodeGenFunction &) { return false; });
5958 CGF.FinishFunction();
5959 return Fn;
5960}
5961
5962/// Emits reduction combiner function:
5963/// \code
5964/// void @.red_comb(void* %arg0, void* %arg1) {
5965/// %lhs = bitcast void* %arg0 to <type>*
5966/// %rhs = bitcast void* %arg1 to <type>*
5967/// %2 = <ReductionOp>(<type>* %lhs, <type>* %rhs)
5968/// store <type> %2, <type>* %lhs
5969/// ret void
5970/// }
5971/// \endcode
5972static llvm::Value *emitReduceCombFunction(CodeGenModule &CGM,
5973 SourceLocation Loc,
5974 ReductionCodeGen &RCG, unsigned N,
5975 const Expr *ReductionOp,
5976 const Expr *LHS, const Expr *RHS,
5977 const Expr *PrivateRef) {
5978 ASTContext &C = CGM.getContext();
5979 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(LHS)->getDecl());
5980 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(RHS)->getDecl());
5981 FunctionArgList Args;
5982 auto *ParamInOut =
5983 ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
5984 C.VoidPtrTy, ImplicitParamKind::Other);
5985 auto *ParamIn =
5986 ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
5987 C.VoidPtrTy, ImplicitParamKind::Other);
5988 Args.emplace_back(ParamInOut);
5989 Args.emplace_back(ParamIn);
5990 const auto &FnInfo =
5991 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5992 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
5993 std::string Name = CGM.getOpenMPRuntime().getName({"red_comb", ""});
5994 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5995 Name, &CGM.getModule());
5996 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
5997 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
5998 Fn->addFnAttr("sample-profile-suffix-elision-policy", "selected");
5999 Fn->setDoesNotRecurse();
6000 CodeGenFunction CGF(CGM);
6001 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
6002 llvm::Value *Size = nullptr;
6003 // If the size of the reduction item is non-constant, load it from global
6004 // threadprivate variable.
6005 if (RCG.getSizes(N).second) {
6007 CGF, CGM.getContext().getSizeType(),
6008 generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N)));
6009 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,
6010 CGM.getContext().getSizeType(), Loc);
6011 }
6012 RCG.emitAggregateType(CGF, N, Size);
6013 // Remap lhs and rhs variables to the addresses of the function arguments.
6014 // %lhs = bitcast void* %arg0 to <type>*
6015 // %rhs = bitcast void* %arg1 to <type>*
6016 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
6017 PrivateScope.addPrivate(
6018 LHSVD,
6019 // Pull out the pointer to the variable.
6021 CGF.GetAddrOfLocalVar(ParamInOut)
6022 .withElementType(CGF.Builder.getPtrTy(0)),
6023 C.getPointerType(LHSVD->getType())->castAs<PointerType>()));
6024 PrivateScope.addPrivate(
6025 RHSVD,
6026 // Pull out the pointer to the variable.
6029 CGF.Builder.getPtrTy(0)),
6030 C.getPointerType(RHSVD->getType())->castAs<PointerType>()));
6031 PrivateScope.Privatize();
6032 // Emit the combiner body:
6033 // %2 = <ReductionOp>(<type> *%lhs, <type> *%rhs)
6034 // store <type> %2, <type>* %lhs
6036 CGF, ReductionOp, PrivateRef, cast<DeclRefExpr>(LHS),
6037 cast<DeclRefExpr>(RHS));
6038 CGF.FinishFunction();
6039 return Fn;
6040}
6041
6042/// Emits reduction finalizer function:
6043/// \code
6044/// void @.red_fini(void* %arg) {
6045/// %0 = bitcast void* %arg to <type>*
6046/// <destroy>(<type>* %0)
6047/// ret void
6048/// }
6049/// \endcode
6050static llvm::Value *emitReduceFiniFunction(CodeGenModule &CGM,
6051 SourceLocation Loc,
6052 ReductionCodeGen &RCG, unsigned N) {
6053 if (!RCG.needCleanups(N))
6054 return nullptr;
6055 ASTContext &C = CGM.getContext();
6056 FunctionArgList Args;
6057 auto *Param =
6058 ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
6059 C.VoidPtrTy, ImplicitParamKind::Other);
6060 Args.emplace_back(Param);
6061 const auto &FnInfo =
6062 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
6063 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
6064 std::string Name = CGM.getOpenMPRuntime().getName({"red_fini", ""});
6065 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
6066 Name, &CGM.getModule());
6067 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
6068 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
6069 Fn->addFnAttr("sample-profile-suffix-elision-policy", "selected");
6070 Fn->setDoesNotRecurse();
6071 CodeGenFunction CGF(CGM);
6072 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
6073 Address PrivateAddr = CGF.EmitLoadOfPointer(
6074 CGF.GetAddrOfLocalVar(Param), C.VoidPtrTy.castAs<PointerType>());
6075 llvm::Value *Size = nullptr;
6076 // If the size of the reduction item is non-constant, load it from global
6077 // threadprivate variable.
6078 if (RCG.getSizes(N).second) {
6080 CGF, CGM.getContext().getSizeType(),
6081 generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N)));
6082 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,
6083 CGM.getContext().getSizeType(), Loc);
6084 }
6085 RCG.emitAggregateType(CGF, N, Size);
6086 // Emit the finalizer body:
6087 // <destroy>(<type>* %0)
6088 RCG.emitCleanups(CGF, N, PrivateAddr);
6089 CGF.FinishFunction(Loc);
6090 return Fn;
6091}
6092
6095 ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) {
6096 if (!CGF.HaveInsertPoint() || Data.ReductionVars.empty())
6097 return nullptr;
6098
6099 // Build typedef struct:
6100 // kmp_taskred_input {
6101 // void *reduce_shar; // shared reduction item
6102 // void *reduce_orig; // original reduction item used for initialization
6103 // size_t reduce_size; // size of data item
6104 // void *reduce_init; // data initialization routine
6105 // void *reduce_fini; // data finalization routine
6106 // void *reduce_comb; // data combiner routine
6107 // kmp_task_red_flags_t flags; // flags for additional info from compiler
6108 // } kmp_taskred_input_t;
6109 ASTContext &C = CGM.getContext();
6110 RecordDecl *RD = C.buildImplicitRecord("kmp_taskred_input_t");
6111 RD->startDefinition();
6112 const FieldDecl *SharedFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);
6113 const FieldDecl *OrigFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);
6114 const FieldDecl *SizeFD = addFieldToRecordDecl(C, RD, C.getSizeType());
6115 const FieldDecl *InitFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);
6116 const FieldDecl *FiniFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);
6117 const FieldDecl *CombFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);
6118 const FieldDecl *FlagsFD = addFieldToRecordDecl(
6119 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/false));
6120 RD->completeDefinition();
6121 CanQualType RDType = C.getCanonicalTagType(RD);
6122 unsigned Size = Data.ReductionVars.size();
6123 llvm::APInt ArraySize(/*numBits=*/64, Size);
6124 QualType ArrayRDType =
6125 C.getConstantArrayType(RDType, ArraySize, nullptr,
6126 ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
6127 // kmp_task_red_input_t .rd_input.[Size];
6128 RawAddress TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input.");
6129 ReductionCodeGen RCG(Data.ReductionVars, Data.ReductionOrigs,
6130 Data.ReductionCopies, Data.ReductionOps);
6131 for (unsigned Cnt = 0; Cnt < Size; ++Cnt) {
6132 // kmp_task_red_input_t &ElemLVal = .rd_input.[Cnt];
6133 llvm::Value *Idxs[] = {llvm::ConstantInt::get(CGM.SizeTy, /*V=*/0),
6134 llvm::ConstantInt::get(CGM.SizeTy, Cnt)};
6135 llvm::Value *GEP = CGF.EmitCheckedInBoundsGEP(
6136 TaskRedInput.getElementType(), TaskRedInput.getPointer(), Idxs,
6137 /*SignedIndices=*/false, /*IsSubtraction=*/false, Loc,
6138 ".rd_input.gep.");
6139 LValue ElemLVal = CGF.MakeNaturalAlignRawAddrLValue(GEP, RDType);
6140 // ElemLVal.reduce_shar = &Shareds[Cnt];
6141 LValue SharedLVal = CGF.EmitLValueForField(ElemLVal, SharedFD);
6142 RCG.emitSharedOrigLValue(CGF, Cnt);
6143 llvm::Value *Shared = RCG.getSharedLValue(Cnt).getPointer(CGF);
6144 CGF.EmitStoreOfScalar(Shared, SharedLVal);
6145 // ElemLVal.reduce_orig = &Origs[Cnt];
6146 LValue OrigLVal = CGF.EmitLValueForField(ElemLVal, OrigFD);
6147 llvm::Value *Orig = RCG.getOrigLValue(Cnt).getPointer(CGF);
6148 CGF.EmitStoreOfScalar(Orig, OrigLVal);
6149 RCG.emitAggregateType(CGF, Cnt);
6150 llvm::Value *SizeValInChars;
6151 llvm::Value *SizeVal;
6152 std::tie(SizeValInChars, SizeVal) = RCG.getSizes(Cnt);
6153 // We use delayed creation/initialization for VLAs and array sections. It is
6154 // required because runtime does not provide the way to pass the sizes of
6155 // VLAs/array sections to initializer/combiner/finalizer functions. Instead
6156 // threadprivate global variables are used to store these values and use
6157 // them in the functions.
6158 bool DelayedCreation = !!SizeVal;
6159 SizeValInChars = CGF.Builder.CreateIntCast(SizeValInChars, CGM.SizeTy,
6160 /*isSigned=*/false);
6161 LValue SizeLVal = CGF.EmitLValueForField(ElemLVal, SizeFD);
6162 CGF.EmitStoreOfScalar(SizeValInChars, SizeLVal);
6163 // ElemLVal.reduce_init = init;
6164 LValue InitLVal = CGF.EmitLValueForField(ElemLVal, InitFD);
6165 llvm::Value *InitAddr = emitReduceInitFunction(CGM, Loc, RCG, Cnt);
6166 CGF.EmitStoreOfScalar(InitAddr, InitLVal);
6167 // ElemLVal.reduce_fini = fini;
6168 LValue FiniLVal = CGF.EmitLValueForField(ElemLVal, FiniFD);
6169 llvm::Value *Fini = emitReduceFiniFunction(CGM, Loc, RCG, Cnt);
6170 llvm::Value *FiniAddr =
6171 Fini ? Fini : llvm::ConstantPointerNull::get(CGM.VoidPtrTy);
6172 CGF.EmitStoreOfScalar(FiniAddr, FiniLVal);
6173 // ElemLVal.reduce_comb = comb;
6174 LValue CombLVal = CGF.EmitLValueForField(ElemLVal, CombFD);
6175 llvm::Value *CombAddr = emitReduceCombFunction(
6176 CGM, Loc, RCG, Cnt, Data.ReductionOps[Cnt], LHSExprs[Cnt],
6177 RHSExprs[Cnt], Data.ReductionCopies[Cnt]);
6178 CGF.EmitStoreOfScalar(CombAddr, CombLVal);
6179 // ElemLVal.flags = 0;
6180 LValue FlagsLVal = CGF.EmitLValueForField(ElemLVal, FlagsFD);
6181 if (DelayedCreation) {
6183 llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/1, /*isSigned=*/true),
6184 FlagsLVal);
6185 } else
6186 CGF.EmitNullInitialization(FlagsLVal.getAddress(), FlagsLVal.getType());
6187 }
6188 if (Data.IsReductionWithTaskMod) {
6189 // Build call void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int
6190 // is_ws, int num, void *data);
6191 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc);
6192 llvm::Value *GTid = CGF.Builder.CreateIntCast(getThreadID(CGF, Loc),
6193 CGM.IntTy, /*isSigned=*/true);
6194 llvm::Value *Args[] = {
6195 IdentTLoc, GTid,
6196 llvm::ConstantInt::get(CGM.IntTy, Data.IsWorksharingReduction ? 1 : 0,
6197 /*isSigned=*/true),
6198 llvm::ConstantInt::get(CGM.IntTy, Size, /*isSigned=*/true),
6200 TaskRedInput.getPointer(), CGM.VoidPtrTy)};
6201 return CGF.EmitRuntimeCall(
6202 OMPBuilder.getOrCreateRuntimeFunction(
6203 CGM.getModule(), OMPRTL___kmpc_taskred_modifier_init),
6204 Args);
6205 }
6206 // Build call void *__kmpc_taskred_init(int gtid, int num_data, void *data);
6207 llvm::Value *Args[] = {
6208 CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), CGM.IntTy,
6209 /*isSigned=*/true),
6210 llvm::ConstantInt::get(CGM.IntTy, Size, /*isSigned=*/true),
6212 CGM.VoidPtrTy)};
6213 return CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
6214 CGM.getModule(), OMPRTL___kmpc_taskred_init),
6215 Args);
6216}
6217
6219 SourceLocation Loc,
6220 bool IsWorksharingReduction) {
6221 // Build call void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int
6222 // is_ws, int num, void *data);
6223 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc);
6224 llvm::Value *GTid = CGF.Builder.CreateIntCast(getThreadID(CGF, Loc),
6225 CGM.IntTy, /*isSigned=*/true);
6226 llvm::Value *Args[] = {IdentTLoc, GTid,
6227 llvm::ConstantInt::get(CGM.IntTy,
6228 IsWorksharingReduction ? 1 : 0,
6229 /*isSigned=*/true)};
6230 (void)CGF.EmitRuntimeCall(
6231 OMPBuilder.getOrCreateRuntimeFunction(
6232 CGM.getModule(), OMPRTL___kmpc_task_reduction_modifier_fini),
6233 Args);
6234}
6235
6237 SourceLocation Loc,
6238 ReductionCodeGen &RCG,
6239 unsigned N) {
6240 auto Sizes = RCG.getSizes(N);
6241 // Emit threadprivate global variable if the type is non-constant
6242 // (Sizes.second = nullptr).
6243 if (Sizes.second) {
6244 llvm::Value *SizeVal = CGF.Builder.CreateIntCast(Sizes.second, CGM.SizeTy,
6245 /*isSigned=*/false);
6247 CGF, CGM.getContext().getSizeType(),
6248 generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N)));
6249 CGF.Builder.CreateStore(SizeVal, SizeAddr, /*IsVolatile=*/false);
6250 }
6251}
6252
6254 SourceLocation Loc,
6255 llvm::Value *ReductionsPtr,
6256 LValue SharedLVal) {
6257 // Build call void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
6258 // *d);
6259 llvm::Value *Args[] = {CGF.Builder.CreateIntCast(getThreadID(CGF, Loc),
6260 CGM.IntTy,
6261 /*isSigned=*/true),
6262 ReductionsPtr,
6264 SharedLVal.getPointer(CGF), CGM.VoidPtrTy)};
6265 return Address(
6266 CGF.EmitRuntimeCall(
6267 OMPBuilder.getOrCreateRuntimeFunction(
6268 CGM.getModule(), OMPRTL___kmpc_task_reduction_get_th_data),
6269 Args),
6270 CGF.Int8Ty, SharedLVal.getAlignment());
6271}
6272
6274 const OMPTaskDataTy &Data) {
6275 if (!CGF.HaveInsertPoint())
6276 return;
6277
6278 if (CGF.CGM.getLangOpts().OpenMPIRBuilder && Data.Dependences.empty()) {
6279 // TODO: Need to support taskwait with dependences in the OpenMPIRBuilder.
6280 OMPBuilder.createTaskwait(CGF.Builder);
6281 } else {
6282 llvm::Value *ThreadID = getThreadID(CGF, Loc);
6283 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
6284 auto &M = CGM.getModule();
6285 Address DependenciesArray = Address::invalid();
6286 llvm::Value *NumOfElements;
6287 std::tie(NumOfElements, DependenciesArray) =
6288 emitDependClause(CGF, Data.Dependences, Loc);
6289 if (!Data.Dependences.empty()) {
6290 llvm::Value *DepWaitTaskArgs[7];
6291 DepWaitTaskArgs[0] = UpLoc;
6292 DepWaitTaskArgs[1] = ThreadID;
6293 DepWaitTaskArgs[2] = NumOfElements;
6294 DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
6295 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
6296 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
6297 DepWaitTaskArgs[6] =
6298 llvm::ConstantInt::get(CGF.Int32Ty, Data.HasNowaitClause);
6299
6300 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
6301
6302 // Build void __kmpc_omp_taskwait_deps_51(ident_t *, kmp_int32 gtid,
6303 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
6304 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list,
6305 // kmp_int32 has_no_wait); if dependence info is specified.
6306 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
6307 M, OMPRTL___kmpc_omp_taskwait_deps_51),
6308 DepWaitTaskArgs);
6309
6310 } else {
6311
6312 // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
6313 // global_tid);
6314 llvm::Value *Args[] = {UpLoc, ThreadID};
6315 // Ignore return result until untied tasks are supported.
6316 CGF.EmitRuntimeCall(
6317 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_omp_taskwait),
6318 Args);
6319 }
6320 }
6321
6322 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
6323 Region->emitUntiedSwitch(CGF);
6324}
6325
6327 OpenMPDirectiveKind InnerKind,
6328 const RegionCodeGenTy &CodeGen,
6329 bool HasCancel) {
6330 if (!CGF.HaveInsertPoint())
6331 return;
6332 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel,
6333 InnerKind != OMPD_critical &&
6334 InnerKind != OMPD_master &&
6335 InnerKind != OMPD_masked);
6336 CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr);
6337}
6338
6339namespace {
6340enum RTCancelKind {
6341 CancelNoreq = 0,
6342 CancelParallel = 1,
6343 CancelLoop = 2,
6344 CancelSections = 3,
6345 CancelTaskgroup = 4
6346};
6347} // anonymous namespace
6348
6349static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion) {
6350 RTCancelKind CancelKind = CancelNoreq;
6351 if (CancelRegion == OMPD_parallel)
6352 CancelKind = CancelParallel;
6353 else if (CancelRegion == OMPD_for)
6354 CancelKind = CancelLoop;
6355 else if (CancelRegion == OMPD_sections)
6356 CancelKind = CancelSections;
6357 else {
6358 assert(CancelRegion == OMPD_taskgroup);
6359 CancelKind = CancelTaskgroup;
6360 }
6361 return CancelKind;
6362}
6363
6366 OpenMPDirectiveKind CancelRegion) {
6367 if (!CGF.HaveInsertPoint())
6368 return;
6369 // Build call kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
6370 // global_tid, kmp_int32 cncl_kind);
6371 if (auto *OMPRegionInfo =
6372 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
6373 // For 'cancellation point taskgroup', the task region info may not have a
6374 // cancel. This may instead happen in another adjacent task.
6375 if (CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) {
6376 llvm::Value *Args[] = {
6377 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
6378 CGF.Builder.getInt32(getCancellationKind(CancelRegion))};
6379 // Ignore return result until untied tasks are supported.
6380 llvm::Value *Result = CGF.EmitRuntimeCall(
6381 OMPBuilder.getOrCreateRuntimeFunction(
6382 CGM.getModule(), OMPRTL___kmpc_cancellationpoint),
6383 Args);
6384 // if (__kmpc_cancellationpoint()) {
6385 // call i32 @__kmpc_cancel_barrier( // for parallel cancellation only
6386 // exit from construct;
6387 // }
6388 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
6389 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
6390 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
6391 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
6392 CGF.EmitBlock(ExitBB);
6393 if (CancelRegion == OMPD_parallel)
6394 emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false);
6395 // exit from construct;
6396 CodeGenFunction::JumpDest CancelDest =
6397 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
6398 CGF.EmitBranchThroughCleanup(CancelDest);
6399 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
6400 }
6401 }
6402}
6403
6405 const Expr *IfCond,
6406 OpenMPDirectiveKind CancelRegion) {
6407 if (!CGF.HaveInsertPoint())
6408 return;
6409 // Build call kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
6410 // kmp_int32 cncl_kind);
6411 auto &M = CGM.getModule();
6412 if (auto *OMPRegionInfo =
6413 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
6414 auto &&ThenGen = [this, &M, Loc, CancelRegion,
6415 OMPRegionInfo](CodeGenFunction &CGF, PrePostActionTy &) {
6416 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
6417 llvm::Value *Args[] = {
6418 RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc),
6419 CGF.Builder.getInt32(getCancellationKind(CancelRegion))};
6420 // Ignore return result until untied tasks are supported.
6421 llvm::Value *Result = CGF.EmitRuntimeCall(
6422 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_cancel), Args);
6423 // if (__kmpc_cancel()) {
6424 // call i32 @__kmpc_cancel_barrier( // for parallel cancellation only
6425 // exit from construct;
6426 // }
6427 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
6428 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
6429 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
6430 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
6431 CGF.EmitBlock(ExitBB);
6432 if (CancelRegion == OMPD_parallel)
6433 RT.emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false);
6434 // exit from construct;
6435 CodeGenFunction::JumpDest CancelDest =
6436 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
6437 CGF.EmitBranchThroughCleanup(CancelDest);
6438 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
6439 };
6440 if (IfCond) {
6441 emitIfClause(CGF, IfCond, ThenGen,
6442 [](CodeGenFunction &, PrePostActionTy &) {});
6443 } else {
6444 RegionCodeGenTy ThenRCG(ThenGen);
6445 ThenRCG(CGF);
6446 }
6447 }
6448}
6449
6450namespace {
6451/// Cleanup action for uses_allocators support.
6452class OMPUsesAllocatorsActionTy final : public PrePostActionTy {
6454 const OMPExecutableDirective &D;
6455 bool IsOffloadEntry;
6456
6457public:
6458 OMPUsesAllocatorsActionTy(
6459 ArrayRef<std::pair<const Expr *, const Expr *>> Allocators,
6460 const OMPExecutableDirective &D, bool IsOffloadEntry)
6461 : Allocators(Allocators), D(D), IsOffloadEntry(IsOffloadEntry) {}
6462 void Enter(CodeGenFunction &CGF) override {
6463 if (!CGF.HaveInsertPoint())
6464 return;
6465 for (const auto &AllocatorData : Allocators) {
6467 CGF, AllocatorData.first, AllocatorData.second);
6468 }
6469 // This kernel does not go through the device-side runtime
6470 // init/deinit sequence (that is GPU-only), but the runtime still
6471 // needs a '<kernel>_kernel_environment' global to know how the
6472 // kernel was configured, so emit it directly here.
6473 if (IsOffloadEntry)
6475 }
6476 void Exit(CodeGenFunction &CGF) override {
6477 if (!CGF.HaveInsertPoint())
6478 return;
6479 for (const auto &AllocatorData : Allocators) {
6481 AllocatorData.first);
6482 }
6483 }
6484};
6485} // namespace
6486
6488 CodeGenFunction &CGF) {
6489 llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs Attrs;
6490 Attrs.ExecFlags = llvm::omp::OMPTgtExecModeFlags::OMP_TGT_EXEC_MODE_GENERIC;
6491 computeMinAndMaxThreadsAndTeams(D, CGF, Attrs);
6492 OMPBuilder.emitKernelEnvironment(CGF.Builder, Attrs);
6493}
6494
6495StringRef
6497 // Regions in a closure are named after its enclosing non-closure function
6498 // (CodeGenFunction::CurFuncDecl), if there is one.
6499 if (const auto *Parent =
6500 dyn_cast_or_null<FunctionDecl>(FD->getNonClosureContext()))
6501 FD = Parent;
6502 if (const auto *Ctor = dyn_cast<CXXConstructorDecl>(FD))
6503 return CGM.getMangledName(GlobalDecl(Ctor, Ctor_Complete));
6504 if (const auto *Dtor = dyn_cast<CXXDestructorDecl>(FD))
6505 return CGM.getMangledName(GlobalDecl(Dtor, Dtor_Complete));
6506 return CGM.getMangledName(GlobalDecl(FD));
6507}
6508
6510 const OMPExecutableDirective &D, StringRef ParentName,
6511 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6512 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
6513 assert(!ParentName.empty() && "Invalid target entry parent name!");
6516 for (const auto *C : D.getClausesOfKind<OMPUsesAllocatorsClause>()) {
6517 for (unsigned I = 0, E = C->getNumberOfAllocators(); I < E; ++I) {
6518 const OMPUsesAllocatorsClause::Data D = C->getAllocatorData(I);
6519 if (!D.AllocatorTraits)
6520 continue;
6521 Allocators.emplace_back(D.Allocator, D.AllocatorTraits);
6522 }
6523 }
6524 OMPUsesAllocatorsActionTy UsesAllocatorAction(Allocators, D, IsOffloadEntry);
6525 CodeGen.setAction(UsesAllocatorAction);
6526 emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,
6527 IsOffloadEntry, CodeGen);
6528}
6529
6531 const Expr *Allocator,
6532 const Expr *AllocatorTraits) {
6533 llvm::Value *ThreadId = getThreadID(CGF, Allocator->getExprLoc());
6534 ThreadId = CGF.Builder.CreateIntCast(ThreadId, CGF.IntTy, /*isSigned=*/true);
6535 // Use default memspace handle.
6536 llvm::Value *MemSpaceHandle = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
6537 llvm::Value *NumTraits = llvm::ConstantInt::get(
6539 AllocatorTraits->getType()->getAsArrayTypeUnsafe())
6540 ->getSize()
6541 .getLimitedValue());
6542 LValue AllocatorTraitsLVal = CGF.EmitLValue(AllocatorTraits);
6544 AllocatorTraitsLVal.getAddress(), CGF.VoidPtrPtrTy, CGF.VoidPtrTy);
6545 AllocatorTraitsLVal = CGF.MakeAddrLValue(Addr, CGF.getContext().VoidPtrTy,
6546 AllocatorTraitsLVal.getBaseInfo(),
6547 AllocatorTraitsLVal.getTBAAInfo());
6548 llvm::Value *Traits = Addr.emitRawPointer(CGF);
6549
6550 llvm::Value *AllocatorVal =
6551 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
6552 CGM.getModule(), OMPRTL___kmpc_init_allocator),
6553 {ThreadId, MemSpaceHandle, NumTraits, Traits});
6554 // Store to allocator.
6556 cast<DeclRefExpr>(Allocator->IgnoreParenImpCasts())->getDecl()));
6557 LValue AllocatorLVal = CGF.EmitLValue(Allocator->IgnoreParenImpCasts());
6558 AllocatorVal =
6559 CGF.EmitScalarConversion(AllocatorVal, CGF.getContext().VoidPtrTy,
6560 Allocator->getType(), Allocator->getExprLoc());
6561 CGF.EmitStoreOfScalar(AllocatorVal, AllocatorLVal);
6562}
6563
6565 const Expr *Allocator) {
6566 llvm::Value *ThreadId = getThreadID(CGF, Allocator->getExprLoc());
6567 ThreadId = CGF.Builder.CreateIntCast(ThreadId, CGF.IntTy, /*isSigned=*/true);
6568 LValue AllocatorLVal = CGF.EmitLValue(Allocator->IgnoreParenImpCasts());
6569 llvm::Value *AllocatorVal =
6570 CGF.EmitLoadOfScalar(AllocatorLVal, Allocator->getExprLoc());
6571 AllocatorVal = CGF.EmitScalarConversion(AllocatorVal, Allocator->getType(),
6572 CGF.getContext().VoidPtrTy,
6573 Allocator->getExprLoc());
6574 (void)CGF.EmitRuntimeCall(
6575 OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
6576 OMPRTL___kmpc_destroy_allocator),
6577 {ThreadId, AllocatorVal});
6578}
6579
6582 llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs) {
6583 assert(Attrs.MaxTeams.size() == 1 && Attrs.MaxThreads.size() == 1 &&
6584 "invalid default attrs structure");
6585 int32_t &MaxTeamsVal = Attrs.MaxTeams.front();
6586 int32_t &MaxThreadsVal = Attrs.MaxThreads.front();
6587
6588 getNumTeamsExprForTargetDirective(CGF, D, Attrs.MinTeams.front(),
6589 MaxTeamsVal);
6590 getNumThreadsExprForTargetDirective(CGF, D, MaxThreadsVal,
6591 /*UpperBoundOnly=*/true);
6592
6593 for (auto *C : D.getClausesOfKind<OMPXAttributeClause>()) {
6594 for (auto *A : C->getAttrs()) {
6595 int32_t AttrMinThreadsVal = 1, AttrMaxThreadsVal = -1;
6596 int32_t AttrMinBlocksVal = 1, AttrMaxBlocksVal = -1;
6597 if (auto *Attr = dyn_cast<CUDALaunchBoundsAttr>(A))
6598 CGM.handleCUDALaunchBoundsAttr(nullptr, Attr, &AttrMaxThreadsVal,
6599 &AttrMinBlocksVal, &AttrMaxBlocksVal);
6600 else if (auto *Attr = dyn_cast<AMDGPUFlatWorkGroupSizeAttr>(A))
6601 CGM.handleAMDGPUFlatWorkGroupSizeAttr(
6602 nullptr, Attr, /*ReqdWGS=*/nullptr, &AttrMinThreadsVal,
6603 &AttrMaxThreadsVal);
6604 else
6605 continue;
6606
6607 Attrs.MinThreads.front() =
6608 std::max(Attrs.MinThreads.front(), AttrMinThreadsVal);
6609 if (AttrMaxThreadsVal > 0)
6610 MaxThreadsVal = MaxThreadsVal > 0
6611 ? std::min(MaxThreadsVal, AttrMaxThreadsVal)
6612 : AttrMaxThreadsVal;
6613 Attrs.MinTeams.front() =
6614 std::max(Attrs.MinTeams.front(), AttrMinBlocksVal);
6615 if (AttrMaxBlocksVal > 0)
6616 MaxTeamsVal = MaxTeamsVal > 0 ? std::min(MaxTeamsVal, AttrMaxBlocksVal)
6617 : AttrMaxBlocksVal;
6618 }
6619 }
6620}
6621
6623 const OMPExecutableDirective &D, StringRef ParentName,
6624 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6625 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
6626
6627 llvm::TargetRegionEntryInfo EntryInfo =
6628 getEntryInfoFromPresumedLoc(CGM, OMPBuilder, D.getBeginLoc(), ParentName);
6629
6630 CodeGenFunction CGF(CGM, true);
6631 llvm::OpenMPIRBuilder::FunctionGenCallback &&GenerateOutlinedFunction =
6632 [&CGF, &D, &CodeGen, this](StringRef EntryFnName) {
6633 const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target);
6634
6635 CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName);
6636 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6637 if (CGM.getLangOpts().OpenMPIsTargetDevice && !isGPU())
6639 return CGF.GenerateOpenMPCapturedStmtFunction(CS, D);
6640 };
6641
6642 cantFail(OMPBuilder.emitTargetRegionFunction(
6643 EntryInfo, GenerateOutlinedFunction, IsOffloadEntry, OutlinedFn,
6644 OutlinedFnID));
6645
6646 if (!OutlinedFn)
6647 return;
6648
6649 // A target body is entered once, from the kernel, and never re-entered by
6650 // the runtime, so it cannot occur in a cycle.
6651 OutlinedFn->setDoesNotRecurse();
6652
6653 CGM.getTargetCodeGenInfo().setTargetAttributes(nullptr, OutlinedFn, CGM);
6654
6655 for (auto *C : D.getClausesOfKind<OMPXAttributeClause>()) {
6656 for (auto *A : C->getAttrs()) {
6657 if (auto *Attr = dyn_cast<AMDGPUWavesPerEUAttr>(A))
6658 CGM.handleAMDGPUWavesPerEUAttr(OutlinedFn, Attr);
6659 }
6660 }
6661 registerVTable(D);
6662}
6663
6664/// Checks if the expression is constant or does not have non-trivial function
6665/// calls.
6666static bool isTrivial(ASTContext &Ctx, const Expr * E) {
6667 // We can skip constant expressions.
6668 // We can skip expressions with trivial calls or simple expressions.
6670 !E->hasNonTrivialCall(Ctx)) &&
6671 !E->HasSideEffects(Ctx, /*IncludePossibleEffects=*/true);
6672}
6673
6675 const Stmt *Body) {
6676 const Stmt *Child = Body->IgnoreContainers();
6677 while (const auto *C = dyn_cast_or_null<CompoundStmt>(Child)) {
6678 Child = nullptr;
6679 for (const Stmt *S : C->body()) {
6680 if (const auto *E = dyn_cast<Expr>(S)) {
6681 if (isTrivial(Ctx, E))
6682 continue;
6683 }
6684 // Some of the statements can be ignored.
6687 continue;
6688 // Analyze declarations.
6689 if (const auto *DS = dyn_cast<DeclStmt>(S)) {
6690 if (llvm::all_of(DS->decls(), [](const Decl *D) {
6691 if (isa<EmptyDecl>(D) || isa<DeclContext>(D) ||
6692 isa<TypeDecl>(D) || isa<PragmaCommentDecl>(D) ||
6693 isa<PragmaDetectMismatchDecl>(D) || isa<UsingDecl>(D) ||
6694 isa<UsingDirectiveDecl>(D) ||
6695 isa<OMPDeclareReductionDecl>(D) ||
6696 isa<OMPThreadPrivateDecl>(D) || isa<OMPAllocateDecl>(D))
6697 return true;
6698 const auto *VD = dyn_cast<VarDecl>(D);
6699 if (!VD)
6700 return false;
6701 return VD->hasGlobalStorage() || !VD->isUsed();
6702 }))
6703 continue;
6704 }
6705 // Found multiple children - cannot get the one child only.
6706 if (Child)
6707 return nullptr;
6708 Child = S;
6709 }
6710 if (Child)
6711 Child = Child->IgnoreContainers();
6712 }
6713 return Child;
6714}
6715
6717 CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &MinTeamsVal,
6718 int32_t &MaxTeamsVal) {
6719
6720 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();
6721 assert(isOpenMPTargetExecutionDirective(DirectiveKind) &&
6722 "Expected target-based executable directive.");
6723 switch (DirectiveKind) {
6724 case OMPD_target: {
6725 const auto *CS = D.getInnermostCapturedStmt();
6726 const auto *Body =
6727 CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
6728 const Stmt *ChildStmt =
6730 if (const auto *NestedDir =
6731 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
6732 if (isOpenMPTeamsDirective(NestedDir->getDirectiveKind())) {
6733 if (NestedDir->hasClausesOfKind<OMPNumTeamsClause>()) {
6734 const Expr *NumTeams = NestedDir->getSingleClause<OMPNumTeamsClause>()
6735 ->getNumTeams()
6736 .front();
6737 if (NumTeams->isIntegerConstantExpr(CGF.getContext()))
6738 if (auto Constant =
6739 NumTeams->getIntegerConstantExpr(CGF.getContext()))
6740 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();
6741 return NumTeams;
6742 }
6743 MinTeamsVal = MaxTeamsVal = 0;
6744 return nullptr;
6745 }
6746 MinTeamsVal = MaxTeamsVal = 1;
6747 return nullptr;
6748 }
6749 // A value of -1 is used to check if we need to emit no teams region
6750 MinTeamsVal = MaxTeamsVal = -1;
6751 return nullptr;
6752 }
6753 case OMPD_target_teams_loop:
6754 case OMPD_target_teams:
6755 case OMPD_target_teams_distribute:
6756 case OMPD_target_teams_distribute_simd:
6757 case OMPD_target_teams_distribute_parallel_for:
6758 case OMPD_target_teams_distribute_parallel_for_simd: {
6759 if (D.hasClausesOfKind<OMPNumTeamsClause>()) {
6760 const Expr *NumTeams =
6761 D.getSingleClause<OMPNumTeamsClause>()->getNumTeams().front();
6762 if (NumTeams->isIntegerConstantExpr(CGF.getContext()))
6763 if (auto Constant = NumTeams->getIntegerConstantExpr(CGF.getContext()))
6764 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();
6765 return NumTeams;
6766 }
6767 MinTeamsVal = MaxTeamsVal = 0;
6768 return nullptr;
6769 }
6770 case OMPD_target_parallel:
6771 case OMPD_target_parallel_for:
6772 case OMPD_target_parallel_for_simd:
6773 case OMPD_target_parallel_loop:
6774 case OMPD_target_simd:
6775 case OMPD_target_loop:
6776 MinTeamsVal = MaxTeamsVal = 1;
6777 return nullptr;
6778 case OMPD_parallel:
6779 case OMPD_for:
6780 case OMPD_parallel_for:
6781 case OMPD_parallel_loop:
6782 case OMPD_parallel_master:
6783 case OMPD_parallel_sections:
6784 case OMPD_for_simd:
6785 case OMPD_parallel_for_simd:
6786 case OMPD_cancel:
6787 case OMPD_cancellation_point:
6788 case OMPD_ordered_standalone:
6789 case OMPD_ordered_blockassoc:
6790 case OMPD_threadprivate:
6791 case OMPD_allocate:
6792 case OMPD_task:
6793 case OMPD_simd:
6794 case OMPD_tile:
6795 case OMPD_unroll:
6796 case OMPD_sections:
6797 case OMPD_section:
6798 case OMPD_single:
6799 case OMPD_master:
6800 case OMPD_critical:
6801 case OMPD_taskyield:
6802 case OMPD_barrier:
6803 case OMPD_taskwait:
6804 case OMPD_taskgroup:
6805 case OMPD_atomic:
6806 case OMPD_flush:
6807 case OMPD_depobj:
6808 case OMPD_scan:
6809 case OMPD_teams:
6810 case OMPD_target_data:
6811 case OMPD_target_exit_data:
6812 case OMPD_target_enter_data:
6813 case OMPD_distribute:
6814 case OMPD_distribute_simd:
6815 case OMPD_distribute_parallel_for:
6816 case OMPD_distribute_parallel_for_simd:
6817 case OMPD_teams_distribute:
6818 case OMPD_teams_distribute_simd:
6819 case OMPD_teams_distribute_parallel_for:
6820 case OMPD_teams_distribute_parallel_for_simd:
6821 case OMPD_target_update:
6822 case OMPD_declare_simd:
6823 case OMPD_declare_variant:
6824 case OMPD_begin_declare_variant:
6825 case OMPD_end_declare_variant:
6826 case OMPD_declare_target:
6827 case OMPD_end_declare_target:
6828 case OMPD_declare_reduction:
6829 case OMPD_declare_mapper:
6830 case OMPD_taskloop:
6831 case OMPD_taskloop_simd:
6832 case OMPD_master_taskloop:
6833 case OMPD_master_taskloop_simd:
6834 case OMPD_parallel_master_taskloop:
6835 case OMPD_parallel_master_taskloop_simd:
6836 case OMPD_requires:
6837 case OMPD_metadirective:
6838 case OMPD_unknown:
6839 break;
6840 default:
6841 break;
6842 }
6843 llvm_unreachable("Unexpected directive kind.");
6844}
6845
6847 CodeGenFunction &CGF, const OMPExecutableDirective &D) {
6848 assert(!CGF.getLangOpts().OpenMPIsTargetDevice &&
6849 "Clauses associated with the teams directive expected to be emitted "
6850 "only for the host!");
6851 CGBuilderTy &Bld = CGF.Builder;
6852 int32_t MinNT = -1, MaxNT = -1;
6853 const Expr *NumTeams =
6854 getNumTeamsExprForTargetDirective(CGF, D, MinNT, MaxNT);
6855 if (NumTeams != nullptr) {
6856 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();
6857
6858 switch (DirectiveKind) {
6859 case OMPD_target: {
6860 const auto *CS = D.getInnermostCapturedStmt();
6861 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6862 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6863 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(NumTeams,
6864 /*IgnoreResultAssign*/ true);
6865 return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty,
6866 /*isSigned=*/true);
6867 }
6868 case OMPD_target_teams:
6869 case OMPD_target_teams_distribute:
6870 case OMPD_target_teams_distribute_simd:
6871 case OMPD_target_teams_distribute_parallel_for:
6872 case OMPD_target_teams_distribute_parallel_for_simd: {
6873 CodeGenFunction::RunCleanupsScope NumTeamsScope(CGF);
6874 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(NumTeams,
6875 /*IgnoreResultAssign*/ true);
6876 return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty,
6877 /*isSigned=*/true);
6878 }
6879 default:
6880 break;
6881 }
6882 }
6883
6884 assert(MinNT == MaxNT && "Num threads ranges require handling here.");
6885 return llvm::ConstantInt::getSigned(CGF.Int32Ty, MinNT);
6886}
6887
6888/// Merge the thread count upper bound \p Val into \p UpperBound.
6889///
6890/// \p UpperBound is -1 while no thread limiting clause has been seen, 0 once
6891/// one has been seen whose value is not known at compile time, and otherwise
6892/// the smallest constant bound found so far.
6893///
6894/// Thread limiting clauses compose by taking the minimum, so a constant bound
6895/// stays valid whatever the clauses that are not compile time constants
6896/// evaluate to. That makes it correct to replace the 0 marker with \p Val, and
6897/// necessary to keep a clause from raising a smaller bound found earlier.
6898static void mergeThreadCountUpperBound(int32_t &UpperBound, int32_t Val) {
6899 UpperBound = UpperBound > 0 ? std::min(UpperBound, Val) : Val;
6900}
6901
6902/// Check for a num threads constant value (stored in \p DefaultVal), or
6903/// expression (stored in \p E). If the value is conditional (via an if-clause),
6904/// store the condition in \p CondVal. If \p E, and \p CondVal respectively, are
6905/// nullptr, no expression evaluation is perfomed.
6906static void getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS,
6907 const Expr **E, int32_t &UpperBound,
6908 bool UpperBoundOnly, llvm::Value **CondVal) {
6910 CGF.getContext(), CS->getCapturedStmt());
6911 const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child);
6912 if (!Dir)
6913 return;
6914
6915 if (isOpenMPParallelDirective(Dir->getDirectiveKind())) {
6916 // Handle if clause. If if clause present, the number of threads is
6917 // calculated as <cond> ? (<numthreads> ? <numthreads> : 0 ) : 1.
6918 if (CondVal && Dir->hasClausesOfKind<OMPIfClause>()) {
6919 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6920 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6921 const OMPIfClause *IfClause = nullptr;
6922 for (const auto *C : Dir->getClausesOfKind<OMPIfClause>()) {
6923 if (C->getNameModifier() == OMPD_unknown ||
6924 C->getNameModifier() == OMPD_parallel) {
6925 IfClause = C;
6926 break;
6927 }
6928 }
6929 if (IfClause) {
6930 const Expr *CondExpr = IfClause->getCondition();
6931 bool Result;
6932 if (CondExpr->EvaluateAsBooleanCondition(Result, CGF.getContext())) {
6933 if (!Result) {
6934 UpperBound = 1;
6935 return;
6936 }
6937 } else {
6939 if (const auto *PreInit =
6940 cast_or_null<DeclStmt>(IfClause->getPreInitStmt())) {
6941 for (const auto *I : PreInit->decls()) {
6942 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6943 CGF.EmitVarDecl(cast<VarDecl>(*I));
6944 } else {
6947 CGF.EmitAutoVarCleanups(Emission);
6948 }
6949 }
6950 *CondVal = CGF.EvaluateExprAsBool(CondExpr);
6951 }
6952 }
6953 }
6954 }
6955 // Check the value of num_threads clause iff if clause was not specified
6956 // or is not evaluated to false.
6957 if (Dir->hasClausesOfKind<OMPNumThreadsClause>()) {
6958 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6959 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6960 const auto *NumThreadsClause =
6961 Dir->getSingleClause<OMPNumThreadsClause>();
6962 const Expr *NTExpr = NumThreadsClause->getNumThreads().front();
6963 if (NTExpr->isIntegerConstantExpr(CGF.getContext()))
6964 if (auto Constant = NTExpr->getIntegerConstantExpr(CGF.getContext()))
6966 UpperBound, static_cast<int32_t>(Constant->getZExtValue()));
6967 // If we haven't found a upper bound, remember we saw a thread limiting
6968 // clause.
6969 if (UpperBound == -1)
6970 UpperBound = 0;
6971 if (!E)
6972 return;
6973 CodeGenFunction::LexicalScope Scope(CGF, NTExpr->getSourceRange());
6974 if (const auto *PreInit =
6975 cast_or_null<DeclStmt>(NumThreadsClause->getPreInitStmt())) {
6976 for (const auto *I : PreInit->decls()) {
6977 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6978 CGF.EmitVarDecl(cast<VarDecl>(*I));
6979 } else {
6982 CGF.EmitAutoVarCleanups(Emission);
6983 }
6984 }
6985 }
6986 *E = NTExpr;
6987 }
6988 return;
6989 }
6990 if (isOpenMPSimdDirective(Dir->getDirectiveKind()))
6991 UpperBound = 1;
6992}
6993
6995 CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &UpperBound,
6996 bool UpperBoundOnly, llvm::Value **CondVal, const Expr **ThreadLimitExpr) {
6997 assert((!CGF.getLangOpts().OpenMPIsTargetDevice || UpperBoundOnly) &&
6998 "Clauses associated with the teams directive expected to be emitted "
6999 "only for the host!");
7000 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();
7001 assert(isOpenMPTargetExecutionDirective(DirectiveKind) &&
7002 "Expected target-based executable directive.");
7003
7004 const Expr *NT = nullptr;
7005 const Expr **NTPtr = UpperBoundOnly ? nullptr : &NT;
7006
7007 auto CheckForConstExpr = [&](const Expr *E, const Expr **EPtr) {
7008 if (E->isIntegerConstantExpr(CGF.getContext())) {
7009 if (auto Constant = E->getIntegerConstantExpr(CGF.getContext()))
7011 UpperBound, static_cast<int32_t>(Constant->getZExtValue()));
7012 }
7013 // If we haven't found a upper bound, remember we saw a thread limiting
7014 // clause.
7015 if (UpperBound == -1)
7016 UpperBound = 0;
7017 if (EPtr)
7018 *EPtr = E;
7019 };
7020
7021 auto ReturnSequential = [&]() {
7022 UpperBound = 1;
7023 return NT;
7024 };
7025
7026 switch (DirectiveKind) {
7027 case OMPD_target: {
7028 const CapturedStmt *CS = D.getInnermostCapturedStmt();
7029 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
7031 CGF.getContext(), CS->getCapturedStmt());
7032 // TODO: The standard is not clear how to resolve two thread limit clauses,
7033 // let's pick the teams one if it's present, otherwise the target one.
7034 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
7035 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
7036 if (const auto *TLC = Dir->getSingleClause<OMPThreadLimitClause>()) {
7037 ThreadLimitClause = TLC;
7038 if (ThreadLimitExpr) {
7039 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
7040 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
7042 CGF,
7043 ThreadLimitClause->getThreadLimit().front()->getSourceRange());
7044 if (const auto *PreInit =
7045 cast_or_null<DeclStmt>(ThreadLimitClause->getPreInitStmt())) {
7046 for (const auto *I : PreInit->decls()) {
7047 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
7048 CGF.EmitVarDecl(cast<VarDecl>(*I));
7049 } else {
7052 CGF.EmitAutoVarCleanups(Emission);
7053 }
7054 }
7055 }
7056 }
7057 }
7058 }
7059 if (ThreadLimitClause)
7060 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
7061 ThreadLimitExpr);
7062 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
7063 if (isOpenMPTeamsDirective(Dir->getDirectiveKind()) &&
7064 !isOpenMPDistributeDirective(Dir->getDirectiveKind())) {
7065 CS = Dir->getInnermostCapturedStmt();
7066 // Now that the 'teams' level has been peeled off, the remainder is
7067 // shaped like a 'target teams' region, so pick up the num_threads of
7068 // the directive nested in it the same way the OMPD_target_teams case
7069 // below does. Without this the upper bound of a construct written as
7070 // 'target' / 'teams' / 'distribute parallel for' would stay at the
7071 // default, while every combined spelling of the same construct honors
7072 // the clause. Only the bound is taken here: passing null for the
7073 // expression and the condition keeps this from emitting anything, so
7074 // the value the host passes to the kernel launch is left as it was.
7075 getNumThreads(CGF, CS, /*E=*/nullptr, UpperBound, UpperBoundOnly,
7076 /*CondVal=*/nullptr);
7078 CGF.getContext(), CS->getCapturedStmt());
7079 Dir = dyn_cast_or_null<OMPExecutableDirective>(Child);
7080 }
7081 if (Dir && isOpenMPParallelDirective(Dir->getDirectiveKind())) {
7082 CS = Dir->getInnermostCapturedStmt();
7083 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
7084 } else if (Dir && isOpenMPSimdDirective(Dir->getDirectiveKind()))
7085 return ReturnSequential();
7086 }
7087 return NT;
7088 }
7089 case OMPD_target_teams: {
7090 if (D.hasClausesOfKind<OMPThreadLimitClause>()) {
7091 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
7092 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
7093 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
7094 ThreadLimitExpr);
7095 }
7096 const CapturedStmt *CS = D.getInnermostCapturedStmt();
7097 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
7099 CGF.getContext(), CS->getCapturedStmt());
7100 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
7101 if (Dir->getDirectiveKind() == OMPD_distribute) {
7102 CS = Dir->getInnermostCapturedStmt();
7103 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);
7104 }
7105 }
7106 return NT;
7107 }
7108 case OMPD_target_teams_distribute:
7109 if (D.hasClausesOfKind<OMPThreadLimitClause>()) {
7110 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
7111 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
7112 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
7113 ThreadLimitExpr);
7114 }
7115 getNumThreads(CGF, D.getInnermostCapturedStmt(), NTPtr, UpperBound,
7116 UpperBoundOnly, CondVal);
7117 return NT;
7118 case OMPD_target_teams_loop:
7119 case OMPD_target_parallel_loop:
7120 case OMPD_target_parallel:
7121 case OMPD_target_parallel_for:
7122 case OMPD_target_parallel_for_simd:
7123 case OMPD_target_teams_distribute_parallel_for:
7124 case OMPD_target_teams_distribute_parallel_for_simd: {
7125 if (CondVal && D.hasClausesOfKind<OMPIfClause>()) {
7126 const OMPIfClause *IfClause = nullptr;
7127 for (const auto *C : D.getClausesOfKind<OMPIfClause>()) {
7128 if (C->getNameModifier() == OMPD_unknown ||
7129 C->getNameModifier() == OMPD_parallel) {
7130 IfClause = C;
7131 break;
7132 }
7133 }
7134 if (IfClause) {
7135 const Expr *Cond = IfClause->getCondition();
7136 bool Result;
7137 if (Cond->EvaluateAsBooleanCondition(Result, CGF.getContext())) {
7138 if (!Result)
7139 return ReturnSequential();
7140 } else {
7142 *CondVal = CGF.EvaluateExprAsBool(Cond);
7143 }
7144 }
7145 }
7146 if (D.hasClausesOfKind<OMPThreadLimitClause>()) {
7147 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
7148 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
7149 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
7150 ThreadLimitExpr);
7151 }
7152 if (D.hasClausesOfKind<OMPNumThreadsClause>()) {
7153 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
7154 const auto *NumThreadsClause = D.getSingleClause<OMPNumThreadsClause>();
7155 CheckForConstExpr(NumThreadsClause->getNumThreads().front(), nullptr);
7156 return NumThreadsClause->getNumThreads().front();
7157 }
7158 return NT;
7159 }
7160 case OMPD_target_teams_distribute_simd:
7161 case OMPD_target_simd:
7162 case OMPD_target_loop:
7163 return ReturnSequential();
7164 default:
7165 break;
7166 }
7167 llvm_unreachable("Unsupported directive kind.");
7168}
7169
7171 CodeGenFunction &CGF, const OMPExecutableDirective &D) {
7172 llvm::Value *NumThreadsVal = nullptr;
7173 llvm::Value *CondVal = nullptr;
7174 llvm::Value *ThreadLimitVal = nullptr;
7175 const Expr *ThreadLimitExpr = nullptr;
7176 int32_t UpperBound = -1;
7177
7179 CGF, D, UpperBound, /* UpperBoundOnly */ false, &CondVal,
7180 &ThreadLimitExpr);
7181
7182 // Thread limit expressions are used below, emit them.
7183 if (ThreadLimitExpr) {
7184 ThreadLimitVal =
7185 CGF.EmitScalarExpr(ThreadLimitExpr, /*IgnoreResultAssign=*/true);
7186 ThreadLimitVal = CGF.Builder.CreateIntCast(ThreadLimitVal, CGF.Int32Ty,
7187 /*isSigned=*/false);
7188 }
7189
7190 // Generate the num teams expression.
7191 if (UpperBound == 1) {
7192 NumThreadsVal = CGF.Builder.getInt32(UpperBound);
7193 } else if (NT) {
7194 NumThreadsVal = CGF.EmitScalarExpr(NT, /*IgnoreResultAssign=*/true);
7195 NumThreadsVal = CGF.Builder.CreateIntCast(NumThreadsVal, CGF.Int32Ty,
7196 /*isSigned=*/false);
7197 } else if (ThreadLimitVal) {
7198 // If we do not have a num threads value but a thread limit, replace the
7199 // former with the latter. We know handled the thread limit expression.
7200 NumThreadsVal = ThreadLimitVal;
7201 ThreadLimitVal = nullptr;
7202 } else {
7203 // Default to "0" which means runtime choice.
7204 assert(!ThreadLimitVal && "Default not applicable with thread limit value");
7205 NumThreadsVal = CGF.Builder.getInt32(0);
7206 }
7207
7208 // Handle if clause. If if clause present, the number of threads is
7209 // calculated as <cond> ? (<numthreads> ? <numthreads> : 0 ) : 1.
7210 if (CondVal) {
7212 NumThreadsVal = CGF.Builder.CreateSelect(CondVal, NumThreadsVal,
7213 CGF.Builder.getInt32(1));
7214 }
7215
7216 // If the thread limit and num teams expression were present, take the
7217 // minimum.
7218 if (ThreadLimitVal) {
7219 NumThreadsVal = CGF.Builder.CreateSelect(
7220 CGF.Builder.CreateICmpULT(ThreadLimitVal, NumThreadsVal),
7221 ThreadLimitVal, NumThreadsVal);
7222 }
7223
7224 return NumThreadsVal;
7225}
7226
7227namespace {
7229
7230// Utility to handle information from clauses associated with a given
7231// construct that use mappable expressions (e.g. 'map' clause, 'to' clause).
7232// It provides a convenient interface to obtain the information and generate
7233// code for that information.
7234class MappableExprsHandler {
7235public:
7236 /// Custom comparator for attach-pointer expressions that compares them by
7237 /// complexity (i.e. their component-depth) first, then by the order in which
7238 /// they were computed by collectAttachPtrExprInfo(), if they are semantically
7239 /// different.
7240 struct AttachPtrExprComparator {
7241 const MappableExprsHandler &Handler;
7242 // Cache of previous equality comparison results.
7243 mutable llvm::DenseMap<std::pair<const Expr *, const Expr *>, bool>
7244 CachedEqualityComparisons;
7245
7246 AttachPtrExprComparator(const MappableExprsHandler &H) : Handler(H) {}
7247 AttachPtrExprComparator() = delete;
7248
7249 // Return true iff LHS is "less than" RHS.
7250 bool operator()(const Expr *LHS, const Expr *RHS) const {
7251 if (LHS == RHS)
7252 return false;
7253
7254 // First, compare by complexity (depth)
7255 const auto ItLHS = Handler.AttachPtrComponentDepthMap.find(LHS);
7256 const auto ItRHS = Handler.AttachPtrComponentDepthMap.find(RHS);
7257
7258 std::optional<size_t> DepthLHS =
7259 (ItLHS != Handler.AttachPtrComponentDepthMap.end()) ? ItLHS->second
7260 : std::nullopt;
7261 std::optional<size_t> DepthRHS =
7262 (ItRHS != Handler.AttachPtrComponentDepthMap.end()) ? ItRHS->second
7263 : std::nullopt;
7264
7265 // std::nullopt (no attach pointer) has lowest complexity
7266 if (!DepthLHS.has_value() && !DepthRHS.has_value()) {
7267 // Both have same complexity, now check semantic equality
7268 if (areEqual(LHS, RHS))
7269 return false;
7270 // Different semantically, compare by computation order
7271 return wasComputedBefore(LHS, RHS);
7272 }
7273 if (!DepthLHS.has_value())
7274 return true; // LHS has lower complexity
7275 if (!DepthRHS.has_value())
7276 return false; // RHS has lower complexity
7277
7278 // Both have values, compare by depth (lower depth = lower complexity)
7279 if (DepthLHS.value() != DepthRHS.value())
7280 return DepthLHS.value() < DepthRHS.value();
7281
7282 // Same complexity, now check semantic equality
7283 if (areEqual(LHS, RHS))
7284 return false;
7285 // Different semantically, compare by computation order
7286 return wasComputedBefore(LHS, RHS);
7287 }
7288
7289 public:
7290 /// Return true if \p LHS and \p RHS are semantically equal. Uses pre-cached
7291 /// results, if available, otherwise does a recursive semantic comparison.
7292 bool areEqual(const Expr *LHS, const Expr *RHS) const {
7293 // Check cache first for faster lookup
7294 const auto CachedResultIt = CachedEqualityComparisons.find({LHS, RHS});
7295 if (CachedResultIt != CachedEqualityComparisons.end())
7296 return CachedResultIt->second;
7297
7298 bool ComparisonResult = areSemanticallyEqual(LHS, RHS);
7299
7300 // Cache the result for future lookups (both orders since semantic
7301 // equality is commutative)
7302 CachedEqualityComparisons[{LHS, RHS}] = ComparisonResult;
7303 CachedEqualityComparisons[{RHS, LHS}] = ComparisonResult;
7304 return ComparisonResult;
7305 }
7306
7307 /// Compare the two attach-ptr expressions by their computation order.
7308 /// Returns true iff LHS was computed before RHS by
7309 /// collectAttachPtrExprInfo().
7310 bool wasComputedBefore(const Expr *LHS, const Expr *RHS) const {
7311 const size_t &OrderLHS = Handler.AttachPtrComputationOrderMap.at(LHS);
7312 const size_t &OrderRHS = Handler.AttachPtrComputationOrderMap.at(RHS);
7313
7314 return OrderLHS < OrderRHS;
7315 }
7316
7317 private:
7318 /// Helper function to compare attach-pointer expressions semantically.
7319 /// This function handles various expression types that can be part of an
7320 /// attach-pointer.
7321 /// TODO: Not urgent, but we should ideally return true when comparing
7322 /// `p[10]`, `*(p + 10)`, `*(p + 5 + 5)`, `p[10:1]` etc.
7323 bool areSemanticallyEqual(const Expr *LHS, const Expr *RHS) const {
7324 if (LHS == RHS)
7325 return true;
7326
7327 // If only one is null, they aren't equal
7328 if (!LHS || !RHS)
7329 return false;
7330
7331 ASTContext &Ctx = Handler.CGF.getContext();
7332 // Strip away parentheses and no-op casts to get to the core expression
7333 LHS = LHS->IgnoreParenNoopCasts(Ctx);
7334 RHS = RHS->IgnoreParenNoopCasts(Ctx);
7335
7336 // Direct pointer comparison of the underlying expressions
7337 if (LHS == RHS)
7338 return true;
7339
7340 // Check if the expression classes match
7341 if (LHS->getStmtClass() != RHS->getStmtClass())
7342 return false;
7343
7344 // Handle DeclRefExpr (variable references)
7345 if (const auto *LD = dyn_cast<DeclRefExpr>(LHS)) {
7346 const auto *RD = dyn_cast<DeclRefExpr>(RHS);
7347 if (!RD)
7348 return false;
7349 return LD->getDecl()->getCanonicalDecl() ==
7350 RD->getDecl()->getCanonicalDecl();
7351 }
7352
7353 // Handle ArraySubscriptExpr (array indexing like a[i])
7354 if (const auto *LA = dyn_cast<ArraySubscriptExpr>(LHS)) {
7355 const auto *RA = dyn_cast<ArraySubscriptExpr>(RHS);
7356 if (!RA)
7357 return false;
7358 return areSemanticallyEqual(LA->getBase(), RA->getBase()) &&
7359 areSemanticallyEqual(LA->getIdx(), RA->getIdx());
7360 }
7361
7362 // Handle MemberExpr (member access like s.m or p->m)
7363 if (const auto *LM = dyn_cast<MemberExpr>(LHS)) {
7364 const auto *RM = dyn_cast<MemberExpr>(RHS);
7365 if (!RM)
7366 return false;
7367 if (LM->getMemberDecl()->getCanonicalDecl() !=
7368 RM->getMemberDecl()->getCanonicalDecl())
7369 return false;
7370 return areSemanticallyEqual(LM->getBase(), RM->getBase());
7371 }
7372
7373 // Handle UnaryOperator (unary operations like *p, &x, etc.)
7374 if (const auto *LU = dyn_cast<UnaryOperator>(LHS)) {
7375 const auto *RU = dyn_cast<UnaryOperator>(RHS);
7376 if (!RU)
7377 return false;
7378 if (LU->getOpcode() != RU->getOpcode())
7379 return false;
7380 return areSemanticallyEqual(LU->getSubExpr(), RU->getSubExpr());
7381 }
7382
7383 // Handle BinaryOperator (binary operations like p + offset)
7384 if (const auto *LB = dyn_cast<BinaryOperator>(LHS)) {
7385 const auto *RB = dyn_cast<BinaryOperator>(RHS);
7386 if (!RB)
7387 return false;
7388 if (LB->getOpcode() != RB->getOpcode())
7389 return false;
7390 return areSemanticallyEqual(LB->getLHS(), RB->getLHS()) &&
7391 areSemanticallyEqual(LB->getRHS(), RB->getRHS());
7392 }
7393
7394 // Handle ArraySectionExpr (array sections like a[0:1])
7395 // Attach pointers should not contain array-sections, but currently we
7396 // don't emit an error.
7397 if (const auto *LAS = dyn_cast<ArraySectionExpr>(LHS)) {
7398 const auto *RAS = dyn_cast<ArraySectionExpr>(RHS);
7399 if (!RAS)
7400 return false;
7401 return areSemanticallyEqual(LAS->getBase(), RAS->getBase()) &&
7402 areSemanticallyEqual(LAS->getLowerBound(),
7403 RAS->getLowerBound()) &&
7404 areSemanticallyEqual(LAS->getLength(), RAS->getLength());
7405 }
7406
7407 // Handle CastExpr (explicit casts)
7408 if (const auto *LC = dyn_cast<CastExpr>(LHS)) {
7409 const auto *RC = dyn_cast<CastExpr>(RHS);
7410 if (!RC)
7411 return false;
7412 if (LC->getCastKind() != RC->getCastKind())
7413 return false;
7414 return areSemanticallyEqual(LC->getSubExpr(), RC->getSubExpr());
7415 }
7416
7417 // Handle CXXThisExpr (this pointer)
7418 if (isa<CXXThisExpr>(LHS) && isa<CXXThisExpr>(RHS))
7419 return true;
7420
7421 // Handle IntegerLiteral (integer constants)
7422 if (const auto *LI = dyn_cast<IntegerLiteral>(LHS)) {
7423 const auto *RI = dyn_cast<IntegerLiteral>(RHS);
7424 if (!RI)
7425 return false;
7426 return LI->getValue() == RI->getValue();
7427 }
7428
7429 // Handle CharacterLiteral (character constants)
7430 if (const auto *LC = dyn_cast<CharacterLiteral>(LHS)) {
7431 const auto *RC = dyn_cast<CharacterLiteral>(RHS);
7432 if (!RC)
7433 return false;
7434 return LC->getValue() == RC->getValue();
7435 }
7436
7437 // Handle FloatingLiteral (floating point constants)
7438 if (const auto *LF = dyn_cast<FloatingLiteral>(LHS)) {
7439 const auto *RF = dyn_cast<FloatingLiteral>(RHS);
7440 if (!RF)
7441 return false;
7442 // Use bitwise comparison for floating point literals
7443 return LF->getValue().bitwiseIsEqual(RF->getValue());
7444 }
7445
7446 // Handle StringLiteral (string constants)
7447 if (const auto *LS = dyn_cast<StringLiteral>(LHS)) {
7448 const auto *RS = dyn_cast<StringLiteral>(RHS);
7449 if (!RS)
7450 return false;
7451 return LS->getString() == RS->getString();
7452 }
7453
7454 // Handle CXXNullPtrLiteralExpr (nullptr)
7456 return true;
7457
7458 // Handle CXXBoolLiteralExpr (true/false)
7459 if (const auto *LB = dyn_cast<CXXBoolLiteralExpr>(LHS)) {
7460 const auto *RB = dyn_cast<CXXBoolLiteralExpr>(RHS);
7461 if (!RB)
7462 return false;
7463 return LB->getValue() == RB->getValue();
7464 }
7465
7466 // Fallback for other forms - use the existing comparison method
7467 return Expr::isSameComparisonOperand(LHS, RHS);
7468 }
7469 };
7470
7471 /// Get the offset of the OMP_MAP_MEMBER_OF field.
7472 static unsigned getFlagMemberOffset() {
7473 unsigned Offset = 0;
7474 for (uint64_t Remain =
7475 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
7476 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
7477 !(Remain & 1); Remain = Remain >> 1)
7478 Offset++;
7479 return Offset;
7480 }
7481
7482 /// Class that holds debugging information for a data mapping to be passed to
7483 /// the runtime library.
7484 class MappingExprInfo {
7485 /// The variable declaration used for the data mapping.
7486 const ValueDecl *MapDecl = nullptr;
7487 /// The original expression used in the map clause, or null if there is
7488 /// none.
7489 const Expr *MapExpr = nullptr;
7490
7491 public:
7492 MappingExprInfo(const ValueDecl *MapDecl, const Expr *MapExpr = nullptr)
7493 : MapDecl(MapDecl), MapExpr(MapExpr) {}
7494
7495 const ValueDecl *getMapDecl() const { return MapDecl; }
7496 const Expr *getMapExpr() const { return MapExpr; }
7497 };
7498
7499 using DeviceInfoTy = llvm::OpenMPIRBuilder::DeviceInfoTy;
7500 using MapBaseValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
7501 using MapValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
7502 using MapFlagsArrayTy = llvm::OpenMPIRBuilder::MapFlagsArrayTy;
7503 using MapDimArrayTy = llvm::OpenMPIRBuilder::MapDimArrayTy;
7504 using MapNonContiguousArrayTy =
7505 llvm::OpenMPIRBuilder::MapNonContiguousArrayTy;
7506 using MapExprsArrayTy = SmallVector<MappingExprInfo, 4>;
7507 using MapValueDeclsArrayTy = SmallVector<const ValueDecl *, 4>;
7508 using MapData =
7510 OpenMPMapClauseKind, ArrayRef<OpenMPMapModifierKind>,
7511 bool /*IsImplicit*/, const ValueDecl *, const Expr *>;
7512 using MapDataArrayTy = SmallVector<MapData, 4>;
7513
7514 /// This structure contains combined information generated for mappable
7515 /// clauses, including base pointers, pointers, sizes, map types, user-defined
7516 /// mappers, and non-contiguous information.
7517 struct MapCombinedInfoTy : llvm::OpenMPIRBuilder::MapInfosTy {
7518 MapExprsArrayTy Exprs;
7519 MapValueDeclsArrayTy Mappers;
7520 MapValueDeclsArrayTy DevicePtrDecls;
7521
7522 /// Append arrays in \a CurInfo.
7523 void append(MapCombinedInfoTy &CurInfo) {
7524 Exprs.append(CurInfo.Exprs.begin(), CurInfo.Exprs.end());
7525 DevicePtrDecls.append(CurInfo.DevicePtrDecls.begin(),
7526 CurInfo.DevicePtrDecls.end());
7527 Mappers.append(CurInfo.Mappers.begin(), CurInfo.Mappers.end());
7528 llvm::OpenMPIRBuilder::MapInfosTy::append(CurInfo);
7529 }
7530 };
7531
7532 /// Map between a struct and the its lowest & highest elements which have been
7533 /// mapped.
7534 /// [ValueDecl *] --> {LE(FieldIndex, Pointer),
7535 /// HE(FieldIndex, Pointer)}
7536 struct StructRangeInfoTy {
7537 MapCombinedInfoTy PreliminaryMapData;
7538 std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> LowestElem = {
7539 0, Address::invalid()};
7540 std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> HighestElem = {
7541 0, Address::invalid()};
7544 bool IsArraySection = false;
7545 bool HasCompleteRecord = false;
7546 };
7547
7548 /// A struct to store the attach pointer and pointee information, to be used
7549 /// when emitting an attach entry.
7550 struct AttachInfoTy {
7551 Address AttachPtrAddr = Address::invalid();
7552 Address AttachPteeAddr = Address::invalid();
7553 const ValueDecl *AttachPtrDecl = nullptr;
7554 const Expr *AttachMapExpr = nullptr;
7555
7556 bool isValid() const {
7557 return AttachPtrAddr.isValid() && AttachPteeAddr.isValid();
7558 }
7559 };
7560
7561 /// Check if there's any component list where the attach pointer expression
7562 /// matches the given captured variable.
7563 bool hasAttachEntryForCapturedVar(const ValueDecl *VD) const {
7564 for (const auto &AttachEntry : AttachPtrExprMap) {
7565 if (AttachEntry.second) {
7566 // Check if the attach pointer expression is a DeclRefExpr that
7567 // references the captured variable
7568 if (const auto *DRE = dyn_cast<DeclRefExpr>(AttachEntry.second))
7569 if (DRE->getDecl() == VD)
7570 return true;
7571 }
7572 }
7573 return false;
7574 }
7575
7576 /// Get the previously-cached attach pointer for a component list, if-any.
7577 const Expr *getAttachPtrExpr(
7579 const {
7580 const auto It = AttachPtrExprMap.find(Components);
7581 if (It != AttachPtrExprMap.end())
7582 return It->second;
7583
7584 return nullptr;
7585 }
7586
7587private:
7588 /// Kind that defines how a device pointer has to be returned.
7589 struct MapInfo {
7592 ArrayRef<OpenMPMapModifierKind> MapModifiers;
7593 ArrayRef<OpenMPMotionModifierKind> MotionModifiers;
7594 bool ReturnDevicePointer = false;
7595 bool IsImplicit = false;
7596 const ValueDecl *Mapper = nullptr;
7597 const Expr *VarRef = nullptr;
7598 bool ForDeviceAddr = false;
7599 bool HasUdpFbNullify = false;
7600
7601 MapInfo() = default;
7602 MapInfo(
7604 OpenMPMapClauseKind MapType,
7605 ArrayRef<OpenMPMapModifierKind> MapModifiers,
7606 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
7607 bool ReturnDevicePointer, bool IsImplicit,
7608 const ValueDecl *Mapper = nullptr, const Expr *VarRef = nullptr,
7609 bool ForDeviceAddr = false, bool HasUdpFbNullify = false)
7610 : Components(Components), MapType(MapType), MapModifiers(MapModifiers),
7611 MotionModifiers(MotionModifiers),
7612 ReturnDevicePointer(ReturnDevicePointer), IsImplicit(IsImplicit),
7613 Mapper(Mapper), VarRef(VarRef), ForDeviceAddr(ForDeviceAddr),
7614 HasUdpFbNullify(HasUdpFbNullify) {}
7615 };
7616
7617 /// The target directive from where the mappable clauses were extracted. It
7618 /// is either a executable directive or a user-defined mapper directive.
7619 llvm::PointerUnion<const OMPExecutableDirective *,
7620 const OMPDeclareMapperDecl *>
7621 CurDir;
7622
7623 /// Function the directive is being generated for.
7624 CodeGenFunction &CGF;
7625
7626 /// Set of all first private variables in the current directive.
7627 /// bool data is set to true if the variable is implicitly marked as
7628 /// firstprivate, false otherwise.
7629 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>, bool> FirstPrivateDecls;
7630
7631 /// Set of defaultmap clause kinds that use firstprivate behavior.
7632 llvm::SmallSet<OpenMPDefaultmapClauseKind, 4> DefaultmapFirstprivateKinds;
7633
7634 /// Map between device pointer declarations and their expression components.
7635 /// The key value for declarations in 'this' is null.
7636 llvm::DenseMap<
7637 const ValueDecl *,
7638 SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>>
7639 DevPointersMap;
7640
7641 /// Map between device addr declarations and their expression components.
7642 /// The key value for declarations in 'this' is null.
7643 llvm::DenseMap<
7644 const ValueDecl *,
7645 SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>>
7646 HasDevAddrsMap;
7647
7648 /// Map between lambda declarations and their map type.
7649 llvm::DenseMap<const ValueDecl *, const OMPMapClause *> LambdasMap;
7650
7651 /// Map from component lists to their attach pointer expressions.
7653 const Expr *>
7654 AttachPtrExprMap;
7655
7656 /// Map from attach pointer expressions to their component depth.
7657 /// nullptr key has std::nullopt depth. This can be used to order attach-ptr
7658 /// expressions with increasing/decreasing depth.
7659 /// The component-depth of `nullptr` (i.e. no attach-ptr) is `std::nullopt`.
7660 /// TODO: Not urgent, but we should ideally use the number of pointer
7661 /// dereferences in an expr as an indicator of its complexity, instead of the
7662 /// component-depth. That would be needed for us to treat `p[1]`, `*(p + 10)`,
7663 /// `*(p + 5 + 5)` together.
7664 llvm::DenseMap<const Expr *, std::optional<size_t>>
7665 AttachPtrComponentDepthMap = {{nullptr, std::nullopt}};
7666
7667 /// Map from attach pointer expressions to the order they were computed in, in
7668 /// collectAttachPtrExprInfo().
7669 llvm::DenseMap<const Expr *, size_t> AttachPtrComputationOrderMap = {
7670 {nullptr, 0}};
7671
7672 /// An instance of attach-ptr-expr comparator that can be used throughout the
7673 /// lifetime of this handler.
7674 AttachPtrExprComparator AttachPtrComparator;
7675
7676 llvm::Value *getExprTypeSize(const Expr *E) const {
7677 QualType ExprTy = E->getType().getCanonicalType();
7678
7679 // Calculate the size for array shaping expression.
7680 if (const auto *OAE = dyn_cast<OMPArrayShapingExpr>(E)) {
7681 llvm::Value *Size =
7682 CGF.getTypeSize(OAE->getBase()->getType()->getPointeeType());
7683 for (const Expr *SE : OAE->getDimensions()) {
7684 llvm::Value *Sz = CGF.EmitScalarExpr(SE);
7685 Sz = CGF.EmitScalarConversion(Sz, SE->getType(),
7686 CGF.getContext().getSizeType(),
7687 SE->getExprLoc());
7688 Size = CGF.Builder.CreateNUWMul(Size, Sz);
7689 }
7690 return Size;
7691 }
7692
7693 // Reference types are ignored for mapping purposes.
7694 if (const auto *RefTy = ExprTy->getAs<ReferenceType>())
7695 ExprTy = RefTy->getPointeeType().getCanonicalType();
7696
7697 // Given that an array section is considered a built-in type, we need to
7698 // do the calculation based on the length of the section instead of relying
7699 // on CGF.getTypeSize(E->getType()).
7700 if (const auto *OAE = dyn_cast<ArraySectionExpr>(E)) {
7701 QualType BaseTy = ArraySectionExpr::getBaseOriginalType(
7702 OAE->getBase()->IgnoreParenImpCasts())
7704
7705 // If there is no length associated with the expression and lower bound is
7706 // not specified too, that means we are using the whole length of the
7707 // base.
7708 if (!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
7709 !OAE->getLowerBound())
7710 return CGF.getTypeSize(BaseTy);
7711
7712 llvm::Value *ElemSize;
7713 if (const auto *PTy = BaseTy->getAs<PointerType>()) {
7714 ElemSize = CGF.getTypeSize(PTy->getPointeeType().getCanonicalType());
7715 } else {
7716 const auto *ATy = cast<ArrayType>(BaseTy.getTypePtr());
7717 assert(ATy && "Expecting array type if not a pointer type.");
7718 ElemSize = CGF.getTypeSize(ATy->getElementType().getCanonicalType());
7719 }
7720
7721 // If we don't have a length at this point, that is because we have an
7722 // array section with a single element.
7723 if (!OAE->getLength() && OAE->getColonLocFirst().isInvalid())
7724 return ElemSize;
7725
7726 if (const Expr *LenExpr = OAE->getLength()) {
7727 llvm::Value *LengthVal = CGF.EmitScalarExpr(LenExpr);
7728 LengthVal = CGF.EmitScalarConversion(LengthVal, LenExpr->getType(),
7729 CGF.getContext().getSizeType(),
7730 LenExpr->getExprLoc());
7731 return CGF.Builder.CreateNUWMul(LengthVal, ElemSize);
7732 }
7733 assert(!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
7734 OAE->getLowerBound() && "expected array_section[lb:].");
7735 // Size = sizetype - lb * elemtype;
7736 llvm::Value *LengthVal = CGF.getTypeSize(BaseTy);
7737 llvm::Value *LBVal = CGF.EmitScalarExpr(OAE->getLowerBound());
7738 LBVal = CGF.EmitScalarConversion(LBVal, OAE->getLowerBound()->getType(),
7739 CGF.getContext().getSizeType(),
7740 OAE->getLowerBound()->getExprLoc());
7741 LBVal = CGF.Builder.CreateNUWMul(LBVal, ElemSize);
7742 llvm::Value *Cmp = CGF.Builder.CreateICmpUGT(LengthVal, LBVal);
7743 llvm::Value *TrueVal = CGF.Builder.CreateNUWSub(LengthVal, LBVal);
7744 LengthVal = CGF.Builder.CreateSelect(
7745 Cmp, TrueVal, llvm::ConstantInt::get(CGF.SizeTy, 0));
7746 return LengthVal;
7747 }
7748 return CGF.getTypeSize(ExprTy);
7749 }
7750
7751 /// Return the corresponding bits for a given map clause modifier. Add
7752 /// a flag marking the map as a pointer if requested. Add a flag marking the
7753 /// map as the first one of a series of maps that relate to the same map
7754 /// expression.
7755 OpenMPOffloadMappingFlags getMapTypeBits(
7756 OpenMPMapClauseKind MapType, ArrayRef<OpenMPMapModifierKind> MapModifiers,
7757 ArrayRef<OpenMPMotionModifierKind> MotionModifiers, bool IsImplicit,
7758 bool AddPtrFlag, bool AddIsTargetParamFlag, bool IsNonContiguous) const {
7759 OpenMPOffloadMappingFlags Bits =
7760 IsImplicit ? OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT
7761 : OpenMPOffloadMappingFlags::OMP_MAP_NONE;
7762 switch (MapType) {
7763 case OMPC_MAP_alloc:
7764 case OMPC_MAP_release:
7765 // alloc and release is the default behavior in the runtime library, i.e.
7766 // if we don't pass any bits alloc/release that is what the runtime is
7767 // going to do. Therefore, we don't need to signal anything for these two
7768 // type modifiers.
7769 break;
7770 case OMPC_MAP_to:
7771 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO;
7772 break;
7773 case OMPC_MAP_from:
7774 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_FROM;
7775 break;
7776 case OMPC_MAP_tofrom:
7777 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO |
7778 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
7779 break;
7780 case OMPC_MAP_delete:
7781 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_DELETE;
7782 break;
7783 case OMPC_MAP_unknown:
7784 llvm_unreachable("Unexpected map type!");
7785 }
7786 if (AddPtrFlag)
7787 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
7788 if (AddIsTargetParamFlag)
7789 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
7790 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_always))
7791 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS;
7792 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_close))
7793 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_CLOSE;
7794 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_present) ||
7795 llvm::is_contained(MotionModifiers, OMPC_MOTION_MODIFIER_present))
7796 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
7797 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_ompx_hold))
7798 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
7799 if (IsNonContiguous)
7800 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG;
7801 return Bits;
7802 }
7803
7804 /// Return true if the provided expression is a final array section. A
7805 /// final array section, is one whose length can't be proved to be one.
7806 bool isFinalArraySectionExpression(const Expr *E) const {
7807 const auto *OASE = dyn_cast<ArraySectionExpr>(E);
7808
7809 // It is not an array section and therefore not a unity-size one.
7810 if (!OASE)
7811 return false;
7812
7813 // An array section with no colon always refer to a single element.
7814 if (OASE->getColonLocFirst().isInvalid())
7815 return false;
7816
7817 const Expr *Length = OASE->getLength();
7818
7819 // If we don't have a length we have to check if the array has size 1
7820 // for this dimension. Also, we should always expect a length if the
7821 // base type is pointer.
7822 if (!Length) {
7823 QualType BaseQTy = ArraySectionExpr::getBaseOriginalType(
7824 OASE->getBase()->IgnoreParenImpCasts())
7826 if (const auto *ATy = dyn_cast<ConstantArrayType>(BaseQTy.getTypePtr()))
7827 return ATy->getSExtSize() != 1;
7828 // If we don't have a constant dimension length, we have to consider
7829 // the current section as having any size, so it is not necessarily
7830 // unitary. If it happen to be unity size, that's user fault.
7831 return true;
7832 }
7833
7834 // Check if the length evaluates to 1.
7835 Expr::EvalResult Result;
7836 if (!Length->EvaluateAsInt(Result, CGF.getContext()))
7837 return true; // Can have more that size 1.
7838
7839 llvm::APSInt ConstLength = Result.Val.getInt();
7840 return ConstLength.getSExtValue() != 1;
7841 }
7842
7843 /// Emit an attach entry into \p CombinedInfo, using the information from \p
7844 /// AttachInfo. For example, for a map of form `int *p; ... map(p[1:10])`,
7845 /// an attach entry has the following form:
7846 /// &p, &p[1], sizeof(void*), ATTACH
7847 void emitAttachEntry(CodeGenFunction &CGF, MapCombinedInfoTy &CombinedInfo,
7848 const AttachInfoTy &AttachInfo) const {
7849 assert(AttachInfo.isValid() &&
7850 "Expected valid attach pointer/pointee information!");
7851
7852 // Size is the size of the pointer itself - use pointer size, not BaseDecl
7853 // size
7854 llvm::Value *PointerSize = CGF.Builder.CreateIntCast(
7855 llvm::ConstantInt::get(
7856 CGF.CGM.SizeTy, CGF.getContext()
7858 .getQuantity()),
7859 CGF.Int64Ty, /*isSigned=*/true);
7860
7861 CombinedInfo.Exprs.emplace_back(AttachInfo.AttachPtrDecl,
7862 AttachInfo.AttachMapExpr);
7863 CombinedInfo.BasePointers.push_back(
7864 AttachInfo.AttachPtrAddr.emitRawPointer(CGF));
7865 CombinedInfo.DevicePtrDecls.push_back(nullptr);
7866 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
7867 CombinedInfo.Pointers.push_back(
7868 AttachInfo.AttachPteeAddr.emitRawPointer(CGF));
7869 CombinedInfo.Sizes.push_back(PointerSize);
7870 CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_ATTACH);
7871 // ATTACH entries themselves don't "have" a base attach-ptr.
7872 CombinedInfo.HasAttachPtr.push_back(false);
7873 CombinedInfo.Mappers.push_back(nullptr);
7874 CombinedInfo.NonContigInfo.Dims.push_back(1);
7875 }
7876
7877 /// A helper class to copy structures with overlapped elements, i.e. those
7878 /// which have mappings of both "s" and "s.mem". Consecutive elements that
7879 /// are not explicitly copied have mapping nodes synthesized for them,
7880 /// taking care to avoid generating zero-sized copies.
7881 class CopyOverlappedEntryGaps {
7882 CodeGenFunction &CGF;
7883 MapCombinedInfoTy &CombinedInfo;
7884 OpenMPOffloadMappingFlags Flags = OpenMPOffloadMappingFlags::OMP_MAP_NONE;
7885 const ValueDecl *MapDecl = nullptr;
7886 const Expr *MapExpr = nullptr;
7888 bool IsNonContiguous = false;
7889 uint64_t DimSize = 0;
7890 // These elements track the position as the struct is iterated over
7891 // (in order of increasing element address).
7892 const RecordDecl *LastParent = nullptr;
7893 uint64_t Cursor = 0;
7894 unsigned LastIndex = -1u;
7896
7897 public:
7898 CopyOverlappedEntryGaps(CodeGenFunction &CGF,
7899 MapCombinedInfoTy &CombinedInfo,
7900 OpenMPOffloadMappingFlags Flags,
7901 const ValueDecl *MapDecl, const Expr *MapExpr,
7902 Address BP, Address LB, bool IsNonContiguous,
7903 uint64_t DimSize)
7904 : CGF(CGF), CombinedInfo(CombinedInfo), Flags(Flags), MapDecl(MapDecl),
7905 MapExpr(MapExpr), BP(BP), IsNonContiguous(IsNonContiguous),
7906 DimSize(DimSize), LB(LB) {}
7907
7908 void processField(
7909 const OMPClauseMappableExprCommon::MappableComponent &MC,
7910 const FieldDecl *FD,
7911 llvm::function_ref<LValue(CodeGenFunction &, const MemberExpr *)>
7912 EmitMemberExprBase) {
7913 const RecordDecl *RD = FD->getParent();
7914 const ASTRecordLayout &RL = CGF.getContext().getASTRecordLayout(RD);
7915 uint64_t FieldOffset = RL.getFieldOffset(FD->getFieldIndex());
7916 uint64_t FieldSize =
7918 Address ComponentLB = Address::invalid();
7919
7920 if (FD->getType()->isLValueReferenceType()) {
7921 const auto *ME = cast<MemberExpr>(MC.getAssociatedExpression());
7922 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
7923 ComponentLB =
7924 CGF.EmitLValueForFieldInitialization(BaseLVal, FD).getAddress();
7925 } else {
7926 ComponentLB =
7928 }
7929
7930 if (!LastParent)
7931 LastParent = RD;
7932 if (FD->getParent() == LastParent) {
7933 if (FD->getFieldIndex() != LastIndex + 1)
7934 copyUntilField(FD, ComponentLB);
7935 } else {
7936 LastParent = FD->getParent();
7937 if (((int64_t)FieldOffset - (int64_t)Cursor) > 0)
7938 copyUntilField(FD, ComponentLB);
7939 }
7940 Cursor = FieldOffset + FieldSize;
7941 LastIndex = FD->getFieldIndex();
7942 LB = CGF.Builder.CreateConstGEP(ComponentLB, 1);
7943 }
7944
7945 void copyUntilField(const FieldDecl *FD, Address ComponentLB) {
7946 llvm::Value *ComponentLBPtr = ComponentLB.emitRawPointer(CGF);
7947 llvm::Value *LBPtr = LB.emitRawPointer(CGF);
7948 llvm::Value *Size = CGF.Builder.CreatePtrDiff(ComponentLBPtr, LBPtr);
7949 copySizedChunk(LBPtr, Size);
7950 }
7951
7952 void copyUntilEnd(Address HB) {
7953 if (LastParent) {
7954 const ASTRecordLayout &RL =
7955 CGF.getContext().getASTRecordLayout(LastParent);
7956 if ((uint64_t)CGF.getContext().toBits(RL.getSize()) <= Cursor)
7957 return;
7958 }
7959 llvm::Value *LBPtr = LB.emitRawPointer(CGF);
7960 llvm::Value *Size = CGF.Builder.CreatePtrDiff(
7961 CGF.Builder.CreateConstGEP(HB, 1).emitRawPointer(CGF), LBPtr);
7962 copySizedChunk(LBPtr, Size);
7963 }
7964
7965 void copySizedChunk(llvm::Value *Base, llvm::Value *Size) {
7966 CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
7967 CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF));
7968 CombinedInfo.DevicePtrDecls.push_back(nullptr);
7969 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
7970 CombinedInfo.Pointers.push_back(Base);
7971 CombinedInfo.Sizes.push_back(
7972 CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/false));
7973 CombinedInfo.Types.push_back(Flags);
7974 CombinedInfo.HasAttachPtr.push_back(false);
7975 CombinedInfo.Mappers.push_back(nullptr);
7976 CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize : 1);
7977 }
7978 };
7979
7980 /// Generate the base pointers, section pointers, sizes, map type bits, and
7981 /// user-defined mappers (all included in \a CombinedInfo) for the provided
7982 /// map type, map or motion modifiers, and expression components.
7983 /// \a IsFirstComponent should be set to true if the provided set of
7984 /// components is the first associated with a capture.
7985 void generateInfoForComponentList(
7986 OpenMPMapClauseKind MapType, ArrayRef<OpenMPMapModifierKind> MapModifiers,
7987 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
7989 MapCombinedInfoTy &CombinedInfo,
7990 MapCombinedInfoTy &StructBaseCombinedInfo,
7991 StructRangeInfoTy &PartialStruct, AttachInfoTy &AttachInfo,
7992 bool IsFirstComponentList, bool IsImplicit,
7993 bool GenerateAllInfoForClauses, const ValueDecl *Mapper = nullptr,
7994 bool ForDeviceAddr = false, const ValueDecl *BaseDecl = nullptr,
7995 const Expr *MapExpr = nullptr,
7996 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
7997 OverlappedElements = {}) const {
7998
7999 // The following summarizes what has to be generated for each map and the
8000 // types below. The generated information is expressed in this order:
8001 // base pointer, section pointer, size, flags
8002 // (to add to the ones that come from the map type and modifier).
8003 // Entries annotated with (+) are only generated for "target" constructs,
8004 // and only if the variable at the beginning of the expression is used in
8005 // the region.
8006 //
8007 // double d;
8008 // int i[100];
8009 // float *p;
8010 // int **a = &i;
8011 //
8012 // struct S1 {
8013 // int i;
8014 // float f[50];
8015 // }
8016 // struct S2 {
8017 // int i;
8018 // float f[50];
8019 // S1 s;
8020 // double *p;
8021 // double *&pref;
8022 // struct S2 *ps;
8023 // int &ref;
8024 // }
8025 // S2 s;
8026 // S2 *ps;
8027 //
8028 // map(d)
8029 // &d, &d, sizeof(double), TARGET_PARAM | TO | FROM
8030 //
8031 // map(i)
8032 // &i, &i, 100*sizeof(int), TARGET_PARAM | TO | FROM
8033 //
8034 // map(i[1:23])
8035 // &i(=&i[0]), &i[1], 23*sizeof(int), TARGET_PARAM | TO | FROM
8036 //
8037 // map(p)
8038 // &p, &p, sizeof(float*), TARGET_PARAM | TO | FROM
8039 //
8040 // map(p[1:24])
8041 // p, &p[1], 24*sizeof(float), TARGET_PARAM | TO | FROM // map pointee
8042 // &p, &p[1], sizeof(void*), ATTACH // attach pointer/pointee, if both
8043 // // are present, and either is new
8044 //
8045 // map(([22])p)
8046 // p, p, 22*sizeof(float), TARGET_PARAM | TO | FROM
8047 // &p, p, sizeof(void*), ATTACH
8048 //
8049 // map((*a)[0:3])
8050 // a, a, 0, TARGET_PARAM | IMPLICIT // (+)
8051 // (*a)[0], &(*a)[0], 3 * sizeof(int), TO | FROM
8052 // &(*a), &(*a)[0], sizeof(void*), ATTACH
8053 // (+) Only on target, if a is used in the region
8054 // Note: Since the attach base-pointer is `*a`, which is not a scalar
8055 // variable, it doesn't determine the clause on `a`. `a` is mapped using
8056 // a zero-length-array-section map by generateDefaultMapInfo, if it is
8057 // referenced in the target region, because it is a pointer.
8058 //
8059 // map(**a)
8060 // a, a, 0, TARGET_PARAM | IMPLICIT // (+)
8061 // &(*a)[0], &(*a)[0], sizeof(int), TO | FROM
8062 // &(*a), &(*a)[0], sizeof(void*), ATTACH
8063 // (+) Only on target, if a is used in the region
8064 //
8065 // map(s)
8066 // FIXME: This needs to also imply map(ref_ptr_ptee: s.ref), since the
8067 // effect is supposed to be same as if the user had a map for every element
8068 // of the struct. We currently do a shallow-map of s.
8069 // &s, &s, sizeof(S2), TARGET_PARAM | TO | FROM
8070 //
8071 // map(s.i)
8072 // &s, &(s.i), sizeof(int), TARGET_PARAM | TO | FROM
8073 //
8074 // map(s.s.f)
8075 // &s, &(s.s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM
8076 //
8077 // map(s.p)
8078 // &s, &(s.p), sizeof(double*), TARGET_PARAM | TO | FROM
8079 //
8080 // map(to: s.p[:22])
8081 // &s, &(s.p), sizeof(double*), TARGET_PARAM | IMPLICIT // (+)
8082 // &(s.p[0]), &(s.p[0]), 22 * sizeof(double*), TO | FROM
8083 // &(s.p), &(s.p[0]), sizeof(void*), ATTACH
8084 //
8085 // map(to: s.ref)
8086 // &s, &(ptr(s.ref)), sizeof(int*), TARGET_PARAM (*)
8087 // &s, &(ptee(s.ref)), sizeof(int), MEMBER_OF(1) | PTR_AND_OBJ | TO (***)
8088 // (*) alloc space for struct members, only this is a target parameter.
8089 // (**) map the pointer (nothing to be mapped in this example) (the compiler
8090 // optimizes this entry out, same in the examples below)
8091 // (***) map the pointee (map: to)
8092 // Note: ptr(s.ref) represents the referring pointer of s.ref
8093 // ptee(s.ref) represents the referenced pointee of s.ref
8094 //
8095 // map(to: s.pref)
8096 // &s, &(ptr(s.pref)), sizeof(double**), TARGET_PARAM
8097 // &s, &(ptee(s.pref)), sizeof(double*), MEMBER_OF(1) | PTR_AND_OBJ | TO
8098 //
8099 // map(to: s.pref[:22])
8100 // &s, &(ptr(s.pref)), sizeof(double**), TARGET_PARAM | IMPLICIT // (+)
8101 // &s, &(ptee(s.pref)), sizeof(double*), MEMBER_OF(1) | PTR_AND_OBJ | TO |
8102 // FROM | IMPLICIT // (+)
8103 // &(ptee(s.pref)[0]), &(ptee(s.pref)[0]), 22 * sizeof(double), TO
8104 // &(ptee(s.pref)), &(ptee(s.pref)[0]), sizeof(void*), ATTACH
8105 //
8106 // map(s.ps)
8107 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM
8108 //
8109 // map(from: s.ps->s.i)
8110 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+)
8111 // &(s.ps[0]), &(s.ps->s.i), sizeof(int), FROM
8112 // &(s.ps), &(s.ps->s.i), sizeof(void*), ATTACH
8113 //
8114 // map(to: s.ps->ps)
8115 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+)
8116 // &(s.ps[0]), &(s.ps->ps), sizeof(S2*), TO
8117 // &(s.ps), &(s.ps->ps), sizeof(void*), ATTACH
8118 //
8119 // map(s.ps->ps->ps)
8120 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+)
8121 // &(s.ps->ps[0]), &(s.ps->ps->ps), sizeof(S2*), TO
8122 // &(s.ps->ps), &(s.ps->ps->ps), sizeof(void*), ATTACH
8123 //
8124 // map(to: s.ps->ps->s.f[:22])
8125 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+)
8126 // &(s.ps->ps[0]), &(s.ps->ps->s.f[0]), 22*sizeof(float), TO
8127 // &(s.ps->ps), &(s.ps->ps->s.f[0]), sizeof(void*), ATTACH
8128 //
8129 // map(ps)
8130 // &ps, &ps, sizeof(S2*), TARGET_PARAM | TO | FROM
8131 //
8132 // map(ps->i)
8133 // ps, &(ps->i), sizeof(int), TARGET_PARAM | TO | FROM
8134 // &ps, &(ps->i), sizeof(void*), ATTACH
8135 //
8136 // map(ps->s.f)
8137 // ps, &(ps->s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM
8138 // &ps, &(ps->s.f[0]), sizeof(ps), ATTACH
8139 //
8140 // map(from: ps->p)
8141 // ps, &(ps->p), sizeof(double*), TARGET_PARAM | FROM
8142 // &ps, &(ps->p), sizeof(ps), ATTACH
8143 //
8144 // map(to: ps->p[:22])
8145 // ps, &(ps[0]), 0, TARGET_PARAM | IMPLICIT // (+)
8146 // &(ps->p[0]), &(ps->p[0]), 22*sizeof(double), TO
8147 // &(ps->p), &(ps->p[0]), sizeof(void*), ATTACH
8148 //
8149 // map(ps->ps)
8150 // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM | TO | FROM
8151 // &ps, &(ps->ps), sizeof(ps), ATTACH
8152 //
8153 // map(from: ps->ps->s.i)
8154 // ps, &(ps[0]), 0, TARGET_PARAM | IMPLICIT // (+)
8155 // &(ps->ps[0]), &(ps->ps->s.i), sizeof(int), FROM
8156 // &(ps->ps), &(ps->ps->s.i), sizeof(void*), ATTACH
8157 //
8158 // map(from: ps->ps->ps)
8159 // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
8160 // &(ps->ps[0]), &(ps->ps->ps), sizeof(S2*), FROM
8161 // &(ps->ps), &(ps->ps->ps), sizeof(void*), ATTACH
8162 //
8163 // map(ps->ps->ps->ps)
8164 // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
8165 // &(ps->ps->ps[0]), &(ps->ps->ps->ps), sizeof(S2*), FROM
8166 // &(ps->ps->ps), &(ps->ps->ps->ps), sizeof(void*), ATTACH
8167 //
8168 // map(to: ps->ps->ps->s.f[:22])
8169 // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
8170 // &(ps->ps->ps[0]), &(ps->ps->ps->s.f[0]), 22*sizeof(float), TO
8171 // &(ps->ps->ps), &(ps->ps->ps->s.f[0]), sizeof(void*), ATTACH
8172 //
8173 // map(to: s.f[:22]) map(from: s.p[:33])
8174 // On target, and if s is used in the region:
8175 //
8176 // &s, &(s.f[0]), 50*sizeof(float) +
8177 // sizeof(struct S1) +
8178 // sizeof(double*) (**), TARGET_PARAM
8179 // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | TO
8180 // &s, &(s.p), sizeof(double*), MEMBER_OF(1) | TO |
8181 // FROM | IMPLICIT
8182 // &(s.p[0]), &(s.p[0]), 33*sizeof(double), FROM
8183 // &(s.p), &(s.p[0]), sizeof(void*), ATTACH
8184 // (**) allocate contiguous space needed to fit all mapped members even if
8185 // we allocate space for members not mapped (in this example,
8186 // s.f[22..49] and s.s are not mapped, yet we must allocate space for
8187 // them as well because they fall between &s.f[0] and &s.p)
8188 //
8189 // On other constructs, and, if s is not used in the region, on target:
8190 // &s, &(s.f[0]), 22*sizeof(float), TO
8191 // &(s.p[0]), &(s.p[0]), 33*sizeof(double), FROM
8192 // &(s.p), &(s.p[0]), sizeof(void*), ATTACH
8193 //
8194 // map(from: s.f[:22]) map(to: ps->p[:33])
8195 // &s, &(s.f[0]), 22*sizeof(float), TARGET_PARAM | FROM
8196 // &ps[0], &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
8197 // &(ps->p[0]), &(ps->p[0]), 33*sizeof(double), TO
8198 // &(ps->p), &(ps->p[0]), sizeof(void*), ATTACH
8199 //
8200 // map(from: s.f[:22], s.s) map(to: ps->p[:33])
8201 // &s, &(s.f[0]), 50*sizeof(float) +
8202 // sizeof(struct S1), TARGET_PARAM
8203 // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | FROM
8204 // &s, &(s.s), sizeof(struct S1), MEMBER_OF(1) | FROM
8205 // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
8206 // &(ps->p[0]), &(ps->p[0]), 33*sizeof(double), TO
8207 // &(ps->p), &(ps->p[0]), sizeof(void*), ATTACH
8208 //
8209 // map(p[:100], p)
8210 // &p, &p, sizeof(float*), TARGET_PARAM | TO | FROM
8211 // p, &p[0], 100*sizeof(float), TO | FROM
8212 // &p, &p[0], sizeof(float*), ATTACH
8213
8214 // Track if the map information being generated is the first for a capture.
8215 bool IsCaptureFirstInfo = IsFirstComponentList;
8216 // When the variable is on a declare target link or in a to clause with
8217 // unified memory, a reference is needed to hold the host/device address
8218 // of the variable.
8219 bool RequiresReference = false;
8220
8221 // Scan the components from the base to the complete expression.
8222 auto CI = Components.rbegin();
8223 auto CE = Components.rend();
8224 auto I = CI;
8225
8226 // Track if the map information being generated is the first for a list of
8227 // components.
8228 bool IsExpressionFirstInfo = true;
8229 bool FirstPointerInComplexData = false;
8231 Address FinalLowestElem = Address::invalid();
8232 const Expr *AssocExpr = I->getAssociatedExpression();
8233 const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr);
8234 const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr);
8235 const auto *OAShE = dyn_cast<OMPArrayShapingExpr>(AssocExpr);
8236
8237 // Get the pointer-attachment base-pointer for the given list, if any.
8238 const Expr *AttachPtrExpr = getAttachPtrExpr(Components);
8239 auto [AttachPtrAddr, AttachPteeBaseAddr] =
8240 getAttachPtrAddrAndPteeBaseAddr(AttachPtrExpr, CGF);
8241
8242 bool HasAttachPtr = AttachPtrExpr != nullptr;
8243 bool FirstComponentIsForAttachPtr = AssocExpr == AttachPtrExpr;
8244 bool SeenAttachPtr = FirstComponentIsForAttachPtr;
8245
8246 if (FirstComponentIsForAttachPtr) {
8247 // No need to process AttachPtr here. It will be processed at the end
8248 // after we have computed the pointee's address.
8249 ++I;
8250 } else if (isa<MemberExpr>(AssocExpr)) {
8251 // The base is the 'this' pointer. The content of the pointer is going
8252 // to be the base of the field being mapped.
8253 BP = CGF.LoadCXXThisAddress();
8254 } else if ((AE && isa<CXXThisExpr>(AE->getBase()->IgnoreParenImpCasts())) ||
8255 (OASE &&
8256 isa<CXXThisExpr>(OASE->getBase()->IgnoreParenImpCasts()))) {
8257 BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress();
8258 } else if (OAShE &&
8259 isa<CXXThisExpr>(OAShE->getBase()->IgnoreParenCasts())) {
8260 BP = Address(
8261 CGF.EmitScalarExpr(OAShE->getBase()),
8262 CGF.ConvertTypeForMem(OAShE->getBase()->getType()->getPointeeType()),
8263 CGF.getContext().getTypeAlignInChars(OAShE->getBase()->getType()));
8264 } else {
8265 // The base is the reference to the variable.
8266 // BP = &Var.
8267 BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress();
8268 if (const auto *VD =
8269 dyn_cast_or_null<VarDecl>(I->getAssociatedDeclaration())) {
8270 if (std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
8271 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) {
8272 if ((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
8273 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
8274 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
8276 RequiresReference = true;
8278 }
8279 }
8280 }
8281
8282 // If the variable is a pointer and is being dereferenced (i.e. is not
8283 // the last component), the base has to be the pointer itself, not its
8284 // reference. References are ignored for mapping purposes.
8285 QualType Ty =
8286 I->getAssociatedDeclaration()->getType().getNonReferenceType();
8287 if (Ty->isAnyPointerType() && std::next(I) != CE) {
8288 // No need to generate individual map information for the pointer, it
8289 // can be associated with the combined storage if shared memory mode is
8290 // active or the base declaration is not global variable.
8291 const auto *VD = dyn_cast<VarDecl>(I->getAssociatedDeclaration());
8293 !VD || VD->hasLocalStorage() || HasAttachPtr)
8294 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());
8295 else
8296 FirstPointerInComplexData = true;
8297 ++I;
8298 }
8299 }
8300
8301 // Track whether a component of the list should be marked as MEMBER_OF some
8302 // combined entry (for partial structs). Only the first PTR_AND_OBJ entry
8303 // in a component list should be marked as MEMBER_OF, all subsequent entries
8304 // do not belong to the base struct. E.g.
8305 // struct S2 s;
8306 // s.ps->ps->ps->f[:]
8307 // (1) (2) (3) (4)
8308 // ps(1) is a member pointer, ps(2) is a pointee of ps(1), so it is a
8309 // PTR_AND_OBJ entry; the PTR is ps(1), so MEMBER_OF the base struct. ps(3)
8310 // is the pointee of ps(2) which is not member of struct s, so it should not
8311 // be marked as such (it is still PTR_AND_OBJ).
8312 // The variable is initialized to false so that PTR_AND_OBJ entries which
8313 // are not struct members are not considered (e.g. array of pointers to
8314 // data).
8315 bool ShouldBeMemberOf = false;
8316
8317 // Variable keeping track of whether or not we have encountered a component
8318 // in the component list which is a member expression. Useful when we have a
8319 // pointer or a final array section, in which case it is the previous
8320 // component in the list which tells us whether we have a member expression.
8321 // E.g. X.f[:]
8322 // While processing the final array section "[:]" it is "f" which tells us
8323 // whether we are dealing with a member of a declared struct.
8324 const MemberExpr *EncounteredME = nullptr;
8325
8326 // Track for the total number of dimension. Start from one for the dummy
8327 // dimension.
8328 uint64_t DimSize = 1;
8329
8330 // Detects non-contiguous updates due to strided accesses.
8331 // Sets the 'IsNonContiguous' flag so that the 'MapType' bits are set
8332 // correctly when generating information to be passed to the runtime. The
8333 // flag is set to true if any array section has a stride not equal to 1, or
8334 // if the stride is not a constant expression (conservatively assumed
8335 // non-contiguous).
8336 bool IsNonContiguous =
8337 CombinedInfo.NonContigInfo.IsNonContiguous ||
8338 any_of(Components, [&](const auto &Component) {
8339 const auto *OASE =
8340 dyn_cast<ArraySectionExpr>(Component.getAssociatedExpression());
8341 if (!OASE)
8342 return false;
8343
8344 const Expr *StrideExpr = OASE->getStride();
8345 if (!StrideExpr)
8346 return false;
8347
8348 assert(StrideExpr->getType()->isIntegerType() &&
8349 "Stride expression must be of integer type");
8350
8351 // If stride is not evaluatable as a constant, treat as
8352 // non-contiguous.
8353 const auto Constant =
8354 StrideExpr->getIntegerConstantExpr(CGF.getContext());
8355 if (!Constant)
8356 return true;
8357
8358 // Treat non-unitary strides as non-contiguous.
8359 return !Constant->isOne();
8360 });
8361
8362 bool IsPrevMemberReference = false;
8363
8364 bool IsPartialMapped =
8365 !PartialStruct.PreliminaryMapData.BasePointers.empty();
8366
8367 // We need to check if we will be encountering any MEs. If we do not
8368 // encounter any ME expression it means we will be mapping the whole struct.
8369 // In that case we need to skip adding an entry for the struct to the
8370 // CombinedInfo list and instead add an entry to the StructBaseCombinedInfo
8371 // list only when generating all info for clauses.
8372 bool IsMappingWholeStruct = true;
8373 if (!GenerateAllInfoForClauses) {
8374 IsMappingWholeStruct = false;
8375 } else {
8376 for (auto TempI = I; TempI != CE; ++TempI) {
8377 const MemberExpr *PossibleME =
8378 dyn_cast<MemberExpr>(TempI->getAssociatedExpression());
8379 if (PossibleME) {
8380 IsMappingWholeStruct = false;
8381 break;
8382 }
8383 }
8384 }
8385
8386 bool SeenFirstNonBinOpExprAfterAttachPtr = false;
8387 for (; I != CE; ++I) {
8388 // If we have a valid attach-ptr, we skip processing all components until
8389 // after the attach-ptr.
8390 if (HasAttachPtr && !SeenAttachPtr) {
8391 SeenAttachPtr = I->getAssociatedExpression() == AttachPtrExpr;
8392 continue;
8393 }
8394
8395 // After finding the attach pointer, skip binary-ops, to skip past
8396 // expressions like (p + 10), for a map like map(*(p + 10)), where p is
8397 // the attach-ptr.
8398 if (HasAttachPtr && !SeenFirstNonBinOpExprAfterAttachPtr) {
8399 const auto *BO = dyn_cast<BinaryOperator>(I->getAssociatedExpression());
8400 if (BO)
8401 continue;
8402
8403 // Found the first non-binary-operator component after attach
8404 SeenFirstNonBinOpExprAfterAttachPtr = true;
8405 BP = AttachPteeBaseAddr;
8406 }
8407
8408 // If the current component is member of a struct (parent struct) mark it.
8409 if (!EncounteredME) {
8410 EncounteredME = dyn_cast<MemberExpr>(I->getAssociatedExpression());
8411 // If we encounter a PTR_AND_OBJ entry from now on it should be marked
8412 // as MEMBER_OF the parent struct.
8413 if (EncounteredME) {
8414 ShouldBeMemberOf = true;
8415 // Do not emit as complex pointer if this is actually not array-like
8416 // expression.
8417 if (FirstPointerInComplexData) {
8418 QualType Ty = std::prev(I)
8419 ->getAssociatedDeclaration()
8420 ->getType()
8421 .getNonReferenceType();
8422 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());
8423 FirstPointerInComplexData = false;
8424 }
8425 }
8426 }
8427
8428 auto Next = std::next(I);
8429
8430 // We need to generate the addresses and sizes if this is the last
8431 // component, if the component is a pointer or if it is an array section
8432 // whose length can't be proved to be one. If this is a pointer, it
8433 // becomes the base address for the following components.
8434
8435 // A final array section, is one whose length can't be proved to be one.
8436 // If the map item is non-contiguous then we don't treat any array section
8437 // as final array section.
8438 bool IsFinalArraySection =
8439 !IsNonContiguous &&
8440 isFinalArraySectionExpression(I->getAssociatedExpression());
8441
8442 // If we have a declaration for the mapping use that, otherwise use
8443 // the base declaration of the map clause.
8444 const ValueDecl *MapDecl = (I->getAssociatedDeclaration())
8445 ? I->getAssociatedDeclaration()
8446 : BaseDecl;
8447 MapExpr = (I->getAssociatedExpression()) ? I->getAssociatedExpression()
8448 : MapExpr;
8449
8450 // Get information on whether the element is a pointer. Have to do a
8451 // special treatment for array sections given that they are built-in
8452 // types.
8453 const auto *OASE =
8454 dyn_cast<ArraySectionExpr>(I->getAssociatedExpression());
8455 const auto *OAShE =
8456 dyn_cast<OMPArrayShapingExpr>(I->getAssociatedExpression());
8457 const auto *UO = dyn_cast<UnaryOperator>(I->getAssociatedExpression());
8458 const auto *BO = dyn_cast<BinaryOperator>(I->getAssociatedExpression());
8459 bool IsPointer =
8460 OAShE ||
8463 ->isAnyPointerType()) ||
8464 I->getAssociatedExpression()->getType()->isAnyPointerType();
8465 bool IsMemberReference = isa<MemberExpr>(I->getAssociatedExpression()) &&
8466 MapDecl &&
8467 MapDecl->getType()->isLValueReferenceType();
8468 bool IsNonDerefPointer = IsPointer &&
8469 !(UO && UO->getOpcode() != UO_Deref) && !BO &&
8470 !IsNonContiguous;
8471
8472 if (OASE)
8473 ++DimSize;
8474
8475 if (Next == CE || IsMemberReference || IsNonDerefPointer ||
8476 IsFinalArraySection) {
8477 // If this is not the last component, we expect the pointer to be
8478 // associated with an array expression or member expression.
8479 assert((Next == CE ||
8480 isa<MemberExpr>(Next->getAssociatedExpression()) ||
8481 isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) ||
8482 isa<ArraySectionExpr>(Next->getAssociatedExpression()) ||
8483 isa<OMPArrayShapingExpr>(Next->getAssociatedExpression()) ||
8484 isa<UnaryOperator>(Next->getAssociatedExpression()) ||
8485 isa<BinaryOperator>(Next->getAssociatedExpression())) &&
8486 "Unexpected expression");
8487
8489 Address LowestElem = Address::invalid();
8490 auto &&EmitMemberExprBase = [](CodeGenFunction &CGF,
8491 const MemberExpr *E) {
8492 const Expr *BaseExpr = E->getBase();
8493 // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a
8494 // scalar.
8495 LValue BaseLV;
8496 if (E->isArrow()) {
8497 LValueBaseInfo BaseInfo;
8498 TBAAAccessInfo TBAAInfo;
8499 Address Addr =
8500 CGF.EmitPointerWithAlignment(BaseExpr, &BaseInfo, &TBAAInfo);
8501 QualType PtrTy = BaseExpr->getType()->getPointeeType();
8502 BaseLV = CGF.MakeAddrLValue(Addr, PtrTy, BaseInfo, TBAAInfo);
8503 } else {
8504 BaseLV = CGF.EmitOMPSharedLValue(BaseExpr);
8505 }
8506 return BaseLV;
8507 };
8508 if (OAShE) {
8509 LowestElem = LB =
8510 Address(CGF.EmitScalarExpr(OAShE->getBase()),
8512 OAShE->getBase()->getType()->getPointeeType()),
8514 OAShE->getBase()->getType()));
8515 } else if (IsMemberReference) {
8516 const auto *ME = cast<MemberExpr>(I->getAssociatedExpression());
8517 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
8518 LowestElem = CGF.EmitLValueForFieldInitialization(
8519 BaseLVal, cast<FieldDecl>(MapDecl))
8520 .getAddress();
8521 LB = CGF.EmitLoadOfReferenceLValue(LowestElem, MapDecl->getType())
8522 .getAddress();
8523 } else {
8524 LowestElem = LB =
8525 CGF.EmitOMPSharedLValue(I->getAssociatedExpression())
8526 .getAddress();
8527 }
8528
8529 // Save the final LowestElem, to use it as the pointee in attach maps,
8530 // if emitted.
8531 if (Next == CE)
8532 FinalLowestElem = LowestElem;
8533
8534 // If this component is a pointer inside the base struct then we don't
8535 // need to create any entry for it - it will be combined with the object
8536 // it is pointing to into a single PTR_AND_OBJ entry.
8537 bool IsMemberPointerOrAddr =
8538 EncounteredME &&
8539 (((IsPointer || ForDeviceAddr) &&
8540 I->getAssociatedExpression() == EncounteredME) ||
8541 (IsPrevMemberReference && !IsPointer) ||
8542 (IsMemberReference && Next != CE &&
8543 !Next->getAssociatedExpression()->getType()->isPointerType()));
8544 if (!OverlappedElements.empty() && Next == CE) {
8545 // Handle base element with the info for overlapped elements.
8546 assert(!PartialStruct.Base.isValid() && "The base element is set.");
8547 assert(!IsPointer &&
8548 "Unexpected base element with the pointer type.");
8549 // Mark the whole struct as the struct that requires allocation on the
8550 // device.
8551 PartialStruct.LowestElem = {0, LowestElem};
8552 CharUnits TypeSize = CGF.getContext().getTypeSizeInChars(
8553 I->getAssociatedExpression()->getType());
8556 LowestElem, CGF.VoidPtrTy, CGF.Int8Ty),
8557 TypeSize.getQuantity() - 1);
8558 PartialStruct.HighestElem = {
8559 std::numeric_limits<decltype(
8560 PartialStruct.HighestElem.first)>::max(),
8561 HB};
8562 PartialStruct.Base = BP;
8563 PartialStruct.LB = LB;
8564 assert(
8565 PartialStruct.PreliminaryMapData.BasePointers.empty() &&
8566 "Overlapped elements must be used only once for the variable.");
8567 std::swap(PartialStruct.PreliminaryMapData, CombinedInfo);
8568 // Emit data for non-overlapped data.
8569 OpenMPOffloadMappingFlags Flags =
8570 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
8571 getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,
8572 /*AddPtrFlag=*/false,
8573 /*AddIsTargetParamFlag=*/false, IsNonContiguous);
8574 CopyOverlappedEntryGaps CopyGaps(CGF, CombinedInfo, Flags, MapDecl,
8575 MapExpr, BP, LB, IsNonContiguous,
8576 DimSize);
8577 // Do bitcopy of all non-overlapped structure elements.
8579 Component : OverlappedElements) {
8580 for (const OMPClauseMappableExprCommon::MappableComponent &MC :
8581 Component) {
8582 if (const ValueDecl *VD = MC.getAssociatedDeclaration()) {
8583 if (const auto *FD = dyn_cast<FieldDecl>(VD)) {
8584 CopyGaps.processField(MC, FD, EmitMemberExprBase);
8585 }
8586 }
8587 }
8588 }
8589 CopyGaps.copyUntilEnd(HB);
8590 break;
8591 }
8592 llvm::Value *Size = getExprTypeSize(I->getAssociatedExpression());
8593 // Skip adding an entry in the CurInfo of this combined entry if the
8594 // whole struct is currently being mapped. The struct needs to be added
8595 // in the first position before any data internal to the struct is being
8596 // mapped.
8597 // Skip adding an entry in the CurInfo of this combined entry if the
8598 // PartialStruct.PreliminaryMapData.BasePointers has been mapped.
8599 if ((!IsMemberPointerOrAddr && !IsPartialMapped) ||
8600 (Next == CE && MapType != OMPC_MAP_unknown)) {
8601 if (!IsMappingWholeStruct) {
8602 CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
8603 CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF));
8604 CombinedInfo.DevicePtrDecls.push_back(nullptr);
8605 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8606 CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF));
8607 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
8608 Size, CGF.Int64Ty, /*isSigned=*/true));
8609 CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize
8610 : 1);
8611 } else {
8612 StructBaseCombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
8613 StructBaseCombinedInfo.BasePointers.push_back(
8614 BP.emitRawPointer(CGF));
8615 StructBaseCombinedInfo.DevicePtrDecls.push_back(nullptr);
8616 StructBaseCombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
8617 StructBaseCombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF));
8618 StructBaseCombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
8619 Size, CGF.Int64Ty, /*isSigned=*/true));
8620 StructBaseCombinedInfo.NonContigInfo.Dims.push_back(
8621 IsNonContiguous ? DimSize : 1);
8622 }
8623
8624 // If Mapper is valid, the last component inherits the mapper.
8625 bool HasMapper = Mapper && Next == CE;
8626 if (!IsMappingWholeStruct)
8627 CombinedInfo.Mappers.push_back(HasMapper ? Mapper : nullptr);
8628 else
8629 StructBaseCombinedInfo.Mappers.push_back(HasMapper ? Mapper
8630 : nullptr);
8631
8632 // We need to add a pointer flag for each map that comes from the
8633 // same expression except for the first one. We also need to signal
8634 // this map is the first one that relates with the current capture
8635 // (there is a set of entries for each capture).
8636 OpenMPOffloadMappingFlags Flags = getMapTypeBits(
8637 MapType, MapModifiers, MotionModifiers, IsImplicit,
8638 !IsExpressionFirstInfo || RequiresReference ||
8639 FirstPointerInComplexData || IsMemberReference,
8640 IsCaptureFirstInfo && !RequiresReference, IsNonContiguous);
8641
8642 if (!IsExpressionFirstInfo || IsMemberReference) {
8643 // If we have a PTR_AND_OBJ pair where the OBJ is a pointer as well,
8644 // then we reset the TO/FROM/ALWAYS/DELETE/CLOSE flags.
8645 if (IsPointer || (IsMemberReference && Next != CE))
8646 Flags &= ~(OpenMPOffloadMappingFlags::OMP_MAP_TO |
8647 OpenMPOffloadMappingFlags::OMP_MAP_FROM |
8648 OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS |
8649 OpenMPOffloadMappingFlags::OMP_MAP_DELETE |
8650 OpenMPOffloadMappingFlags::OMP_MAP_CLOSE);
8651
8652 if (ShouldBeMemberOf) {
8653 // Set placeholder value MEMBER_OF=FFFF to indicate that the flag
8654 // should be later updated with the correct value of MEMBER_OF.
8655 Flags |= OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;
8656 // From now on, all subsequent PTR_AND_OBJ entries should not be
8657 // marked as MEMBER_OF.
8658 ShouldBeMemberOf = false;
8659 }
8660 }
8661
8662 if (!IsMappingWholeStruct) {
8663 CombinedInfo.Types.push_back(Flags);
8664 // HasAttachPtr marks pointee entries, which have a base attach-ptr.
8665 CombinedInfo.HasAttachPtr.push_back(HasAttachPtr);
8666 } else {
8667 StructBaseCombinedInfo.Types.push_back(Flags);
8668 StructBaseCombinedInfo.HasAttachPtr.push_back(HasAttachPtr);
8669 }
8670 }
8671
8672 // If we have encountered a member expression so far, keep track of the
8673 // mapped member. If the parent is "*this", then the value declaration
8674 // is nullptr.
8675 if (EncounteredME) {
8676 const auto *FD = cast<FieldDecl>(EncounteredME->getMemberDecl());
8677 unsigned FieldIndex = FD->getFieldIndex();
8678
8679 // Update info about the lowest and highest elements for this struct
8680 if (!PartialStruct.Base.isValid()) {
8681 PartialStruct.LowestElem = {FieldIndex, LowestElem};
8682 if (IsFinalArraySection && OASE) {
8683 Address HB =
8684 CGF.EmitArraySectionExpr(OASE, /*IsLowerBound=*/false)
8685 .getAddress();
8686 PartialStruct.HighestElem = {FieldIndex, HB};
8687 } else {
8688 PartialStruct.HighestElem = {FieldIndex, LowestElem};
8689 }
8690 PartialStruct.Base = BP;
8691 PartialStruct.LB = BP;
8692 } else if (FieldIndex < PartialStruct.LowestElem.first) {
8693 PartialStruct.LowestElem = {FieldIndex, LowestElem};
8694 } else if (FieldIndex > PartialStruct.HighestElem.first) {
8695 if (IsFinalArraySection && OASE) {
8696 Address HB =
8697 CGF.EmitArraySectionExpr(OASE, /*IsLowerBound=*/false)
8698 .getAddress();
8699 PartialStruct.HighestElem = {FieldIndex, HB};
8700 } else {
8701 PartialStruct.HighestElem = {FieldIndex, LowestElem};
8702 }
8703 }
8704 }
8705
8706 // Need to emit combined struct for array sections.
8707 if (IsFinalArraySection || IsNonContiguous)
8708 PartialStruct.IsArraySection = true;
8709
8710 // If we have a final array section, we are done with this expression.
8711 if (IsFinalArraySection)
8712 break;
8713
8714 // The pointer becomes the base for the next element.
8715 if (Next != CE)
8716 BP = IsMemberReference ? LowestElem : LB;
8717 if (!IsPartialMapped)
8718 IsExpressionFirstInfo = false;
8719 IsCaptureFirstInfo = false;
8720 FirstPointerInComplexData = false;
8721 IsPrevMemberReference = IsMemberReference;
8722 } else if (FirstPointerInComplexData) {
8723 QualType Ty = Components.rbegin()
8724 ->getAssociatedDeclaration()
8725 ->getType()
8726 .getNonReferenceType();
8727 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());
8728 FirstPointerInComplexData = false;
8729 }
8730 }
8731 // If ran into the whole component - allocate the space for the whole
8732 // record.
8733 if (!EncounteredME)
8734 PartialStruct.HasCompleteRecord = true;
8735
8736 // Populate ATTACH information for later processing by emitAttachEntry.
8737 if (shouldEmitAttachEntry(AttachPtrExpr, BaseDecl, CGF, CurDir)) {
8738 AttachInfo.AttachPtrAddr = AttachPtrAddr;
8739 AttachInfo.AttachPteeAddr = FinalLowestElem;
8740 AttachInfo.AttachPtrDecl = BaseDecl;
8741 AttachInfo.AttachMapExpr = MapExpr;
8742 }
8743
8744 if (!IsNonContiguous)
8745 return;
8746
8747 const ASTContext &Context = CGF.getContext();
8748
8749 // For supporting stride in array section, we need to initialize the first
8750 // dimension size as 1, first offset as 0, and first count as 1
8751 MapValuesArrayTy CurOffsets = {llvm::ConstantInt::get(CGF.CGM.Int64Ty, 0)};
8752 MapValuesArrayTy CurCounts;
8753 MapValuesArrayTy CurStrides = {llvm::ConstantInt::get(CGF.CGM.Int64Ty, 1)};
8754 MapValuesArrayTy DimSizes{llvm::ConstantInt::get(CGF.CGM.Int64Ty, 1)};
8755 uint64_t ElementTypeSize;
8756
8757 // Collect Size information for each dimension and get the element size as
8758 // the first Stride. For example, for `int arr[10][10]`, the DimSizes
8759 // should be [10, 10] and the first stride is 4 btyes.
8760 for (const OMPClauseMappableExprCommon::MappableComponent &Component :
8761 Components) {
8762 const Expr *AssocExpr = Component.getAssociatedExpression();
8763 const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr);
8764
8765 if (!OASE)
8766 continue;
8767
8768 QualType Ty = ArraySectionExpr::getBaseOriginalType(OASE->getBase());
8769 auto *CAT = Context.getAsConstantArrayType(Ty);
8770 auto *VAT = Context.getAsVariableArrayType(Ty);
8771
8772 // We need all the dimension size except for the last dimension.
8773 assert((VAT || CAT || &Component == &*Components.begin()) &&
8774 "Should be either ConstantArray or VariableArray if not the "
8775 "first Component");
8776
8777 // Get element size if CurCounts is empty.
8778 if (CurCounts.empty()) {
8779 const Type *ElementType = nullptr;
8780 if (CAT)
8781 ElementType = CAT->getElementType().getTypePtr();
8782 else if (VAT)
8783 ElementType = VAT->getElementType().getTypePtr();
8784 else if (&Component == &*Components.begin()) {
8785 // If the base is a raw pointer (e.g. T *data with data[a:b:c]),
8786 // there was no earlier CAT/VAT/array handling to establish
8787 // ElementType. Capture the pointee type now so that subsequent
8788 // components (offset/length/stride) have a concrete element type to
8789 // work with. This makes pointer-backed sections behave consistently
8790 // with CAT/VAT/array bases.
8791 if (const auto *PtrType = Ty->getAs<PointerType>())
8792 ElementType = PtrType->getPointeeType().getTypePtr();
8793 } else {
8794 // Any component after the first should never have a raw pointer type;
8795 // by this point. ElementType must already be known (set above or in
8796 // prior array / CAT / VAT handling).
8797 assert(!Ty->isPointerType() &&
8798 "Non-first components should not be raw pointers");
8799 }
8800
8801 // At this stage, if ElementType was a base pointer and we are in the
8802 // first iteration, it has been computed.
8803 if (ElementType) {
8804 // For the case that having pointer as base, we need to remove one
8805 // level of indirection.
8806 if (&Component != &*Components.begin())
8807 ElementType = ElementType->getPointeeOrArrayElementType();
8808 ElementTypeSize =
8809 Context.getTypeSizeInChars(ElementType).getQuantity();
8810 CurCounts.push_back(
8811 llvm::ConstantInt::get(CGF.Int64Ty, ElementTypeSize));
8812 }
8813 }
8814 // Get dimension value except for the last dimension since we don't need
8815 // it.
8816 if (DimSizes.size() < Components.size() - 1) {
8817 if (CAT)
8818 DimSizes.push_back(
8819 llvm::ConstantInt::get(CGF.Int64Ty, CAT->getZExtSize()));
8820 else if (VAT)
8821 DimSizes.push_back(CGF.Builder.CreateIntCast(
8822 CGF.EmitScalarExpr(VAT->getSizeExpr()), CGF.Int64Ty,
8823 /*IsSigned=*/false));
8824 }
8825 }
8826
8827 // Skip the dummy dimension since we have already have its information.
8828 auto *DI = DimSizes.begin() + 1;
8829 // Product of dimension.
8830 llvm::Value *DimProd =
8831 llvm::ConstantInt::get(CGF.CGM.Int64Ty, ElementTypeSize);
8832
8833 // Collect info for non-contiguous. Notice that offset, count, and stride
8834 // are only meaningful for array-section, so we insert a null for anything
8835 // other than array-section.
8836 // Also, the size of offset, count, and stride are not the same as
8837 // pointers, base_pointers, sizes, or dims. Instead, the size of offset,
8838 // count, and stride are the same as the number of non-contiguous
8839 // declaration in target update to/from clause.
8840 for (const OMPClauseMappableExprCommon::MappableComponent &Component :
8841 Components) {
8842 const Expr *AssocExpr = Component.getAssociatedExpression();
8843
8844 if (const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr)) {
8845 llvm::Value *Offset = CGF.Builder.CreateIntCast(
8846 CGF.EmitScalarExpr(AE->getIdx()), CGF.Int64Ty,
8847 /*isSigned=*/false);
8848 CurOffsets.push_back(Offset);
8849 CurCounts.push_back(llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/1));
8850 CurStrides.push_back(CurStrides.back());
8851 continue;
8852 }
8853
8854 const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr);
8855
8856 if (!OASE)
8857 continue;
8858
8859 // Offset
8860 const Expr *OffsetExpr = OASE->getLowerBound();
8861 llvm::Value *Offset = nullptr;
8862 if (!OffsetExpr) {
8863 // If offset is absent, then we just set it to zero.
8864 Offset = llvm::ConstantInt::get(CGF.Int64Ty, 0);
8865 } else {
8866 Offset = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(OffsetExpr),
8867 CGF.Int64Ty,
8868 /*isSigned=*/false);
8869 }
8870
8871 // Count
8872 const Expr *CountExpr = OASE->getLength();
8873 llvm::Value *Count = nullptr;
8874 if (!CountExpr) {
8875 // In Clang, once a high dimension is an array section, we construct all
8876 // the lower dimension as array section, however, for case like
8877 // arr[0:2][2], Clang construct the inner dimension as an array section
8878 // but it actually is not in an array section form according to spec.
8879 if (!OASE->getColonLocFirst().isValid() &&
8880 !OASE->getColonLocSecond().isValid()) {
8881 Count = llvm::ConstantInt::get(CGF.Int64Ty, 1);
8882 } else {
8883 // OpenMP 5.0, 2.1.5 Array Sections, Description.
8884 // When the length is absent it defaults to ⌈(size −
8885 // lower-bound)/stride⌉, where size is the size of the array
8886 // dimension.
8887 const Expr *StrideExpr = OASE->getStride();
8888 llvm::Value *Stride =
8889 StrideExpr
8890 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(StrideExpr),
8891 CGF.Int64Ty, /*isSigned=*/false)
8892 : nullptr;
8893 if (Stride)
8894 Count = CGF.Builder.CreateUDiv(
8895 CGF.Builder.CreateNUWSub(*DI, Offset), Stride);
8896 else
8897 Count = CGF.Builder.CreateNUWSub(*DI, Offset);
8898 }
8899 } else {
8900 Count = CGF.EmitScalarExpr(CountExpr);
8901 }
8902 Count = CGF.Builder.CreateIntCast(Count, CGF.Int64Ty, /*isSigned=*/false);
8903 CurCounts.push_back(Count);
8904
8905 // Stride_n' = Stride_n * (D_0 * D_1 ... * D_n-1) * Unit size
8906 // Offset_n' = Offset_n * (D_0 * D_1 ... * D_n-1) * Unit size
8907 // Take `int arr[5][5][5]` and `arr[0:2:2][1:2:1][0:2:2]` as an example:
8908 // Offset Count Stride
8909 // D0 0 4 1 (int) <- dummy dimension
8910 // D1 0 2 8 (2 * (1) * 4)
8911 // D2 100 2 20 (1 * (1 * 5) * 4)
8912 // D3 0 2 200 (2 * (1 * 5 * 4) * 4)
8913 const Expr *StrideExpr = OASE->getStride();
8914 llvm::Value *Stride =
8915 StrideExpr
8916 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(StrideExpr),
8917 CGF.Int64Ty, /*isSigned=*/false)
8918 : nullptr;
8919 DimProd = CGF.Builder.CreateNUWMul(DimProd, *(DI - 1));
8920 if (Stride)
8921 CurStrides.push_back(CGF.Builder.CreateNUWMul(DimProd, Stride));
8922 else
8923 CurStrides.push_back(DimProd);
8924
8925 Offset = CGF.Builder.CreateNUWMul(DimProd, Offset);
8926 CurOffsets.push_back(Offset);
8927
8928 if (DI != DimSizes.end())
8929 ++DI;
8930 }
8931
8932 CombinedInfo.NonContigInfo.Offsets.push_back(CurOffsets);
8933 CombinedInfo.NonContigInfo.Counts.push_back(CurCounts);
8934 CombinedInfo.NonContigInfo.Strides.push_back(CurStrides);
8935 }
8936
8937 /// Return the adjusted map modifiers if the declaration a capture refers to
8938 /// appears in a first-private clause. This is expected to be used only with
8939 /// directives that start with 'target'.
8940 OpenMPOffloadMappingFlags
8941 getMapModifiersForPrivateClauses(const CapturedStmt::Capture &Cap) const {
8942 assert(Cap.capturesVariable() && "Expected capture by reference only!");
8943
8944 // A first private variable captured by reference will use only the
8945 // 'private ptr' and 'map to' flag. Return the right flags if the captured
8946 // declaration is known as first-private in this handler.
8947 if (FirstPrivateDecls.count(Cap.getCapturedVar())) {
8948 if (Cap.getCapturedVar()->getType()->isAnyPointerType())
8949 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
8950 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
8951 return OpenMPOffloadMappingFlags::OMP_MAP_PRIVATE |
8952 OpenMPOffloadMappingFlags::OMP_MAP_TO;
8953 }
8954 auto I = LambdasMap.find(Cap.getCapturedVar()->getCanonicalDecl());
8955 if (I != LambdasMap.end())
8956 // for map(to: lambda): using user specified map type.
8957 return getMapTypeBits(
8958 I->getSecond()->getMapType(), I->getSecond()->getMapTypeModifiers(),
8959 /*MotionModifiers=*/{}, I->getSecond()->isImplicit(),
8960 /*AddPtrFlag=*/false,
8961 /*AddIsTargetParamFlag=*/false,
8962 /*isNonContiguous=*/false);
8963 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
8964 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
8965 }
8966
8967 void getPlainLayout(const CXXRecordDecl *RD,
8968 llvm::SmallVectorImpl<const FieldDecl *> &Layout,
8969 bool AsBase) const {
8970 const CGRecordLayout &RL = CGF.getTypes().getCGRecordLayout(RD);
8971
8972 llvm::StructType *St =
8973 AsBase ? RL.getBaseSubobjectLLVMType() : RL.getLLVMType();
8974
8975 unsigned NumElements = St->getNumElements();
8976 llvm::SmallVector<
8977 llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>, 4>
8978 RecordLayout(NumElements);
8979
8980 // Fill bases.
8981 for (const auto &I : RD->bases()) {
8982 if (I.isVirtual())
8983 continue;
8984
8985 QualType BaseTy = I.getType();
8986 const auto *Base = BaseTy->getAsCXXRecordDecl();
8987 // Ignore empty bases.
8989 CGF.getContext()
8990 .getASTRecordLayout(Base)
8992 .isZero())
8993 continue;
8994
8995 unsigned FieldIndex = RL.getNonVirtualBaseLLVMFieldNo(Base);
8996 RecordLayout[FieldIndex] = Base;
8997 }
8998 // Fill in virtual bases.
8999 for (const auto &I : RD->vbases()) {
9000 QualType BaseTy = I.getType();
9001 // Ignore empty bases.
9003 continue;
9004
9005 const auto *Base = BaseTy->getAsCXXRecordDecl();
9006 unsigned FieldIndex = RL.getVirtualBaseIndex(Base);
9007 if (RecordLayout[FieldIndex])
9008 continue;
9009 RecordLayout[FieldIndex] = Base;
9010 }
9011 // Fill in all the fields.
9012 assert(!RD->isUnion() && "Unexpected union.");
9013 for (const auto *Field : RD->fields()) {
9014 // Fill in non-bitfields. (Bitfields always use a zero pattern, which we
9015 // will fill in later.)
9016 if (!Field->isBitField() &&
9018 unsigned FieldIndex = RL.getLLVMFieldNo(Field);
9019 RecordLayout[FieldIndex] = Field;
9020 }
9021 }
9022 for (const llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>
9023 &Data : RecordLayout) {
9024 if (Data.isNull())
9025 continue;
9026 if (const auto *Base = dyn_cast<const CXXRecordDecl *>(Data))
9027 getPlainLayout(Base, Layout, /*AsBase=*/true);
9028 else
9029 Layout.push_back(cast<const FieldDecl *>(Data));
9030 }
9031 }
9032
9033 /// Returns the address corresponding to \p PointerExpr.
9034 static Address getAttachPtrAddr(const Expr *PointerExpr,
9035 CodeGenFunction &CGF) {
9036 assert(PointerExpr && "Cannot get addr from null attach-ptr expr");
9037 Address AttachPtrAddr = Address::invalid();
9038
9039 if (auto *DRE = dyn_cast<DeclRefExpr>(PointerExpr)) {
9040 // If the pointer is a variable, we can use its address directly.
9041 AttachPtrAddr = CGF.EmitLValue(DRE).getAddress();
9042 } else if (auto *OASE = dyn_cast<ArraySectionExpr>(PointerExpr)) {
9043 AttachPtrAddr =
9044 CGF.EmitArraySectionExpr(OASE, /*IsLowerBound=*/true).getAddress();
9045 } else if (auto *ASE = dyn_cast<ArraySubscriptExpr>(PointerExpr)) {
9046 AttachPtrAddr = CGF.EmitLValue(ASE).getAddress();
9047 } else if (auto *ME = dyn_cast<MemberExpr>(PointerExpr)) {
9048 AttachPtrAddr = CGF.EmitMemberExpr(ME).getAddress();
9049 } else if (auto *UO = dyn_cast<UnaryOperator>(PointerExpr)) {
9050 assert(UO->getOpcode() == UO_Deref &&
9051 "Unexpected unary-operator on attach-ptr-expr");
9052 AttachPtrAddr = CGF.EmitLValue(UO).getAddress();
9053 }
9054 assert(AttachPtrAddr.isValid() &&
9055 "Failed to get address for attach pointer expression");
9056 return AttachPtrAddr;
9057 }
9058
9059 /// Get the address of the attach pointer, and a load from it, to get the
9060 /// pointee base address.
9061 /// \return A pair containing AttachPtrAddr and AttachPteeBaseAddr. The pair
9062 /// contains invalid addresses if \p AttachPtrExpr is null.
9063 static std::pair<Address, Address>
9064 getAttachPtrAddrAndPteeBaseAddr(const Expr *AttachPtrExpr,
9065 CodeGenFunction &CGF) {
9066
9067 if (!AttachPtrExpr)
9068 return {Address::invalid(), Address::invalid()};
9069
9070 Address AttachPtrAddr = getAttachPtrAddr(AttachPtrExpr, CGF);
9071 assert(AttachPtrAddr.isValid() && "Invalid attach pointer addr");
9072
9073 QualType AttachPtrType =
9076
9077 Address AttachPteeBaseAddr = CGF.EmitLoadOfPointer(
9078 AttachPtrAddr, AttachPtrType->castAs<PointerType>());
9079 assert(AttachPteeBaseAddr.isValid() && "Invalid attach pointee base addr");
9080
9081 return {AttachPtrAddr, AttachPteeBaseAddr};
9082 }
9083
9084 /// Returns whether an attach entry should be emitted for a map on
9085 /// \p MapBaseDecl on the directive \p CurDir.
9086 static bool
9087 shouldEmitAttachEntry(const Expr *PointerExpr, const ValueDecl *MapBaseDecl,
9088 CodeGenFunction &CGF,
9089 llvm::PointerUnion<const OMPExecutableDirective *,
9090 const OMPDeclareMapperDecl *>
9091 CurDir) {
9092 if (!PointerExpr)
9093 return false;
9094
9095 // Pointer attachment is needed at map-entering time or for declare
9096 // mappers.
9097 return isa<const OMPDeclareMapperDecl *>(CurDir) ||
9100 ->getDirectiveKind());
9101 }
9102
9103 /// Computes the attach-ptr expr for \p Components, and updates various maps
9104 /// with the information.
9105 /// It internally calls OMPClauseMappableExprCommon::findAttachPtrExpr()
9106 /// with the OpenMPDirectiveKind extracted from \p CurDir.
9107 /// It updates AttachPtrComputationOrderMap, AttachPtrComponentDepthMap, and
9108 /// AttachPtrExprMap.
9109 void collectAttachPtrExprInfo(
9111 llvm::PointerUnion<const OMPExecutableDirective *,
9112 const OMPDeclareMapperDecl *>
9113 CurDir) {
9114
9115 OpenMPDirectiveKind CurDirectiveID =
9117 ? OMPD_declare_mapper
9118 : cast<const OMPExecutableDirective *>(CurDir)->getDirectiveKind();
9119
9120 const auto &[AttachPtrExpr, Depth] =
9122 CurDirectiveID);
9123
9124 AttachPtrComputationOrderMap.try_emplace(
9125 AttachPtrExpr, AttachPtrComputationOrderMap.size());
9126 AttachPtrComponentDepthMap.try_emplace(AttachPtrExpr, Depth);
9127 AttachPtrExprMap.try_emplace(Components, AttachPtrExpr);
9128 }
9129
9130 /// Generate all the base pointers, section pointers, sizes, map types, and
9131 /// mappers for the extracted mappable expressions (all included in \a
9132 /// CombinedInfo). Also, for each item that relates with a device pointer, a
9133 /// pair of the relevant declaration and index where it occurs is appended to
9134 /// the device pointers info array.
9135 void generateAllInfoForClauses(
9136 ArrayRef<const OMPClause *> Clauses, MapCombinedInfoTy &CombinedInfo,
9137 llvm::OpenMPIRBuilder &OMPBuilder,
9138 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkipVarSet =
9139 llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) const {
9140 // We have to process the component lists that relate with the same
9141 // declaration in a single chunk so that we can generate the map flags
9142 // correctly. Therefore, we organize all lists in a map.
9143 enum MapKind { Present, Allocs, Other, Total };
9144 llvm::MapVector<CanonicalDeclPtr<const Decl>,
9145 SmallVector<SmallVector<MapInfo, 8>, 4>>
9146 Info;
9147
9148 // Helper function to fill the information map for the different supported
9149 // clauses.
9150 auto &&InfoGen =
9151 [&Info, &SkipVarSet](
9152 const ValueDecl *D, MapKind Kind,
9154 OpenMPMapClauseKind MapType,
9155 ArrayRef<OpenMPMapModifierKind> MapModifiers,
9156 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
9157 bool ReturnDevicePointer, bool IsImplicit, const ValueDecl *Mapper,
9158 const Expr *VarRef = nullptr, bool ForDeviceAddr = false) {
9159 if (SkipVarSet.contains(D))
9160 return;
9161 auto It = Info.try_emplace(D, Total).first;
9162 It->second[Kind].emplace_back(
9163 L, MapType, MapModifiers, MotionModifiers, ReturnDevicePointer,
9164 IsImplicit, Mapper, VarRef, ForDeviceAddr);
9165 };
9166
9167 for (const auto *Cl : Clauses) {
9168 const auto *C = dyn_cast<OMPMapClause>(Cl);
9169 if (!C)
9170 continue;
9171 MapKind Kind = Other;
9172 if (llvm::is_contained(C->getMapTypeModifiers(),
9173 OMPC_MAP_MODIFIER_present))
9174 Kind = Present;
9175 else if (C->getMapType() == OMPC_MAP_alloc)
9176 Kind = Allocs;
9177 const auto *EI = C->getVarRefs().begin();
9178 for (const auto L : C->component_lists()) {
9179 const Expr *E = (C->getMapLoc().isValid()) ? *EI : nullptr;
9180 InfoGen(std::get<0>(L), Kind, std::get<1>(L), C->getMapType(),
9181 C->getMapTypeModifiers(), {},
9182 /*ReturnDevicePointer=*/false, C->isImplicit(), std::get<2>(L),
9183 E);
9184 ++EI;
9185 }
9186 }
9187 for (const auto *Cl : Clauses) {
9188 const auto *C = dyn_cast<OMPToClause>(Cl);
9189 if (!C)
9190 continue;
9191 MapKind Kind = Other;
9192 if (llvm::is_contained(C->getMotionModifiers(),
9193 OMPC_MOTION_MODIFIER_present))
9194 Kind = Present;
9195 if (llvm::is_contained(C->getMotionModifiers(),
9196 OMPC_MOTION_MODIFIER_iterator)) {
9197 if (auto *IteratorExpr = dyn_cast<OMPIteratorExpr>(
9198 C->getIteratorModifier()->IgnoreParenImpCasts())) {
9199 const auto *VD = cast<VarDecl>(IteratorExpr->getIteratorDecl(0));
9200 CGF.EmitVarDecl(*VD);
9201 }
9202 }
9203
9204 const auto *EI = C->getVarRefs().begin();
9205 for (const auto L : C->component_lists()) {
9206 InfoGen(std::get<0>(L), Kind, std::get<1>(L), OMPC_MAP_to, {},
9207 C->getMotionModifiers(), /*ReturnDevicePointer=*/false,
9208 C->isImplicit(), std::get<2>(L), *EI);
9209 ++EI;
9210 }
9211 }
9212 for (const auto *Cl : Clauses) {
9213 const auto *C = dyn_cast<OMPFromClause>(Cl);
9214 if (!C)
9215 continue;
9216 MapKind Kind = Other;
9217 if (llvm::is_contained(C->getMotionModifiers(),
9218 OMPC_MOTION_MODIFIER_present))
9219 Kind = Present;
9220 if (llvm::is_contained(C->getMotionModifiers(),
9221 OMPC_MOTION_MODIFIER_iterator)) {
9222 if (auto *IteratorExpr = dyn_cast<OMPIteratorExpr>(
9223 C->getIteratorModifier()->IgnoreParenImpCasts())) {
9224 const auto *VD = cast<VarDecl>(IteratorExpr->getIteratorDecl(0));
9225 CGF.EmitVarDecl(*VD);
9226 }
9227 }
9228
9229 const auto *EI = C->getVarRefs().begin();
9230 for (const auto L : C->component_lists()) {
9231 InfoGen(std::get<0>(L), Kind, std::get<1>(L), OMPC_MAP_from, {},
9232 C->getMotionModifiers(),
9233 /*ReturnDevicePointer=*/false, C->isImplicit(), std::get<2>(L),
9234 *EI);
9235 ++EI;
9236 }
9237 }
9238
9239 // Look at the use_device_ptr and use_device_addr clauses information and
9240 // mark the existing map entries as such. If there is no map information for
9241 // an entry in the use_device_ptr and use_device_addr list, we create one
9242 // with map type 'return_param' and zero size section. It is the user's
9243 // fault if that was not mapped before. If there is no map information, then
9244 // we defer the emission of that entry until all the maps for the same VD
9245 // have been handled.
9246 MapCombinedInfoTy UseDeviceDataCombinedInfo;
9247
9248 auto &&UseDeviceDataCombinedInfoGen =
9249 [&UseDeviceDataCombinedInfo](const ValueDecl *VD, llvm::Value *Ptr,
9250 CodeGenFunction &CGF, bool IsDevAddr,
9251 bool HasUdpFbNullify = false) {
9252 UseDeviceDataCombinedInfo.Exprs.push_back(VD);
9253 UseDeviceDataCombinedInfo.BasePointers.emplace_back(Ptr);
9254 UseDeviceDataCombinedInfo.DevicePtrDecls.emplace_back(VD);
9255 UseDeviceDataCombinedInfo.DevicePointers.emplace_back(
9256 IsDevAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer);
9257 // FIXME: For use_device_addr on array-sections, this should
9258 // be the starting address of the section.
9259 // e.g. int *p;
9260 // ... use_device_addr(p[3])
9261 // &p[0], &p[3], /*size=*/0, RETURN_PARAM
9262 UseDeviceDataCombinedInfo.Pointers.push_back(Ptr);
9263 UseDeviceDataCombinedInfo.Sizes.push_back(
9264 llvm::Constant::getNullValue(CGF.Int64Ty));
9265 OpenMPOffloadMappingFlags Flags =
9266 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
9267 if (HasUdpFbNullify)
9268 Flags |= OpenMPOffloadMappingFlags::OMP_MAP_FB_NULLIFY;
9269 UseDeviceDataCombinedInfo.Types.push_back(Flags);
9270 UseDeviceDataCombinedInfo.HasAttachPtr.push_back(false);
9271 UseDeviceDataCombinedInfo.Mappers.push_back(nullptr);
9272 };
9273
9274 auto &&MapInfoGen =
9275 [&UseDeviceDataCombinedInfoGen](
9276 CodeGenFunction &CGF, const Expr *IE, const ValueDecl *VD,
9278 Components,
9279 bool IsDevAddr, bool IEIsAttachPtrForDevAddr = false,
9280 bool HasUdpFbNullify = false) {
9281 // We didn't find any match in our map information - generate a zero
9282 // size array section.
9283 llvm::Value *Ptr;
9284 if (IsDevAddr && !IEIsAttachPtrForDevAddr) {
9285 if (IE->isGLValue())
9286 Ptr = CGF.EmitLValue(IE).getPointer(CGF);
9287 else
9288 Ptr = CGF.EmitScalarExpr(IE);
9289 } else {
9290 Ptr = CGF.EmitLoadOfScalar(CGF.EmitLValue(IE), IE->getExprLoc());
9291 }
9292 bool TreatDevAddrAsDevPtr = IEIsAttachPtrForDevAddr;
9293 // For the purpose of address-translation, treat something like the
9294 // following:
9295 // int *p;
9296 // ... use_device_addr(p[1])
9297 // equivalent to
9298 // ... use_device_ptr(p)
9299 UseDeviceDataCombinedInfoGen(VD, Ptr, CGF, /*IsDevAddr=*/IsDevAddr &&
9300 !TreatDevAddrAsDevPtr,
9301 HasUdpFbNullify);
9302 };
9303
9304 auto &&IsMapInfoExist =
9305 [&Info, this](CodeGenFunction &CGF, const ValueDecl *VD, const Expr *IE,
9306 const Expr *DesiredAttachPtrExpr, bool IsDevAddr,
9307 bool HasUdpFbNullify = false) -> bool {
9308 // We potentially have map information for this declaration already.
9309 // Look for the first set of components that refer to it. If found,
9310 // return true.
9311 // If the first component is a member expression, we have to look into
9312 // 'this', which maps to null in the map of map information. Otherwise
9313 // look directly for the information.
9314 auto It = Info.find(isa<MemberExpr>(IE) ? nullptr : VD);
9315 if (It != Info.end()) {
9316 bool Found = false;
9317 for (auto &Data : It->second) {
9318 MapInfo *CI = nullptr;
9319 // We potentially have multiple maps for the same decl. We need to
9320 // only consider those for which the attach-ptr matches the desired
9321 // attach-ptr.
9322 auto *It = llvm::find_if(Data, [&](const MapInfo &MI) {
9323 if (MI.Components.back().getAssociatedDeclaration() != VD)
9324 return false;
9325
9326 const Expr *MapAttachPtr = getAttachPtrExpr(MI.Components);
9327 bool Match = AttachPtrComparator.areEqual(MapAttachPtr,
9328 DesiredAttachPtrExpr);
9329 return Match;
9330 });
9331
9332 if (It != Data.end())
9333 CI = &*It;
9334
9335 if (CI) {
9336 if (IsDevAddr) {
9337 CI->ForDeviceAddr = true;
9338 CI->ReturnDevicePointer = true;
9339 CI->HasUdpFbNullify = HasUdpFbNullify;
9340 Found = true;
9341 break;
9342 } else {
9343 auto PrevCI = std::next(CI->Components.rbegin());
9344 const auto *VarD = dyn_cast<VarDecl>(VD);
9345 const Expr *AttachPtrExpr = getAttachPtrExpr(CI->Components);
9346 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
9347 isa<MemberExpr>(IE) ||
9348 !VD->getType().getNonReferenceType()->isPointerType() ||
9349 PrevCI == CI->Components.rend() ||
9350 isa<MemberExpr>(PrevCI->getAssociatedExpression()) || !VarD ||
9351 VarD->hasLocalStorage() ||
9352 (isa_and_nonnull<DeclRefExpr>(AttachPtrExpr) &&
9353 VD == cast<DeclRefExpr>(AttachPtrExpr)->getDecl())) {
9354 CI->ForDeviceAddr = IsDevAddr;
9355 CI->ReturnDevicePointer = true;
9356 CI->HasUdpFbNullify = HasUdpFbNullify;
9357 Found = true;
9358 break;
9359 }
9360 }
9361 }
9362 }
9363 return Found;
9364 }
9365 return false;
9366 };
9367
9368 // Look at the use_device_ptr clause information and mark the existing map
9369 // entries as such. If there is no map information for an entry in the
9370 // use_device_ptr list, we create one with map type 'alloc' and zero size
9371 // section. It is the user fault if that was not mapped before. If there is
9372 // no map information and the pointer is a struct member, then we defer the
9373 // emission of that entry until the whole struct has been processed.
9374 for (const auto *Cl : Clauses) {
9375 const auto *C = dyn_cast<OMPUseDevicePtrClause>(Cl);
9376 if (!C)
9377 continue;
9378 bool HasUdpFbNullify =
9379 C->getFallbackModifier() == OMPC_USE_DEVICE_PTR_FALLBACK_fb_nullify;
9380 for (const auto L : C->component_lists()) {
9382 std::get<1>(L);
9383 assert(!Components.empty() &&
9384 "Not expecting empty list of components!");
9385 const ValueDecl *VD = Components.back().getAssociatedDeclaration();
9387 const Expr *IE = Components.back().getAssociatedExpression();
9388 // For use_device_ptr, we match an existing map clause if its attach-ptr
9389 // is same as the use_device_ptr operand. e.g.
9390 // map expr | use_device_ptr expr | current behavior
9391 // ---------|---------------------|-----------------
9392 // p[1] | p | match
9393 // ps->a | ps | match
9394 // p | p | no match
9395 const Expr *UDPOperandExpr =
9396 Components.front().getAssociatedExpression();
9397 if (IsMapInfoExist(CGF, VD, IE,
9398 /*DesiredAttachPtrExpr=*/UDPOperandExpr,
9399 /*IsDevAddr=*/false, HasUdpFbNullify))
9400 continue;
9401 MapInfoGen(CGF, IE, VD, Components, /*IsDevAddr=*/false,
9402 /*IEIsAttachPtrForDevAddr=*/false, HasUdpFbNullify);
9403 }
9404 }
9405
9406 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
9407 for (const auto *Cl : Clauses) {
9408 const auto *C = dyn_cast<OMPUseDeviceAddrClause>(Cl);
9409 if (!C)
9410 continue;
9411 for (const auto L : C->component_lists()) {
9413 std::get<1>(L);
9414 assert(!std::get<1>(L).empty() &&
9415 "Not expecting empty list of components!");
9416 const ValueDecl *VD = std::get<1>(L).back().getAssociatedDeclaration();
9417 if (!Processed.insert(VD).second)
9418 continue;
9420 // For use_device_addr, we match an existing map clause if the
9421 // use_device_addr operand's attach-ptr matches the map operand's
9422 // attach-ptr.
9423 // We chould also restrict to only match cases when there is a full
9424 // match between the map/use_device_addr clause exprs, but that may be
9425 // unnecessary.
9426 //
9427 // map expr | use_device_addr expr | current | possible restrictive/
9428 // | | behavior | safer behavior
9429 // ---------|----------------------|-----------|-----------------------
9430 // p | p | match | match
9431 // p[0] | p[0] | match | match
9432 // p[0:1] | p[0] | match | no match
9433 // p[0:1] | p[2:1] | match | no match
9434 // p[1] | p[0] | match | no match
9435 // ps->a | ps->b | match | no match
9436 // p | p[0] | no match | no match
9437 // pp | pp[0][0] | no match | no match
9438 const Expr *UDAAttachPtrExpr = getAttachPtrExpr(Components);
9439 const Expr *IE = std::get<1>(L).back().getAssociatedExpression();
9440 assert((!UDAAttachPtrExpr || UDAAttachPtrExpr == IE) &&
9441 "use_device_addr operand has an attach-ptr, but does not match "
9442 "last component's expr.");
9443 if (IsMapInfoExist(CGF, VD, IE,
9444 /*DesiredAttachPtrExpr=*/UDAAttachPtrExpr,
9445 /*IsDevAddr=*/true))
9446 continue;
9447 MapInfoGen(CGF, IE, VD, Components,
9448 /*IsDevAddr=*/true,
9449 /*IEIsAttachPtrForDevAddr=*/UDAAttachPtrExpr != nullptr);
9450 }
9451 }
9452
9453 for (const auto &Data : Info) {
9454 MapCombinedInfoTy CurInfo;
9455 const Decl *D = Data.first;
9456 const ValueDecl *VD = cast_or_null<ValueDecl>(D);
9457 // Group component lists by their AttachPtrExpr and process them in order
9458 // of increasing complexity (nullptr first, then simple expressions like
9459 // p, then more complex ones like p[0], etc.)
9460 //
9461 // This is similar to how generateInfoForCaptureFromClauseInfo handles
9462 // grouping for target constructs.
9463 SmallVector<std::pair<const Expr *, MapInfo>, 16> AttachPtrMapInfoPairs;
9464
9465 // First, collect all MapData entries with their attach-ptr exprs.
9466 for (const auto &M : Data.second) {
9467 for (const MapInfo &L : M) {
9468 assert(!L.Components.empty() &&
9469 "Not expecting declaration with no component lists.");
9470
9471 const Expr *AttachPtrExpr = getAttachPtrExpr(L.Components);
9472 AttachPtrMapInfoPairs.emplace_back(AttachPtrExpr, L);
9473 }
9474 }
9475
9476 // Next, sort by increasing order of their complexity.
9477 llvm::stable_sort(AttachPtrMapInfoPairs,
9478 [this](const auto &LHS, const auto &RHS) {
9479 return AttachPtrComparator(LHS.first, RHS.first);
9480 });
9481
9482 // And finally, process them all in order, grouping those with
9483 // equivalent attach-ptr exprs together.
9484 auto *It = AttachPtrMapInfoPairs.begin();
9485 while (It != AttachPtrMapInfoPairs.end()) {
9486 const Expr *AttachPtrExpr = It->first;
9487
9488 SmallVector<MapInfo, 8> GroupLists;
9489 while (It != AttachPtrMapInfoPairs.end() &&
9490 (It->first == AttachPtrExpr ||
9491 AttachPtrComparator.areEqual(It->first, AttachPtrExpr))) {
9492 GroupLists.push_back(It->second);
9493 ++It;
9494 }
9495 assert(!GroupLists.empty() && "GroupLists should not be empty");
9496
9497 StructRangeInfoTy PartialStruct;
9498 AttachInfoTy AttachInfo;
9499 MapCombinedInfoTy GroupCurInfo;
9500 // Current group's struct base information:
9501 MapCombinedInfoTy GroupStructBaseCurInfo;
9502 for (const MapInfo &L : GroupLists) {
9503 // Remember the current base pointer index.
9504 unsigned CurrentBasePointersIdx = GroupCurInfo.BasePointers.size();
9505 unsigned StructBasePointersIdx =
9506 GroupStructBaseCurInfo.BasePointers.size();
9507
9508 GroupCurInfo.NonContigInfo.IsNonContiguous =
9509 L.Components.back().isNonContiguous();
9510 generateInfoForComponentList(
9511 L.MapType, L.MapModifiers, L.MotionModifiers, L.Components,
9512 GroupCurInfo, GroupStructBaseCurInfo, PartialStruct, AttachInfo,
9513 /*IsFirstComponentList=*/false, L.IsImplicit,
9514 /*GenerateAllInfoForClauses*/ true, L.Mapper, L.ForDeviceAddr, VD,
9515 L.VarRef, /*OverlappedElements*/ {});
9516
9517 // If this entry relates to a device pointer, set the relevant
9518 // declaration and add the 'return pointer' flag.
9519 if (L.ReturnDevicePointer) {
9520 // Check whether a value was added to either GroupCurInfo or
9521 // GroupStructBaseCurInfo and error if no value was added to either
9522 // of them:
9523 assert((CurrentBasePointersIdx < GroupCurInfo.BasePointers.size() ||
9524 StructBasePointersIdx <
9525 GroupStructBaseCurInfo.BasePointers.size()) &&
9526 "Unexpected number of mapped base pointers.");
9527
9528 // Choose a base pointer index which is always valid:
9529 const ValueDecl *RelevantVD =
9530 L.Components.back().getAssociatedDeclaration();
9531 assert(RelevantVD &&
9532 "No relevant declaration related with device pointer??");
9533
9534 // If GroupStructBaseCurInfo has been updated this iteration then
9535 // work on the first new entry added to it i.e. make sure that when
9536 // multiple values are added to any of the lists, the first value
9537 // added is being modified by the assignments below (not the last
9538 // value added).
9539 auto SetDevicePointerInfo = [&](MapCombinedInfoTy &Info,
9540 unsigned Idx) {
9541 Info.DevicePtrDecls[Idx] = RelevantVD;
9542 Info.DevicePointers[Idx] = L.ForDeviceAddr
9543 ? DeviceInfoTy::Address
9544 : DeviceInfoTy::Pointer;
9545 Info.Types[Idx] |=
9546 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
9547 if (L.HasUdpFbNullify)
9548 Info.Types[Idx] |=
9549 OpenMPOffloadMappingFlags::OMP_MAP_FB_NULLIFY;
9550 };
9551
9552 if (StructBasePointersIdx <
9553 GroupStructBaseCurInfo.BasePointers.size())
9554 SetDevicePointerInfo(GroupStructBaseCurInfo,
9555 StructBasePointersIdx);
9556 else
9557 SetDevicePointerInfo(GroupCurInfo, CurrentBasePointersIdx);
9558 }
9559 }
9560
9561 // Unify entries in one list making sure the struct mapping precedes the
9562 // individual fields:
9563 MapCombinedInfoTy GroupUnionCurInfo;
9564 GroupUnionCurInfo.append(GroupStructBaseCurInfo);
9565 GroupUnionCurInfo.append(GroupCurInfo);
9566
9567 // If there is an entry in PartialStruct it means we have a struct with
9568 // individual members mapped. Emit an extra combined entry.
9569 if (PartialStruct.Base.isValid()) {
9570 // Prepend a synthetic dimension of length 1 to represent the
9571 // aggregated struct object. Using 1 (not 0, as 0 produced an
9572 // incorrect non-contiguous descriptor (DimSize==1), causing the
9573 // non-contiguous motion clause path to be skipped.) is important:
9574 // * It preserves the correct rank so targetDataUpdate() computes
9575 // DimSize == 2 for cases like strided array sections originating
9576 // from user-defined mappers (e.g. test with s.data[0:8:2]).
9577 GroupUnionCurInfo.NonContigInfo.Dims.insert(
9578 GroupUnionCurInfo.NonContigInfo.Dims.begin(), 1);
9579 emitCombinedEntry(
9580 CurInfo, GroupUnionCurInfo.Types, PartialStruct, AttachInfo,
9581 /*IsMapThis=*/!VD, OMPBuilder, VD,
9582 /*OffsetForMemberOfFlag=*/CombinedInfo.BasePointers.size(),
9583 /*NotTargetParams=*/true);
9584 }
9585
9586 // Append this group's results to the overall CurInfo in the correct
9587 // order: combined-entry -> original-field-entries -> attach-entry
9588 CurInfo.append(GroupUnionCurInfo);
9589 if (AttachInfo.isValid())
9590 emitAttachEntry(CGF, CurInfo, AttachInfo);
9591 }
9592
9593 // We need to append the results of this capture to what we already have.
9594 CombinedInfo.append(CurInfo);
9595 }
9596 // Append data for use_device_ptr/addr clauses.
9597 CombinedInfo.append(UseDeviceDataCombinedInfo);
9598 }
9599
9600public:
9601 MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF)
9602 : CurDir(&Dir), CGF(CGF), AttachPtrComparator(*this) {
9603 // Extract firstprivate clause information.
9604 for (const auto *C : Dir.getClausesOfKind<OMPFirstprivateClause>())
9605 for (const auto *D : C->varlist()) {
9606 const ValueDecl *VD = cast<DeclRefExpr>(D)->getDecl();
9607 if (const auto *BD = dyn_cast<BindingDecl>(VD))
9608 VD = cast<VarDecl>(BD->getDecomposedDecl());
9609 FirstPrivateDecls.try_emplace(cast<VarDecl>(VD), C->isImplicit());
9610 }
9611 // Extract implicit firstprivates from uses_allocators clauses.
9612 for (const auto *C : Dir.getClausesOfKind<OMPUsesAllocatorsClause>()) {
9613 for (unsigned I = 0, E = C->getNumberOfAllocators(); I < E; ++I) {
9614 OMPUsesAllocatorsClause::Data D = C->getAllocatorData(I);
9615 if (const auto *DRE = dyn_cast_or_null<DeclRefExpr>(D.AllocatorTraits))
9616 FirstPrivateDecls.try_emplace(cast<VarDecl>(DRE->getDecl()),
9617 /*Implicit=*/true);
9618 else if (const auto *VD = dyn_cast<VarDecl>(
9619 cast<DeclRefExpr>(D.Allocator->IgnoreParenImpCasts())
9620 ->getDecl()))
9621 FirstPrivateDecls.try_emplace(VD, /*Implicit=*/true);
9622 }
9623 }
9624 // Extract defaultmap clause information.
9625 for (const auto *C : Dir.getClausesOfKind<OMPDefaultmapClause>())
9626 if (C->getDefaultmapModifier() == OMPC_DEFAULTMAP_MODIFIER_firstprivate)
9627 DefaultmapFirstprivateKinds.insert(C->getDefaultmapKind());
9628 // Extract device pointer clause information.
9629 for (const auto *C : Dir.getClausesOfKind<OMPIsDevicePtrClause>())
9630 for (auto L : C->component_lists())
9631 DevPointersMap[std::get<0>(L)].push_back(std::get<1>(L));
9632 // Extract device addr clause information.
9633 for (const auto *C : Dir.getClausesOfKind<OMPHasDeviceAddrClause>())
9634 for (auto L : C->component_lists())
9635 HasDevAddrsMap[std::get<0>(L)].push_back(std::get<1>(L));
9636 // Extract map information.
9637 for (const auto *C : Dir.getClausesOfKind<OMPMapClause>()) {
9638 if (C->getMapType() != OMPC_MAP_to)
9639 continue;
9640 for (auto L : C->component_lists()) {
9641 const ValueDecl *VD = std::get<0>(L);
9642 const auto *RD = VD ? VD->getType()
9643 .getCanonicalType()
9644 .getNonReferenceType()
9645 ->getAsCXXRecordDecl()
9646 : nullptr;
9647 if (RD && RD->isLambda())
9648 LambdasMap.try_emplace(std::get<0>(L), C);
9649 }
9650 }
9651
9652 auto CollectAttachPtrExprsForClauseComponents = [this](const auto *C) {
9653 for (auto L : C->component_lists()) {
9655 std::get<1>(L);
9656 if (!Components.empty())
9657 collectAttachPtrExprInfo(Components, CurDir);
9658 }
9659 };
9660
9661 // Populate the AttachPtrExprMap for all component lists from map-related
9662 // clauses.
9663 for (const auto *C : Dir.getClausesOfKind<OMPMapClause>())
9664 CollectAttachPtrExprsForClauseComponents(C);
9665 for (const auto *C : Dir.getClausesOfKind<OMPToClause>())
9666 CollectAttachPtrExprsForClauseComponents(C);
9667 for (const auto *C : Dir.getClausesOfKind<OMPFromClause>())
9668 CollectAttachPtrExprsForClauseComponents(C);
9669 for (const auto *C : Dir.getClausesOfKind<OMPUseDevicePtrClause>())
9670 CollectAttachPtrExprsForClauseComponents(C);
9671 for (const auto *C : Dir.getClausesOfKind<OMPUseDeviceAddrClause>())
9672 CollectAttachPtrExprsForClauseComponents(C);
9673 for (const auto *C : Dir.getClausesOfKind<OMPIsDevicePtrClause>())
9674 CollectAttachPtrExprsForClauseComponents(C);
9675 for (const auto *C : Dir.getClausesOfKind<OMPHasDeviceAddrClause>())
9676 CollectAttachPtrExprsForClauseComponents(C);
9677 }
9678
9679 /// Constructor for the declare mapper directive.
9680 MappableExprsHandler(const OMPDeclareMapperDecl &Dir, CodeGenFunction &CGF)
9681 : CurDir(&Dir), CGF(CGF), AttachPtrComparator(*this) {
9682 auto CollectAttachPtrExprsForClauseComponents = [this](const auto *C) {
9683 for (auto L : C->component_lists()) {
9685 std::get<1>(L);
9686 if (!Components.empty())
9687 collectAttachPtrExprInfo(Components, CurDir);
9688 }
9689 };
9690
9691 // Populate the AttachPtrExprMap for all component lists from map-related
9692 // clauses in the declare mapper directive, to enable attach-style mapping
9693 // for mappers.
9694 for (const auto *Cl : Dir.clauses()) {
9695 if (const auto *C = dyn_cast<OMPMapClause>(Cl))
9696 CollectAttachPtrExprsForClauseComponents(C);
9697 else if (const auto *C = dyn_cast<OMPToClause>(Cl))
9698 CollectAttachPtrExprsForClauseComponents(C);
9699 else if (const auto *C = dyn_cast<OMPFromClause>(Cl))
9700 CollectAttachPtrExprsForClauseComponents(C);
9701 }
9702 }
9703
9704 /// Generate code for the combined entry if we have a partially mapped struct
9705 /// and take care of the mapping flags of the arguments corresponding to
9706 /// individual struct members.
9707 /// If a valid \p AttachInfo exists, its pointee addr will be updated to point
9708 /// to the combined-entry's begin address, if emitted.
9709 /// \p PartialStruct contains attach base-pointer information.
9710 /// \returns The index of the combined entry if one was added, std::nullopt
9711 /// otherwise.
9712 void emitCombinedEntry(MapCombinedInfoTy &CombinedInfo,
9713 MapFlagsArrayTy &CurTypes,
9714 const StructRangeInfoTy &PartialStruct,
9715 AttachInfoTy &AttachInfo, bool IsMapThis,
9716 llvm::OpenMPIRBuilder &OMPBuilder, const ValueDecl *VD,
9717 unsigned OffsetForMemberOfFlag,
9718 bool NotTargetParams) const {
9719 if (CurTypes.size() == 1 &&
9720 ((CurTypes.back() & OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) !=
9721 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) &&
9722 !PartialStruct.IsArraySection)
9723 return;
9724 Address LBAddr = PartialStruct.LowestElem.second;
9725 Address HBAddr = PartialStruct.HighestElem.second;
9726 if (PartialStruct.HasCompleteRecord) {
9727 LBAddr = PartialStruct.LB;
9728 HBAddr = PartialStruct.LB;
9729 }
9730 CombinedInfo.Exprs.push_back(VD);
9731 // Base is the base of the struct
9732 CombinedInfo.BasePointers.push_back(PartialStruct.Base.emitRawPointer(CGF));
9733 CombinedInfo.DevicePtrDecls.push_back(nullptr);
9734 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
9735 // Pointer is the address of the lowest element
9736 llvm::Value *LB = LBAddr.emitRawPointer(CGF);
9737 const CXXMethodDecl *MD =
9738 CGF.CurFuncDecl ? dyn_cast<CXXMethodDecl>(CGF.CurFuncDecl) : nullptr;
9739 const CXXRecordDecl *RD = MD ? MD->getParent() : nullptr;
9740 bool HasBaseClass = RD && IsMapThis ? RD->getNumBases() > 0 : false;
9741 // There should not be a mapper for a combined entry.
9742 if (HasBaseClass) {
9743 // OpenMP 5.2 148:21:
9744 // If the target construct is within a class non-static member function,
9745 // and a variable is an accessible data member of the object for which the
9746 // non-static data member function is invoked, the variable is treated as
9747 // if the this[:1] expression had appeared in a map clause with a map-type
9748 // of tofrom.
9749 // Emit this[:1]
9750 CombinedInfo.Pointers.push_back(PartialStruct.Base.emitRawPointer(CGF));
9751 QualType Ty = MD->getFunctionObjectParameterType();
9752 llvm::Value *Size =
9753 CGF.Builder.CreateIntCast(CGF.getTypeSize(Ty), CGF.Int64Ty,
9754 /*isSigned=*/true);
9755 CombinedInfo.Sizes.push_back(Size);
9756 } else {
9757 CombinedInfo.Pointers.push_back(LB);
9758 // Size is (addr of {highest+1} element) - (addr of lowest element)
9759 llvm::Value *HB = HBAddr.emitRawPointer(CGF);
9760 llvm::Value *HAddr = CGF.Builder.CreateConstGEP1_32(
9761 HBAddr.getElementType(), HB, /*Idx0=*/1);
9762 llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(LB, CGF.VoidPtrTy);
9763 llvm::Value *CHAddr = CGF.Builder.CreatePointerCast(HAddr, CGF.VoidPtrTy);
9764 llvm::Value *Diff = CGF.Builder.CreatePtrDiff(CHAddr, CLAddr);
9765 llvm::Value *Size = CGF.Builder.CreateIntCast(Diff, CGF.Int64Ty,
9766 /*isSigned=*/false);
9767 CombinedInfo.Sizes.push_back(Size);
9768 }
9769 CombinedInfo.Mappers.push_back(nullptr);
9770 // Map type is always TARGET_PARAM, if generate info for captures.
9771 CombinedInfo.Types.push_back(
9772 NotTargetParams ? OpenMPOffloadMappingFlags::OMP_MAP_NONE
9773 : !PartialStruct.PreliminaryMapData.BasePointers.empty()
9774 ? OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ
9775 : OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
9776 // A combined entry has a base attach-ptr if its constituents do. e.g.:
9777 // map(s2.s1p->x, s2.s1p->y)
9778 // combined entry:
9779 // s2.s1p[0], s2.s1p->x, sizeof(s1p->x..y), ALLOC
9780 // here s2.s1p is the attach-ptr for the combined entry.
9781 // See the inline comments in emitUserDefinedMapper's definition for how
9782 // entries with an attach-ptr are treated.
9783 CombinedInfo.HasAttachPtr.push_back(AttachInfo.isValid());
9784 // If any element has the present modifier, then make sure the runtime
9785 // doesn't attempt to allocate the struct.
9786 if (CurTypes.end() !=
9787 llvm::find_if(CurTypes, [](OpenMPOffloadMappingFlags Type) {
9788 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
9789 Type & OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
9790 }))
9791 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
9792 // Remove TARGET_PARAM flag from the first element
9793 (*CurTypes.begin()) &= ~OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
9794 // If any element has the ompx_hold modifier, then make sure the runtime
9795 // uses the hold reference count for the struct as a whole so that it won't
9796 // be unmapped by an extra dynamic reference count decrement. Add it to all
9797 // elements as well so the runtime knows which reference count to check
9798 // when determining whether it's time for device-to-host transfers of
9799 // individual elements.
9800 if (CurTypes.end() !=
9801 llvm::find_if(CurTypes, [](OpenMPOffloadMappingFlags Type) {
9802 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
9803 Type & OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD);
9804 })) {
9805 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
9806 for (auto &M : CurTypes)
9807 M |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
9808 }
9809
9810 // All other current entries will be MEMBER_OF the combined entry
9811 // (except for PTR_AND_OBJ entries which do not have a placeholder value
9812 // 0xFFFF in the MEMBER_OF field, or ATTACH entries since they are expected
9813 // to be handled by themselves, after all other maps).
9814 OpenMPOffloadMappingFlags MemberOfFlag = OMPBuilder.getMemberOfFlag(
9815 OffsetForMemberOfFlag + CombinedInfo.BasePointers.size() - 1);
9816 for (auto &M : CurTypes)
9817 OMPBuilder.setCorrectMemberOfFlag(M, MemberOfFlag);
9818
9819 // When we are emitting a combined entry. If there were any pending
9820 // attachments to be done, we do them to the begin address of the combined
9821 // entry. Note that this means only one attachment per combined-entry will
9822 // be done. So, for instance, if we have:
9823 // S *ps;
9824 // ... map(ps->a, ps->b)
9825 // When we are emitting a combined entry. If AttachInfo is valid,
9826 // update the pointee address to point to the begin address of the combined
9827 // entry. This ensures that if we have multiple maps like:
9828 // `map(ps->a, ps->b)`, we still get a single ATTACH entry, like:
9829 //
9830 // &ps[0], &ps->a, sizeof(ps->a to ps->b), ALLOC // combined-entry
9831 // &ps[0], &ps->a, sizeof(ps->a), TO | FROM
9832 // &ps[0], &ps->b, sizeof(ps->b), TO | FROM
9833 // &ps, &ps->a, sizeof(void*), ATTACH // Use combined-entry's LB
9834 if (AttachInfo.isValid())
9835 AttachInfo.AttachPteeAddr = LBAddr;
9836 }
9837
9838 /// Generate all the base pointers, section pointers, sizes, map types, and
9839 /// mappers for the extracted mappable expressions (all included in \a
9840 /// CombinedInfo). Also, for each item that relates with a device pointer, a
9841 /// pair of the relevant declaration and index where it occurs is appended to
9842 /// the device pointers info array.
9843 void generateAllInfo(
9844 MapCombinedInfoTy &CombinedInfo, llvm::OpenMPIRBuilder &OMPBuilder,
9845 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkipVarSet =
9846 llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) const {
9847 assert(isa<const OMPExecutableDirective *>(CurDir) &&
9848 "Expect a executable directive");
9849 const auto *CurExecDir = cast<const OMPExecutableDirective *>(CurDir);
9850 generateAllInfoForClauses(CurExecDir->clauses(), CombinedInfo, OMPBuilder,
9851 SkipVarSet);
9852 }
9853
9854 /// Generate all the base pointers, section pointers, sizes, map types, and
9855 /// mappers for the extracted map clauses of user-defined mapper (all included
9856 /// in \a CombinedInfo).
9857 void generateAllInfoForMapper(MapCombinedInfoTy &CombinedInfo,
9858 llvm::OpenMPIRBuilder &OMPBuilder) const {
9859 assert(isa<const OMPDeclareMapperDecl *>(CurDir) &&
9860 "Expect a declare mapper directive");
9861 const auto *CurMapperDir = cast<const OMPDeclareMapperDecl *>(CurDir);
9862 generateAllInfoForClauses(CurMapperDir->clauses(), CombinedInfo,
9863 OMPBuilder);
9864 }
9865
9866 /// Emit capture info for lambdas for variables captured by reference.
9867 void generateInfoForLambdaCaptures(
9868 const ValueDecl *VD, llvm::Value *Arg, MapCombinedInfoTy &CombinedInfo,
9869 llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers) const {
9870 QualType VDType = VD->getType().getCanonicalType().getNonReferenceType();
9871 const auto *RD = VDType->getAsCXXRecordDecl();
9872 if (!RD || !RD->isLambda())
9873 return;
9874 Address VDAddr(Arg, CGF.ConvertTypeForMem(VDType),
9875 CGF.getContext().getDeclAlign(VD));
9876 LValue VDLVal = CGF.MakeAddrLValue(VDAddr, VDType);
9877 llvm::DenseMap<const ValueDecl *, FieldDecl *> Captures;
9878 FieldDecl *ThisCapture = nullptr;
9879 RD->getCaptureFields(Captures, ThisCapture);
9880 if (ThisCapture) {
9881 LValue ThisLVal =
9882 CGF.EmitLValueForFieldInitialization(VDLVal, ThisCapture);
9883 LValue ThisLValVal = CGF.EmitLValueForField(VDLVal, ThisCapture);
9884 LambdaPointers.try_emplace(ThisLVal.getPointer(CGF),
9885 VDLVal.getPointer(CGF));
9886 CombinedInfo.Exprs.push_back(VD);
9887 CombinedInfo.BasePointers.push_back(ThisLVal.getPointer(CGF));
9888 CombinedInfo.DevicePtrDecls.push_back(nullptr);
9889 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
9890 CombinedInfo.Pointers.push_back(ThisLValVal.getPointer(CGF));
9891 CombinedInfo.Sizes.push_back(
9892 CGF.Builder.CreateIntCast(CGF.getTypeSize(CGF.getContext().VoidPtrTy),
9893 CGF.Int64Ty, /*isSigned=*/true));
9894 CombinedInfo.Types.push_back(
9895 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
9896 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9897 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
9898 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
9899 CombinedInfo.HasAttachPtr.push_back(false);
9900 CombinedInfo.Mappers.push_back(nullptr);
9901 }
9902 for (const LambdaCapture &LC : RD->captures()) {
9903 if (!LC.capturesVariable())
9904 continue;
9905 const VarDecl *VD = cast<VarDecl>(LC.getCapturedVar());
9906 if (LC.getCaptureKind() != LCK_ByRef && !VD->getType()->isPointerType())
9907 continue;
9908 auto It = Captures.find(VD);
9909 assert(It != Captures.end() && "Found lambda capture without field.");
9910 LValue VarLVal = CGF.EmitLValueForFieldInitialization(VDLVal, It->second);
9911 if (LC.getCaptureKind() == LCK_ByRef) {
9912 LValue VarLValVal = CGF.EmitLValueForField(VDLVal, It->second);
9913 LambdaPointers.try_emplace(VarLVal.getPointer(CGF),
9914 VDLVal.getPointer(CGF));
9915 CombinedInfo.Exprs.push_back(VD);
9916 CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF));
9917 CombinedInfo.DevicePtrDecls.push_back(nullptr);
9918 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
9919 CombinedInfo.Pointers.push_back(VarLValVal.getPointer(CGF));
9920 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
9921 CGF.getTypeSize(
9923 CGF.Int64Ty, /*isSigned=*/true));
9924 } else {
9925 RValue VarRVal = CGF.EmitLoadOfLValue(VarLVal, RD->getLocation());
9926 LambdaPointers.try_emplace(VarLVal.getPointer(CGF),
9927 VDLVal.getPointer(CGF));
9928 CombinedInfo.Exprs.push_back(VD);
9929 CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF));
9930 CombinedInfo.DevicePtrDecls.push_back(nullptr);
9931 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
9932 CombinedInfo.Pointers.push_back(VarRVal.getScalarVal());
9933 CombinedInfo.Sizes.push_back(llvm::ConstantInt::get(CGF.Int64Ty, 0));
9934 }
9935 CombinedInfo.Types.push_back(
9936 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
9937 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9938 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
9939 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
9940 CombinedInfo.HasAttachPtr.push_back(false);
9941 CombinedInfo.Mappers.push_back(nullptr);
9942 }
9943 }
9944
9945 /// Set correct indices for lambdas captures.
9946 void adjustMemberOfForLambdaCaptures(
9947 llvm::OpenMPIRBuilder &OMPBuilder,
9948 const llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers,
9949 MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
9950 MapFlagsArrayTy &Types) const {
9951 for (unsigned I = 0, E = Types.size(); I < E; ++I) {
9952 // Set correct member_of idx for all implicit lambda captures.
9953 if (Types[I] != (OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
9954 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9955 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
9956 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT))
9957 continue;
9958 llvm::Value *BasePtr = LambdaPointers.lookup(BasePointers[I]);
9959 assert(BasePtr && "Unable to find base lambda address.");
9960 int TgtIdx = -1;
9961 for (unsigned J = I; J > 0; --J) {
9962 unsigned Idx = J - 1;
9963 if (Pointers[Idx] != BasePtr)
9964 continue;
9965 TgtIdx = Idx;
9966 break;
9967 }
9968 assert(TgtIdx != -1 && "Unable to find parent lambda.");
9969 // All other current entries will be MEMBER_OF the combined entry
9970 // (except for PTR_AND_OBJ entries which do not have a placeholder value
9971 // 0xFFFF in the MEMBER_OF field).
9972 OpenMPOffloadMappingFlags MemberOfFlag =
9973 OMPBuilder.getMemberOfFlag(TgtIdx);
9974 OMPBuilder.setCorrectMemberOfFlag(Types[I], MemberOfFlag);
9975 }
9976 }
9977
9978 /// Populate component lists for non-lambda captured variables from map,
9979 /// is_device_ptr and has_device_addr clause info.
9980 void populateComponentListsForNonLambdaCaptureFromClauses(
9981 const ValueDecl *VD, MapDataArrayTy &DeclComponentLists,
9982 SmallVectorImpl<
9983 SmallVector<OMPClauseMappableExprCommon::MappableComponent, 8>>
9984 &StorageForImplicitlyAddedComponentLists) const {
9985 if (VD && LambdasMap.count(VD))
9986 return;
9987
9988 // For member fields list in is_device_ptr, store it in
9989 // DeclComponentLists for generating components info.
9991 auto It = DevPointersMap.find(VD);
9992 if (It != DevPointersMap.end())
9993 for (const auto &MCL : It->second)
9994 DeclComponentLists.emplace_back(MCL, OMPC_MAP_to, Unknown,
9995 /*IsImpicit = */ true, nullptr,
9996 nullptr);
9997 auto I = HasDevAddrsMap.find(VD);
9998 if (I != HasDevAddrsMap.end())
9999 for (const auto &MCL : I->second)
10000 DeclComponentLists.emplace_back(MCL, OMPC_MAP_tofrom, Unknown,
10001 /*IsImpicit = */ true, nullptr,
10002 nullptr);
10003 assert(isa<const OMPExecutableDirective *>(CurDir) &&
10004 "Expect a executable directive");
10005 const auto *CurExecDir = cast<const OMPExecutableDirective *>(CurDir);
10006 for (const auto *C : CurExecDir->getClausesOfKind<OMPMapClause>()) {
10007 const auto *EI = C->getVarRefs().begin();
10008 for (const auto L : C->decl_component_lists(VD)) {
10009 const ValueDecl *VDecl, *Mapper;
10010 // The Expression is not correct if the mapping is implicit
10011 const Expr *E = (C->getMapLoc().isValid()) ? *EI : nullptr;
10013 std::tie(VDecl, Components, Mapper) = L;
10014 assert(VDecl == VD && "We got information for the wrong declaration??");
10015 assert(!Components.empty() &&
10016 "Not expecting declaration with no component lists.");
10017 DeclComponentLists.emplace_back(Components, C->getMapType(),
10018 C->getMapTypeModifiers(),
10019 C->isImplicit(), Mapper, E);
10020 ++EI;
10021 }
10022 }
10023
10024 // For the target construct, if there's a map with a base-pointer that's
10025 // a member of an implicitly captured struct, of the current class,
10026 // we need to emit an implicit map on the pointer.
10027 if (isOpenMPTargetExecutionDirective(CurExecDir->getDirectiveKind()))
10028 addImplicitMapForAttachPtrBaseIfMemberOfCapturedVD(
10029 VD, DeclComponentLists, StorageForImplicitlyAddedComponentLists);
10030
10031 llvm::stable_sort(DeclComponentLists, [](const MapData &LHS,
10032 const MapData &RHS) {
10033 ArrayRef<OpenMPMapModifierKind> MapModifiers = std::get<2>(LHS);
10034 OpenMPMapClauseKind MapType = std::get<1>(RHS);
10035 bool HasPresent =
10036 llvm::is_contained(MapModifiers, clang::OMPC_MAP_MODIFIER_present);
10037 bool HasAllocs = MapType == OMPC_MAP_alloc;
10038 MapModifiers = std::get<2>(RHS);
10039 MapType = std::get<1>(LHS);
10040 bool HasPresentR =
10041 llvm::is_contained(MapModifiers, clang::OMPC_MAP_MODIFIER_present);
10042 bool HasAllocsR = MapType == OMPC_MAP_alloc;
10043 return (HasPresent && !HasPresentR) || (HasAllocs && !HasAllocsR);
10044 });
10045 }
10046
10047 /// On a target construct, if there's an implicit map on a struct, or that of
10048 /// this[:], and an explicit map with a member of that struct/class as the
10049 /// base-pointer, we need to make sure that base-pointer is implicitly mapped,
10050 /// to make sure we don't map the full struct/class. For example:
10051 ///
10052 /// \code
10053 /// struct S {
10054 /// int dummy[10000];
10055 /// int *p;
10056 /// void f1() {
10057 /// #pragma omp target map(p[0:1])
10058 /// (void)this;
10059 /// }
10060 /// }; S s;
10061 ///
10062 /// void f2() {
10063 /// #pragma omp target map(s.p[0:10])
10064 /// (void)s;
10065 /// }
10066 /// \endcode
10067 ///
10068 /// Only `this-p` and `s.p` should be mapped in the two cases above.
10069 //
10070 // OpenMP 6.0: 7.9.6 map clause, pg 285
10071 // If a list item with an implicitly determined data-mapping attribute does
10072 // not have any corresponding storage in the device data environment prior to
10073 // a task encountering the construct associated with the map clause, and one
10074 // or more contiguous parts of the original storage are either list items or
10075 // base pointers to list items that are explicitly mapped on the construct,
10076 // only those parts of the original storage will have corresponding storage in
10077 // the device data environment as a result of the map clauses on the
10078 // construct.
10079 void addImplicitMapForAttachPtrBaseIfMemberOfCapturedVD(
10080 const ValueDecl *CapturedVD, MapDataArrayTy &DeclComponentLists,
10081 SmallVectorImpl<
10082 SmallVector<OMPClauseMappableExprCommon::MappableComponent, 8>>
10083 &ComponentVectorStorage) const {
10084 bool IsThisCapture = CapturedVD == nullptr;
10085
10086 for (const auto &ComponentsAndAttachPtr : AttachPtrExprMap) {
10088 ComponentsWithAttachPtr = ComponentsAndAttachPtr.first;
10089 const Expr *AttachPtrExpr = ComponentsAndAttachPtr.second;
10090 if (!AttachPtrExpr)
10091 continue;
10092
10093 const auto *ME = dyn_cast<MemberExpr>(AttachPtrExpr);
10094 if (!ME)
10095 continue;
10096
10097 const Expr *Base = ME->getBase()->IgnoreParenImpCasts();
10098
10099 // If we are handling a "this" capture, then we are looking for
10100 // attach-ptrs of form `this->p`, either explicitly or implicitly.
10101 if (IsThisCapture && !ME->isImplicitCXXThis() && !isa<CXXThisExpr>(Base))
10102 continue;
10103
10104 if (!IsThisCapture && (!isa<DeclRefExpr>(Base) ||
10105 cast<DeclRefExpr>(Base)->getDecl() != CapturedVD))
10106 continue;
10107
10108 // For non-this captures, we are looking for attach-ptrs of form
10109 // `s.p`.
10110 // For non-this captures, we are looking for attach-ptrs like `s.p`.
10111 if (!IsThisCapture && (ME->isArrow() || !isa<DeclRefExpr>(Base) ||
10112 cast<DeclRefExpr>(Base)->getDecl() != CapturedVD))
10113 continue;
10114
10115 // Check if we have an existing map on either:
10116 // this[:], s, this->p, or s.p, in which case, we don't need to add
10117 // an implicit one for the attach-ptr s.p/this->p.
10118 bool FoundExistingMap = false;
10119 for (const MapData &ExistingL : DeclComponentLists) {
10121 ExistingComponents = std::get<0>(ExistingL);
10122
10123 if (ExistingComponents.empty())
10124 continue;
10125
10126 // First check if we have a map like map(this->p) or map(s.p).
10127 const auto &FirstComponent = ExistingComponents.front();
10128 const Expr *FirstExpr = FirstComponent.getAssociatedExpression();
10129
10130 if (!FirstExpr)
10131 continue;
10132
10133 // First check if we have a map like map(this->p) or map(s.p).
10134 if (AttachPtrComparator.areEqual(FirstExpr, AttachPtrExpr)) {
10135 FoundExistingMap = true;
10136 break;
10137 }
10138
10139 // Check if we have a map like this[0:1]
10140 if (IsThisCapture) {
10141 if (const auto *OASE = dyn_cast<ArraySectionExpr>(FirstExpr)) {
10142 if (isa<CXXThisExpr>(OASE->getBase()->IgnoreParenImpCasts())) {
10143 FoundExistingMap = true;
10144 break;
10145 }
10146 }
10147 continue;
10148 }
10149
10150 // When the attach-ptr is something like `s.p`, check if
10151 // `s` itself is mapped explicitly.
10152 if (const auto *DRE = dyn_cast<DeclRefExpr>(FirstExpr)) {
10153 if (DRE->getDecl() == CapturedVD) {
10154 FoundExistingMap = true;
10155 break;
10156 }
10157 }
10158 }
10159
10160 if (FoundExistingMap)
10161 continue;
10162
10163 // If no base map is found, we need to create an implicit map for the
10164 // attach-pointer expr.
10165
10166 ComponentVectorStorage.emplace_back();
10167 auto &AttachPtrComponents = ComponentVectorStorage.back();
10168
10170 bool SeenAttachPtrComponent = false;
10171 // For creating a map on the attach-ptr `s.p/this->p`, we copy all
10172 // components from the component-list which has `s.p/this->p`
10173 // as the attach-ptr, starting from the component which matches
10174 // `s.p/this->p`. This way, we'll have component-lists of
10175 // `s.p` -> `s`, and `this->p` -> `this`.
10176 for (size_t i = 0; i < ComponentsWithAttachPtr.size(); ++i) {
10177 const auto &Component = ComponentsWithAttachPtr[i];
10178 const Expr *ComponentExpr = Component.getAssociatedExpression();
10179
10180 if (!SeenAttachPtrComponent && ComponentExpr != AttachPtrExpr)
10181 continue;
10182 SeenAttachPtrComponent = true;
10183
10184 AttachPtrComponents.emplace_back(Component.getAssociatedExpression(),
10185 Component.getAssociatedDeclaration(),
10186 Component.isNonContiguous());
10187 }
10188 assert(!AttachPtrComponents.empty() &&
10189 "Could not populate component-lists for mapping attach-ptr");
10190
10191 DeclComponentLists.emplace_back(
10192 AttachPtrComponents, OMPC_MAP_tofrom, Unknown,
10193 /*IsImplicit=*/true, /*mapper=*/nullptr, AttachPtrExpr);
10194 }
10195 }
10196
10197 /// For a capture that has an associated clause, generate the base pointers,
10198 /// section pointers, sizes, map types, and mappers (all included in
10199 /// \a CurCaptureVarInfo).
10200 void generateInfoForCaptureFromClauseInfo(
10201 const MapDataArrayTy &DeclComponentListsFromClauses,
10202 const CapturedStmt::Capture *Cap, llvm::Value *Arg,
10203 MapCombinedInfoTy &CurCaptureVarInfo, llvm::OpenMPIRBuilder &OMPBuilder,
10204 unsigned OffsetForMemberOfFlag) const {
10205 assert(!Cap->capturesVariableArrayType() &&
10206 "Not expecting to generate map info for a variable array type!");
10207
10208 // We need to know when we generating information for the first component
10209 const ValueDecl *VD = Cap->capturesThis()
10210 ? nullptr
10211 : Cap->getCapturedVar()->getCanonicalDecl();
10212
10213 // for map(to: lambda): skip here, processing it in
10214 // generateDefaultMapInfo
10215 if (LambdasMap.count(VD))
10216 return;
10217
10218 // If this declaration appears in a is_device_ptr clause we just have to
10219 // pass the pointer by value. If it is a reference to a declaration, we just
10220 // pass its value.
10221 if (VD && (DevPointersMap.count(VD) || HasDevAddrsMap.count(VD))) {
10222 CurCaptureVarInfo.Exprs.push_back(VD);
10223 CurCaptureVarInfo.BasePointers.emplace_back(Arg);
10224 CurCaptureVarInfo.DevicePtrDecls.emplace_back(VD);
10225 CurCaptureVarInfo.DevicePointers.emplace_back(DeviceInfoTy::Pointer);
10226 CurCaptureVarInfo.Pointers.push_back(Arg);
10227 CurCaptureVarInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
10228 CGF.getTypeSize(CGF.getContext().VoidPtrTy), CGF.Int64Ty,
10229 /*isSigned=*/true));
10230 CurCaptureVarInfo.Types.push_back(
10231 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
10232 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
10233 CurCaptureVarInfo.HasAttachPtr.push_back(false);
10234 CurCaptureVarInfo.Mappers.push_back(nullptr);
10235 return;
10236 }
10237
10238 auto GenerateInfoForComponentLists =
10239 [&](ArrayRef<MapData> DeclComponentListsFromClauses,
10240 bool IsEligibleForTargetParamFlag) {
10241 MapCombinedInfoTy CurInfoForComponentLists;
10242 StructRangeInfoTy PartialStruct;
10243 AttachInfoTy AttachInfo;
10244
10245 if (DeclComponentListsFromClauses.empty())
10246 return;
10247
10248 generateInfoForCaptureFromComponentLists(
10249 VD, DeclComponentListsFromClauses, CurInfoForComponentLists,
10250 PartialStruct, AttachInfo, IsEligibleForTargetParamFlag);
10251
10252 // If there is an entry in PartialStruct it means we have a
10253 // struct with individual members mapped. Emit an extra combined
10254 // entry.
10255 if (PartialStruct.Base.isValid()) {
10256 CurCaptureVarInfo.append(PartialStruct.PreliminaryMapData);
10257 emitCombinedEntry(
10258 CurCaptureVarInfo, CurInfoForComponentLists.Types,
10259 PartialStruct, AttachInfo, Cap->capturesThis(), OMPBuilder,
10260 /*VD=*/nullptr, OffsetForMemberOfFlag,
10261 /*NotTargetParams*/ !IsEligibleForTargetParamFlag);
10262 }
10263
10264 // We do the appends to get the entries in the following order:
10265 // combined-entry -> individual-field-entries -> attach-entry,
10266 CurCaptureVarInfo.append(CurInfoForComponentLists);
10267 if (AttachInfo.isValid())
10268 emitAttachEntry(CGF, CurCaptureVarInfo, AttachInfo);
10269 };
10270
10271 // Group component lists by their AttachPtrExpr and process them in order
10272 // of increasing complexity (nullptr first, then simple expressions like p,
10273 // then more complex ones like p[0], etc.)
10274 //
10275 // This ensure that we:
10276 // * handle maps that can contribute towards setting the kernel argument,
10277 // (e.g. map(ps), or map(ps[0])), before any that cannot (e.g. ps->pt->d).
10278 // * allocate a single contiguous storage for all exprs with the same
10279 // captured var and having the same attach-ptr.
10280 //
10281 // Example: The map clauses below should be handled grouped together based
10282 // on their attachable-base-pointers:
10283 // map-clause | attachable-base-pointer
10284 // --------------------------+------------------------
10285 // map(p, ps) | nullptr
10286 // map(p[0]) | p
10287 // map(p[0]->b, p[0]->c) | p[0]
10288 // map(ps->d, ps->e, ps->pt) | ps
10289 // map(ps->pt->d, ps->pt->e) | ps->pt
10290
10291 // First, collect all MapData entries with their attach-ptr exprs.
10292 SmallVector<std::pair<const Expr *, MapData>, 16> AttachPtrMapDataPairs;
10293
10294 for (const MapData &L : DeclComponentListsFromClauses) {
10296 std::get<0>(L);
10297 const Expr *AttachPtrExpr = getAttachPtrExpr(Components);
10298 AttachPtrMapDataPairs.emplace_back(AttachPtrExpr, L);
10299 }
10300
10301 // Next, sort by increasing order of their complexity.
10302 llvm::stable_sort(AttachPtrMapDataPairs,
10303 [this](const auto &LHS, const auto &RHS) {
10304 return AttachPtrComparator(LHS.first, RHS.first);
10305 });
10306
10307 bool NoDefaultMappingDoneForVD = CurCaptureVarInfo.BasePointers.empty();
10308 bool IsFirstGroup = true;
10309
10310 // And finally, process them all in order, grouping those with
10311 // equivalent attach-ptr exprs together.
10312 auto *It = AttachPtrMapDataPairs.begin();
10313 while (It != AttachPtrMapDataPairs.end()) {
10314 const Expr *AttachPtrExpr = It->first;
10315
10316 MapDataArrayTy GroupLists;
10317 while (It != AttachPtrMapDataPairs.end() &&
10318 (It->first == AttachPtrExpr ||
10319 AttachPtrComparator.areEqual(It->first, AttachPtrExpr))) {
10320 GroupLists.push_back(It->second);
10321 ++It;
10322 }
10323 assert(!GroupLists.empty() && "GroupLists should not be empty");
10324
10325 // Determine if this group of component-lists is eligible for TARGET_PARAM
10326 // flag. Only the first group processed should be eligible, and only if no
10327 // default mapping was done.
10328 bool IsEligibleForTargetParamFlag =
10329 IsFirstGroup && NoDefaultMappingDoneForVD;
10330
10331 GenerateInfoForComponentLists(GroupLists, IsEligibleForTargetParamFlag);
10332 IsFirstGroup = false;
10333 }
10334 }
10335
10336 /// Generate the base pointers, section pointers, sizes, map types, and
10337 /// mappers associated to \a DeclComponentLists for a given capture
10338 /// \a VD (all included in \a CurComponentListInfo).
10339 void generateInfoForCaptureFromComponentLists(
10340 const ValueDecl *VD, ArrayRef<MapData> DeclComponentLists,
10341 MapCombinedInfoTy &CurComponentListInfo, StructRangeInfoTy &PartialStruct,
10342 AttachInfoTy &AttachInfo, bool IsListEligibleForTargetParamFlag) const {
10343 // Find overlapping elements (including the offset from the base element).
10344 llvm::SmallDenseMap<
10345 const MapData *,
10346 llvm::SmallVector<
10348 4>
10349 OverlappedData;
10350 size_t Count = 0;
10351 for (const MapData &L : DeclComponentLists) {
10353 OpenMPMapClauseKind MapType;
10354 ArrayRef<OpenMPMapModifierKind> MapModifiers;
10355 bool IsImplicit;
10356 const ValueDecl *Mapper;
10357 const Expr *VarRef;
10358 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =
10359 L;
10360 ++Count;
10361 for (const MapData &L1 : ArrayRef(DeclComponentLists).slice(Count)) {
10363 std::tie(Components1, MapType, MapModifiers, IsImplicit, Mapper,
10364 VarRef) = L1;
10365 auto CI = Components.rbegin();
10366 auto CE = Components.rend();
10367 auto SI = Components1.rbegin();
10368 auto SE = Components1.rend();
10369 for (; CI != CE && SI != SE; ++CI, ++SI) {
10370 if (CI->getAssociatedExpression()->getStmtClass() !=
10371 SI->getAssociatedExpression()->getStmtClass())
10372 break;
10373 // Are we dealing with different variables/fields?
10374 if (CI->getAssociatedDeclaration() != SI->getAssociatedDeclaration())
10375 break;
10376 }
10377 // Found overlapping if, at least for one component, reached the head
10378 // of the components list.
10379 if (CI == CE || SI == SE) {
10380 // Ignore it if it is the same component.
10381 if (CI == CE && SI == SE)
10382 continue;
10383 const auto It = (SI == SE) ? CI : SI;
10384 // If one component is a pointer and another one is a kind of
10385 // dereference of this pointer (array subscript, section, dereference,
10386 // etc.), it is not an overlapping.
10387 // Same, if one component is a base and another component is a
10388 // dereferenced pointer memberexpr with the same base.
10389 if (!isa<MemberExpr>(It->getAssociatedExpression()) ||
10390 (std::prev(It)->getAssociatedDeclaration() &&
10391 std::prev(It)
10392 ->getAssociatedDeclaration()
10393 ->getType()
10394 ->isPointerType()) ||
10395 (It->getAssociatedDeclaration() &&
10396 It->getAssociatedDeclaration()->getType()->isPointerType() &&
10397 std::next(It) != CE && std::next(It) != SE))
10398 continue;
10399 const MapData &BaseData = CI == CE ? L : L1;
10401 SI == SE ? Components : Components1;
10402 OverlappedData[&BaseData].push_back(SubData);
10403 }
10404 }
10405 }
10406 // Sort the overlapped elements for each item.
10407 llvm::SmallVector<const FieldDecl *, 4> Layout;
10408 if (!OverlappedData.empty()) {
10409 const Type *BaseType = VD->getType().getCanonicalType().getTypePtr();
10410 const Type *OrigType = BaseType->getPointeeOrArrayElementType();
10411 while (BaseType != OrigType) {
10412 BaseType = OrigType->getCanonicalTypeInternal().getTypePtr();
10413 OrigType = BaseType->getPointeeOrArrayElementType();
10414 }
10415
10416 if (const auto *CRD = BaseType->getAsCXXRecordDecl())
10417 getPlainLayout(CRD, Layout, /*AsBase=*/false);
10418 else {
10419 const auto *RD = BaseType->getAsRecordDecl();
10420 Layout.append(RD->field_begin(), RD->field_end());
10421 }
10422 }
10423 for (auto &Pair : OverlappedData) {
10424 llvm::stable_sort(
10425 Pair.getSecond(),
10426 [&Layout](
10429 Second) {
10430 auto CI = First.rbegin();
10431 auto CE = First.rend();
10432 auto SI = Second.rbegin();
10433 auto SE = Second.rend();
10434 for (; CI != CE && SI != SE; ++CI, ++SI) {
10435 if (CI->getAssociatedExpression()->getStmtClass() !=
10436 SI->getAssociatedExpression()->getStmtClass())
10437 break;
10438 // Are we dealing with different variables/fields?
10439 if (CI->getAssociatedDeclaration() !=
10440 SI->getAssociatedDeclaration())
10441 break;
10442 }
10443
10444 // Lists contain the same elements.
10445 if (CI == CE && SI == SE)
10446 return false;
10447
10448 // List with less elements is less than list with more elements.
10449 if (CI == CE || SI == SE)
10450 return CI == CE;
10451
10452 const auto *FD1 = cast<FieldDecl>(CI->getAssociatedDeclaration());
10453 const auto *FD2 = cast<FieldDecl>(SI->getAssociatedDeclaration());
10454 if (FD1->getParent() == FD2->getParent())
10455 return FD1->getFieldIndex() < FD2->getFieldIndex();
10456 const auto *It =
10457 llvm::find_if(Layout, [FD1, FD2](const FieldDecl *FD) {
10458 return FD == FD1 || FD == FD2;
10459 });
10460 return *It == FD1;
10461 });
10462 }
10463
10464 // Associated with a capture, because the mapping flags depend on it.
10465 // Go through all of the elements with the overlapped elements.
10466 bool AddTargetParamFlag = IsListEligibleForTargetParamFlag;
10467 MapCombinedInfoTy StructBaseCombinedInfo;
10468 for (const auto &Pair : OverlappedData) {
10469 const MapData &L = *Pair.getFirst();
10471 OpenMPMapClauseKind MapType;
10472 ArrayRef<OpenMPMapModifierKind> MapModifiers;
10473 bool IsImplicit;
10474 const ValueDecl *Mapper;
10475 const Expr *VarRef;
10476 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =
10477 L;
10478 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
10479 OverlappedComponents = Pair.getSecond();
10480 generateInfoForComponentList(
10481 MapType, MapModifiers, {}, Components, CurComponentListInfo,
10482 StructBaseCombinedInfo, PartialStruct, AttachInfo, AddTargetParamFlag,
10483 IsImplicit, /*GenerateAllInfoForClauses*/ false, Mapper,
10484 /*ForDeviceAddr=*/false, VD, VarRef, OverlappedComponents);
10485 AddTargetParamFlag = false;
10486 }
10487 // Go through other elements without overlapped elements.
10488 for (const MapData &L : DeclComponentLists) {
10490 OpenMPMapClauseKind MapType;
10491 ArrayRef<OpenMPMapModifierKind> MapModifiers;
10492 bool IsImplicit;
10493 const ValueDecl *Mapper;
10494 const Expr *VarRef;
10495 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =
10496 L;
10497 auto It = OverlappedData.find(&L);
10498 if (It == OverlappedData.end())
10499 generateInfoForComponentList(
10500 MapType, MapModifiers, {}, Components, CurComponentListInfo,
10501 StructBaseCombinedInfo, PartialStruct, AttachInfo,
10502 AddTargetParamFlag, IsImplicit, /*GenerateAllInfoForClauses*/ false,
10503 Mapper, /*ForDeviceAddr=*/false, VD, VarRef,
10504 /*OverlappedElements*/ {});
10505 AddTargetParamFlag = false;
10506 }
10507 }
10508
10509 /// Check if a variable should be treated as firstprivate due to explicit
10510 /// firstprivate clause or defaultmap(firstprivate:...).
10511 bool isEffectivelyFirstprivate(const VarDecl *VD, QualType Type) const {
10512 // Check explicit firstprivate clauses (not implicit from defaultmap)
10513 auto I = FirstPrivateDecls.find(VD);
10514 if (I != FirstPrivateDecls.end() && !I->getSecond())
10515 return true; // Explicit firstprivate only
10516
10517 // Check defaultmap(firstprivate:scalar) for scalar types
10518 if (DefaultmapFirstprivateKinds.count(OMPC_DEFAULTMAP_scalar)) {
10519 if (Type->isScalarType())
10520 return true;
10521 }
10522
10523 // Check defaultmap(firstprivate:pointer) for pointer types
10524 if (DefaultmapFirstprivateKinds.count(OMPC_DEFAULTMAP_pointer)) {
10525 if (Type->isAnyPointerType())
10526 return true;
10527 }
10528
10529 // Check defaultmap(firstprivate:aggregate) for aggregate types
10530 if (DefaultmapFirstprivateKinds.count(OMPC_DEFAULTMAP_aggregate)) {
10531 if (Type->isAggregateType())
10532 return true;
10533 }
10534
10535 // Check defaultmap(firstprivate:all) for all types
10536 return DefaultmapFirstprivateKinds.count(OMPC_DEFAULTMAP_all);
10537 }
10538
10539 /// Generate the default map information for a given capture \a CI,
10540 /// record field declaration \a RI and captured value \a CV.
10541 void generateDefaultMapInfo(const CapturedStmt::Capture &CI,
10542 const FieldDecl &RI, llvm::Value *CV,
10543 MapCombinedInfoTy &CombinedInfo) const {
10544 bool IsImplicit = true;
10545 // Do the default mapping.
10546 if (CI.capturesThis()) {
10547 CombinedInfo.Exprs.push_back(nullptr);
10548 CombinedInfo.BasePointers.push_back(CV);
10549 CombinedInfo.DevicePtrDecls.push_back(nullptr);
10550 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
10551 CombinedInfo.Pointers.push_back(CV);
10552 const auto *PtrTy = cast<PointerType>(RI.getType().getTypePtr());
10553 CombinedInfo.Sizes.push_back(
10554 CGF.Builder.CreateIntCast(CGF.getTypeSize(PtrTy->getPointeeType()),
10555 CGF.Int64Ty, /*isSigned=*/true));
10556 // Default map type.
10557 CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_TO |
10558 OpenMPOffloadMappingFlags::OMP_MAP_FROM);
10559 } else if (CI.capturesVariableByCopy()) {
10560 const VarDecl *VD = CI.getCapturedVar();
10561 CombinedInfo.Exprs.push_back(VD->getCanonicalDecl());
10562 CombinedInfo.BasePointers.push_back(CV);
10563 CombinedInfo.DevicePtrDecls.push_back(nullptr);
10564 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
10565 CombinedInfo.Pointers.push_back(CV);
10566 bool IsFirstprivate =
10567 isEffectivelyFirstprivate(VD, RI.getType().getNonReferenceType());
10568
10569 if (!RI.getType()->isAnyPointerType()) {
10570 // We have to signal to the runtime captures passed by value that are
10571 // not pointers.
10572 CombinedInfo.Types.push_back(
10573 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
10574 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
10575 CGF.getTypeSize(RI.getType()), CGF.Int64Ty, /*isSigned=*/true));
10576 } else if (IsFirstprivate) {
10577 // Firstprivate pointers should be passed by value (as literals)
10578 // without performing a present table lookup at runtime.
10579 CombinedInfo.Types.push_back(
10580 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
10581 // Use zero size for pointer literals (just passing the pointer value)
10582 CombinedInfo.Sizes.push_back(llvm::Constant::getNullValue(CGF.Int64Ty));
10583 } else {
10584 // Pointers are implicitly mapped with a zero size and no flags
10585 // (other than first map that is added for all implicit maps).
10586 CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_NONE);
10587 CombinedInfo.Sizes.push_back(llvm::Constant::getNullValue(CGF.Int64Ty));
10588 }
10589 auto I = FirstPrivateDecls.find(VD);
10590 if (I != FirstPrivateDecls.end())
10591 IsImplicit = I->getSecond();
10592 } else {
10593 assert(CI.capturesVariable() && "Expected captured reference.");
10594 const auto *PtrTy = cast<ReferenceType>(RI.getType().getTypePtr());
10595 QualType ElementType = PtrTy->getPointeeType();
10596 const VarDecl *VD = CI.getCapturedVar();
10597 bool IsFirstprivate = isEffectivelyFirstprivate(VD, ElementType);
10598 CombinedInfo.Exprs.push_back(VD->getCanonicalDecl());
10599 CombinedInfo.BasePointers.push_back(CV);
10600 CombinedInfo.DevicePtrDecls.push_back(nullptr);
10601 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
10602
10603 // For firstprivate pointers, pass by value instead of dereferencing
10604 if (IsFirstprivate && ElementType->isAnyPointerType()) {
10605 // Treat as a literal value (pass the pointer value itself)
10606 CombinedInfo.Pointers.push_back(CV);
10607 // Use zero size for pointer literals
10608 CombinedInfo.Sizes.push_back(llvm::Constant::getNullValue(CGF.Int64Ty));
10609 CombinedInfo.Types.push_back(
10610 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
10611 } else {
10612 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
10613 CGF.getTypeSize(ElementType), CGF.Int64Ty, /*isSigned=*/true));
10614 // The default map type for a scalar/complex type is 'to' because by
10615 // default the value doesn't have to be retrieved. For an aggregate
10616 // type, the default is 'tofrom'.
10617 CombinedInfo.Types.push_back(getMapModifiersForPrivateClauses(CI));
10618 CombinedInfo.Pointers.push_back(CV);
10619 }
10620 auto I = FirstPrivateDecls.find(VD);
10621 if (I != FirstPrivateDecls.end())
10622 IsImplicit = I->getSecond();
10623 }
10624 // Every default map produces a single argument which is a target parameter.
10625 CombinedInfo.Types.back() |=
10626 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
10627
10628 // Add flag stating this is an implicit map.
10629 if (IsImplicit)
10630 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
10631
10632 CombinedInfo.HasAttachPtr.push_back(false);
10633 // No user-defined mapper for default mapping.
10634 CombinedInfo.Mappers.push_back(nullptr);
10635 }
10636};
10637} // anonymous namespace
10638
10639// Try to extract the base declaration from a `this->x` expression if possible.
10641 if (!E)
10642 return nullptr;
10643
10644 if (const auto *OASE = dyn_cast<ArraySectionExpr>(E->IgnoreParenCasts()))
10645 if (const MemberExpr *ME =
10646 dyn_cast<MemberExpr>(OASE->getBase()->IgnoreParenImpCasts()))
10647 return ME->getMemberDecl();
10648 return nullptr;
10649}
10650
10651/// Emit a string constant containing the names of the values mapped to the
10652/// offloading runtime library.
10653static llvm::Constant *
10654emitMappingInformation(CodeGenFunction &CGF, llvm::OpenMPIRBuilder &OMPBuilder,
10655 MappableExprsHandler::MappingExprInfo &MapExprs) {
10656
10657 uint32_t SrcLocStrSize;
10658 if (!MapExprs.getMapDecl() && !MapExprs.getMapExpr())
10659 return OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
10660
10661 SourceLocation Loc;
10662 if (!MapExprs.getMapDecl() && MapExprs.getMapExpr()) {
10663 if (const ValueDecl *VD = getDeclFromThisExpr(MapExprs.getMapExpr()))
10664 Loc = VD->getLocation();
10665 else
10666 Loc = MapExprs.getMapExpr()->getExprLoc();
10667 } else {
10668 Loc = MapExprs.getMapDecl()->getLocation();
10669 }
10670
10671 std::string ExprName;
10672 if (MapExprs.getMapExpr()) {
10674 llvm::raw_string_ostream OS(ExprName);
10675 MapExprs.getMapExpr()->printPretty(OS, nullptr, P);
10676 } else {
10677 ExprName = MapExprs.getMapDecl()->getNameAsString();
10678 }
10679
10680 std::string FileName;
10682 if (auto *DbgInfo = CGF.getDebugInfo())
10683 FileName = DbgInfo->remapDIPath(PLoc.getFilename());
10684 else
10685 FileName = PLoc.getFilename();
10686 return OMPBuilder.getOrCreateSrcLocStr(FileName, ExprName, PLoc.getLine(),
10687 PLoc.getColumn(), SrcLocStrSize);
10688}
10689/// Emit the arrays used to pass the captures and map information to the
10690/// offloading runtime library. If there is no map or capture information,
10691/// return nullptr by reference.
10693 CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
10694 CGOpenMPRuntime::TargetDataInfo &Info, llvm::OpenMPIRBuilder &OMPBuilder,
10695 bool IsNonContiguous = false, bool ForEndCall = false) {
10696 CodeGenModule &CGM = CGF.CGM;
10697
10698 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
10699 InsertPointTy AllocaIP(CGF.AllocaInsertPt->getIterator());
10700 InsertPointTy CodeGenIP(CGF.Builder.GetInsertPoint());
10701
10702 auto DeviceAddrCB = [&](unsigned int I, llvm::Value *NewDecl) {
10703 if (const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
10704 Info.CaptureDeviceAddrMap.try_emplace(DevVD, NewDecl);
10705 }
10706 };
10707
10708 auto CustomMapperCB = [&](unsigned int I) {
10709 llvm::Function *MFunc = nullptr;
10710 if (CombinedInfo.Mappers[I]) {
10711 Info.HasMapper = true;
10713 cast<OMPDeclareMapperDecl>(CombinedInfo.Mappers[I]));
10714 }
10715 return MFunc;
10716 };
10717 cantFail(OMPBuilder.emitOffloadingArraysAndArgs(
10718 AllocaIP, CodeGenIP, Info, Info.RTArgs, CombinedInfo, CustomMapperCB,
10719 IsNonContiguous, ForEndCall, DeviceAddrCB));
10720}
10721
10722/// Check for inner distribute directive.
10723static const OMPExecutableDirective *
10725 const auto *CS = D.getInnermostCapturedStmt();
10726 const auto *Body =
10727 CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
10728 const Stmt *ChildStmt =
10730
10731 if (const auto *NestedDir =
10732 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
10733 OpenMPDirectiveKind DKind = NestedDir->getDirectiveKind();
10734 switch (D.getDirectiveKind()) {
10735 case OMPD_target:
10736 // For now, treat 'target' with nested 'teams loop' as if it's
10737 // distributed (target teams distribute).
10738 if (isOpenMPDistributeDirective(DKind) || DKind == OMPD_teams_loop)
10739 return NestedDir;
10740 if (DKind == OMPD_teams) {
10741 Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers(
10742 /*IgnoreCaptured=*/true);
10743 if (!Body)
10744 return nullptr;
10745 ChildStmt = CGOpenMPSIMDRuntime::getSingleCompoundChild(Ctx, Body);
10746 if (const auto *NND =
10747 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
10748 DKind = NND->getDirectiveKind();
10749 if (isOpenMPDistributeDirective(DKind))
10750 return NND;
10751 }
10752 }
10753 return nullptr;
10754 case OMPD_target_teams:
10755 if (isOpenMPDistributeDirective(DKind))
10756 return NestedDir;
10757 return nullptr;
10758 case OMPD_target_parallel:
10759 case OMPD_target_simd:
10760 case OMPD_target_loop:
10761 case OMPD_target_parallel_for:
10762 case OMPD_target_parallel_for_simd:
10763 return nullptr;
10764 case OMPD_target_teams_distribute:
10765 case OMPD_target_teams_distribute_simd:
10766 case OMPD_target_teams_distribute_parallel_for:
10767 case OMPD_target_teams_distribute_parallel_for_simd:
10768 case OMPD_parallel:
10769 case OMPD_for:
10770 case OMPD_parallel_for:
10771 case OMPD_parallel_master:
10772 case OMPD_parallel_sections:
10773 case OMPD_for_simd:
10774 case OMPD_parallel_for_simd:
10775 case OMPD_cancel:
10776 case OMPD_cancellation_point:
10777 case OMPD_ordered_standalone:
10778 case OMPD_ordered_blockassoc:
10779 case OMPD_threadprivate:
10780 case OMPD_allocate:
10781 case OMPD_task:
10782 case OMPD_simd:
10783 case OMPD_tile:
10784 case OMPD_unroll:
10785 case OMPD_sections:
10786 case OMPD_section:
10787 case OMPD_single:
10788 case OMPD_master:
10789 case OMPD_critical:
10790 case OMPD_taskyield:
10791 case OMPD_barrier:
10792 case OMPD_taskwait:
10793 case OMPD_taskgroup:
10794 case OMPD_atomic:
10795 case OMPD_flush:
10796 case OMPD_depobj:
10797 case OMPD_scan:
10798 case OMPD_teams:
10799 case OMPD_target_data:
10800 case OMPD_target_exit_data:
10801 case OMPD_target_enter_data:
10802 case OMPD_distribute:
10803 case OMPD_distribute_simd:
10804 case OMPD_distribute_parallel_for:
10805 case OMPD_distribute_parallel_for_simd:
10806 case OMPD_teams_distribute:
10807 case OMPD_teams_distribute_simd:
10808 case OMPD_teams_distribute_parallel_for:
10809 case OMPD_teams_distribute_parallel_for_simd:
10810 case OMPD_target_update:
10811 case OMPD_declare_simd:
10812 case OMPD_declare_variant:
10813 case OMPD_begin_declare_variant:
10814 case OMPD_end_declare_variant:
10815 case OMPD_declare_target:
10816 case OMPD_end_declare_target:
10817 case OMPD_declare_reduction:
10818 case OMPD_declare_mapper:
10819 case OMPD_taskloop:
10820 case OMPD_taskloop_simd:
10821 case OMPD_master_taskloop:
10822 case OMPD_master_taskloop_simd:
10823 case OMPD_parallel_master_taskloop:
10824 case OMPD_parallel_master_taskloop_simd:
10825 case OMPD_requires:
10826 case OMPD_metadirective:
10827 case OMPD_unknown:
10828 default:
10829 llvm_unreachable("Unexpected directive.");
10830 }
10831 }
10832
10833 return nullptr;
10834}
10835
10836/// Emit the user-defined mapper function. The code generation follows the
10837/// pattern in the example below.
10838/// \code
10839/// void .omp_mapper.<type_name>.<mapper_id>.(void *rt_mapper_handle,
10840/// void *base, void *begin,
10841/// int64_t size, int64_t type,
10842/// void *name = nullptr) {
10843/// // Allocate space for an array section first.
10844/// if ((size > 1 || (base != begin)) && !maptype.IsDelete)
10845/// __tgt_push_mapper_component(rt_mapper_handle, base, begin,
10846/// size*sizeof(Ty), clearToFromMember(type));
10847/// // Map members.
10848/// for (unsigned i = 0; i < size; i++) {
10849/// N = __tgt_mapper_num_components(rt_mapper_handle);
10850/// // For each component specified by this mapper:
10851/// for (auto c : begin[i]->all_components) {
10852/// // MEMBER_OF grouping: tie this component to the current array element
10853/// // (component N) by adding N<<48. Exceptions:
10854/// // - ATTACH entries are not members of any struct storage range.
10855/// // - Pointee entries (reached via a pointer member) occupy separate
10856/// // storage; their inner MEMBER_OF bits are shifted by N instead.
10857/// if (c.isAttach() || c.isPointee())
10858/// member_type = c.arg_type + (c.hasInnerMemberOf() ? N<<48 : 0);
10859/// else
10860/// member_type = c.arg_type + N<<48;
10861/// // Map-type-modifying bits (ALWAYS, DELETE, CLOSE) from the outer map
10862/// // clause are propagated to each component, except ATTACH entries
10863/// // (ATTACH|ALWAYS is reserved for attach(always), and other modifier
10864/// // bits have no meaning for ATTACH). PRESENT is additionally
10865/// // propagated to components with HasAttachPtr (the pointee data) at
10866/// // OpenMP >= 6.0.
10867/// present_bit = (v60 && c.hasAttachPtr()) ? PRESENT : 0;
10868/// imported_modifier_bits =
10869/// type & (ALWAYS | DELETE | CLOSE | present_bit);
10870/// effective_type = c.isAttach() ? member_type
10871/// : member_type | imported_modifier_bits;
10872/// if (c.hasMapper())
10873/// (*c.Mapper())(rt_mapper_handle, c.arg_base, c.arg_begin, c.arg_size,
10874/// effective_type, c.arg_name);
10875/// else
10876/// __tgt_push_mapper_component(rt_mapper_handle, c.arg_base,
10877/// c.arg_begin, c.arg_size, effective_type,
10878/// c.arg_name);
10879/// }
10880/// }
10881/// // Delete the array section.
10882/// if (size > 1 && maptype.IsDelete)
10883/// __tgt_push_mapper_component(rt_mapper_handle, base, begin,
10884/// size*sizeof(Ty), clearToFromMember(type));
10885/// }
10886/// \endcode
10888 CodeGenFunction *CGF) {
10889 if (UDMMap.count(D) > 0)
10890 return;
10891 ASTContext &C = CGM.getContext();
10892 QualType Ty = D->getType();
10893 auto *MapperVarDecl =
10895 CharUnits ElementSize = C.getTypeSizeInChars(Ty);
10896 llvm::Type *ElemTy = CGM.getTypes().ConvertTypeForMem(Ty);
10897
10898 CodeGenFunction MapperCGF(CGM);
10899 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
10900 auto PrivatizeAndGenMapInfoCB =
10901 [&](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP, llvm::Value *PtrPHI,
10902 llvm::Value *BeginArg) -> llvm::OpenMPIRBuilder::MapInfosTy & {
10903 MapperCGF.Builder.restoreIP(CodeGenIP);
10904
10905 // Privatize the declared variable of mapper to be the current array
10906 // element.
10907 Address PtrCurrent(
10908 PtrPHI, ElemTy,
10909 Address(BeginArg, MapperCGF.VoidPtrTy, CGM.getPointerAlign())
10910 .getAlignment()
10911 .alignmentOfArrayElement(ElementSize));
10913 Scope.addPrivate(MapperVarDecl, PtrCurrent);
10914 (void)Scope.Privatize();
10915
10916 // Get map clause information.
10917 MappableExprsHandler MEHandler(*D, MapperCGF);
10918 MEHandler.generateAllInfoForMapper(CombinedInfo, OMPBuilder);
10919
10920 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
10921 return emitMappingInformation(MapperCGF, OMPBuilder, MapExpr);
10922 };
10923 if (CGM.getCodeGenOpts().getDebugInfo() !=
10924 llvm::codegenoptions::NoDebugInfo) {
10925 CombinedInfo.Names.resize(CombinedInfo.Exprs.size());
10926 llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),
10927 FillInfoMap);
10928 }
10929
10930 return CombinedInfo;
10931 };
10932
10933 auto CustomMapperCB = [&](unsigned I) {
10934 llvm::Function *MapperFunc = nullptr;
10935 if (CombinedInfo.Mappers[I]) {
10936 // Call the corresponding mapper function.
10938 cast<OMPDeclareMapperDecl>(CombinedInfo.Mappers[I]));
10939 assert(MapperFunc && "Expect a valid mapper function is available.");
10940 }
10941 return MapperFunc;
10942 };
10943
10944 SmallString<64> TyStr;
10945 llvm::raw_svector_ostream Out(TyStr);
10946 CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(Ty, Out);
10947 std::string Name = getName({"omp_mapper", TyStr, D->getName()});
10948
10949 // Propagate the PRESENT modifier to the pointee entries (those with
10950 // HasAttachPtr) only for OpenMP >= 6.0; before 6.0 the present modifier does
10951 // not apply to the pointee (see the OpenMP 6.0 erratum on the present motion
10952 // vs. map-type modifier divergence).
10953 bool PropagatePresentToPointeeStorage = CGM.getLangOpts().OpenMP >= 60;
10954 // Keep non-pointee storage present propagation disabled until Clang stops
10955 // emitting PTR_AND_OBJ maps and uses attach-style maps throughout.
10956 bool PropagatePresentToNonPointeeStorage = false;
10957 llvm::Function *NewFn = cantFail(OMPBuilder.emitUserDefinedMapper(
10958 PrivatizeAndGenMapInfoCB, ElemTy, Name, CustomMapperCB,
10959 /*PreserveMemberOfFlags=*/false, PropagatePresentToPointeeStorage,
10960 PropagatePresentToNonPointeeStorage));
10961 UDMMap.try_emplace(D, NewFn);
10962 if (CGF)
10963 FunctionUDMMap[CGF->CurFn].push_back(D);
10964}
10965
10967 const OMPDeclareMapperDecl *D) {
10968 auto I = UDMMap.find(D);
10969 if (I != UDMMap.end())
10970 return I->second;
10972 return UDMMap.lookup(D);
10973}
10974
10977 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
10978 const OMPLoopDirective &D)>
10979 SizeEmitter) {
10980 OpenMPDirectiveKind Kind = D.getDirectiveKind();
10981 const OMPExecutableDirective *TD = &D;
10982 // Get nested teams distribute kind directive, if any. For now, treat
10983 // 'target_teams_loop' as if it's really a target_teams_distribute.
10984 if ((!isOpenMPDistributeDirective(Kind) || !isOpenMPTeamsDirective(Kind)) &&
10985 Kind != OMPD_target_teams_loop)
10986 TD = getNestedDistributeDirective(CGM.getContext(), D);
10987 if (!TD)
10988 return llvm::ConstantInt::get(CGF.Int64Ty, 0);
10989
10990 const auto *LD = cast<OMPLoopDirective>(TD);
10991 if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD))
10992 return NumIterations;
10993 return llvm::ConstantInt::get(CGF.Int64Ty, 0);
10994}
10995
10996static void
10997emitTargetCallFallback(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn,
10998 const OMPExecutableDirective &D,
11000 bool RequiresOuterTask, const CapturedStmt &CS,
11001 bool OffloadingMandatory, CodeGenFunction &CGF) {
11002 if (OffloadingMandatory) {
11003 CGF.Builder.CreateUnreachable();
11004 } else {
11005 if (RequiresOuterTask) {
11006 CapturedVars.clear();
11007 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
11008 }
11009 llvm::SmallVector<llvm::Value *, 16> Args(CapturedVars.begin(),
11010 CapturedVars.end());
11011 Args.push_back(llvm::Constant::getNullValue(CGF.Builder.getPtrTy()));
11012 OMPRuntime->emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn,
11013 Args);
11014 }
11015}
11016
11017static llvm::Value *emitDeviceID(
11018 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
11019 CodeGenFunction &CGF) {
11020 // Emit device ID if any.
11021 llvm::Value *DeviceID;
11022 if (Device.getPointer()) {
11023 assert((Device.getInt() == OMPC_DEVICE_unknown ||
11024 Device.getInt() == OMPC_DEVICE_device_num) &&
11025 "Expected device_num modifier.");
11026 llvm::Value *DevVal = CGF.EmitScalarExpr(Device.getPointer());
11027 DeviceID =
11028 CGF.Builder.CreateIntCast(DevVal, CGF.Int64Ty, /*isSigned=*/true);
11029 } else {
11030 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
11031 }
11032 return DeviceID;
11033}
11034
11035static std::pair<llvm::Value *, OMPDynGroupprivateFallbackType>
11037 llvm::Value *DynGP = CGF.Builder.getInt32(0);
11038 auto DynGPFallback = OMPDynGroupprivateFallbackType::Abort;
11039
11040 if (auto *DynGPClause = D.getSingleClause<OMPDynGroupprivateClause>()) {
11041 CodeGenFunction::RunCleanupsScope DynGPScope(CGF);
11042 llvm::Value *DynGPVal =
11043 CGF.EmitScalarExpr(DynGPClause->getSize(), /*IgnoreResultAssign=*/true);
11044 DynGP = CGF.Builder.CreateIntCast(DynGPVal, CGF.Int32Ty,
11045 /*isSigned=*/false);
11046 auto FallbackModifier = DynGPClause->getDynGroupprivateFallbackModifier();
11047 switch (FallbackModifier) {
11048 case OMPC_DYN_GROUPPRIVATE_FALLBACK_abort:
11049 DynGPFallback = OMPDynGroupprivateFallbackType::Abort;
11050 break;
11051 case OMPC_DYN_GROUPPRIVATE_FALLBACK_null:
11052 DynGPFallback = OMPDynGroupprivateFallbackType::Null;
11053 break;
11054 case OMPC_DYN_GROUPPRIVATE_FALLBACK_default_mem:
11056 // This is the default for dyn_groupprivate.
11057 DynGPFallback = OMPDynGroupprivateFallbackType::DefaultMem;
11058 break;
11059 default:
11060 llvm_unreachable("Unknown fallback modifier for OpenMP dyn_groupprivate");
11061 }
11062 } else if (auto *OMPXDynCGClause =
11063 D.getSingleClause<OMPXDynCGroupMemClause>()) {
11064 CodeGenFunction::RunCleanupsScope DynCGMemScope(CGF);
11065 llvm::Value *DynCGMemVal = CGF.EmitScalarExpr(OMPXDynCGClause->getSize(),
11066 /*IgnoreResultAssign=*/true);
11067 DynGP = CGF.Builder.CreateIntCast(DynCGMemVal, CGF.Int32Ty,
11068 /*isSigned=*/false);
11069 }
11070 return {DynGP, DynGPFallback};
11071}
11072
11074 MappableExprsHandler &MEHandler, CodeGenFunction &CGF,
11075 const CapturedStmt &CS, llvm::SmallVectorImpl<llvm::Value *> &CapturedVars,
11076 llvm::OpenMPIRBuilder &OMPBuilder,
11077 llvm::DenseSet<CanonicalDeclPtr<const Decl>> &MappedVarSet,
11078 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) {
11079
11080 llvm::DenseMap<llvm::Value *, llvm::Value *> LambdaPointers;
11081 auto RI = CS.getCapturedRecordDecl()->field_begin();
11082 auto *CV = CapturedVars.begin();
11084 CE = CS.capture_end();
11085 CI != CE; ++CI, ++RI, ++CV) {
11086 MappableExprsHandler::MapCombinedInfoTy CurInfo;
11087
11088 // VLA sizes are passed to the outlined region by copy and do not have map
11089 // information associated.
11090 if (CI->capturesVariableArrayType()) {
11091 CurInfo.Exprs.push_back(nullptr);
11092 CurInfo.BasePointers.push_back(*CV);
11093 CurInfo.DevicePtrDecls.push_back(nullptr);
11094 CurInfo.DevicePointers.push_back(
11095 MappableExprsHandler::DeviceInfoTy::None);
11096 CurInfo.Pointers.push_back(*CV);
11097 CurInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
11098 CGF.getTypeSize(RI->getType()), CGF.Int64Ty, /*isSigned=*/true));
11099 // Copy to the device as an argument. No need to retrieve it.
11100 CurInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
11101 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM |
11102 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
11103 CurInfo.HasAttachPtr.push_back(false);
11104 CurInfo.Mappers.push_back(nullptr);
11105 } else {
11106 const ValueDecl *CapturedVD =
11107 CI->capturesThis() ? nullptr
11109 bool HasEntryWithCVAsAttachPtr = false;
11110 if (CapturedVD)
11111 HasEntryWithCVAsAttachPtr =
11112 MEHandler.hasAttachEntryForCapturedVar(CapturedVD);
11113
11114 // Populate component lists for the captured variable from clauses.
11115 MappableExprsHandler::MapDataArrayTy DeclComponentLists;
11118 StorageForImplicitlyAddedComponentLists;
11119 MEHandler.populateComponentListsForNonLambdaCaptureFromClauses(
11120 CapturedVD, DeclComponentLists,
11121 StorageForImplicitlyAddedComponentLists);
11122
11123 // OpenMP 6.0, 15.8, target construct, restrictions:
11124 // * A list item in a map clause that is specified on a target construct
11125 // must have a base variable or base pointer.
11126 //
11127 // Map clauses on a target construct must either have a base pointer, or a
11128 // base-variable. So, if we don't have a base-pointer, that means that it
11129 // must have a base-variable, i.e. we have a map like `map(s)`, `map(s.x)`
11130 // etc. In such cases, we do not need to handle default map generation
11131 // for `s`.
11132 bool HasEntryWithoutAttachPtr =
11133 llvm::any_of(DeclComponentLists, [&](const auto &MapData) {
11135 Components = std::get<0>(MapData);
11136 return !MEHandler.getAttachPtrExpr(Components);
11137 });
11138
11139 // Generate default map info first if there's no direct map with CV as
11140 // the base-variable, or attach pointer.
11141 if (DeclComponentLists.empty() ||
11142 (!HasEntryWithCVAsAttachPtr && !HasEntryWithoutAttachPtr))
11143 MEHandler.generateDefaultMapInfo(*CI, **RI, *CV, CurInfo);
11144
11145 // If we have any information in the map clause, we use it, otherwise we
11146 // just do a default mapping.
11147 MEHandler.generateInfoForCaptureFromClauseInfo(
11148 DeclComponentLists, CI, *CV, CurInfo, OMPBuilder,
11149 /*OffsetForMemberOfFlag=*/CombinedInfo.BasePointers.size());
11150
11151 if (!CI->capturesThis())
11152 MappedVarSet.insert(CI->getCapturedVar());
11153 else
11154 MappedVarSet.insert(nullptr);
11155
11156 // Generate correct mapping for variables captured by reference in
11157 // lambdas.
11158 if (CI->capturesVariable())
11159 MEHandler.generateInfoForLambdaCaptures(CI->getCapturedVar(), *CV,
11160 CurInfo, LambdaPointers);
11161 }
11162 // We expect to have at least an element of information for this capture.
11163 assert(!CurInfo.BasePointers.empty() &&
11164 "Non-existing map pointer for capture!");
11165 assert(CurInfo.BasePointers.size() == CurInfo.Pointers.size() &&
11166 CurInfo.BasePointers.size() == CurInfo.Sizes.size() &&
11167 CurInfo.BasePointers.size() == CurInfo.Types.size() &&
11168 CurInfo.BasePointers.size() == CurInfo.Mappers.size() &&
11169 "Inconsistent map information sizes!");
11170
11171 // We need to append the results of this capture to what we already have.
11172 CombinedInfo.append(CurInfo);
11173 }
11174 // Adjust MEMBER_OF flags for the lambdas captures.
11175 MEHandler.adjustMemberOfForLambdaCaptures(
11176 OMPBuilder, LambdaPointers, CombinedInfo.BasePointers,
11177 CombinedInfo.Pointers, CombinedInfo.Types);
11178}
11179static void
11180genMapInfo(MappableExprsHandler &MEHandler, CodeGenFunction &CGF,
11181 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
11182 llvm::OpenMPIRBuilder &OMPBuilder,
11183 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkippedVarSet =
11184 llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) {
11185
11186 CodeGenModule &CGM = CGF.CGM;
11187 // Map any list items in a map clause that were not captures because they
11188 // weren't referenced within the construct.
11189 MEHandler.generateAllInfo(CombinedInfo, OMPBuilder, SkippedVarSet);
11190
11191 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
11192 return emitMappingInformation(CGF, OMPBuilder, MapExpr);
11193 };
11194 if (CGM.getCodeGenOpts().getDebugInfo() !=
11195 llvm::codegenoptions::NoDebugInfo) {
11196 CombinedInfo.Names.resize(CombinedInfo.Exprs.size());
11197 llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),
11198 FillInfoMap);
11199 }
11200}
11201
11203 const CapturedStmt &CS,
11205 llvm::OpenMPIRBuilder &OMPBuilder,
11206 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) {
11207 // Get mappable expression information.
11208 MappableExprsHandler MEHandler(D, CGF);
11209 llvm::DenseSet<CanonicalDeclPtr<const Decl>> MappedVarSet;
11210
11211 genMapInfoForCaptures(MEHandler, CGF, CS, CapturedVars, OMPBuilder,
11212 MappedVarSet, CombinedInfo);
11213 genMapInfo(MEHandler, CGF, CombinedInfo, OMPBuilder, MappedVarSet);
11214}
11215
11216template <typename ClauseTy>
11217static void
11219 const OMPExecutableDirective &D,
11221 const auto *C = D.getSingleClause<ClauseTy>();
11222 assert(!C->varlist_empty() &&
11223 "ompx_bare requires explicit num_teams and thread_limit");
11225 for (auto *E : C->varlist()) {
11226 llvm::Value *V = CGF.EmitScalarExpr(E);
11227 Values.push_back(
11228 CGF.Builder.CreateIntCast(V, CGF.Int32Ty, /*isSigned=*/true));
11229 }
11230}
11231
11233 CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn,
11234 const OMPExecutableDirective &D,
11235 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars, bool RequiresOuterTask,
11236 const CapturedStmt &CS, bool OffloadingMandatory,
11237 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
11238 llvm::Value *OutlinedFnID, CodeGenFunction::OMPTargetDataInfo &InputInfo,
11239 llvm::Value *&MapTypesArray, llvm::Value *&MapNamesArray,
11240 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
11241 const OMPLoopDirective &D)>
11242 SizeEmitter,
11243 CodeGenFunction &CGF, CodeGenModule &CGM) {
11244 llvm::OpenMPIRBuilder &OMPBuilder = OMPRuntime->getOMPBuilder();
11245
11246 // Fill up the arrays with all the captured variables.
11247 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
11249 genMapInfo(D, CGF, CS, CapturedVars, OMPBuilder, CombinedInfo);
11250
11251 // Append a null entry for the implicit dyn_ptr argument.
11252 using OpenMPOffloadMappingFlags = llvm::omp::OpenMPOffloadMappingFlags;
11253 auto *NullPtr = llvm::Constant::getNullValue(CGF.Builder.getPtrTy());
11254 CombinedInfo.BasePointers.push_back(NullPtr);
11255 CombinedInfo.Pointers.push_back(NullPtr);
11256 CombinedInfo.DevicePointers.push_back(
11257 llvm::OpenMPIRBuilder::DeviceInfoTy::None);
11258 CombinedInfo.Sizes.push_back(CGF.Builder.getInt64(0));
11259 CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM |
11260 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
11261 CombinedInfo.HasAttachPtr.push_back(false);
11262 if (!CombinedInfo.Names.empty())
11263 CombinedInfo.Names.push_back(NullPtr);
11264 CombinedInfo.Exprs.push_back(nullptr);
11265 CombinedInfo.Mappers.push_back(nullptr);
11266 CombinedInfo.DevicePtrDecls.push_back(nullptr);
11267
11268 emitOffloadingArraysAndArgs(CGF, CombinedInfo, Info, OMPBuilder,
11269 /*IsNonContiguous=*/true, /*ForEndCall=*/false);
11270
11271 InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
11272 InputInfo.BasePointersArray = Address(Info.RTArgs.BasePointersArray,
11273 CGF.VoidPtrTy, CGM.getPointerAlign());
11274 InputInfo.PointersArray =
11275 Address(Info.RTArgs.PointersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
11276 InputInfo.SizesArray =
11277 Address(Info.RTArgs.SizesArray, CGF.Int64Ty, CGM.getPointerAlign());
11278 InputInfo.MappersArray =
11279 Address(Info.RTArgs.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
11280 MapTypesArray = Info.RTArgs.MapTypesArray;
11281 MapNamesArray = Info.RTArgs.MapNamesArray;
11282
11283 auto &&ThenGen = [&OMPRuntime, OutlinedFn, &D, &CapturedVars,
11284 RequiresOuterTask, &CS, OffloadingMandatory, Device,
11285 OutlinedFnID, &InputInfo, &MapTypesArray, &MapNamesArray,
11286 SizeEmitter](CodeGenFunction &CGF, PrePostActionTy &) {
11287 bool IsReverseOffloading = Device.getInt() == OMPC_DEVICE_ancestor;
11288
11289 if (IsReverseOffloading) {
11290 // Reverse offloading is not supported, so just execute on the host.
11291 // FIXME: This fallback solution is incorrect since it ignores the
11292 // OMP_TARGET_OFFLOAD environment variable. Instead it would be better to
11293 // assert here and ensure SEMA emits an error.
11294 emitTargetCallFallback(OMPRuntime, OutlinedFn, D, CapturedVars,
11295 RequiresOuterTask, CS, OffloadingMandatory, CGF);
11296 return;
11297 }
11298
11299 bool HasNoWait = D.hasClausesOfKind<OMPNowaitClause>();
11300 unsigned NumTargetItems = InputInfo.NumberOfTargetItems;
11301
11302 llvm::Value *BasePointersArray =
11303 InputInfo.BasePointersArray.emitRawPointer(CGF);
11304 llvm::Value *PointersArray = InputInfo.PointersArray.emitRawPointer(CGF);
11305 llvm::Value *SizesArray = InputInfo.SizesArray.emitRawPointer(CGF);
11306 llvm::Value *MappersArray = InputInfo.MappersArray.emitRawPointer(CGF);
11307
11308 auto &&EmitTargetCallFallbackCB =
11309 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
11310 OffloadingMandatory, &CGF](llvm::OpenMPIRBuilder::InsertPointTy IP)
11311 -> llvm::OpenMPIRBuilder::InsertPointTy {
11312 CGF.Builder.restoreIP(IP);
11313 emitTargetCallFallback(OMPRuntime, OutlinedFn, D, CapturedVars,
11314 RequiresOuterTask, CS, OffloadingMandatory, CGF);
11315 return CGF.Builder.saveIP();
11316 };
11317
11318 bool IsBare = D.hasClausesOfKind<OMPXBareClause>();
11321 if (IsBare) {
11324 NumThreads);
11325 } else {
11326 NumTeams.push_back(OMPRuntime->emitNumTeamsForTargetDirective(CGF, D));
11327 NumThreads.push_back(
11328 OMPRuntime->emitNumThreadsForTargetDirective(CGF, D));
11329 }
11330
11331 llvm::Value *DeviceID = emitDeviceID(Device, CGF);
11332 llvm::Value *RTLoc = OMPRuntime->emitUpdateLocation(CGF, D.getBeginLoc());
11333 llvm::Value *NumIterations =
11334 OMPRuntime->emitTargetNumIterationsCall(CGF, D, SizeEmitter);
11335 auto [DynCGroupMem, DynCGroupMemFallback] = emitDynCGroupMem(D, CGF);
11336 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
11337 CGF.AllocaInsertPt->getIterator());
11338
11339 llvm::OpenMPIRBuilder::TargetDataRTArgs RTArgs(
11340 BasePointersArray, PointersArray, SizesArray, MapTypesArray,
11341 nullptr /* MapTypesArrayEnd */, MappersArray, MapNamesArray);
11342
11343 llvm::OpenMPIRBuilder::TargetKernelArgs Args(
11344 NumTargetItems, RTArgs, NumIterations, NumTeams, NumThreads,
11345 DynCGroupMem, HasNoWait, /*StrictBlocks=*/IsBare,
11346 /*StrictThreads=*/IsBare, DynCGroupMemFallback);
11347
11348 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
11349 cantFail(OMPRuntime->getOMPBuilder().emitKernelLaunch(
11350 CGF.Builder, OutlinedFnID, EmitTargetCallFallbackCB, Args, DeviceID,
11351 RTLoc, AllocaIP));
11352 CGF.Builder.restoreIP(AfterIP);
11353 };
11354
11355 if (RequiresOuterTask)
11356 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo);
11357 else
11358 OMPRuntime->emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen);
11359}
11360
11361static void
11362emitTargetCallElse(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn,
11363 const OMPExecutableDirective &D,
11365 bool RequiresOuterTask, const CapturedStmt &CS,
11366 bool OffloadingMandatory, CodeGenFunction &CGF) {
11367
11368 // Notify that the host version must be executed.
11369 auto &&ElseGen =
11370 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
11371 OffloadingMandatory](CodeGenFunction &CGF, PrePostActionTy &) {
11372 emitTargetCallFallback(OMPRuntime, OutlinedFn, D, CapturedVars,
11373 RequiresOuterTask, CS, OffloadingMandatory, CGF);
11374 };
11375
11376 if (RequiresOuterTask) {
11378 CGF.EmitOMPTargetTaskBasedDirective(D, ElseGen, InputInfo);
11379 } else {
11380 OMPRuntime->emitInlinedDirective(CGF, D.getDirectiveKind(), ElseGen);
11381 }
11382}
11383
11386 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
11387 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
11388 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
11389 const OMPLoopDirective &D)>
11390 SizeEmitter) {
11391 if (!CGF.HaveInsertPoint())
11392 return;
11393
11394 const bool OffloadingMandatory = !CGM.getLangOpts().OpenMPIsTargetDevice &&
11395 CGM.getLangOpts().OpenMPOffloadMandatory;
11396
11397 assert((OffloadingMandatory || OutlinedFn) && "Invalid outlined function!");
11398
11399 const bool RequiresOuterTask =
11400 D.hasClausesOfKind<OMPDependClause>() ||
11401 D.hasClausesOfKind<OMPNowaitClause>() ||
11402 D.hasClausesOfKind<OMPInReductionClause>() ||
11403 (CGM.getLangOpts().OpenMP >= 51 &&
11404 needsTaskBasedThreadLimit(D.getDirectiveKind()) &&
11405 D.hasClausesOfKind<OMPThreadLimitClause>());
11407 const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target);
11408 auto &&ArgsCodegen = [&CS, &CapturedVars](CodeGenFunction &CGF,
11409 PrePostActionTy &) {
11410 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
11411 };
11412 emitInlinedDirective(CGF, OMPD_unknown, ArgsCodegen);
11413
11415 llvm::Value *MapTypesArray = nullptr;
11416 llvm::Value *MapNamesArray = nullptr;
11417
11418 auto &&TargetThenGen = [this, OutlinedFn, &D, &CapturedVars,
11419 RequiresOuterTask, &CS, OffloadingMandatory, Device,
11420 OutlinedFnID, &InputInfo, &MapTypesArray,
11421 &MapNamesArray, SizeEmitter](CodeGenFunction &CGF,
11422 PrePostActionTy &) {
11423 emitTargetCallKernelLaunch(this, OutlinedFn, D, CapturedVars,
11424 RequiresOuterTask, CS, OffloadingMandatory,
11425 Device, OutlinedFnID, InputInfo, MapTypesArray,
11426 MapNamesArray, SizeEmitter, CGF, CGM);
11427 };
11428
11429 auto &&TargetElseGen =
11430 [this, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
11431 OffloadingMandatory](CodeGenFunction &CGF, PrePostActionTy &) {
11432 emitTargetCallElse(this, OutlinedFn, D, CapturedVars, RequiresOuterTask,
11433 CS, OffloadingMandatory, CGF);
11434 };
11435
11436 // If we have a target function ID it means that we need to support
11437 // offloading, otherwise, just execute on the host. We need to execute on host
11438 // regardless of the conditional in the if clause if, e.g., the user do not
11439 // specify target triples.
11440 if (OutlinedFnID) {
11441 if (IfCond) {
11442 emitIfClause(CGF, IfCond, TargetThenGen, TargetElseGen);
11443 } else {
11444 RegionCodeGenTy ThenRCG(TargetThenGen);
11445 ThenRCG(CGF);
11446 }
11447 } else {
11448 RegionCodeGenTy ElseRCG(TargetElseGen);
11449 ElseRCG(CGF);
11450 }
11451}
11452
11454 StringRef ParentName) {
11455 if (!S)
11456 return;
11457
11458 // Register vtable from device for target data and target directives.
11459 // Add this block here since scanForTargetRegionsFunctions ignores
11460 // target data by checking if S is a executable directive (target).
11461 if (auto *E = dyn_cast<OMPExecutableDirective>(S);
11462 E && isOpenMPTargetDataManagementDirective(E->getDirectiveKind())) {
11463 // Don't need to check if it's device compile
11464 // since scanForTargetRegionsFunctions currently only called
11465 // in device compilation.
11466 registerVTable(*E);
11467 }
11468
11469 // Codegen OMP target directives that offload compute to the device.
11470 bool RequiresDeviceCodegen =
11473 cast<OMPExecutableDirective>(S)->getDirectiveKind());
11474
11475 if (RequiresDeviceCodegen) {
11476 const auto &E = *cast<OMPExecutableDirective>(S);
11477
11478 llvm::TargetRegionEntryInfo EntryInfo = getEntryInfoFromPresumedLoc(
11479 CGM, OMPBuilder, E.getBeginLoc(), ParentName);
11480
11481 // Is this a target region that should not be emitted as an entry point? If
11482 // so just signal we are done with this target region.
11483 if (!OMPBuilder.OffloadInfoManager.hasTargetRegionEntryInfo(EntryInfo))
11484 return;
11485
11486 switch (E.getDirectiveKind()) {
11487 case OMPD_target:
11490 break;
11491 case OMPD_target_parallel:
11493 CGM, ParentName, cast<OMPTargetParallelDirective>(E));
11494 break;
11495 case OMPD_target_teams:
11497 CGM, ParentName, cast<OMPTargetTeamsDirective>(E));
11498 break;
11499 case OMPD_target_teams_distribute:
11502 break;
11503 case OMPD_target_teams_distribute_simd:
11506 break;
11507 case OMPD_target_parallel_for:
11510 break;
11511 case OMPD_target_parallel_for_simd:
11514 break;
11515 case OMPD_target_simd:
11516 case OMPD_target_loop:
11518 CGM, ParentName, cast<OMPLoopDirective>(E));
11519 break;
11520 case OMPD_target_teams_distribute_parallel_for:
11522 CGM, ParentName,
11524 break;
11525 case OMPD_target_teams_distribute_parallel_for_simd:
11528 CGM, ParentName,
11530 break;
11531 case OMPD_target_teams_loop:
11534 break;
11535 case OMPD_target_parallel_loop:
11538 break;
11539 case OMPD_parallel:
11540 case OMPD_for:
11541 case OMPD_parallel_for:
11542 case OMPD_parallel_master:
11543 case OMPD_parallel_sections:
11544 case OMPD_for_simd:
11545 case OMPD_parallel_for_simd:
11546 case OMPD_cancel:
11547 case OMPD_cancellation_point:
11548 case OMPD_ordered_standalone:
11549 case OMPD_ordered_blockassoc:
11550 case OMPD_threadprivate:
11551 case OMPD_allocate:
11552 case OMPD_task:
11553 case OMPD_simd:
11554 case OMPD_tile:
11555 case OMPD_unroll:
11556 case OMPD_sections:
11557 case OMPD_section:
11558 case OMPD_single:
11559 case OMPD_master:
11560 case OMPD_critical:
11561 case OMPD_taskyield:
11562 case OMPD_barrier:
11563 case OMPD_taskwait:
11564 case OMPD_taskgroup:
11565 case OMPD_atomic:
11566 case OMPD_flush:
11567 case OMPD_depobj:
11568 case OMPD_scan:
11569 case OMPD_teams:
11570 case OMPD_target_data:
11571 case OMPD_target_exit_data:
11572 case OMPD_target_enter_data:
11573 case OMPD_distribute:
11574 case OMPD_distribute_simd:
11575 case OMPD_distribute_parallel_for:
11576 case OMPD_distribute_parallel_for_simd:
11577 case OMPD_teams_distribute:
11578 case OMPD_teams_distribute_simd:
11579 case OMPD_teams_distribute_parallel_for:
11580 case OMPD_teams_distribute_parallel_for_simd:
11581 case OMPD_target_update:
11582 case OMPD_declare_simd:
11583 case OMPD_declare_variant:
11584 case OMPD_begin_declare_variant:
11585 case OMPD_end_declare_variant:
11586 case OMPD_declare_target:
11587 case OMPD_end_declare_target:
11588 case OMPD_declare_reduction:
11589 case OMPD_declare_mapper:
11590 case OMPD_taskloop:
11591 case OMPD_taskloop_simd:
11592 case OMPD_master_taskloop:
11593 case OMPD_master_taskloop_simd:
11594 case OMPD_parallel_master_taskloop:
11595 case OMPD_parallel_master_taskloop_simd:
11596 case OMPD_requires:
11597 case OMPD_metadirective:
11598 case OMPD_unknown:
11599 default:
11600 llvm_unreachable("Unknown target directive for OpenMP device codegen.");
11601 }
11602 return;
11603 }
11604
11605 // A metadirective has no associated statement; look into its variants.
11606 if (const auto *MD = dyn_cast<OMPMetaDirective>(S)) {
11607 for (const Stmt *V : MD->variants())
11608 scanForTargetRegionsFunctions(V, ParentName);
11609 return;
11610 }
11611
11612 if (const auto *E = dyn_cast<OMPExecutableDirective>(S)) {
11613 if (!E->hasAssociatedStmt() || !E->getAssociatedStmt())
11614 return;
11615
11616 scanForTargetRegionsFunctions(E->getRawStmt(), ParentName);
11617 return;
11618 }
11619
11620 // Default arguments and member initializers are not children of their uses.
11621 if (const auto *E = dyn_cast<CXXDefaultArgExpr>(S)) {
11622 scanForTargetRegionsFunctions(E->getExpr(), ParentName);
11623 return;
11624 }
11625 if (const auto *E = dyn_cast<CXXDefaultInitExpr>(S)) {
11626 scanForTargetRegionsFunctions(E->getExpr(), ParentName);
11627 return;
11628 }
11629
11630 // Implicit and inheriting constructors are only reachable from their uses.
11631 const CXXConstructorDecl *Ctor = nullptr;
11632 if (const auto *E = dyn_cast<CXXConstructExpr>(S))
11633 Ctor = E->getConstructor();
11634 else if (const auto *E = dyn_cast<CXXInheritedCtorInitExpr>(S))
11635 Ctor = E->getConstructor();
11636 if (Ctor && (!Ctor->isUserProvided() || Ctor->isInheritingConstructor()) &&
11637 Ctor->hasBody() && ScannedImplicitCtors.insert(Ctor).second)
11638 scanConstructorForTargetRegions(Ctor, ParentName);
11639
11640 // If this is a lambda function, look into its body.
11641 if (const auto *L = dyn_cast<LambdaExpr>(S)) {
11642 const CXXMethodDecl *CallOp = L->getCallOperator();
11643 // Specializations of a generic lambda are scanned as top-level decls.
11644 if (CallOp->isTemplated())
11645 return;
11646 ParentName = getTargetRegionParentName(CallOp);
11647 S = L->getBody();
11648 }
11649
11650 // Keep looking for target regions recursively.
11651 for (const Stmt *II : S->children())
11652 scanForTargetRegionsFunctions(II, ParentName);
11653}
11654
11656 const CXXConstructorDecl *Ctor, StringRef ParentName) {
11657 const auto *Def = dyn_cast_or_null<CXXConstructorDecl>(Ctor->getDefinition());
11658 if (!Def)
11659 return;
11660 for (const CXXCtorInitializer *Init : Def->inits())
11661 scanForTargetRegionsFunctions(Init->getInit(), ParentName);
11662 scanForTargetRegionsFunctions(Def->getBody(), ParentName);
11663}
11664
11665static bool isAssumedToBeNotEmitted(const ValueDecl *VD, bool IsDevice) {
11666 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
11667 OMPDeclareTargetDeclAttr::getDeviceType(VD);
11668 if (!DevTy)
11669 return false;
11670 // Do not emit device_type(nohost) functions for the host.
11671 if (!IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_NoHost)
11672 return true;
11673 // Do not emit device_type(host) functions for the device.
11674 if (IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_Host)
11675 return true;
11676 return false;
11677}
11678
11680 // If emitting code for the host, we do not process FD here. Instead we do
11681 // the normal code generation.
11682 if (!CGM.getLangOpts().OpenMPIsTargetDevice) {
11683 if (const auto *FD = dyn_cast<FunctionDecl>(GD.getDecl()))
11685 CGM.getLangOpts().OpenMPIsTargetDevice))
11686 return true;
11687 return false;
11688 }
11689
11690 const ValueDecl *VD = cast<ValueDecl>(GD.getDecl());
11691 // Try to detect target regions in the function.
11692 if (const auto *FD = dyn_cast<FunctionDecl>(VD)) {
11693 StringRef Name = getTargetRegionParentName(FD);
11694 if (const auto *Ctor = dyn_cast<CXXConstructorDecl>(FD))
11696 else
11699 CGM.getLangOpts().OpenMPIsTargetDevice))
11700 return true;
11701 }
11702
11703 // Do not emit function if it is not marked as declare target.
11704 return !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD) &&
11705 AlreadyEmittedTargetDecls.count(VD) == 0;
11706}
11707
11709 if (!CGM.getLangOpts().OpenMPIsTargetDevice)
11711 /*IsDevice=*/false);
11712
11713 // Check if there are Ctors/Dtors in this declaration and look for target
11714 // regions in it. We use the complete variant to produce the kernel name
11715 // mangling.
11716 const auto *VD = cast<VarDecl>(GD.getDecl());
11717 QualType RDTy = VD->getType();
11718 if (const auto *RD = RDTy->getBaseElementTypeUnsafe()->getAsCXXRecordDecl()) {
11719 for (const CXXConstructorDecl *Ctor : RD->ctors())
11721 if (const CXXDestructorDecl *Dtor = RD->getDestructor())
11722 scanForTargetRegionsFunctions(Dtor->getBody(),
11724 }
11725
11726 // Lambdas in the initializer are reached from nowhere else.
11727 if (const Expr *Init = VD->getInit())
11728 scanForTargetRegionsFunctions(Init, CGM.getMangledName(GD));
11729
11730 if (isAssumedToBeNotEmitted(VD, /*IsDevice=*/true))
11731 return true;
11732
11733 // Do not emit variable if it is not marked as declare target.
11734 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
11735 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
11736 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
11737 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
11738 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
11740 DeferredGlobalVariables.insert(VD);
11741 return true;
11742 }
11743 return false;
11744}
11745
11747 llvm::Constant *Addr) {
11748 if (CGM.getLangOpts().OMPTargetTriples.empty() &&
11749 !CGM.getLangOpts().OpenMPIsTargetDevice)
11750 return;
11751
11752 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
11753 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
11754
11755 // If this is an 'extern' declaration we defer to the canonical definition and
11756 // do not emit an offloading entry.
11757 if (Res && *Res != OMPDeclareTargetDeclAttr::MT_Link &&
11758 VD->hasExternalStorage())
11759 return;
11760
11761 // MT_Local variables use direct access with no host-device mapping.
11762 // No offload entry needed — the device global keeps its own initializer.
11763 if (Res && *Res == OMPDeclareTargetDeclAttr::MT_Local)
11764 return;
11765
11766 if (!Res) {
11767 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
11768 // Register non-target variables being emitted in device code (debug info
11769 // may cause this).
11770 StringRef VarName = CGM.getMangledName(VD);
11771 EmittedNonTargetVariables.try_emplace(VarName, Addr);
11772 }
11773 return;
11774 }
11775
11776 auto AddrOfGlobal = [&VD, this]() { return CGM.GetAddrOfGlobal(VD); };
11777 auto LinkageForVariable = [&VD, this]() {
11778 return CGM.getLLVMLinkageVarDefinition(VD);
11779 };
11780
11781 std::vector<llvm::GlobalVariable *> GeneratedRefs;
11782 OMPBuilder.registerTargetGlobalVariable(
11784 VD->hasDefinition(CGM.getContext()) == VarDecl::DeclarationOnly,
11785 VD->isExternallyVisible(),
11787 VD->getCanonicalDecl()->getBeginLoc()),
11788 CGM.getMangledName(VD), GeneratedRefs, CGM.getLangOpts().OpenMPSimd,
11789 CGM.getLangOpts().OMPTargetTriples, AddrOfGlobal, LinkageForVariable,
11790 CGM.getTypes().ConvertTypeForMem(
11791 CGM.getContext().getPointerType(VD->getType())),
11792 Addr);
11793
11794 for (auto *ref : GeneratedRefs)
11795 CGM.addCompilerUsedGlobal(ref);
11796}
11797
11799 if (isa<FunctionDecl>(GD.getDecl()) ||
11801 return emitTargetFunctions(GD);
11802
11803 return emitTargetGlobalVariable(GD);
11804}
11805
11807 for (const VarDecl *VD : DeferredGlobalVariables) {
11808 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
11809 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
11810 if (!Res)
11811 continue;
11812 // MT_Local and MT_To/MT_Enter without USM are always emitted.
11813 if (*Res == OMPDeclareTargetDeclAttr::MT_Local ||
11814 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
11815 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
11817 CGM.EmitGlobal(VD);
11818 } else {
11819 assert((*Res == OMPDeclareTargetDeclAttr::MT_Link ||
11820 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
11821 *Res == OMPDeclareTargetDeclAttr::MT_Enter ||
11822 *Res == OMPDeclareTargetDeclAttr::MT_Local) &&
11824 "Expected link clause or to clause with unified memory.");
11825 (void)CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
11826 }
11827 }
11828}
11829
11831 CodeGenFunction &CGF, const OMPExecutableDirective &D) const {
11832 assert(isOpenMPTargetExecutionDirective(D.getDirectiveKind()) &&
11833 " Expected target-based directive.");
11834}
11835
11837 for (const OMPClause *Clause : D->clauselists()) {
11838 if (Clause->getClauseKind() == OMPC_unified_shared_memory) {
11840 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(true);
11841 } else if (const auto *AC =
11842 dyn_cast<OMPAtomicDefaultMemOrderClause>(Clause)) {
11843 switch (AC->getAtomicDefaultMemOrderKind()) {
11844 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_acq_rel:
11845 RequiresAtomicOrdering = llvm::AtomicOrdering::AcquireRelease;
11846 break;
11847 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_seq_cst:
11848 RequiresAtomicOrdering = llvm::AtomicOrdering::SequentiallyConsistent;
11849 break;
11850 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_relaxed:
11851 RequiresAtomicOrdering = llvm::AtomicOrdering::Monotonic;
11852 break;
11854 break;
11855 }
11856 }
11857 }
11858}
11859
11860llvm::AtomicOrdering CGOpenMPRuntime::getDefaultMemoryOrdering() const {
11862}
11863
11865 LangAS &AS) {
11866 if (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())
11867 return false;
11868 const auto *A = VD->getAttr<OMPAllocateDeclAttr>();
11869 switch(A->getAllocatorType()) {
11870 case OMPAllocateDeclAttr::OMPNullMemAlloc:
11871 case OMPAllocateDeclAttr::OMPDefaultMemAlloc:
11872 // Not supported, fallback to the default mem space.
11873 case OMPAllocateDeclAttr::OMPLargeCapMemAlloc:
11874 case OMPAllocateDeclAttr::OMPCGroupMemAlloc:
11875 case OMPAllocateDeclAttr::OMPHighBWMemAlloc:
11876 case OMPAllocateDeclAttr::OMPLowLatMemAlloc:
11877 case OMPAllocateDeclAttr::OMPThreadMemAlloc:
11878 case OMPAllocateDeclAttr::OMPConstMemAlloc:
11879 case OMPAllocateDeclAttr::OMPPTeamMemAlloc:
11880 AS = LangAS::Default;
11881 return true;
11882 case OMPAllocateDeclAttr::OMPUserDefinedMemAlloc:
11883 llvm_unreachable("Expected predefined allocator for the variables with the "
11884 "static storage.");
11885 }
11886 return false;
11887}
11888
11892
11894 CodeGenModule &CGM)
11895 : CGM(CGM) {
11896 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
11897 SavedShouldMarkAsGlobal = CGM.getOpenMPRuntime().ShouldMarkAsGlobal;
11898 CGM.getOpenMPRuntime().ShouldMarkAsGlobal = false;
11899 }
11900}
11901
11903 if (CGM.getLangOpts().OpenMPIsTargetDevice)
11904 CGM.getOpenMPRuntime().ShouldMarkAsGlobal = SavedShouldMarkAsGlobal;
11905}
11906
11908 if (!CGM.getLangOpts().OpenMPIsTargetDevice || !ShouldMarkAsGlobal)
11909 return true;
11910
11911 const auto *D = cast<FunctionDecl>(GD.getDecl());
11912 // Do not emit function if it is marked as declare target as it was already
11913 // emitted.
11914 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(D)) {
11915 if (D->hasBody() && AlreadyEmittedTargetDecls.count(D) == 0) {
11916 if (auto *F = dyn_cast_or_null<llvm::Function>(
11917 CGM.GetGlobalValue(CGM.getMangledName(GD))))
11918 return !F->isDeclaration();
11919 return false;
11920 }
11921 return true;
11922 }
11923
11924 return !AlreadyEmittedTargetDecls.insert(D).second;
11925}
11926
11928 const OMPExecutableDirective &D,
11929 SourceLocation Loc,
11930 llvm::Function *OutlinedFn,
11931 ArrayRef<llvm::Value *> CapturedVars) {
11932 if (!CGF.HaveInsertPoint())
11933 return;
11934
11935 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
11937
11938 // Build call __kmpc_fork_teams(loc, n, microtask, var1, .., varn);
11939 llvm::Value *Args[] = {
11940 RTLoc,
11941 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
11942 OutlinedFn};
11944 RealArgs.append(std::begin(Args), std::end(Args));
11945 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
11946
11947 llvm::FunctionCallee RTLFn = OMPBuilder.getOrCreateRuntimeFunction(
11948 CGM.getModule(), OMPRTL___kmpc_fork_teams);
11949 CGF.EmitRuntimeCall(RTLFn, RealArgs);
11950}
11951
11953 const Expr *NumTeams,
11954 const Expr *ThreadLimit,
11955 SourceLocation Loc) {
11956 if (!CGF.HaveInsertPoint())
11957 return;
11958
11959 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
11960
11961 llvm::Value *NumTeamsVal =
11962 NumTeams
11963 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(NumTeams),
11964 CGF.CGM.Int32Ty, /* isSigned = */ true)
11965 : CGF.Builder.getInt32(0);
11966
11967 llvm::Value *ThreadLimitVal =
11968 ThreadLimit
11969 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(ThreadLimit),
11970 CGF.CGM.Int32Ty, /* isSigned = */ true)
11971 : CGF.Builder.getInt32(0);
11972
11973 // Build call __kmpc_push_num_teamss(&loc, global_tid, num_teams, thread_limit)
11974 llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal,
11975 ThreadLimitVal};
11976 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
11977 CGM.getModule(), OMPRTL___kmpc_push_num_teams),
11978 PushNumTeamsArgs);
11979}
11980
11982 const Expr *ThreadLimit,
11983 SourceLocation Loc) {
11984 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
11985 llvm::Value *ThreadLimitVal =
11986 ThreadLimit
11987 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(ThreadLimit),
11988 CGF.CGM.Int32Ty, /* isSigned = */ true)
11989 : CGF.Builder.getInt32(0);
11990
11991 // Build call __kmpc_set_thread_limit(&loc, global_tid, thread_limit)
11992 llvm::Value *ThreadLimitArgs[] = {RTLoc, getThreadID(CGF, Loc),
11993 ThreadLimitVal};
11994 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
11995 CGM.getModule(), OMPRTL___kmpc_set_thread_limit),
11996 ThreadLimitArgs);
11997}
11998
12000 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
12001 const Expr *Device, const RegionCodeGenTy &CodeGen,
12003 if (!CGF.HaveInsertPoint())
12004 return;
12005
12006 // Action used to replace the default codegen action and turn privatization
12007 // off.
12008 PrePostActionTy NoPrivAction;
12009
12010 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
12011
12012 llvm::Value *IfCondVal = nullptr;
12013 if (IfCond)
12014 IfCondVal = CGF.EvaluateExprAsBool(IfCond);
12015
12016 // Emit device ID if any.
12017 llvm::Value *DeviceID = nullptr;
12018 if (Device) {
12019 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
12020 CGF.Int64Ty, /*isSigned=*/true);
12021 } else {
12022 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
12023 }
12024
12025 // Fill up the arrays with all the mapped variables.
12026 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
12027 auto GenMapInfoCB =
12028 [&](InsertPointTy CodeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {
12029 CGF.Builder.restoreIP(CodeGenIP);
12030 // Get map clause information.
12031 MappableExprsHandler MEHandler(D, CGF);
12032 MEHandler.generateAllInfo(CombinedInfo, OMPBuilder);
12033
12034 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
12035 return emitMappingInformation(CGF, OMPBuilder, MapExpr);
12036 };
12037 if (CGM.getCodeGenOpts().getDebugInfo() !=
12038 llvm::codegenoptions::NoDebugInfo) {
12039 CombinedInfo.Names.resize(CombinedInfo.Exprs.size());
12040 llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),
12041 FillInfoMap);
12042 }
12043
12044 return CombinedInfo;
12045 };
12046 using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;
12047 auto BodyCB = [&](InsertPointTy CodeGenIP, BodyGenTy BodyGenType) {
12048 CGF.Builder.restoreIP(CodeGenIP);
12049 switch (BodyGenType) {
12050 case BodyGenTy::Priv:
12051 if (!Info.CaptureDeviceAddrMap.empty())
12052 CodeGen(CGF);
12053 break;
12054 case BodyGenTy::DupNoPriv:
12055 if (!Info.CaptureDeviceAddrMap.empty()) {
12056 CodeGen.setAction(NoPrivAction);
12057 CodeGen(CGF);
12058 }
12059 break;
12060 case BodyGenTy::NoPriv:
12061 if (Info.CaptureDeviceAddrMap.empty()) {
12062 CodeGen.setAction(NoPrivAction);
12063 CodeGen(CGF);
12064 }
12065 break;
12066 }
12067 return InsertPointTy(CGF.Builder.GetInsertPoint());
12068 };
12069
12070 auto DeviceAddrCB = [&](unsigned int I, llvm::Value *NewDecl) {
12071 if (const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
12072 Info.CaptureDeviceAddrMap.try_emplace(DevVD, NewDecl);
12073 }
12074 };
12075
12076 auto CustomMapperCB = [&](unsigned int I) {
12077 llvm::Function *MFunc = nullptr;
12078 if (CombinedInfo.Mappers[I]) {
12079 Info.HasMapper = true;
12081 cast<OMPDeclareMapperDecl>(CombinedInfo.Mappers[I]));
12082 }
12083 return MFunc;
12084 };
12085
12086 // Source location for the ident struct
12087 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc());
12088
12089 InsertPointTy AllocaIP(CGF.AllocaInsertPt->getIterator());
12090 InsertPointTy CodeGenIP(CGF.Builder.GetInsertPoint());
12091 llvm::OpenMPIRBuilder::LocationDescription OmpLoc(CGF.Builder);
12092 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
12093 cantFail(OMPBuilder.createTargetData(
12094 OmpLoc, AllocaIP, CodeGenIP, /*DeallocBlocks=*/{}, DeviceID,
12095 IfCondVal, Info, GenMapInfoCB, CustomMapperCB,
12096 /*MapperFunc=*/nullptr, BodyCB, DeviceAddrCB, RTLoc));
12097 CGF.Builder.restoreIP(AfterIP);
12098}
12099
12101 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
12102 const Expr *Device) {
12103 if (!CGF.HaveInsertPoint())
12104 return;
12105
12109 "Expecting either target enter, exit data, or update directives.");
12110
12112 llvm::Value *MapTypesArray = nullptr;
12113 llvm::Value *MapNamesArray = nullptr;
12114 // Generate the code for the opening of the data environment.
12115 auto &&ThenGen = [this, &D, Device, &InputInfo, &MapTypesArray,
12116 &MapNamesArray](CodeGenFunction &CGF, PrePostActionTy &) {
12117 // Emit device ID if any.
12118 llvm::Value *DeviceID = nullptr;
12119 if (Device) {
12120 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
12121 CGF.Int64Ty, /*isSigned=*/true);
12122 } else {
12123 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
12124 }
12125
12126 // Emit the number of elements in the offloading arrays.
12127 llvm::Constant *PointerNum =
12128 CGF.Builder.getInt32(InputInfo.NumberOfTargetItems);
12129
12130 // Source location for the ident struct
12131 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc());
12132
12133 SmallVector<llvm::Value *, 13> OffloadingArgs(
12134 {RTLoc, DeviceID, PointerNum,
12135 InputInfo.BasePointersArray.emitRawPointer(CGF),
12136 InputInfo.PointersArray.emitRawPointer(CGF),
12137 InputInfo.SizesArray.emitRawPointer(CGF), MapTypesArray, MapNamesArray,
12138 InputInfo.MappersArray.emitRawPointer(CGF)});
12139
12140 // Select the right runtime function call for each standalone
12141 // directive.
12142 const bool HasNowait = D.hasClausesOfKind<OMPNowaitClause>();
12143 RuntimeFunction RTLFn;
12144 switch (D.getDirectiveKind()) {
12145 case OMPD_target_enter_data:
12146 RTLFn = HasNowait ? OMPRTL___tgt_target_data_begin_nowait_mapper
12147 : OMPRTL___tgt_target_data_begin_mapper;
12148 break;
12149 case OMPD_target_exit_data:
12150 RTLFn = HasNowait ? OMPRTL___tgt_target_data_end_nowait_mapper
12151 : OMPRTL___tgt_target_data_end_mapper;
12152 break;
12153 case OMPD_target_update:
12154 RTLFn = HasNowait ? OMPRTL___tgt_target_data_update_nowait_mapper
12155 : OMPRTL___tgt_target_data_update_mapper;
12156 break;
12157 case OMPD_parallel:
12158 case OMPD_for:
12159 case OMPD_parallel_for:
12160 case OMPD_parallel_master:
12161 case OMPD_parallel_sections:
12162 case OMPD_for_simd:
12163 case OMPD_parallel_for_simd:
12164 case OMPD_cancel:
12165 case OMPD_cancellation_point:
12166 case OMPD_ordered_standalone:
12167 case OMPD_ordered_blockassoc:
12168 case OMPD_threadprivate:
12169 case OMPD_allocate:
12170 case OMPD_task:
12171 case OMPD_simd:
12172 case OMPD_tile:
12173 case OMPD_unroll:
12174 case OMPD_sections:
12175 case OMPD_section:
12176 case OMPD_single:
12177 case OMPD_master:
12178 case OMPD_critical:
12179 case OMPD_taskyield:
12180 case OMPD_barrier:
12181 case OMPD_taskwait:
12182 case OMPD_taskgroup:
12183 case OMPD_atomic:
12184 case OMPD_flush:
12185 case OMPD_depobj:
12186 case OMPD_scan:
12187 case OMPD_teams:
12188 case OMPD_target_data:
12189 case OMPD_distribute:
12190 case OMPD_distribute_simd:
12191 case OMPD_distribute_parallel_for:
12192 case OMPD_distribute_parallel_for_simd:
12193 case OMPD_teams_distribute:
12194 case OMPD_teams_distribute_simd:
12195 case OMPD_teams_distribute_parallel_for:
12196 case OMPD_teams_distribute_parallel_for_simd:
12197 case OMPD_declare_simd:
12198 case OMPD_declare_variant:
12199 case OMPD_begin_declare_variant:
12200 case OMPD_end_declare_variant:
12201 case OMPD_declare_target:
12202 case OMPD_end_declare_target:
12203 case OMPD_declare_reduction:
12204 case OMPD_declare_mapper:
12205 case OMPD_taskloop:
12206 case OMPD_taskloop_simd:
12207 case OMPD_master_taskloop:
12208 case OMPD_master_taskloop_simd:
12209 case OMPD_parallel_master_taskloop:
12210 case OMPD_parallel_master_taskloop_simd:
12211 case OMPD_target:
12212 case OMPD_target_simd:
12213 case OMPD_target_teams_distribute:
12214 case OMPD_target_teams_distribute_simd:
12215 case OMPD_target_teams_distribute_parallel_for:
12216 case OMPD_target_teams_distribute_parallel_for_simd:
12217 case OMPD_target_teams:
12218 case OMPD_target_parallel:
12219 case OMPD_target_parallel_for:
12220 case OMPD_target_parallel_for_simd:
12221 case OMPD_requires:
12222 case OMPD_metadirective:
12223 case OMPD_unknown:
12224 default:
12225 llvm_unreachable("Unexpected standalone target data directive.");
12226 break;
12227 }
12228 if (HasNowait) {
12229 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.Int32Ty));
12230 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.VoidPtrTy));
12231 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.Int32Ty));
12232 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.VoidPtrTy));
12233 }
12234 CGF.EmitRuntimeCall(
12235 OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(), RTLFn),
12236 OffloadingArgs);
12237 };
12238
12239 auto &&TargetThenGen = [this, &ThenGen, &D, &InputInfo, &MapTypesArray,
12240 &MapNamesArray](CodeGenFunction &CGF,
12241 PrePostActionTy &) {
12242 // Fill up the arrays with all the mapped variables.
12243 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
12245 MappableExprsHandler MEHandler(D, CGF);
12246 genMapInfo(MEHandler, CGF, CombinedInfo, OMPBuilder);
12247 emitOffloadingArraysAndArgs(CGF, CombinedInfo, Info, OMPBuilder,
12248 /*IsNonContiguous=*/true, /*ForEndCall=*/false);
12249
12250 bool RequiresOuterTask = D.hasClausesOfKind<OMPDependClause>() ||
12251 D.hasClausesOfKind<OMPNowaitClause>();
12252
12253 InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
12254 InputInfo.BasePointersArray = Address(Info.RTArgs.BasePointersArray,
12255 CGF.VoidPtrTy, CGM.getPointerAlign());
12256 InputInfo.PointersArray = Address(Info.RTArgs.PointersArray, CGF.VoidPtrTy,
12257 CGM.getPointerAlign());
12258 InputInfo.SizesArray =
12259 Address(Info.RTArgs.SizesArray, CGF.Int64Ty, CGM.getPointerAlign());
12260 InputInfo.MappersArray =
12261 Address(Info.RTArgs.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
12262 MapTypesArray = Info.RTArgs.MapTypesArray;
12263 MapNamesArray = Info.RTArgs.MapNamesArray;
12264 if (RequiresOuterTask)
12265 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo);
12266 else
12267 emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen);
12268 };
12269
12270 if (IfCond) {
12271 emitIfClause(CGF, IfCond, TargetThenGen,
12272 [](CodeGenFunction &CGF, PrePostActionTy &) {});
12273 } else {
12274 RegionCodeGenTy ThenRCG(TargetThenGen);
12275 ThenRCG(CGF);
12276 }
12277}
12278
12279static unsigned
12282 // Every vector variant of a SIMD-enabled function has a vector length (VLEN).
12283 // If OpenMP clause "simdlen" is used, the VLEN is the value of the argument
12284 // of that clause. The VLEN value must be power of 2.
12285 // In other case the notion of the function`s "characteristic data type" (CDT)
12286 // is used to compute the vector length.
12287 // CDT is defined in the following order:
12288 // a) For non-void function, the CDT is the return type.
12289 // b) If the function has any non-uniform, non-linear parameters, then the
12290 // CDT is the type of the first such parameter.
12291 // c) If the CDT determined by a) or b) above is struct, union, or class
12292 // type which is pass-by-value (except for the type that maps to the
12293 // built-in complex data type), the characteristic data type is int.
12294 // d) If none of the above three cases is applicable, the CDT is int.
12295 // The VLEN is then determined based on the CDT and the size of vector
12296 // register of that ISA for which current vector version is generated. The
12297 // VLEN is computed using the formula below:
12298 // VLEN = sizeof(vector_register) / sizeof(CDT),
12299 // where vector register size specified in section 3.2.1 Registers and the
12300 // Stack Frame of original AMD64 ABI document.
12301 QualType RetType = FD->getReturnType();
12302 if (RetType.isNull())
12303 return 0;
12304 ASTContext &C = FD->getASTContext();
12305 QualType CDT;
12306 if (!RetType.isNull() && !RetType->isVoidType()) {
12307 CDT = RetType;
12308 } else {
12309 unsigned Offset = 0;
12310 if (const auto *MD = dyn_cast<CXXMethodDecl>(FD)) {
12311 if (ParamAttrs[Offset].Kind ==
12312 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector)
12313 CDT = C.getPointerType(C.getCanonicalTagType(MD->getParent()));
12314 ++Offset;
12315 }
12316 if (CDT.isNull()) {
12317 for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) {
12318 if (ParamAttrs[I + Offset].Kind ==
12319 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector) {
12320 CDT = FD->getParamDecl(I)->getType();
12321 break;
12322 }
12323 }
12324 }
12325 }
12326 if (CDT.isNull())
12327 CDT = C.IntTy;
12328 CDT = CDT->getCanonicalTypeUnqualified();
12329 if (CDT->isRecordType() || CDT->isUnionType())
12330 CDT = C.IntTy;
12331 return C.getTypeSize(CDT);
12332}
12333
12334// This are the Functions that are needed to mangle the name of the
12335// vector functions generated by the compiler, according to the rules
12336// defined in the "Vector Function ABI specifications for AArch64",
12337// available at
12338// https://developer.arm.com/products/software-development-tools/hpc/arm-compiler-for-hpc/vector-function-abi.
12339
12340/// Maps To Vector (MTV), as defined in 4.1.1 of the AAVFABI (2021Q1).
12342 llvm::OpenMPIRBuilder::DeclareSimdKindTy Kind) {
12343 QT = QT.getCanonicalType();
12344
12345 if (QT->isVoidType())
12346 return false;
12347
12348 if (Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::Uniform)
12349 return false;
12350
12351 if (Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal ||
12352 Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef)
12353 return false;
12354
12355 if ((Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear ||
12356 Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal) &&
12357 !QT->isReferenceType())
12358 return false;
12359
12360 return true;
12361}
12362
12363/// Pass By Value (PBV), as defined in 3.1.2 of the AAVFABI.
12365 QT = QT.getCanonicalType();
12366 unsigned Size = C.getTypeSize(QT);
12367
12368 // Only scalars and complex within 16 bytes wide set PVB to true.
12369 if (Size != 8 && Size != 16 && Size != 32 && Size != 64 && Size != 128)
12370 return false;
12371
12372 if (QT->isFloatingType())
12373 return true;
12374
12375 if (QT->isIntegerType())
12376 return true;
12377
12378 if (QT->isPointerType())
12379 return true;
12380
12381 // TODO: Add support for complex types (section 3.1.2, item 2).
12382
12383 return false;
12384}
12385
12386/// Computes the lane size (LS) of a return type or of an input parameter,
12387/// as defined by `LS(P)` in 3.2.1 of the AAVFABI.
12388/// TODO: Add support for references, section 3.2.1, item 1.
12389static unsigned getAArch64LS(QualType QT,
12390 llvm::OpenMPIRBuilder::DeclareSimdKindTy Kind,
12391 ASTContext &C) {
12392 if (!getAArch64MTV(QT, Kind) && QT.getCanonicalType()->isPointerType()) {
12394 if (getAArch64PBV(PTy, C))
12395 return C.getTypeSize(PTy);
12396 }
12397 if (getAArch64PBV(QT, C))
12398 return C.getTypeSize(QT);
12399
12400 return C.getTypeSize(C.getUIntPtrType());
12401}
12402
12403// Get Narrowest Data Size (NDS) and Widest Data Size (WDS) from the
12404// signature of the scalar function, as defined in 3.2.2 of the
12405// AAVFABI.
12406static std::tuple<unsigned, unsigned, bool>
12409 QualType RetType = FD->getReturnType().getCanonicalType();
12410
12411 ASTContext &C = FD->getASTContext();
12412
12413 bool OutputBecomesInput = false;
12414
12416 if (!RetType->isVoidType()) {
12417 Sizes.push_back(getAArch64LS(
12418 RetType, llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector, C));
12419 if (!getAArch64PBV(RetType, C) && getAArch64MTV(RetType, {}))
12420 OutputBecomesInput = true;
12421 }
12422 for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) {
12424 Sizes.push_back(getAArch64LS(QT, ParamAttrs[I].Kind, C));
12425 }
12426
12427 assert(!Sizes.empty() && "Unable to determine NDS and WDS.");
12428 // The LS of a function parameter / return value can only be a power
12429 // of 2, starting from 8 bits, up to 128.
12430 assert(llvm::all_of(Sizes,
12431 [](unsigned Size) {
12432 return Size == 8 || Size == 16 || Size == 32 ||
12433 Size == 64 || Size == 128;
12434 }) &&
12435 "Invalid size");
12436
12437 return std::make_tuple(*llvm::min_element(Sizes), *llvm::max_element(Sizes),
12438 OutputBecomesInput);
12439}
12440
12441static llvm::OpenMPIRBuilder::DeclareSimdBranch
12442convertDeclareSimdBranch(OMPDeclareSimdDeclAttr::BranchStateTy State) {
12443 switch (State) {
12444 case OMPDeclareSimdDeclAttr::BS_Undefined:
12445 return llvm::OpenMPIRBuilder::DeclareSimdBranch::Undefined;
12446 case OMPDeclareSimdDeclAttr::BS_Inbranch:
12447 return llvm::OpenMPIRBuilder::DeclareSimdBranch::Inbranch;
12448 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
12449 return llvm::OpenMPIRBuilder::DeclareSimdBranch::Notinbranch;
12450 }
12451 llvm_unreachable("unexpected declare simd branch state");
12452}
12453
12454// Check the values provided via `simdlen` by the user.
12456 unsigned UserVLEN, unsigned WDS, char ISA) {
12457 // 1. A `simdlen(1)` doesn't produce vector signatures.
12458 if (UserVLEN == 1) {
12459 CGM.getDiags().Report(SLoc, diag::warn_simdlen_1_no_effect);
12460 return false;
12461 }
12462
12463 // 2. Section 3.3.1, item 1: user input must be a power of 2 for Advanced
12464 // SIMD.
12465 if (ISA == 'n' && UserVLEN && !llvm::isPowerOf2_32(UserVLEN)) {
12466 CGM.getDiags().Report(SLoc, diag::warn_simdlen_requires_power_of_2);
12467 return false;
12468 }
12469
12470 // 3. Section 3.4.1: SVE fixed length must obey the architectural limits.
12471 if (ISA == 's' && UserVLEN != 0 &&
12472 ((UserVLEN * WDS > 2048) || (UserVLEN * WDS % 128 != 0))) {
12473 CGM.getDiags().Report(SLoc, diag::warn_simdlen_must_fit_lanes) << WDS;
12474 return false;
12475 }
12476
12477 return true;
12478}
12479
12481 llvm::Function *Fn) {
12482 ASTContext &C = CGM.getContext();
12483 FD = FD->getMostRecentDecl();
12484 while (FD) {
12485 // Map params to their positions in function decl.
12486 llvm::DenseMap<const Decl *, unsigned> ParamPositions;
12487 if (isa<CXXMethodDecl>(FD))
12488 ParamPositions.try_emplace(FD, 0);
12489 unsigned ParamPos = ParamPositions.size();
12490 for (const ParmVarDecl *P : FD->parameters()) {
12491 ParamPositions.try_emplace(P->getCanonicalDecl(), ParamPos);
12492 ++ParamPos;
12493 }
12494 for (const auto *Attr : FD->specific_attrs<OMPDeclareSimdDeclAttr>()) {
12496 ParamPositions.size());
12497 // Mark uniform parameters.
12498 for (const Expr *E : Attr->uniforms()) {
12499 E = E->IgnoreParenImpCasts();
12500 unsigned Pos;
12501 if (isa<CXXThisExpr>(E)) {
12502 Pos = ParamPositions[FD];
12503 } else {
12504 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
12505 ->getCanonicalDecl();
12506 auto It = ParamPositions.find(PVD);
12507 assert(It != ParamPositions.end() && "Function parameter not found");
12508 Pos = It->second;
12509 }
12510 ParamAttrs[Pos].Kind =
12511 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Uniform;
12512 }
12513 // Get alignment info.
12514 auto *NI = Attr->alignments_begin();
12515 for (const Expr *E : Attr->aligneds()) {
12516 E = E->IgnoreParenImpCasts();
12517 unsigned Pos;
12518 QualType ParmTy;
12519 if (isa<CXXThisExpr>(E)) {
12520 Pos = ParamPositions[FD];
12521 ParmTy = E->getType();
12522 } else {
12523 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
12524 ->getCanonicalDecl();
12525 auto It = ParamPositions.find(PVD);
12526 assert(It != ParamPositions.end() && "Function parameter not found");
12527 Pos = It->second;
12528 ParmTy = PVD->getType();
12529 }
12530 ParamAttrs[Pos].Alignment =
12531 (*NI)
12532 ? (*NI)->EvaluateKnownConstInt(C)
12533 : llvm::APSInt::getUnsigned(
12534 C.toCharUnitsFromBits(C.getOpenMPDefaultSimdAlign(ParmTy))
12535 .getQuantity());
12536 ++NI;
12537 }
12538 // Mark linear parameters.
12539 auto *SI = Attr->steps_begin();
12540 auto *MI = Attr->modifiers_begin();
12541 for (const Expr *E : Attr->linears()) {
12542 E = E->IgnoreParenImpCasts();
12543 unsigned Pos;
12544 bool IsReferenceType = false;
12545 // Rescaling factor needed to compute the linear parameter
12546 // value in the mangled name.
12547 unsigned PtrRescalingFactor = 1;
12548 if (isa<CXXThisExpr>(E)) {
12549 Pos = ParamPositions[FD];
12550 auto *P = cast<PointerType>(E->getType());
12551 PtrRescalingFactor = CGM.getContext()
12552 .getTypeSizeInChars(P->getPointeeType())
12553 .getQuantity();
12554 } else {
12555 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
12556 ->getCanonicalDecl();
12557 auto It = ParamPositions.find(PVD);
12558 assert(It != ParamPositions.end() && "Function parameter not found");
12559 Pos = It->second;
12560 if (auto *P = dyn_cast<PointerType>(PVD->getType()))
12561 PtrRescalingFactor = CGM.getContext()
12562 .getTypeSizeInChars(P->getPointeeType())
12563 .getQuantity();
12564 else if (PVD->getType()->isReferenceType()) {
12565 IsReferenceType = true;
12566 PtrRescalingFactor =
12567 CGM.getContext()
12568 .getTypeSizeInChars(PVD->getType().getNonReferenceType())
12569 .getQuantity();
12570 }
12571 }
12572 llvm::OpenMPIRBuilder::DeclareSimdAttrTy &ParamAttr = ParamAttrs[Pos];
12573 if (*MI == OMPC_LINEAR_ref)
12574 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef;
12575 else if (*MI == OMPC_LINEAR_uval)
12576 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal;
12577 else if (IsReferenceType)
12578 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal;
12579 else
12580 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear;
12581 // Assuming a stride of 1, for `linear` without modifiers.
12582 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(1);
12583 if (*SI) {
12585 if (!(*SI)->EvaluateAsInt(Result, C, Expr::SE_AllowSideEffects)) {
12586 if (const auto *DRE =
12587 cast<DeclRefExpr>((*SI)->IgnoreParenImpCasts())) {
12588 if (const auto *StridePVD =
12589 dyn_cast<ParmVarDecl>(DRE->getDecl())) {
12590 ParamAttr.HasVarStride = true;
12591 auto It = ParamPositions.find(StridePVD->getCanonicalDecl());
12592 assert(It != ParamPositions.end() &&
12593 "Function parameter not found");
12594 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(It->second);
12595 }
12596 }
12597 } else {
12598 ParamAttr.StrideOrArg = Result.Val.getInt();
12599 }
12600 }
12601 // If we are using a linear clause on a pointer, we need to
12602 // rescale the value of linear_step with the byte size of the
12603 // pointee type.
12604 if (!ParamAttr.HasVarStride &&
12605 (ParamAttr.Kind ==
12606 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear ||
12607 ParamAttr.Kind ==
12608 llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef))
12609 ParamAttr.StrideOrArg = ParamAttr.StrideOrArg * PtrRescalingFactor;
12610 ++SI;
12611 ++MI;
12612 }
12613 llvm::APSInt VLENVal;
12614 SourceLocation ExprLoc;
12615 const Expr *VLENExpr = Attr->getSimdlen();
12616 if (VLENExpr) {
12617 VLENVal = VLENExpr->EvaluateKnownConstInt(C);
12618 ExprLoc = VLENExpr->getExprLoc();
12619 }
12620 llvm::OpenMPIRBuilder::DeclareSimdBranch State =
12621 convertDeclareSimdBranch(Attr->getBranchState());
12622 if (CGM.getTriple().isX86()) {
12623 unsigned NumElts = evaluateCDTSize(FD, ParamAttrs);
12624 assert(NumElts && "Non-zero simdlen/cdtsize expected");
12625 OMPBuilder.emitX86DeclareSimdFunction(Fn, NumElts, VLENVal, ParamAttrs,
12626 State);
12627 } else if (CGM.getTriple().getArch() == llvm::Triple::aarch64) {
12628 unsigned VLEN = VLENVal.getExtValue();
12629 // Get basic data for building the vector signature.
12630 const auto Data = getNDSWDS(FD, ParamAttrs);
12631 const unsigned NDS = std::get<0>(Data);
12632 const unsigned WDS = std::get<1>(Data);
12633 const bool OutputBecomesInput = std::get<2>(Data);
12634 if (CGM.getTarget().hasFeature("sve")) {
12635 if (validateAArch64Simdlen(CGM, ExprLoc, VLEN, WDS, 's'))
12636 OMPBuilder.emitAArch64DeclareSimdFunction(
12637 Fn, VLEN, ParamAttrs, State, 's', NDS, OutputBecomesInput);
12638 } else if (CGM.getTarget().hasFeature("neon")) {
12639 if (validateAArch64Simdlen(CGM, ExprLoc, VLEN, WDS, 'n'))
12640 OMPBuilder.emitAArch64DeclareSimdFunction(
12641 Fn, VLEN, ParamAttrs, State, 'n', NDS, OutputBecomesInput);
12642 }
12643 }
12644 }
12645 FD = FD->getPreviousDecl();
12646 }
12647}
12648
12649namespace {
12650/// Cleanup action for doacross support.
12651class DoacrossCleanupTy final : public EHScopeStack::Cleanup {
12652public:
12653 static const int DoacrossFinArgs = 2;
12654
12655private:
12656 llvm::FunctionCallee RTLFn;
12657 llvm::Value *Args[DoacrossFinArgs];
12658
12659public:
12660 DoacrossCleanupTy(llvm::FunctionCallee RTLFn,
12661 ArrayRef<llvm::Value *> CallArgs)
12662 : RTLFn(RTLFn) {
12663 assert(CallArgs.size() == DoacrossFinArgs);
12664 std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args));
12665 }
12666 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
12667 if (!CGF.HaveInsertPoint())
12668 return;
12669 CGF.EmitRuntimeCall(RTLFn, Args);
12670 }
12671};
12672} // namespace
12673
12675 const OMPLoopDirective &D,
12676 ArrayRef<Expr *> NumIterations) {
12677 if (!CGF.HaveInsertPoint())
12678 return;
12679
12680 ASTContext &C = CGM.getContext();
12681 QualType Int64Ty = C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/true);
12682 RecordDecl *RD;
12683 if (KmpDimTy.isNull()) {
12684 // Build struct kmp_dim { // loop bounds info casted to kmp_int64
12685 // kmp_int64 lo; // lower
12686 // kmp_int64 up; // upper
12687 // kmp_int64 st; // stride
12688 // };
12689 RD = C.buildImplicitRecord("kmp_dim");
12690 RD->startDefinition();
12691 addFieldToRecordDecl(C, RD, Int64Ty);
12692 addFieldToRecordDecl(C, RD, Int64Ty);
12693 addFieldToRecordDecl(C, RD, Int64Ty);
12694 RD->completeDefinition();
12695 KmpDimTy = C.getCanonicalTagType(RD);
12696 } else {
12697 RD = KmpDimTy->castAsRecordDecl();
12698 }
12699 llvm::APInt Size(/*numBits=*/32, NumIterations.size());
12700 QualType ArrayTy = C.getConstantArrayType(KmpDimTy, Size, nullptr,
12702
12703 Address DimsAddr = CGF.CreateMemTemp(ArrayTy, "dims");
12704 CGF.EmitNullInitialization(DimsAddr, ArrayTy);
12705 enum { LowerFD = 0, UpperFD, StrideFD };
12706 // Fill dims with data.
12707 for (unsigned I = 0, E = NumIterations.size(); I < E; ++I) {
12708 LValue DimsLVal = CGF.MakeAddrLValue(
12709 CGF.Builder.CreateConstArrayGEP(DimsAddr, I), KmpDimTy);
12710 // dims.upper = num_iterations;
12711 LValue UpperLVal = CGF.EmitLValueForField(
12712 DimsLVal, *std::next(RD->field_begin(), UpperFD));
12713 llvm::Value *NumIterVal = CGF.EmitScalarConversion(
12714 CGF.EmitScalarExpr(NumIterations[I]), NumIterations[I]->getType(),
12715 Int64Ty, NumIterations[I]->getExprLoc());
12716 CGF.EmitStoreOfScalar(NumIterVal, UpperLVal);
12717 // dims.stride = 1;
12718 LValue StrideLVal = CGF.EmitLValueForField(
12719 DimsLVal, *std::next(RD->field_begin(), StrideFD));
12720 CGF.EmitStoreOfScalar(llvm::ConstantInt::getSigned(CGM.Int64Ty, /*V=*/1),
12721 StrideLVal);
12722 }
12723
12724 // Build call void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid,
12725 // kmp_int32 num_dims, struct kmp_dim * dims);
12726 llvm::Value *Args[] = {
12727 emitUpdateLocation(CGF, D.getBeginLoc()),
12728 getThreadID(CGF, D.getBeginLoc()),
12729 llvm::ConstantInt::getSigned(CGM.Int32Ty, NumIterations.size()),
12731 CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).emitRawPointer(CGF),
12732 CGM.VoidPtrTy)};
12733
12734 llvm::FunctionCallee RTLFn = OMPBuilder.getOrCreateRuntimeFunction(
12735 CGM.getModule(), OMPRTL___kmpc_doacross_init);
12736 CGF.EmitRuntimeCall(RTLFn, Args);
12737 llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = {
12738 emitUpdateLocation(CGF, D.getEndLoc()), getThreadID(CGF, D.getEndLoc())};
12739 llvm::FunctionCallee FiniRTLFn = OMPBuilder.getOrCreateRuntimeFunction(
12740 CGM.getModule(), OMPRTL___kmpc_doacross_fini);
12741 CGF.EHStack.pushCleanup<DoacrossCleanupTy>(NormalAndEHCleanup, FiniRTLFn,
12742 llvm::ArrayRef(FiniArgs));
12743}
12744
12745template <typename T>
12747 const T *C, llvm::Value *ULoc,
12748 llvm::Value *ThreadID) {
12749 QualType Int64Ty =
12750 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
12751 llvm::APInt Size(/*numBits=*/32, C->getNumLoops());
12753 Int64Ty, Size, nullptr, ArraySizeModifier::Normal, 0);
12754 Address CntAddr = CGF.CreateMemTemp(ArrayTy, ".cnt.addr");
12755 for (unsigned I = 0, E = C->getNumLoops(); I < E; ++I) {
12756 const Expr *CounterVal = C->getLoopData(I);
12757 assert(CounterVal);
12758 llvm::Value *CntVal = CGF.EmitScalarConversion(
12759 CGF.EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty,
12760 CounterVal->getExprLoc());
12761 CGF.EmitStoreOfScalar(CntVal, CGF.Builder.CreateConstArrayGEP(CntAddr, I),
12762 /*Volatile=*/false, Int64Ty);
12763 }
12764 llvm::Value *Args[] = {
12765 ULoc, ThreadID,
12766 CGF.Builder.CreateConstArrayGEP(CntAddr, 0).emitRawPointer(CGF)};
12767 llvm::FunctionCallee RTLFn;
12768 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
12769 OMPDoacrossKind<T> ODK;
12770 if (ODK.isSource(C)) {
12771 RTLFn = OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
12772 OMPRTL___kmpc_doacross_post);
12773 } else {
12774 assert(ODK.isSink(C) && "Expect sink modifier.");
12775 RTLFn = OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
12776 OMPRTL___kmpc_doacross_wait);
12777 }
12778 CGF.EmitRuntimeCall(RTLFn, Args);
12779}
12780
12782 const OMPDependClause *C) {
12784 CGF, CGM, C, emitUpdateLocation(CGF, C->getBeginLoc()),
12785 getThreadID(CGF, C->getBeginLoc()));
12786}
12787
12789 const OMPDoacrossClause *C) {
12791 CGF, CGM, C, emitUpdateLocation(CGF, C->getBeginLoc()),
12792 getThreadID(CGF, C->getBeginLoc()));
12793}
12794
12796 llvm::FunctionCallee Callee,
12797 ArrayRef<llvm::Value *> Args) const {
12798 assert(Loc.isValid() && "Outlined function call location must be valid.");
12800
12801 if (auto *Fn = dyn_cast<llvm::Function>(Callee.getCallee())) {
12802 if (Fn->doesNotThrow()) {
12803 CGF.EmitNounwindRuntimeCall(Fn, Args);
12804 return;
12805 }
12806 }
12807 CGF.EmitRuntimeCall(Callee, Args);
12808}
12809
12811 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn,
12812 ArrayRef<llvm::Value *> Args) const {
12813 emitCall(CGF, Loc, OutlinedFn, Args);
12814}
12815
12817 if (const auto *FD = dyn_cast<FunctionDecl>(D))
12818 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(FD))
12820}
12821
12823 const VarDecl *NativeParam,
12824 const VarDecl *TargetParam) const {
12825 return CGF.GetAddrOfLocalVar(NativeParam);
12826}
12827
12828/// Return allocator value from expression, or return a null allocator (default
12829/// when no allocator specified).
12830static llvm::Value *getAllocatorVal(CodeGenFunction &CGF,
12831 const Expr *Allocator) {
12832 llvm::Value *AllocVal;
12833 if (Allocator) {
12834 AllocVal = CGF.EmitScalarExpr(Allocator);
12835 // According to the standard, the original allocator type is a enum
12836 // (integer). Convert to pointer type, if required.
12837 AllocVal = CGF.EmitScalarConversion(AllocVal, Allocator->getType(),
12838 CGF.getContext().VoidPtrTy,
12839 Allocator->getExprLoc());
12840 } else {
12841 // If no allocator specified, it defaults to the null allocator.
12842 AllocVal = llvm::Constant::getNullValue(
12844 }
12845 return AllocVal;
12846}
12847
12848/// Return the alignment from an allocate directive if present.
12849static llvm::Value *getAlignmentValue(CodeGenModule &CGM, const VarDecl *VD) {
12850 std::optional<CharUnits> AllocateAlignment = CGM.getOMPAllocateAlignment(VD);
12851
12852 if (!AllocateAlignment)
12853 return nullptr;
12854
12855 return llvm::ConstantInt::get(CGM.SizeTy, AllocateAlignment->getQuantity());
12856}
12857
12859 const VarDecl *VD) {
12860 if (!VD)
12861 return Address::invalid();
12862 Address UntiedAddr = Address::invalid();
12863 Address UntiedRealAddr = Address::invalid();
12864 auto It = FunctionToUntiedTaskStackMap.find(CGF.CurFn);
12865 if (It != FunctionToUntiedTaskStackMap.end()) {
12866 const UntiedLocalVarsAddressesMap &UntiedData =
12867 UntiedLocalVarsStack[It->second];
12868 auto I = UntiedData.find(VD);
12869 if (I != UntiedData.end()) {
12870 UntiedAddr = I->second.first;
12871 UntiedRealAddr = I->second.second;
12872 }
12873 }
12874 const VarDecl *CVD = VD->getCanonicalDecl();
12875 if (CVD->hasAttr<OMPAllocateDeclAttr>()) {
12876 // Use the default allocation.
12877 if (!isAllocatableDecl(VD))
12878 return UntiedAddr;
12879 llvm::Value *Size;
12880 CharUnits Align = CGM.getContext().getDeclAlign(CVD);
12881 if (CVD->getType()->isVariablyModifiedType()) {
12882 Size = CGF.getTypeSize(CVD->getType());
12883 // Align the size: ((size + align - 1) / align) * align
12884 Size = CGF.Builder.CreateNUWAdd(
12885 Size, CGM.getSize(Align - CharUnits::fromQuantity(1)));
12886 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align));
12887 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align));
12888 } else {
12889 CharUnits Sz = CGM.getContext().getTypeSizeInChars(CVD->getType());
12890 Size = CGM.getSize(Sz.alignTo(Align));
12891 }
12892 llvm::Value *ThreadID = getThreadID(CGF, CVD->getBeginLoc());
12893 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
12894 const Expr *Allocator = AA->getAllocator();
12895 llvm::Value *AllocVal = getAllocatorVal(CGF, Allocator);
12896 llvm::Value *Alignment = getAlignmentValue(CGM, CVD);
12898 Args.push_back(ThreadID);
12899 if (Alignment)
12900 Args.push_back(Alignment);
12901 Args.push_back(Size);
12902 Args.push_back(AllocVal);
12903 llvm::omp::RuntimeFunction FnID =
12904 Alignment ? OMPRTL___kmpc_aligned_alloc : OMPRTL___kmpc_alloc;
12905 llvm::Value *Addr = CGF.EmitRuntimeCall(
12906 OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(), FnID), Args,
12907 getName({CVD->getName(), ".void.addr"}));
12908 llvm::FunctionCallee FiniRTLFn = OMPBuilder.getOrCreateRuntimeFunction(
12909 CGM.getModule(), OMPRTL___kmpc_free);
12910 QualType Ty = CGM.getContext().getPointerType(CVD->getType());
12912 Addr, CGF.ConvertTypeForMem(Ty), getName({CVD->getName(), ".addr"}));
12913 if (UntiedAddr.isValid())
12914 CGF.EmitStoreOfScalar(Addr, UntiedAddr, /*Volatile=*/false, Ty);
12915
12916 // Cleanup action for allocate support.
12917 class OMPAllocateCleanupTy final : public EHScopeStack::Cleanup {
12918 llvm::FunctionCallee RTLFn;
12919 SourceLocation::UIntTy LocEncoding;
12920 Address Addr;
12921 const Expr *AllocExpr;
12922
12923 public:
12924 OMPAllocateCleanupTy(llvm::FunctionCallee RTLFn,
12925 SourceLocation::UIntTy LocEncoding, Address Addr,
12926 const Expr *AllocExpr)
12927 : RTLFn(RTLFn), LocEncoding(LocEncoding), Addr(Addr),
12928 AllocExpr(AllocExpr) {}
12929 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
12930 if (!CGF.HaveInsertPoint())
12931 return;
12932 llvm::Value *Args[3];
12933 Args[0] = CGF.CGM.getOpenMPRuntime().getThreadID(
12934 CGF, SourceLocation::getFromRawEncoding(LocEncoding));
12936 Addr.emitRawPointer(CGF), CGF.VoidPtrTy);
12937 llvm::Value *AllocVal = getAllocatorVal(CGF, AllocExpr);
12938 Args[2] = AllocVal;
12939 CGF.EmitRuntimeCall(RTLFn, Args);
12940 }
12941 };
12942 Address VDAddr =
12943 UntiedRealAddr.isValid()
12944 ? UntiedRealAddr
12945 : Address(Addr, CGF.ConvertTypeForMem(CVD->getType()), Align);
12946 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(
12947 NormalAndEHCleanup, FiniRTLFn, CVD->getLocation().getRawEncoding(),
12948 VDAddr, Allocator);
12949 if (UntiedRealAddr.isValid())
12950 if (auto *Region =
12951 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
12952 Region->emitUntiedSwitch(CGF);
12953 return VDAddr;
12954 }
12955 return UntiedAddr;
12956}
12957
12959 const VarDecl *VD) const {
12960 auto It = FunctionToUntiedTaskStackMap.find(CGF.CurFn);
12961 if (It == FunctionToUntiedTaskStackMap.end())
12962 return false;
12963 return UntiedLocalVarsStack[It->second].count(VD) > 0;
12964}
12965
12967 CodeGenModule &CGM, const OMPLoopDirective &S)
12968 : CGM(CGM), NeedToPush(S.hasClausesOfKind<OMPNontemporalClause>()) {
12969 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");
12970 if (!NeedToPush)
12971 return;
12973 CGM.getOpenMPRuntime().NontemporalDeclsStack.emplace_back();
12974 for (const auto *C : S.getClausesOfKind<OMPNontemporalClause>()) {
12975 for (const Stmt *Ref : C->private_refs()) {
12976 const auto *SimpleRefExpr = cast<Expr>(Ref)->IgnoreParenImpCasts();
12977 const ValueDecl *VD;
12978 if (const auto *DRE = dyn_cast<DeclRefExpr>(SimpleRefExpr)) {
12979 VD = DRE->getDecl();
12980 } else {
12981 const auto *ME = cast<MemberExpr>(SimpleRefExpr);
12982 assert((ME->isImplicitCXXThis() ||
12983 isa<CXXThisExpr>(ME->getBase()->IgnoreParenImpCasts())) &&
12984 "Expected member of current class.");
12985 VD = ME->getMemberDecl();
12986 }
12987 DS.insert(VD);
12988 }
12989 }
12990}
12991
12993 if (!NeedToPush)
12994 return;
12995 CGM.getOpenMPRuntime().NontemporalDeclsStack.pop_back();
12996}
12997
12999 CodeGenFunction &CGF,
13000 const llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
13001 std::pair<Address, Address>> &LocalVars)
13002 : CGM(CGF.CGM), NeedToPush(!LocalVars.empty()) {
13003 if (!NeedToPush)
13004 return;
13005 CGM.getOpenMPRuntime().FunctionToUntiedTaskStackMap.try_emplace(
13006 CGF.CurFn, CGM.getOpenMPRuntime().UntiedLocalVarsStack.size());
13007 CGM.getOpenMPRuntime().UntiedLocalVarsStack.push_back(LocalVars);
13008}
13009
13011 if (!NeedToPush)
13012 return;
13013 CGM.getOpenMPRuntime().UntiedLocalVarsStack.pop_back();
13014}
13015
13017 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");
13018
13019 return llvm::any_of(
13020 CGM.getOpenMPRuntime().NontemporalDeclsStack,
13021 [VD](const NontemporalDeclsSet &Set) { return Set.contains(VD); });
13022}
13023
13024void CGOpenMPRuntime::LastprivateConditionalRAII::tryToDisableInnerAnalysis(
13025 const OMPExecutableDirective &S,
13026 llvm::DenseSet<CanonicalDeclPtr<const Decl>> &NeedToAddForLPCsAsDisabled)
13027 const {
13028 llvm::DenseSet<CanonicalDeclPtr<const Decl>> NeedToCheckForLPCs;
13029 // Vars in target/task regions must be excluded completely.
13030 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()) ||
13031 isOpenMPTaskingDirective(S.getDirectiveKind())) {
13033 getOpenMPCaptureRegions(CaptureRegions, S.getDirectiveKind());
13034 const CapturedStmt *CS = S.getCapturedStmt(CaptureRegions.front());
13035 for (const CapturedStmt::Capture &Cap : CS->captures()) {
13036 if (Cap.capturesVariable() || Cap.capturesVariableByCopy())
13037 NeedToCheckForLPCs.insert(Cap.getCapturedVar());
13038 }
13039 }
13040 // Exclude vars in private clauses.
13041 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
13042 for (const Expr *Ref : C->varlist()) {
13043 if (!Ref->getType()->isScalarType())
13044 continue;
13045 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
13046 if (!DRE)
13047 continue;
13048 NeedToCheckForLPCs.insert(DRE->getDecl());
13049 }
13050 }
13051 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
13052 for (const Expr *Ref : C->varlist()) {
13053 if (!Ref->getType()->isScalarType())
13054 continue;
13055 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
13056 if (!DRE)
13057 continue;
13058 NeedToCheckForLPCs.insert(DRE->getDecl());
13059 }
13060 }
13061 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
13062 for (const Expr *Ref : C->varlist()) {
13063 if (!Ref->getType()->isScalarType())
13064 continue;
13065 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
13066 if (!DRE)
13067 continue;
13068 NeedToCheckForLPCs.insert(DRE->getDecl());
13069 }
13070 }
13071 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
13072 for (const Expr *Ref : C->varlist()) {
13073 if (!Ref->getType()->isScalarType())
13074 continue;
13075 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
13076 if (!DRE)
13077 continue;
13078 NeedToCheckForLPCs.insert(DRE->getDecl());
13079 }
13080 }
13081 for (const auto *C : S.getClausesOfKind<OMPLinearClause>()) {
13082 for (const Expr *Ref : C->varlist()) {
13083 if (!Ref->getType()->isScalarType())
13084 continue;
13085 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
13086 if (!DRE)
13087 continue;
13088 NeedToCheckForLPCs.insert(DRE->getDecl());
13089 }
13090 }
13091 for (const Decl *VD : NeedToCheckForLPCs) {
13092 for (const LastprivateConditionalData &Data :
13093 llvm::reverse(CGM.getOpenMPRuntime().LastprivateConditionalStack)) {
13094 if (Data.DeclToUniqueName.count(VD) > 0) {
13095 if (!Data.Disabled)
13096 NeedToAddForLPCsAsDisabled.insert(VD);
13097 break;
13098 }
13099 }
13100 }
13101}
13102
13103CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
13104 CodeGenFunction &CGF, const OMPExecutableDirective &S, LValue IVLVal)
13105 : CGM(CGF.CGM),
13106 Action((CGM.getLangOpts().OpenMP >= 50 &&
13107 llvm::any_of(S.getClausesOfKind<OMPLastprivateClause>(),
13108 [](const OMPLastprivateClause *C) {
13109 return C->getKind() ==
13110 OMPC_LASTPRIVATE_conditional;
13111 }))
13112 ? ActionToDo::PushAsLastprivateConditional
13113 : ActionToDo::DoNotPush) {
13114 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");
13115 if (CGM.getLangOpts().OpenMP < 50 || Action == ActionToDo::DoNotPush)
13116 return;
13117 assert(Action == ActionToDo::PushAsLastprivateConditional &&
13118 "Expected a push action.");
13120 CGM.getOpenMPRuntime().LastprivateConditionalStack.emplace_back();
13121 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
13122 if (C->getKind() != OMPC_LASTPRIVATE_conditional)
13123 continue;
13124
13125 for (const Expr *Ref : C->varlist()) {
13126 Data.DeclToUniqueName.insert(std::make_pair(
13127 cast<DeclRefExpr>(Ref->IgnoreParenImpCasts())->getDecl(),
13128 SmallString<16>(generateUniqueName(CGM, "pl_cond", Ref))));
13129 }
13130 }
13131 Data.IVLVal = IVLVal;
13132 Data.Fn = CGF.CurFn;
13133}
13134
13135CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
13137 : CGM(CGF.CGM), Action(ActionToDo::DoNotPush) {
13138 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");
13139 if (CGM.getLangOpts().OpenMP < 50)
13140 return;
13141 llvm::DenseSet<CanonicalDeclPtr<const Decl>> NeedToAddForLPCsAsDisabled;
13142 tryToDisableInnerAnalysis(S, NeedToAddForLPCsAsDisabled);
13143 if (!NeedToAddForLPCsAsDisabled.empty()) {
13144 Action = ActionToDo::DisableLastprivateConditional;
13145 LastprivateConditionalData &Data =
13147 for (const Decl *VD : NeedToAddForLPCsAsDisabled)
13148 Data.DeclToUniqueName.try_emplace(VD);
13149 Data.Fn = CGF.CurFn;
13150 Data.Disabled = true;
13151 }
13152}
13153
13154CGOpenMPRuntime::LastprivateConditionalRAII
13156 CodeGenFunction &CGF, const OMPExecutableDirective &S) {
13157 return LastprivateConditionalRAII(CGF, S);
13158}
13159
13161 if (CGM.getLangOpts().OpenMP < 50)
13162 return;
13163 if (Action == ActionToDo::DisableLastprivateConditional) {
13164 assert(CGM.getOpenMPRuntime().LastprivateConditionalStack.back().Disabled &&
13165 "Expected list of disabled private vars.");
13166 CGM.getOpenMPRuntime().LastprivateConditionalStack.pop_back();
13167 }
13168 if (Action == ActionToDo::PushAsLastprivateConditional) {
13169 assert(
13170 !CGM.getOpenMPRuntime().LastprivateConditionalStack.back().Disabled &&
13171 "Expected list of lastprivate conditional vars.");
13172 CGM.getOpenMPRuntime().LastprivateConditionalStack.pop_back();
13173 }
13174}
13175
13177 const VarDecl *VD) {
13178 ASTContext &C = CGM.getContext();
13179 auto I = LastprivateConditionalToTypes.try_emplace(CGF.CurFn).first;
13180 QualType NewType;
13181 const FieldDecl *VDField;
13182 const FieldDecl *FiredField;
13183 LValue BaseLVal;
13184 auto VI = I->getSecond().find(VD);
13185 if (VI == I->getSecond().end()) {
13186 RecordDecl *RD = C.buildImplicitRecord("lasprivate.conditional");
13187 RD->startDefinition();
13188 VDField = addFieldToRecordDecl(C, RD, VD->getType().getNonReferenceType());
13189 FiredField = addFieldToRecordDecl(C, RD, C.CharTy);
13190 RD->completeDefinition();
13191 NewType = C.getCanonicalTagType(RD);
13192 Address Addr = CGF.CreateMemTemp(NewType, C.getDeclAlign(VD), VD->getName());
13193 BaseLVal = CGF.MakeAddrLValue(Addr, NewType, AlignmentSource::Decl);
13194 I->getSecond().try_emplace(VD, NewType, VDField, FiredField, BaseLVal);
13195 } else {
13196 NewType = std::get<0>(VI->getSecond());
13197 VDField = std::get<1>(VI->getSecond());
13198 FiredField = std::get<2>(VI->getSecond());
13199 BaseLVal = std::get<3>(VI->getSecond());
13200 }
13201 LValue FiredLVal =
13202 CGF.EmitLValueForField(BaseLVal, FiredField);
13204 llvm::ConstantInt::getNullValue(CGF.ConvertTypeForMem(C.CharTy)),
13205 FiredLVal);
13206 return CGF.EmitLValueForField(BaseLVal, VDField).getAddress();
13207}
13208
13209namespace {
13210/// Checks if the lastprivate conditional variable is referenced in LHS.
13211class LastprivateConditionalRefChecker final
13212 : public ConstStmtVisitor<LastprivateConditionalRefChecker, bool> {
13214 const Expr *FoundE = nullptr;
13215 const Decl *FoundD = nullptr;
13216 StringRef UniqueDeclName;
13217 LValue IVLVal;
13218 llvm::Function *FoundFn = nullptr;
13219 SourceLocation Loc;
13220
13221public:
13222 bool VisitDeclRefExpr(const DeclRefExpr *E) {
13224 llvm::reverse(LPM)) {
13225 auto It = D.DeclToUniqueName.find(E->getDecl());
13226 if (It == D.DeclToUniqueName.end())
13227 continue;
13228 if (D.Disabled)
13229 return false;
13230 FoundE = E;
13231 FoundD = E->getDecl()->getCanonicalDecl();
13232 UniqueDeclName = It->second;
13233 IVLVal = D.IVLVal;
13234 FoundFn = D.Fn;
13235 break;
13236 }
13237 return FoundE == E;
13238 }
13239 bool VisitMemberExpr(const MemberExpr *E) {
13241 return false;
13243 llvm::reverse(LPM)) {
13244 auto It = D.DeclToUniqueName.find(E->getMemberDecl());
13245 if (It == D.DeclToUniqueName.end())
13246 continue;
13247 if (D.Disabled)
13248 return false;
13249 FoundE = E;
13250 FoundD = E->getMemberDecl()->getCanonicalDecl();
13251 UniqueDeclName = It->second;
13252 IVLVal = D.IVLVal;
13253 FoundFn = D.Fn;
13254 break;
13255 }
13256 return FoundE == E;
13257 }
13258 bool VisitStmt(const Stmt *S) {
13259 for (const Stmt *Child : S->children()) {
13260 if (!Child)
13261 continue;
13262 if (const auto *E = dyn_cast<Expr>(Child))
13263 if (!E->isGLValue())
13264 continue;
13265 if (Visit(Child))
13266 return true;
13267 }
13268 return false;
13269 }
13270 explicit LastprivateConditionalRefChecker(
13271 ArrayRef<CGOpenMPRuntime::LastprivateConditionalData> LPM)
13272 : LPM(LPM) {}
13273 std::tuple<const Expr *, const Decl *, StringRef, LValue, llvm::Function *>
13274 getFoundData() const {
13275 return std::make_tuple(FoundE, FoundD, UniqueDeclName, IVLVal, FoundFn);
13276 }
13277};
13278} // namespace
13279
13281 LValue IVLVal,
13282 StringRef UniqueDeclName,
13283 LValue LVal,
13284 SourceLocation Loc) {
13285 // Last updated loop counter for the lastprivate conditional var.
13286 // int<xx> last_iv = 0;
13287 llvm::Type *LLIVTy = CGF.ConvertTypeForMem(IVLVal.getType());
13288 llvm::Constant *LastIV = OMPBuilder.getOrCreateInternalVariable(
13289 LLIVTy, getName({UniqueDeclName, "iv"}));
13290 cast<llvm::GlobalVariable>(LastIV)->setAlignment(
13291 IVLVal.getAlignment().getAsAlign());
13292 LValue LastIVLVal =
13293 CGF.MakeNaturalAlignRawAddrLValue(LastIV, IVLVal.getType());
13294
13295 // Last value of the lastprivate conditional.
13296 // decltype(priv_a) last_a;
13297 llvm::GlobalVariable *Last = OMPBuilder.getOrCreateInternalVariable(
13298 CGF.ConvertTypeForMem(LVal.getType()), UniqueDeclName);
13299 cast<llvm::GlobalVariable>(Last)->setAlignment(
13300 LVal.getAlignment().getAsAlign());
13301 LValue LastLVal =
13302 CGF.MakeRawAddrLValue(Last, LVal.getType(), LVal.getAlignment());
13303
13304 // Global loop counter. Required to handle inner parallel-for regions.
13305 // iv
13306 llvm::Value *IVVal = CGF.EmitLoadOfScalar(IVLVal, Loc);
13307
13308 // #pragma omp critical(a)
13309 // if (last_iv <= iv) {
13310 // last_iv = iv;
13311 // last_a = priv_a;
13312 // }
13313 auto &&CodeGen = [&LastIVLVal, &IVLVal, IVVal, &LVal, &LastLVal,
13314 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
13315 Action.Enter(CGF);
13316 llvm::Value *LastIVVal = CGF.EmitLoadOfScalar(LastIVLVal, Loc);
13317 // (last_iv <= iv) ? Check if the variable is updated and store new
13318 // value in global var.
13319 llvm::Value *CmpRes;
13320 if (IVLVal.getType()->isSignedIntegerType()) {
13321 CmpRes = CGF.Builder.CreateICmpSLE(LastIVVal, IVVal);
13322 } else {
13323 assert(IVLVal.getType()->isUnsignedIntegerType() &&
13324 "Loop iteration variable must be integer.");
13325 CmpRes = CGF.Builder.CreateICmpULE(LastIVVal, IVVal);
13326 }
13327 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("lp_cond_then");
13328 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("lp_cond_exit");
13329 CGF.Builder.CreateCondBr(CmpRes, ThenBB, ExitBB);
13330 // {
13331 CGF.EmitBlock(ThenBB);
13332
13333 // last_iv = iv;
13334 CGF.EmitStoreOfScalar(IVVal, LastIVLVal);
13335
13336 // last_a = priv_a;
13337 switch (CGF.getEvaluationKind(LVal.getType())) {
13338 case TEK_Scalar: {
13339 llvm::Value *PrivVal = CGF.EmitLoadOfScalar(LVal, Loc);
13340 CGF.EmitStoreOfScalar(PrivVal, LastLVal);
13341 break;
13342 }
13343 case TEK_Complex: {
13344 CodeGenFunction::ComplexPairTy PrivVal = CGF.EmitLoadOfComplex(LVal, Loc);
13345 CGF.EmitStoreOfComplex(PrivVal, LastLVal, /*isInit=*/false);
13346 break;
13347 }
13348 case TEK_Aggregate:
13349 llvm_unreachable(
13350 "Aggregates are not supported in lastprivate conditional.");
13351 }
13352 // }
13353 CGF.EmitBranch(ExitBB);
13354 // There is no need to emit line number for unconditional branch.
13356 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
13357 };
13358
13359 if (CGM.getLangOpts().OpenMPSimd) {
13360 // Do not emit as a critical region as no parallel region could be emitted.
13361 RegionCodeGenTy ThenRCG(CodeGen);
13362 ThenRCG(CGF);
13363 } else {
13364 emitCriticalRegion(CGF, UniqueDeclName, CodeGen, Loc);
13365 }
13366}
13367
13369 const Expr *LHS) {
13370 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty())
13371 return;
13372 LastprivateConditionalRefChecker Checker(LastprivateConditionalStack);
13373 if (!Checker.Visit(LHS))
13374 return;
13375 const Expr *FoundE;
13376 const Decl *FoundD;
13377 StringRef UniqueDeclName;
13378 LValue IVLVal;
13379 llvm::Function *FoundFn;
13380 std::tie(FoundE, FoundD, UniqueDeclName, IVLVal, FoundFn) =
13381 Checker.getFoundData();
13382 if (FoundFn != CGF.CurFn) {
13383 // Special codegen for inner parallel regions.
13384 // ((struct.lastprivate.conditional*)&priv_a)->Fired = 1;
13385 auto It = LastprivateConditionalToTypes[FoundFn].find(FoundD);
13386 assert(It != LastprivateConditionalToTypes[FoundFn].end() &&
13387 "Lastprivate conditional is not found in outer region.");
13388 QualType StructTy = std::get<0>(It->getSecond());
13389 const FieldDecl* FiredDecl = std::get<2>(It->getSecond());
13390 LValue PrivLVal = CGF.EmitLValue(FoundE);
13392 PrivLVal.getAddress(),
13393 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(StructTy)),
13394 CGF.ConvertTypeForMem(StructTy));
13395 LValue BaseLVal =
13396 CGF.MakeAddrLValue(StructAddr, StructTy, AlignmentSource::Decl);
13397 LValue FiredLVal = CGF.EmitLValueForField(BaseLVal, FiredDecl);
13398 CGF.EmitAtomicStore(RValue::get(llvm::ConstantInt::get(
13399 CGF.ConvertTypeForMem(FiredDecl->getType()), 1)),
13400 FiredLVal, llvm::AtomicOrdering::Unordered,
13401 /*IsVolatile=*/true, /*isInit=*/false);
13402 return;
13403 }
13404
13405 // Private address of the lastprivate conditional in the current context.
13406 // priv_a
13407 LValue LVal = CGF.EmitLValue(FoundE);
13408 emitLastprivateConditionalUpdate(CGF, IVLVal, UniqueDeclName, LVal,
13409 FoundE->getExprLoc());
13410}
13411
13414 const llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> &IgnoredDecls) {
13415 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty())
13416 return;
13417 auto Range = llvm::reverse(LastprivateConditionalStack);
13418 auto It = llvm::find_if(
13419 Range, [](const LastprivateConditionalData &D) { return !D.Disabled; });
13420 if (It == Range.end() || It->Fn != CGF.CurFn)
13421 return;
13422 auto LPCI = LastprivateConditionalToTypes.find(It->Fn);
13423 assert(LPCI != LastprivateConditionalToTypes.end() &&
13424 "Lastprivates must be registered already.");
13426 getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind());
13427 const CapturedStmt *CS = D.getCapturedStmt(CaptureRegions.back());
13428 for (const auto &Pair : It->DeclToUniqueName) {
13429 const auto *VD = cast<VarDecl>(Pair.first->getCanonicalDecl());
13430 if (!CS->capturesVariable(VD) || IgnoredDecls.contains(VD))
13431 continue;
13432 auto I = LPCI->getSecond().find(Pair.first);
13433 assert(I != LPCI->getSecond().end() &&
13434 "Lastprivate must be rehistered already.");
13435 // bool Cmp = priv_a.Fired != 0;
13436 LValue BaseLVal = std::get<3>(I->getSecond());
13437 LValue FiredLVal =
13438 CGF.EmitLValueForField(BaseLVal, std::get<2>(I->getSecond()));
13439 llvm::Value *Res = CGF.EmitLoadOfScalar(FiredLVal, D.getBeginLoc());
13440 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Res);
13441 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("lpc.then");
13442 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("lpc.done");
13443 // if (Cmp) {
13444 CGF.Builder.CreateCondBr(Cmp, ThenBB, DoneBB);
13445 CGF.EmitBlock(ThenBB);
13446 Address Addr = CGF.GetAddrOfLocalVar(VD);
13447 LValue LVal;
13448 if (VD->getType()->isReferenceType())
13449 LVal = CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(),
13451 else
13452 LVal = CGF.MakeAddrLValue(Addr, VD->getType().getNonReferenceType(),
13454 emitLastprivateConditionalUpdate(CGF, It->IVLVal, Pair.second, LVal,
13455 D.getBeginLoc());
13457 CGF.EmitBlock(DoneBB, /*IsFinal=*/true);
13458 // }
13459 }
13460}
13461
13463 CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD,
13464 SourceLocation Loc) {
13465 if (CGF.getLangOpts().OpenMP < 50)
13466 return;
13467 auto It = LastprivateConditionalStack.back().DeclToUniqueName.find(VD);
13468 assert(It != LastprivateConditionalStack.back().DeclToUniqueName.end() &&
13469 "Unknown lastprivate conditional variable.");
13470 StringRef UniqueName = It->second;
13471 llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(UniqueName);
13472 // The variable was not updated in the region - exit.
13473 if (!GV)
13474 return;
13475 LValue LPLVal = CGF.MakeRawAddrLValue(
13476 GV, PrivLVal.getType().getNonReferenceType(), PrivLVal.getAlignment());
13477 llvm::Value *Res = CGF.EmitLoadOfScalar(LPLVal, Loc);
13478 CGF.EmitStoreOfScalar(Res, PrivLVal);
13479}
13480
13483 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
13484 const RegionCodeGenTy &CodeGen) {
13485 llvm_unreachable("Not supported in SIMD-only mode");
13486}
13487
13490 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
13491 const RegionCodeGenTy &CodeGen) {
13492 llvm_unreachable("Not supported in SIMD-only mode");
13493}
13494
13496 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
13497 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
13498 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
13499 bool Tied, unsigned &NumberOfParts) {
13500 llvm_unreachable("Not supported in SIMD-only mode");
13501}
13502
13504 CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn,
13505 ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond,
13506 llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier,
13507 OpenMPSeverityClauseKind Severity, const Expr *Message) {
13508 llvm_unreachable("Not supported in SIMD-only mode");
13509}
13510
13512 CodeGenFunction &CGF, StringRef CriticalName,
13513 const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc,
13514 const Expr *Hint) {
13515 llvm_unreachable("Not supported in SIMD-only mode");
13516}
13517
13519 const RegionCodeGenTy &MasterOpGen,
13520 SourceLocation Loc) {
13521 llvm_unreachable("Not supported in SIMD-only mode");
13522}
13523
13525 const RegionCodeGenTy &MasterOpGen,
13526 SourceLocation Loc,
13527 const Expr *Filter) {
13528 llvm_unreachable("Not supported in SIMD-only mode");
13529}
13530
13532 SourceLocation Loc) {
13533 llvm_unreachable("Not supported in SIMD-only mode");
13534}
13535
13537 CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen,
13538 SourceLocation Loc) {
13539 llvm_unreachable("Not supported in SIMD-only mode");
13540}
13541
13543 CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen,
13544 SourceLocation Loc, ArrayRef<const Expr *> CopyprivateVars,
13546 ArrayRef<const Expr *> AssignmentOps) {
13547 llvm_unreachable("Not supported in SIMD-only mode");
13548}
13549
13551 const RegionCodeGenTy &OrderedOpGen,
13552 SourceLocation Loc,
13553 bool IsThreads) {
13554 llvm_unreachable("Not supported in SIMD-only mode");
13555}
13556
13558 SourceLocation Loc,
13560 bool EmitChecks,
13561 bool ForceSimpleCall) {
13562 llvm_unreachable("Not supported in SIMD-only mode");
13563}
13564
13567 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
13568 bool Ordered, const DispatchRTInput &DispatchValues) {
13569 llvm_unreachable("Not supported in SIMD-only mode");
13570}
13571
13573 SourceLocation Loc) {
13574 llvm_unreachable("Not supported in SIMD-only mode");
13575}
13576
13579 const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) {
13580 llvm_unreachable("Not supported in SIMD-only mode");
13581}
13582
13585 OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values) {
13586 llvm_unreachable("Not supported in SIMD-only mode");
13587}
13588
13590 SourceLocation Loc,
13591 unsigned IVSize,
13592 bool IVSigned) {
13593 llvm_unreachable("Not supported in SIMD-only mode");
13594}
13595
13597 SourceLocation Loc,
13598 OpenMPDirectiveKind DKind) {
13599 llvm_unreachable("Not supported in SIMD-only mode");
13600}
13601
13603 SourceLocation Loc,
13604 unsigned IVSize, bool IVSigned,
13605 Address IL, Address LB,
13606 Address UB, Address ST) {
13607 llvm_unreachable("Not supported in SIMD-only mode");
13608}
13609
13611 CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc,
13613 SourceLocation SeverityLoc, const Expr *Message,
13614 SourceLocation MessageLoc) {
13615 llvm_unreachable("Not supported in SIMD-only mode");
13616}
13617
13619 ProcBindKind ProcBind,
13620 SourceLocation Loc) {
13621 llvm_unreachable("Not supported in SIMD-only mode");
13622}
13623
13625 const VarDecl *VD,
13626 Address VDAddr,
13627 SourceLocation Loc) {
13628 llvm_unreachable("Not supported in SIMD-only mode");
13629}
13630
13632 const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit,
13633 CodeGenFunction *CGF) {
13634 llvm_unreachable("Not supported in SIMD-only mode");
13635}
13636
13638 CodeGenFunction &CGF, QualType VarType, StringRef Name) {
13639 llvm_unreachable("Not supported in SIMD-only mode");
13640}
13641
13644 SourceLocation Loc,
13645 llvm::AtomicOrdering AO) {
13646 llvm_unreachable("Not supported in SIMD-only mode");
13647}
13648
13650 const OMPExecutableDirective &D,
13651 llvm::Function *TaskFunction,
13652 QualType SharedsTy, Address Shareds,
13653 const Expr *IfCond,
13654 const OMPTaskDataTy &Data) {
13655 llvm_unreachable("Not supported in SIMD-only mode");
13656}
13657
13660 llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds,
13661 const Expr *IfCond, const OMPTaskDataTy &Data) {
13662 llvm_unreachable("Not supported in SIMD-only mode");
13663}
13664
13668 ArrayRef<const Expr *> ReductionOps, ReductionOptionsTy Options) {
13669 assert(Options.SimpleReduction && "Only simple reduction is expected.");
13670 CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs,
13671 ReductionOps, Options);
13672}
13673
13676 ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) {
13677 llvm_unreachable("Not supported in SIMD-only mode");
13678}
13679
13681 SourceLocation Loc,
13682 bool IsWorksharingReduction) {
13683 llvm_unreachable("Not supported in SIMD-only mode");
13684}
13685
13687 SourceLocation Loc,
13688 ReductionCodeGen &RCG,
13689 unsigned N) {
13690 llvm_unreachable("Not supported in SIMD-only mode");
13691}
13692
13694 SourceLocation Loc,
13695 llvm::Value *ReductionsPtr,
13696 LValue SharedLVal) {
13697 llvm_unreachable("Not supported in SIMD-only mode");
13698}
13699
13701 SourceLocation Loc,
13702 const OMPTaskDataTy &Data) {
13703 llvm_unreachable("Not supported in SIMD-only mode");
13704}
13705
13708 OpenMPDirectiveKind CancelRegion) {
13709 llvm_unreachable("Not supported in SIMD-only mode");
13710}
13711
13713 SourceLocation Loc, const Expr *IfCond,
13714 OpenMPDirectiveKind CancelRegion) {
13715 llvm_unreachable("Not supported in SIMD-only mode");
13716}
13717
13719 const OMPExecutableDirective &D, StringRef ParentName,
13720 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
13721 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
13722 llvm_unreachable("Not supported in SIMD-only mode");
13723}
13724
13727 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
13728 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
13729 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
13730 const OMPLoopDirective &D)>
13731 SizeEmitter) {
13732 llvm_unreachable("Not supported in SIMD-only mode");
13733}
13734
13736 llvm_unreachable("Not supported in SIMD-only mode");
13737}
13738
13740 llvm_unreachable("Not supported in SIMD-only mode");
13741}
13742
13744 return false;
13745}
13746
13748 const OMPExecutableDirective &D,
13749 SourceLocation Loc,
13750 llvm::Function *OutlinedFn,
13751 ArrayRef<llvm::Value *> CapturedVars) {
13752 llvm_unreachable("Not supported in SIMD-only mode");
13753}
13754
13756 const Expr *NumTeams,
13757 const Expr *ThreadLimit,
13758 SourceLocation Loc) {
13759 llvm_unreachable("Not supported in SIMD-only mode");
13760}
13761
13763 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
13764 const Expr *Device, const RegionCodeGenTy &CodeGen,
13766 llvm_unreachable("Not supported in SIMD-only mode");
13767}
13768
13770 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
13771 const Expr *Device) {
13772 llvm_unreachable("Not supported in SIMD-only mode");
13773}
13774
13776 const OMPLoopDirective &D,
13777 ArrayRef<Expr *> NumIterations) {
13778 llvm_unreachable("Not supported in SIMD-only mode");
13779}
13780
13782 const OMPDependClause *C) {
13783 llvm_unreachable("Not supported in SIMD-only mode");
13784}
13785
13787 const OMPDoacrossClause *C) {
13788 llvm_unreachable("Not supported in SIMD-only mode");
13789}
13790
13791const VarDecl *
13793 const VarDecl *NativeParam) const {
13794 llvm_unreachable("Not supported in SIMD-only mode");
13795}
13796
13797Address
13799 const VarDecl *NativeParam,
13800 const VarDecl *TargetParam) const {
13801 llvm_unreachable("Not supported in SIMD-only mode");
13802}
#define V(N, I)
static llvm::Value * emitCopyprivateCopyFunction(CodeGenModule &CGM, llvm::Type *ArgsElemType, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps, SourceLocation Loc)
static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF, SourceLocation Loc, SmallString< 128 > &Buffer)
static void emitOffloadingArraysAndArgs(CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo, CGOpenMPRuntime::TargetDataInfo &Info, llvm::OpenMPIRBuilder &OMPBuilder, bool IsNonContiguous=false, bool ForEndCall=false)
Emit the arrays used to pass the captures and map information to the offloading runtime library.
static RecordDecl * createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy, ArrayRef< PrivateDataTy > Privates)
static void emitInitWithReductionInitializer(CodeGenFunction &CGF, const OMPDeclareReductionDecl *DRD, const Expr *InitOp, Address Private, Address Original, QualType Ty)
static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, Address OriginalBaseAddress, llvm::Value *Addr)
static void emitPrivatesInit(CodeGenFunction &CGF, const OMPExecutableDirective &D, Address KmpTaskSharedsPtr, LValue TDBase, const RecordDecl *KmpTaskTWithPrivatesQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef< PrivateDataTy > Privates, bool ForDup)
Emit initialization for private variables in task-based directives.
static void emitClauseForBareTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &Values)
static llvm::Value * emitDestructorsFunction(CodeGenModule &CGM, SourceLocation Loc, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy)
static void EmitOMPAggregateReduction(CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar, const VarDecl *RHSVar, const llvm::function_ref< void(CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *)> &RedOpGen, const Expr *XExpr=nullptr, const Expr *EExpr=nullptr, const Expr *UpExpr=nullptr)
Emit reduction operation for each element of array (required for array sections) LHS op = RHS.
static void emitTargetCallFallback(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, CodeGenFunction &CGF)
static llvm::Value * emitReduceInitFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Emits reduction initializer function:
static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion)
static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy, llvm::PointerUnion< unsigned *, LValue * > Pos, const OMPTaskDataTy::DependData &Data, Address DependenciesArray)
static llvm::Value * emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc, const OMPTaskDataTy &Data, QualType PrivatesQTy, ArrayRef< PrivateDataTy > Privates)
Emit a privates mapping function for correct handling of private and firstprivate variables.
static llvm::Value * emitReduceCombFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N, const Expr *ReductionOp, const Expr *LHS, const Expr *RHS, const Expr *PrivateRef)
Emits reduction combiner function:
static RecordDecl * createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef< PrivateDataTy > Privates)
static llvm::Value * getAllocatorVal(CodeGenFunction &CGF, const Expr *Allocator)
Return allocator value from expression, or return a null allocator (default when no allocator specifi...
static llvm::Function * emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy, QualType SharedsPtrTy, llvm::Function *TaskFunction, llvm::Value *TaskPrivatesMap)
Emit a proxy function which accepts kmp_task_t as the second argument.
static bool isAllocatableDecl(const VarDecl *VD)
static llvm::Value * getAlignmentValue(CodeGenModule &CGM, const VarDecl *VD)
Return the alignment from an allocate directive if present.
static void emitTargetCallKernelLaunch(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::Value *OutlinedFnID, CodeGenFunction::OMPTargetDataInfo &InputInfo, llvm::Value *&MapTypesArray, llvm::Value *&MapNamesArray, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter, CodeGenFunction &CGF, CodeGenModule &CGM)
static const OMPExecutableDirective * getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D)
Check for inner distribute directive.
static std::pair< llvm::Value *, llvm::Value * > getPointerAndSize(CodeGenFunction &CGF, const Expr *E)
static const VarDecl * getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE)
static bool getAArch64MTV(QualType QT, llvm::OpenMPIRBuilder::DeclareSimdKindTy Kind)
Maps To Vector (MTV), as defined in 4.1.1 of the AAVFABI (2021Q1).
static bool isTrivial(ASTContext &Ctx, const Expr *E)
Checks if the expression is constant or does not have non-trivial function calls.
static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind, bool Chunked, bool Ordered)
Map the OpenMP loop schedule to the runtime enumeration.
static void getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS, const Expr **E, int32_t &UpperBound, bool UpperBoundOnly, llvm::Value **CondVal)
Check for a num threads constant value (stored in DefaultVal), or expression (stored in E).
static llvm::Value * emitDeviceID(llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, CodeGenFunction &CGF)
static const OMPDeclareReductionDecl * getReductionInit(const Expr *ReductionOp)
Check if the combiner is a call to UDR combiner and if it is so return the UDR decl used for reductio...
static bool checkInitIsRequired(CodeGenFunction &CGF, ArrayRef< PrivateDataTy > Privates)
Check if duplication function is required for taskloops.
static bool validateAArch64Simdlen(CodeGenModule &CGM, SourceLocation SLoc, unsigned UserVLEN, unsigned WDS, char ISA)
static bool checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD, ArrayRef< PrivateDataTy > Privates)
Checks if destructor function is required to be generated.
static llvm::TargetRegionEntryInfo getEntryInfoFromPresumedLoc(CodeGenModule &CGM, llvm::OpenMPIRBuilder &OMPBuilder, SourceLocation BeginLoc, llvm::StringRef ParentName="")
static void genMapInfo(MappableExprsHandler &MEHandler, CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo, llvm::OpenMPIRBuilder &OMPBuilder, const llvm::DenseSet< CanonicalDeclPtr< const Decl > > &SkippedVarSet=llvm::DenseSet< CanonicalDeclPtr< const Decl > >())
static unsigned getAArch64LS(QualType QT, llvm::OpenMPIRBuilder::DeclareSimdKindTy Kind, ASTContext &C)
Computes the lane size (LS) of a return type or of an input parameter, as defined by LS(P) in 3....
static llvm::OpenMPIRBuilder::DeclareSimdBranch convertDeclareSimdBranch(OMPDeclareSimdDeclAttr::BranchStateTy State)
static void emitForStaticInitCall(CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId, llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2, const CGOpenMPRuntime::StaticRTInput &Values)
static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, LValue BaseLV)
static void getKmpAffinityType(ASTContext &C, QualType &KmpTaskAffinityInfoTy)
Builds kmp_depend_info, if it is not built yet, and builds flags type.
static llvm::Constant * emitMappingInformation(CodeGenFunction &CGF, llvm::OpenMPIRBuilder &OMPBuilder, MappableExprsHandler::MappingExprInfo &MapExprs)
Emit a string constant containing the names of the values mapped to the offloading runtime library.
static void getDependTypes(ASTContext &C, QualType &KmpDependInfoTy, QualType &FlagsTy)
Builds kmp_depend_info, if it is not built yet, and builds flags type.
static llvm::Value * emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc, const OMPExecutableDirective &D, QualType KmpTaskTWithPrivatesPtrQTy, const RecordDecl *KmpTaskTWithPrivatesQTyRD, const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef< PrivateDataTy > Privates, bool WithLastIter)
Emit task_dup function (for initialization of private/firstprivate/lastprivate vars and last_iter fla...
static std::pair< llvm::Value *, OMPDynGroupprivateFallbackType > emitDynCGroupMem(const OMPExecutableDirective &D, CodeGenFunction &CGF)
static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind convertDeviceClause(const VarDecl *VD)
static llvm::Value * emitReduceFiniFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Emits reduction finalizer function:
static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, QualType Type, bool EmitDeclareReductionInit, const Expr *Init, const OMPDeclareReductionDecl *DRD, Address SrcAddr=Address::invalid())
Emit initialization of arrays of complex types.
static bool getAArch64PBV(QualType QT, ASTContext &C)
Pass By Value (PBV), as defined in 3.1.2 of the AAVFABI.
static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM, const T *C, llvm::Value *ULoc, llvm::Value *ThreadID)
static RTLDependenceKindTy translateDependencyKind(OpenMPDependClauseKind K)
Translates internal dependency kind into the runtime kind.
static void emitTargetCallElse(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, CodeGenFunction &CGF)
static llvm::Function * emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty, const Expr *CombinerInitializer, const VarDecl *In, const VarDecl *Out, bool IsCombiner)
static void emitReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp)
Emit reduction combiner.
static std::tuple< unsigned, unsigned, bool > getNDSWDS(const FunctionDecl *FD, ArrayRef< llvm::OpenMPIRBuilder::DeclareSimdAttrTy > ParamAttrs)
static std::string generateUniqueName(CodeGenModule &CGM, llvm::StringRef Prefix, const Expr *Ref)
static llvm::Function * emitParallelOrTeamsOutlinedFunction(CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen)
static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array, unsigned Index, const VarDecl *Var)
Given an array of pointers to variables, project the address of a given variable.
static const VarDecl * getOriginalVarDecl(const ValueDecl *Decl)
For BindingDecls, returns the DecomposedDecl as the original VarDecl.
static void mergeThreadCountUpperBound(int32_t &UpperBound, int32_t Val)
Merge the thread count upper bound Val into UpperBound.
static FieldDecl * addFieldToRecordDecl(ASTContext &C, DeclContext *DC, QualType FieldTy)
static unsigned evaluateCDTSize(const FunctionDecl *FD, ArrayRef< llvm::OpenMPIRBuilder::DeclareSimdAttrTy > ParamAttrs)
static ValueDecl * getDeclFromThisExpr(const Expr *E)
static void genMapInfoForCaptures(MappableExprsHandler &MEHandler, CodeGenFunction &CGF, const CapturedStmt &CS, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars, llvm::OpenMPIRBuilder &OMPBuilder, llvm::DenseSet< CanonicalDeclPtr< const Decl > > &MappedVarSet, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo)
static RecordDecl * createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpRoutineEntryPointerQTy)
static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2)
static mlir::omp::DeclareTargetCaptureClause convertCaptureClause(OMPDeclareTargetDeclAttr::MapTypeTy mapTy)
static bool isAssumedToBeNotEmitted(const ValueDecl *vd, bool isDevice)
Returns true if the declaration should be skipped based on its device_type attribute and the current ...
Expr::Classification Cl
TokenType getType() const
Returns the token's type, e.g.
FormatToken * Next
The next token in the unwrapped line.
Result
Implement __builtin_bit_cast and related operations.
#define X(type, name)
Definition Value.h:97
This file defines OpenMP AST classes for clauses.
Defines some OpenMP-specific enums and functions.
llvm::json::Array Array
Defines the SourceManager interface.
This file defines OpenMP AST classes for executable directives and clauses.
__DEVICE__ int max(int __a, int __b)
This represents clause 'affinity' in the 'pragma omp task'-based directives.
static std::pair< const Expr *, std::optional< size_t > > findAttachPtrExpr(MappableExprComponentListRef Components, OpenMPDirectiveKind CurDirKind)
Find the attach pointer expression from a list of mappable expression components.
static QualType getComponentExprElementType(const Expr *Exp)
Get the type of an element of a ComponentList Expr Exp.
ArrayRef< MappableComponent > MappableExprComponentListRef
This represents implicit clause 'depend' for the 'pragma omp task' directive.
This represents 'detach' clause in the 'pragma omp task' directive.
This represents 'device' clause in the 'pragma omp ...' directive.
This represents the 'doacross' clause for the 'pragma omp ordered' directive.
This represents 'dyn_groupprivate' clause in 'pragma omp target ...' and 'pragma omp teams ....
This is a common base class for loop directives ('omp simd', 'omp for', 'omp for simd' etc....
Expr * getLowerBoundVariable() const
Expr * getUpperBoundVariable() const
Expr * getStrideVariable() const
This represents clause 'map' in the 'pragma omp ...' directives.
This represents clause 'nontemporal' in the 'pragma omp ...' directives.
This represents 'num_teams' clause in the 'pragma omp ...' directive.
This represents 'thread_limit' clause in the 'pragma omp ...' directive.
This represents clause 'uses_allocators' in the 'pragma omp target'-based directives.
This represents 'ompx_attribute' clause in a directive that might generate an outlined function.
This represents 'ompx_bare' clause in the 'pragma omp target teams ...' directive.
This represents 'ompx_dyn_cgroup_mem' clause in the 'pragma omp target ...' directive.
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition ASTContext.h:245
SourceManager & getSourceManager()
Definition ASTContext.h:913
const ConstantArrayType * getAsConstantArrayType(QualType T) const
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
const ASTRecordLayout & getASTRecordLayout(const RecordDecl *D) const
Get or compute information about the layout of the specified record (struct/union/class) D,...
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
CanQualType VoidPtrTy
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
const LangOptions & getLangOpts() const
CanQualType BoolTy
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth,...
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
static bool hasSameType(QualType T1, QualType T2)
Determine whether the given types T1 and T2 are equivalent.
const VariableArrayType * getAsVariableArrayType(QualType T) const
QualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
unsigned getTypeAlign(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in bits.
CharUnits getSize() const
getSize - Get the record size in characters.
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
CharUnits getNonVirtualSize() const
getNonVirtualSize - Get the non-virtual size (in chars) of an object, which is the size of the object...
static QualType getBaseOriginalType(const Expr *Base)
Return original type of the base expression for array section.
Definition Expr.cpp:5442
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Definition TypeBase.h:3820
Attr - This represents one attribute.
Definition Attr.h:47
A binding in a decomposition declaration.
Definition DeclCXX.h:4232
Expr * getBinding() const
Get the expression to which this declaration is bound.
Definition DeclCXX.h:4258
DecompositionDecl * getDecomposedDecl() const
Get the decomposition declaration that this binding represents a decomposition of.
Definition DeclCXX.h:4265
Represents a base class of a C++ class.
Definition DeclCXX.h:146
Represents a C++ constructor within a class.
Definition DeclCXX.h:2645
bool isInheritingConstructor() const
Determine whether this is an implicit constructor synthesized to model a call to a constructor inheri...
Definition DeclCXX.h:2883
Represents a C++ base or member initializer.
Definition DeclCXX.h:2410
Represents a C++ destructor within a class.
Definition DeclCXX.h:2924
Represents a static or instance method of a struct/union/class.
Definition DeclCXX.h:2153
const CXXRecordDecl * getParent() const
Return the parent of this method declaration, which is the class in which this method is defined.
Definition DeclCXX.h:2296
QualType getFunctionObjectParameterType() const
Definition DeclCXX.h:2320
Represents a C++ struct/union/class.
Definition DeclCXX.h:258
base_class_range bases()
Definition DeclCXX.h:609
bool isLambda() const
Determine whether this class describes a lambda function object.
Definition DeclCXX.h:1028
void getCaptureFields(llvm::DenseMap< const ValueDecl *, FieldDecl * > &Captures, FieldDecl *&ThisCapture) const
For a closure type, retrieve the mapping from captured variables and this to the non-static data memb...
Definition DeclCXX.cpp:1792
unsigned getNumBases() const
Retrieves the number of base classes of this class.
Definition DeclCXX.h:603
base_class_range vbases()
Definition DeclCXX.h:626
capture_const_range captures() const
Definition DeclCXX.h:1107
ctor_range ctors() const
Definition DeclCXX.h:671
CXXDestructorDecl * getDestructor() const
Returns the destructor decl for this class.
Definition DeclCXX.cpp:2129
CanProxy< U > castAs() const
A wrapper class around a pointer that always points to its canonical declaration.
Describes the capture of either a variable, or 'this', or variable-length array type.
Definition Stmt.h:3963
bool capturesVariableByCopy() const
Determine whether this capture handles a variable by copy.
Definition Stmt.h:3997
VarDecl * getCapturedVar() const
Retrieve the declaration of the variable being captured.
Definition Stmt.cpp:1391
bool capturesVariableArrayType() const
Determine whether this capture handles a variable-length array type.
Definition Stmt.h:4003
bool capturesThis() const
Determine whether this capture handles the C++ 'this' pointer.
Definition Stmt.h:3991
bool capturesVariable() const
Determine whether this capture handles a variable (by reference).
Definition Stmt.h:3994
This captures a statement into a function.
Definition Stmt.h:3950
const Capture * const_capture_iterator
Definition Stmt.h:4084
capture_iterator capture_end() const
Retrieve an iterator pointing past the end of the sequence of captures.
Definition Stmt.h:4101
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Definition Stmt.h:4071
Stmt * getCapturedStmt()
Retrieve the statement being captured.
Definition Stmt.h:4054
bool capturesVariable(const VarDecl *Var) const
True if this variable has been captured.
Definition Stmt.cpp:1517
capture_iterator capture_begin()
Retrieve an iterator pointing to the first capture.
Definition Stmt.h:4096
capture_range captures()
Definition Stmt.h:4088
This is an opaque type for sizes expressed in character units.
Definition CharUnits.h:38
bool isZero() const
Test whether the quantity equals zero.
Definition CharUnits.h:101
CharUnits alignTo(CharUnits Align) const
Returns the next integer (mod 2**64) that is greater than or equal to this quantity and is a multiple...
Definition CharUnits.h:171
llvm::Align getAsAlign() const
Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit bytes.
Definition CharUnits.h:159
QuantityType getQuantity() const
Get the raw integer representation of this quantity.
Definition CharUnits.h:155
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
Definition CharUnits.h:184
static CharUnits fromQuantity(QuantityType Quantity)
Construct a CharUnits quantity from a raw integer type.
Definition CharUnits.h:58
std::string SampleProfileFile
Name of the profile file to use with -fprofile-sample-use.
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
Definition Address.h:128
static Address invalid()
Definition Address.h:176
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
Definition Address.h:253
CharUnits getAlignment() const
Definition Address.h:194
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Definition Address.h:209
Address withPointer(llvm::Value *NewPointer, KnownNonNull_t IsKnownNonNull) const
Return address with different pointer, but same element type and alignment.
Definition Address.h:261
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Definition Address.h:276
bool isValid() const
Definition Address.h:177
llvm::PointerType * getType() const
Return the type of the pointer value.
Definition Address.h:204
static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateDefaultArtificial(CodeGenFunction &CGF, SourceLocation TemporaryLocation)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF)
Set the IRBuilder to not attach debug locations.
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
Definition CGBuilder.h:156
Address CreateGEP(CodeGenFunction &CGF, Address Addr, llvm::Value *Index, const llvm::Twine &Name="")
Definition CGBuilder.h:312
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Definition CGBuilder.h:223
Address CreateConstArrayGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = [n x T]* ... produce name = getelementptr inbounds addr, i64 0, i64 index where i64 is a...
Definition CGBuilder.h:261
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Definition CGBuilder.h:128
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
Definition CGBuilder.h:407
Address CreateConstGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = T* ... produce name = getelementptr inbounds addr, i64 index where i64 is actually the t...
Definition CGBuilder.h:298
Address CreateAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Definition CGBuilder.h:209
CGFunctionInfo - Class to encapsulate the information about a function definition.
static LastprivateConditionalRAII disable(CodeGenFunction &CGF, const OMPExecutableDirective &S)
NontemporalDeclsRAII(CodeGenModule &CGM, const OMPLoopDirective &S)
Struct that keeps all the relevant information that should be kept throughout a 'target data' region.
llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap
Map between the a declaration of a capture and the corresponding new llvm address where the runtime r...
UntiedTaskLocalDeclsRAII(CodeGenFunction &CGF, const llvm::MapVector< CanonicalDeclPtr< const VarDecl >, std::pair< Address, Address > > &LocalVars)
virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc)
Emits address of the word in a memory where current thread id is stored.
llvm::StringSet ThreadPrivateWithDefinition
Set of threadprivate variables with the generated initializer.
void emitUpdateDependObjectsClause(CodeGenFunction &CGF, LValue DepobjLVal, OpenMPDependClauseKind NewDepKind, SourceLocation Loc)
Updates the dependency kind in the specified depobj object.
virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
Emit task region for the task directive.
void createOffloadEntriesAndInfoMetadata()
Creates all the offload entries in the current compilation unit along with the associated metadata.
const Expr * getNumTeamsExprForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &MinTeamsVal, int32_t &MaxTeamsVal)
Emit the number of teams for a target directive.
virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc)
Returns address of the threadprivate variable for the current thread.
void emitDeferredTargetDecls() const
Emit deferred declare target variables marked for deferred emission.
virtual llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST)
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
bool markAsGlobalTarget(GlobalDecl GD)
Marks the declaration as already emitted for the device code and returns true, if it was marked alrea...
virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond, llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, const Expr *Message=nullptr)
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
llvm::SmallDenseSet< CanonicalDeclPtr< const Decl > > NontemporalDeclsSet
virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device)
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
virtual void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc, OpenMPNumThreadsClauseModifier Modifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, SourceLocation SeverityLoc=SourceLocation(), const Expr *Message=nullptr, SourceLocation MessageLoc=SourceLocation())
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32global_tid, kmp_int32 num_threads) ...
QualType SavedKmpTaskloopTQTy
Saved kmp_task_t for taskloop-based directive.
virtual void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps)
Emits a single region.
virtual bool emitTargetGlobal(GlobalDecl GD)
Emit the global GD if it is meaningful for the target.
void setLocThreadIdInsertPt(CodeGenFunction &CGF, bool AtCurrentPoint=false)
std::string getOutlinedHelperName(StringRef Name) const
Get the function name of an outlined region.
StringRef getTargetRegionParentName(const FunctionDecl *FD) const
Returns the parent name of the target regions in FD: its enclosing non-closure function if FD is a cl...
bool HasEmittedDeclareTargetRegion
Flag for keeping track of weather a device routine has been emitted.
llvm::Constant * getOrCreateThreadPrivateCache(const VarDecl *VD)
If the specified mangled name is not in the module, create and return threadprivate cache object.
virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal)
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
virtual void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc)
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
void emitCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee Callee, ArrayRef< llvm::Value * > Args={}) const
Emits Callee function call with arguments Args with location Loc.
virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF, const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const
Choose default schedule type and chunk value for the schedule clause.
virtual std::pair< llvm::Function *, llvm::Function * > getUserDefinedReduction(const OMPDeclareReductionDecl *D)
Get combiner/initializer for the specified user-defined reduction, if any.
virtual bool isGPU() const
Returns true if the current target is a GPU.
static const Stmt * getSingleCompoundChild(ASTContext &Ctx, const Stmt *Body)
Checks if the Body is the CompoundStmt and returns its child statement iff there is only one that is ...
virtual void emitDeclareTargetFunction(const FunctionDecl *FD, llvm::GlobalValue *GV)
Emit code for handling declare target functions in the runtime.
bool HasRequiresUnifiedSharedMemory
Flag for keeping track of weather a requires unified_shared_memory directive is present.
llvm::Value * emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc, unsigned Flags=0, bool EmitLoc=false)
Emits object of ident_t type with info for source location.
bool isLocalVarInUntiedTask(CodeGenFunction &CGF, const VarDecl *VD) const
Returns true if the variable is a local variable in untied task.
virtual void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars)
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
virtual void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancellation point' construct.
virtual llvm::Function * emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF=nullptr)
Emit a code for initialization of threadprivate variable.
virtual ConstantAddress getAddrOfDeclareTargetVar(const VarDecl *VD)
Returns the address of the variable marked as declare target with link clause OR as declare target wi...
llvm::Function * getOrCreateUserDefinedMapperFunc(const OMPDeclareMapperDecl *D)
Get the function for the specified user-defined mapper.
OpenMPLocThreadIDMapTy OpenMPLocThreadIDMap
virtual void functionFinished(CodeGenFunction &CGF)
Cleans up references to the objects in finished function.
virtual llvm::Function * emitTeamsOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP teams directive D.
QualType KmpTaskTQTy
Type typedef struct kmp_task { void * shareds; /‍**< pointer to block of pointers to shared vars ‍/ k...
llvm::OpenMPIRBuilder OMPBuilder
An OpenMP-IR-Builder instance.
virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr * > NumIterations)
Emit initialization for doacross loop nesting support.
virtual void adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF, const OMPExecutableDirective &D) const
Adjust some parameters for the target-based directives, like addresses of the variables captured by r...
virtual void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, CGOpenMPRuntime::TargetDataInfo &Info)
Emit the target data mapping code associated with D.
virtual unsigned getDefaultLocationReserved2Flags() const
Returns additional flags that can be stored in reserved_2 field of the default location.
virtual Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const
Gets the address of the native argument basing on the address of the target-specific parameter.
void emitUsesAllocatorsFini(CodeGenFunction &CGF, const Expr *Allocator)
Destroys user defined allocators specified in the uses_allocators clause.
QualType KmpTaskAffinityInfoTy
Type typedef struct kmp_task_affinity_info { kmp_intptr_t base_addr; size_t len; struct { bool flag1 ...
void emitPrivateReduction(CodeGenFunction &CGF, SourceLocation Loc, const Expr *Privates, const Expr *LHSExprs, const Expr *RHSExprs, const Expr *ReductionOps)
Emits code for private variable reduction.
llvm::Value * emitNumTeamsForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D)
virtual void emitTargetOutlinedFunctionHelper(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Helper to emit outlined function for 'target' directive.
void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName)
Start scanning from statement S and emit all target regions found along the way.
SmallVector< llvm::Value *, 4 > emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy, const OMPTaskDataTy::DependData &Data)
virtual llvm::Value * emitMessageClause(CodeGenFunction &CGF, const Expr *Message, SourceLocation Loc)
virtual void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc)
Emit a taskgroup region.
llvm::DenseMap< llvm::Function *, llvm::DenseMap< CanonicalDeclPtr< const Decl >, std::tuple< QualType, const FieldDecl *, const FieldDecl *, LValue > > > LastprivateConditionalToTypes
Maps local variables marked as lastprivate conditional to their internal types.
virtual bool emitTargetGlobalVariable(GlobalDecl GD)
Emit the global variable if it is a valid device global variable.
virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32global_tid, kmp_int32 num_teams,...
bool hasRequiresUnifiedSharedMemory() const
Return whether the unified_shared_memory has been specified.
virtual Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, QualType VarType, StringRef Name)
Creates artificial threadprivate variable with name Name and type VarType.
void emitUserDefinedMapper(const OMPDeclareMapperDecl *D, CodeGenFunction *CGF=nullptr)
Emit the function for the user defined mapper construct.
bool HasEmittedTargetRegion
Flag for keeping track of weather a target region has been emitted.
void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy, LValue PosLVal, const OMPTaskDataTy::DependData &Data, Address DependenciesArray)
std::string getReductionFuncName(StringRef Name) const
Get the function name of a reduction function.
virtual void processRequiresDirective(const OMPRequiresDecl *D)
Perform check on requires decl to ensure that target architecture supports unified addressing.
llvm::DenseSet< CanonicalDeclPtr< const Decl > > AlreadyEmittedTargetDecls
List of the emitted declarations.
virtual llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data)
Emit a code for initialization of task reduction clause.
llvm::Value * getThreadID(CodeGenFunction &CGF, SourceLocation Loc)
Gets thread id value for the current thread.
virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD, SourceLocation Loc)
Gets the address of the global copy used for lastprivate conditional update, if any.
llvm::MapVector< CanonicalDeclPtr< const VarDecl >, std::pair< Address, Address > > UntiedLocalVarsAddressesMap
virtual void emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, Expr *ME, bool IsFatal)
Emit __kmpc_error call for error directive extern void __kmpc_error(ident_t *loc, int severity,...
void clearLocThreadIdInsertPt(CodeGenFunction &CGF)
virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc)
Emits code for a taskyield directive.
std::string getName(ArrayRef< StringRef > Parts) const
Get the platform-specific name separator.
void computeMinAndMaxThreadsAndTeams(const OMPExecutableDirective &D, CodeGenFunction &CGF, llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs)
Helper to determine the min/max number of threads/teams for D.
virtual void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc, llvm::AtomicOrdering AO)
Emit flush of the variables specified in 'omp flush' directive.
virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPTaskDataTy &Data)
Emit code for 'taskwait' directive.
virtual void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc)
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32global_tid, int proc_bind) to generate...
void emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LValue IVLVal, StringRef UniqueDeclName, LValue LVal, SourceLocation Loc)
Emit update for lastprivate conditional data.
virtual void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
Emit task region for the taskloop directive.
virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false)
Emit an implicit/explicit barrier for OpenMP threads.
static unsigned getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind)
Returns default flags for the barriers depending on the directive, for which this barier is going to ...
virtual bool emitTargetFunctions(GlobalDecl GD)
Emit the target regions enclosed in GD function definition or the function itself in case it is a val...
TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const OMPTaskDataTy &Data)
Emit task region for the task directive.
llvm::Value * emitTargetNumIterationsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Return the trip count of loops associated with constructs / 'target teams distribute' and 'teams dist...
llvm::StringMap< llvm::AssertingVH< llvm::GlobalVariable >, llvm::BumpPtrAllocator > InternalVars
An ordered map of auto-generated variables to their unique names.
virtual void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values)
llvm::SmallVector< UntiedLocalVarsAddressesMap, 4 > UntiedLocalVarsStack
virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind)
Call the appropriate runtime routine to notify that we finished all the work with current loop.
virtual void emitThreadLimitClause(CodeGenFunction &CGF, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_set_thread_limit(ident_t *loc, kmp_int32global_tid, kmp_int32 thread_limit)...
void emitIfClause(CodeGenFunction &CGF, const Expr *Cond, const RegionCodeGenTy &ThenGen, const RegionCodeGenTy &ElseGen)
Emits code for OpenMP 'if' clause using specified CodeGen function.
Address emitDepobjDependClause(CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies, SourceLocation Loc)
Emits list of dependecies based on the provided data (array of dependence/expression pairs) for depob...
bool isNontemporalDecl(const ValueDecl *VD) const
Checks if the VD variable is marked as nontemporal declaration in current context.
virtual llvm::Function * emitParallelOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP parallel directive D.
const Expr * getNumThreadsExprForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &UpperBound, bool UpperBoundOnly, llvm::Value **CondExpr=nullptr, const Expr **ThreadLimitExpr=nullptr)
Check for a number of threads upper bound constant value (stored in UpperBound), or expression (retur...
virtual void registerVTableOffloadEntry(llvm::GlobalVariable *VTable, const VarDecl *VD)
Register VTable to OpenMP offload entry.
virtual llvm::Value * emitSeverityClause(OpenMPSeverityClauseKind Severity, SourceLocation Loc)
llvm::SmallVector< LastprivateConditionalData, 4 > LastprivateConditionalStack
Stack for list of addresses of declarations in current context marked as lastprivate conditional.
virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values)
Call the appropriate runtime routine to initialize it before start of loop.
virtual void emitDeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn)
Marks function Fn with properly mangled versions of vector functions.
llvm::AtomicOrdering getDefaultMemoryOrdering() const
Gets default memory ordering as specified in requires directive.
virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static non-chunked.
virtual void emitAndRegisterVTable(CodeGenModule &CGM, CXXRecordDecl *CXXRecord, const VarDecl *VD)
Emit and register VTable for the C++ class in OpenMP offload entry.
void scanConstructorForTargetRegions(const CXXConstructorDecl *Ctor, StringRef ParentName)
Scans the mem-initializers and the body of Ctor for target regions.
llvm::Value * getCriticalRegionLock(StringRef CriticalName)
Returns corresponding lock object for the specified critical region name.
virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancel' construct.
QualType SavedKmpTaskTQTy
Saved kmp_task_t for task directive.
virtual void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc)
Emits a master region.
virtual llvm::Function * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts)
Emits outlined function for the OpenMP task directive D.
llvm::DenseMap< llvm::Function *, unsigned > FunctionToUntiedTaskStackMap
Maps function to the position of the untied task locals stack.
void emitHostKernelEnvironment(const OMPExecutableDirective &D, CodeGenFunction &CGF)
Emit the '<kernel>_kernel_environment' global for a target region that is compiled for a non-GPU (hos...
void emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal, SourceLocation Loc)
Emits the code to destroy the dependency object provided in depobj directive.
virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Required to resolve existing problems in the runtime.
llvm::ArrayType * KmpCriticalNameTy
Type kmp_critical_name, originally defined as typedef kmp_int32 kmp_critical_name[8];.
virtual void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C)
Emit code for doacross ordered directive with 'depend' clause.
llvm::DenseMap< const OMPDeclareMapperDecl *, llvm::Function * > UDMMap
Map from the user-defined mapper declaration to its corresponding functions.
virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF, const Expr *LHS)
Checks if the provided LVal is lastprivate conditional and emits the code to update the value of the ...
std::pair< llvm::Value *, LValue > getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal, SourceLocation Loc)
Returns the number of the elements and the address of the depobj dependency array.
llvm::SmallDenseSet< const VarDecl * > DeferredGlobalVariables
List of variables that can become declare target implicitly and, thus, must be emitted.
void emitUsesAllocatorsInit(CodeGenFunction &CGF, const Expr *Allocator, const Expr *AllocatorTraits)
Initializes user defined allocators specified in the uses_allocators clauses.
virtual void registerVTable(const OMPExecutableDirective &D)
Emit code for registering vtable by scanning through map clause in OpenMP target region.
llvm::Type * KmpRoutineEntryPtrTy
Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *);.
llvm::Type * getIdentTyPointerTy()
Returns pointer to ident_t type.
void emitSingleReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp, const Expr *PrivateRef, const DeclRefExpr *LHS, const DeclRefExpr *RHS)
Emits single reduction combiner.
llvm::OpenMPIRBuilder & getOMPBuilder()
virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Emit outilined function for 'target' directive.
virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr)
Emits a critical region.
virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned)
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
virtual void emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, ArrayRef< llvm::Value * > Args={}) const
Emits call of the outlined function with the provided arguments, translating these arguments to corre...
llvm::Value * emitNumThreadsForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D)
Emit an expression that denotes the number of threads a target region shall use.
void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor, llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc)
Emits initialization code for the threadprivate variables.
virtual void emitUserDefinedReduction(CodeGenFunction *CGF, const OMPDeclareReductionDecl *D)
Emit code for the specified user defined reduction construct.
virtual void checkAndEmitSharedLastprivateConditional(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::DenseSet< CanonicalDeclPtr< const VarDecl > > &IgnoredDecls)
Checks if the lastprivate conditional was updated in inner region and writes the value.
QualType KmpDimTy
struct kmp_dim { // loop bounds info casted to kmp_int64 kmp_int64 lo; // lower kmp_int64 up; // uppe...
llvm::SmallPtrSet< const CXXConstructorDecl *, 8 > ScannedImplicitCtors
Implicitly-defined constructors already scanned for target regions; only lambdas can hold such region...
virtual void emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool HasCancel=false)
Emit code for the directive that does not require outlining.
virtual void registerTargetGlobalVariable(const VarDecl *VD, llvm::Constant *Addr)
Checks if the provided global decl GD is a declare target variable and registers it when emitting cod...
virtual void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D)
Emits OpenMP-specific function prolog.
void emitKmpRoutineEntryT(QualType KmpInt32Ty)
Build type kmp_routine_entry_t (if not built yet).
virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static chunked.
virtual void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Emit the target offloading code associated with D.
virtual bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS)
Checks if the variable has associated OMPAllocateDeclAttr attribute with the predefined allocator and...
llvm::AtomicOrdering RequiresAtomicOrdering
Atomic ordering from the omp requires directive.
virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps, ReductionOptionsTy Options)
Emit a code for reduction clause.
std::pair< llvm::Value *, Address > emitDependClause(CodeGenFunction &CGF, ArrayRef< OMPTaskDataTy::DependData > Dependencies, SourceLocation Loc)
Emits list of dependecies based on the provided data (array of dependence/expression pairs).
llvm::StringMap< llvm::WeakTrackingVH > EmittedNonTargetVariables
List of the global variables with their addresses that should not be emitted for the target.
virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const
Check if the specified ScheduleKind is dynamic.
Address emitLastprivateConditionalInit(CodeGenFunction &CGF, const VarDecl *VD)
Create specialized alloca to handle lastprivate conditionals.
virtual void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads)
Emit an ordered region.
virtual Address getAddressOfLocalVariable(CodeGenFunction &CGF, const VarDecl *VD)
Gets the OpenMP-specific address of the local variable.
virtual void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, bool IsWorksharingReduction)
Emits the following code for reduction clause with task modifier:
virtual void emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter=nullptr)
Emits a masked region.
QualType KmpDependInfoTy
Type typedef struct kmp_depend_info { kmp_intptr_t base_addr; size_t len; struct { bool in:1; bool ou...
llvm::Function * emitReductionFunction(StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps)
Emits reduction function.
virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues)
Call the appropriate runtime routine to initialize it before start of loop.
Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal) override
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr) override
Emits a critical region.
void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values) override
void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) override
Call the appropriate runtime routine to initialize it before start of loop.
bool emitTargetGlobalVariable(GlobalDecl GD) override
Emit the global variable if it is a valid device global variable.
llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST) override
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
llvm::Function * emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF=nullptr) override
Emit a code for initialization of threadprivate variable.
void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device) override
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
llvm::Function * emitTeamsOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) override
Emits outlined function for the specified OpenMP teams directive D.
void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond, llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, const Expr *Message=nullptr) override
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps, ReductionOptionsTy Options) override
Emit a code for reduction clause.
void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc, llvm::AtomicOrdering AO) override
Emit flush of the variables specified in 'omp flush' directive.
void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C) override
Emit code for doacross ordered directive with 'depend' clause.
void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc) override
Emits a masked region.
Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, QualType VarType, StringRef Name) override
Creates artificial threadprivate variable with name Name and type VarType.
Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc) override
Returns address of the threadprivate variable for the current thread.
void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps) override
Emits a single region.
void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N) override
Required to resolve existing problems in the runtime.
llvm::Function * emitParallelOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) override
Emits outlined function for the specified OpenMP parallel directive D.
void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion) override
Emit code for 'cancellation point' construct.
void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false) override
Emit an implicit/explicit barrier for OpenMP threads.
Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const override
Gets the address of the native argument basing on the address of the target-specific parameter.
void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars) override
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned) override
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
bool emitTargetGlobal(GlobalDecl GD) override
Emit the global GD if it is meaningful for the target.
void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, bool IsWorksharingReduction) override
Emits the following code for reduction clause with task modifier:
void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads) override
Emit an ordered region.
void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind) override
Call the appropriate runtime routine to notify that we finished all the work with current loop.
llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data) override
Emit a code for initialization of task reduction clause.
void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc) override
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32global_tid, int proc_bind) to generate...
void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) override
Emit outilined function for 'target' directive.
void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc) override
Emits a master region.
void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc) override
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32global_tid, kmp_int32 num_teams,...
void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc) override
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
const VarDecl * translateParameter(const FieldDecl *FD, const VarDecl *NativeParam) const override
Translates the native parameter of outlined function if this is required for target.
void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc, OpenMPNumThreadsClauseModifier Modifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, SourceLocation SeverityLoc=SourceLocation(), const Expr *Message=nullptr, SourceLocation MessageLoc=SourceLocation()) override
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32global_tid, kmp_int32 num_threads) ...
void emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter=nullptr) override
Emits a masked region.
void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data) override
Emit task region for the task directive.
void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter) override
Emit the target offloading code associated with D.
bool emitTargetFunctions(GlobalDecl GD) override
Emit the target regions enclosed in GD function definition or the function itself in case it is a val...
void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr * > NumIterations) override
Emit initialization for doacross loop nesting support.
void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion) override
Emit code for 'cancel' construct.
void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPTaskDataTy &Data) override
Emit code for 'taskwait' directive.
void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc) override
Emit a taskgroup region.
void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, CGOpenMPRuntime::TargetDataInfo &Info) override
Emit the target data mapping code associated with D.
void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues) override
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
llvm::Function * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts) override
Emits outlined function for the OpenMP task directive D.
void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data) override
Emit task region for the taskloop directive.
unsigned getNonVirtualBaseLLVMFieldNo(const CXXRecordDecl *RD) const
llvm::StructType * getLLVMType() const
Return the "complete object" LLVM type associated with this record.
llvm::StructType * getBaseSubobjectLLVMType() const
Return the "base subobject" LLVM type associated with this record.
unsigned getLLVMFieldNo(const FieldDecl *FD) const
Return llvm::StructType element number that corresponds to the field FD.
unsigned getVirtualBaseIndex(const CXXRecordDecl *base) const
Return the LLVM field index corresponding to the given virtual base.
API for captured statement code generation.
virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S)
Emit the captured statement body.
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
RAII for correct setting/restoring of CapturedStmtInfo.
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
bool Privatize()
Privatizes local variables previously registered as private.
bool addPrivate(const ValueDecl *LocalVD, Address Addr)
Registers LocalVD variable as a private with Addr as the address of the corresponding private variabl...
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited.
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
LValue EmitLoadOfReferenceLValue(LValue RefLVal)
Definition CGExpr.cpp:3444
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount, Stmt::Likelihood LH=Stmt::LH_None, const Expr *ConditionalOp=nullptr, const VarDecl *ConditionalDecl=nullptr)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
void emitDestroy(Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray)
emitDestroy - Immediately perform the destruction of the given object.
Definition CGDecl.cpp:2479
JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target)
The given basic block lies in the current EH scope, but may be a target of a potentially scope-crossi...
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
void EmitNullInitialization(Address DestPtr, QualType Ty)
EmitNullInitialization - Generate code to set a value of the given type to null, If the type contains...
CGCapturedStmtInfo * CapturedStmtInfo
ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc)
EmitLoadOfComplex - Load a complex number from the specified l-value.
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
llvm::Function * GenerateOpenMPCapturedStmtFunctionAggregate(const CapturedStmt &S, const OMPExecutableDirective &D)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
const LangOptions & getLangOpts() const
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
EmitAutoVarAlloca - Emit the alloca and debug information for a local variable.
Definition CGDecl.cpp:1491
void pushDestroy(QualType::DestructionKind dtorKind, Address addr, QualType type)
pushDestroy - Push the standard destructor for the given type as at least a normal cleanup.
Definition CGDecl.cpp:2362
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPLoopDirective &S)
Emit device code for the target simd and target loop directives.
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Load a pointer with type PtrTy stored at address Ptr.
Definition CGExpr.cpp:3453
DeclMapTy::iterator localDeclMapEnd()
void EmitBranchThroughCleanup(JumpDest Dest)
EmitBranchThroughCleanup - Emit a branch from the current insert block through the normal cleanup han...
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
Destroyer * getDestroyer(QualType::DestructionKind destructionKind)
Definition CGDecl.cpp:2335
llvm::AssertingVH< llvm::Instruction > AllocaInsertPt
AllocaInsertPoint - This is an instruction in the entry block before which we prefer to insert alloca...
void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy)
Emit an aggregate assignment.
JumpDest ReturnBlock
ReturnBlock - Unified return block.
void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy, AggValueSlot::Overlap_t MayOverlap, bool isVolatile=false)
EmitAggregateCopy - Emit an aggregate copy.
LValue EmitLValueForField(LValue Base, const FieldDecl *Field, bool IsInBounds=true)
Definition CGExpr.cpp:5966
RawAddress CreateDefaultAlignTempAlloca(llvm::Type *Ty, const Twine &Name="tmp")
CreateDefaultAlignedTempAlloca - This creates an alloca with the default ABI alignment of the given L...
Definition CGExpr.cpp:187
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value * > &CapturedVars)
DeclMapTy::iterator findLocalDecl(const Decl *D)
Accessors for LocalDeclMap.
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
Definition CGExpr.cpp:263
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
Definition CGExpr.cpp:2549
std::pair< DeclMapTy::iterator, bool > insertLocalDecl(const Decl *D, Address Addr)
LValue EmitArraySectionExpr(const ArraySectionExpr *E, bool IsLowerBound=true)
Definition CGExpr.cpp:5492
LValue EmitOMPSharedLValue(const Expr *E)
Emits the lvalue for the expression with possibly captured variable.
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
Definition CGExpr.cpp:244
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
llvm::Value * emitArrayLength(const ArrayType *arrayType, QualType &baseType, Address &addr)
emitArrayLength - Compute the length of an array, even if it's a VLA, and drill down to the base elem...
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
LValue MakeRawAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment, AlignmentSource Source=AlignmentSource::Type)
Same as MakeAddrLValue above except that the pointer is known to be unsigned.
LValue EmitLValueForFieldInitialization(LValue Base, const FieldDecl *Field)
EmitLValueForFieldInitialization - Like EmitLValueForField, except that if the Field is a reference,...
Definition CGExpr.cpp:6125
RawAddress CreateMemTempWithoutCast(QualType T, const Twine &Name="tmp")
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen without...
Definition CGExpr.cpp:236
VlaSizePair getVLASize(const VariableArrayType *vla)
Returns an LLVM value that corresponds to the size, in non-variably-sized elements,...
llvm::CallInst * EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
void EmitAutoVarCleanups(const AutoVarEmission &emission)
Definition CGDecl.cpp:2281
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
Definition CGExpr.cpp:2801
LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy)
Definition CGExpr.cpp:3463
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
Definition CGExpr.cpp:314
bool needsEHCleanup(QualType::DestructionKind kind)
Determines whether an EH cleanup is required to destroy a type with the given destruction kind.
llvm::DenseMap< const ValueDecl *, FieldDecl * > LambdaCaptureFields
llvm::CallInst * EmitRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
llvm::Type * ConvertTypeForMem(QualType T)
static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S)
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
CodeGenTypes & getTypes() const
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
Definition CGExpr.cpp:1661
static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Emit device code for the target teams distribute parallel for simd directive.
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block,...
Definition CGStmt.cpp:678
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S, const OMPExecutableDirective &D)
RawAddress CreateMemTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Definition CGExpr.cpp:200
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
Definition CGDecl.cpp:211
llvm::Value * EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr, ArrayRef< llvm::Value * > IdxList, bool SignedIndices, bool IsSubtraction, SourceLocation Loc, const Twine &Name="")
Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to detect undefined behavior whe...
static void EmitOMPTargetParallelGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelGenericLoopDirective &S)
Emit device code for the target parallel loop directive.
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
static bool IsWrappedCXXThis(const Expr *E)
Check if E is a C++ "this" pointer wrapped in value-preserving casts.
Definition CGExpr.cpp:1719
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant,...
static void EmitOMPTargetTeamsGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsGenericLoopDirective &S)
Emit device code for the target teams loop directive.
LValue EmitMemberExpr(const MemberExpr *E)
Definition CGExpr.cpp:5761
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
Definition CGExpr.cpp:1777
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler's counter for the given statement by StepV.
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
void EmitVariablyModifiedType(QualType Ty)
EmitVLASize - Capture all the sizes for the VLA expressions in the given variably-modified type and s...
bool isTrivialInitializer(const Expr *Init)
Determine whether the given initializer is trivial in the sense that it requires no code to be genera...
Definition CGDecl.cpp:1861
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
Definition CGStmt.cpp:658
void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit)
EmitExprAsInit - Emits the code necessary to initialize a location in memory with the given initializ...
Definition CGDecl.cpp:2171
LValue MakeNaturalAlignRawAddrLValue(llvm::Value *V, QualType T)
This class organizes the cross-function state that is used while generating LLVM code.
void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
llvm::Module & getModule() const
const IntrusiveRefCntPtr< llvm::vfs::FileSystem > & getFileSystem() const
DiagnosticsEngine & getDiags() const
const LangOptions & getLangOpts() const
CharUnits getNaturalTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, bool forPointeeType=false)
const llvm::DataLayout & getDataLayout() const
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
TBAAAccessInfo getTBAAInfoForSubobject(LValue Base, QualType AccessType)
getTBAAInfoForSubobject - Get TBAA information for an access with a given base lvalue.
ASTContext & getContext() const
const CodeGenOptions & getCodeGenOpts() const
StringRef getMangledName(GlobalDecl GD)
std::optional< CharUnits > getOMPAllocateAlignment(const VarDecl *VD)
Return the alignment specified in an allocate directive, if present.
Definition CGDecl.cpp:3031
llvm::Constant * EmitNullConstant(QualType T)
Return the result of value-initializing the given type, i.e.
llvm::Type * ConvertType(QualType T)
ConvertType - Convert type T into a llvm::Type.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
Definition CGCall.cpp:2092
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
Definition CGCall.cpp:774
const CGRecordLayout & getCGRecordLayout(const RecordDecl *)
getCGRecordLayout - Return record layout info for the given record decl.
llvm::GlobalVariable * GetAddrOfVTable(const CXXRecordDecl *RD)
GetAddrOfVTable - Get the address of the VTable for the given record decl.
Definition CGVTables.cpp:41
A specialization of Address that requires the address to be an LLVM Constant.
Definition Address.h:296
static ConstantAddress invalid()
Definition Address.h:304
void pushTerminate()
Push a terminate handler on the stack.
void popTerminate()
Pops a terminate handler off the stack.
Definition CGCleanup.h:658
FunctionArgList - Type for representing both the decl and type of parameters to a function.
Definition CGCall.h:378
LValue - This represents an lvalue references.
Definition CGValue.h:183
CharUnits getAlignment() const
Definition CGValue.h:355
llvm::Value * getPointer(CodeGenFunction &CGF) const
const Qualifiers & getQuals() const
Definition CGValue.h:350
Address getAddress() const
Definition CGValue.h:373
LValueBaseInfo getBaseInfo() const
Definition CGValue.h:358
QualType getType() const
Definition CGValue.h:303
TBAAAccessInfo getTBAAInfo() const
Definition CGValue.h:347
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
virtual void Enter(CodeGenFunction &CGF)
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
Definition CGValue.h:42
static RValue get(llvm::Value *V)
Definition CGValue.h:99
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
Definition CGValue.h:109
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
Definition CGValue.h:72
An abstract representation of an aligned address.
Definition Address.h:42
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Definition Address.h:77
llvm::Value * getPointer() const
Definition Address.h:66
static RawAddress invalid()
Definition Address.h:61
Class intended to support codegen of all kind of the reduction clauses.
LValue getSharedLValue(unsigned N) const
Returns LValue for the reduction item.
const Expr * getRefExpr(unsigned N) const
Returns the base declaration of the reduction item.
LValue getOrigLValue(unsigned N) const
Returns LValue for the original reduction item.
bool needCleanups(unsigned N)
Returns true if the private copy requires cleanups.
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
const VarDecl * getBaseDecl(unsigned N) const
Returns the base declaration of the reduction item.
QualType getPrivateType(unsigned N) const
Return the type of the private item.
bool usesReductionInitializer(unsigned N) const
Returns true if the initialization of the reduction item uses initializer from declare reduction cons...
void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N)
Emits lvalue for the shared and original reduction item.
void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, llvm::function_ref< bool(CodeGenFunction &)> DefaultInit)
Performs initialization of the private copy for the reduction item.
std::pair< llvm::Value *, llvm::Value * > getSizes(unsigned N) const
Returns the size of the reduction item (in chars and total number of elements in the item),...
ReductionCodeGen(ArrayRef< const Expr * > Shareds, ArrayRef< const Expr * > Origs, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > ReductionOps)
void emitCleanups(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Emits cleanup code for the reduction item.
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations.
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
void operator()(CodeGenFunction &CGF) const
void setAction(PrePostActionTy &Action) const
ConstStmtVisitor - This class implements a simple visitor for Stmt subclasses.
DeclContext - This is used only as base class of specific decl types that can act as declaration cont...
Definition DeclBase.h:1467
void addDecl(Decl *D)
Add the declaration D into this context.
A reference to a declared variable, function, enum, etc.
Definition Expr.h:1315
ValueDecl * getDecl()
Definition Expr.h:1383
Decl - This represents one declaration (or definition), e.g.
Definition DeclBase.h:86
T * getAttr() const
Definition DeclBase.h:582
bool hasAttrs() const
Definition DeclBase.h:527
ASTContext & getASTContext() const LLVM_READONLY
Definition DeclBase.cpp:550
void addAttr(Attr *A)
virtual Stmt * getBody() const
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
Definition DeclBase.h:1105
bool isTemplated() const
Determine whether this declaration is a templated entity (whether it is.
Definition DeclBase.cpp:308
Decl * getNonClosureContext()
Find the innermost non-closure ancestor of this declaration, walking up through blocks,...
llvm::iterator_range< specific_attr_iterator< T > > specific_attrs() const
Definition DeclBase.h:568
SourceLocation getLocation() const
Definition DeclBase.h:448
DeclContext * getDeclContext()
Definition DeclBase.h:457
AttrVec & getAttrs()
Definition DeclBase.h:533
bool hasAttr() const
Definition DeclBase.h:586
virtual Decl * getCanonicalDecl()
Retrieves the "canonical" declaration of the given declaration.
Definition DeclBase.h:996
SourceLocation getBeginLoc() const LLVM_READONLY
Definition Decl.h:833
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
This represents one expression.
Definition Expr.h:114
bool isIntegerConstantExpr(const ASTContext &Ctx) const
bool isGLValue() const
Definition Expr.h:289
Expr * IgnoreParenNoopCasts(const ASTContext &Ctx) LLVM_READONLY
Skip past any parentheses and casts which do not change the value (including ptr->int casts of the sa...
Definition Expr.cpp:3156
@ SE_AllowSideEffects
Allow any unmodeled side effect.
Definition Expr.h:703
@ SE_AllowUndefinedBehavior
Allow UB that we can give a value, but not arbitrary unmodeled side effects.
Definition Expr.h:701
Expr * IgnoreParenCasts() LLVM_READONLY
Skip past any parentheses and casts which might surround this expression until reaching a fixed point...
Definition Expr.cpp:3134
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
Definition Expr.cpp:3129
bool isEvaluatable(const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects) const
isEvaluatable - Call EvaluateAsRValue to see if this expression can be constant folded without side-e...
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
Definition Expr.cpp:3728
std::optional< llvm::APSInt > getIntegerConstantExpr(const ASTContext &Ctx, bool AllowRelaxedEval=false) const
isIntegerConstantExpr - Return the value if this expression is a valid integer constant expression.
bool EvaluateAsBooleanCondition(bool &Result, const ASTContext &Ctx, bool InConstantContext=false) const
EvaluateAsBooleanCondition - Return true if this is a constant which we can fold and convert to a boo...
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Definition Expr.cpp:283
static bool isSameComparisonOperand(const Expr *E1, const Expr *E2)
Checks that the two Expr's will refer to the same value as a comparison operand.
Definition Expr.cpp:4369
QualType getType() const
Definition Expr.h:146
bool hasNonTrivialCall(const ASTContext &Ctx) const
Determine whether this expression involves a call to any function that is not trivial.
Definition Expr.cpp:4098
Represents a member of a struct/union/class.
Definition Decl.h:3341
unsigned getFieldIndex() const
Returns the index of this field within its record, as appropriate for passing to ASTRecordLayout::get...
Definition Decl.h:3426
const RecordDecl * getParent() const
Returns the parent of this field declaration, which is the struct in which this field is defined.
Definition Decl.h:3579
static FieldDecl * Create(const ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, const IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, Expr *BW, bool Mutable, InClassInitStyle InitStyle)
Definition Decl.cpp:4803
Represents a function declaration or definition.
Definition Decl.h:2082
const ParmVarDecl * getParamDecl(unsigned i) const
Definition Decl.h:2974
QualType getReturnType() const
Definition Decl.h:3022
ArrayRef< ParmVarDecl * > parameters() const
Definition Decl.h:2947
FunctionDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
Definition Decl.cpp:3821
FunctionDecl * getMostRecentDecl()
Returns the most recent (re)declaration of this declaration.
FunctionDecl * getDefinition()
Get the definition for this declaration.
Definition Decl.h:2438
bool isUserProvided() const
True if this method is user-declared and was not deleted or defaulted on its first declaration.
Definition Decl.h:2579
unsigned getNumParams() const
Return the number of parameters this function must have based on its FunctionType.
Definition Decl.cpp:3900
bool hasBody(const FunctionDecl *&Definition) const
Returns true if the function has a body.
Definition Decl.cpp:3205
FunctionDecl * getPreviousDecl()
Return the previous declaration of this declaration or NULL if this is the first declaration.
GlobalDecl - represents a global declaration.
Definition GlobalDecl.h:60
const Decl * getDecl() const
Definition GlobalDecl.h:115
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, const IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
Definition Decl.cpp:5739
static IntegerLiteral * Create(const ASTContext &C, const llvm::APInt &V, QualType type, SourceLocation l)
Returns a new integer literal with value 'V' and type 'type'.
Definition Expr.cpp:985
An lvalue reference type, per C++11 [dcl.ref].
Definition TypeBase.h:3715
MemberExpr - [C99 6.5.2.3] Structure and Union Members.
Definition Expr.h:3433
ValueDecl * getMemberDecl() const
Retrieve the member declaration to which this expression refers.
Definition Expr.h:3516
Expr * getBase() const
Definition Expr.h:3510
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
Definition Decl.h:303
bool isExternallyVisible() const
Definition Decl.h:435
const Stmt * getPreInitStmt() const
Get pre-initialization statement for the clause.
This is a basic class for representing single OpenMP clause.
ArrayRef< OMPClause * > clauses() const
Definition DeclOpenMP.h:91
This represents 'pragma omp declare mapper ...' directive.
Definition DeclOpenMP.h:349
Expr * getMapperVarRef()
Get the variable declared in the mapper.
Definition DeclOpenMP.h:411
This represents 'pragma omp declare reduction ...' directive.
Definition DeclOpenMP.h:239
Expr * getInitializer()
Get initializer expression (if specified) of the declare reduction construct.
Definition DeclOpenMP.h:300
Expr * getInitPriv()
Get Priv variable of the initializer.
Definition DeclOpenMP.h:311
Expr * getCombinerOut()
Get Out variable of the combiner.
Definition DeclOpenMP.h:288
Expr * getCombinerIn()
Get In variable of the combiner.
Definition DeclOpenMP.h:285
Expr * getCombiner()
Get combiner expression of the declare reduction construct.
Definition DeclOpenMP.h:282
Expr * getInitOrig()
Get Orig variable of the initializer.
Definition DeclOpenMP.h:308
OMPDeclareReductionInitKind getInitializerKind() const
Get initializer kind.
Definition DeclOpenMP.h:303
This represents 'if' clause in the 'pragma omp ...' directive.
Expr * getCondition() const
Returns condition.
OMPIteratorHelperData & getHelper(unsigned I)
Fetches helper data for the specified iteration space.
Definition Expr.cpp:5650
unsigned numOfIterators() const
Returns number of iterator definitions.
Definition ExprOpenMP.h:275
This represents 'num_threads' clause in the 'pragma omp ...' directive.
This represents 'pragma omp requires...' directive.
Definition DeclOpenMP.h:479
clauselist_range clauselists()
Definition DeclOpenMP.h:504
This represents 'threadset' clause in the 'pragma omp task ...' directive.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class.
Definition Expr.h:1223
Represents a parameter to a function.
Definition Decl.h:1832
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition TypeBase.h:3403
Represents an unpacked "presumed" location which can be presented to the user.
unsigned getColumn() const
Return the presumed column number of this location.
const char * getFilename() const
Return the presumed filename of this location.
unsigned getLine() const
Return the presumed line number of this location.
A (possibly-)qualified type.
Definition TypeBase.h:938
void addRestrict()
Add the restrict qualifier to this QualType.
Definition TypeBase.h:1188
QualType withRestrict() const
Definition TypeBase.h:1191
bool isNull() const
Return true if this QualType doesn't point to a type yet.
Definition TypeBase.h:1005
const Type * getTypePtr() const
Retrieves a pointer to the underlying (unqualified) type.
Definition TypeBase.h:8578
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
Definition TypeBase.h:8618
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Definition TypeBase.h:8763
QualType getCanonicalType() const
Definition TypeBase.h:8630
DestructionKind isDestructedType() const
Returns a nonzero value if objects of this type require non-trivial work to clean up after.
Definition TypeBase.h:1561
Represents a struct/union/class.
Definition Decl.h:4508
field_iterator field_end() const
Definition Decl.h:4714
field_range fields() const
Definition Decl.h:4711
virtual void completeDefinition()
Note that the definition of this type is now complete.
Definition Decl.cpp:5428
bool field_empty() const
Definition Decl.h:4719
field_iterator field_begin() const
Definition Decl.cpp:5412
Scope - A scope is a transient data structure that is used while parsing the program.
Definition Scope.h:41
Encodes a location in the source.
static SourceLocation getFromRawEncoding(UIntTy Encoding)
Turn a raw encoding of a SourceLocation object into a real SourceLocation.
bool isValid() const
Return true if this is a valid SourceLocation object.
UIntTy getRawEncoding() const
When a SourceLocation itself cannot be used, this returns an (opaque) 32-bit integer encoding for it.
This class handles loading and caching of source files into memory.
PresumedLoc getPresumedLoc(SourceLocation Loc, bool UseLineDirectives=true) const
Returns the "presumed" location of a SourceLocation specifies.
Stmt - This represents one statement.
Definition Stmt.h:85
child_range children()
Definition Stmt.cpp:304
StmtClass getStmtClass() const
Definition Stmt.h:1504
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
Definition Stmt.cpp:343
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top,...
Definition Stmt.cpp:210
SourceLocation getBeginLoc() const LLVM_READONLY
Definition Stmt.cpp:355
void startDefinition()
Starts the definition of this tag declaration.
Definition Decl.cpp:5043
bool isUnion() const
Definition Decl.h:4111
The base class of the type hierarchy.
Definition TypeBase.h:1879
bool isVoidType() const
Definition TypeBase.h:9200
const Type * getPointeeOrArrayElementType() const
If this is a pointer type, return the pointee type.
Definition TypeBase.h:9391
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
Definition Type.cpp:2392
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
Definition Type.h:26
RecordDecl * getAsRecordDecl() const
Retrieves the RecordDecl this type refers to.
Definition Type.h:41
bool isArrayType() const
Definition TypeBase.h:8914
bool isPointerType() const
Definition TypeBase.h:8815
CanQualType getCanonicalTypeUnqualified() const
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
Definition TypeBase.h:9248
const T * castAs() const
Member-template castAs<specific type>.
Definition TypeBase.h:9498
bool isReferenceType() const
Definition TypeBase.h:8839
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
Definition Type.cpp:885
bool isLValueReferenceType() const
Definition TypeBase.h:8843
bool isAggregateType() const
Determines whether the type is a C++ aggregate type or C aggregate or union type.
Definition Type.cpp:2631
RecordDecl * castAsRecordDecl() const
Definition Type.h:48
QualType getCanonicalTypeInternal() const
Definition TypeBase.h:3200
const Type * getBaseElementTypeUnsafe() const
Get the base element type of this type, potentially discarding type qualifiers.
Definition TypeBase.h:9384
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
Definition TypeBase.h:2881
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type.
Definition TypeBase.h:9484
bool isFloatingType() const
Definition Type.cpp:2517
bool isUnsignedIntegerType() const
Return true if this is an integer type that is unsigned, according to C99 6.2.5p6 [which returns true...
Definition Type.cpp:2460
bool isAnyPointerType() const
Definition TypeBase.h:8823
const T * getAs() const
Member-template getAs<specific type>'.
Definition TypeBase.h:9431
bool isRecordType() const
Definition TypeBase.h:8942
bool isUnionType() const
Definition Type.cpp:851
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Definition Decl.h:714
QualType getType() const
Definition Decl.h:725
Represents a variable declaration or definition.
Definition Decl.h:934
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
Definition Decl.cpp:2238
VarDecl * getDefinition(ASTContext &)
Get the real (not just tentative) definition for this declaration.
Definition Decl.cpp:2368
const Expr * getInit() const
Definition Decl.h:1393
bool hasExternalStorage() const
Returns true if a variable has extern or private_extern storage.
Definition Decl.h:1240
@ DeclarationOnly
This declaration is only a declaration.
Definition Decl.h:1320
DefinitionKind hasDefinition(ASTContext &) const
Check whether this variable is defined in this translation unit.
Definition Decl.cpp:2377
bool isLocalVarDeclOrParm() const
Similar to isLocalVarDecl but also includes parameters.
Definition Decl.h:1287
const Expr * getAnyInitializer() const
Get the initializer for this variable, no matter which declaration it is attached to.
Definition Decl.h:1383
Represents a C array with a specified size that is not an integer-constant-expression.
Definition TypeBase.h:4064
Expr * getSizeExpr() const
Definition TypeBase.h:4078
specific_attr_iterator - Iterates over a subrange of an AttrVec, only providing attributes that are o...
Definition SPIR.cpp:35
bool isEmptyFieldForLayout(const ASTContext &Ctx, const FieldDecl *FD)
Return true iff the field is "empty", that is, either a zero-width bit-field or an isEmptyRecordForLa...
bool isEmptyRecordForLayout(const ASTContext &Ctx, QualType T)
Return true iff a structure contains only empty base classes (per isEmptyRecordForLayout) and fields ...
@ Type
The l-value was considered opaque, so the alignment was determined from a type.
Definition CGValue.h:155
@ Decl
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
Definition CGValue.h:146
ComparisonResult
Indicates the result of a tentative comparison.
@ Address
A pointer to a ValueDecl.
Definition Primitives.h:28
Top level wrappers for InstallAPI frontend operations.
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
CanQual< Type > CanQualType
Represents a canonical, potentially-qualified type.
bool needsTaskBasedThreadLimit(OpenMPDirectiveKind DKind)
Checks if the specified target directive, combined or not, needs task based thread_limit.
@ Match
This is not an overload because the signature exactly matches an existing declaration.
Definition Sema.h:823
@ Ctor_Complete
Complete object ctor.
Definition ABI.h:25
Privates[]
This class represents the 'transparent' clause in the 'pragma omp task' directive.
bool isa(CodeGen::Address addr)
Definition Address.h:330
if(T->getSizeExpr()) TRY_TO(TraverseStmt(const_cast< Expr * >(T -> getSizeExpr())))
bool isOpenMPTargetDataManagementDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target data offload directive.
static bool classof(const OMPClause *T)
Stmt Stmt * Callback
Definition StmtOpenMP.h:930
@ Conditional
A conditional (?:) operator.
Definition Sema.h:662
@ ICIS_NoInit
No in-class initializer.
Definition Specifiers.h:276
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
@ LCK_ByRef
Capturing by reference.
Definition Lambda.h:37
LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE()
@ Private
'private' clause, allowed on 'parallel', 'serial', 'loop', 'parallel loop', and 'serial loop' constru...
@ Reduction
'reduction' clause, allowed on Parallel, Serial, Loop, and the combined constructs.
@ Present
'present' clause, allowed on Compute and Combined constructs, plus 'data' and 'declare'.
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
Definition OpenMPKinds.h:39
@ OMPC_SCHEDULE_MODIFIER_last
Definition OpenMPKinds.h:44
@ OMPC_SCHEDULE_MODIFIER_unknown
Definition OpenMPKinds.h:40
@ AS_public
Definition Specifiers.h:128
nullptr
This class represents a compute construct, representing a 'Kind' of ‘parallel’, 'serial',...
@ CR_OpenMP
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
OpenMPDistScheduleClauseKind
OpenMP attributes for 'dist_schedule' clause.
bool isOpenMPTaskingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of tasking directives - task or any taskloop-based dire...
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
@ OMPC_DYN_GROUPPRIVATE_FALLBACK_unknown
@ Result
The result type of a method or function.
Definition TypeBase.h:906
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
const FunctionProtoType * T
OpenMPDependClauseKind
OpenMP attributes for 'depend' clause.
Definition OpenMPKinds.h:55
@ OMPC_DEPEND_unknown
Definition OpenMPKinds.h:59
@ Dtor_Complete
Complete object dtor.
Definition ABI.h:36
@ Union
The "union" keyword.
Definition TypeBase.h:6179
bool isOpenMPTargetMapEnteringDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a map-entering target directive.
bool isOpenMPLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a directive with an associated loop construct.
OpenMPSeverityClauseKind
OpenMP attributes for 'severity' clause.
LangAS
Defines the address space values used by the address space qualifier of QualType.
llvm::omp::Directive OpenMPDirectiveKind
OpenMP directives.
Definition OpenMPKinds.h:25
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
@ VK_PRValue
A pr-value expression (in the C++11 taxonomy) produces a temporary value.
Definition Specifiers.h:139
@ VK_LValue
An l-value expression is a reference to an object with independent storage.
Definition Specifiers.h:143
for(const auto &A :T->param_types())
void getOpenMPCaptureRegions(llvm::SmallVectorImpl< OpenMPDirectiveKind > &CaptureRegions, OpenMPDirectiveKind DKind)
Return the captured regions of an OpenMP directive.
OpenMPNumThreadsClauseModifier
@ OMPC_ATOMIC_DEFAULT_MEM_ORDER_unknown
U cast(CodeGen::Address addr)
Definition Address.h:327
@ OMPC_DEVICE_unknown
Definition OpenMPKinds.h:51
OpenMPMapModifierKind
OpenMP modifier kind for 'map' clause.
Definition OpenMPKinds.h:79
@ OMPC_MAP_MODIFIER_unknown
Definition OpenMPKinds.h:80
@ Other
Other implicit parameter.
Definition Decl.h:1787
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
Definition OpenMPKinds.h:31
@ OMPC_SCHEDULE_unknown
Definition OpenMPKinds.h:35
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
OpenMPThreadsetKind
OpenMP modifiers for 'threadset' clause.
OpenMPMapClauseKind
OpenMP mapping kind for 'map' clause.
Definition OpenMPKinds.h:71
@ OMPC_MAP_unknown
Definition OpenMPKinds.h:75
Diagnostic wrappers for TextAPI types for error reporting.
Definition Dominators.h:30
__builtin_bitreverse32 uint32_t __builtin_riscv_aaddu_u32 __builtin_riscv_asubu_u32 int32_t
__builtin_riscv_nclipru_u32 int64_t
#define false
Definition stdbool.h:26
Data for list of allocators.
Expr * AllocatorTraits
Allocator traits.
struct with the values to be passed to the dispatch runtime function
llvm::Value * Chunk
Chunk size specified using 'schedule' clause (nullptr if chunk was not specified)
Maps the expression for the lastprivate variable to the global copy used to store new value because o...
Struct with the values to be passed to the static runtime function.
bool IVSigned
Sign of the iteration variable.
Address UB
Address of the output variable in which the upper iteration number is returned.
Address IL
Address of the output variable in which the flag of the last iteration is returned.
llvm::Value * Chunk
Value of the chunk for the static_chunked scheduled loop.
unsigned IVSize
Size of the iteration variable in bits.
Address ST
Address of the output variable in which the stride value is returned necessary to generated the stati...
bool Ordered
true if loop is ordered, false otherwise.
Address LB
Address of the output variable in which the lower iteration number is returned.
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::CallingConv::ID getRuntimeCC() const
SmallVector< const Expr *, 4 > DepExprs
EvalResult is a struct with detailed info about an evaluated expression.
Definition Expr.h:667
Extra information about a function prototype.
Definition TypeBase.h:5633
Expr * CounterUpdate
Updater for the internal counter: ++CounterVD;.
Definition ExprOpenMP.h:121
Scheduling data for loop-based OpenMP directives.
bool UseFusedDistChunkSchedule
Request the fused distr_static_chunk + static_chunkone runtime schedule in for_static_init.
OpenMPScheduleClauseModifier M2
OpenMPScheduleClauseModifier M1
OpenMPScheduleClauseKind Schedule
Describes how types, statements, expressions, and declarations should be printed.