clang 24.0.0git
CIRGenCUDANV.cpp
Go to the documentation of this file.
1//========- CIRGenCUDANV.cpp - Interface to NVIDIA CUDA Runtime -----=========//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for CUDA code generation targeting the NVIDIA CUDA
10// runtime library.
11//
12//===----------------------------------------------------------------------===//
13
14#include "CIRGenCUDARuntime.h"
15#include "CIRGenCXXABI.h"
16#include "CIRGenFunction.h"
17#include "CIRGenModule.h"
18#include "mlir/IR/Operation.h"
20#include "clang/AST/Attrs.inc"
21#include "clang/AST/Decl.h"
24#include "clang/Basic/Cuda.h"
27#include "llvm/Support/Casting.h"
28
29using namespace clang;
30using namespace clang::CIRGen;
31
32namespace {
33
34class CIRGenNVCUDARuntime : public CIRGenCUDARuntime {
35protected:
36 StringRef prefix;
37
38 // Map a device stub function to a symbol for identifying kernel in host
39 // code. For CUDA, the symbol for identifying the kernel is the same as the
40 // device stub function. For HIP, they are different.
41 llvm::StringMap<mlir::Operation *> kernelHandles;
42
43 // Map a kernel handle to the kernel stub.
44 llvm::DenseMap<mlir::Operation *, mlir::Operation *> kernelStubs;
45
46 struct VarInfo {
47 cir::GlobalOp var;
48 const VarDecl *d;
49 cir::CUDADeviceVarKind flags;
50 };
51 llvm::SmallVector<VarInfo, 16> deviceVars;
52
53 // Mangle context for device.
54 std::unique_ptr<MangleContext> deviceMC;
55
56private:
57 void emitDeviceStubBodyNew(CIRGenFunction &cgf, cir::FuncOp fn,
58 FunctionArgList &args);
59 mlir::Value prepareKernelArgs(CIRGenFunction &cgf, mlir::Location loc,
60 FunctionArgList &args);
61 mlir::Operation *getKernelHandle(cir::FuncOp fn, GlobalDecl gd) override;
62
63 mlir::Operation *getKernelStub(mlir::Operation *handle) override {
64 auto it = kernelStubs.find(handle);
65 assert(it != kernelStubs.end());
66 return it->second;
67 }
68 std::string addPrefixToName(StringRef funcName) const;
69 std::string addUnderscoredPrefixToName(StringRef funcName) const;
70
71public:
72 CIRGenNVCUDARuntime(CIRGenModule &cgm);
73 ~CIRGenNVCUDARuntime();
74
75 void emitDeviceStub(CIRGenFunction &cgf, cir::FuncOp fn,
76 FunctionArgList &args) override;
77
78 void handleVarRegistration(const VarDecl *vd, cir::GlobalOp var) override;
79 void finalizeModule() override;
80 void handleGlobalReplace(cir::GlobalOp oldGV, cir::GlobalOp newGV) override;
81
82 void internalizeDeviceSideVar(const VarDecl *d,
83 cir::GlobalLinkageKind &linkage) override;
84
85 std::string getDeviceSideName(const NamedDecl *nd) override;
86
87 void registerDeviceVar(const VarDecl *vd, cir::GlobalOp &var, bool isExtern,
88 bool isConstant) {
89 // Attach the device var attribute to the GlobalOp
90 auto &builder = cgm.getBuilder();
91 var->setAttr(cir::CUDAVarRegistrationInfoAttr::getMnemonic(),
92 cir::CUDAVarRegistrationInfoAttr::get(
93 builder.getContext(),
94 getDeviceSideName(cast<NamedDecl>(vd)),
95 cir::CUDADeviceVarKind::Variable, isExtern, isConstant,
96 vd->hasAttr<HIPManagedAttr>()));
97 deviceVars.push_back({
98 var,
99 vd,
100 cir::CUDADeviceVarKind::Variable,
101 });
102 }
103
104 void registerDeviceSurf(const VarDecl *vd, cir::GlobalOp &var,
105 bool isExtern) {
106 auto &builder = cgm.getBuilder();
107
108 var->setAttr(cir::CUDAVarRegistrationInfoAttr::getMnemonic(),
109 cir::CUDAVarRegistrationInfoAttr::get(
110 builder.getContext(),
111 getDeviceSideName(cast<NamedDecl>(vd)),
112 cir::CUDADeviceVarKind::Surface, isExtern,
113 /*isConstant=*/false,
114 /*isManaged=*/false));
115
116 deviceVars.push_back({
117 var,
118 vd,
119 cir::CUDADeviceVarKind::Surface,
120 });
121 }
122
123 void registerDeviceTex(const VarDecl *vd, cir::GlobalOp &var, bool isExtern) {
124 auto &builder = cgm.getBuilder();
125
126 var->setAttr(cir::CUDAVarRegistrationInfoAttr::getMnemonic(),
127 cir::CUDAVarRegistrationInfoAttr::get(
128 builder.getContext(),
129 getDeviceSideName(cast<NamedDecl>(vd)),
130 cir::CUDADeviceVarKind::Texture, isExtern,
131 /*isConstant=*/false,
132 /*isManaged=*/false));
133
134 deviceVars.push_back({
135 var,
136 vd,
137 cir::CUDADeviceVarKind::Texture,
138 });
139 }
140};
141
142} // namespace
143
144std::string CIRGenNVCUDARuntime::addPrefixToName(StringRef funcName) const {
145 return (prefix + funcName).str();
146}
147
148std::string
149CIRGenNVCUDARuntime::addUnderscoredPrefixToName(StringRef funcName) const {
150 return ("__" + prefix + funcName).str();
151}
152
153CIRGenNVCUDARuntime::CIRGenNVCUDARuntime(CIRGenModule &cgm)
154 : CIRGenCUDARuntime(cgm),
155 deviceMC(cgm.getASTContext().cudaNVInitDeviceMC()) {
156 if (cgm.getLangOpts().OffloadViaLLVM)
157 cgm.errorNYI("CIRGenNVCUDARuntime: Offload via LLVM");
158 else if (cgm.getLangOpts().HIP)
159 prefix = "hip";
160 else
161 prefix = "cuda";
162}
163
164mlir::Value CIRGenNVCUDARuntime::prepareKernelArgs(CIRGenFunction &cgf,
165 mlir::Location loc,
166 FunctionArgList &args) {
167 CIRGenBuilderTy &builder = cgm.getBuilder();
168
169 // Build void *args[] and populate with the addresses of kernel arguments.
170 auto voidPtrArrayTy = cir::ArrayType::get(cgm.voidPtrTy, args.size());
171 mlir::Value kernelArgs =
172 builder.createAlloca(loc, cir::PointerType::get(voidPtrArrayTy),
173 "kernel_args", CharUnits::fromQuantity(16));
174
175 mlir::Value kernelArgsDecayed =
176 builder.createCast(cir::CastKind::array_to_ptrdecay, kernelArgs,
177 cir::PointerType::get(cgm.voidPtrTy));
178
179 for (const auto &[i, arg] : llvm::enumerate(args)) {
180 mlir::Value index =
181 builder.getConstInt(loc, llvm::APInt(/*numBits=*/32, i));
182 mlir::Value storePos =
183 builder.createPtrStride(loc, kernelArgsDecayed, index);
184 mlir::Value argAddr = cgf.getAddrOfLocalVar(arg).getPointer();
185 mlir::Value argAsVoid = builder.createBitcast(argAddr, cgm.voidPtrTy);
186
187 builder.CIRBaseBuilderTy::createStore(loc, argAsVoid, storePos);
188 }
189
190 return kernelArgsDecayed;
191}
192
193// CUDA 9.0+ uses new way to launch kernels. Parameters are packed in a local
194// array and kernels are launched using cudaLaunchKernel().
195void CIRGenNVCUDARuntime::emitDeviceStubBodyNew(CIRGenFunction &cgf,
196 cir::FuncOp fn,
197 FunctionArgList &args) {
198
199 // This requires arguments to be sent to kernels in a different way.
200 if (cgm.getLangOpts().OffloadViaLLVM)
201 cgm.errorNYI("CIRGenNVCUDARuntime: Offload via LLVM");
202
203 CIRGenBuilderTy &builder = cgm.getBuilder();
204 mlir::Location loc = fn.getLoc();
205
206 // For [cuda|hip]LaunchKernel, we must add another layer of indirection
207 // to arguments. For example, for function `add(int a, float b)`,
208 // we need to pass it as `void *args[2] = { &a, &b }`.
209 mlir::Value kernelArgs = prepareKernelArgs(cgf, loc, args);
210
211 // Lookup cudaLaunchKernel/hipLaunchKernel function.
212 // HIP kernel launching API name depends on -fgpu-default-stream option. For
213 // the default value 'legacy', it is hipLaunchKernel. For 'per-thread',
214 // it is hipLaunchKernel_spt.
215 // cudaError_t cudaLaunchKernel(const void *func, dim3 gridDim, dim3 blockDim,
216 // void **args, size_t sharedMem,
217 // cudaStream_t stream);
218 // hipError_t hipLaunchKernel[_spt](const void *func, dim3 gridDim,
219 // dim3 blockDim, void **args,
220 // size_t sharedMem, hipStream_t stream);
221 TranslationUnitDecl *tuDecl = cgm.getASTContext().getTranslationUnitDecl();
222 DeclContext *dc = TranslationUnitDecl::castToDeclContext(tuDecl);
223
224 // The default stream is usually stream 0 (the legacy default stream).
225 // For per-thread default stream, we need a different LaunchKernel function.
226 std::string kernelLaunchAPI = "LaunchKernel";
227 if (cgm.getLangOpts().GPUDefaultStream ==
228 LangOptions::GPUDefaultStreamKind::PerThread) {
229 if (cgm.getLangOpts().HIP)
230 kernelLaunchAPI += "_spt";
231 else if (cgm.getLangOpts().CUDA)
232 kernelLaunchAPI += "_ptsz";
233 }
234
235 std::string launchKernelName = addPrefixToName(kernelLaunchAPI);
236 const IdentifierInfo &launchII =
237 cgm.getASTContext().Idents.get(launchKernelName);
238 FunctionDecl *cudaLaunchKernelFD = nullptr;
239 for (NamedDecl *result : dc->lookup(&launchII)) {
240 if (FunctionDecl *fd = dyn_cast<FunctionDecl>(result))
241 cudaLaunchKernelFD = fd;
242 }
243
244 if (cudaLaunchKernelFD == nullptr) {
245 cgm.error(cgf.curFuncDecl->getLocation(),
246 "Can't find declaration for " + launchKernelName);
247 return;
248 }
249
250 // Use this function to retrieve arguments for cudaLaunchKernel:
251 // int __[cuda|hip]PopCallConfiguration(dim3 *gridDim, dim3 *blockDim, size_t
252 // *sharedMem, cudaStream_t *stream)
253 //
254 // Here [cuda|hip]Stream_t, while also being the 6th argument of
255 // [cuda|hip]LaunchKernel, is a pointer to some opaque struct.
256
257 mlir::Type dim3Ty = cgf.getTypes().convertType(
258 cudaLaunchKernelFD->getParamDecl(1)->getType());
259 mlir::Type streamTy = cgf.getTypes().convertType(
260 cudaLaunchKernelFD->getParamDecl(5)->getType());
261
262 mlir::Value gridDim =
263 builder.createAlloca(loc, cir::PointerType::get(dim3Ty), "grid_dim",
265 mlir::Value blockDim =
266 builder.createAlloca(loc, cir::PointerType::get(dim3Ty), "block_dim",
268 mlir::Value sharedMem = builder.createAlloca(
269 loc, cir::PointerType::get(cgm.sizeTy), "shared_mem", cgm.getSizeAlign());
270 mlir::Value stream = builder.createAlloca(
271 loc, cir::PointerType::get(streamTy), "stream", cgm.getPointerAlign());
272
273 cir::FuncOp popConfig = cgm.createRuntimeFunction(
274 cir::FuncType::get({gridDim.getType(), blockDim.getType(),
275 sharedMem.getType(), stream.getType()},
276 cgm.sInt32Ty),
277 addUnderscoredPrefixToName("PopCallConfiguration"));
278 cgf.emitRuntimeCall(loc, popConfig, {gridDim, blockDim, sharedMem, stream});
279
280 // Now emit the call to cudaLaunchKernel
281 // [cuda|hip]Error_t [cuda|hip]LaunchKernel(const void *func, dim3 gridDim,
282 // dim3 blockDim,
283 // void **args, size_t sharedMem,
284 // [cuda|hip]Stream_t stream);
285
286 // We now either pick the function or the stub global for cuda, hip
287 // respectively.
288 mlir::Value kernel = [&]() -> mlir::Value {
289 if (cir::GlobalOp globalOp = llvm::dyn_cast_or_null<cir::GlobalOp>(
290 kernelHandles[fn.getSymName()])) {
291 cir::PointerType kernelTy = cir::PointerType::get(globalOp.getSymType());
292 mlir::Value kernelVal = cir::GetGlobalOp::create(builder, loc, kernelTy,
293 globalOp.getSymName());
294 mlir::Value func = builder.createBitcast(kernelVal, cgm.voidPtrTy);
295 return func;
296 }
297 if (cir::FuncOp funcOp = llvm::dyn_cast_or_null<cir::FuncOp>(
298 kernelHandles[fn.getSymName()])) {
299 cir::PointerType kernelTy =
300 cir::PointerType::get(funcOp.getFunctionType());
301 mlir::Value kernelVal =
302 cir::GetGlobalOp::create(builder, loc, kernelTy, funcOp.getSymName());
303 mlir::Value func = builder.createBitcast(kernelVal, cgm.voidPtrTy);
304 return func;
305 }
306 llvm_unreachable("Expected stub handle to be cir::GlobalOp or FuncOp");
307 }();
308
309 CallArgList launchArgs;
310 launchArgs.add(RValue::get(kernel),
311 cudaLaunchKernelFD->getParamDecl(0)->getType());
312 launchArgs.add(
314 cudaLaunchKernelFD->getParamDecl(1)->getType());
315 launchArgs.add(
317 cudaLaunchKernelFD->getParamDecl(2)->getType());
318 launchArgs.add(RValue::get(kernelArgs),
319 cudaLaunchKernelFD->getParamDecl(3)->getType());
320 launchArgs.add(
321 RValue::get(builder.CIRBaseBuilderTy::createLoad(loc, sharedMem)),
322 cudaLaunchKernelFD->getParamDecl(4)->getType());
323 launchArgs.add(RValue::get(builder.CIRBaseBuilderTy::createLoad(loc, stream)),
324 cudaLaunchKernelFD->getParamDecl(5)->getType());
325
326 mlir::Type launchTy =
327 cgm.getTypes().convertType(cudaLaunchKernelFD->getType());
328 mlir::Operation *cudaKernelLauncherFn = cgm.createRuntimeFunction(
329 cast<cir::FuncType>(launchTy), launchKernelName);
330 const CIRGenFunctionInfo &callInfo =
331 cgm.getTypes().arrangeFunctionDeclaration(cudaLaunchKernelFD);
332 cgf.emitCall(callInfo, CIRGenCallee::forDirect(cudaKernelLauncherFn),
333 ReturnValueSlot(), launchArgs, /*isMustTail=*/false);
334
336 !cgf.getLangOpts().HIP)
337 cgm.errorNYI("MSVC CUDA stub handling");
338}
339
340void CIRGenNVCUDARuntime::emitDeviceStub(CIRGenFunction &cgf, cir::FuncOp fn,
341 FunctionArgList &args) {
342
343 if (auto globalOp =
344 llvm::dyn_cast<cir::GlobalOp>(kernelHandles[fn.getSymName()])) {
345 CIRGenBuilderTy &builder = cgm.getBuilder();
346 mlir::Type fnPtrTy = globalOp.getSymType();
347 auto sym = mlir::FlatSymbolRefAttr::get(fn.getSymNameAttr());
348 auto gv = cir::GlobalViewAttr::get(fnPtrTy, sym);
349
350 globalOp->setAttr("initial_value", gv);
351 globalOp->removeAttr("sym_visibility");
352 globalOp->setAttr("alignment", builder.getI64IntegerAttr(
354 }
355
356 // CUDA 9.0 changed the way to launch kernels.
358 CudaFeature::CUDA_USES_NEW_LAUNCH) ||
359 (cgm.getLangOpts().HIP && cgm.getLangOpts().HIPUseNewLaunchAPI) ||
360 cgm.getLangOpts().OffloadViaLLVM)
361 emitDeviceStubBodyNew(cgf, fn, args);
362 else
363 cgm.errorNYI("Emit Stub Body Legacy");
364}
365
367 return new CIRGenNVCUDARuntime(cgm);
368}
369
370CIRGenNVCUDARuntime::~CIRGenNVCUDARuntime() {}
371
372mlir::Operation *CIRGenNVCUDARuntime::getKernelHandle(cir::FuncOp fn,
373 GlobalDecl gd) {
374
375 // Check if we already have a kernel handle for this function
376 auto it = kernelHandles.find(fn.getSymName());
377 if (it != kernelHandles.end()) {
378 mlir::Operation *oldHandle = it->second;
379 // Here we know that the fn did not change. Return it
380 if (kernelStubs[oldHandle] == fn)
381 return oldHandle;
382
383 // We've found the function name, but F itself has changed, so we need to
384 // update the references.
385 if (cgm.getLangOpts().HIP) {
386 // For HIP compilation the handle itself does not change, so we only need
387 // to update the Stub value.
388 kernelStubs[oldHandle] = fn;
389 return oldHandle;
390 }
391 // For non-HIP compilation, erase the old Stub and fall-through to creating
392 // new entries.
393 kernelStubs.erase(oldHandle);
394 }
395
396 // If not targeting HIP, store the function itself
397 if (!cgm.getLangOpts().HIP) {
398 kernelHandles[fn.getSymName()] = fn;
399 kernelStubs[fn] = fn;
400 return fn;
401 }
402
403 // Create a new CIR global variable to represent the kernel handle
404 CIRGenBuilderTy &builder = cgm.getBuilder();
405 StringRef globalName = cgm.getMangledName(
406 gd.getWithKernelReferenceKind(KernelReferenceKind::Kernel));
407 cir::PointerType fnPtrTy = builder.getPointerTo(fn.getFunctionType());
408 cir::GlobalOp globalOp =
409 cgm.createGlobalOp(fn.getLoc(), globalName, fnPtrTy, /*isConstant=*/true);
410
411 globalOp->setAttr("alignment", builder.getI64IntegerAttr(
413
414 // Store references
415 kernelHandles[fn.getSymName()] = globalOp;
416 kernelStubs[globalOp] = fn;
417
418 return globalOp;
419}
420
421void CIRGenNVCUDARuntime::internalizeDeviceSideVar(
422 const VarDecl *d, cir::GlobalLinkageKind &linkage) {
423 if (cgm.getLangOpts().GPURelocatableDeviceCode)
424 cgm.errorNYI(d->getSourceRange(),
425 "internalizeDeviceSideVar: GPU Relocatable Device Code (RDC)");
426
427 // __shared__ variables are odd. Shadows do get created, but
428 // they are not registered with the CUDA runtime, so they
429 // can't really be used to access their device-side
430 // counterparts. It's not clear yet whether it's nvcc's bug or
431 // a feature, but we've got to do the same for compatibility.
432 if (d->hasAttr<CUDADeviceAttr>() || d->hasAttr<CUDAConstantAttr>() ||
433 d->hasAttr<CUDASharedAttr>()) {
434 linkage = cir::GlobalLinkageKind::InternalLinkage;
435 }
436
439 cgm.errorNYI(d->getSourceRange(),
440 "internalizeDeviceSideVar: CUDA Surface/Texture support");
441}
442
443std::string CIRGenNVCUDARuntime::getDeviceSideName(const NamedDecl *nd) {
444 GlobalDecl gd;
445 // nd could be either a kernel or a variable.
446 if (auto *fd = dyn_cast<FunctionDecl>(nd))
447 gd = GlobalDecl(fd, KernelReferenceKind::Kernel);
448 else
449 gd = GlobalDecl(nd);
450 std::string deviceSideName;
451 MangleContext *mc;
452 if (cgm.getLangOpts().CUDAIsDevice)
453 mc = &cgm.getCXXABI().getMangleContext();
454 else
455 mc = deviceMC.get();
456 if (mc->shouldMangleDeclName(nd)) {
457 SmallString<256> buffer;
458 llvm::raw_svector_ostream out(buffer);
459 mc->mangleName(gd, out);
460 deviceSideName = std::string(out.str());
461 } else
462 deviceSideName = std::string(nd->getIdentifier()->getName());
463
464 // Make unique name for device side static file-scope variable for HIP.
465 if (cgm.getASTContext().shouldExternalize(nd) &&
466 cgm.getLangOpts().GPURelocatableDeviceCode) {
467 SmallString<256> buffer;
468 llvm::raw_svector_ostream out(buffer);
469 out << deviceSideName;
471 deviceSideName = std::string(out.str());
472 }
473 return deviceSideName;
474}
475
476void CIRGenNVCUDARuntime::handleVarRegistration(const VarDecl *vd,
477 cir::GlobalOp var) {
478 if (vd->hasAttr<CUDADeviceAttr>() || vd->hasAttr<CUDAConstantAttr>()) {
479 // Shadow variables and their properties must be registered with CUDA
480 // runtime. Skip Extern global variables, which will be registered in
481 // the TU where they are defined.
482 //
483 // Don't register a C++17 inline variable. The local symbol can be
484 // discarded and referencing a discarded local symbol from outside the
485 // comdat (__cuda_register_globals) is disallowed by the ELF spec.
486 //
487 // HIP managed variables need to be always recorded in device and host
488 // compilations for transformation.
489 //
490 // HIP managed variables and variables in CUDADeviceVarODRUsedByHost are
491 // added to llvm.compiler-used, therefore they are safe to be registered.
492 if ((!vd->hasExternalStorage() && !vd->isInline()) ||
493 cgm.getASTContext().CUDADeviceVarODRUsedByHost.contains(vd) ||
494 vd->hasAttr<HIPManagedAttr>()) {
495 registerDeviceVar(vd, var, !vd->hasDefinition(),
496 vd->hasAttr<CUDAConstantAttr>());
497 }
498 } else if (vd->getType()->isCUDADeviceBuiltinSurfaceType()) {
499 // Builtin surfaces and their template arguments are also registered
500 // with CUDA runtime.
501 if (!vd->hasExternalStorage())
502 registerDeviceSurf(vd, var, !vd->hasDefinition());
503
504 } else if (vd->getType()->isCUDADeviceBuiltinTextureType()) {
505 // Builtin textures and their template arguments are also registered
506 // with CUDA runtime.
507 if (!vd->hasExternalStorage())
508 registerDeviceTex(vd, var, !vd->hasDefinition());
509 }
510}
511
512void CIRGenNVCUDARuntime::handleGlobalReplace(cir::GlobalOp oldGV,
513 cir::GlobalOp newGV) {
514 for (auto &info : deviceVars) {
515 if (info.var == oldGV)
516 info.var = newGV;
517 }
518}
519
520void CIRGenNVCUDARuntime::finalizeModule() {
521 if (!cgm.getLangOpts().CUDAIsDevice)
522 return;
523
524 // Mark ODR-used device variables as compiler used to prevent them from being
525 // eliminated by optimization. This is necessary for device variables
526 // ODR-used by host functions. Sema correctly marks them as ODR-used no
527 // matter whether they are ODR-used by device or host functions.
528 //
529 // We do not need to do this if the variable has used attribute since it
530 // has already been added.
531 //
532 // Static device variables have been externalized at this point, therefore
533 // variables with private or internal linkage need not be added.
534 for (auto &&info : deviceVars) {
535 auto kind = info.flags;
536 bool isDecl = info.var.isDeclaration();
537 bool isLocalLinkage = cir::isLocalLinkage(info.var.getLinkage());
538 bool isVarOrSurfaceOrTexture = (kind == cir::CUDADeviceVarKind::Variable ||
539 kind == cir::CUDADeviceVarKind::Surface ||
540 kind == cir::CUDADeviceVarKind::Texture);
541 bool isUsed = info.d->isUsed();
542 bool hasUsedAttr = info.d->hasAttr<UsedAttr>();
543 if (!isDecl && !isLocalLinkage && isVarOrSurfaceOrTexture && isUsed &&
544 !hasUsedAttr) {
545 if (auto globalValue = mlir::dyn_cast<cir::CIRGlobalValueInterface>(
546 info.var.getOperation())) {
547 cgm.addCompilerUsedGlobal(globalValue);
548 }
549 }
550 }
551}
Defines the clang::ASTContext interface.
Provides definitions for the various language-specific address spaces.
*collection of selector each with an associated kind and an ordered *collection of selectors A selector has a kind
__CUDA_BUILTIN_VAR __cuda_builtin_blockDim_t blockDim
__CUDA_BUILTIN_VAR __cuda_builtin_gridDim_t gridDim
mlir::Value createCast(mlir::Location loc, cir::CastKind kind, mlir::Value src, mlir::Type newTy)
cir::PtrStrideOp createPtrStride(mlir::Location loc, mlir::Value base, mlir::Value stride)
cir::PointerType getPointerTo(mlir::Type ty)
mlir::Value createAlloca(mlir::Location loc, cir::PointerType addrType, llvm::StringRef name, mlir::IntegerAttr alignment, mlir::Value dynAllocSize)
mlir::Value createBitcast(mlir::Value src, mlir::Type newTy)
TranslationUnitDecl * getTranslationUnitDecl() const
bool shouldExternalize(const Decl *D) const
Whether a C++ static variable or CUDA/HIP kernel should be externalized.
IdentifierTable & Idents
Definition ASTContext.h:828
llvm::SetVector< const VarDecl * > CUDADeviceVarODRUsedByHost
Keep track of CUDA/HIP device-side variables ODR-used by host code.
const TargetInfo & getTargetInfo() const
Definition ASTContext.h:947
mlir::Value getPointer() const
Definition Address.h:98
cir::ConstantOp getConstInt(mlir::Location loc, llvm::APSInt intVal)
clang::MangleContext & getMangleContext()
Gets the mangle context.
static CIRGenCallee forDirect(mlir::Operation *funcPtr, const CIRGenCalleeInfo &abstractInfo=CIRGenCalleeInfo())
Definition CIRGenCall.h:92
CIRGenTypes & getTypes() const
const clang::LangOptions & getLangOpts() const
const clang::Decl * curFuncDecl
Address getAddrOfLocalVar(const clang::VarDecl *vd)
Return the address of a local variable.
mlir::Value emitRuntimeCall(mlir::Location loc, cir::FuncOp callee, llvm::ArrayRef< mlir::Value > args={}, mlir::NamedAttrList attrs={})
RValue emitCall(const CIRGenFunctionInfo &funcInfo, const CIRGenCallee &callee, ReturnValueSlot returnValue, const CallArgList &args, cir::CIRCallOpInterface *callOp, bool isMustTail, mlir::Location loc)
This class organizes the cross-function state that is used while generating CIR code.
llvm::StringRef getMangledName(clang::GlobalDecl gd)
DiagnosticBuilder errorNYI(SourceLocation, llvm::StringRef)
Helpers to emit "not yet implemented" error diagnostics.
clang::ASTContext & getASTContext() const
CIRGenBuilderTy & getBuilder()
const clang::TargetInfo & getTarget() const
void error(SourceLocation loc, llvm::StringRef error)
Emit a general error that something can't be done.
cir::FuncOp createRuntimeFunction(cir::FuncType ty, llvm::StringRef name, mlir::NamedAttrList extraAttrs={}, bool isLocal=false, bool assumeConvergent=false)
const clang::LangOptions & getLangOpts() const
void printPostfixForExternalizedDecl(llvm::raw_ostream &os, const Decl *d)
Print the postfix for externalized static variable or kernels for single source offloading languages ...
cir::GlobalOp createGlobalOp(mlir::Location loc, llvm::StringRef name, mlir::Type t, bool isConstant=false, mlir::ptr::MemorySpaceAttrInterface addrSpace={}, mlir::Operation *insertPoint=nullptr)
void addCompilerUsedGlobal(cir::CIRGlobalValueInterface gv)
Add a global value to the llvmCompilerUsed list.
CIRGenCXXABI & getCXXABI() const
const CIRGenFunctionInfo & arrangeFunctionDeclaration(const clang::FunctionDecl *fd)
Free functions are functions that are compatible with an ordinary C function pointer type.
mlir::Type convertType(clang::QualType type)
Convert a Clang type into a mlir::Type.
void add(RValue rvalue, clang::QualType type)
Definition CIRGenCall.h:239
Type for representing both the decl and type of parameters to a function.
Definition CIRGenCall.h:193
static RValue get(mlir::Value v)
Definition CIRGenValue.h:83
static RValue getAggregate(Address addr, bool isVolatile=false)
Convert an Address to an RValue.
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Definition CharUnits.h:185
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
Definition CharUnits.h:63
lookup_result lookup(DeclarationName Name) const
lookup - Find the declarations (if any) with the given Name in this context.
SourceLocation getLocation() const
Definition DeclBase.h:447
bool hasAttr() const
Definition DeclBase.h:585
const ParmVarDecl * getParamDecl(unsigned i) const
Definition Decl.h:2928
GlobalDecl - represents a global declaration.
Definition GlobalDecl.h:60
GlobalDecl getWithKernelReferenceKind(KernelReferenceKind Kind)
Definition GlobalDecl.h:200
StringRef getName() const
Return the actual identifier string.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
GPUDefaultStreamKind GPUDefaultStream
The default stream kind used for HIP kernel launching.
bool shouldMangleDeclName(const NamedDecl *D)
Definition Mangle.cpp:129
void mangleName(GlobalDecl GD, raw_ostream &)
Definition Mangle.cpp:245
IdentifierInfo * getIdentifier() const
Get the identifier that names this declaration, if there is one.
Definition Decl.h:296
bool isMicrosoft() const
Is this ABI an MSVC-compatible ABI?
TargetCXXABI getCXXABI() const
Get the C++ ABI currently in use.
const llvm::VersionTuple & getSDKVersion() const
static DeclContext * castToDeclContext(const TranslationUnitDecl *D)
Definition Decl.h:152
bool isCUDADeviceBuiltinSurfaceType() const
Check if the type is the CUDA device builtin surface type.
Definition Type.cpp:5511
bool isCUDADeviceBuiltinTextureType() const
Check if the type is the CUDA device builtin texture type.
Definition Type.cpp:5520
QualType getType() const
Definition Decl.h:724
SourceRange getSourceRange() const override LLVM_READONLY
Source range that this declaration covers.
Definition Decl.cpp:2172
bool isInline() const
Whether this variable is (C++1z) inline.
Definition Decl.h:1576
bool hasExternalStorage() const
Returns true if a variable has extern or private_extern storage.
Definition Decl.h:1239
DefinitionKind hasDefinition(ASTContext &) const
Check whether this variable is defined in this translation unit.
Definition Decl.cpp:2357
static bool isLocalLinkage(GlobalLinkageKind linkage)
Definition CIROpsEnums.h:51
CIRGenCUDARuntime * createNVCUDARuntime(CIRGenModule &cgm)
constexpr Variable var(Literal L)
Returns the variable of L.
Definition CNFFormula.h:64
@ Address
A pointer to a ValueDecl.
Definition Primitives.h:28
void info(bool Verbose, unsigned Level, const char *Fmt, Ts &&...Args)
Prints an indented note to stderr when Verbose is set.
Definition Utils.h:57
Top level wrappers for InstallAPI frontend operations.
if(T->getSizeExpr()) TRY_TO(TraverseStmt(const_cast< Expr * >(T -> getSizeExpr())))
bool CudaFeatureEnabled(llvm::VersionTuple, CudaFeature)
Definition Cuda.cpp:119
U cast(CodeGen::Address addr)
Definition Address.h:327
clang::CharUnits getPointerAlign() const
clang::CharUnits getSizeAlign() const
cir::PointerType voidPtrTy
void* in address space 0