18#include "mlir/IR/Operation.h"
20#include "clang/AST/Attrs.inc"
27#include "llvm/Support/Casting.h"
41 llvm::StringMap<mlir::Operation *> kernelHandles;
44 llvm::DenseMap<mlir::Operation *, mlir::Operation *> kernelStubs;
49 cir::CUDADeviceVarKind flags;
51 llvm::SmallVector<VarInfo, 16> deviceVars;
54 std::unique_ptr<MangleContext> deviceMC;
57 void emitDeviceStubBodyNew(CIRGenFunction &cgf, cir::FuncOp fn,
58 FunctionArgList &args);
59 mlir::Value prepareKernelArgs(CIRGenFunction &cgf, mlir::Location loc,
60 FunctionArgList &args);
61 mlir::Operation *getKernelHandle(cir::FuncOp fn, GlobalDecl gd)
override;
63 mlir::Operation *getKernelStub(mlir::Operation *handle)
override {
64 auto it = kernelStubs.find(handle);
65 assert(it != kernelStubs.end());
68 std::string addPrefixToName(StringRef funcName)
const;
69 std::string addUnderscoredPrefixToName(StringRef funcName)
const;
72 CIRGenNVCUDARuntime(CIRGenModule &cgm);
73 ~CIRGenNVCUDARuntime();
75 void emitDeviceStub(CIRGenFunction &cgf, cir::FuncOp fn,
76 FunctionArgList &args)
override;
78 void handleVarRegistration(
const VarDecl *vd, cir::GlobalOp var)
override;
79 void finalizeModule()
override;
80 void handleGlobalReplace(cir::GlobalOp oldGV, cir::GlobalOp newGV)
override;
82 void internalizeDeviceSideVar(
const VarDecl *d,
83 cir::GlobalLinkageKind &linkage)
override;
85 std::string getDeviceSideName(
const NamedDecl *nd)
override;
87 void registerDeviceVar(
const VarDecl *vd, cir::GlobalOp &var,
bool isExtern,
90 auto &builder = cgm.getBuilder();
91 var->setAttr(cir::CUDAVarRegistrationInfoAttr::getMnemonic(),
92 cir::CUDAVarRegistrationInfoAttr::get(
95 cir::CUDADeviceVarKind::Variable, isExtern, isConstant,
96 vd->
hasAttr<HIPManagedAttr>()));
97 deviceVars.push_back({
100 cir::CUDADeviceVarKind::Variable,
104 void registerDeviceSurf(
const VarDecl *vd, cir::GlobalOp &var,
106 auto &builder = cgm.getBuilder();
108 var->setAttr(cir::CUDAVarRegistrationInfoAttr::getMnemonic(),
109 cir::CUDAVarRegistrationInfoAttr::get(
110 builder.getContext(),
112 cir::CUDADeviceVarKind::Surface, isExtern,
116 deviceVars.push_back({
119 cir::CUDADeviceVarKind::Surface,
123 void registerDeviceTex(
const VarDecl *vd, cir::GlobalOp &var,
bool isExtern) {
124 auto &builder = cgm.getBuilder();
126 var->setAttr(cir::CUDAVarRegistrationInfoAttr::getMnemonic(),
127 cir::CUDAVarRegistrationInfoAttr::get(
128 builder.getContext(),
130 cir::CUDADeviceVarKind::Texture, isExtern,
134 deviceVars.push_back({
137 cir::CUDADeviceVarKind::Texture,
144std::string CIRGenNVCUDARuntime::addPrefixToName(StringRef funcName)
const {
145 return (prefix + funcName).str();
149CIRGenNVCUDARuntime::addUnderscoredPrefixToName(StringRef funcName)
const {
150 return (
"__" + prefix + funcName).str();
153CIRGenNVCUDARuntime::CIRGenNVCUDARuntime(CIRGenModule &cgm)
154 : CIRGenCUDARuntime(cgm),
155 deviceMC(cgm.getASTContext().cudaNVInitDeviceMC()) {
157 cgm.
errorNYI(
"CIRGenNVCUDARuntime: Offload via LLVM");
164mlir::Value CIRGenNVCUDARuntime::prepareKernelArgs(
CIRGenFunction &cgf,
170 auto voidPtrArrayTy = cir::ArrayType::get(cgm.
voidPtrTy, args.size());
171 mlir::Value kernelArgs =
172 builder.
createAlloca(loc, cir::PointerType::get(voidPtrArrayTy),
175 mlir::Value kernelArgsDecayed =
176 builder.
createCast(cir::CastKind::array_to_ptrdecay, kernelArgs,
179 for (
const auto &[i, arg] : llvm::enumerate(args)) {
182 mlir::Value storePos =
187 builder.CIRBaseBuilderTy::createStore(loc, argAsVoid, storePos);
190 return kernelArgsDecayed;
195void CIRGenNVCUDARuntime::emitDeviceStubBodyNew(CIRGenFunction &cgf,
197 FunctionArgList &args) {
201 cgm.
errorNYI(
"CIRGenNVCUDARuntime: Offload via LLVM");
204 mlir::Location loc = fn.getLoc();
209 mlir::Value kernelArgs = prepareKernelArgs(cgf, loc, args);
226 std::string kernelLaunchAPI =
"LaunchKernel";
228 LangOptions::GPUDefaultStreamKind::PerThread) {
230 kernelLaunchAPI +=
"_spt";
232 kernelLaunchAPI +=
"_ptsz";
235 std::string launchKernelName = addPrefixToName(kernelLaunchAPI);
236 const IdentifierInfo &launchII =
238 FunctionDecl *cudaLaunchKernelFD =
nullptr;
239 for (NamedDecl *result : dc->
lookup(&launchII)) {
240 if (FunctionDecl *fd = dyn_cast<FunctionDecl>(result))
241 cudaLaunchKernelFD = fd;
244 if (cudaLaunchKernelFD ==
nullptr) {
246 "Can't find declaration for " + launchKernelName);
263 builder.
createAlloca(loc, cir::PointerType::get(dim3Ty),
"grid_dim",
266 builder.
createAlloca(loc, cir::PointerType::get(dim3Ty),
"block_dim",
271 loc, cir::PointerType::get(streamTy),
"stream", cgm.
getPointerAlign());
275 sharedMem.getType(), stream.getType()},
277 addUnderscoredPrefixToName(
"PopCallConfiguration"));
288 mlir::Value kernel = [&]() -> mlir::Value {
289 if (cir::GlobalOp globalOp = llvm::dyn_cast_or_null<cir::GlobalOp>(
290 kernelHandles[fn.getSymName()])) {
291 cir::PointerType kernelTy = cir::PointerType::get(globalOp.getSymType());
292 mlir::Value kernelVal = cir::GetGlobalOp::create(builder, loc, kernelTy,
293 globalOp.getSymName());
297 if (cir::FuncOp funcOp = llvm::dyn_cast_or_null<cir::FuncOp>(
298 kernelHandles[fn.getSymName()])) {
299 cir::PointerType kernelTy =
300 cir::PointerType::get(funcOp.getFunctionType());
301 mlir::Value kernelVal =
302 cir::GetGlobalOp::create(builder, loc, kernelTy, funcOp.getSymName());
306 llvm_unreachable(
"Expected stub handle to be cir::GlobalOp or FuncOp");
309 CallArgList launchArgs;
321 RValue::get(builder.CIRBaseBuilderTy::createLoad(loc, sharedMem)),
323 launchArgs.
add(
RValue::get(builder.CIRBaseBuilderTy::createLoad(loc, stream)),
326 mlir::Type launchTy =
330 const CIRGenFunctionInfo &callInfo =
333 ReturnValueSlot(), launchArgs,
false);
337 cgm.
errorNYI(
"MSVC CUDA stub handling");
340void CIRGenNVCUDARuntime::emitDeviceStub(CIRGenFunction &cgf, cir::FuncOp fn,
341 FunctionArgList &args) {
344 llvm::dyn_cast<cir::GlobalOp>(kernelHandles[fn.getSymName()])) {
346 mlir::Type fnPtrTy = globalOp.getSymType();
347 auto sym = mlir::FlatSymbolRefAttr::get(fn.getSymNameAttr());
348 auto gv = cir::GlobalViewAttr::get(fnPtrTy, sym);
350 globalOp->setAttr(
"initial_value", gv);
351 globalOp->removeAttr(
"sym_visibility");
352 globalOp->setAttr(
"alignment", builder.getI64IntegerAttr(
358 CudaFeature::CUDA_USES_NEW_LAUNCH) ||
361 emitDeviceStubBodyNew(cgf, fn, args);
363 cgm.
errorNYI(
"Emit Stub Body Legacy");
367 return new CIRGenNVCUDARuntime(cgm);
370CIRGenNVCUDARuntime::~CIRGenNVCUDARuntime() {}
372mlir::Operation *CIRGenNVCUDARuntime::getKernelHandle(cir::FuncOp fn,
376 auto it = kernelHandles.find(fn.getSymName());
377 if (it != kernelHandles.end()) {
378 mlir::Operation *oldHandle = it->second;
380 if (kernelStubs[oldHandle] == fn)
388 kernelStubs[oldHandle] = fn;
393 kernelStubs.erase(oldHandle);
398 kernelHandles[fn.getSymName()] = fn;
399 kernelStubs[fn] = fn;
407 cir::PointerType fnPtrTy = builder.
getPointerTo(fn.getFunctionType());
408 cir::GlobalOp globalOp =
411 globalOp->setAttr(
"alignment", builder.getI64IntegerAttr(
415 kernelHandles[fn.getSymName()] = globalOp;
416 kernelStubs[globalOp] = fn;
421void CIRGenNVCUDARuntime::internalizeDeviceSideVar(
422 const VarDecl *d, cir::GlobalLinkageKind &linkage) {
425 "internalizeDeviceSideVar: GPU Relocatable Device Code (RDC)");
432 if (d->
hasAttr<CUDADeviceAttr>() || d->
hasAttr<CUDAConstantAttr>() ||
433 d->
hasAttr<CUDASharedAttr>()) {
434 linkage = cir::GlobalLinkageKind::InternalLinkage;
440 "internalizeDeviceSideVar: CUDA Surface/Texture support");
443std::string CIRGenNVCUDARuntime::getDeviceSideName(
const NamedDecl *nd) {
446 if (
auto *fd = dyn_cast<FunctionDecl>(nd))
447 gd = GlobalDecl(fd, KernelReferenceKind::Kernel);
450 std::string deviceSideName;
457 SmallString<256> buffer;
458 llvm::raw_svector_ostream
out(buffer);
460 deviceSideName = std::string(
out.str());
467 SmallString<256> buffer;
468 llvm::raw_svector_ostream
out(buffer);
469 out << deviceSideName;
471 deviceSideName = std::string(
out.str());
473 return deviceSideName;
476void CIRGenNVCUDARuntime::handleVarRegistration(
const VarDecl *vd,
478 if (vd->
hasAttr<CUDADeviceAttr>() || vd->
hasAttr<CUDAConstantAttr>()) {
494 vd->
hasAttr<HIPManagedAttr>()) {
496 vd->
hasAttr<CUDAConstantAttr>());
512void CIRGenNVCUDARuntime::handleGlobalReplace(cir::GlobalOp oldGV,
513 cir::GlobalOp newGV) {
514 for (
auto &info : deviceVars) {
515 if (
info.var == oldGV)
520void CIRGenNVCUDARuntime::finalizeModule() {
534 for (
auto &&info : deviceVars) {
536 bool isDecl =
info.var.isDeclaration();
538 bool isVarOrSurfaceOrTexture = (
kind == cir::CUDADeviceVarKind::Variable ||
539 kind == cir::CUDADeviceVarKind::Surface ||
540 kind == cir::CUDADeviceVarKind::Texture);
541 bool isUsed =
info.d->isUsed();
542 bool hasUsedAttr =
info.d->hasAttr<UsedAttr>();
543 if (!isDecl && !isLocalLinkage && isVarOrSurfaceOrTexture && isUsed &&
545 if (
auto globalValue = mlir::dyn_cast<cir::CIRGlobalValueInterface>(
546 info.var.getOperation())) {
Defines the clang::ASTContext interface.
Provides definitions for the various language-specific address spaces.
*collection of selector each with an associated kind and an ordered *collection of selectors A selector has a kind
__CUDA_BUILTIN_VAR __cuda_builtin_blockDim_t blockDim
__CUDA_BUILTIN_VAR __cuda_builtin_gridDim_t gridDim
mlir::Value createCast(mlir::Location loc, cir::CastKind kind, mlir::Value src, mlir::Type newTy)
cir::PtrStrideOp createPtrStride(mlir::Location loc, mlir::Value base, mlir::Value stride)
cir::PointerType getPointerTo(mlir::Type ty)
mlir::Value createAlloca(mlir::Location loc, cir::PointerType addrType, llvm::StringRef name, mlir::IntegerAttr alignment, mlir::Value dynAllocSize)
mlir::Value createBitcast(mlir::Value src, mlir::Type newTy)
TranslationUnitDecl * getTranslationUnitDecl() const
bool shouldExternalize(const Decl *D) const
Whether a C++ static variable or CUDA/HIP kernel should be externalized.
llvm::SetVector< const VarDecl * > CUDADeviceVarODRUsedByHost
Keep track of CUDA/HIP device-side variables ODR-used by host code.
const TargetInfo & getTargetInfo() const
mlir::Value getPointer() const
cir::ConstantOp getConstInt(mlir::Location loc, llvm::APSInt intVal)
clang::MangleContext & getMangleContext()
Gets the mangle context.
static CIRGenCallee forDirect(mlir::Operation *funcPtr, const CIRGenCalleeInfo &abstractInfo=CIRGenCalleeInfo())
CIRGenTypes & getTypes() const
const clang::LangOptions & getLangOpts() const
const clang::Decl * curFuncDecl
Address getAddrOfLocalVar(const clang::VarDecl *vd)
Return the address of a local variable.
mlir::Value emitRuntimeCall(mlir::Location loc, cir::FuncOp callee, llvm::ArrayRef< mlir::Value > args={}, mlir::NamedAttrList attrs={})
RValue emitCall(const CIRGenFunctionInfo &funcInfo, const CIRGenCallee &callee, ReturnValueSlot returnValue, const CallArgList &args, cir::CIRCallOpInterface *callOp, bool isMustTail, mlir::Location loc)
This class organizes the cross-function state that is used while generating CIR code.
llvm::StringRef getMangledName(clang::GlobalDecl gd)
DiagnosticBuilder errorNYI(SourceLocation, llvm::StringRef)
Helpers to emit "not yet implemented" error diagnostics.
clang::ASTContext & getASTContext() const
CIRGenBuilderTy & getBuilder()
const clang::TargetInfo & getTarget() const
void error(SourceLocation loc, llvm::StringRef error)
Emit a general error that something can't be done.
cir::FuncOp createRuntimeFunction(cir::FuncType ty, llvm::StringRef name, mlir::NamedAttrList extraAttrs={}, bool isLocal=false, bool assumeConvergent=false)
const clang::LangOptions & getLangOpts() const
void printPostfixForExternalizedDecl(llvm::raw_ostream &os, const Decl *d)
Print the postfix for externalized static variable or kernels for single source offloading languages ...
cir::GlobalOp createGlobalOp(mlir::Location loc, llvm::StringRef name, mlir::Type t, bool isConstant=false, mlir::ptr::MemorySpaceAttrInterface addrSpace={}, mlir::Operation *insertPoint=nullptr)
void addCompilerUsedGlobal(cir::CIRGlobalValueInterface gv)
Add a global value to the llvmCompilerUsed list.
CIRGenCXXABI & getCXXABI() const
const CIRGenFunctionInfo & arrangeFunctionDeclaration(const clang::FunctionDecl *fd)
Free functions are functions that are compatible with an ordinary C function pointer type.
mlir::Type convertType(clang::QualType type)
Convert a Clang type into a mlir::Type.
void add(RValue rvalue, clang::QualType type)
Type for representing both the decl and type of parameters to a function.
static RValue get(mlir::Value v)
static RValue getAggregate(Address addr, bool isVolatile=false)
Convert an Address to an RValue.
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
lookup_result lookup(DeclarationName Name) const
lookup - Find the declarations (if any) with the given Name in this context.
SourceLocation getLocation() const
const ParmVarDecl * getParamDecl(unsigned i) const
GlobalDecl - represents a global declaration.
GlobalDecl getWithKernelReferenceKind(KernelReferenceKind Kind)
StringRef getName() const
Return the actual identifier string.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
GPUDefaultStreamKind GPUDefaultStream
The default stream kind used for HIP kernel launching.
bool shouldMangleDeclName(const NamedDecl *D)
void mangleName(GlobalDecl GD, raw_ostream &)
IdentifierInfo * getIdentifier() const
Get the identifier that names this declaration, if there is one.
bool isMicrosoft() const
Is this ABI an MSVC-compatible ABI?
TargetCXXABI getCXXABI() const
Get the C++ ABI currently in use.
const llvm::VersionTuple & getSDKVersion() const
static DeclContext * castToDeclContext(const TranslationUnitDecl *D)
bool isCUDADeviceBuiltinSurfaceType() const
Check if the type is the CUDA device builtin surface type.
bool isCUDADeviceBuiltinTextureType() const
Check if the type is the CUDA device builtin texture type.
SourceRange getSourceRange() const override LLVM_READONLY
Source range that this declaration covers.
bool isInline() const
Whether this variable is (C++1z) inline.
bool hasExternalStorage() const
Returns true if a variable has extern or private_extern storage.
DefinitionKind hasDefinition(ASTContext &) const
Check whether this variable is defined in this translation unit.
static bool isLocalLinkage(GlobalLinkageKind linkage)
CIRGenCUDARuntime * createNVCUDARuntime(CIRGenModule &cgm)
constexpr Variable var(Literal L)
Returns the variable of L.
@ Address
A pointer to a ValueDecl.
void info(bool Verbose, unsigned Level, const char *Fmt, Ts &&...Args)
Prints an indented note to stderr when Verbose is set.
Top level wrappers for InstallAPI frontend operations.
if(T->getSizeExpr()) TRY_TO(TraverseStmt(const_cast< Expr * >(T -> getSizeExpr())))
bool CudaFeatureEnabled(llvm::VersionTuple, CudaFeature)
U cast(CodeGen::Address addr)
clang::CharUnits getPointerAlign() const
clang::CharUnits getSizeAlign() const
cir::PointerType voidPtrTy
void* in address space 0