clang 24.0.0git
CIRGenBuiltinNVPTX.cpp
Go to the documentation of this file.
1//===---- CIRGenBuiltinNVPTX.cpp - Emit CIR for NVPTX builtins ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit NVPTX Builtin calls.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CIRGenFunction.h"
14
15#include "mlir/IR/Value.h"
16#include "clang/Basic/TargetBuiltins.h"
17#include "clang/CIR/Dialect/IR/CIRDialect.h"
18#include "llvm/Support/NVPTXAddrSpace.h"
19
20using namespace clang;
21using namespace clang::CIRGen;
22
23static mlir::Value makeLdu(CIRGenFunction &cgf, const CallExpr *expr,
24 llvm::StringRef intrinsicName) {
25 auto &builder = cgf.getBuilder();
26 Address ptr = cgf.emitPointerWithAlignment(expr->getArg(0));
27 QualType argType = expr->getArg(0)->getType();
28 mlir::Type elemTy = cgf.convertTypeForMem(argType->getPointeeType());
29 clang::CharUnits align = ptr.getAlignment();
30 mlir::Location loc = cgf.getLoc(expr->getExprLoc());
31 mlir::Value alignVal =
32 builder.getConstantInt(loc, builder.getSInt32Ty(), align.getQuantity());
33 return cir::LLVMIntrinsicCallOp::create(
34 builder, loc, builder.getStringAttr(intrinsicName), elemTy,
35 {ptr.emitRawPointer(), alignVal})
36 .getResult();
37}
38
39static mlir::Value makeLdg(CIRGenFunction &cgf, const CallExpr *expr) {
40 auto &builder = cgf.getBuilder();
41 Address ptr = cgf.emitPointerWithAlignment(expr->getArg(0));
42 QualType argType = expr->getArg(0)->getType();
43 mlir::Type elemTy = cgf.convertTypeForMem(argType->getPointeeType());
44 mlir::Location loc = cgf.getLoc(expr->getExprLoc());
45
46 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL.
47 mlir::Type globalPtrTy = cir::PointerType::get(
48 elemTy, cir::TargetAddressSpaceAttr::get(
49 builder.getContext(), llvm::NVPTXAS::ADDRESS_SPACE_GLOBAL));
50 mlir::Value asc =
51 builder.createAddrSpaceCast(loc, ptr.getPointer(), globalPtrTy);
52 cir::LoadOp load =
53 builder.createAlignedLoad(loc, elemTy, asc, ptr.getAlignment());
54 load.setInvariant(true);
55 return load.getResult();
56}
57
58/// Emit a CIR LLVMIntrinsicCallOp for a unary NVVM intrinsic.
59/// The result type is inferred from the single argument.
60static mlir::Value emitUnaryNVVMIntrinsic(CIRGenFunction &cgf,
61 const CallExpr *expr,
62 llvm::StringRef intrinsicName) {
63 auto &builder = cgf.getBuilder();
64 mlir::Value arg = cgf.emitScalarExpr(expr->getArg(0));
65 return cir::LLVMIntrinsicCallOp::create(
66 builder, cgf.getLoc(expr->getExprLoc()),
67 builder.getStringAttr(intrinsicName), arg.getType(), {arg})
68 .getResult();
69}
70
71/// Emit a CIR LLVMIntrinsicCallOp for an NVVM fadd/fmul intrinsic, which takes
72/// the rounding mode as a trailing operand.
73static mlir::Value emitNVVMFPArith(CIRGenFunction &cgf, const CallExpr *expr,
74 llvm::StringRef intrinsicName,
75 llvm::APFloat::roundingMode rm) {
76 auto &builder = cgf.getBuilder();
77 mlir::Location loc = cgf.getLoc(expr->getExprLoc());
78 mlir::Value lhs = cgf.emitScalarExpr(expr->getArg(0));
79 mlir::Value rhs = cgf.emitScalarExpr(expr->getArg(1));
80 mlir::Value rnd =
81 builder.getConstInt(loc, builder.getSInt32Ty(), static_cast<int>(rm));
82 return cir::LLVMIntrinsicCallOp::create(builder, loc,
83 builder.getStringAttr(intrinsicName),
84 lhs.getType(), {lhs, rhs, rnd})
85 .getResult();
86}
87
88static mlir::Value emitBar0Reduction(CIRGenFunction &cgf, const CallExpr *expr,
89 llvm::StringRef intrinsicName,
90 bool returnsPred) {
91 CIRGenBuilderTy &builder = cgf.getBuilder();
92 mlir::Location loc = cgf.getLoc(expr->getExprLoc());
93 mlir::Type si32Ty = builder.getSInt32Ty();
94 mlir::Value zero = builder.getNullValue(si32Ty, loc);
95 mlir::Value pred = builder.createCompare(
96 loc, cir::CmpOpKind::ne, cgf.emitScalarExpr(expr->getArg(0)), zero);
97 mlir::Type resultTy = returnsPred ? mlir::Type(builder.getBoolTy()) : si32Ty;
98 mlir::Value result = builder.emitIntrinsicCallOp(
99 loc, intrinsicName, resultTy, mlir::ValueRange{zero, pred});
100 if (returnsPred)
101 result = builder.createBoolToInt(result, si32Ty);
102 return result;
103}
104
105static mlir::Value makeScopedAtomicRMW(CIRGenFunction &cgf,
106 const CallExpr *expr,
107 cir::AtomicFetchKind kind,
108 cir::SyncScopeKind scope) {
109 auto &builder = cgf.getBuilder();
110 Address destAddr = cgf.emitPointerWithAlignment(expr->getArg(0));
111 mlir::Value destValue = destAddr.emitRawPointer();
112 mlir::Value val = cgf.emitScalarExpr(expr->getArg(1));
113 auto rmwi = cir::AtomicFetchOp::create(
114 builder, cgf.getLoc(expr->getSourceRange()), destValue, val, kind,
115 cir::MemOrder::Relaxed, scope, /*is_volatile=*/false,
116 /*fetch_first=*/true);
117 return rmwi->getResult(0);
118}
119
120static mlir::Value makeScopedAtomicXchg(CIRGenFunction &cgf,
121 const CallExpr *expr,
122 cir::SyncScopeKind scope) {
123 auto &builder = cgf.getBuilder();
124 Address destAddr = cgf.emitPointerWithAlignment(expr->getArg(0));
125 mlir::Value destValue = destAddr.emitRawPointer();
126 mlir::Value val = cgf.emitScalarExpr(expr->getArg(1));
127 auto xchg = cir::AtomicXchgOp::create(
128 builder, cgf.getLoc(expr->getSourceRange()), destValue, val,
129 cir::MemOrder::Relaxed, scope, /*is_volatile=*/false);
130 return xchg.getResult();
131}
132
133std::optional<mlir::Value>
134CIRGenFunction::emitNVPTXBuiltinExpr(unsigned builtinId, const CallExpr *expr) {
135 switch (builtinId) {
136 case NVPTX::BI__nvvm_atom_add_gen_i:
137 case NVPTX::BI__nvvm_atom_add_gen_l:
138 case NVPTX::BI__nvvm_atom_add_gen_ll:
139 return makeBinaryAtomicValue(cir::AtomicFetchKind::Add, expr,
140 /*originalArgType=*/nullptr,
141 /*emittedArgValue=*/nullptr,
142 cir::MemOrder::Relaxed);
143 case NVPTX::BI__nvvm_atom_sub_gen_i:
144 case NVPTX::BI__nvvm_atom_sub_gen_l:
145 case NVPTX::BI__nvvm_atom_sub_gen_ll:
146 return makeBinaryAtomicValue(cir::AtomicFetchKind::Sub, expr,
147 /*originalArgType=*/nullptr,
148 /*emittedArgValue=*/nullptr,
149 cir::MemOrder::Relaxed);
150 case NVPTX::BI__nvvm_atom_and_gen_i:
151 case NVPTX::BI__nvvm_atom_and_gen_l:
152 case NVPTX::BI__nvvm_atom_and_gen_ll:
153 return makeBinaryAtomicValue(cir::AtomicFetchKind::And, expr,
154 /*originalArgType=*/nullptr,
155 /*emittedArgValue=*/nullptr,
156 cir::MemOrder::Relaxed);
157 case NVPTX::BI__nvvm_atom_or_gen_i:
158 case NVPTX::BI__nvvm_atom_or_gen_l:
159 case NVPTX::BI__nvvm_atom_or_gen_ll:
160 return makeBinaryAtomicValue(cir::AtomicFetchKind::Or, expr,
161 /*originalArgType=*/nullptr,
162 /*emittedArgValue=*/nullptr,
163 cir::MemOrder::Relaxed);
164 case NVPTX::BI__nvvm_atom_xor_gen_i:
165 case NVPTX::BI__nvvm_atom_xor_gen_l:
166 case NVPTX::BI__nvvm_atom_xor_gen_ll:
167 return makeBinaryAtomicValue(cir::AtomicFetchKind::Xor, expr,
168 /*originalArgType=*/nullptr,
169 /*emittedArgValue=*/nullptr,
170 cir::MemOrder::Relaxed);
171 case NVPTX::BI__nvvm_atom_xchg_gen_i:
172 case NVPTX::BI__nvvm_atom_xchg_gen_l:
173 case NVPTX::BI__nvvm_atom_xchg_gen_ll:
174 return makeScopedAtomicXchg(*this, expr, cir::SyncScopeKind::System);
175 case NVPTX::BI__nvvm_atom_max_gen_i:
176 case NVPTX::BI__nvvm_atom_max_gen_l:
177 case NVPTX::BI__nvvm_atom_max_gen_ll:
178 return makeBinaryAtomicValue(cir::AtomicFetchKind::Max, expr,
179 /*originalArgType=*/nullptr,
180 /*emittedArgValue=*/nullptr,
181 cir::MemOrder::Relaxed);
182 case NVPTX::BI__nvvm_atom_max_gen_ui:
183 case NVPTX::BI__nvvm_atom_max_gen_ul:
184 case NVPTX::BI__nvvm_atom_max_gen_ull:
185 return makeBinaryAtomicValue(cir::AtomicFetchKind::Max, expr,
186 /*originalArgType=*/nullptr,
187 /*emittedArgValue=*/nullptr,
188 cir::MemOrder::Relaxed);
189 case NVPTX::BI__nvvm_atom_min_gen_i:
190 case NVPTX::BI__nvvm_atom_min_gen_l:
191 case NVPTX::BI__nvvm_atom_min_gen_ll:
192 return makeBinaryAtomicValue(cir::AtomicFetchKind::Min, expr,
193 /*originalArgType=*/nullptr,
194 /*emittedArgValue=*/nullptr,
195 cir::MemOrder::Relaxed);
196 case NVPTX::BI__nvvm_atom_min_gen_ui:
197 case NVPTX::BI__nvvm_atom_min_gen_ul:
198 case NVPTX::BI__nvvm_atom_min_gen_ull:
199 return makeBinaryAtomicValue(cir::AtomicFetchKind::Min, expr,
200 /*originalArgType=*/nullptr,
201 /*emittedArgValue=*/nullptr,
202 cir::MemOrder::Relaxed);
203 case NVPTX::BI__nvvm_atom_cas_gen_us:
204 case NVPTX::BI__nvvm_atom_cas_gen_i:
205 case NVPTX::BI__nvvm_atom_cas_gen_l:
206 case NVPTX::BI__nvvm_atom_cas_gen_ll:
207 return emitAtomicCmpXchg(expr, /*returnBool=*/false, cir::MemOrder::Relaxed,
208 cir::MemOrder::Relaxed,
209 cir::SyncScopeKind::System);
210 case NVPTX::BI__nvvm_atom_add_gen_f:
211 case NVPTX::BI__nvvm_atom_add_gen_d:
212 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Add,
213 cir::SyncScopeKind::System);
214 case NVPTX::BI__nvvm_atom_inc_gen_ui:
215 return makeBinaryAtomicValue(cir::AtomicFetchKind::UIncWrap, expr,
216 /*originalArgType=*/nullptr,
217 /*emittedArgValue=*/nullptr,
218 cir::MemOrder::Relaxed);
219 case NVPTX::BI__nvvm_atom_dec_gen_ui:
220 return makeBinaryAtomicValue(cir::AtomicFetchKind::UDecWrap, expr,
221 /*originalArgType=*/nullptr,
222 /*emittedArgValue=*/nullptr,
223 cir::MemOrder::Relaxed);
224 case NVPTX::BI__nvvm_ldg_c:
225 case NVPTX::BI__nvvm_ldg_sc:
226 case NVPTX::BI__nvvm_ldg_c2:
227 case NVPTX::BI__nvvm_ldg_sc2:
228 case NVPTX::BI__nvvm_ldg_c4:
229 case NVPTX::BI__nvvm_ldg_sc4:
230 case NVPTX::BI__nvvm_ldg_s:
231 case NVPTX::BI__nvvm_ldg_s2:
232 case NVPTX::BI__nvvm_ldg_s4:
233 case NVPTX::BI__nvvm_ldg_i:
234 case NVPTX::BI__nvvm_ldg_i2:
235 case NVPTX::BI__nvvm_ldg_i4:
236 case NVPTX::BI__nvvm_ldg_l:
237 case NVPTX::BI__nvvm_ldg_l2:
238 case NVPTX::BI__nvvm_ldg_ll:
239 case NVPTX::BI__nvvm_ldg_ll2:
240 case NVPTX::BI__nvvm_ldg_uc:
241 case NVPTX::BI__nvvm_ldg_uc2:
242 case NVPTX::BI__nvvm_ldg_uc4:
243 case NVPTX::BI__nvvm_ldg_us:
244 case NVPTX::BI__nvvm_ldg_us2:
245 case NVPTX::BI__nvvm_ldg_us4:
246 case NVPTX::BI__nvvm_ldg_ui:
247 case NVPTX::BI__nvvm_ldg_ui2:
248 case NVPTX::BI__nvvm_ldg_ui4:
249 case NVPTX::BI__nvvm_ldg_ul:
250 case NVPTX::BI__nvvm_ldg_ul2:
251 case NVPTX::BI__nvvm_ldg_ull:
252 case NVPTX::BI__nvvm_ldg_ull2:
253 case NVPTX::BI__nvvm_ldg_f:
254 case NVPTX::BI__nvvm_ldg_f2:
255 case NVPTX::BI__nvvm_ldg_f4:
256 case NVPTX::BI__nvvm_ldg_d:
257 case NVPTX::BI__nvvm_ldg_d2:
258 return makeLdg(*this, expr);
259 case NVPTX::BI__nvvm_ldu_c:
260 case NVPTX::BI__nvvm_ldu_sc:
261 case NVPTX::BI__nvvm_ldu_c2:
262 case NVPTX::BI__nvvm_ldu_sc2:
263 case NVPTX::BI__nvvm_ldu_c4:
264 case NVPTX::BI__nvvm_ldu_sc4:
265 case NVPTX::BI__nvvm_ldu_s:
266 case NVPTX::BI__nvvm_ldu_s2:
267 case NVPTX::BI__nvvm_ldu_s4:
268 case NVPTX::BI__nvvm_ldu_i:
269 case NVPTX::BI__nvvm_ldu_i2:
270 case NVPTX::BI__nvvm_ldu_i4:
271 case NVPTX::BI__nvvm_ldu_l:
272 case NVPTX::BI__nvvm_ldu_l2:
273 case NVPTX::BI__nvvm_ldu_ll:
274 case NVPTX::BI__nvvm_ldu_ll2:
275 case NVPTX::BI__nvvm_ldu_uc:
276 case NVPTX::BI__nvvm_ldu_uc2:
277 case NVPTX::BI__nvvm_ldu_uc4:
278 case NVPTX::BI__nvvm_ldu_us:
279 case NVPTX::BI__nvvm_ldu_us2:
280 case NVPTX::BI__nvvm_ldu_us4:
281 case NVPTX::BI__nvvm_ldu_ui:
282 case NVPTX::BI__nvvm_ldu_ui2:
283 case NVPTX::BI__nvvm_ldu_ui4:
284 case NVPTX::BI__nvvm_ldu_ul:
285 case NVPTX::BI__nvvm_ldu_ul2:
286 case NVPTX::BI__nvvm_ldu_ull:
287 case NVPTX::BI__nvvm_ldu_ull2:
288 return makeLdu(*this, expr, "nvvm.ldu.global.i");
289 case NVPTX::BI__nvvm_ldu_f:
290 case NVPTX::BI__nvvm_ldu_f2:
291 case NVPTX::BI__nvvm_ldu_f4:
292 case NVPTX::BI__nvvm_ldu_d:
293 case NVPTX::BI__nvvm_ldu_d2:
294 return makeLdu(*this, expr, "nvvm.ldu.global.f");
295 case NVPTX::BI__nvvm_atom_cta_add_gen_i:
296 case NVPTX::BI__nvvm_atom_cta_add_gen_l:
297 case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
298 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Add,
299 cir::SyncScopeKind::Workgroup);
300 case NVPTX::BI__nvvm_atom_sys_add_gen_i:
301 case NVPTX::BI__nvvm_atom_sys_add_gen_l:
302 case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
303 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Add,
304 cir::SyncScopeKind::System);
305 case NVPTX::BI__nvvm_atom_cta_add_gen_f:
306 case NVPTX::BI__nvvm_atom_cta_add_gen_d:
307 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Add,
308 cir::SyncScopeKind::Workgroup);
309 case NVPTX::BI__nvvm_atom_sys_add_gen_f:
310 case NVPTX::BI__nvvm_atom_sys_add_gen_d:
311 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Add,
312 cir::SyncScopeKind::System);
313 case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
314 case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
315 case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
316 return makeScopedAtomicXchg(*this, expr, cir::SyncScopeKind::Workgroup);
317 case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
318 case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
319 case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
320 return makeScopedAtomicXchg(*this, expr, cir::SyncScopeKind::System);
321 case NVPTX::BI__nvvm_atom_cta_max_gen_i:
322 case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
323 case NVPTX::BI__nvvm_atom_cta_max_gen_l:
324 case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
325 case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
326 case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
327 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Max,
328 cir::SyncScopeKind::Workgroup);
329 case NVPTX::BI__nvvm_atom_sys_max_gen_i:
330 case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
331 case NVPTX::BI__nvvm_atom_sys_max_gen_l:
332 case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
333 case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
334 case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
335 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Max,
336 cir::SyncScopeKind::System);
337 case NVPTX::BI__nvvm_atom_cta_min_gen_i:
338 case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
339 case NVPTX::BI__nvvm_atom_cta_min_gen_l:
340 case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
341 case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
342 case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
343 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Min,
344 cir::SyncScopeKind::Workgroup);
345 case NVPTX::BI__nvvm_atom_sys_min_gen_i:
346 case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
347 case NVPTX::BI__nvvm_atom_sys_min_gen_l:
348 case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
349 case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
350 case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
351 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Min,
352 cir::SyncScopeKind::System);
353 case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
354 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::UIncWrap,
355 cir::SyncScopeKind::Workgroup);
356 case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
357 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::UDecWrap,
358 cir::SyncScopeKind::Workgroup);
359 case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
360 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::UIncWrap,
361 cir::SyncScopeKind::System);
362 case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
363 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::UDecWrap,
364 cir::SyncScopeKind::System);
365 case NVPTX::BI__nvvm_atom_cta_and_gen_i:
366 case NVPTX::BI__nvvm_atom_cta_and_gen_l:
367 case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
368 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::And,
369 cir::SyncScopeKind::Workgroup);
370 case NVPTX::BI__nvvm_atom_sys_and_gen_i:
371 case NVPTX::BI__nvvm_atom_sys_and_gen_l:
372 case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
373 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::And,
374 cir::SyncScopeKind::System);
375 case NVPTX::BI__nvvm_atom_cta_or_gen_i:
376 case NVPTX::BI__nvvm_atom_cta_or_gen_l:
377 case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
378 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Or,
379 cir::SyncScopeKind::Workgroup);
380 case NVPTX::BI__nvvm_atom_sys_or_gen_i:
381 case NVPTX::BI__nvvm_atom_sys_or_gen_l:
382 case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
383 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Or,
384 cir::SyncScopeKind::System);
385 case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
386 case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
387 case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
388 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Xor,
389 cir::SyncScopeKind::Workgroup);
390 case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
391 case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
392 case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
393 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Xor,
394 cir::SyncScopeKind::System);
395 case NVPTX::BI__nvvm_atom_cta_cas_gen_us:
396 case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
397 case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
398 case NVPTX::BI__nvvm_atom_cta_cas_gen_ll:
399 return emitAtomicCmpXchg(expr, /*returnBool=*/false, cir::MemOrder::Relaxed,
400 cir::MemOrder::Relaxed,
401 cir::SyncScopeKind::Workgroup);
402 case NVPTX::BI__nvvm_atom_sys_cas_gen_us:
403 case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
404 case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
405 case NVPTX::BI__nvvm_atom_sys_cas_gen_ll:
406 return emitAtomicCmpXchg(expr, /*returnBool=*/false, cir::MemOrder::Relaxed,
407 cir::MemOrder::Relaxed,
408 cir::SyncScopeKind::System);
409 case NVPTX::BI__nvvm_match_all_sync_i32p:
410 case NVPTX::BI__nvvm_match_all_sync_i64p:
411 cgm.errorNYI(expr->getSourceRange(),
412 std::string("unimplemented NVPTX builtin call: ") +
413 getContext().BuiltinInfo.getName(builtinId));
414 return mlir::Value{};
415 // FP MMA loads
416 case NVPTX::BI__hmma_m16n16k16_ld_a:
417 case NVPTX::BI__hmma_m16n16k16_ld_b:
418 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
419 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
420 case NVPTX::BI__hmma_m32n8k16_ld_a:
421 case NVPTX::BI__hmma_m32n8k16_ld_b:
422 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
423 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
424 case NVPTX::BI__hmma_m8n32k16_ld_a:
425 case NVPTX::BI__hmma_m8n32k16_ld_b:
426 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
427 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
428 cgm.errorNYI(expr->getSourceRange(),
429 std::string("unimplemented NVPTX builtin call: ") +
430 getContext().BuiltinInfo.getName(builtinId));
431 return mlir::Value{};
432 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
433 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
434 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
435 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
436 case NVPTX::BI__imma_m16n16k16_ld_c:
437 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
438 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
439 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
440 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
441 case NVPTX::BI__imma_m32n8k16_ld_c:
442 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
443 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
444 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
445 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
446 case NVPTX::BI__imma_m8n32k16_ld_c:
447 cgm.errorNYI(expr->getSourceRange(),
448 std::string("unimplemented NVPTX builtin call: ") +
449 getContext().BuiltinInfo.getName(builtinId));
450 return mlir::Value{};
451 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
452 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
453 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
454 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
455 case NVPTX::BI__imma_m8n8k32_ld_c:
456 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
457 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
458 case NVPTX::BI__bmma_m8n8k128_ld_c:
459 cgm.errorNYI(expr->getSourceRange(),
460 std::string("unimplemented NVPTX builtin call: ") +
461 getContext().BuiltinInfo.getName(builtinId));
462 return mlir::Value{};
463 case NVPTX::BI__dmma_m8n8k4_ld_a:
464 case NVPTX::BI__dmma_m8n8k4_ld_b:
465 case NVPTX::BI__dmma_m8n8k4_ld_c:
466 cgm.errorNYI(expr->getSourceRange(),
467 std::string("unimplemented NVPTX builtin call: ") +
468 getContext().BuiltinInfo.getName(builtinId));
469 return mlir::Value{};
470 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
471 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
472 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
473 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
474 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
475 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
476 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
477 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
478 case NVPTX::BI__mma_tf32_m16n16k8_ld_c:
479 cgm.errorNYI(expr->getSourceRange(),
480 std::string("unimplemented NVPTX builtin call: ") +
481 getContext().BuiltinInfo.getName(builtinId));
482 return mlir::Value{};
483 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
484 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
485 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
486 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
487 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
488 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
489 case NVPTX::BI__imma_m16n16k16_st_c_i32:
490 case NVPTX::BI__imma_m32n8k16_st_c_i32:
491 case NVPTX::BI__imma_m8n32k16_st_c_i32:
492 case NVPTX::BI__imma_m8n8k32_st_c_i32:
493 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
494 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
495 case NVPTX::BI__mma_m16n16k8_st_c_f32:
496 cgm.errorNYI(expr->getSourceRange(),
497 std::string("unimplemented NVPTX builtin call: ") +
498 getContext().BuiltinInfo.getName(builtinId));
499 return mlir::Value{};
500 // BI__hmma_m16n16k16_mma_<Dtype><CType>(d, a, b, c, layout, satf) -->
501 // Intrinsic::nvvm_wmma_m16n16k16_mma_sync<layout A,B><DType><CType><Satf>
502 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
503 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
504 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
505 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
506 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
507 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
508 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
509 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
510 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
511 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
512 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
513 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
514 case NVPTX::BI__imma_m16n16k16_mma_s8:
515 case NVPTX::BI__imma_m16n16k16_mma_u8:
516 case NVPTX::BI__imma_m32n8k16_mma_s8:
517 case NVPTX::BI__imma_m32n8k16_mma_u8:
518 case NVPTX::BI__imma_m8n32k16_mma_s8:
519 case NVPTX::BI__imma_m8n32k16_mma_u8:
520 case NVPTX::BI__imma_m8n8k32_mma_s4:
521 case NVPTX::BI__imma_m8n8k32_mma_u4:
522 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
523 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
524 case NVPTX::BI__dmma_m8n8k4_mma_f64:
525 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
526 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
527 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
528 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32:
529 cgm.errorNYI(expr->getSourceRange(),
530 std::string("unimplemented NVPTX builtin call: ") +
531 getContext().BuiltinInfo.getName(builtinId));
532 return mlir::Value{};
533 // The following builtins require half type support
534 case NVPTX::BI__nvvm_ff2f16x2_rn:
535 cgm.errorNYI(expr->getSourceRange(),
536 std::string("unimplemented NVPTX builtin call: ") +
537 getContext().BuiltinInfo.getName(builtinId));
538 return mlir::Value{};
539 case NVPTX::BI__nvvm_ff2f16x2_rn_relu:
540 cgm.errorNYI(expr->getSourceRange(),
541 std::string("unimplemented NVPTX builtin call: ") +
542 getContext().BuiltinInfo.getName(builtinId));
543 return mlir::Value{};
544 case NVPTX::BI__nvvm_ff2f16x2_rz:
545 cgm.errorNYI(expr->getSourceRange(),
546 std::string("unimplemented NVPTX builtin call: ") +
547 getContext().BuiltinInfo.getName(builtinId));
548 return mlir::Value{};
549 case NVPTX::BI__nvvm_ff2f16x2_rz_relu:
550 cgm.errorNYI(expr->getSourceRange(),
551 std::string("unimplemented NVPTX builtin call: ") +
552 getContext().BuiltinInfo.getName(builtinId));
553 return mlir::Value{};
554 case NVPTX::BI__nvvm_fma_rn_f16:
555 return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.f16")
556 .getValue();
557 case NVPTX::BI__nvvm_fma_rn_f16x2:
558 return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.f16x2")
559 .getValue();
560 case NVPTX::BI__nvvm_fma_rn_ftz_f16:
561 return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.ftz.f16")
562 .getValue();
563 case NVPTX::BI__nvvm_fma_rn_ftz_f16x2:
564 return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.ftz.f16x2")
565 .getValue();
566 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16:
567 return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.ftz.relu.f16")
568 .getValue();
569 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16x2:
570 return emitBuiltinWithOneOverloadedType<3>(expr,
571 "nvvm.fma.rn.ftz.relu.f16x2")
572 .getValue();
573 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16:
574 return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.ftz.sat.f16")
575 .getValue();
576 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16x2:
577 return emitBuiltinWithOneOverloadedType<3>(expr,
578 "nvvm.fma.rn.ftz.sat.f16x2")
579 .getValue();
580 case NVPTX::BI__nvvm_fma_rn_relu_f16:
581 return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.relu.f16")
582 .getValue();
583 case NVPTX::BI__nvvm_fma_rn_relu_f16x2:
584 return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.relu.f16x2")
585 .getValue();
586 case NVPTX::BI__nvvm_fma_rn_sat_f16:
587 return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.sat.f16")
588 .getValue();
589 case NVPTX::BI__nvvm_fma_rn_sat_f16x2:
590 return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.sat.f16x2")
591 .getValue();
592 case NVPTX::BI__nvvm_fma_rn_oob_f16:
593 case NVPTX::BI__nvvm_fma_rn_oob_f16x2:
594 case NVPTX::BI__nvvm_fma_rn_oob_bf16:
595 case NVPTX::BI__nvvm_fma_rn_oob_bf16x2:
596 return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.oob")
597 .getValue();
598 case NVPTX::BI__nvvm_fma_rn_oob_relu_f16:
599 case NVPTX::BI__nvvm_fma_rn_oob_relu_f16x2:
600 case NVPTX::BI__nvvm_fma_rn_oob_relu_bf16:
601 case NVPTX::BI__nvvm_fma_rn_oob_relu_bf16x2:
602 return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.oob.relu")
603 .getValue();
604 case NVPTX::BI__nvvm_fmax_f16:
605 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmax.f16")
606 .getValue();
607 case NVPTX::BI__nvvm_fmax_f16x2:
608 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmax.f16x2")
609 .getValue();
610 case NVPTX::BI__nvvm_fmax_ftz_f16:
611 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmax.ftz.f16")
612 .getValue();
613 case NVPTX::BI__nvvm_fmax_ftz_f16x2:
614 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmax.ftz.f16x2")
615 .getValue();
616 case NVPTX::BI__nvvm_fmax_ftz_nan_f16:
617 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmax.ftz.nan.f16")
618 .getValue();
619 case NVPTX::BI__nvvm_fmax_ftz_nan_f16x2:
620 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmax.ftz.nan.f16x2")
621 .getValue();
622 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16:
623 return emitBuiltinWithOneOverloadedType<2>(
624 expr, "nvvm.fmax.ftz.nan.xorsign.abs.f16")
625 .getValue();
626 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16x2:
627 return emitBuiltinWithOneOverloadedType<2>(
628 expr, "nvvm.fmax.ftz.nan.xorsign.abs.f16x2")
629 .getValue();
630 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16:
631 return emitBuiltinWithOneOverloadedType<2>(expr,
632 "nvvm.fmax.ftz.xorsign.abs.f16")
633 .getValue();
634 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16x2:
635 return emitBuiltinWithOneOverloadedType<2>(
636 expr, "nvvm.fmax.ftz.xorsign.abs.f16x2")
637 .getValue();
638 case NVPTX::BI__nvvm_fmax_nan_f16:
639 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmax.nan.f16")
640 .getValue();
641 case NVPTX::BI__nvvm_fmax_nan_f16x2:
642 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmax.nan.f16x2")
643 .getValue();
644 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16:
645 return emitBuiltinWithOneOverloadedType<2>(expr,
646 "nvvm.fmax.nan.xorsign.abs.f16")
647 .getValue();
648 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16x2:
649 return emitBuiltinWithOneOverloadedType<2>(
650 expr, "nvvm.fmax.nan.xorsign.abs.f16x2")
651 .getValue();
652 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16:
653 return emitBuiltinWithOneOverloadedType<2>(expr,
654 "nvvm.fmax.xorsign.abs.f16")
655 .getValue();
656 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16x2:
657 return emitBuiltinWithOneOverloadedType<2>(expr,
658 "nvvm.fmax.xorsign.abs.f16x2")
659 .getValue();
660 case NVPTX::BI__nvvm_fmin_f16:
661 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmin.f16")
662 .getValue();
663 case NVPTX::BI__nvvm_fmin_f16x2:
664 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmin.f16x2")
665 .getValue();
666 case NVPTX::BI__nvvm_fmin_ftz_f16:
667 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmin.ftz.f16")
668 .getValue();
669 case NVPTX::BI__nvvm_fmin_ftz_f16x2:
670 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmin.ftz.f16x2")
671 .getValue();
672 case NVPTX::BI__nvvm_fmin_ftz_nan_f16:
673 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmin.ftz.nan.f16")
674 .getValue();
675 case NVPTX::BI__nvvm_fmin_ftz_nan_f16x2:
676 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmin.ftz.nan.f16x2")
677 .getValue();
678 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16:
679 return emitBuiltinWithOneOverloadedType<2>(
680 expr, "nvvm.fmin.ftz.nan.xorsign.abs.f16")
681 .getValue();
682 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16x2:
683 return emitBuiltinWithOneOverloadedType<2>(
684 expr, "nvvm.fmin.ftz.nan.xorsign.abs.f16x2")
685 .getValue();
686 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16:
687 return emitBuiltinWithOneOverloadedType<2>(expr,
688 "nvvm.fmin.ftz.xorsign.abs.f16")
689 .getValue();
690 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16x2:
691 return emitBuiltinWithOneOverloadedType<2>(
692 expr, "nvvm.fmin.ftz.xorsign.abs.f16x2")
693 .getValue();
694 case NVPTX::BI__nvvm_fmin_nan_f16:
695 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmin.nan.f16")
696 .getValue();
697 case NVPTX::BI__nvvm_fmin_nan_f16x2:
698 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmin.nan.f16x2")
699 .getValue();
700 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16:
701 return emitBuiltinWithOneOverloadedType<2>(expr,
702 "nvvm.fmin.nan.xorsign.abs.f16")
703 .getValue();
704 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16x2:
705 return emitBuiltinWithOneOverloadedType<2>(
706 expr, "nvvm.fmin.nan.xorsign.abs.f16x2")
707 .getValue();
708 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16:
709 return emitBuiltinWithOneOverloadedType<2>(expr,
710 "nvvm.fmin.xorsign.abs.f16")
711 .getValue();
712 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16x2:
713 return emitBuiltinWithOneOverloadedType<2>(expr,
714 "nvvm.fmin.xorsign.abs.f16x2")
715 .getValue();
716 case NVPTX::BI__nvvm_fabs_f:
717 case NVPTX::BI__nvvm_abs_bf16:
718 case NVPTX::BI__nvvm_abs_bf16x2:
719 case NVPTX::BI__nvvm_fabs_f16:
720 case NVPTX::BI__nvvm_fabs_f16x2:
721 return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.fabs");
722 case NVPTX::BI__nvvm_fabs_ftz_f:
723 case NVPTX::BI__nvvm_fabs_ftz_f16:
724 case NVPTX::BI__nvvm_fabs_ftz_f16x2:
725 return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.fabs.ftz");
726 case NVPTX::BI__nvvm_fabs_d:
727 return emitUnaryNVVMIntrinsic(*this, expr, "fabs");
728 case NVPTX::BI__nvvm_ex2_approx_d:
729 case NVPTX::BI__nvvm_ex2_approx_f:
730 case NVPTX::BI__nvvm_ex2_approx_f16:
731 case NVPTX::BI__nvvm_ex2_approx_f16x2:
732 return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.ex2.approx");
733 case NVPTX::BI__nvvm_ex2_approx_ftz_f:
734 return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.ex2.approx.ftz");
735 case NVPTX::BI__nvvm_add_rn_f:
736 case NVPTX::BI__nvvm_add_rn_d:
737 return emitNVVMFPArith(*this, expr, "nvvm.fadd",
738 llvm::APFloat::rmNearestTiesToEven);
739 case NVPTX::BI__nvvm_add_rz_f:
740 case NVPTX::BI__nvvm_add_rz_d:
741 return emitNVVMFPArith(*this, expr, "nvvm.fadd",
742 llvm::APFloat::rmTowardZero);
743 case NVPTX::BI__nvvm_add_rm_f:
744 case NVPTX::BI__nvvm_add_rm_d:
745 return emitNVVMFPArith(*this, expr, "nvvm.fadd",
746 llvm::APFloat::rmTowardNegative);
747 case NVPTX::BI__nvvm_add_rp_f:
748 case NVPTX::BI__nvvm_add_rp_d:
749 return emitNVVMFPArith(*this, expr, "nvvm.fadd",
750 llvm::APFloat::rmTowardPositive);
751 case NVPTX::BI__nvvm_add_rn_ftz_f:
752 return emitNVVMFPArith(*this, expr, "nvvm.fadd.ftz",
753 llvm::APFloat::rmNearestTiesToEven);
754 case NVPTX::BI__nvvm_add_rz_ftz_f:
755 return emitNVVMFPArith(*this, expr, "nvvm.fadd.ftz",
756 llvm::APFloat::rmTowardZero);
757 case NVPTX::BI__nvvm_add_rm_ftz_f:
758 return emitNVVMFPArith(*this, expr, "nvvm.fadd.ftz",
759 llvm::APFloat::rmTowardNegative);
760 case NVPTX::BI__nvvm_add_rp_ftz_f:
761 return emitNVVMFPArith(*this, expr, "nvvm.fadd.ftz",
762 llvm::APFloat::rmTowardPositive);
763 case NVPTX::BI__nvvm_add_rn_sat_f:
764 case NVPTX::BI__nvvm_add_rn_sat_f16:
765 case NVPTX::BI__nvvm_add_rn_sat_v2f16:
766 return emitNVVMFPArith(*this, expr, "nvvm.fadd.sat",
767 llvm::APFloat::rmNearestTiesToEven);
768 case NVPTX::BI__nvvm_add_rz_sat_f:
769 return emitNVVMFPArith(*this, expr, "nvvm.fadd.sat",
770 llvm::APFloat::rmTowardZero);
771 case NVPTX::BI__nvvm_add_rm_sat_f:
772 return emitNVVMFPArith(*this, expr, "nvvm.fadd.sat",
773 llvm::APFloat::rmTowardNegative);
774 case NVPTX::BI__nvvm_add_rp_sat_f:
775 return emitNVVMFPArith(*this, expr, "nvvm.fadd.sat",
776 llvm::APFloat::rmTowardPositive);
777 case NVPTX::BI__nvvm_add_rn_ftz_sat_f:
778 case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
779 case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
780 return emitNVVMFPArith(*this, expr, "nvvm.fadd.ftz.sat",
781 llvm::APFloat::rmNearestTiesToEven);
782 case NVPTX::BI__nvvm_add_rz_ftz_sat_f:
783 return emitNVVMFPArith(*this, expr, "nvvm.fadd.ftz.sat",
784 llvm::APFloat::rmTowardZero);
785 case NVPTX::BI__nvvm_add_rm_ftz_sat_f:
786 return emitNVVMFPArith(*this, expr, "nvvm.fadd.ftz.sat",
787 llvm::APFloat::rmTowardNegative);
788 case NVPTX::BI__nvvm_add_rp_ftz_sat_f:
789 return emitNVVMFPArith(*this, expr, "nvvm.fadd.ftz.sat",
790 llvm::APFloat::rmTowardPositive);
791 case NVPTX::BI__nvvm_mul_rn_f:
792 case NVPTX::BI__nvvm_mul_rn_d:
793 return emitNVVMFPArith(*this, expr, "nvvm.fmul",
794 llvm::APFloat::rmNearestTiesToEven);
795 case NVPTX::BI__nvvm_mul_rz_f:
796 case NVPTX::BI__nvvm_mul_rz_d:
797 return emitNVVMFPArith(*this, expr, "nvvm.fmul",
798 llvm::APFloat::rmTowardZero);
799 case NVPTX::BI__nvvm_mul_rm_f:
800 case NVPTX::BI__nvvm_mul_rm_d:
801 return emitNVVMFPArith(*this, expr, "nvvm.fmul",
802 llvm::APFloat::rmTowardNegative);
803 case NVPTX::BI__nvvm_mul_rp_f:
804 case NVPTX::BI__nvvm_mul_rp_d:
805 return emitNVVMFPArith(*this, expr, "nvvm.fmul",
806 llvm::APFloat::rmTowardPositive);
807 case NVPTX::BI__nvvm_mul_rn_ftz_f:
808 return emitNVVMFPArith(*this, expr, "nvvm.fmul.ftz",
809 llvm::APFloat::rmNearestTiesToEven);
810 case NVPTX::BI__nvvm_mul_rz_ftz_f:
811 return emitNVVMFPArith(*this, expr, "nvvm.fmul.ftz",
812 llvm::APFloat::rmTowardZero);
813 case NVPTX::BI__nvvm_mul_rm_ftz_f:
814 return emitNVVMFPArith(*this, expr, "nvvm.fmul.ftz",
815 llvm::APFloat::rmTowardNegative);
816 case NVPTX::BI__nvvm_mul_rp_ftz_f:
817 return emitNVVMFPArith(*this, expr, "nvvm.fmul.ftz",
818 llvm::APFloat::rmTowardPositive);
819 case NVPTX::BI__nvvm_mul_rn_sat_f16:
820 case NVPTX::BI__nvvm_mul_rn_sat_v2f16:
821 return emitNVVMFPArith(*this, expr, "nvvm.fmul.sat",
822 llvm::APFloat::rmNearestTiesToEven);
823 case NVPTX::BI__nvvm_mul_rn_ftz_sat_f16:
824 case NVPTX::BI__nvvm_mul_rn_ftz_sat_v2f16:
825 return emitNVVMFPArith(*this, expr, "nvvm.fmul.ftz.sat",
826 llvm::APFloat::rmNearestTiesToEven);
827 case NVPTX::BI__nvvm_ldg_h:
828 case NVPTX::BI__nvvm_ldg_h2:
829 cgm.errorNYI(expr->getSourceRange(),
830 std::string("unimplemented NVPTX builtin call: ") +
831 getContext().BuiltinInfo.getName(builtinId));
832 return mlir::Value{};
833 case NVPTX::BI__nvvm_ldu_h:
834 case NVPTX::BI__nvvm_ldu_h2:
835 return makeLdu(*this, expr, "nvvm.ldu.global.f");
836 case NVPTX::BI__nvvm_cp_async_ca_shared_global_4:
837 cgm.errorNYI(expr->getSourceRange(),
838 std::string("unimplemented NVPTX builtin call: ") +
839 getContext().BuiltinInfo.getName(builtinId));
840 return mlir::Value{};
841 case NVPTX::BI__nvvm_cp_async_ca_shared_global_8:
842 cgm.errorNYI(expr->getSourceRange(),
843 std::string("unimplemented NVPTX builtin call: ") +
844 getContext().BuiltinInfo.getName(builtinId));
845 return mlir::Value{};
846 case NVPTX::BI__nvvm_cp_async_ca_shared_global_16:
847 cgm.errorNYI(expr->getSourceRange(),
848 std::string("unimplemented NVPTX builtin call: ") +
849 getContext().BuiltinInfo.getName(builtinId));
850 return mlir::Value{};
851 case NVPTX::BI__nvvm_cp_async_cg_shared_global_16:
852 cgm.errorNYI(expr->getSourceRange(),
853 std::string("unimplemented NVPTX builtin call: ") +
854 getContext().BuiltinInfo.getName(builtinId));
855 return mlir::Value{};
856 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_x:
857 cgm.errorNYI(expr->getSourceRange(),
858 std::string("unimplemented NVPTX builtin call: ") +
859 getContext().BuiltinInfo.getName(builtinId));
860 return mlir::Value{};
861 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_y:
862 cgm.errorNYI(expr->getSourceRange(),
863 std::string("unimplemented NVPTX builtin call: ") +
864 getContext().BuiltinInfo.getName(builtinId));
865 return mlir::Value{};
866 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_z:
867 cgm.errorNYI(expr->getSourceRange(),
868 std::string("unimplemented NVPTX builtin call: ") +
869 getContext().BuiltinInfo.getName(builtinId));
870 return mlir::Value{};
871 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_w:
872 cgm.errorNYI(expr->getSourceRange(),
873 std::string("unimplemented NVPTX builtin call: ") +
874 getContext().BuiltinInfo.getName(builtinId));
875 return mlir::Value{};
876 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_x:
877 cgm.errorNYI(expr->getSourceRange(),
878 std::string("unimplemented NVPTX builtin call: ") +
879 getContext().BuiltinInfo.getName(builtinId));
880 return mlir::Value{};
881 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_y:
882 cgm.errorNYI(expr->getSourceRange(),
883 std::string("unimplemented NVPTX builtin call: ") +
884 getContext().BuiltinInfo.getName(builtinId));
885 return mlir::Value{};
886 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_z:
887 cgm.errorNYI(expr->getSourceRange(),
888 std::string("unimplemented NVPTX builtin call: ") +
889 getContext().BuiltinInfo.getName(builtinId));
890 return mlir::Value{};
891 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_w:
892 cgm.errorNYI(expr->getSourceRange(),
893 std::string("unimplemented NVPTX builtin call: ") +
894 getContext().BuiltinInfo.getName(builtinId));
895 return mlir::Value{};
896 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_x:
897 cgm.errorNYI(expr->getSourceRange(),
898 std::string("unimplemented NVPTX builtin call: ") +
899 getContext().BuiltinInfo.getName(builtinId));
900 return mlir::Value{};
901 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_y:
902 cgm.errorNYI(expr->getSourceRange(),
903 std::string("unimplemented NVPTX builtin call: ") +
904 getContext().BuiltinInfo.getName(builtinId));
905 return mlir::Value{};
906 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_z:
907 cgm.errorNYI(expr->getSourceRange(),
908 std::string("unimplemented NVPTX builtin call: ") +
909 getContext().BuiltinInfo.getName(builtinId));
910 return mlir::Value{};
911 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_w:
912 cgm.errorNYI(expr->getSourceRange(),
913 std::string("unimplemented NVPTX builtin call: ") +
914 getContext().BuiltinInfo.getName(builtinId));
915 return mlir::Value{};
916 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_x:
917 cgm.errorNYI(expr->getSourceRange(),
918 std::string("unimplemented NVPTX builtin call: ") +
919 getContext().BuiltinInfo.getName(builtinId));
920 return mlir::Value{};
921 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_y:
922 cgm.errorNYI(expr->getSourceRange(),
923 std::string("unimplemented NVPTX builtin call: ") +
924 getContext().BuiltinInfo.getName(builtinId));
925 return mlir::Value{};
926 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_z:
927 cgm.errorNYI(expr->getSourceRange(),
928 std::string("unimplemented NVPTX builtin call: ") +
929 getContext().BuiltinInfo.getName(builtinId));
930 return mlir::Value{};
931 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_w:
932 cgm.errorNYI(expr->getSourceRange(),
933 std::string("unimplemented NVPTX builtin call: ") +
934 getContext().BuiltinInfo.getName(builtinId));
935 return mlir::Value{};
936 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctarank:
937 cgm.errorNYI(expr->getSourceRange(),
938 std::string("unimplemented NVPTX builtin call: ") +
939 getContext().BuiltinInfo.getName(builtinId));
940 return mlir::Value{};
941 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctarank:
942 cgm.errorNYI(expr->getSourceRange(),
943 std::string("unimplemented NVPTX builtin call: ") +
944 getContext().BuiltinInfo.getName(builtinId));
945 return mlir::Value{};
946 case NVPTX::BI__nvvm_is_explicit_cluster:
947 cgm.errorNYI(expr->getSourceRange(),
948 std::string("unimplemented NVPTX builtin call: ") +
949 getContext().BuiltinInfo.getName(builtinId));
950 return mlir::Value{};
951 case NVPTX::BI__nvvm_isspacep_shared_cluster:
952 cgm.errorNYI(expr->getSourceRange(),
953 std::string("unimplemented NVPTX builtin call: ") +
954 getContext().BuiltinInfo.getName(builtinId));
955 return mlir::Value{};
956 case NVPTX::BI__nvvm_mapa:
957 cgm.errorNYI(expr->getSourceRange(),
958 std::string("unimplemented NVPTX builtin call: ") +
959 getContext().BuiltinInfo.getName(builtinId));
960 return mlir::Value{};
961 case NVPTX::BI__nvvm_mapa_shared_cluster:
962 cgm.errorNYI(expr->getSourceRange(),
963 std::string("unimplemented NVPTX builtin call: ") +
964 getContext().BuiltinInfo.getName(builtinId));
965 return mlir::Value{};
966 case NVPTX::BI__nvvm_getctarank:
967 cgm.errorNYI(expr->getSourceRange(),
968 std::string("unimplemented NVPTX builtin call: ") +
969 getContext().BuiltinInfo.getName(builtinId));
970 return mlir::Value{};
971 case NVPTX::BI__nvvm_getctarank_shared_cluster:
972 cgm.errorNYI(expr->getSourceRange(),
973 std::string("unimplemented NVPTX builtin call: ") +
974 getContext().BuiltinInfo.getName(builtinId));
975 return mlir::Value{};
976 case NVPTX::BI__nvvm_barrier_cluster_arrive:
977 return builder.emitIntrinsicCallOp(getLoc(expr->getExprLoc()),
978 "nvvm.barrier.cluster.arrive",
979 builder.getVoidTy());
980 case NVPTX::BI__nvvm_barrier_cluster_arrive_relaxed:
981 return builder.emitIntrinsicCallOp(getLoc(expr->getExprLoc()),
982 "nvvm.barrier.cluster.arrive.relaxed",
983 builder.getVoidTy());
984 case NVPTX::BI__nvvm_barrier_cluster_wait:
985 return builder.emitIntrinsicCallOp(getLoc(expr->getExprLoc()),
986 "nvvm.barrier.cluster.wait",
987 builder.getVoidTy());
988 case NVPTX::BI__nvvm_fence_sc_cluster:
989 return builder.emitIntrinsicCallOp(getLoc(expr->getExprLoc()),
990 "nvvm.fence.sc.cluster",
991 builder.getVoidTy());
992 case NVPTX::BI__nvvm_bar_sync:
993 return builder.emitIntrinsicCallOp(
994 getLoc(expr->getExprLoc()), "nvvm.barrier.cta.sync.aligned.all",
995 builder.getVoidTy(), mlir::ValueRange{emitScalarExpr(expr->getArg(0))});
996 case NVPTX::BI__syncthreads:
997 return builder.emitIntrinsicCallOp(
998 getLoc(expr->getExprLoc()), "nvvm.barrier.cta.sync.aligned.all",
999 builder.getVoidTy(),
1000 mlir::ValueRange{builder.getConstInt(getLoc(expr->getExprLoc()),
1001 builder.getSInt32Ty(), 0)});
1002 case NVPTX::BI__nvvm_barrier_sync:
1003 return builder.emitIntrinsicCallOp(
1004 getLoc(expr->getExprLoc()), "nvvm.barrier.cta.sync.all",
1005 builder.getVoidTy(), mlir::ValueRange{emitScalarExpr(expr->getArg(0))});
1006 case NVPTX::BI__nvvm_barrier_sync_cnt:
1007 return builder.emitIntrinsicCallOp(
1008 getLoc(expr->getExprLoc()), "nvvm.barrier.cta.sync.count",
1009 builder.getVoidTy(),
1010 mlir::ValueRange{emitScalarExpr(expr->getArg(0)),
1011 emitScalarExpr(expr->getArg(1))});
1012 case NVPTX::BI__nvvm_bar0_and:
1013 return emitBar0Reduction(*this, expr,
1014 "nvvm.barrier.cta.red.and.aligned.all",
1015 /*returnsPred=*/true);
1016 case NVPTX::BI__nvvm_bar0_or:
1017 return emitBar0Reduction(*this, expr, "nvvm.barrier.cta.red.or.aligned.all",
1018 /*returnsPred=*/true);
1019 case NVPTX::BI__nvvm_bar0_popc:
1020 return emitBar0Reduction(*this, expr,
1021 "nvvm.barrier.cta.red.popc.aligned.all",
1022 /*returnsPred=*/false);
1023
1024 default:
1025 return std::nullopt;
1026 }
1027}
makeScopedAtomicRMW
static mlir::Value makeScopedAtomicRMW(CIRGenFunction &cgf, const CallExpr *expr, cir::AtomicFetchKind kind, cir::SyncScopeKind scope)
Definition CIRGenBuiltinNVPTX.cpp:105
makeLdg
static mlir::Value makeLdg(CIRGenFunction &cgf, const CallExpr *expr)
Definition CIRGenBuiltinNVPTX.cpp:39
emitBar0Reduction
static mlir::Value emitBar0Reduction(CIRGenFunction &cgf, const CallExpr *expr, llvm::StringRef intrinsicName, bool returnsPred)
Definition CIRGenBuiltinNVPTX.cpp:88
emitNVVMFPArith
static mlir::Value emitNVVMFPArith(CIRGenFunction &cgf, const CallExpr *expr, llvm::StringRef intrinsicName, llvm::APFloat::roundingMode rm)
Emit a CIR LLVMIntrinsicCallOp for an NVVM fadd/fmul intrinsic, which takes the rounding mode as a tr...
Definition CIRGenBuiltinNVPTX.cpp:73
makeScopedAtomicXchg
static mlir::Value makeScopedAtomicXchg(CIRGenFunction &cgf, const CallExpr *expr, cir::SyncScopeKind scope)
Definition CIRGenBuiltinNVPTX.cpp:120
makeLdu
static mlir::Value makeLdu(CIRGenFunction &cgf, const CallExpr *expr, llvm::StringRef intrinsicName)
Definition CIRGenBuiltinNVPTX.cpp:23
emitUnaryNVVMIntrinsic
static mlir::Value emitUnaryNVVMIntrinsic(CIRGenFunction &cgf, const CallExpr *expr, llvm::StringRef intrinsicName)
Emit a CIR LLVMIntrinsicCallOp for a unary NVVM intrinsic.
Definition CIRGenBuiltinNVPTX.cpp:60
kind
*collection of selector each with an associated kind and an ordered *collection of selectors A selector has a kind
Definition OpenMPClause.h:10087
TargetBuiltins.h
Enumerates target-specific builtins in their own namespaces within namespace clang.
cir::CIRBaseBuilderTy::getNullValue
cir::ConstantOp getNullValue(mlir::Type ty, mlir::Location loc)
Definition CIRBaseBuilder.h:105
cir::CIRBaseBuilderTy::createBoolToInt
mlir::Value createBoolToInt(mlir::Value src, mlir::Type newTy)
Definition CIRBaseBuilder.h:733
cir::CIRBaseBuilderTy::createCompare
cir::CmpOp createCompare(mlir::Location loc, cir::CmpOpKind kind, mlir::Value lhs, mlir::Value rhs)
Definition CIRBaseBuilder.h:946
cir::CIRBaseBuilderTy::getBoolTy
cir::BoolType getBoolTy()
Definition CIRBaseBuilder.h:159
clang::CIRGen::Address::getPointer
mlir::Value getPointer() const
Definition Address.h:98
clang::CIRGen::Address::getAlignment
clang::CharUnits getAlignment() const
Definition Address.h:138
clang::CIRGen::Address::emitRawPointer
mlir::Value emitRawPointer() const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
Definition Address.h:112
clang::CIRGen::CIRGenBuilderTy::emitIntrinsicCallOp
mlir::Value emitIntrinsicCallOp(mlir::Location loc, const llvm::StringRef str, const mlir::Type &resTy, Operands &&...op)
Definition CIRGenBuilder.h:893
clang::CIRGen::CIRGenFunction::emitPointerWithAlignment
Address emitPointerWithAlignment(const clang::Expr *expr, LValueBaseInfo *baseInfo=nullptr)
Given an expression with a pointer type, emit the value and compute our best estimate of the alignmen...
Definition CIRGenExpr.cpp:127
clang::CIRGen::CIRGenFunction::getLoc
mlir::Location getLoc(clang::SourceLocation srcLoc)
Helpers to convert Clang's SourceLocation to a MLIR Location.
Definition CIRGenFunction.cpp:127
clang::CIRGen::CIRGenFunction::makeBinaryAtomicValue
mlir::Value makeBinaryAtomicValue(cir::AtomicFetchKind kind, const clang::CallExpr *expr, mlir::Type *originalArgType=nullptr, mlir::Value *emittedArgValue=nullptr, cir::MemOrder ordering=cir::MemOrder::SequentiallyConsistent)
Utility to insert an atomic instruction based on Intrinsic::ID and the expression node.
Definition CIRGenBuiltin.cpp:326
clang::CIRGen::CIRGenFunction::convertTypeForMem
mlir::Type convertTypeForMem(QualType t)
Definition CIRGenFunction.cpp:119
clang::CIRGen::CIRGenFunction::emitBuiltinWithOneOverloadedType
RValue emitBuiltinWithOneOverloadedType(const CallExpr *e, llvm::StringRef intrinName, mlir::Type resultType={})
Emit a simple LLVM intrinsic that takes N scalar arguments.
Definition CIRGenFunction.h:2011
clang::CIRGen::CIRGenFunction::emitScalarExpr
mlir::Value emitScalarExpr(const clang::Expr *e, bool ignoreResultAssign=false)
Emit the computation of the specified expression of scalar type.
Definition CIRGenExprScalar.cpp:1906
clang::CIRGen::CIRGenFunction::emitAtomicCmpXchg
mlir::Value emitAtomicCmpXchg(const clang::CallExpr *expr, bool returnBool, cir::MemOrder successOrder=cir::MemOrder::SequentiallyConsistent, cir::MemOrder failureOrder=cir::MemOrder::SequentiallyConsistent, cir::SyncScopeKind scope=cir::SyncScopeKind::System)
Emit cir.atomic.cmpxchg.
Definition CIRGenBuiltin.cpp:410
clang::CIRGen::CIRGenFunction::getBuilder
CIRGenBuilderTy & getBuilder()
Definition CIRGenFunction.h:326
clang::CIRGen::CIRGenFunction::emitNVPTXBuiltinExpr
std::optional< mlir::Value > emitNVPTXBuiltinExpr(unsigned builtinID, const CallExpr *expr)
Emit a call to an NVPTX builtin function.
Definition CIRGenBuiltinNVPTX.cpp:134
clang::CIRGen::CIRGenFunction::getContext
clang::ASTContext & getContext() const
Definition CIRGenFunction.h:324
clang::CIRGen::RValue::getValue
mlir::Value getValue() const
Return the value of this scalar value.
Definition CIRGenValue.h:57
clang::CallExpr
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
Definition Expr.h:2991
clang::CharUnits
This is an opaque type for sizes expressed in character units.
Definition CharUnits.h:38
clang::QualType
A (possibly-)qualified type.
Definition TypeBase.h:938
clang::Type::getPointeeType
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
Definition Type.cpp:885
clang::ast_matchers::expr
const internal::VariadicDynCastAllOfMatcher< Stmt, Expr > expr
Matches expressions.
Definition ASTMatchersInternal.cpp:937
clang
Top level wrappers for InstallAPI frontend operations.
Definition CalledOnceCheck.h:17
Generated on for clang by doxygen 1.14.0