clang 24.0.0git
CIRGenBuiltinNVPTX.cpp
Go to the documentation of this file.
1//===---- CIRGenBuiltinNVPTX.cpp - Emit CIR for NVPTX builtins ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit NVPTX Builtin calls.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CIRGenFunction.h"
14
15#include "mlir/IR/Value.h"
16#include "clang/Basic/TargetBuiltins.h"
17#include "clang/CIR/Dialect/IR/CIRDataLayout.h"
18#include "clang/CIR/Dialect/IR/CIRDialect.h"
19
20using namespace clang;
21using namespace clang::CIRGen;
22
23static mlir::Value makeLdu(CIRGenFunction &cgf, const CallExpr *expr,
24 llvm::StringRef intrinsicName) {
25 auto &builder = cgf.getBuilder();
26 Address ptr = cgf.emitPointerWithAlignment(expr->getArg(0));
27 QualType argType = expr->getArg(0)->getType();
28 mlir::Type elemTy = cgf.convertTypeForMem(argType->getPointeeType());
29 clang::CharUnits align = ptr.getAlignment();
30 mlir::Location loc = cgf.getLoc(expr->getExprLoc());
31 mlir::Value alignVal =
32 builder.getConstantInt(loc, builder.getSInt32Ty(), align.getQuantity());
33 return cir::LLVMIntrinsicCallOp::create(
34 builder, loc, builder.getStringAttr(intrinsicName), elemTy,
35 {ptr.emitRawPointer(), alignVal})
36 .getResult();
37}
38
39/// Emit a CIR LLVMIntrinsicCallOp for a unary NVVM intrinsic.
40/// The result type is inferred from the single argument.
41static mlir::Value emitUnaryNVVMIntrinsic(CIRGenFunction &cgf,
42 const CallExpr *expr,
43 llvm::StringRef intrinsicName) {
44 auto &builder = cgf.getBuilder();
45 mlir::Value arg = cgf.emitScalarExpr(expr->getArg(0));
46 return cir::LLVMIntrinsicCallOp::create(
47 builder, cgf.getLoc(expr->getExprLoc()),
48 builder.getStringAttr(intrinsicName), arg.getType(), {arg})
49 .getResult();
50}
51
52static mlir::Value makeScopedAtomicRMW(CIRGenFunction &cgf,
53 const CallExpr *expr,
54 cir::AtomicFetchKind kind,
55 cir::SyncScopeKind scope) {
56 auto &builder = cgf.getBuilder();
57 Address destAddr = cgf.emitPointerWithAlignment(expr->getArg(0));
58 mlir::Value destValue = destAddr.emitRawPointer();
59 mlir::Value val = cgf.emitScalarExpr(expr->getArg(1));
60 auto rmwi = cir::AtomicFetchOp::create(
61 builder, cgf.getLoc(expr->getSourceRange()), destValue, val, kind,
62 cir::MemOrder::Relaxed, scope, /*is_volatile=*/false,
63 /*fetch_first=*/true);
64 return rmwi->getResult(0);
65}
66
67std::optional<mlir::Value>
68CIRGenFunction::emitNVPTXBuiltinExpr(unsigned builtinId, const CallExpr *expr) {
69 switch (builtinId) {
70 case NVPTX::BI__nvvm_atom_add_gen_i:
71 case NVPTX::BI__nvvm_atom_add_gen_l:
72 case NVPTX::BI__nvvm_atom_add_gen_ll:
73 return makeBinaryAtomicValue(cir::AtomicFetchKind::Add, expr,
74 /*originalArgType=*/nullptr,
75 /*emittedArgValue=*/nullptr,
76 cir::MemOrder::Relaxed);
77 case NVPTX::BI__nvvm_atom_sub_gen_i:
78 case NVPTX::BI__nvvm_atom_sub_gen_l:
79 case NVPTX::BI__nvvm_atom_sub_gen_ll:
80 return makeBinaryAtomicValue(cir::AtomicFetchKind::Sub, expr,
81 /*originalArgType=*/nullptr,
82 /*emittedArgValue=*/nullptr,
83 cir::MemOrder::Relaxed);
84 case NVPTX::BI__nvvm_atom_and_gen_i:
85 case NVPTX::BI__nvvm_atom_and_gen_l:
86 case NVPTX::BI__nvvm_atom_and_gen_ll:
87 return makeBinaryAtomicValue(cir::AtomicFetchKind::And, expr,
88 /*originalArgType=*/nullptr,
89 /*emittedArgValue=*/nullptr,
90 cir::MemOrder::Relaxed);
91 case NVPTX::BI__nvvm_atom_or_gen_i:
92 case NVPTX::BI__nvvm_atom_or_gen_l:
93 case NVPTX::BI__nvvm_atom_or_gen_ll:
94 return makeBinaryAtomicValue(cir::AtomicFetchKind::Or, expr,
95 /*originalArgType=*/nullptr,
96 /*emittedArgValue=*/nullptr,
97 cir::MemOrder::Relaxed);
98 case NVPTX::BI__nvvm_atom_xor_gen_i:
99 case NVPTX::BI__nvvm_atom_xor_gen_l:
100 case NVPTX::BI__nvvm_atom_xor_gen_ll:
101 return makeBinaryAtomicValue(cir::AtomicFetchKind::Xor, expr,
102 /*originalArgType=*/nullptr,
103 /*emittedArgValue=*/nullptr,
104 cir::MemOrder::Relaxed);
105 case NVPTX::BI__nvvm_atom_xchg_gen_i:
106 case NVPTX::BI__nvvm_atom_xchg_gen_l:
107 case NVPTX::BI__nvvm_atom_xchg_gen_ll:
108 cgm.errorNYI(expr->getSourceRange(),
109 std::string("unimplemented NVPTX builtin call: ") +
110 getContext().BuiltinInfo.getName(builtinId));
111 return mlir::Value{};
112 case NVPTX::BI__nvvm_atom_max_gen_i:
113 case NVPTX::BI__nvvm_atom_max_gen_l:
114 case NVPTX::BI__nvvm_atom_max_gen_ll:
115 return makeBinaryAtomicValue(cir::AtomicFetchKind::Max, expr,
116 /*originalArgType=*/nullptr,
117 /*emittedArgValue=*/nullptr,
118 cir::MemOrder::Relaxed);
119 case NVPTX::BI__nvvm_atom_max_gen_ui:
120 case NVPTX::BI__nvvm_atom_max_gen_ul:
121 case NVPTX::BI__nvvm_atom_max_gen_ull:
122 return makeBinaryAtomicValue(cir::AtomicFetchKind::Max, expr,
123 /*originalArgType=*/nullptr,
124 /*emittedArgValue=*/nullptr,
125 cir::MemOrder::Relaxed);
126 case NVPTX::BI__nvvm_atom_min_gen_i:
127 case NVPTX::BI__nvvm_atom_min_gen_l:
128 case NVPTX::BI__nvvm_atom_min_gen_ll:
129 return makeBinaryAtomicValue(cir::AtomicFetchKind::Min, expr,
130 /*originalArgType=*/nullptr,
131 /*emittedArgValue=*/nullptr,
132 cir::MemOrder::Relaxed);
133 case NVPTX::BI__nvvm_atom_min_gen_ui:
134 case NVPTX::BI__nvvm_atom_min_gen_ul:
135 case NVPTX::BI__nvvm_atom_min_gen_ull:
136 return makeBinaryAtomicValue(cir::AtomicFetchKind::Min, expr,
137 /*originalArgType=*/nullptr,
138 /*emittedArgValue=*/nullptr,
139 cir::MemOrder::Relaxed);
140 case NVPTX::BI__nvvm_atom_cas_gen_us:
141 case NVPTX::BI__nvvm_atom_cas_gen_i:
142 case NVPTX::BI__nvvm_atom_cas_gen_l:
143 case NVPTX::BI__nvvm_atom_cas_gen_ll:
144 cgm.errorNYI(expr->getSourceRange(),
145 std::string("unimplemented NVPTX builtin call: ") +
146 getContext().BuiltinInfo.getName(builtinId));
147 return mlir::Value{};
148 // success flag.
149 case NVPTX::BI__nvvm_atom_add_gen_f:
150 case NVPTX::BI__nvvm_atom_add_gen_d:
151 cgm.errorNYI(expr->getSourceRange(),
152 std::string("unimplemented NVPTX builtin call: ") +
153 getContext().BuiltinInfo.getName(builtinId));
154 return mlir::Value{};
155 case NVPTX::BI__nvvm_atom_inc_gen_ui:
156 return makeBinaryAtomicValue(cir::AtomicFetchKind::UIncWrap, expr,
157 /*originalArgType=*/nullptr,
158 /*emittedArgValue=*/nullptr,
159 cir::MemOrder::Relaxed);
160 case NVPTX::BI__nvvm_atom_dec_gen_ui:
161 return makeBinaryAtomicValue(cir::AtomicFetchKind::UDecWrap, expr,
162 /*originalArgType=*/nullptr,
163 /*emittedArgValue=*/nullptr,
164 cir::MemOrder::Relaxed);
165 case NVPTX::BI__nvvm_ldg_c:
166 case NVPTX::BI__nvvm_ldg_sc:
167 case NVPTX::BI__nvvm_ldg_c2:
168 case NVPTX::BI__nvvm_ldg_sc2:
169 case NVPTX::BI__nvvm_ldg_c4:
170 case NVPTX::BI__nvvm_ldg_sc4:
171 case NVPTX::BI__nvvm_ldg_s:
172 case NVPTX::BI__nvvm_ldg_s2:
173 case NVPTX::BI__nvvm_ldg_s4:
174 case NVPTX::BI__nvvm_ldg_i:
175 case NVPTX::BI__nvvm_ldg_i2:
176 case NVPTX::BI__nvvm_ldg_i4:
177 case NVPTX::BI__nvvm_ldg_l:
178 case NVPTX::BI__nvvm_ldg_l2:
179 case NVPTX::BI__nvvm_ldg_ll:
180 case NVPTX::BI__nvvm_ldg_ll2:
181 case NVPTX::BI__nvvm_ldg_uc:
182 case NVPTX::BI__nvvm_ldg_uc2:
183 case NVPTX::BI__nvvm_ldg_uc4:
184 case NVPTX::BI__nvvm_ldg_us:
185 case NVPTX::BI__nvvm_ldg_us2:
186 case NVPTX::BI__nvvm_ldg_us4:
187 case NVPTX::BI__nvvm_ldg_ui:
188 case NVPTX::BI__nvvm_ldg_ui2:
189 case NVPTX::BI__nvvm_ldg_ui4:
190 case NVPTX::BI__nvvm_ldg_ul:
191 case NVPTX::BI__nvvm_ldg_ul2:
192 case NVPTX::BI__nvvm_ldg_ull:
193 case NVPTX::BI__nvvm_ldg_ull2:
194 case NVPTX::BI__nvvm_ldg_f:
195 case NVPTX::BI__nvvm_ldg_f2:
196 case NVPTX::BI__nvvm_ldg_f4:
197 case NVPTX::BI__nvvm_ldg_d:
198 case NVPTX::BI__nvvm_ldg_d2:
199 cgm.errorNYI(expr->getSourceRange(),
200 std::string("unimplemented NVPTX builtin call: ") +
201 getContext().BuiltinInfo.getName(builtinId));
202 return mlir::Value{};
203 case NVPTX::BI__nvvm_ldu_c:
204 case NVPTX::BI__nvvm_ldu_sc:
205 case NVPTX::BI__nvvm_ldu_c2:
206 case NVPTX::BI__nvvm_ldu_sc2:
207 case NVPTX::BI__nvvm_ldu_c4:
208 case NVPTX::BI__nvvm_ldu_sc4:
209 case NVPTX::BI__nvvm_ldu_s:
210 case NVPTX::BI__nvvm_ldu_s2:
211 case NVPTX::BI__nvvm_ldu_s4:
212 case NVPTX::BI__nvvm_ldu_i:
213 case NVPTX::BI__nvvm_ldu_i2:
214 case NVPTX::BI__nvvm_ldu_i4:
215 case NVPTX::BI__nvvm_ldu_l:
216 case NVPTX::BI__nvvm_ldu_l2:
217 case NVPTX::BI__nvvm_ldu_ll:
218 case NVPTX::BI__nvvm_ldu_ll2:
219 case NVPTX::BI__nvvm_ldu_uc:
220 case NVPTX::BI__nvvm_ldu_uc2:
221 case NVPTX::BI__nvvm_ldu_uc4:
222 case NVPTX::BI__nvvm_ldu_us:
223 case NVPTX::BI__nvvm_ldu_us2:
224 case NVPTX::BI__nvvm_ldu_us4:
225 case NVPTX::BI__nvvm_ldu_ui:
226 case NVPTX::BI__nvvm_ldu_ui2:
227 case NVPTX::BI__nvvm_ldu_ui4:
228 case NVPTX::BI__nvvm_ldu_ul:
229 case NVPTX::BI__nvvm_ldu_ul2:
230 case NVPTX::BI__nvvm_ldu_ull:
231 case NVPTX::BI__nvvm_ldu_ull2:
232 return makeLdu(*this, expr, "nvvm.ldu.global.i");
233 case NVPTX::BI__nvvm_ldu_f:
234 case NVPTX::BI__nvvm_ldu_f2:
235 case NVPTX::BI__nvvm_ldu_f4:
236 case NVPTX::BI__nvvm_ldu_d:
237 case NVPTX::BI__nvvm_ldu_d2:
238 return makeLdu(*this, expr, "nvvm.ldu.global.f");
239 case NVPTX::BI__nvvm_atom_cta_add_gen_i:
240 case NVPTX::BI__nvvm_atom_cta_add_gen_l:
241 case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
242 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Add,
243 cir::SyncScopeKind::Workgroup);
244 case NVPTX::BI__nvvm_atom_sys_add_gen_i:
245 case NVPTX::BI__nvvm_atom_sys_add_gen_l:
246 case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
247 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Add,
248 cir::SyncScopeKind::System);
249 case NVPTX::BI__nvvm_atom_cta_add_gen_f:
250 case NVPTX::BI__nvvm_atom_cta_add_gen_d:
251 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Add,
252 cir::SyncScopeKind::Workgroup);
253 case NVPTX::BI__nvvm_atom_sys_add_gen_f:
254 case NVPTX::BI__nvvm_atom_sys_add_gen_d:
255 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Add,
256 cir::SyncScopeKind::System);
257 case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
258 case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
259 case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
260 cgm.errorNYI(expr->getSourceRange(),
261 std::string("unimplemented NVPTX builtin call: ") +
262 getContext().BuiltinInfo.getName(builtinId));
263 return mlir::Value{};
264 case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
265 case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
266 case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
267 cgm.errorNYI(expr->getSourceRange(),
268 std::string("unimplemented NVPTX builtin call: ") +
269 getContext().BuiltinInfo.getName(builtinId));
270 return mlir::Value{};
271 case NVPTX::BI__nvvm_atom_cta_max_gen_i:
272 case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
273 case NVPTX::BI__nvvm_atom_cta_max_gen_l:
274 case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
275 case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
276 case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
277 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Max,
278 cir::SyncScopeKind::Workgroup);
279 case NVPTX::BI__nvvm_atom_sys_max_gen_i:
280 case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
281 case NVPTX::BI__nvvm_atom_sys_max_gen_l:
282 case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
283 case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
284 case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
285 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Max,
286 cir::SyncScopeKind::System);
287 case NVPTX::BI__nvvm_atom_cta_min_gen_i:
288 case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
289 case NVPTX::BI__nvvm_atom_cta_min_gen_l:
290 case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
291 case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
292 case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
293 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Min,
294 cir::SyncScopeKind::Workgroup);
295 case NVPTX::BI__nvvm_atom_sys_min_gen_i:
296 case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
297 case NVPTX::BI__nvvm_atom_sys_min_gen_l:
298 case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
299 case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
300 case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
301 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Min,
302 cir::SyncScopeKind::System);
303 case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
304 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::UIncWrap,
305 cir::SyncScopeKind::Workgroup);
306 case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
307 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::UDecWrap,
308 cir::SyncScopeKind::Workgroup);
309 case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
310 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::UIncWrap,
311 cir::SyncScopeKind::System);
312 case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
313 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::UDecWrap,
314 cir::SyncScopeKind::System);
315 case NVPTX::BI__nvvm_atom_cta_and_gen_i:
316 case NVPTX::BI__nvvm_atom_cta_and_gen_l:
317 case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
318 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::And,
319 cir::SyncScopeKind::Workgroup);
320 case NVPTX::BI__nvvm_atom_sys_and_gen_i:
321 case NVPTX::BI__nvvm_atom_sys_and_gen_l:
322 case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
323 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::And,
324 cir::SyncScopeKind::System);
325 case NVPTX::BI__nvvm_atom_cta_or_gen_i:
326 case NVPTX::BI__nvvm_atom_cta_or_gen_l:
327 case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
328 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Or,
329 cir::SyncScopeKind::Workgroup);
330 case NVPTX::BI__nvvm_atom_sys_or_gen_i:
331 case NVPTX::BI__nvvm_atom_sys_or_gen_l:
332 case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
333 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Or,
334 cir::SyncScopeKind::System);
335 case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
336 case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
337 case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
338 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Xor,
339 cir::SyncScopeKind::Workgroup);
340 case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
341 case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
342 case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
343 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Xor,
344 cir::SyncScopeKind::System);
345 case NVPTX::BI__nvvm_atom_cta_cas_gen_us:
346 case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
347 case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
348 case NVPTX::BI__nvvm_atom_cta_cas_gen_ll:
349 cgm.errorNYI(expr->getSourceRange(),
350 std::string("unimplemented NVPTX builtin call: ") +
351 getContext().BuiltinInfo.getName(builtinId));
352 return mlir::Value{};
353 case NVPTX::BI__nvvm_atom_sys_cas_gen_us:
354 case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
355 case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
356 case NVPTX::BI__nvvm_atom_sys_cas_gen_ll:
357 cgm.errorNYI(expr->getSourceRange(),
358 std::string("unimplemented NVPTX builtin call: ") +
359 getContext().BuiltinInfo.getName(builtinId));
360 return mlir::Value{};
361 case NVPTX::BI__nvvm_match_all_sync_i32p:
362 case NVPTX::BI__nvvm_match_all_sync_i64p:
363 cgm.errorNYI(expr->getSourceRange(),
364 std::string("unimplemented NVPTX builtin call: ") +
365 getContext().BuiltinInfo.getName(builtinId));
366 return mlir::Value{};
367 // FP MMA loads
368 case NVPTX::BI__hmma_m16n16k16_ld_a:
369 case NVPTX::BI__hmma_m16n16k16_ld_b:
370 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
371 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
372 case NVPTX::BI__hmma_m32n8k16_ld_a:
373 case NVPTX::BI__hmma_m32n8k16_ld_b:
374 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
375 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
376 case NVPTX::BI__hmma_m8n32k16_ld_a:
377 case NVPTX::BI__hmma_m8n32k16_ld_b:
378 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
379 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
380 cgm.errorNYI(expr->getSourceRange(),
381 std::string("unimplemented NVPTX builtin call: ") +
382 getContext().BuiltinInfo.getName(builtinId));
383 return mlir::Value{};
384 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
385 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
386 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
387 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
388 case NVPTX::BI__imma_m16n16k16_ld_c:
389 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
390 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
391 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
392 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
393 case NVPTX::BI__imma_m32n8k16_ld_c:
394 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
395 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
396 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
397 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
398 case NVPTX::BI__imma_m8n32k16_ld_c:
399 cgm.errorNYI(expr->getSourceRange(),
400 std::string("unimplemented NVPTX builtin call: ") +
401 getContext().BuiltinInfo.getName(builtinId));
402 return mlir::Value{};
403 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
404 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
405 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
406 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
407 case NVPTX::BI__imma_m8n8k32_ld_c:
408 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
409 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
410 case NVPTX::BI__bmma_m8n8k128_ld_c:
411 cgm.errorNYI(expr->getSourceRange(),
412 std::string("unimplemented NVPTX builtin call: ") +
413 getContext().BuiltinInfo.getName(builtinId));
414 return mlir::Value{};
415 case NVPTX::BI__dmma_m8n8k4_ld_a:
416 case NVPTX::BI__dmma_m8n8k4_ld_b:
417 case NVPTX::BI__dmma_m8n8k4_ld_c:
418 cgm.errorNYI(expr->getSourceRange(),
419 std::string("unimplemented NVPTX builtin call: ") +
420 getContext().BuiltinInfo.getName(builtinId));
421 return mlir::Value{};
422 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
423 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
424 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
425 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
426 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
427 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
428 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
429 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
430 case NVPTX::BI__mma_tf32_m16n16k8_ld_c:
431 cgm.errorNYI(expr->getSourceRange(),
432 std::string("unimplemented NVPTX builtin call: ") +
433 getContext().BuiltinInfo.getName(builtinId));
434 return mlir::Value{};
435 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
436 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
437 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
438 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
439 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
440 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
441 case NVPTX::BI__imma_m16n16k16_st_c_i32:
442 case NVPTX::BI__imma_m32n8k16_st_c_i32:
443 case NVPTX::BI__imma_m8n32k16_st_c_i32:
444 case NVPTX::BI__imma_m8n8k32_st_c_i32:
445 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
446 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
447 case NVPTX::BI__mma_m16n16k8_st_c_f32:
448 cgm.errorNYI(expr->getSourceRange(),
449 std::string("unimplemented NVPTX builtin call: ") +
450 getContext().BuiltinInfo.getName(builtinId));
451 return mlir::Value{};
452 // BI__hmma_m16n16k16_mma_<Dtype><CType>(d, a, b, c, layout, satf) -->
453 // Intrinsic::nvvm_wmma_m16n16k16_mma_sync<layout A,B><DType><CType><Satf>
454 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
455 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
456 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
457 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
458 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
459 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
460 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
461 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
462 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
463 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
464 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
465 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
466 case NVPTX::BI__imma_m16n16k16_mma_s8:
467 case NVPTX::BI__imma_m16n16k16_mma_u8:
468 case NVPTX::BI__imma_m32n8k16_mma_s8:
469 case NVPTX::BI__imma_m32n8k16_mma_u8:
470 case NVPTX::BI__imma_m8n32k16_mma_s8:
471 case NVPTX::BI__imma_m8n32k16_mma_u8:
472 case NVPTX::BI__imma_m8n8k32_mma_s4:
473 case NVPTX::BI__imma_m8n8k32_mma_u4:
474 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
475 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
476 case NVPTX::BI__dmma_m8n8k4_mma_f64:
477 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
478 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
479 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
480 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32:
481 cgm.errorNYI(expr->getSourceRange(),
482 std::string("unimplemented NVPTX builtin call: ") +
483 getContext().BuiltinInfo.getName(builtinId));
484 return mlir::Value{};
485 // The following builtins require half type support
486 case NVPTX::BI__nvvm_ex2_approx_f16:
487 cgm.errorNYI(expr->getSourceRange(),
488 std::string("unimplemented NVPTX builtin call: ") +
489 getContext().BuiltinInfo.getName(builtinId));
490 return mlir::Value{};
491 case NVPTX::BI__nvvm_ex2_approx_f16x2:
492 cgm.errorNYI(expr->getSourceRange(),
493 std::string("unimplemented NVPTX builtin call: ") +
494 getContext().BuiltinInfo.getName(builtinId));
495 return mlir::Value{};
496 case NVPTX::BI__nvvm_ff2f16x2_rn:
497 cgm.errorNYI(expr->getSourceRange(),
498 std::string("unimplemented NVPTX builtin call: ") +
499 getContext().BuiltinInfo.getName(builtinId));
500 return mlir::Value{};
501 case NVPTX::BI__nvvm_ff2f16x2_rn_relu:
502 cgm.errorNYI(expr->getSourceRange(),
503 std::string("unimplemented NVPTX builtin call: ") +
504 getContext().BuiltinInfo.getName(builtinId));
505 return mlir::Value{};
506 case NVPTX::BI__nvvm_ff2f16x2_rz:
507 cgm.errorNYI(expr->getSourceRange(),
508 std::string("unimplemented NVPTX builtin call: ") +
509 getContext().BuiltinInfo.getName(builtinId));
510 return mlir::Value{};
511 case NVPTX::BI__nvvm_ff2f16x2_rz_relu:
512 cgm.errorNYI(expr->getSourceRange(),
513 std::string("unimplemented NVPTX builtin call: ") +
514 getContext().BuiltinInfo.getName(builtinId));
515 return mlir::Value{};
516 case NVPTX::BI__nvvm_fma_rn_f16:
517 cgm.errorNYI(expr->getSourceRange(),
518 std::string("unimplemented NVPTX builtin call: ") +
519 getContext().BuiltinInfo.getName(builtinId));
520 return mlir::Value{};
521 case NVPTX::BI__nvvm_fma_rn_f16x2:
522 cgm.errorNYI(expr->getSourceRange(),
523 std::string("unimplemented NVPTX builtin call: ") +
524 getContext().BuiltinInfo.getName(builtinId));
525 return mlir::Value{};
526 case NVPTX::BI__nvvm_fma_rn_ftz_f16:
527 cgm.errorNYI(expr->getSourceRange(),
528 std::string("unimplemented NVPTX builtin call: ") +
529 getContext().BuiltinInfo.getName(builtinId));
530 return mlir::Value{};
531 case NVPTX::BI__nvvm_fma_rn_ftz_f16x2:
532 cgm.errorNYI(expr->getSourceRange(),
533 std::string("unimplemented NVPTX builtin call: ") +
534 getContext().BuiltinInfo.getName(builtinId));
535 return mlir::Value{};
536 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16:
537 cgm.errorNYI(expr->getSourceRange(),
538 std::string("unimplemented NVPTX builtin call: ") +
539 getContext().BuiltinInfo.getName(builtinId));
540 return mlir::Value{};
541 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16x2:
542 cgm.errorNYI(expr->getSourceRange(),
543 std::string("unimplemented NVPTX builtin call: ") +
544 getContext().BuiltinInfo.getName(builtinId));
545 return mlir::Value{};
546 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16:
547 cgm.errorNYI(expr->getSourceRange(),
548 std::string("unimplemented NVPTX builtin call: ") +
549 getContext().BuiltinInfo.getName(builtinId));
550 return mlir::Value{};
551 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16x2:
552 cgm.errorNYI(expr->getSourceRange(),
553 std::string("unimplemented NVPTX builtin call: ") +
554 getContext().BuiltinInfo.getName(builtinId));
555 return mlir::Value{};
556 case NVPTX::BI__nvvm_fma_rn_relu_f16:
557 cgm.errorNYI(expr->getSourceRange(),
558 std::string("unimplemented NVPTX builtin call: ") +
559 getContext().BuiltinInfo.getName(builtinId));
560 return mlir::Value{};
561 case NVPTX::BI__nvvm_fma_rn_relu_f16x2:
562 cgm.errorNYI(expr->getSourceRange(),
563 std::string("unimplemented NVPTX builtin call: ") +
564 getContext().BuiltinInfo.getName(builtinId));
565 return mlir::Value{};
566 case NVPTX::BI__nvvm_fma_rn_sat_f16:
567 cgm.errorNYI(expr->getSourceRange(),
568 std::string("unimplemented NVPTX builtin call: ") +
569 getContext().BuiltinInfo.getName(builtinId));
570 return mlir::Value{};
571 case NVPTX::BI__nvvm_fma_rn_sat_f16x2:
572 cgm.errorNYI(expr->getSourceRange(),
573 std::string("unimplemented NVPTX builtin call: ") +
574 getContext().BuiltinInfo.getName(builtinId));
575 return mlir::Value{};
576 case NVPTX::BI__nvvm_fma_rn_oob_f16:
577 cgm.errorNYI(expr->getSourceRange(),
578 std::string("unimplemented NVPTX builtin call: ") +
579 getContext().BuiltinInfo.getName(builtinId));
580 return mlir::Value{};
581 case NVPTX::BI__nvvm_fma_rn_oob_f16x2:
582 cgm.errorNYI(expr->getSourceRange(),
583 std::string("unimplemented NVPTX builtin call: ") +
584 getContext().BuiltinInfo.getName(builtinId));
585 return mlir::Value{};
586 case NVPTX::BI__nvvm_fma_rn_oob_bf16:
587 cgm.errorNYI(expr->getSourceRange(),
588 std::string("unimplemented NVPTX builtin call: ") +
589 getContext().BuiltinInfo.getName(builtinId));
590 return mlir::Value{};
591 case NVPTX::BI__nvvm_fma_rn_oob_bf16x2:
592 cgm.errorNYI(expr->getSourceRange(),
593 std::string("unimplemented NVPTX builtin call: ") +
594 getContext().BuiltinInfo.getName(builtinId));
595 return mlir::Value{};
596 case NVPTX::BI__nvvm_fma_rn_oob_relu_f16:
597 cgm.errorNYI(expr->getSourceRange(),
598 std::string("unimplemented NVPTX builtin call: ") +
599 getContext().BuiltinInfo.getName(builtinId));
600 return mlir::Value{};
601 case NVPTX::BI__nvvm_fma_rn_oob_relu_f16x2:
602 cgm.errorNYI(expr->getSourceRange(),
603 std::string("unimplemented NVPTX builtin call: ") +
604 getContext().BuiltinInfo.getName(builtinId));
605 return mlir::Value{};
606 case NVPTX::BI__nvvm_fma_rn_oob_relu_bf16:
607 cgm.errorNYI(expr->getSourceRange(),
608 std::string("unimplemented NVPTX builtin call: ") +
609 getContext().BuiltinInfo.getName(builtinId));
610 return mlir::Value{};
611 case NVPTX::BI__nvvm_fma_rn_oob_relu_bf16x2:
612 cgm.errorNYI(expr->getSourceRange(),
613 std::string("unimplemented NVPTX builtin call: ") +
614 getContext().BuiltinInfo.getName(builtinId));
615 return mlir::Value{};
616 case NVPTX::BI__nvvm_fmax_f16:
617 cgm.errorNYI(expr->getSourceRange(),
618 std::string("unimplemented NVPTX builtin call: ") +
619 getContext().BuiltinInfo.getName(builtinId));
620 return mlir::Value{};
621 case NVPTX::BI__nvvm_fmax_f16x2:
622 cgm.errorNYI(expr->getSourceRange(),
623 std::string("unimplemented NVPTX builtin call: ") +
624 getContext().BuiltinInfo.getName(builtinId));
625 return mlir::Value{};
626 case NVPTX::BI__nvvm_fmax_ftz_f16:
627 cgm.errorNYI(expr->getSourceRange(),
628 std::string("unimplemented NVPTX builtin call: ") +
629 getContext().BuiltinInfo.getName(builtinId));
630 return mlir::Value{};
631 case NVPTX::BI__nvvm_fmax_ftz_f16x2:
632 cgm.errorNYI(expr->getSourceRange(),
633 std::string("unimplemented NVPTX builtin call: ") +
634 getContext().BuiltinInfo.getName(builtinId));
635 return mlir::Value{};
636 case NVPTX::BI__nvvm_fmax_ftz_nan_f16:
637 cgm.errorNYI(expr->getSourceRange(),
638 std::string("unimplemented NVPTX builtin call: ") +
639 getContext().BuiltinInfo.getName(builtinId));
640 return mlir::Value{};
641 case NVPTX::BI__nvvm_fmax_ftz_nan_f16x2:
642 cgm.errorNYI(expr->getSourceRange(),
643 std::string("unimplemented NVPTX builtin call: ") +
644 getContext().BuiltinInfo.getName(builtinId));
645 return mlir::Value{};
646 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16:
647 cgm.errorNYI(expr->getSourceRange(),
648 std::string("unimplemented NVPTX builtin call: ") +
649 getContext().BuiltinInfo.getName(builtinId));
650 return mlir::Value{};
651 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16x2:
652 cgm.errorNYI(expr->getSourceRange(),
653 std::string("unimplemented NVPTX builtin call: ") +
654 getContext().BuiltinInfo.getName(builtinId));
655 return mlir::Value{};
656 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16:
657 cgm.errorNYI(expr->getSourceRange(),
658 std::string("unimplemented NVPTX builtin call: ") +
659 getContext().BuiltinInfo.getName(builtinId));
660 return mlir::Value{};
661 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16x2:
662 cgm.errorNYI(expr->getSourceRange(),
663 std::string("unimplemented NVPTX builtin call: ") +
664 getContext().BuiltinInfo.getName(builtinId));
665 return mlir::Value{};
666 case NVPTX::BI__nvvm_fmax_nan_f16:
667 cgm.errorNYI(expr->getSourceRange(),
668 std::string("unimplemented NVPTX builtin call: ") +
669 getContext().BuiltinInfo.getName(builtinId));
670 return mlir::Value{};
671 case NVPTX::BI__nvvm_fmax_nan_f16x2:
672 cgm.errorNYI(expr->getSourceRange(),
673 std::string("unimplemented NVPTX builtin call: ") +
674 getContext().BuiltinInfo.getName(builtinId));
675 return mlir::Value{};
676 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16:
677 cgm.errorNYI(expr->getSourceRange(),
678 std::string("unimplemented NVPTX builtin call: ") +
679 getContext().BuiltinInfo.getName(builtinId));
680 return mlir::Value{};
681 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16x2:
682 cgm.errorNYI(expr->getSourceRange(),
683 std::string("unimplemented NVPTX builtin call: ") +
684 getContext().BuiltinInfo.getName(builtinId));
685 return mlir::Value{};
686 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16:
687 cgm.errorNYI(expr->getSourceRange(),
688 std::string("unimplemented NVPTX builtin call: ") +
689 getContext().BuiltinInfo.getName(builtinId));
690 return mlir::Value{};
691 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16x2:
692 cgm.errorNYI(expr->getSourceRange(),
693 std::string("unimplemented NVPTX builtin call: ") +
694 getContext().BuiltinInfo.getName(builtinId));
695 return mlir::Value{};
696 case NVPTX::BI__nvvm_fmin_f16:
697 cgm.errorNYI(expr->getSourceRange(),
698 std::string("unimplemented NVPTX builtin call: ") +
699 getContext().BuiltinInfo.getName(builtinId));
700 return mlir::Value{};
701 case NVPTX::BI__nvvm_fmin_f16x2:
702 cgm.errorNYI(expr->getSourceRange(),
703 std::string("unimplemented NVPTX builtin call: ") +
704 getContext().BuiltinInfo.getName(builtinId));
705 return mlir::Value{};
706 case NVPTX::BI__nvvm_fmin_ftz_f16:
707 cgm.errorNYI(expr->getSourceRange(),
708 std::string("unimplemented NVPTX builtin call: ") +
709 getContext().BuiltinInfo.getName(builtinId));
710 return mlir::Value{};
711 case NVPTX::BI__nvvm_fmin_ftz_f16x2:
712 cgm.errorNYI(expr->getSourceRange(),
713 std::string("unimplemented NVPTX builtin call: ") +
714 getContext().BuiltinInfo.getName(builtinId));
715 return mlir::Value{};
716 case NVPTX::BI__nvvm_fmin_ftz_nan_f16:
717 cgm.errorNYI(expr->getSourceRange(),
718 std::string("unimplemented NVPTX builtin call: ") +
719 getContext().BuiltinInfo.getName(builtinId));
720 return mlir::Value{};
721 case NVPTX::BI__nvvm_fmin_ftz_nan_f16x2:
722 cgm.errorNYI(expr->getSourceRange(),
723 std::string("unimplemented NVPTX builtin call: ") +
724 getContext().BuiltinInfo.getName(builtinId));
725 return mlir::Value{};
726 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16:
727 cgm.errorNYI(expr->getSourceRange(),
728 std::string("unimplemented NVPTX builtin call: ") +
729 getContext().BuiltinInfo.getName(builtinId));
730 return mlir::Value{};
731 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16x2:
732 cgm.errorNYI(expr->getSourceRange(),
733 std::string("unimplemented NVPTX builtin call: ") +
734 getContext().BuiltinInfo.getName(builtinId));
735 return mlir::Value{};
736 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16:
737 cgm.errorNYI(expr->getSourceRange(),
738 std::string("unimplemented NVPTX builtin call: ") +
739 getContext().BuiltinInfo.getName(builtinId));
740 return mlir::Value{};
741 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16x2:
742 cgm.errorNYI(expr->getSourceRange(),
743 std::string("unimplemented NVPTX builtin call: ") +
744 getContext().BuiltinInfo.getName(builtinId));
745 return mlir::Value{};
746 case NVPTX::BI__nvvm_fmin_nan_f16:
747 cgm.errorNYI(expr->getSourceRange(),
748 std::string("unimplemented NVPTX builtin call: ") +
749 getContext().BuiltinInfo.getName(builtinId));
750 return mlir::Value{};
751 case NVPTX::BI__nvvm_fmin_nan_f16x2:
752 cgm.errorNYI(expr->getSourceRange(),
753 std::string("unimplemented NVPTX builtin call: ") +
754 getContext().BuiltinInfo.getName(builtinId));
755 return mlir::Value{};
756 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16:
757 cgm.errorNYI(expr->getSourceRange(),
758 std::string("unimplemented NVPTX builtin call: ") +
759 getContext().BuiltinInfo.getName(builtinId));
760 return mlir::Value{};
761 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16x2:
762 cgm.errorNYI(expr->getSourceRange(),
763 std::string("unimplemented NVPTX builtin call: ") +
764 getContext().BuiltinInfo.getName(builtinId));
765 return mlir::Value{};
766 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16:
767 cgm.errorNYI(expr->getSourceRange(),
768 std::string("unimplemented NVPTX builtin call: ") +
769 getContext().BuiltinInfo.getName(builtinId));
770 return mlir::Value{};
771 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16x2:
772 cgm.errorNYI(expr->getSourceRange(),
773 std::string("unimplemented NVPTX builtin call: ") +
774 getContext().BuiltinInfo.getName(builtinId));
775 return mlir::Value{};
776 case NVPTX::BI__nvvm_fabs_f:
777 case NVPTX::BI__nvvm_abs_bf16:
778 case NVPTX::BI__nvvm_abs_bf16x2:
779 case NVPTX::BI__nvvm_fabs_f16:
780 case NVPTX::BI__nvvm_fabs_f16x2:
781 return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.fabs");
782 case NVPTX::BI__nvvm_fabs_ftz_f:
783 case NVPTX::BI__nvvm_fabs_ftz_f16:
784 case NVPTX::BI__nvvm_fabs_ftz_f16x2:
785 return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.fabs.ftz");
786 case NVPTX::BI__nvvm_fabs_d:
787 return emitUnaryNVVMIntrinsic(*this, expr, "fabs");
788 case NVPTX::BI__nvvm_ex2_approx_d:
789 case NVPTX::BI__nvvm_ex2_approx_f:
790 return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.ex2.approx");
791 case NVPTX::BI__nvvm_ex2_approx_ftz_f:
792 return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.ex2.approx.ftz");
793 case NVPTX::BI__nvvm_ldg_h:
794 case NVPTX::BI__nvvm_ldg_h2:
795 cgm.errorNYI(expr->getSourceRange(),
796 std::string("unimplemented NVPTX builtin call: ") +
797 getContext().BuiltinInfo.getName(builtinId));
798 return mlir::Value{};
799 case NVPTX::BI__nvvm_ldu_h:
800 case NVPTX::BI__nvvm_ldu_h2:
801 return makeLdu(*this, expr, "nvvm.ldu.global.f");
802 case NVPTX::BI__nvvm_cp_async_ca_shared_global_4:
803 cgm.errorNYI(expr->getSourceRange(),
804 std::string("unimplemented NVPTX builtin call: ") +
805 getContext().BuiltinInfo.getName(builtinId));
806 return mlir::Value{};
807 case NVPTX::BI__nvvm_cp_async_ca_shared_global_8:
808 cgm.errorNYI(expr->getSourceRange(),
809 std::string("unimplemented NVPTX builtin call: ") +
810 getContext().BuiltinInfo.getName(builtinId));
811 return mlir::Value{};
812 case NVPTX::BI__nvvm_cp_async_ca_shared_global_16:
813 cgm.errorNYI(expr->getSourceRange(),
814 std::string("unimplemented NVPTX builtin call: ") +
815 getContext().BuiltinInfo.getName(builtinId));
816 return mlir::Value{};
817 case NVPTX::BI__nvvm_cp_async_cg_shared_global_16:
818 cgm.errorNYI(expr->getSourceRange(),
819 std::string("unimplemented NVPTX builtin call: ") +
820 getContext().BuiltinInfo.getName(builtinId));
821 return mlir::Value{};
822 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_x:
823 cgm.errorNYI(expr->getSourceRange(),
824 std::string("unimplemented NVPTX builtin call: ") +
825 getContext().BuiltinInfo.getName(builtinId));
826 return mlir::Value{};
827 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_y:
828 cgm.errorNYI(expr->getSourceRange(),
829 std::string("unimplemented NVPTX builtin call: ") +
830 getContext().BuiltinInfo.getName(builtinId));
831 return mlir::Value{};
832 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_z:
833 cgm.errorNYI(expr->getSourceRange(),
834 std::string("unimplemented NVPTX builtin call: ") +
835 getContext().BuiltinInfo.getName(builtinId));
836 return mlir::Value{};
837 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_w:
838 cgm.errorNYI(expr->getSourceRange(),
839 std::string("unimplemented NVPTX builtin call: ") +
840 getContext().BuiltinInfo.getName(builtinId));
841 return mlir::Value{};
842 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_x:
843 cgm.errorNYI(expr->getSourceRange(),
844 std::string("unimplemented NVPTX builtin call: ") +
845 getContext().BuiltinInfo.getName(builtinId));
846 return mlir::Value{};
847 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_y:
848 cgm.errorNYI(expr->getSourceRange(),
849 std::string("unimplemented NVPTX builtin call: ") +
850 getContext().BuiltinInfo.getName(builtinId));
851 return mlir::Value{};
852 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_z:
853 cgm.errorNYI(expr->getSourceRange(),
854 std::string("unimplemented NVPTX builtin call: ") +
855 getContext().BuiltinInfo.getName(builtinId));
856 return mlir::Value{};
857 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_w:
858 cgm.errorNYI(expr->getSourceRange(),
859 std::string("unimplemented NVPTX builtin call: ") +
860 getContext().BuiltinInfo.getName(builtinId));
861 return mlir::Value{};
862 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_x:
863 cgm.errorNYI(expr->getSourceRange(),
864 std::string("unimplemented NVPTX builtin call: ") +
865 getContext().BuiltinInfo.getName(builtinId));
866 return mlir::Value{};
867 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_y:
868 cgm.errorNYI(expr->getSourceRange(),
869 std::string("unimplemented NVPTX builtin call: ") +
870 getContext().BuiltinInfo.getName(builtinId));
871 return mlir::Value{};
872 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_z:
873 cgm.errorNYI(expr->getSourceRange(),
874 std::string("unimplemented NVPTX builtin call: ") +
875 getContext().BuiltinInfo.getName(builtinId));
876 return mlir::Value{};
877 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_w:
878 cgm.errorNYI(expr->getSourceRange(),
879 std::string("unimplemented NVPTX builtin call: ") +
880 getContext().BuiltinInfo.getName(builtinId));
881 return mlir::Value{};
882 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_x:
883 cgm.errorNYI(expr->getSourceRange(),
884 std::string("unimplemented NVPTX builtin call: ") +
885 getContext().BuiltinInfo.getName(builtinId));
886 return mlir::Value{};
887 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_y:
888 cgm.errorNYI(expr->getSourceRange(),
889 std::string("unimplemented NVPTX builtin call: ") +
890 getContext().BuiltinInfo.getName(builtinId));
891 return mlir::Value{};
892 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_z:
893 cgm.errorNYI(expr->getSourceRange(),
894 std::string("unimplemented NVPTX builtin call: ") +
895 getContext().BuiltinInfo.getName(builtinId));
896 return mlir::Value{};
897 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_w:
898 cgm.errorNYI(expr->getSourceRange(),
899 std::string("unimplemented NVPTX builtin call: ") +
900 getContext().BuiltinInfo.getName(builtinId));
901 return mlir::Value{};
902 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctarank:
903 cgm.errorNYI(expr->getSourceRange(),
904 std::string("unimplemented NVPTX builtin call: ") +
905 getContext().BuiltinInfo.getName(builtinId));
906 return mlir::Value{};
907 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctarank:
908 cgm.errorNYI(expr->getSourceRange(),
909 std::string("unimplemented NVPTX builtin call: ") +
910 getContext().BuiltinInfo.getName(builtinId));
911 return mlir::Value{};
912 case NVPTX::BI__nvvm_is_explicit_cluster:
913 cgm.errorNYI(expr->getSourceRange(),
914 std::string("unimplemented NVPTX builtin call: ") +
915 getContext().BuiltinInfo.getName(builtinId));
916 return mlir::Value{};
917 case NVPTX::BI__nvvm_isspacep_shared_cluster:
918 cgm.errorNYI(expr->getSourceRange(),
919 std::string("unimplemented NVPTX builtin call: ") +
920 getContext().BuiltinInfo.getName(builtinId));
921 return mlir::Value{};
922 case NVPTX::BI__nvvm_mapa:
923 cgm.errorNYI(expr->getSourceRange(),
924 std::string("unimplemented NVPTX builtin call: ") +
925 getContext().BuiltinInfo.getName(builtinId));
926 return mlir::Value{};
927 case NVPTX::BI__nvvm_mapa_shared_cluster:
928 cgm.errorNYI(expr->getSourceRange(),
929 std::string("unimplemented NVPTX builtin call: ") +
930 getContext().BuiltinInfo.getName(builtinId));
931 return mlir::Value{};
932 case NVPTX::BI__nvvm_getctarank:
933 cgm.errorNYI(expr->getSourceRange(),
934 std::string("unimplemented NVPTX builtin call: ") +
935 getContext().BuiltinInfo.getName(builtinId));
936 return mlir::Value{};
937 case NVPTX::BI__nvvm_getctarank_shared_cluster:
938 cgm.errorNYI(expr->getSourceRange(),
939 std::string("unimplemented NVPTX builtin call: ") +
940 getContext().BuiltinInfo.getName(builtinId));
941 return mlir::Value{};
942 case NVPTX::BI__nvvm_barrier_cluster_arrive:
943 return builder.emitIntrinsicCallOp(getLoc(expr->getExprLoc()),
944 "nvvm.barrier.cluster.arrive",
945 builder.getVoidTy());
946 case NVPTX::BI__nvvm_barrier_cluster_arrive_relaxed:
947 return builder.emitIntrinsicCallOp(getLoc(expr->getExprLoc()),
948 "nvvm.barrier.cluster.arrive.relaxed",
949 builder.getVoidTy());
950 case NVPTX::BI__nvvm_barrier_cluster_wait:
951 return builder.emitIntrinsicCallOp(getLoc(expr->getExprLoc()),
952 "nvvm.barrier.cluster.wait",
953 builder.getVoidTy());
954 case NVPTX::BI__nvvm_fence_sc_cluster:
955 return builder.emitIntrinsicCallOp(getLoc(expr->getExprLoc()),
956 "nvvm.fence.sc.cluster",
957 builder.getVoidTy());
958 case NVPTX::BI__nvvm_bar_sync:
959 return builder.emitIntrinsicCallOp(
960 getLoc(expr->getExprLoc()), "nvvm.barrier.cta.sync.aligned.all",
961 builder.getVoidTy(), mlir::ValueRange{emitScalarExpr(expr->getArg(0))});
962 case NVPTX::BI__syncthreads:
963 return builder.emitIntrinsicCallOp(
964 getLoc(expr->getExprLoc()), "nvvm.barrier.cta.sync.aligned.all",
965 builder.getVoidTy(),
966 mlir::ValueRange{builder.getConstInt(getLoc(expr->getExprLoc()),
967 builder.getSInt32Ty(), 0)});
968 case NVPTX::BI__nvvm_barrier_sync:
969 return builder.emitIntrinsicCallOp(
970 getLoc(expr->getExprLoc()), "nvvm.barrier.cta.sync.all",
971 builder.getVoidTy(), mlir::ValueRange{emitScalarExpr(expr->getArg(0))});
972 case NVPTX::BI__nvvm_barrier_sync_cnt:
973 return builder.emitIntrinsicCallOp(
974 getLoc(expr->getExprLoc()), "nvvm.barrier.cta.sync.count",
975 builder.getVoidTy(),
976 mlir::ValueRange{emitScalarExpr(expr->getArg(0)),
977 emitScalarExpr(expr->getArg(1))});
978 case NVPTX::BI__nvvm_bar0_and:
979 cgm.errorNYI(expr->getSourceRange(),
980 std::string("unimplemented NVPTX builtin call: ") +
981 getContext().BuiltinInfo.getName(builtinId));
982 return mlir::Value{};
983 case NVPTX::BI__nvvm_bar0_or:
984 cgm.errorNYI(expr->getSourceRange(),
985 std::string("unimplemented NVPTX builtin call: ") +
986 getContext().BuiltinInfo.getName(builtinId));
987 return mlir::Value{};
988 case NVPTX::BI__nvvm_bar0_popc:
989 cgm.errorNYI(expr->getSourceRange(),
990 std::string("unimplemented NVPTX builtin call: ") +
991 getContext().BuiltinInfo.getName(builtinId));
992 return mlir::Value{};
993
994 default:
995 return std::nullopt;
996 }
997}
998
999// vprintf takes two args: A format string, and a pointer to a buffer containing
1000// the varargs.
1001//
1002// For example, the call
1003//
1004// printf("format string", arg1, arg2, arg3);
1005//
1006// is converted into something resembling
1007//
1008// struct Tmp {
1009// Arg1 a1;
1010// Arg2 a2;
1011// Arg3 a3;
1012// };
1013// char* buf = alloca(sizeof(Tmp));
1014// *(Tmp*)buf = {a1, a2, a3};
1015// vprintf("format string", buf);
1016//
1017// `buf` is aligned to the max of {alignof(Arg1), ...}. Furthermore, each of
1018// the args is itself aligned to its preferred alignment.
1019//
1020// Note that by the time this function runs, the arguments have already
1021// undergone the standard C vararg promotion (short -> int, float -> double
1022// etc). In this function we pack the arguments into the buffer described above.
1023static mlir::Value packArgsIntoNVPTXFormatBuffer(CIRGenFunction &cgf,
1024 const CallArgList &args,
1025 mlir::Location loc) {
1026 const cir::CIRDataLayout dataLayout = cgf.cgm.getDataLayout();
1027 CIRGenBuilderTy &builder = cgf.getBuilder();
1028
1029 if (args.size() <= 1)
1030 // If there are no arguments other than the format string,
1031 // pass a nullptr to vprintf.
1032 return builder.getNullPtr(builder.getVoidPtrTy(), loc);
1033
1034 llvm::SmallVector<mlir::Type> argTypes;
1035 for (const auto &arg : llvm::drop_begin(args))
1036 argTypes.push_back(arg.getKnownRValue().getValue().getType());
1037
1038 // We can directly store the arguments into a struct, and the alignment
1039 // would automatically be correct. That's because vprintf does not
1040 // accept aggregates.
1041 mlir::Type allocaTy = builder.getAnonRecordTy(argTypes);
1042 auto allocaAlign = clang::CharUnits::fromQuantity(
1043 dataLayout.getABITypeAlign(allocaTy).value());
1044 Address allocaAddr =
1045 cgf.createTempAlloca(allocaTy, allocaAlign, loc, "printf_args");
1046 mlir::Value alloca = allocaAddr.getPointer();
1047
1048 for (auto [i, arg] : llvm::enumerate(llvm::drop_begin(args))) {
1049 mlir::Value member = builder.createGetMember(
1050 loc, cir::PointerType::get(argTypes[i]), alloca, /*name=*/"",
1051 /*index=*/i);
1052 auto abiAlign = clang::CharUnits::fromQuantity(
1053 dataLayout.getABITypeAlign(argTypes[i]).value());
1054 cir::StoreOp::create(builder, loc, arg.getKnownRValue().getValue(), member,
1055 /*is_volatile=*/false,
1056 /*isNontemporal=*/false,
1057 builder.getAlignmentAttr(abiAlign),
1058 /*sync_scope=*/cir::SyncScopeKindAttr{},
1059 /*mem_order=*/cir::MemOrderAttr{});
1060 }
1061
1062 return builder.createBitcast(alloca, builder.getVoidPtrTy());
1063}
1064
1065mlir::Value
1066CIRGenFunction::emitNVPTXDevicePrintfCallExpr(const CallExpr *expr) {
1067 assert(cgm.getTriple().isNVPTX());
1068 assert(expr->getBuiltinCallee() == Builtin::BIprintf ||
1069 expr->getBuiltinCallee() == Builtin::BI__builtin_printf);
1070 assert(expr->getNumArgs() >= 1); // printf always has at least one arg.
1071 CallArgList args;
1072 emitCallArgs(args,
1073 expr->getDirectCallee()->getType()->getAs<FunctionProtoType>(),
1074 expr->arguments(), expr->getDirectCallee());
1075
1076 mlir::Location loc = getLoc(expr->getBeginLoc());
1077
1078 // We don't know how to emit non-scalar varargs.
1079 bool hasNonScalar =
1080 llvm::any_of(llvm::drop_begin(args), [&](const CallArg &a) {
1081 return a.hasLValue() || !a.getKnownRValue().isScalar();
1082 });
1083 if (hasNonScalar) {
1084 cgm.errorUnsupported(expr, "non-scalar args to printf");
1085 return builder.getConstInt(loc, builder.getSInt32Ty(), 0);
1086 }
1087
1088 mlir::Value packedData = packArgsIntoNVPTXFormatBuffer(*this, args, loc);
1089
1090 // int vprintf(char *format, void *packedData);
1091 auto vprintf = cgm.createRuntimeFunction(
1092 cir::FuncType::get(
1093 {cir::PointerType::get(builder.getSInt8Ty()), builder.getVoidPtrTy()},
1094 builder.getSInt32Ty()),
1095 "vprintf");
1096 auto formatString = args[0].getKnownRValue().getValue();
1097 return builder
1098 .createCallOp(loc, vprintf, mlir::ValueRange{formatString, packedData})
1099 .getResult();
1100}
makeScopedAtomicRMW
static mlir::Value makeScopedAtomicRMW(CIRGenFunction &cgf, const CallExpr *expr, cir::AtomicFetchKind kind, cir::SyncScopeKind scope)
Definition CIRGenBuiltinNVPTX.cpp:52
packArgsIntoNVPTXFormatBuffer
static mlir::Value packArgsIntoNVPTXFormatBuffer(CIRGenFunction &cgf, const CallArgList &args, mlir::Location loc)
Definition CIRGenBuiltinNVPTX.cpp:1023
makeLdu
static mlir::Value makeLdu(CIRGenFunction &cgf, const CallExpr *expr, llvm::StringRef intrinsicName)
Definition CIRGenBuiltinNVPTX.cpp:23
emitUnaryNVVMIntrinsic
static mlir::Value emitUnaryNVVMIntrinsic(CIRGenFunction &cgf, const CallExpr *expr, llvm::StringRef intrinsicName)
Emit a CIR LLVMIntrinsicCallOp for a unary NVVM intrinsic.
Definition CIRGenBuiltinNVPTX.cpp:41
kind
*collection of selector each with an associated kind and an ordered *collection of selectors A selector has a kind
Definition OpenMPClause.h:9960
TargetBuiltins.h
Enumerates target-specific builtins in their own namespaces within namespace clang.
cir::CIRBaseBuilderTy::getNullPtr
cir::ConstantOp getNullPtr(mlir::Type ty, mlir::Location loc)
Definition CIRBaseBuilder.h:913
cir::CIRBaseBuilderTy::createBitcast
mlir::Value createBitcast(mlir::Value src, mlir::Type newTy)
Definition CIRBaseBuilder.h:588
cir::CIRBaseBuilderTy::getAlignmentAttr
mlir::IntegerAttr getAlignmentAttr(clang::CharUnits alignment)
Definition CIRBaseBuilder.h:889
cir::CIRBaseBuilderTy::getVoidPtrTy
cir::PointerType getVoidPtrTy(clang::LangAS langAS=clang::LangAS::Default)
Definition CIRBaseBuilder.h:182
cir::CIRDataLayout::getABITypeAlign
llvm::Align getABITypeAlign(mlir::Type ty) const
Definition CIRDataLayout.h:46
clang::CIRGen::Address::getPointer
mlir::Value getPointer() const
Definition Address.h:98
clang::CIRGen::Address::getAlignment
clang::CharUnits getAlignment() const
Definition Address.h:138
clang::CIRGen::Address::emitRawPointer
mlir::Value emitRawPointer() const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
Definition Address.h:112
clang::CIRGen::CIRGenBuilderTy::createGetMember
Address createGetMember(mlir::Location loc, Address base, llvm::StringRef name, unsigned index)
Definition CIRGenBuilder.h:668
clang::CIRGen::CIRGenBuilderTy::getAnonRecordTy
cir::StructType getAnonRecordTy(llvm::ArrayRef< mlir::Type > members, bool packed=false, bool padded=false)
Get a CIR anonymous struct type.
Definition CIRGenBuilder.h:446
clang::CIRGen::CIRGenFunction::emitCallArgs
void emitCallArgs(CallArgList &args, PrototypeWrapper prototype, llvm::iterator_range< clang::CallExpr::const_arg_iterator > argRange, AbstractCallee callee=AbstractCallee(), unsigned paramsToSkip=0)
Definition CIRGenCall.cpp:1502
clang::CIRGen::CIRGenFunction::emitPointerWithAlignment
Address emitPointerWithAlignment(const clang::Expr *expr, LValueBaseInfo *baseInfo=nullptr)
Given an expression with a pointer type, emit the value and compute our best estimate of the alignmen...
Definition CIRGenExpr.cpp:117
clang::CIRGen::CIRGenFunction::createTempAlloca
cir::AllocaOp createTempAlloca(mlir::Type ty, mlir::Location loc, const Twine &name="tmp", mlir::Value arraySize=nullptr, bool insertIntoFnEntryBlock=false)
This creates an alloca and inserts it into the entry block if ArraySize is nullptr,...
Definition CIRGenExpr.cpp:2927
clang::CIRGen::CIRGenFunction::emitNVPTXDevicePrintfCallExpr
mlir::Value emitNVPTXDevicePrintfCallExpr(const CallExpr *expr)
Emit a device-side printf call for NVPTX targets.
Definition CIRGenBuiltinNVPTX.cpp:1066
clang::CIRGen::CIRGenFunction::getLoc
mlir::Location getLoc(clang::SourceLocation srcLoc)
Helpers to convert Clang's SourceLocation to a MLIR Location.
Definition CIRGenFunction.cpp:108
clang::CIRGen::CIRGenFunction::makeBinaryAtomicValue
mlir::Value makeBinaryAtomicValue(cir::AtomicFetchKind kind, const clang::CallExpr *expr, mlir::Type *originalArgType=nullptr, mlir::Value *emittedArgValue=nullptr, cir::MemOrder ordering=cir::MemOrder::SequentiallyConsistent)
Utility to insert an atomic instruction based on Intrinsic::ID and the expression node.
Definition CIRGenBuiltin.cpp:166
clang::CIRGen::CIRGenFunction::convertTypeForMem
mlir::Type convertTypeForMem(QualType t)
Definition CIRGenFunction.cpp:100
clang::CIRGen::CIRGenFunction::emitScalarExpr
mlir::Value emitScalarExpr(const clang::Expr *e, bool ignoreResultAssign=false)
Emit the computation of the specified expression of scalar type.
Definition CIRGenExprScalar.cpp:1760
clang::CIRGen::CIRGenFunction::getBuilder
CIRGenBuilderTy & getBuilder()
Definition CIRGenFunction.h:222
clang::CIRGen::CIRGenFunction::emitNVPTXBuiltinExpr
std::optional< mlir::Value > emitNVPTXBuiltinExpr(unsigned builtinID, const CallExpr *expr)
Emit a call to an NVPTX builtin function.
Definition CIRGenBuiltinNVPTX.cpp:68
clang::CIRGen::CIRGenFunction::getContext
clang::ASTContext & getContext() const
Definition CIRGenFunction.h:220
clang::CIRGen::CIRGenModule::getDataLayout
const cir::CIRDataLayout getDataLayout() const
Definition CIRGenModule.h:177
clang::CIRGen::RValue::isScalar
bool isScalar() const
Definition CIRGenValue.h:49
clang::CallExpr
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
Definition Expr.h:2949
clang::CharUnits
CharUnits - This is an opaque type for sizes expressed in character units.
Definition CharUnits.h:38
clang::CharUnits::fromQuantity
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
Definition CharUnits.h:63
clang::FunctionProtoType
Represents a prototype with parameter type info, e.g.
Definition TypeBase.h:5406
clang::QualType
A (possibly-)qualified type.
Definition TypeBase.h:938
clang::Type::getPointeeType
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
Definition Type.cpp:789
clang::ast_matchers::expr
const internal::VariadicDynCastAllOfMatcher< Stmt, Expr > expr
Matches expressions.
Definition ASTMatchersInternal.cpp:937
clang
The JSON file list parser is used to communicate input to InstallAPI.
Definition CalledOnceCheck.h:17
clang::CIRGen::CallArg::hasLValue
bool hasLValue() const
Definition CIRGenCall.h:216
clang::CIRGen::CallArg::getKnownRValue
RValue getKnownRValue() const
Definition CIRGenCall.h:227
Generated on for clang by doxygen 1.14.0