clang 23.0.0git
CIRGenBuiltinNVPTX.cpp
Go to the documentation of this file.
1//===---- CIRGenBuiltinNVPTX.cpp - Emit CIR for NVPTX builtins ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit NVPTX Builtin calls.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CIRGenFunction.h"
14
15#include "mlir/IR/Value.h"
16#include "clang/Basic/TargetBuiltins.h"
17#include "clang/CIR/Dialect/IR/CIRDataLayout.h"
18#include "clang/CIR/Dialect/IR/CIRDialect.h"
19
20using namespace clang;
21using namespace clang::CIRGen;
22
23/// Emit a CIR LLVMIntrinsicCallOp for a unary NVVM intrinsic.
24/// The result type is inferred from the single argument.
25static mlir::Value emitUnaryNVVMIntrinsic(CIRGenFunction &cgf,
26 const CallExpr *expr,
27 llvm::StringRef intrinsicName) {
28 auto &builder = cgf.getBuilder();
29 mlir::Value arg = cgf.emitScalarExpr(expr->getArg(0));
30 return cir::LLVMIntrinsicCallOp::create(
31 builder, cgf.getLoc(expr->getExprLoc()),
32 builder.getStringAttr(intrinsicName), arg.getType(), {arg})
33 .getResult();
34}
35
36std::optional<mlir::Value>
37CIRGenFunction::emitNVPTXBuiltinExpr(unsigned builtinId, const CallExpr *expr) {
38 switch (builtinId) {
39 case NVPTX::BI__nvvm_atom_add_gen_i:
40 case NVPTX::BI__nvvm_atom_add_gen_l:
41 case NVPTX::BI__nvvm_atom_add_gen_ll:
42 cgm.errorNYI(expr->getSourceRange(),
43 std::string("unimplemented NVPTX builtin call: ") +
44 getContext().BuiltinInfo.getName(builtinId));
45 return mlir::Value{};
46 case NVPTX::BI__nvvm_atom_sub_gen_i:
47 case NVPTX::BI__nvvm_atom_sub_gen_l:
48 case NVPTX::BI__nvvm_atom_sub_gen_ll:
49 cgm.errorNYI(expr->getSourceRange(),
50 std::string("unimplemented NVPTX builtin call: ") +
51 getContext().BuiltinInfo.getName(builtinId));
52 return mlir::Value{};
53 case NVPTX::BI__nvvm_atom_and_gen_i:
54 case NVPTX::BI__nvvm_atom_and_gen_l:
55 case NVPTX::BI__nvvm_atom_and_gen_ll:
56 cgm.errorNYI(expr->getSourceRange(),
57 std::string("unimplemented NVPTX builtin call: ") +
58 getContext().BuiltinInfo.getName(builtinId));
59 return mlir::Value{};
60 case NVPTX::BI__nvvm_atom_or_gen_i:
61 case NVPTX::BI__nvvm_atom_or_gen_l:
62 case NVPTX::BI__nvvm_atom_or_gen_ll:
63 cgm.errorNYI(expr->getSourceRange(),
64 std::string("unimplemented NVPTX builtin call: ") +
65 getContext().BuiltinInfo.getName(builtinId));
66 return mlir::Value{};
67 case NVPTX::BI__nvvm_atom_xor_gen_i:
68 case NVPTX::BI__nvvm_atom_xor_gen_l:
69 case NVPTX::BI__nvvm_atom_xor_gen_ll:
70 cgm.errorNYI(expr->getSourceRange(),
71 std::string("unimplemented NVPTX builtin call: ") +
72 getContext().BuiltinInfo.getName(builtinId));
73 return mlir::Value{};
74 case NVPTX::BI__nvvm_atom_xchg_gen_i:
75 case NVPTX::BI__nvvm_atom_xchg_gen_l:
76 case NVPTX::BI__nvvm_atom_xchg_gen_ll:
77 cgm.errorNYI(expr->getSourceRange(),
78 std::string("unimplemented NVPTX builtin call: ") +
79 getContext().BuiltinInfo.getName(builtinId));
80 return mlir::Value{};
81 case NVPTX::BI__nvvm_atom_max_gen_i:
82 case NVPTX::BI__nvvm_atom_max_gen_l:
83 case NVPTX::BI__nvvm_atom_max_gen_ll:
84 cgm.errorNYI(expr->getSourceRange(),
85 std::string("unimplemented NVPTX builtin call: ") +
86 getContext().BuiltinInfo.getName(builtinId));
87 return mlir::Value{};
88 case NVPTX::BI__nvvm_atom_max_gen_ui:
89 case NVPTX::BI__nvvm_atom_max_gen_ul:
90 case NVPTX::BI__nvvm_atom_max_gen_ull:
91 cgm.errorNYI(expr->getSourceRange(),
92 std::string("unimplemented NVPTX builtin call: ") +
93 getContext().BuiltinInfo.getName(builtinId));
94 return mlir::Value{};
95 case NVPTX::BI__nvvm_atom_min_gen_i:
96 case NVPTX::BI__nvvm_atom_min_gen_l:
97 case NVPTX::BI__nvvm_atom_min_gen_ll:
98 cgm.errorNYI(expr->getSourceRange(),
99 std::string("unimplemented NVPTX builtin call: ") +
100 getContext().BuiltinInfo.getName(builtinId));
101 return mlir::Value{};
102 case NVPTX::BI__nvvm_atom_min_gen_ui:
103 case NVPTX::BI__nvvm_atom_min_gen_ul:
104 case NVPTX::BI__nvvm_atom_min_gen_ull:
105 cgm.errorNYI(expr->getSourceRange(),
106 std::string("unimplemented NVPTX builtin call: ") +
107 getContext().BuiltinInfo.getName(builtinId));
108 return mlir::Value{};
109 case NVPTX::BI__nvvm_atom_cas_gen_us:
110 case NVPTX::BI__nvvm_atom_cas_gen_i:
111 case NVPTX::BI__nvvm_atom_cas_gen_l:
112 case NVPTX::BI__nvvm_atom_cas_gen_ll:
113 cgm.errorNYI(expr->getSourceRange(),
114 std::string("unimplemented NVPTX builtin call: ") +
115 getContext().BuiltinInfo.getName(builtinId));
116 return mlir::Value{};
117 // success flag.
118 case NVPTX::BI__nvvm_atom_add_gen_f:
119 case NVPTX::BI__nvvm_atom_add_gen_d:
120 cgm.errorNYI(expr->getSourceRange(),
121 std::string("unimplemented NVPTX builtin call: ") +
122 getContext().BuiltinInfo.getName(builtinId));
123 return mlir::Value{};
124 case NVPTX::BI__nvvm_atom_inc_gen_ui:
125 cgm.errorNYI(expr->getSourceRange(),
126 std::string("unimplemented NVPTX builtin call: ") +
127 getContext().BuiltinInfo.getName(builtinId));
128 return mlir::Value{};
129 case NVPTX::BI__nvvm_atom_dec_gen_ui:
130 cgm.errorNYI(expr->getSourceRange(),
131 std::string("unimplemented NVPTX builtin call: ") +
132 getContext().BuiltinInfo.getName(builtinId));
133 return mlir::Value{};
134 case NVPTX::BI__nvvm_ldg_c:
135 case NVPTX::BI__nvvm_ldg_sc:
136 case NVPTX::BI__nvvm_ldg_c2:
137 case NVPTX::BI__nvvm_ldg_sc2:
138 case NVPTX::BI__nvvm_ldg_c4:
139 case NVPTX::BI__nvvm_ldg_sc4:
140 case NVPTX::BI__nvvm_ldg_s:
141 case NVPTX::BI__nvvm_ldg_s2:
142 case NVPTX::BI__nvvm_ldg_s4:
143 case NVPTX::BI__nvvm_ldg_i:
144 case NVPTX::BI__nvvm_ldg_i2:
145 case NVPTX::BI__nvvm_ldg_i4:
146 case NVPTX::BI__nvvm_ldg_l:
147 case NVPTX::BI__nvvm_ldg_l2:
148 case NVPTX::BI__nvvm_ldg_ll:
149 case NVPTX::BI__nvvm_ldg_ll2:
150 case NVPTX::BI__nvvm_ldg_uc:
151 case NVPTX::BI__nvvm_ldg_uc2:
152 case NVPTX::BI__nvvm_ldg_uc4:
153 case NVPTX::BI__nvvm_ldg_us:
154 case NVPTX::BI__nvvm_ldg_us2:
155 case NVPTX::BI__nvvm_ldg_us4:
156 case NVPTX::BI__nvvm_ldg_ui:
157 case NVPTX::BI__nvvm_ldg_ui2:
158 case NVPTX::BI__nvvm_ldg_ui4:
159 case NVPTX::BI__nvvm_ldg_ul:
160 case NVPTX::BI__nvvm_ldg_ul2:
161 case NVPTX::BI__nvvm_ldg_ull:
162 case NVPTX::BI__nvvm_ldg_ull2:
163 case NVPTX::BI__nvvm_ldg_f:
164 case NVPTX::BI__nvvm_ldg_f2:
165 case NVPTX::BI__nvvm_ldg_f4:
166 case NVPTX::BI__nvvm_ldg_d:
167 case NVPTX::BI__nvvm_ldg_d2:
168 cgm.errorNYI(expr->getSourceRange(),
169 std::string("unimplemented NVPTX builtin call: ") +
170 getContext().BuiltinInfo.getName(builtinId));
171 return mlir::Value{};
172 case NVPTX::BI__nvvm_ldu_c:
173 case NVPTX::BI__nvvm_ldu_sc:
174 case NVPTX::BI__nvvm_ldu_c2:
175 case NVPTX::BI__nvvm_ldu_sc2:
176 case NVPTX::BI__nvvm_ldu_c4:
177 case NVPTX::BI__nvvm_ldu_sc4:
178 case NVPTX::BI__nvvm_ldu_s:
179 case NVPTX::BI__nvvm_ldu_s2:
180 case NVPTX::BI__nvvm_ldu_s4:
181 case NVPTX::BI__nvvm_ldu_i:
182 case NVPTX::BI__nvvm_ldu_i2:
183 case NVPTX::BI__nvvm_ldu_i4:
184 case NVPTX::BI__nvvm_ldu_l:
185 case NVPTX::BI__nvvm_ldu_l2:
186 case NVPTX::BI__nvvm_ldu_ll:
187 case NVPTX::BI__nvvm_ldu_ll2:
188 case NVPTX::BI__nvvm_ldu_uc:
189 case NVPTX::BI__nvvm_ldu_uc2:
190 case NVPTX::BI__nvvm_ldu_uc4:
191 case NVPTX::BI__nvvm_ldu_us:
192 case NVPTX::BI__nvvm_ldu_us2:
193 case NVPTX::BI__nvvm_ldu_us4:
194 case NVPTX::BI__nvvm_ldu_ui:
195 case NVPTX::BI__nvvm_ldu_ui2:
196 case NVPTX::BI__nvvm_ldu_ui4:
197 case NVPTX::BI__nvvm_ldu_ul:
198 case NVPTX::BI__nvvm_ldu_ul2:
199 case NVPTX::BI__nvvm_ldu_ull:
200 case NVPTX::BI__nvvm_ldu_ull2:
201 cgm.errorNYI(expr->getSourceRange(),
202 std::string("unimplemented NVPTX builtin call: ") +
203 getContext().BuiltinInfo.getName(builtinId));
204 return mlir::Value{};
205 case NVPTX::BI__nvvm_ldu_f:
206 case NVPTX::BI__nvvm_ldu_f2:
207 case NVPTX::BI__nvvm_ldu_f4:
208 case NVPTX::BI__nvvm_ldu_d:
209 case NVPTX::BI__nvvm_ldu_d2:
210 cgm.errorNYI(expr->getSourceRange(),
211 std::string("unimplemented NVPTX builtin call: ") +
212 getContext().BuiltinInfo.getName(builtinId));
213 return mlir::Value{};
214 case NVPTX::BI__nvvm_atom_cta_add_gen_i:
215 case NVPTX::BI__nvvm_atom_cta_add_gen_l:
216 case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
217 cgm.errorNYI(expr->getSourceRange(),
218 std::string("unimplemented NVPTX builtin call: ") +
219 getContext().BuiltinInfo.getName(builtinId));
220 return mlir::Value{};
221 case NVPTX::BI__nvvm_atom_sys_add_gen_i:
222 case NVPTX::BI__nvvm_atom_sys_add_gen_l:
223 case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
224 cgm.errorNYI(expr->getSourceRange(),
225 std::string("unimplemented NVPTX builtin call: ") +
226 getContext().BuiltinInfo.getName(builtinId));
227 return mlir::Value{};
228 case NVPTX::BI__nvvm_atom_cta_add_gen_f:
229 case NVPTX::BI__nvvm_atom_cta_add_gen_d:
230 cgm.errorNYI(expr->getSourceRange(),
231 std::string("unimplemented NVPTX builtin call: ") +
232 getContext().BuiltinInfo.getName(builtinId));
233 return mlir::Value{};
234 case NVPTX::BI__nvvm_atom_sys_add_gen_f:
235 case NVPTX::BI__nvvm_atom_sys_add_gen_d:
236 cgm.errorNYI(expr->getSourceRange(),
237 std::string("unimplemented NVPTX builtin call: ") +
238 getContext().BuiltinInfo.getName(builtinId));
239 return mlir::Value{};
240 case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
241 case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
242 case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
243 cgm.errorNYI(expr->getSourceRange(),
244 std::string("unimplemented NVPTX builtin call: ") +
245 getContext().BuiltinInfo.getName(builtinId));
246 return mlir::Value{};
247 case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
248 case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
249 case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
250 cgm.errorNYI(expr->getSourceRange(),
251 std::string("unimplemented NVPTX builtin call: ") +
252 getContext().BuiltinInfo.getName(builtinId));
253 return mlir::Value{};
254 case NVPTX::BI__nvvm_atom_cta_max_gen_i:
255 case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
256 case NVPTX::BI__nvvm_atom_cta_max_gen_l:
257 case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
258 case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
259 case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
260 cgm.errorNYI(expr->getSourceRange(),
261 std::string("unimplemented NVPTX builtin call: ") +
262 getContext().BuiltinInfo.getName(builtinId));
263 return mlir::Value{};
264 case NVPTX::BI__nvvm_atom_sys_max_gen_i:
265 case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
266 case NVPTX::BI__nvvm_atom_sys_max_gen_l:
267 case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
268 case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
269 case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
270 cgm.errorNYI(expr->getSourceRange(),
271 std::string("unimplemented NVPTX builtin call: ") +
272 getContext().BuiltinInfo.getName(builtinId));
273 return mlir::Value{};
274 case NVPTX::BI__nvvm_atom_cta_min_gen_i:
275 case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
276 case NVPTX::BI__nvvm_atom_cta_min_gen_l:
277 case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
278 case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
279 case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
280 cgm.errorNYI(expr->getSourceRange(),
281 std::string("unimplemented NVPTX builtin call: ") +
282 getContext().BuiltinInfo.getName(builtinId));
283 return mlir::Value{};
284 case NVPTX::BI__nvvm_atom_sys_min_gen_i:
285 case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
286 case NVPTX::BI__nvvm_atom_sys_min_gen_l:
287 case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
288 case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
289 case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
290 cgm.errorNYI(expr->getSourceRange(),
291 std::string("unimplemented NVPTX builtin call: ") +
292 getContext().BuiltinInfo.getName(builtinId));
293 return mlir::Value{};
294 case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
295 cgm.errorNYI(expr->getSourceRange(),
296 std::string("unimplemented NVPTX builtin call: ") +
297 getContext().BuiltinInfo.getName(builtinId));
298 return mlir::Value{};
299 case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
300 cgm.errorNYI(expr->getSourceRange(),
301 std::string("unimplemented NVPTX builtin call: ") +
302 getContext().BuiltinInfo.getName(builtinId));
303 return mlir::Value{};
304 case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
305 cgm.errorNYI(expr->getSourceRange(),
306 std::string("unimplemented NVPTX builtin call: ") +
307 getContext().BuiltinInfo.getName(builtinId));
308 return mlir::Value{};
309 case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
310 cgm.errorNYI(expr->getSourceRange(),
311 std::string("unimplemented NVPTX builtin call: ") +
312 getContext().BuiltinInfo.getName(builtinId));
313 return mlir::Value{};
314 case NVPTX::BI__nvvm_atom_cta_and_gen_i:
315 case NVPTX::BI__nvvm_atom_cta_and_gen_l:
316 case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
317 cgm.errorNYI(expr->getSourceRange(),
318 std::string("unimplemented NVPTX builtin call: ") +
319 getContext().BuiltinInfo.getName(builtinId));
320 return mlir::Value{};
321 case NVPTX::BI__nvvm_atom_sys_and_gen_i:
322 case NVPTX::BI__nvvm_atom_sys_and_gen_l:
323 case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
324 cgm.errorNYI(expr->getSourceRange(),
325 std::string("unimplemented NVPTX builtin call: ") +
326 getContext().BuiltinInfo.getName(builtinId));
327 return mlir::Value{};
328 case NVPTX::BI__nvvm_atom_cta_or_gen_i:
329 case NVPTX::BI__nvvm_atom_cta_or_gen_l:
330 case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
331 cgm.errorNYI(expr->getSourceRange(),
332 std::string("unimplemented NVPTX builtin call: ") +
333 getContext().BuiltinInfo.getName(builtinId));
334 return mlir::Value{};
335 case NVPTX::BI__nvvm_atom_sys_or_gen_i:
336 case NVPTX::BI__nvvm_atom_sys_or_gen_l:
337 case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
338 cgm.errorNYI(expr->getSourceRange(),
339 std::string("unimplemented NVPTX builtin call: ") +
340 getContext().BuiltinInfo.getName(builtinId));
341 return mlir::Value{};
342 case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
343 case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
344 case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
345 cgm.errorNYI(expr->getSourceRange(),
346 std::string("unimplemented NVPTX builtin call: ") +
347 getContext().BuiltinInfo.getName(builtinId));
348 return mlir::Value{};
349 case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
350 case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
351 case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
352 cgm.errorNYI(expr->getSourceRange(),
353 std::string("unimplemented NVPTX builtin call: ") +
354 getContext().BuiltinInfo.getName(builtinId));
355 return mlir::Value{};
356 case NVPTX::BI__nvvm_atom_cta_cas_gen_us:
357 case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
358 case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
359 case NVPTX::BI__nvvm_atom_cta_cas_gen_ll:
360 cgm.errorNYI(expr->getSourceRange(),
361 std::string("unimplemented NVPTX builtin call: ") +
362 getContext().BuiltinInfo.getName(builtinId));
363 return mlir::Value{};
364 case NVPTX::BI__nvvm_atom_sys_cas_gen_us:
365 case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
366 case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
367 case NVPTX::BI__nvvm_atom_sys_cas_gen_ll:
368 cgm.errorNYI(expr->getSourceRange(),
369 std::string("unimplemented NVPTX builtin call: ") +
370 getContext().BuiltinInfo.getName(builtinId));
371 return mlir::Value{};
372 case NVPTX::BI__nvvm_match_all_sync_i32p:
373 case NVPTX::BI__nvvm_match_all_sync_i64p:
374 cgm.errorNYI(expr->getSourceRange(),
375 std::string("unimplemented NVPTX builtin call: ") +
376 getContext().BuiltinInfo.getName(builtinId));
377 return mlir::Value{};
378 // FP MMA loads
379 case NVPTX::BI__hmma_m16n16k16_ld_a:
380 case NVPTX::BI__hmma_m16n16k16_ld_b:
381 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
382 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
383 case NVPTX::BI__hmma_m32n8k16_ld_a:
384 case NVPTX::BI__hmma_m32n8k16_ld_b:
385 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
386 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
387 case NVPTX::BI__hmma_m8n32k16_ld_a:
388 case NVPTX::BI__hmma_m8n32k16_ld_b:
389 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
390 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
391 cgm.errorNYI(expr->getSourceRange(),
392 std::string("unimplemented NVPTX builtin call: ") +
393 getContext().BuiltinInfo.getName(builtinId));
394 return mlir::Value{};
395 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
396 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
397 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
398 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
399 case NVPTX::BI__imma_m16n16k16_ld_c:
400 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
401 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
402 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
403 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
404 case NVPTX::BI__imma_m32n8k16_ld_c:
405 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
406 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
407 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
408 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
409 case NVPTX::BI__imma_m8n32k16_ld_c:
410 cgm.errorNYI(expr->getSourceRange(),
411 std::string("unimplemented NVPTX builtin call: ") +
412 getContext().BuiltinInfo.getName(builtinId));
413 return mlir::Value{};
414 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
415 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
416 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
417 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
418 case NVPTX::BI__imma_m8n8k32_ld_c:
419 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
420 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
421 case NVPTX::BI__bmma_m8n8k128_ld_c:
422 cgm.errorNYI(expr->getSourceRange(),
423 std::string("unimplemented NVPTX builtin call: ") +
424 getContext().BuiltinInfo.getName(builtinId));
425 return mlir::Value{};
426 case NVPTX::BI__dmma_m8n8k4_ld_a:
427 case NVPTX::BI__dmma_m8n8k4_ld_b:
428 case NVPTX::BI__dmma_m8n8k4_ld_c:
429 cgm.errorNYI(expr->getSourceRange(),
430 std::string("unimplemented NVPTX builtin call: ") +
431 getContext().BuiltinInfo.getName(builtinId));
432 return mlir::Value{};
433 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
434 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
435 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
436 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
437 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
438 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
439 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
440 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
441 case NVPTX::BI__mma_tf32_m16n16k8_ld_c:
442 cgm.errorNYI(expr->getSourceRange(),
443 std::string("unimplemented NVPTX builtin call: ") +
444 getContext().BuiltinInfo.getName(builtinId));
445 return mlir::Value{};
446 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
447 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
448 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
449 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
450 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
451 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
452 case NVPTX::BI__imma_m16n16k16_st_c_i32:
453 case NVPTX::BI__imma_m32n8k16_st_c_i32:
454 case NVPTX::BI__imma_m8n32k16_st_c_i32:
455 case NVPTX::BI__imma_m8n8k32_st_c_i32:
456 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
457 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
458 case NVPTX::BI__mma_m16n16k8_st_c_f32:
459 cgm.errorNYI(expr->getSourceRange(),
460 std::string("unimplemented NVPTX builtin call: ") +
461 getContext().BuiltinInfo.getName(builtinId));
462 return mlir::Value{};
463 // BI__hmma_m16n16k16_mma_<Dtype><CType>(d, a, b, c, layout, satf) -->
464 // Intrinsic::nvvm_wmma_m16n16k16_mma_sync<layout A,B><DType><CType><Satf>
465 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
466 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
467 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
468 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
469 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
470 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
471 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
472 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
473 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
474 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
475 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
476 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
477 case NVPTX::BI__imma_m16n16k16_mma_s8:
478 case NVPTX::BI__imma_m16n16k16_mma_u8:
479 case NVPTX::BI__imma_m32n8k16_mma_s8:
480 case NVPTX::BI__imma_m32n8k16_mma_u8:
481 case NVPTX::BI__imma_m8n32k16_mma_s8:
482 case NVPTX::BI__imma_m8n32k16_mma_u8:
483 case NVPTX::BI__imma_m8n8k32_mma_s4:
484 case NVPTX::BI__imma_m8n8k32_mma_u4:
485 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
486 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
487 case NVPTX::BI__dmma_m8n8k4_mma_f64:
488 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
489 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
490 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
491 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32:
492 cgm.errorNYI(expr->getSourceRange(),
493 std::string("unimplemented NVPTX builtin call: ") +
494 getContext().BuiltinInfo.getName(builtinId));
495 return mlir::Value{};
496 // The following builtins require half type support
497 case NVPTX::BI__nvvm_ex2_approx_f16:
498 cgm.errorNYI(expr->getSourceRange(),
499 std::string("unimplemented NVPTX builtin call: ") +
500 getContext().BuiltinInfo.getName(builtinId));
501 return mlir::Value{};
502 case NVPTX::BI__nvvm_ex2_approx_f16x2:
503 cgm.errorNYI(expr->getSourceRange(),
504 std::string("unimplemented NVPTX builtin call: ") +
505 getContext().BuiltinInfo.getName(builtinId));
506 return mlir::Value{};
507 case NVPTX::BI__nvvm_ff2f16x2_rn:
508 cgm.errorNYI(expr->getSourceRange(),
509 std::string("unimplemented NVPTX builtin call: ") +
510 getContext().BuiltinInfo.getName(builtinId));
511 return mlir::Value{};
512 case NVPTX::BI__nvvm_ff2f16x2_rn_relu:
513 cgm.errorNYI(expr->getSourceRange(),
514 std::string("unimplemented NVPTX builtin call: ") +
515 getContext().BuiltinInfo.getName(builtinId));
516 return mlir::Value{};
517 case NVPTX::BI__nvvm_ff2f16x2_rz:
518 cgm.errorNYI(expr->getSourceRange(),
519 std::string("unimplemented NVPTX builtin call: ") +
520 getContext().BuiltinInfo.getName(builtinId));
521 return mlir::Value{};
522 case NVPTX::BI__nvvm_ff2f16x2_rz_relu:
523 cgm.errorNYI(expr->getSourceRange(),
524 std::string("unimplemented NVPTX builtin call: ") +
525 getContext().BuiltinInfo.getName(builtinId));
526 return mlir::Value{};
527 case NVPTX::BI__nvvm_fma_rn_f16:
528 cgm.errorNYI(expr->getSourceRange(),
529 std::string("unimplemented NVPTX builtin call: ") +
530 getContext().BuiltinInfo.getName(builtinId));
531 return mlir::Value{};
532 case NVPTX::BI__nvvm_fma_rn_f16x2:
533 cgm.errorNYI(expr->getSourceRange(),
534 std::string("unimplemented NVPTX builtin call: ") +
535 getContext().BuiltinInfo.getName(builtinId));
536 return mlir::Value{};
537 case NVPTX::BI__nvvm_fma_rn_ftz_f16:
538 cgm.errorNYI(expr->getSourceRange(),
539 std::string("unimplemented NVPTX builtin call: ") +
540 getContext().BuiltinInfo.getName(builtinId));
541 return mlir::Value{};
542 case NVPTX::BI__nvvm_fma_rn_ftz_f16x2:
543 cgm.errorNYI(expr->getSourceRange(),
544 std::string("unimplemented NVPTX builtin call: ") +
545 getContext().BuiltinInfo.getName(builtinId));
546 return mlir::Value{};
547 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16:
548 cgm.errorNYI(expr->getSourceRange(),
549 std::string("unimplemented NVPTX builtin call: ") +
550 getContext().BuiltinInfo.getName(builtinId));
551 return mlir::Value{};
552 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16x2:
553 cgm.errorNYI(expr->getSourceRange(),
554 std::string("unimplemented NVPTX builtin call: ") +
555 getContext().BuiltinInfo.getName(builtinId));
556 return mlir::Value{};
557 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16:
558 cgm.errorNYI(expr->getSourceRange(),
559 std::string("unimplemented NVPTX builtin call: ") +
560 getContext().BuiltinInfo.getName(builtinId));
561 return mlir::Value{};
562 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16x2:
563 cgm.errorNYI(expr->getSourceRange(),
564 std::string("unimplemented NVPTX builtin call: ") +
565 getContext().BuiltinInfo.getName(builtinId));
566 return mlir::Value{};
567 case NVPTX::BI__nvvm_fma_rn_relu_f16:
568 cgm.errorNYI(expr->getSourceRange(),
569 std::string("unimplemented NVPTX builtin call: ") +
570 getContext().BuiltinInfo.getName(builtinId));
571 return mlir::Value{};
572 case NVPTX::BI__nvvm_fma_rn_relu_f16x2:
573 cgm.errorNYI(expr->getSourceRange(),
574 std::string("unimplemented NVPTX builtin call: ") +
575 getContext().BuiltinInfo.getName(builtinId));
576 return mlir::Value{};
577 case NVPTX::BI__nvvm_fma_rn_sat_f16:
578 cgm.errorNYI(expr->getSourceRange(),
579 std::string("unimplemented NVPTX builtin call: ") +
580 getContext().BuiltinInfo.getName(builtinId));
581 return mlir::Value{};
582 case NVPTX::BI__nvvm_fma_rn_sat_f16x2:
583 cgm.errorNYI(expr->getSourceRange(),
584 std::string("unimplemented NVPTX builtin call: ") +
585 getContext().BuiltinInfo.getName(builtinId));
586 return mlir::Value{};
587 case NVPTX::BI__nvvm_fma_rn_oob_f16:
588 cgm.errorNYI(expr->getSourceRange(),
589 std::string("unimplemented NVPTX builtin call: ") +
590 getContext().BuiltinInfo.getName(builtinId));
591 return mlir::Value{};
592 case NVPTX::BI__nvvm_fma_rn_oob_f16x2:
593 cgm.errorNYI(expr->getSourceRange(),
594 std::string("unimplemented NVPTX builtin call: ") +
595 getContext().BuiltinInfo.getName(builtinId));
596 return mlir::Value{};
597 case NVPTX::BI__nvvm_fma_rn_oob_bf16:
598 cgm.errorNYI(expr->getSourceRange(),
599 std::string("unimplemented NVPTX builtin call: ") +
600 getContext().BuiltinInfo.getName(builtinId));
601 return mlir::Value{};
602 case NVPTX::BI__nvvm_fma_rn_oob_bf16x2:
603 cgm.errorNYI(expr->getSourceRange(),
604 std::string("unimplemented NVPTX builtin call: ") +
605 getContext().BuiltinInfo.getName(builtinId));
606 return mlir::Value{};
607 case NVPTX::BI__nvvm_fma_rn_oob_relu_f16:
608 cgm.errorNYI(expr->getSourceRange(),
609 std::string("unimplemented NVPTX builtin call: ") +
610 getContext().BuiltinInfo.getName(builtinId));
611 return mlir::Value{};
612 case NVPTX::BI__nvvm_fma_rn_oob_relu_f16x2:
613 cgm.errorNYI(expr->getSourceRange(),
614 std::string("unimplemented NVPTX builtin call: ") +
615 getContext().BuiltinInfo.getName(builtinId));
616 return mlir::Value{};
617 case NVPTX::BI__nvvm_fma_rn_oob_relu_bf16:
618 cgm.errorNYI(expr->getSourceRange(),
619 std::string("unimplemented NVPTX builtin call: ") +
620 getContext().BuiltinInfo.getName(builtinId));
621 return mlir::Value{};
622 case NVPTX::BI__nvvm_fma_rn_oob_relu_bf16x2:
623 cgm.errorNYI(expr->getSourceRange(),
624 std::string("unimplemented NVPTX builtin call: ") +
625 getContext().BuiltinInfo.getName(builtinId));
626 return mlir::Value{};
627 case NVPTX::BI__nvvm_fmax_f16:
628 cgm.errorNYI(expr->getSourceRange(),
629 std::string("unimplemented NVPTX builtin call: ") +
630 getContext().BuiltinInfo.getName(builtinId));
631 return mlir::Value{};
632 case NVPTX::BI__nvvm_fmax_f16x2:
633 cgm.errorNYI(expr->getSourceRange(),
634 std::string("unimplemented NVPTX builtin call: ") +
635 getContext().BuiltinInfo.getName(builtinId));
636 return mlir::Value{};
637 case NVPTX::BI__nvvm_fmax_ftz_f16:
638 cgm.errorNYI(expr->getSourceRange(),
639 std::string("unimplemented NVPTX builtin call: ") +
640 getContext().BuiltinInfo.getName(builtinId));
641 return mlir::Value{};
642 case NVPTX::BI__nvvm_fmax_ftz_f16x2:
643 cgm.errorNYI(expr->getSourceRange(),
644 std::string("unimplemented NVPTX builtin call: ") +
645 getContext().BuiltinInfo.getName(builtinId));
646 return mlir::Value{};
647 case NVPTX::BI__nvvm_fmax_ftz_nan_f16:
648 cgm.errorNYI(expr->getSourceRange(),
649 std::string("unimplemented NVPTX builtin call: ") +
650 getContext().BuiltinInfo.getName(builtinId));
651 return mlir::Value{};
652 case NVPTX::BI__nvvm_fmax_ftz_nan_f16x2:
653 cgm.errorNYI(expr->getSourceRange(),
654 std::string("unimplemented NVPTX builtin call: ") +
655 getContext().BuiltinInfo.getName(builtinId));
656 return mlir::Value{};
657 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16:
658 cgm.errorNYI(expr->getSourceRange(),
659 std::string("unimplemented NVPTX builtin call: ") +
660 getContext().BuiltinInfo.getName(builtinId));
661 return mlir::Value{};
662 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16x2:
663 cgm.errorNYI(expr->getSourceRange(),
664 std::string("unimplemented NVPTX builtin call: ") +
665 getContext().BuiltinInfo.getName(builtinId));
666 return mlir::Value{};
667 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16:
668 cgm.errorNYI(expr->getSourceRange(),
669 std::string("unimplemented NVPTX builtin call: ") +
670 getContext().BuiltinInfo.getName(builtinId));
671 return mlir::Value{};
672 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16x2:
673 cgm.errorNYI(expr->getSourceRange(),
674 std::string("unimplemented NVPTX builtin call: ") +
675 getContext().BuiltinInfo.getName(builtinId));
676 return mlir::Value{};
677 case NVPTX::BI__nvvm_fmax_nan_f16:
678 cgm.errorNYI(expr->getSourceRange(),
679 std::string("unimplemented NVPTX builtin call: ") +
680 getContext().BuiltinInfo.getName(builtinId));
681 return mlir::Value{};
682 case NVPTX::BI__nvvm_fmax_nan_f16x2:
683 cgm.errorNYI(expr->getSourceRange(),
684 std::string("unimplemented NVPTX builtin call: ") +
685 getContext().BuiltinInfo.getName(builtinId));
686 return mlir::Value{};
687 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16:
688 cgm.errorNYI(expr->getSourceRange(),
689 std::string("unimplemented NVPTX builtin call: ") +
690 getContext().BuiltinInfo.getName(builtinId));
691 return mlir::Value{};
692 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16x2:
693 cgm.errorNYI(expr->getSourceRange(),
694 std::string("unimplemented NVPTX builtin call: ") +
695 getContext().BuiltinInfo.getName(builtinId));
696 return mlir::Value{};
697 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16:
698 cgm.errorNYI(expr->getSourceRange(),
699 std::string("unimplemented NVPTX builtin call: ") +
700 getContext().BuiltinInfo.getName(builtinId));
701 return mlir::Value{};
702 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16x2:
703 cgm.errorNYI(expr->getSourceRange(),
704 std::string("unimplemented NVPTX builtin call: ") +
705 getContext().BuiltinInfo.getName(builtinId));
706 return mlir::Value{};
707 case NVPTX::BI__nvvm_fmin_f16:
708 cgm.errorNYI(expr->getSourceRange(),
709 std::string("unimplemented NVPTX builtin call: ") +
710 getContext().BuiltinInfo.getName(builtinId));
711 return mlir::Value{};
712 case NVPTX::BI__nvvm_fmin_f16x2:
713 cgm.errorNYI(expr->getSourceRange(),
714 std::string("unimplemented NVPTX builtin call: ") +
715 getContext().BuiltinInfo.getName(builtinId));
716 return mlir::Value{};
717 case NVPTX::BI__nvvm_fmin_ftz_f16:
718 cgm.errorNYI(expr->getSourceRange(),
719 std::string("unimplemented NVPTX builtin call: ") +
720 getContext().BuiltinInfo.getName(builtinId));
721 return mlir::Value{};
722 case NVPTX::BI__nvvm_fmin_ftz_f16x2:
723 cgm.errorNYI(expr->getSourceRange(),
724 std::string("unimplemented NVPTX builtin call: ") +
725 getContext().BuiltinInfo.getName(builtinId));
726 return mlir::Value{};
727 case NVPTX::BI__nvvm_fmin_ftz_nan_f16:
728 cgm.errorNYI(expr->getSourceRange(),
729 std::string("unimplemented NVPTX builtin call: ") +
730 getContext().BuiltinInfo.getName(builtinId));
731 return mlir::Value{};
732 case NVPTX::BI__nvvm_fmin_ftz_nan_f16x2:
733 cgm.errorNYI(expr->getSourceRange(),
734 std::string("unimplemented NVPTX builtin call: ") +
735 getContext().BuiltinInfo.getName(builtinId));
736 return mlir::Value{};
737 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16:
738 cgm.errorNYI(expr->getSourceRange(),
739 std::string("unimplemented NVPTX builtin call: ") +
740 getContext().BuiltinInfo.getName(builtinId));
741 return mlir::Value{};
742 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16x2:
743 cgm.errorNYI(expr->getSourceRange(),
744 std::string("unimplemented NVPTX builtin call: ") +
745 getContext().BuiltinInfo.getName(builtinId));
746 return mlir::Value{};
747 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16:
748 cgm.errorNYI(expr->getSourceRange(),
749 std::string("unimplemented NVPTX builtin call: ") +
750 getContext().BuiltinInfo.getName(builtinId));
751 return mlir::Value{};
752 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16x2:
753 cgm.errorNYI(expr->getSourceRange(),
754 std::string("unimplemented NVPTX builtin call: ") +
755 getContext().BuiltinInfo.getName(builtinId));
756 return mlir::Value{};
757 case NVPTX::BI__nvvm_fmin_nan_f16:
758 cgm.errorNYI(expr->getSourceRange(),
759 std::string("unimplemented NVPTX builtin call: ") +
760 getContext().BuiltinInfo.getName(builtinId));
761 return mlir::Value{};
762 case NVPTX::BI__nvvm_fmin_nan_f16x2:
763 cgm.errorNYI(expr->getSourceRange(),
764 std::string("unimplemented NVPTX builtin call: ") +
765 getContext().BuiltinInfo.getName(builtinId));
766 return mlir::Value{};
767 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16:
768 cgm.errorNYI(expr->getSourceRange(),
769 std::string("unimplemented NVPTX builtin call: ") +
770 getContext().BuiltinInfo.getName(builtinId));
771 return mlir::Value{};
772 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16x2:
773 cgm.errorNYI(expr->getSourceRange(),
774 std::string("unimplemented NVPTX builtin call: ") +
775 getContext().BuiltinInfo.getName(builtinId));
776 return mlir::Value{};
777 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16:
778 cgm.errorNYI(expr->getSourceRange(),
779 std::string("unimplemented NVPTX builtin call: ") +
780 getContext().BuiltinInfo.getName(builtinId));
781 return mlir::Value{};
782 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16x2:
783 cgm.errorNYI(expr->getSourceRange(),
784 std::string("unimplemented NVPTX builtin call: ") +
785 getContext().BuiltinInfo.getName(builtinId));
786 return mlir::Value{};
787 case NVPTX::BI__nvvm_fabs_f:
788 case NVPTX::BI__nvvm_abs_bf16:
789 case NVPTX::BI__nvvm_abs_bf16x2:
790 case NVPTX::BI__nvvm_fabs_f16:
791 case NVPTX::BI__nvvm_fabs_f16x2:
792 return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.fabs");
793 case NVPTX::BI__nvvm_fabs_ftz_f:
794 case NVPTX::BI__nvvm_fabs_ftz_f16:
795 case NVPTX::BI__nvvm_fabs_ftz_f16x2:
796 return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.fabs.ftz");
797 case NVPTX::BI__nvvm_fabs_d:
798 return emitUnaryNVVMIntrinsic(*this, expr, "fabs");
799 case NVPTX::BI__nvvm_ex2_approx_d:
800 case NVPTX::BI__nvvm_ex2_approx_f:
801 return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.ex2.approx");
802 case NVPTX::BI__nvvm_ex2_approx_ftz_f:
803 return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.ex2.approx.ftz");
804 case NVPTX::BI__nvvm_ldg_h:
805 case NVPTX::BI__nvvm_ldg_h2:
806 cgm.errorNYI(expr->getSourceRange(),
807 std::string("unimplemented NVPTX builtin call: ") +
808 getContext().BuiltinInfo.getName(builtinId));
809 return mlir::Value{};
810 case NVPTX::BI__nvvm_ldu_h:
811 case NVPTX::BI__nvvm_ldu_h2:
812 cgm.errorNYI(expr->getSourceRange(),
813 std::string("unimplemented NVPTX builtin call: ") +
814 getContext().BuiltinInfo.getName(builtinId));
815 return mlir::Value{};
816 case NVPTX::BI__nvvm_cp_async_ca_shared_global_4:
817 cgm.errorNYI(expr->getSourceRange(),
818 std::string("unimplemented NVPTX builtin call: ") +
819 getContext().BuiltinInfo.getName(builtinId));
820 return mlir::Value{};
821 case NVPTX::BI__nvvm_cp_async_ca_shared_global_8:
822 cgm.errorNYI(expr->getSourceRange(),
823 std::string("unimplemented NVPTX builtin call: ") +
824 getContext().BuiltinInfo.getName(builtinId));
825 return mlir::Value{};
826 case NVPTX::BI__nvvm_cp_async_ca_shared_global_16:
827 cgm.errorNYI(expr->getSourceRange(),
828 std::string("unimplemented NVPTX builtin call: ") +
829 getContext().BuiltinInfo.getName(builtinId));
830 return mlir::Value{};
831 case NVPTX::BI__nvvm_cp_async_cg_shared_global_16:
832 cgm.errorNYI(expr->getSourceRange(),
833 std::string("unimplemented NVPTX builtin call: ") +
834 getContext().BuiltinInfo.getName(builtinId));
835 return mlir::Value{};
836 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_x:
837 cgm.errorNYI(expr->getSourceRange(),
838 std::string("unimplemented NVPTX builtin call: ") +
839 getContext().BuiltinInfo.getName(builtinId));
840 return mlir::Value{};
841 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_y:
842 cgm.errorNYI(expr->getSourceRange(),
843 std::string("unimplemented NVPTX builtin call: ") +
844 getContext().BuiltinInfo.getName(builtinId));
845 return mlir::Value{};
846 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_z:
847 cgm.errorNYI(expr->getSourceRange(),
848 std::string("unimplemented NVPTX builtin call: ") +
849 getContext().BuiltinInfo.getName(builtinId));
850 return mlir::Value{};
851 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_w:
852 cgm.errorNYI(expr->getSourceRange(),
853 std::string("unimplemented NVPTX builtin call: ") +
854 getContext().BuiltinInfo.getName(builtinId));
855 return mlir::Value{};
856 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_x:
857 cgm.errorNYI(expr->getSourceRange(),
858 std::string("unimplemented NVPTX builtin call: ") +
859 getContext().BuiltinInfo.getName(builtinId));
860 return mlir::Value{};
861 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_y:
862 cgm.errorNYI(expr->getSourceRange(),
863 std::string("unimplemented NVPTX builtin call: ") +
864 getContext().BuiltinInfo.getName(builtinId));
865 return mlir::Value{};
866 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_z:
867 cgm.errorNYI(expr->getSourceRange(),
868 std::string("unimplemented NVPTX builtin call: ") +
869 getContext().BuiltinInfo.getName(builtinId));
870 return mlir::Value{};
871 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_w:
872 cgm.errorNYI(expr->getSourceRange(),
873 std::string("unimplemented NVPTX builtin call: ") +
874 getContext().BuiltinInfo.getName(builtinId));
875 return mlir::Value{};
876 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_x:
877 cgm.errorNYI(expr->getSourceRange(),
878 std::string("unimplemented NVPTX builtin call: ") +
879 getContext().BuiltinInfo.getName(builtinId));
880 return mlir::Value{};
881 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_y:
882 cgm.errorNYI(expr->getSourceRange(),
883 std::string("unimplemented NVPTX builtin call: ") +
884 getContext().BuiltinInfo.getName(builtinId));
885 return mlir::Value{};
886 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_z:
887 cgm.errorNYI(expr->getSourceRange(),
888 std::string("unimplemented NVPTX builtin call: ") +
889 getContext().BuiltinInfo.getName(builtinId));
890 return mlir::Value{};
891 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_w:
892 cgm.errorNYI(expr->getSourceRange(),
893 std::string("unimplemented NVPTX builtin call: ") +
894 getContext().BuiltinInfo.getName(builtinId));
895 return mlir::Value{};
896 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_x:
897 cgm.errorNYI(expr->getSourceRange(),
898 std::string("unimplemented NVPTX builtin call: ") +
899 getContext().BuiltinInfo.getName(builtinId));
900 return mlir::Value{};
901 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_y:
902 cgm.errorNYI(expr->getSourceRange(),
903 std::string("unimplemented NVPTX builtin call: ") +
904 getContext().BuiltinInfo.getName(builtinId));
905 return mlir::Value{};
906 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_z:
907 cgm.errorNYI(expr->getSourceRange(),
908 std::string("unimplemented NVPTX builtin call: ") +
909 getContext().BuiltinInfo.getName(builtinId));
910 return mlir::Value{};
911 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_w:
912 cgm.errorNYI(expr->getSourceRange(),
913 std::string("unimplemented NVPTX builtin call: ") +
914 getContext().BuiltinInfo.getName(builtinId));
915 return mlir::Value{};
916 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctarank:
917 cgm.errorNYI(expr->getSourceRange(),
918 std::string("unimplemented NVPTX builtin call: ") +
919 getContext().BuiltinInfo.getName(builtinId));
920 return mlir::Value{};
921 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctarank:
922 cgm.errorNYI(expr->getSourceRange(),
923 std::string("unimplemented NVPTX builtin call: ") +
924 getContext().BuiltinInfo.getName(builtinId));
925 return mlir::Value{};
926 case NVPTX::BI__nvvm_is_explicit_cluster:
927 cgm.errorNYI(expr->getSourceRange(),
928 std::string("unimplemented NVPTX builtin call: ") +
929 getContext().BuiltinInfo.getName(builtinId));
930 return mlir::Value{};
931 case NVPTX::BI__nvvm_isspacep_shared_cluster:
932 cgm.errorNYI(expr->getSourceRange(),
933 std::string("unimplemented NVPTX builtin call: ") +
934 getContext().BuiltinInfo.getName(builtinId));
935 return mlir::Value{};
936 case NVPTX::BI__nvvm_mapa:
937 cgm.errorNYI(expr->getSourceRange(),
938 std::string("unimplemented NVPTX builtin call: ") +
939 getContext().BuiltinInfo.getName(builtinId));
940 return mlir::Value{};
941 case NVPTX::BI__nvvm_mapa_shared_cluster:
942 cgm.errorNYI(expr->getSourceRange(),
943 std::string("unimplemented NVPTX builtin call: ") +
944 getContext().BuiltinInfo.getName(builtinId));
945 return mlir::Value{};
946 case NVPTX::BI__nvvm_getctarank:
947 cgm.errorNYI(expr->getSourceRange(),
948 std::string("unimplemented NVPTX builtin call: ") +
949 getContext().BuiltinInfo.getName(builtinId));
950 return mlir::Value{};
951 case NVPTX::BI__nvvm_getctarank_shared_cluster:
952 cgm.errorNYI(expr->getSourceRange(),
953 std::string("unimplemented NVPTX builtin call: ") +
954 getContext().BuiltinInfo.getName(builtinId));
955 return mlir::Value{};
956 case NVPTX::BI__nvvm_barrier_cluster_arrive:
957 return builder.emitIntrinsicCallOp(getLoc(expr->getExprLoc()),
958 "nvvm.barrier.cluster.arrive",
959 builder.getVoidTy());
960 case NVPTX::BI__nvvm_barrier_cluster_arrive_relaxed:
961 return builder.emitIntrinsicCallOp(getLoc(expr->getExprLoc()),
962 "nvvm.barrier.cluster.arrive.relaxed",
963 builder.getVoidTy());
964 case NVPTX::BI__nvvm_barrier_cluster_wait:
965 return builder.emitIntrinsicCallOp(getLoc(expr->getExprLoc()),
966 "nvvm.barrier.cluster.wait",
967 builder.getVoidTy());
968 case NVPTX::BI__nvvm_fence_sc_cluster:
969 return builder.emitIntrinsicCallOp(getLoc(expr->getExprLoc()),
970 "nvvm.fence.sc.cluster",
971 builder.getVoidTy());
972 case NVPTX::BI__nvvm_bar_sync:
973 return builder.emitIntrinsicCallOp(
974 getLoc(expr->getExprLoc()), "nvvm.barrier.cta.sync.aligned.all",
975 builder.getVoidTy(), mlir::ValueRange{emitScalarExpr(expr->getArg(0))});
976 case NVPTX::BI__syncthreads:
977 return builder.emitIntrinsicCallOp(
978 getLoc(expr->getExprLoc()), "nvvm.barrier.cta.sync.aligned.all",
979 builder.getVoidTy(),
980 mlir::ValueRange{builder.getConstInt(getLoc(expr->getExprLoc()),
981 builder.getSInt32Ty(), 0)});
982 case NVPTX::BI__nvvm_barrier_sync:
983 return builder.emitIntrinsicCallOp(
984 getLoc(expr->getExprLoc()), "nvvm.barrier.cta.sync.all",
985 builder.getVoidTy(), mlir::ValueRange{emitScalarExpr(expr->getArg(0))});
986 case NVPTX::BI__nvvm_barrier_sync_cnt:
987 return builder.emitIntrinsicCallOp(
988 getLoc(expr->getExprLoc()), "nvvm.barrier.cta.sync.count",
989 builder.getVoidTy(),
990 mlir::ValueRange{emitScalarExpr(expr->getArg(0)),
991 emitScalarExpr(expr->getArg(1))});
992 case NVPTX::BI__nvvm_bar0_and:
993 cgm.errorNYI(expr->getSourceRange(),
994 std::string("unimplemented NVPTX builtin call: ") +
995 getContext().BuiltinInfo.getName(builtinId));
996 return mlir::Value{};
997 case NVPTX::BI__nvvm_bar0_or:
998 cgm.errorNYI(expr->getSourceRange(),
999 std::string("unimplemented NVPTX builtin call: ") +
1000 getContext().BuiltinInfo.getName(builtinId));
1001 return mlir::Value{};
1002 case NVPTX::BI__nvvm_bar0_popc:
1003 cgm.errorNYI(expr->getSourceRange(),
1004 std::string("unimplemented NVPTX builtin call: ") +
1005 getContext().BuiltinInfo.getName(builtinId));
1006 return mlir::Value{};
1007
1008 default:
1009 return std::nullopt;
1010 }
1011}
1012
1013// vprintf takes two args: A format string, and a pointer to a buffer containing
1014// the varargs.
1015//
1016// For example, the call
1017//
1018// printf("format string", arg1, arg2, arg3);
1019//
1020// is converted into something resembling
1021//
1022// struct Tmp {
1023// Arg1 a1;
1024// Arg2 a2;
1025// Arg3 a3;
1026// };
1027// char* buf = alloca(sizeof(Tmp));
1028// *(Tmp*)buf = {a1, a2, a3};
1029// vprintf("format string", buf);
1030//
1031// `buf` is aligned to the max of {alignof(Arg1), ...}. Furthermore, each of
1032// the args is itself aligned to its preferred alignment.
1033//
1034// Note that by the time this function runs, the arguments have already
1035// undergone the standard C vararg promotion (short -> int, float -> double
1036// etc). In this function we pack the arguments into the buffer described above.
1037static mlir::Value packArgsIntoNVPTXFormatBuffer(CIRGenFunction &cgf,
1038 const CallArgList &args,
1039 mlir::Location loc) {
1040 const cir::CIRDataLayout dataLayout = cgf.cgm.getDataLayout();
1041 CIRGenBuilderTy &builder = cgf.getBuilder();
1042
1043 if (args.size() <= 1)
1044 // If there are no arguments other than the format string,
1045 // pass a nullptr to vprintf.
1046 return builder.getNullPtr(builder.getVoidPtrTy(), loc);
1047
1048 llvm::SmallVector<mlir::Type> argTypes;
1049 for (const auto &arg : llvm::drop_begin(args))
1050 argTypes.push_back(arg.getKnownRValue().getValue().getType());
1051
1052 // We can directly store the arguments into a struct, and the alignment
1053 // would automatically be correct. That's because vprintf does not
1054 // accept aggregates.
1055 mlir::Type allocaTy = builder.getAnonRecordTy(argTypes);
1056 auto allocaAlign = clang::CharUnits::fromQuantity(
1057 dataLayout.getABITypeAlign(allocaTy).value());
1058 Address allocaAddr =
1059 cgf.createTempAlloca(allocaTy, allocaAlign, loc, "printf_args");
1060 mlir::Value alloca = allocaAddr.getPointer();
1061
1062 for (auto [i, arg] : llvm::enumerate(llvm::drop_begin(args))) {
1063 mlir::Value member = builder.createGetMember(
1064 loc, cir::PointerType::get(argTypes[i]), alloca, /*name=*/"",
1065 /*index=*/i);
1066 auto abiAlign = clang::CharUnits::fromQuantity(
1067 dataLayout.getABITypeAlign(argTypes[i]).value());
1068 cir::StoreOp::create(builder, loc, arg.getKnownRValue().getValue(), member,
1069 /*is_volatile=*/false,
1070 builder.getAlignmentAttr(abiAlign),
1071 /*sync_scope=*/cir::SyncScopeKindAttr{},
1072 /*mem_order=*/cir::MemOrderAttr{});
1073 }
1074
1075 return builder.createBitcast(alloca, builder.getVoidPtrTy());
1076}
1077
1078mlir::Value
1079CIRGenFunction::emitNVPTXDevicePrintfCallExpr(const CallExpr *expr) {
1080 assert(cgm.getTriple().isNVPTX());
1081 assert(expr->getBuiltinCallee() == Builtin::BIprintf ||
1082 expr->getBuiltinCallee() == Builtin::BI__builtin_printf);
1083 assert(expr->getNumArgs() >= 1); // printf always has at least one arg.
1084 CallArgList args;
1085 emitCallArgs(args,
1086 expr->getDirectCallee()->getType()->getAs<FunctionProtoType>(),
1087 expr->arguments(), expr->getDirectCallee());
1088
1089 mlir::Location loc = getLoc(expr->getBeginLoc());
1090
1091 // We don't know how to emit non-scalar varargs.
1092 bool hasNonScalar =
1093 llvm::any_of(llvm::drop_begin(args), [&](const CallArg &a) {
1094 return a.hasLValue() || !a.getKnownRValue().isScalar();
1095 });
1096 if (hasNonScalar) {
1097 cgm.errorUnsupported(expr, "non-scalar args to printf");
1098 return builder.getConstInt(loc, builder.getSInt32Ty(), 0);
1099 }
1100
1101 mlir::Value packedData = packArgsIntoNVPTXFormatBuffer(*this, args, loc);
1102
1103 // int vprintf(char *format, void *packedData);
1104 auto vprintf = cgm.createRuntimeFunction(
1105 cir::FuncType::get(
1106 {cir::PointerType::get(builder.getSInt8Ty()), builder.getVoidPtrTy()},
1107 builder.getSInt32Ty()),
1108 "vprintf");
1109 auto formatString = args[0].getKnownRValue().getValue();
1110 return builder
1111 .createCallOp(loc, vprintf, mlir::ValueRange{formatString, packedData})
1112 .getResult();
1113}
packArgsIntoNVPTXFormatBuffer
static mlir::Value packArgsIntoNVPTXFormatBuffer(CIRGenFunction &cgf, const CallArgList &args, mlir::Location loc)
Definition CIRGenBuiltinNVPTX.cpp:1037
emitUnaryNVVMIntrinsic
static mlir::Value emitUnaryNVVMIntrinsic(CIRGenFunction &cgf, const CallExpr *expr, llvm::StringRef intrinsicName)
Emit a CIR LLVMIntrinsicCallOp for a unary NVVM intrinsic.
Definition CIRGenBuiltinNVPTX.cpp:25
TargetBuiltins.h
Enumerates target-specific builtins in their own namespaces within namespace clang.
cir::CIRBaseBuilderTy::createGetMember
cir::GetMemberOp createGetMember(mlir::Location loc, mlir::Type resultTy, mlir::Value base, llvm::StringRef name, unsigned index)
Definition CIRBaseBuilder.h:414
cir::CIRBaseBuilderTy::getNullPtr
cir::ConstantOp getNullPtr(mlir::Type ty, mlir::Location loc)
Definition CIRBaseBuilder.h:817
cir::CIRBaseBuilderTy::createBitcast
mlir::Value createBitcast(mlir::Value src, mlir::Type newTy)
Definition CIRBaseBuilder.h:536
cir::CIRBaseBuilderTy::getAlignmentAttr
mlir::IntegerAttr getAlignmentAttr(clang::CharUnits alignment)
Definition CIRBaseBuilder.h:793
cir::CIRBaseBuilderTy::getVoidPtrTy
cir::PointerType getVoidPtrTy(clang::LangAS langAS=clang::LangAS::Default)
Definition CIRBaseBuilder.h:179
cir::CIRDataLayout::getABITypeAlign
llvm::Align getABITypeAlign(mlir::Type ty) const
Definition CIRDataLayout.h:46
clang::CIRGen::Address::getPointer
mlir::Value getPointer() const
Definition Address.h:98
clang::CIRGen::CIRGenBuilderTy::getAnonRecordTy
cir::RecordType getAnonRecordTy(llvm::ArrayRef< mlir::Type > members, bool packed=false, bool padded=false)
Get a CIR anonymous record type.
Definition CIRGenBuilder.h:441
clang::CIRGen::CIRGenFunction::emitCallArgs
void emitCallArgs(CallArgList &args, PrototypeWrapper prototype, llvm::iterator_range< clang::CallExpr::const_arg_iterator > argRange, AbstractCallee callee=AbstractCallee(), unsigned paramsToSkip=0)
Definition CIRGenCall.cpp:1490
clang::CIRGen::CIRGenFunction::createTempAlloca
cir::AllocaOp createTempAlloca(mlir::Type ty, mlir::Location loc, const Twine &name="tmp", mlir::Value arraySize=nullptr, bool insertIntoFnEntryBlock=false)
This creates an alloca and inserts it into the entry block if ArraySize is nullptr,...
Definition CIRGenExpr.cpp:2860
clang::CIRGen::CIRGenFunction::emitNVPTXDevicePrintfCallExpr
mlir::Value emitNVPTXDevicePrintfCallExpr(const CallExpr *expr)
Emit a device-side printf call for NVPTX targets.
Definition CIRGenBuiltinNVPTX.cpp:1079
clang::CIRGen::CIRGenFunction::getLoc
mlir::Location getLoc(clang::SourceLocation srcLoc)
Helpers to convert Clang's SourceLocation to a MLIR Location.
Definition CIRGenFunction.cpp:108
clang::CIRGen::CIRGenFunction::emitScalarExpr
mlir::Value emitScalarExpr(const clang::Expr *e, bool ignoreResultAssign=false)
Emit the computation of the specified expression of scalar type.
Definition CIRGenExprScalar.cpp:1757
clang::CIRGen::CIRGenFunction::getBuilder
CIRGenBuilderTy & getBuilder()
Definition CIRGenFunction.h:222
clang::CIRGen::CIRGenFunction::emitNVPTXBuiltinExpr
std::optional< mlir::Value > emitNVPTXBuiltinExpr(unsigned builtinID, const CallExpr *expr)
Emit a call to an NVPTX builtin function.
Definition CIRGenBuiltinNVPTX.cpp:37
clang::CIRGen::CIRGenFunction::getContext
clang::ASTContext & getContext() const
Definition CIRGenFunction.h:220
clang::CIRGen::CIRGenModule::getDataLayout
const cir::CIRDataLayout getDataLayout() const
Definition CIRGenModule.h:177
clang::CIRGen::RValue::isScalar
bool isScalar() const
Definition CIRGenValue.h:49
clang::CallExpr
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
Definition Expr.h:2946
clang::CharUnits::fromQuantity
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
Definition CharUnits.h:63
clang::FunctionProtoType
Represents a prototype with parameter type info, e.g.
Definition TypeBase.h:5369
clang::ast_matchers::expr
const internal::VariadicDynCastAllOfMatcher< Stmt, Expr > expr
Matches expressions.
Definition ASTMatchersInternal.cpp:937
clang
The JSON file list parser is used to communicate input to InstallAPI.
Definition CalledOnceCheck.h:17
clang::CIRGen::CallArg::hasLValue
bool hasLValue() const
Definition CIRGenCall.h:216
clang::CIRGen::CallArg::getKnownRValue
RValue getKnownRValue() const
Definition CIRGenCall.h:227
Generated on for clang by doxygen 1.14.0